Add vector search index feature and end-to-end logging

.github/workflows/coverage.yml

@@ -25,6 +25,7 @@ jobs:
         chmod +x ./coverage.sh
         ./coverage.sh 80
       env:
+        OLLAMA_AVAILABLE: "false"
         MIN_COVERAGE: 80
 
     - name: Upload coverage report

.gitignore

@@ -6,6 +6,7 @@ archived/
 node_modules/
 obj/
 bin/
+__pycache__/
 _dev/
 .dev/
 .vs/
@@ -68,4 +69,4 @@ publish/
 *.crt
 *.key
 *.pem
-certs/
+certs/

e2e-tests.sh

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+
+set -e
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)"
+cd "$ROOT"
+
+echo "======================================="
+echo "  Running E2E Tests"
+echo "======================================="
+echo ""
+
+# Choose build configuration (default Release to align with build.sh)
+CONFIGURATION="${CONFIGURATION:-Release}"
+KM_BIN="$ROOT/src/Main/bin/$CONFIGURATION/net10.0/KernelMemory.Main.dll"
+
+# Ensure km binary is built at the selected configuration
+if [ ! -f "$KM_BIN" ]; then
+    echo "km binary not found at $KM_BIN. Building ($CONFIGURATION)..."
+    dotnet build src/Main/Main.csproj -c "$CONFIGURATION"
+fi
+
+if [ ! -f "$KM_BIN" ]; then
+    echo "❌ km binary still not found at $KM_BIN after build. Set KM_BIN to a valid path."
+    exit 1
+fi
+
+export KM_BIN
+
+FAILED=0
+PASSED=0
+
+# Run each test file
+for test_file in tests/e2e/test_*.py; do
+    if [ -f "$test_file" ]; then
+        echo ""
+        echo "Running: $(basename "$test_file")"
+        echo "---------------------------------------"
+
+        if python3 "$test_file"; then
+            PASSED=$((PASSED + 1))
+        else
+            FAILED=$((FAILED + 1))
+        fi
+    fi
+done
+
+echo ""
+echo "======================================="
+echo "  E2E Test Results"
+echo "======================================="
+echo "Passed: $PASSED"
+echo "Failed: $FAILED"
+echo "======================================="
+
+if [ $FAILED -gt 0 ]; then
+    exit 1
+fi
+
+exit 0

format.sh

@@ -1 +1,14 @@
-dotnet format
+#!/usr/bin/env bash
+
+set -e
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)"
+cd "$ROOT"
+TMPDIR="$ROOT/.tmp"
+mkdir -p "$TMPDIR"
+export TMPDIR
+
+dotnet format src/Core/Core.csproj
+dotnet format src/Main/Main.csproj
+dotnet format tests/Core.Tests/Core.Tests.csproj
+dotnet format tests/Main.Tests/Main.Tests.csproj

src/Core/Config/AppConfig.cs

@@ -83,7 +83,8 @@ public static AppConfig CreateDefault()
 
     /// <summary>
     /// Creates a default configuration with a single "personal" node
-    /// using local SQLite storage in the specified base directory
+    /// using local SQLite storage in the specified base directory.
+    /// Includes embeddings cache for efficient vector search operations.
     /// </summary>
     /// <param name="baseDir">Base directory for data storage</param>
     public static AppConfig CreateDefault(string baseDir)
@@ -95,8 +96,10 @@ public static AppConfig CreateDefault(string baseDir)
             Nodes = new Dictionary<string, NodeConfig>
             {
                 ["personal"] = NodeConfig.CreateDefaultPersonalNode(personalNodeDir)
-            }
-            // EmbeddingsCache and LLMCache intentionally omitted - add when features are implemented
+            },
+            EmbeddingsCache = CacheConfig.CreateDefaultSqliteCache(
+                Path.Combine(baseDir, "embeddings-cache.db"))
+            // LLMCache intentionally omitted - add when LLM features are implemented
         };
     }
 }

src/Core/Config/ConfigParser.cs

@@ -1,5 +1,6 @@
 // Copyright (c) Microsoft. All rights reserved.
 using System.Text.Json;
+using System.Text.Json.Serialization;
 using System.Text.Json.Serialization.Metadata;
 using KernelMemory.Core.Config.Cache;
 using KernelMemory.Core.Config.ContentIndex;
@@ -28,7 +29,8 @@ public static class ConfigParser
         ReadCommentHandling = JsonCommentHandling.Skip,
         AllowTrailingCommas = true,
         PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
-        TypeInfoResolver = new DefaultJsonTypeInfoResolver()
+        TypeInfoResolver = new DefaultJsonTypeInfoResolver(),
+        Converters = { new JsonStringEnumConverter() }
     };
 
     /// <summary>
@@ -46,13 +48,14 @@ public static class ConfigParser
 
     /// <summary>
     /// Loads configuration from a file, or creates default config if file doesn't exist.
-    /// The config file is always ensured to exist on disk after loading.
+    /// Optionally ensures the config file exists on disk after loading (for write operations).
     /// Performs tilde expansion on paths (~/ → home directory)
     /// </summary>
     /// <param name="filePath">Path to configuration file</param>
+    /// <param name="ensureFileExists">If true, writes config to disk if missing (default: true for backward compatibility)</param>
     /// <returns>Validated AppConfig instance</returns>
     /// <exception cref="ConfigException">Thrown when file exists but parsing or validation fails</exception>
-    public static AppConfig LoadFromFile(string filePath)
+    public static AppConfig LoadFromFile(string filePath, bool ensureFileExists = true)
     {
         AppConfig config;
 
@@ -65,8 +68,11 @@ public static AppConfig LoadFromFile(string filePath)
             // Create default config relative to config file location
             config = AppConfig.CreateDefault(baseDir);
 
-            // Write the config file
-            WriteConfigFile(filePath, config);
+            // Write the config file only if requested
+            if (ensureFileExists)
+            {
+                WriteConfigFile(filePath, config);
+            }
 
             return config;
         }
@@ -82,8 +88,11 @@ public static AppConfig LoadFromFile(string filePath)
             // Expand tilde paths
             ExpandTildePaths(config);
 
-            // Always ensure the config file exists (recreate if deleted between load and save)
-            WriteConfigFileIfMissing(filePath, config);
+            // Optionally ensure the config file exists (recreate if deleted between load and save)
+            if (ensureFileExists)
+            {
+                WriteConfigFileIfMissing(filePath, config);
+            }
 
             return config;
         }

src/Core/Config/Embeddings/HuggingFaceEmbeddingsConfig.cs

@@ -2,7 +2,6 @@
 using System.Text.Json.Serialization;
 using KernelMemory.Core.Config.Enums;
 using KernelMemory.Core.Config.Validation;
-using KernelMemory.Core.Embeddings;
 
 namespace KernelMemory.Core.Config.Embeddings;
 
@@ -20,7 +19,7 @@ public sealed class HuggingFaceEmbeddingsConfig : EmbeddingsConfig
     /// HuggingFace model name (e.g., "sentence-transformers/all-MiniLM-L6-v2", "BAAI/bge-base-en-v1.5").
     /// </summary>
     [JsonPropertyName("model")]
-    public string Model { get; set; } = EmbeddingConstants.DefaultHuggingFaceModel;
+    public string Model { get; set; } = Constants.EmbeddingDefaults.DefaultHuggingFaceModel;
 
     /// <summary>
     /// HuggingFace API key (token).
@@ -35,7 +34,7 @@ public sealed class HuggingFaceEmbeddingsConfig : EmbeddingsConfig
     /// Can be changed for custom inference endpoints.
     /// </summary>
     [JsonPropertyName("baseUrl")]
-    public string BaseUrl { get; set; } = EmbeddingConstants.DefaultHuggingFaceBaseUrl;
+    public string BaseUrl { get; set; } = Constants.EmbeddingDefaults.DefaultHuggingFaceBaseUrl;
 
     /// <inheritdoc />
     public override void Validate(string path)

src/Core/Config/NodeConfig.cs

@@ -1,6 +1,7 @@
 // Copyright (c) Microsoft. All rights reserved.
 using System.Text.Json.Serialization;
 using KernelMemory.Core.Config.ContentIndex;
+using KernelMemory.Core.Config.Embeddings;
 using KernelMemory.Core.Config.Enums;
 using KernelMemory.Core.Config.SearchIndex;
 using KernelMemory.Core.Config.Storage;
@@ -106,7 +107,8 @@ public void Validate(string path)
     }
 
     /// <summary>
-    /// Creates a default "personal" node configuration
+    /// Creates a default "personal" node configuration with FTS and vector search.
+    /// Uses Ollama with qwen3-embedding model (1024 dimensions) for local, offline-capable vector search.
     /// </summary>
     /// <param name="nodeDir"></param>
     internal static NodeConfig CreateDefaultPersonalNode(string nodeDir)
@@ -128,7 +130,21 @@ internal static NodeConfig CreateDefaultPersonalNode(string nodeDir)
                     Id = "sqlite-fts",
                     Type = SearchIndexTypes.SqliteFTS,
                     Path = Path.Combine(nodeDir, "fts.db"),
-                    EnableStemming = true
+                    EnableStemming = true,
+                    Required = true
+                },
+                new VectorSearchIndexConfig
+                {
+                    Id = "sqlite-vector",
+                    Type = SearchIndexTypes.SqliteVector,
+                    Path = Path.Combine(nodeDir, "vector.db"),
+                    Dimensions = 1024,
+                    UseSqliteVec = false,
+                    Embeddings = new OllamaEmbeddingsConfig
+                    {
+                        Model = Constants.EmbeddingDefaults.DefaultOllamaModel,
+                        BaseUrl = Constants.EmbeddingDefaults.DefaultOllamaBaseUrl
+                    }
                 }
             }
         };