major perf improvements

Author: lofcz

src/Infidex.Benchmark/Infidex.Benchmark.csproj

@@ -21,6 +21,9 @@
       <None Update="movies.csv">
         <CopyToOutputDirectory>Always</CopyToOutputDirectory>
       </None>
+      <None Update="movies1M.csv">
+        <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+      </None>
     </ItemGroup>
 
     <ItemGroup>

src/Infidex.Benchmark/MovieData.cs

@@ -34,10 +34,30 @@ public static List<MovieRecord> LoadMovies()
         var config = new CsvConfiguration(CultureInfo.InvariantCulture)
         {
             PrepareHeaderForMatch = args => args.Header.ToLower(),
-            MissingFieldFound = null // Ignore missing fields
+            MissingFieldFound = null, // Ignore missing fields
+            HeaderValidated = null // Ignore missing headers
         };
 
-        string filePath = Path.Combine(AppContext.BaseDirectory, "movies.csv");
+        string filePath = Path.Combine(AppContext.BaseDirectory, "movies1M.csv");
+        
+        // Try to find 1M dataset
+        string[] possiblePaths = 
+        {
+            Path.Combine(AppContext.BaseDirectory, "../../../../src/Infidex.Tests/movies1M.csv"),
+            Path.Combine(AppContext.BaseDirectory, "../../../../src/Infidex.Example/movies.csv"),
+            filePath
+        };
+        
+        foreach (var path in possiblePaths)
+        {
+            if (File.Exists(path))
+            {
+                filePath = path;
+                Console.WriteLine($"Loading movies from: {filePath}");
+                break;
+            }
+        }
+        
         using (var reader = new StreamReader(filePath))
         using (var csv = new CsvReader(reader, config))
         {
@@ -47,4 +67,3 @@ public static List<MovieRecord> LoadMovies()
         return _cachedMovies;
     }
 }
-

src/Infidex.Benchmark/Program.cs

@@ -11,4 +11,4 @@ static void Main(string[] args)
 
         //BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args);
     }
-}
+}

src/Infidex.Benchmark/ResultComparison.cs

@@ -88,7 +88,10 @@ public static void Run()
         // Test queries
         string[] testQueries = 
         {
-            "redeption sh"
+            "redeption sh",
+            "shaaawshank",
+            "shaa awashank",
+            "redeption"
         };
 
         foreach (var queryText in testQueries)
@@ -99,7 +102,10 @@ public static void Run()
 
             // Infidex results
             Console.WriteLine("--- INFIDEX RESULTS (Top 5) ---");
+            var sw = System.Diagnostics.Stopwatch.StartNew();
             var infidexResults = infidexEngine.Search(new Infidex.Api.Query(queryText, 5));
+            sw.Stop();
+            Console.WriteLine($"Search took: {sw.Elapsed.TotalMilliseconds:F2} ms");
             foreach (var result in infidexResults.Records.Take(5))
             {
                 var doc = infidexEngine.GetDocument(result.DocumentId);
@@ -157,4 +163,3 @@ public static void Run()
         luceneDirectory.Dispose();
     }
 }
-