public void TestCommit() { using var indexBuilder = new CodeIndexBuilder("ABC", new LucenePoolLight(TempCodeIndexDir), new LucenePoolLight(TempHintIndexDir), Log); File.AppendAllText(Path.Combine(MonitorFolder, "A.txt"), "ABCD EEEE"); Assert.AreEqual(IndexBuildResults.Successful, indexBuilder.CreateIndex(new FileInfo(Path.Combine(MonitorFolder, "A.txt")))); Assert.IsFalse(IndexBuilderHelper.IndexExists(TempCodeIndexDir)); indexBuilder.Commit(); Assert.IsTrue(IndexBuilderHelper.IndexExists(TempCodeIndexDir)); using var dir = Lucene.Net.Store.FSDirectory.Open(TempCodeIndexDir); using var reader = DirectoryReader.Open(dir); var searcher = new IndexSearcher(reader); Assert.AreEqual(1, searcher.Search(new MatchAllDocsQuery(), 1).TotalHits); }
void InitializeIndexCore(bool forceRebuild) { var folders = IndexConfig.GetFolders(CodeIndexConfiguration.LuceneIndex); IndexBuilder = new CodeIndexBuilder( IndexConfig.IndexName, new LucenePoolLight(folders.CodeIndexFolder), new LucenePoolLight(folders.HintIndexFolder), Log); IndexBuilder.InitIndexFolderIfNeeded(); Status = IndexStatus.Initializing_ComponentInitializeFinished; ChangedSources = new ConcurrentQueue <ChangedSource>(); PendingRetryCodeSources = new ConcurrentQueue <PendingRetrySource>(); FilesWatcher = FilesWatcherHelper.StartWatch(IndexConfig.MonitorFolder, OnChange, OnRename); Log.LogInformation($"{IndexConfig.IndexName}: Start Watch files change"); var allFiles = FilesFetcher.FetchAllFiles(IndexConfig.MonitorFolder, IndexConfig.ExcludedExtensionsArray, IndexConfig.ExcludedPathsArray, includedExtensions: IndexConfig.IncludedExtensionsArray, isInLinux: CodeIndexConfiguration.IsInLinux).ToList(); Log.LogInformation($"{IndexConfig.IndexName}: Fetching {allFiles.Count} files need to indexing"); List <FileInfo> needToBuildIndex = null; var failedUpdateOrDeleteFiles = new List <string>(); var brandNewBuild = false; if (IndexBuilderHelper.IndexExists(IndexBuilder.CodeIndexPool.LuceneIndex)) { if (forceRebuild) { brandNewBuild = true; Log.LogInformation($"{IndexConfig.IndexName}: Force rebuild all indexes"); IndexBuilder.DeleteAllIndex(); } else { Log.LogInformation($"{IndexConfig.IndexName}: Compare index difference"); var allCodeSource = IndexBuilder.GetAllIndexedCodeSource(); GC.Collect(); // Run GC after fetching massive documents from index needToBuildIndex = new List <FileInfo>(); var allFilesDictionary = allFiles.ToDictionary(u => u.FullName); foreach (var codeSource in allCodeSource) { TokenSource.Token.ThrowIfCancellationRequested(); if (allFilesDictionary.TryGetValue(codeSource.FilePath, out var fileInfo)) { if (fileInfo.LastWriteTimeUtc != codeSource.LastWriteTimeUtc) { Log.LogInformation($"{IndexConfig.IndexName}: File {fileInfo.FullName} modified"); if (IndexBuilder.UpdateIndex(fileInfo, TokenSource.Token) != IndexBuildResults.Successful) { failedUpdateOrDeleteFiles.Add(codeSource.FilePath); } } allFilesDictionary.Remove(codeSource.FilePath); } else { Log.LogInformation($"{IndexConfig.IndexName}: File {codeSource.FilePath} deleted"); if (!IndexBuilder.DeleteIndex(codeSource.FilePath)) { failedUpdateOrDeleteFiles.Add(codeSource.FilePath); } } } foreach (var needToCreateFiles in allFilesDictionary) { Log.LogInformation($"{IndexConfig.IndexName}: Found new file {needToCreateFiles.Value.FullName}"); needToBuildIndex.Add(needToCreateFiles.Value); } } } else { brandNewBuild = true; } AddNewIndexFiles(needToBuildIndex ?? allFiles, out var failedIndexFiles, brandNewBuild); GC.Collect(); // Run GC to save the memory IndexBuilder.Commit(); if (failedIndexFiles.Count > 0 || failedUpdateOrDeleteFiles.Count > 0) { Log.LogError($"{IndexConfig.IndexName}: Initialize finished for {IndexConfig.MonitorFolder}, failed with these file(s): {string.Join(", ", failedIndexFiles.Select(u => u.FullName).Concat(failedUpdateOrDeleteFiles))}"); } else { Log.LogInformation($"{IndexConfig.IndexName}: Initialize finished for {IndexConfig.MonitorFolder}"); } }