void DeleteIndex(string fullPath) { try { CodeIndexBuilder.DeleteIndex(config.LuceneIndexForCode, GetNoneTokenizeFieldTerm(nameof(CodeSource.FilePath), fullPath)); pendingChanges++; } catch (Exception ex) { log?.Error(ex.ToString()); } }
void DeleteAllDocumentsIndexUnder(string forderOldFullPath) { try { var term = new PrefixQuery(GetNoneTokenizeFieldTerm(nameof(CodeSource.FilePath), forderOldFullPath)); CodeIndexBuilder.DeleteIndex(config.LuceneIndexForCode, term); pendingChanges++; } catch (Exception ex) { log?.Error(ex.ToString()); } }
public void TestDeleteIndex() { using var indexBuilder = new CodeIndexBuilder("ABC", new LucenePoolLight(TempCodeIndexDir), new LucenePoolLight(TempHintIndexDir), Log); File.AppendAllText(Path.Combine(MonitorFolder, "A.txt"), "ABCD EEEE dddd"); File.AppendAllText(Path.Combine(MonitorFolder, "B.txt"), "ABCD DDDD eeEE"); Assert.AreEqual(IndexBuildResults.Successful, indexBuilder.CreateIndex(new FileInfo(Path.Combine(MonitorFolder, "A.txt")))); Assert.AreEqual(IndexBuildResults.Successful, indexBuilder.CreateIndex(new FileInfo(Path.Combine(MonitorFolder, "B.txt")))); Assert.AreEqual(2, indexBuilder.CodeIndexPool.SearchCode(new MatchAllDocsQuery()).Length); CollectionAssert.AreEquivalent(new[] { "ABCD", "EEEE", "DDDD", "dddd", "eeEE" }, indexBuilder.HintIndexPool.SearchWord(new MatchAllDocsQuery()).Select(u => u.Word)); Assert.IsTrue(indexBuilder.DeleteIndex(Path.Combine(MonitorFolder, "A.txt"))); Assert.AreEqual(1, indexBuilder.CodeIndexPool.SearchCode(new MatchAllDocsQuery()).Length); CollectionAssert.AreEquivalent(new[] { "ABCD", "DDDD", "eeEE" }, indexBuilder.HintIndexPool.SearchWord(new MatchAllDocsQuery()).Select(u => u.Word), "Words Been Removed If Needed"); }
public void TestDeleteIndex() { BuildIndex(); LucenePool.SaveResultsAndClearLucenePool(Config.LuceneIndexForCode); var generator = new QueryGenerator(); var result = CodeIndexSearcher.Search(Config.LuceneIndexForCode, generator.GetQueryFromStr("FFFF test"), 10); Assert.That(result.Length, Is.EqualTo(2)); CodeIndexBuilder.DeleteIndex(Config.LuceneIndexForCode, new Term(nameof(CodeSource.FileExtension), "xml")); LucenePool.SaveResultsAndClearLucenePool(Config.LuceneIndexForCode); result = CodeIndexSearcher.Search(Config.LuceneIndexForCode, generator.GetQueryFromStr("FFFF test"), 10); Assert.That(result.Length, Is.EqualTo(1)); CodeIndexBuilder.DeleteIndex(Config.LuceneIndexForCode, generator.GetQueryFromStr("Test")); LucenePool.SaveResultsAndClearLucenePool(Config.LuceneIndexForCode); result = CodeIndexSearcher.Search(Config.LuceneIndexForCode, generator.GetQueryFromStr("FFFF test"), 10); Assert.That(result.Length, Is.EqualTo(0)); }
public void TestDeleteIndex_FolderDelete() { using var indexBuilder = new CodeIndexBuilder("ABC", new LucenePoolLight(TempCodeIndexDir), new LucenePoolLight(TempHintIndexDir), Log); Directory.CreateDirectory(Path.Combine(MonitorFolder, "SubDir")); File.AppendAllText(Path.Combine(MonitorFolder, "SubDir.txt"), "1234"); File.AppendAllText(Path.Combine(MonitorFolder, "SubDir", "A.txt"), "5678"); File.AppendAllText(Path.Combine(MonitorFolder, "SubDir", "B.txt"), "5678"); Assert.AreEqual(IndexBuildResults.Successful, indexBuilder.CreateIndex(new FileInfo(Path.Combine(MonitorFolder, "SubDir.txt")))); Assert.AreEqual(IndexBuildResults.Successful, indexBuilder.CreateIndex(new FileInfo(Path.Combine(MonitorFolder, "SubDir", "A.txt")))); Assert.AreEqual(IndexBuildResults.Successful, indexBuilder.CreateIndex(new FileInfo(Path.Combine(MonitorFolder, "SubDir", "B.txt")))); Assert.AreEqual(3, indexBuilder.CodeIndexPool.SearchCode(new MatchAllDocsQuery()).Length); Assert.IsTrue(indexBuilder.DeleteIndex(Path.Combine(MonitorFolder, "SubDir"))); var results = indexBuilder.CodeIndexPool.SearchCode(new MatchAllDocsQuery()); Assert.AreEqual(1, results.Length, "Should not delete the index of SubDir.txt"); Assert.AreEqual("SubDir.txt", results[0].FileName); }
void InitializeIndexCore(bool forceRebuild) { var folders = IndexConfig.GetFolders(CodeIndexConfiguration.LuceneIndex); IndexBuilder = new CodeIndexBuilder( IndexConfig.IndexName, new LucenePoolLight(folders.CodeIndexFolder), new LucenePoolLight(folders.HintIndexFolder), Log); IndexBuilder.InitIndexFolderIfNeeded(); Status = IndexStatus.Initializing_ComponentInitializeFinished; ChangedSources = new ConcurrentQueue <ChangedSource>(); PendingRetryCodeSources = new ConcurrentQueue <PendingRetrySource>(); FilesWatcher = FilesWatcherHelper.StartWatch(IndexConfig.MonitorFolder, OnChange, OnRename); Log.LogInformation($"{IndexConfig.IndexName}: Start Watch files change"); var allFiles = FilesFetcher.FetchAllFiles(IndexConfig.MonitorFolder, IndexConfig.ExcludedExtensionsArray, IndexConfig.ExcludedPathsArray, includedExtensions: IndexConfig.IncludedExtensionsArray, isInLinux: CodeIndexConfiguration.IsInLinux).ToList(); Log.LogInformation($"{IndexConfig.IndexName}: Fetching {allFiles.Count} files need to indexing"); List <FileInfo> needToBuildIndex = null; var failedUpdateOrDeleteFiles = new List <string>(); var brandNewBuild = false; if (IndexBuilderHelper.IndexExists(IndexBuilder.CodeIndexPool.LuceneIndex)) { if (forceRebuild) { brandNewBuild = true; Log.LogInformation($"{IndexConfig.IndexName}: Force rebuild all indexes"); IndexBuilder.DeleteAllIndex(); } else { Log.LogInformation($"{IndexConfig.IndexName}: Compare index difference"); var allCodeSource = IndexBuilder.GetAllIndexedCodeSource(); GC.Collect(); // Run GC after fetching massive documents from index needToBuildIndex = new List <FileInfo>(); var allFilesDictionary = allFiles.ToDictionary(u => u.FullName); foreach (var codeSource in allCodeSource) { TokenSource.Token.ThrowIfCancellationRequested(); if (allFilesDictionary.TryGetValue(codeSource.FilePath, out var fileInfo)) { if (fileInfo.LastWriteTimeUtc != codeSource.LastWriteTimeUtc) { Log.LogInformation($"{IndexConfig.IndexName}: File {fileInfo.FullName} modified"); if (IndexBuilder.UpdateIndex(fileInfo, TokenSource.Token) != IndexBuildResults.Successful) { failedUpdateOrDeleteFiles.Add(codeSource.FilePath); } } allFilesDictionary.Remove(codeSource.FilePath); } else { Log.LogInformation($"{IndexConfig.IndexName}: File {codeSource.FilePath} deleted"); if (!IndexBuilder.DeleteIndex(codeSource.FilePath)) { failedUpdateOrDeleteFiles.Add(codeSource.FilePath); } } } foreach (var needToCreateFiles in allFilesDictionary) { Log.LogInformation($"{IndexConfig.IndexName}: Found new file {needToCreateFiles.Value.FullName}"); needToBuildIndex.Add(needToCreateFiles.Value); } } } else { brandNewBuild = true; } AddNewIndexFiles(needToBuildIndex ?? allFiles, out var failedIndexFiles, brandNewBuild); GC.Collect(); // Run GC to save the memory IndexBuilder.Commit(); if (failedIndexFiles.Count > 0 || failedUpdateOrDeleteFiles.Count > 0) { Log.LogError($"{IndexConfig.IndexName}: Initialize finished for {IndexConfig.MonitorFolder}, failed with these file(s): {string.Join(", ", failedIndexFiles.Select(u => u.FullName).Concat(failedUpdateOrDeleteFiles))}"); } else { Log.LogInformation($"{IndexConfig.IndexName}: Initialize finished for {IndexConfig.MonitorFolder}"); } }
public void InitializeIndex(CodeIndexConfiguration config, out List <FileInfo> failedIndexFiles, bool forceDeleteAllIndex = false) { config.RequireNotNull(nameof(config)); log?.Info($"Initialize start for {config.LuceneIndex}"); var allFiles = FilesFetcher.FetchAllFiles(config.MonitorFolder, config.ExcludedExtensionsArray, config.ExcludedPathsArray, includedExtensions: config.IncludedExtensionsArray, isInLinux: config.IsInLinux).ToList(); List <FileInfo> needToBuildIndex = null; var firstInitialize = true; CodeIndexBuilder.InitIndexFolderIfNeeded(config, log); if (CodeIndexBuilder.IndexExists(config.LuceneIndexForCode)) { if (forceDeleteAllIndex) { log?.Info("Delete exist index"); CodeIndexBuilder.DeleteAllIndex(config); } else { firstInitialize = false; log?.Info("Compare index difference"); var allCodeSource = CodeIndexBuilder.GetAllIndexedCodeSource(config.LuceneIndexForCode); needToBuildIndex = new List <FileInfo>(); foreach (var codeSource in allCodeSource) { var fileInfo = allFiles.FirstOrDefault(u => u.FullName == codeSource.FilePath); if (fileInfo != null) { if (fileInfo.LastWriteTimeUtc != codeSource.LastWriteTimeUtc) { log?.Info($"File {fileInfo.FullName} modified"); CodeIndexBuilder.DeleteIndex(config.LuceneIndexForCode, CodeFilesIndexMaintainer.GetNoneTokenizeFieldTerm(nameof(CodeSource.FilePath), codeSource.FilePath)); needToBuildIndex.Add(fileInfo); } allFiles.Remove(fileInfo); } else { log?.Info($"File {codeSource.FilePath} deleted"); CodeIndexBuilder.DeleteIndex(config.LuceneIndexForCode, CodeFilesIndexMaintainer.GetNoneTokenizeFieldTerm(nameof(CodeSource.FilePath), codeSource.FilePath)); } } foreach (var needToCreateFiles in allFiles) { log?.Info($"Found new file {needToCreateFiles.FullName}"); needToBuildIndex.Add(needToCreateFiles); } } } CodeIndexBuilder.BuildIndexByBatch(config, true, true, true, needToBuildIndex ?? allFiles, false, log, out failedIndexFiles); LucenePool.SaveResultsAndClearLucenePool(config.LuceneIndexForCode); WordsHintBuilder.BuildIndexByBatch(config, true, true, true, log, firstInitialize); log?.Info($"Initialize finished for {config.LuceneIndex}"); }