public void TestFetchFiles() { Directory.CreateDirectory(Path.Combine(TempDir, "SubDir")); Directory.CreateDirectory(Path.Combine(TempDir, "bin\\debug")); File.Create(Path.Combine(TempDir, "AAA.cs")).Close(); File.Create(Path.Combine(TempDir, "SubDir", "ddd.txt")).Close(); File.Create(Path.Combine(TempDir, "SubDir", "ddd.xml")).Close(); File.Create(Path.Combine(TempDir, "bin\\debug", "ddd.txt")).Close(); var files = FilesFetcher.FetchAllFiles(TempDir, new[] { ".xml" }, new[] { "BIN\\" }).ToArray(); Assert.That(files.Length, Is.EqualTo(2)); CollectionAssert.AreEquivalent(files.Select(u => u.Name), new[] { "AAA.cs", "ddd.txt" }); files = FilesFetcher.FetchAllFiles(TempDir, new[] { ".xml" }, new[] { "BIN\\" }, "*.cs").ToArray(); Assert.That(files.Length, Is.EqualTo(1)); CollectionAssert.AreEquivalent(files.Select(u => u.Name), new[] { "AAA.cs" }); }
void InitializeIndexCore(bool forceRebuild) { var folders = IndexConfig.GetFolders(CodeIndexConfiguration.LuceneIndex); IndexBuilder = new CodeIndexBuilder( IndexConfig.IndexName, new LucenePoolLight(folders.CodeIndexFolder), new LucenePoolLight(folders.HintIndexFolder), Log); IndexBuilder.InitIndexFolderIfNeeded(); Status = IndexStatus.Initializing_ComponentInitializeFinished; ChangedSources = new ConcurrentQueue <ChangedSource>(); PendingRetryCodeSources = new ConcurrentQueue <PendingRetrySource>(); FilesWatcher = FilesWatcherHelper.StartWatch(IndexConfig.MonitorFolder, OnChange, OnRename); Log.LogInformation($"{IndexConfig.IndexName}: Start Watch files change"); var allFiles = FilesFetcher.FetchAllFiles(IndexConfig.MonitorFolder, IndexConfig.ExcludedExtensionsArray, IndexConfig.ExcludedPathsArray, includedExtensions: IndexConfig.IncludedExtensionsArray, isInLinux: CodeIndexConfiguration.IsInLinux).ToList(); Log.LogInformation($"{IndexConfig.IndexName}: Fetching {allFiles.Count} files need to indexing"); List <FileInfo> needToBuildIndex = null; var failedUpdateOrDeleteFiles = new List <string>(); var brandNewBuild = false; if (IndexBuilderHelper.IndexExists(IndexBuilder.CodeIndexPool.LuceneIndex)) { if (forceRebuild) { brandNewBuild = true; Log.LogInformation($"{IndexConfig.IndexName}: Force rebuild all indexes"); IndexBuilder.DeleteAllIndex(); } else { Log.LogInformation($"{IndexConfig.IndexName}: Compare index difference"); var allCodeSource = IndexBuilder.GetAllIndexedCodeSource(); GC.Collect(); // Run GC after fetching massive documents from index needToBuildIndex = new List <FileInfo>(); var allFilesDictionary = allFiles.ToDictionary(u => u.FullName); foreach (var codeSource in allCodeSource) { TokenSource.Token.ThrowIfCancellationRequested(); if (allFilesDictionary.TryGetValue(codeSource.FilePath, out var fileInfo)) { if (fileInfo.LastWriteTimeUtc != codeSource.LastWriteTimeUtc) { Log.LogInformation($"{IndexConfig.IndexName}: File {fileInfo.FullName} modified"); if (IndexBuilder.UpdateIndex(fileInfo, TokenSource.Token) != IndexBuildResults.Successful) { failedUpdateOrDeleteFiles.Add(codeSource.FilePath); } } allFilesDictionary.Remove(codeSource.FilePath); } else { Log.LogInformation($"{IndexConfig.IndexName}: File {codeSource.FilePath} deleted"); if (!IndexBuilder.DeleteIndex(codeSource.FilePath)) { failedUpdateOrDeleteFiles.Add(codeSource.FilePath); } } } foreach (var needToCreateFiles in allFilesDictionary) { Log.LogInformation($"{IndexConfig.IndexName}: Found new file {needToCreateFiles.Value.FullName}"); needToBuildIndex.Add(needToCreateFiles.Value); } } } else { brandNewBuild = true; } AddNewIndexFiles(needToBuildIndex ?? allFiles, out var failedIndexFiles, brandNewBuild); GC.Collect(); // Run GC to save the memory IndexBuilder.Commit(); if (failedIndexFiles.Count > 0 || failedUpdateOrDeleteFiles.Count > 0) { Log.LogError($"{IndexConfig.IndexName}: Initialize finished for {IndexConfig.MonitorFolder}, failed with these file(s): {string.Join(", ", failedIndexFiles.Select(u => u.FullName).Concat(failedUpdateOrDeleteFiles))}"); } else { Log.LogInformation($"{IndexConfig.IndexName}: Initialize finished for {IndexConfig.MonitorFolder}"); } }
public void InitializeIndex(CodeIndexConfiguration config, out List <FileInfo> failedIndexFiles, bool forceDeleteAllIndex = false) { config.RequireNotNull(nameof(config)); log?.Info($"Initialize start for {config.LuceneIndex}"); var allFiles = FilesFetcher.FetchAllFiles(config.MonitorFolder, config.ExcludedExtensionsArray, config.ExcludedPathsArray, includedExtensions: config.IncludedExtensionsArray, isInLinux: config.IsInLinux).ToList(); List <FileInfo> needToBuildIndex = null; var firstInitialize = true; CodeIndexBuilder.InitIndexFolderIfNeeded(config, log); if (CodeIndexBuilder.IndexExists(config.LuceneIndexForCode)) { if (forceDeleteAllIndex) { log?.Info("Delete exist index"); CodeIndexBuilder.DeleteAllIndex(config); } else { firstInitialize = false; log?.Info("Compare index difference"); var allCodeSource = CodeIndexBuilder.GetAllIndexedCodeSource(config.LuceneIndexForCode); needToBuildIndex = new List <FileInfo>(); foreach (var codeSource in allCodeSource) { var fileInfo = allFiles.FirstOrDefault(u => u.FullName == codeSource.FilePath); if (fileInfo != null) { if (fileInfo.LastWriteTimeUtc != codeSource.LastWriteTimeUtc) { log?.Info($"File {fileInfo.FullName} modified"); CodeIndexBuilder.DeleteIndex(config.LuceneIndexForCode, CodeFilesIndexMaintainer.GetNoneTokenizeFieldTerm(nameof(CodeSource.FilePath), codeSource.FilePath)); needToBuildIndex.Add(fileInfo); } allFiles.Remove(fileInfo); } else { log?.Info($"File {codeSource.FilePath} deleted"); CodeIndexBuilder.DeleteIndex(config.LuceneIndexForCode, CodeFilesIndexMaintainer.GetNoneTokenizeFieldTerm(nameof(CodeSource.FilePath), codeSource.FilePath)); } } foreach (var needToCreateFiles in allFiles) { log?.Info($"Found new file {needToCreateFiles.FullName}"); needToBuildIndex.Add(needToCreateFiles); } } } CodeIndexBuilder.BuildIndexByBatch(config, true, true, true, needToBuildIndex ?? allFiles, false, log, out failedIndexFiles); LucenePool.SaveResultsAndClearLucenePool(config.LuceneIndexForCode); WordsHintBuilder.BuildIndexByBatch(config, true, true, true, log, firstInitialize); log?.Info($"Initialize finished for {config.LuceneIndex}"); }