public void TestBuildIndexByBatch() { using var indexBuilder = new CodeIndexBuilder("ABC", new LucenePoolLight(TempCodeIndexDir), new LucenePoolLight(TempHintIndexDir), Log); var fileName1 = Path.Combine(MonitorFolder, "A.txt"); var fileName2 = Path.Combine(MonitorFolder, "B.txt"); File.AppendAllText(fileName1, "ABCD ABCD" + Environment.NewLine + "ABCD"); File.AppendAllText(fileName2, "ABCD EFGH"); var failedFiles = indexBuilder.BuildIndexByBatch(new[] { new FileInfo(fileName1), new FileInfo(fileName2) }, true, true, true, CancellationToken.None, true); CollectionAssert.IsEmpty(failedFiles); var results = indexBuilder.CodeIndexPool.SearchCode(Generator.GetQueryFromStr("ABCD", false)); Assert.AreEqual(2, results.Length); Assert.AreEqual("A.txt", results[0].FileName); Assert.AreEqual("txt", results[0].FileExtension); Assert.AreEqual(fileName1, results[0].FilePath); Assert.AreEqual("ABCD ABCD" + Environment.NewLine + "ABCD", results[0].Content); Assert.GreaterOrEqual(DateTime.UtcNow, results[0].IndexDate); Assert.AreEqual("B.txt", results[1].FileName); Assert.AreEqual("txt", results[1].FileExtension); Assert.AreEqual(fileName2, results[1].FilePath); Assert.AreEqual("ABCD EFGH", results[1].Content); Assert.GreaterOrEqual(DateTime.UtcNow, results[1].IndexDate); Assert.AreEqual(1, indexBuilder.CodeIndexPool.SearchCode(Generator.GetQueryFromStr("EFGH", false)).Length); Assert.AreEqual(1, indexBuilder.HintIndexPool.SearchWord(new TermQuery(new Term(nameof(CodeWord.Word), "ABCD"))).Length); Assert.AreEqual(1, indexBuilder.HintIndexPool.SearchWord(new TermQuery(new Term(nameof(CodeWord.Word), "EFGH"))).Length); }
public void TestBuildIndexByBatch_ReturnFailedFiles() { File.WriteAllText(Path.Combine(TempDir, "A.txt"), "ABCD"); CodeIndexBuilder.BuildIndexByBatch(Config, true, true, true, new[] { new FileInfo(Path.Combine(TempDir, "A.txt")) }, false, new DummyLog { ThrowExceptionWhenLogContains = "Add index For " + Path.Combine(TempDir, "A.txt") }, out var failedIndexFiles); Assert.AreEqual(1, failedIndexFiles.Count); CodeIndexBuilder.BuildIndexByBatch(Config, true, true, true, new[] { new FileInfo(Path.Combine(TempDir, "A.txt")), new FileInfo("BlaBla\\a.txt") }, false, null, out failedIndexFiles); Assert.AreEqual(0, failedIndexFiles.Count); }
public void TestNotBrandNewBuild() { using var indexBuilder = new CodeIndexBuilder("ABC", new LucenePoolLight(TempCodeIndexDir), new LucenePoolLight(TempHintIndexDir), Log); Assert.IsFalse(Directory.Exists(TempCodeIndexDir)); Assert.IsFalse(Directory.Exists(TempHintIndexDir)); var fileName1 = Path.Combine(MonitorFolder, "A.txt"); var fileName2 = Path.Combine(MonitorFolder, "B.txt"); File.AppendAllText(fileName1, "ABCD ABCD"); File.AppendAllText(fileName2, "ABCD EFGH"); var failedFiles = indexBuilder.BuildIndexByBatch(new[] { new FileInfo(fileName1) }, true, true, true, CancellationToken.None, true); CollectionAssert.IsEmpty(failedFiles); Assert.AreEqual(1, indexBuilder.CodeIndexPool.SearchCode(new MatchAllDocsQuery()).Length); Assert.AreEqual(1, indexBuilder.HintIndexPool.SearchWord(new MatchAllDocsQuery()).Length); indexBuilder.BuildIndexByBatch(new[] { new FileInfo(fileName2) }, true, true, true, CancellationToken.None, false); CollectionAssert.IsEmpty(failedFiles); Assert.AreEqual(2, indexBuilder.CodeIndexPool.SearchCode(new MatchAllDocsQuery()).Length); Assert.AreEqual(2, indexBuilder.HintIndexPool.SearchWord(new MatchAllDocsQuery()).Length, "When not brand new build, will do update hint index rather than add without check"); }
public void TestGetAllIndexedCodeSource() { using var indexBuilder = new CodeIndexBuilder("ABC", new LucenePoolLight(TempCodeIndexDir), new LucenePoolLight(TempHintIndexDir), Log); var fileName1 = Path.Combine(MonitorFolder, "A.txt"); var fileName2 = Path.Combine(MonitorFolder, "B.txt"); File.AppendAllText(fileName1, "ABCD ABCD"); File.AppendAllText(fileName2, "EFGH"); var file1Info = new FileInfo(fileName1); var file2Info = new FileInfo(fileName2); var failedFiles = indexBuilder.BuildIndexByBatch(new[] { new FileInfo(fileName1), new FileInfo(fileName2) }, true, true, true, CancellationToken.None, true); CollectionAssert.IsEmpty(failedFiles); CollectionAssert.AreEquivalent(new[] { (file1Info.FullName, file1Info.LastWriteTimeUtc), (file2Info.FullName, file2Info.LastWriteTimeUtc) }, indexBuilder.GetAllIndexedCodeSource());
public void TestBuildIndexByBatch_FailedFiles() { using var indexBuilder = new CodeIndexBuilder("ABC", new LucenePoolLight(TempCodeIndexDir), new LucenePoolLight(TempHintIndexDir), Log); var fileName1 = Path.Combine(MonitorFolder, "A.txt"); var fileName2 = Path.Combine(MonitorFolder, "B.txt"); var file = File.Create(fileName1); File.AppendAllText(fileName2, "ABCD ABCD"); try { var failedFiles = indexBuilder.BuildIndexByBatch(new[] { new FileInfo(fileName1), new FileInfo(fileName2) }, true, true, true, CancellationToken.None, true); CollectionAssert.AreEquivalent(new[] { fileName1 }, failedFiles.Select(u => u.FullName)); } finally { file.Dispose(); } }
public void TestDeleteAllIndex() { using var indexBuilder = new CodeIndexBuilder("ABC", new LucenePoolLight(TempCodeIndexDir), new LucenePoolLight(TempHintIndexDir), Log); Assert.IsFalse(Directory.Exists(TempCodeIndexDir)); Assert.IsFalse(Directory.Exists(TempHintIndexDir)); var fileName1 = Path.Combine(MonitorFolder, "A.txt"); var fileName2 = Path.Combine(MonitorFolder, "B.txt"); File.AppendAllText(fileName1, "ABCD ABCD"); File.AppendAllText(fileName2, "ABCD EFGH"); var failedFiles = indexBuilder.BuildIndexByBatch(new[] { new FileInfo(fileName1), new FileInfo(fileName2) }, true, true, true, CancellationToken.None, true); CollectionAssert.IsEmpty(failedFiles); Assert.AreEqual(2, indexBuilder.CodeIndexPool.SearchCode(new MatchAllDocsQuery()).Length); Assert.AreEqual(2, indexBuilder.HintIndexPool.SearchWord(new MatchAllDocsQuery()).Length); indexBuilder.DeleteAllIndex(); Assert.AreEqual(0, indexBuilder.CodeIndexPool.SearchCode(new MatchAllDocsQuery()).Length); Assert.AreEqual(0, indexBuilder.HintIndexPool.SearchWord(new MatchAllDocsQuery()).Length); }
public void InitializeIndex(CodeIndexConfiguration config, out List <FileInfo> failedIndexFiles, bool forceDeleteAllIndex = false) { config.RequireNotNull(nameof(config)); log?.Info($"Initialize start for {config.LuceneIndex}"); var allFiles = FilesFetcher.FetchAllFiles(config.MonitorFolder, config.ExcludedExtensionsArray, config.ExcludedPathsArray, includedExtensions: config.IncludedExtensionsArray, isInLinux: config.IsInLinux).ToList(); List <FileInfo> needToBuildIndex = null; var firstInitialize = true; CodeIndexBuilder.InitIndexFolderIfNeeded(config, log); if (CodeIndexBuilder.IndexExists(config.LuceneIndexForCode)) { if (forceDeleteAllIndex) { log?.Info("Delete exist index"); CodeIndexBuilder.DeleteAllIndex(config); } else { firstInitialize = false; log?.Info("Compare index difference"); var allCodeSource = CodeIndexBuilder.GetAllIndexedCodeSource(config.LuceneIndexForCode); needToBuildIndex = new List <FileInfo>(); foreach (var codeSource in allCodeSource) { var fileInfo = allFiles.FirstOrDefault(u => u.FullName == codeSource.FilePath); if (fileInfo != null) { if (fileInfo.LastWriteTimeUtc != codeSource.LastWriteTimeUtc) { log?.Info($"File {fileInfo.FullName} modified"); CodeIndexBuilder.DeleteIndex(config.LuceneIndexForCode, CodeFilesIndexMaintainer.GetNoneTokenizeFieldTerm(nameof(CodeSource.FilePath), codeSource.FilePath)); needToBuildIndex.Add(fileInfo); } allFiles.Remove(fileInfo); } else { log?.Info($"File {codeSource.FilePath} deleted"); CodeIndexBuilder.DeleteIndex(config.LuceneIndexForCode, CodeFilesIndexMaintainer.GetNoneTokenizeFieldTerm(nameof(CodeSource.FilePath), codeSource.FilePath)); } } foreach (var needToCreateFiles in allFiles) { log?.Info($"Found new file {needToCreateFiles.FullName}"); needToBuildIndex.Add(needToCreateFiles); } } } CodeIndexBuilder.BuildIndexByBatch(config, true, true, true, needToBuildIndex ?? allFiles, false, log, out failedIndexFiles); LucenePool.SaveResultsAndClearLucenePool(config.LuceneIndexForCode); WordsHintBuilder.BuildIndexByBatch(config, true, true, true, log, firstInitialize); log?.Info($"Initialize finished for {config.LuceneIndex}"); }