public void TestBuildIndexByBatch()
{
using var indexBuilder = new CodeIndexBuilder("ABC", new LucenePoolLight(TempCodeIndexDir), new LucenePoolLight(TempHintIndexDir), Log);
var fileName1 = Path.Combine(MonitorFolder, "A.txt");
var fileName2 = Path.Combine(MonitorFolder, "B.txt");
File.AppendAllText(fileName1, "ABCD ABCD" + Environment.NewLine + "ABCD");
File.AppendAllText(fileName2, "ABCD EFGH");
var failedFiles = indexBuilder.BuildIndexByBatch(new[] { new FileInfo(fileName1), new FileInfo(fileName2) }, true, true, true, CancellationToken.None, true);
CollectionAssert.IsEmpty(failedFiles);
var results = indexBuilder.CodeIndexPool.SearchCode(Generator.GetQueryFromStr("ABCD", false));
Assert.AreEqual(2, results.Length);
Assert.AreEqual("A.txt", results[0].FileName);
Assert.AreEqual("txt", results[0].FileExtension);
Assert.AreEqual(fileName1, results[0].FilePath);
Assert.AreEqual("ABCD ABCD" + Environment.NewLine + "ABCD", results[0].Content);
Assert.GreaterOrEqual(DateTime.UtcNow, results[0].IndexDate);
Assert.AreEqual("B.txt", results[1].FileName);
Assert.AreEqual("txt", results[1].FileExtension);
Assert.AreEqual(fileName2, results[1].FilePath);
Assert.AreEqual("ABCD EFGH", results[1].Content);
Assert.GreaterOrEqual(DateTime.UtcNow, results[1].IndexDate);
Assert.AreEqual(1, indexBuilder.CodeIndexPool.SearchCode(Generator.GetQueryFromStr("EFGH", false)).Length);
Assert.AreEqual(1, indexBuilder.HintIndexPool.SearchWord(new TermQuery(new Term(nameof(CodeWord.Word), "ABCD"))).Length);
Assert.AreEqual(1, indexBuilder.HintIndexPool.SearchWord(new TermQuery(new Term(nameof(CodeWord.Word), "EFGH"))).Length);
}
public void TestBuildIndexByBatch_ReturnFailedFiles()
{
File.WriteAllText(Path.Combine(TempDir, "A.txt"), "ABCD");
CodeIndexBuilder.BuildIndexByBatch(Config, true, true, true, new[] { new FileInfo(Path.Combine(TempDir, "A.txt")) }, false, new DummyLog {
ThrowExceptionWhenLogContains = "Add index For " + Path.Combine(TempDir, "A.txt")
}, out var failedIndexFiles);
Assert.AreEqual(1, failedIndexFiles.Count);
CodeIndexBuilder.BuildIndexByBatch(Config, true, true, true, new[] { new FileInfo(Path.Combine(TempDir, "A.txt")), new FileInfo("BlaBla\\a.txt") }, false, null, out failedIndexFiles);
Assert.AreEqual(0, failedIndexFiles.Count);
}
public void TestNotBrandNewBuild()
{
using var indexBuilder = new CodeIndexBuilder("ABC", new LucenePoolLight(TempCodeIndexDir), new LucenePoolLight(TempHintIndexDir), Log);
Assert.IsFalse(Directory.Exists(TempCodeIndexDir));
Assert.IsFalse(Directory.Exists(TempHintIndexDir));
var fileName1 = Path.Combine(MonitorFolder, "A.txt");
var fileName2 = Path.Combine(MonitorFolder, "B.txt");
File.AppendAllText(fileName1, "ABCD ABCD");
File.AppendAllText(fileName2, "ABCD EFGH");
var failedFiles = indexBuilder.BuildIndexByBatch(new[] { new FileInfo(fileName1) }, true, true, true, CancellationToken.None, true);
CollectionAssert.IsEmpty(failedFiles);
Assert.AreEqual(1, indexBuilder.CodeIndexPool.SearchCode(new MatchAllDocsQuery()).Length);
Assert.AreEqual(1, indexBuilder.HintIndexPool.SearchWord(new MatchAllDocsQuery()).Length);
indexBuilder.BuildIndexByBatch(new[] { new FileInfo(fileName2) }, true, true, true, CancellationToken.None, false);
CollectionAssert.IsEmpty(failedFiles);
Assert.AreEqual(2, indexBuilder.CodeIndexPool.SearchCode(new MatchAllDocsQuery()).Length);
Assert.AreEqual(2, indexBuilder.HintIndexPool.SearchWord(new MatchAllDocsQuery()).Length, "When not brand new build, will do update hint index rather than add without check");
}
public void TestGetAllIndexedCodeSource()
{
using var indexBuilder = new CodeIndexBuilder("ABC", new LucenePoolLight(TempCodeIndexDir), new LucenePoolLight(TempHintIndexDir), Log);
var fileName1 = Path.Combine(MonitorFolder, "A.txt");
var fileName2 = Path.Combine(MonitorFolder, "B.txt");
File.AppendAllText(fileName1, "ABCD ABCD");
File.AppendAllText(fileName2, "EFGH");
var file1Info = new FileInfo(fileName1);
var file2Info = new FileInfo(fileName2);
var failedFiles = indexBuilder.BuildIndexByBatch(new[] { new FileInfo(fileName1), new FileInfo(fileName2) }, true, true, true, CancellationToken.None, true);
CollectionAssert.IsEmpty(failedFiles);
CollectionAssert.AreEquivalent(new[] { (file1Info.FullName, file1Info.LastWriteTimeUtc), (file2Info.FullName, file2Info.LastWriteTimeUtc) }, indexBuilder.GetAllIndexedCodeSource());
public void TestBuildIndexByBatch_FailedFiles()
{
using var indexBuilder = new CodeIndexBuilder("ABC", new LucenePoolLight(TempCodeIndexDir), new LucenePoolLight(TempHintIndexDir), Log);
var fileName1 = Path.Combine(MonitorFolder, "A.txt");
var fileName2 = Path.Combine(MonitorFolder, "B.txt");
var file = File.Create(fileName1);
File.AppendAllText(fileName2, "ABCD ABCD");
try
{
var failedFiles = indexBuilder.BuildIndexByBatch(new[] { new FileInfo(fileName1), new FileInfo(fileName2) }, true, true, true, CancellationToken.None, true);
CollectionAssert.AreEquivalent(new[] { fileName1 }, failedFiles.Select(u => u.FullName));
}
finally
{
file.Dispose();
}
}
public void TestDeleteAllIndex()
{
using var indexBuilder = new CodeIndexBuilder("ABC", new LucenePoolLight(TempCodeIndexDir), new LucenePoolLight(TempHintIndexDir), Log);
Assert.IsFalse(Directory.Exists(TempCodeIndexDir));
Assert.IsFalse(Directory.Exists(TempHintIndexDir));
var fileName1 = Path.Combine(MonitorFolder, "A.txt");
var fileName2 = Path.Combine(MonitorFolder, "B.txt");
File.AppendAllText(fileName1, "ABCD ABCD");
File.AppendAllText(fileName2, "ABCD EFGH");
var failedFiles = indexBuilder.BuildIndexByBatch(new[] { new FileInfo(fileName1), new FileInfo(fileName2) }, true, true, true, CancellationToken.None, true);
CollectionAssert.IsEmpty(failedFiles);
Assert.AreEqual(2, indexBuilder.CodeIndexPool.SearchCode(new MatchAllDocsQuery()).Length);
Assert.AreEqual(2, indexBuilder.HintIndexPool.SearchWord(new MatchAllDocsQuery()).Length);
indexBuilder.DeleteAllIndex();
Assert.AreEqual(0, indexBuilder.CodeIndexPool.SearchCode(new MatchAllDocsQuery()).Length);
Assert.AreEqual(0, indexBuilder.HintIndexPool.SearchWord(new MatchAllDocsQuery()).Length);
}
public void InitializeIndex(CodeIndexConfiguration config, out List <FileInfo> failedIndexFiles, bool forceDeleteAllIndex = false)
{
config.RequireNotNull(nameof(config));
log?.Info($"Initialize start for {config.LuceneIndex}");
var allFiles = FilesFetcher.FetchAllFiles(config.MonitorFolder, config.ExcludedExtensionsArray, config.ExcludedPathsArray, includedExtensions: config.IncludedExtensionsArray, isInLinux: config.IsInLinux).ToList();
List <FileInfo> needToBuildIndex = null;
var firstInitialize = true;
CodeIndexBuilder.InitIndexFolderIfNeeded(config, log);
if (CodeIndexBuilder.IndexExists(config.LuceneIndexForCode))
{
if (forceDeleteAllIndex)
{
log?.Info("Delete exist index");
CodeIndexBuilder.DeleteAllIndex(config);
}
else
{
firstInitialize = false;
log?.Info("Compare index difference");
var allCodeSource = CodeIndexBuilder.GetAllIndexedCodeSource(config.LuceneIndexForCode);
needToBuildIndex = new List <FileInfo>();
foreach (var codeSource in allCodeSource)
{
var fileInfo = allFiles.FirstOrDefault(u => u.FullName == codeSource.FilePath);
if (fileInfo != null)
{
if (fileInfo.LastWriteTimeUtc != codeSource.LastWriteTimeUtc)
{
log?.Info($"File {fileInfo.FullName} modified");
CodeIndexBuilder.DeleteIndex(config.LuceneIndexForCode, CodeFilesIndexMaintainer.GetNoneTokenizeFieldTerm(nameof(CodeSource.FilePath), codeSource.FilePath));
needToBuildIndex.Add(fileInfo);
}
allFiles.Remove(fileInfo);
}
else
{
log?.Info($"File {codeSource.FilePath} deleted");
CodeIndexBuilder.DeleteIndex(config.LuceneIndexForCode, CodeFilesIndexMaintainer.GetNoneTokenizeFieldTerm(nameof(CodeSource.FilePath), codeSource.FilePath));
}
}
foreach (var needToCreateFiles in allFiles)
{
log?.Info($"Found new file {needToCreateFiles.FullName}");
needToBuildIndex.Add(needToCreateFiles);
}
}
}
CodeIndexBuilder.BuildIndexByBatch(config, true, true, true, needToBuildIndex ?? allFiles, false, log, out failedIndexFiles);
LucenePool.SaveResultsAndClearLucenePool(config.LuceneIndexForCode);
WordsHintBuilder.BuildIndexByBatch(config, true, true, true, log, firstInitialize);
log?.Info($"Initialize finished for {config.LuceneIndex}");
}