public void TestUpdateIndex()
{
BuildIndex();
LucenePool.SaveResultsAndClearLucenePool(Config.LuceneIndexForCode);
var result = CodeIndexSearcher.Search(Config.LuceneIndexForCode, new TermQuery(new Term(nameof(CodeSource.FilePath) + Constants.NoneTokenizeFieldSuffix, @"D:\DDDD\A new Name.cs")), 10);
Assert.That(result.Length, Is.EqualTo(1));
CodeIndexBuilder.UpdateIndex(Config.LuceneIndexForCode, new Term(nameof(CodeSource.FilePath) + Constants.NoneTokenizeFieldSuffix, @"d:\dddd\a new name.cs"), CodeIndexBuilder.GetDocumentFromSource(new CodeSource()
{
Content = "AAA",
FileExtension = "CCC",
FilePath = "BBB",
FileName = "DDD",
IndexDate = new DateTime(1999, 12, 31),
LastWriteTimeUtc = new DateTime(2000, 1, 1)
}));
LucenePool.SaveResultsAndClearLucenePool(Config.LuceneIndexForCode);
result = CodeIndexSearcher.Search(Config.LuceneIndexForCode, new TermQuery(new Term(nameof(CodeSource.FilePath) + Constants.NoneTokenizeFieldSuffix, @"d:\dddd\a new name.cs")), 10);
Assert.That(result.Length, Is.EqualTo(0));
result = CodeIndexSearcher.Search(Config.LuceneIndexForCode, new TermQuery(new Term(nameof(CodeSource.FilePath) + Constants.NoneTokenizeFieldSuffix, "BBB")), 10);
Assert.That(result.Length, Is.EqualTo(1));
Assert.AreEqual(@"DDD", result[0].Get(nameof(CodeSource.FileName)));
Assert.AreEqual(@"CCC", result[0].Get(nameof(CodeSource.FileExtension)));
Assert.AreEqual(@"BBB", result[0].Get(nameof(CodeSource.FilePath)));
Assert.AreEqual("AAA", result[0].Get(nameof(CodeSource.Content)));
Assert.AreEqual(new DateTime(1999, 12, 31).Ticks, result[0].GetField(nameof(CodeSource.IndexDate)).GetInt64Value());
Assert.AreEqual(new DateTime(2000, 1, 1).Ticks, result[0].GetField(nameof(CodeSource.LastWriteTimeUtc)).GetInt64Value());
}
public void Dispose()
{
FileSystemWatcher.EnableRaisingEvents = false;
FileSystemWatcher.Dispose();
tokenSource.Cancel();
LucenePool.SaveResultsAndClearLucenePool(config);
}
public void TestBuildIndex()
{
BuildIndex();
LucenePool.SaveResultsAndClearLucenePool(Config.LuceneIndexForCode);
var result1 = CodeIndexSearcher.Search(Config.LuceneIndexForCode, Generator.GetQueryFromStr($"{nameof(CodeSource.FileName)}:\"Dummy File\""), 10);
Assert.That(result1.Length, Is.EqualTo(1));
Assert.AreEqual(@"Dummy File", result1[0].Get(nameof(CodeSource.FileName)));
Assert.AreEqual(@"cs", result1[0].Get(nameof(CodeSource.FileExtension)));
Assert.AreEqual(@"C:\Dummy File.cs", result1[0].Get(nameof(CodeSource.FilePath)));
Assert.AreEqual(@"C:\Dummy File.cs", result1[0].Get(nameof(CodeSource.FilePath) + Constants.NoneTokenizeFieldSuffix));
Assert.AreEqual("Test Content" + Environment.NewLine + "A New Line For Test", result1[0].Get(nameof(CodeSource.Content)));
Assert.AreEqual(new DateTime(2020, 1, 1).Ticks, result1[0].GetField(nameof(CodeSource.IndexDate)).GetInt64Value());
var generator = new QueryGenerator();
var result2 = CodeIndexSearcher.Search(Config.LuceneIndexForCode, generator.GetQueryFromStr("FFFF test"), 10);
Assert.That(result2.Length, Is.EqualTo(2));
Assert.IsTrue(result2.Any(u => u.Get(nameof(CodeSource.FileName)) == "Dummy File"));
Assert.IsTrue(result2.Any(u => u.Get(nameof(CodeSource.FileName)) == "A new File"));
var result3 = CodeIndexSearcher.Search(Config.LuceneIndexForCode, generator.GetQueryFromStr("FFFF"), 10);
Assert.That(result3.Length, Is.EqualTo(1));
Assert.IsTrue(result3.Any(u => u.Get(nameof(CodeSource.FileName)) == "A new File"));
}
public void TestSearch_NewReader()
{
CodeIndexBuilder.BuildIndex(Config, true, true, true, new[] { new CodeSource
{
FileName = "Dummy File 1",
FileExtension = "cs",
FilePath = @"C:\Dummy File 1.cs",
Content = "Test Content" + Environment.NewLine + "A New Line For Test"
},
new CodeSource
{
FileName = "Dummy File 2",
FileExtension = "cs",
FilePath = @"C:\Dummy File 2.cs",
Content = "Test Content" + Environment.NewLine + "A New Line For Test"
},
new CodeSource
{
FileName = "Dummy File 2",
FileExtension = "xml",
FilePath = @"C:\Dummy File.xml",
Content = "Test Content" + Environment.NewLine + "A New Line For Test"
} });
LucenePool.SaveResultsAndClearLucenePool(Config.LuceneIndexForCode);
var results1 = CodeIndexSearcher.Search(Config.LuceneIndexForCode, new TermQuery(new Term(nameof(CodeSource.FileExtension), "cs")), 10);
Assert.That(results1.Length, Is.EqualTo(2));
var results2 = CodeIndexSearcher.Search(Config.LuceneIndexForCode, new TermQuery(new Term(nameof(CodeSource.FileExtension), "cs")), 1);
Assert.That(results2.Length, Is.EqualTo(1));
}
protected virtual void TearDown()
{
LucenePool.SaveResultsAndClearLucenePool(Config);
WordsHintBuilder.Words.Clear();
DeleteAllFilesInTempDir(TempDir);
}
public void TestThreadSafeForIndexReader()
{
CodeIndexBuilder.BuildIndex(Config, true, true, true, new[] {
new CodeSource
{
FileName = "Dummy File 1",
FileExtension = "cs",
FilePath = @"C:\Dummy File 1.cs",
Content = "Test Content" + Environment.NewLine + "A New Line For Test"
},
new CodeSource
{
FileName = "Dummy File 2",
FileExtension = "cs",
FilePath = @"C:\Dummy File 2.cs",
Content = "Test Content" + Environment.NewLine + "A New Line For Test"
},
new CodeSource
{
FileName = "Dummy File 2",
FileExtension = "xml",
FilePath = @"C:\Dummy File.xml",
Content = "Test Content" + Environment.NewLine + "A New Line For Test"
}
});
LucenePool.SaveResultsAndClearLucenePool(Config);
var taskList = new List <Task>();
for (int i = 0; i < 20; i++)
{
taskList.Add(Task.Run(() =>
{
for (int j = 0; j < 50; j++)
{
if (j % 4 == 0)
{
CodeIndexBuilder.BuildIndex(Config, true, true, true, new[] {
new CodeSource
{
FileName = $"Dummy File 1 {i} {j}",
FileExtension = "cs",
FilePath = $@"C:\Dummy File 1 {i} {j}.cs",
Content = "Test Content" + Environment.NewLine + "A New Line For Test"
}
});
}
LucenePool.Search(Config.LuceneIndexForCode, new MatchAllDocsQuery(), int.MaxValue);
LucenePool.Search(Config.LuceneIndexForCode, new MatchAllDocsQuery(), int.MaxValue);
}
}));
}
Assert.DoesNotThrowAsync(async() => await Task.WhenAll(taskList));
}
public void TestDeleteAllIndex()
{
BuildIndex();
LucenePool.SaveResultsAndClearLucenePool(Config.LuceneIndexForCode);
Assert.AreEqual(1, CodeIndexSearcher.Search(Config.LuceneIndexForCode, Generator.GetQueryFromStr($"{nameof(CodeSource.FileName)}:\"Dummy File\""), 10).Length);
Assert.AreEqual(1, CodeIndexSearcher.Search(Config.LuceneIndexForCode, Generator.GetQueryFromStr($"{nameof(CodeSource.FileName)}:\"A new File\""), 10).Length);
CodeIndexBuilder.DeleteAllIndex(Config);
Assert.AreEqual(0, CodeIndexSearcher.Search(Config.LuceneIndexForCode, Generator.GetQueryFromStr($"{nameof(CodeSource.FileName)}:\"Dummy File\""), 10).Length);
Assert.AreEqual(0, CodeIndexSearcher.Search(Config.LuceneIndexForCode, Generator.GetQueryFromStr($"{nameof(CodeSource.FileName)}:\"A new File\""), 10).Length);
}
public void TestIndexExists()
{
Assert.IsFalse(CodeIndexBuilder.IndexExists(Config.LuceneIndexForCode));
BuildIndex();
Assert.IsFalse(CodeIndexBuilder.IndexExists(Config.LuceneIndexForCode));
LucenePool.SaveResultsAndClearLucenePool(Config.LuceneIndexForCode);
Assert.IsTrue(CodeIndexBuilder.IndexExists(Config.LuceneIndexForCode));
}
void SaveLuceneResultsWhenNeeded(CancellationToken token)
{
while (!token.IsCancellationRequested)
{
token.ThrowIfCancellationRequested();
if (InitializeFinished && (pendingChanges > 100 || pendingChanges > 0 && (DateTime.UtcNow - lastSaveDate).Seconds >= saveIntervalSeconds))
{
pendingChanges = 0;
LucenePool.SaveResultsAndClearLucenePool(config);
lastSaveDate = DateTime.UtcNow;
log?.Info($"Save all pending changes successful");
}
else
{
Thread.Sleep(saveIntervalSeconds * 100); // Sleep when nothing need to save
}
}
}
public void TestDeleteIndex()
{
BuildIndex();
LucenePool.SaveResultsAndClearLucenePool(Config.LuceneIndexForCode);
var generator = new QueryGenerator();
var result = CodeIndexSearcher.Search(Config.LuceneIndexForCode, generator.GetQueryFromStr("FFFF test"), 10);
Assert.That(result.Length, Is.EqualTo(2));
CodeIndexBuilder.DeleteIndex(Config.LuceneIndexForCode, new Term(nameof(CodeSource.FileExtension), "xml"));
LucenePool.SaveResultsAndClearLucenePool(Config.LuceneIndexForCode);
result = CodeIndexSearcher.Search(Config.LuceneIndexForCode, generator.GetQueryFromStr("FFFF test"), 10);
Assert.That(result.Length, Is.EqualTo(1));
CodeIndexBuilder.DeleteIndex(Config.LuceneIndexForCode, generator.GetQueryFromStr("Test"));
LucenePool.SaveResultsAndClearLucenePool(Config.LuceneIndexForCode);
result = CodeIndexSearcher.Search(Config.LuceneIndexForCode, generator.GetQueryFromStr("FFFF test"), 10);
Assert.That(result.Length, Is.EqualTo(0));
}
public void TestBuildIndex_DeleteOldIndexWithSamePath()
{
BuildIndex();
LucenePool.SaveResultsAndClearLucenePool(Config.LuceneIndexForCode);
var result = CodeIndexSearcher.Search(Config.LuceneIndexForCode, Generator.GetQueryFromStr($"{nameof(CodeSource.FileName)}:\"Dummy File\""), 10);
Assert.AreEqual("Test Content" + Environment.NewLine + "A New Line For Test", result.Single().Get(nameof(CodeSource.Content)));
CodeIndexBuilder.BuildIndex(Config, true, true, true, new[] { new CodeSource
{
FileName = "Dummy File New",
FileExtension = "cs",
FilePath = @"C:\Dummy File.cs",
Content = "ABC",
IndexDate = new DateTime(2020, 1, 1)
} });
LucenePool.SaveResultsAndClearLucenePool(Config.LuceneIndexForCode);
result = CodeIndexSearcher.Search(Config.LuceneIndexForCode, Generator.GetQueryFromStr($"{nameof(CodeSource.FileName)}:\"Dummy File New\""), 10);
Assert.AreEqual("ABC", result.Single().Get(nameof(CodeSource.Content)));
}
public void TestMaintainerIndex()
{
Config.ExcludedExtensions = ".dll";
Config.SaveIntervalSeconds = 1;
var waitMS = 1500;
Directory.CreateDirectory(MonitorFolder);
Directory.CreateDirectory(Path.Combine(MonitorFolder, "FolderA"));
Directory.CreateDirectory(Path.Combine(MonitorFolder, "FolderB"));
var fileAPath = Path.Combine(MonitorFolder, "FolderA", "AAA.cs");
File.Create(fileAPath).Close();
File.AppendAllText(fileAPath, "12345");
var fileBPath = Path.Combine(MonitorFolder, "FolderB", "BBB.xml");
File.Create(fileBPath).Close();
File.AppendAllText(fileBPath, "this is a content for test, that's it\r\na new line;");
var fileCPath = Path.Combine(MonitorFolder, "CCC.xml");
File.Create(fileCPath).Close();
File.AppendAllText(fileCPath, "this is a content for test");
var fileDPath = Path.Combine(MonitorFolder, "DDD.txt");
CodeIndexBuilder.BuildIndex(Config, true, true, true,
new[]
{
CodeSource.GetCodeSource(new FileInfo(fileAPath), "12345"),
CodeSource.GetCodeSource(new FileInfo(fileBPath), "this is a content for test, that's it\r\na new line;"),
CodeSource.GetCodeSource(new FileInfo(fileCPath), "this is a content for test")
});
LucenePool.SaveResultsAndClearLucenePool(Config.LuceneIndexForCode);
var codeSources = CodeIndexSearcher.SearchCode(Config.LuceneIndexForCode, new MatchAllDocsQuery(), 100);
CollectionAssert.AreEquivalent(new[] { "AAA.cs", "BBB.xml", "CCC.xml" }, codeSources.Select(u => u.FileName));
using var maintainer = new CodeFilesIndexMaintainer(Config, new DummyLog());
maintainer.StartWatch();
maintainer.SetInitalizeFinishedToTrue();
File.AppendAllText(fileAPath, "56789"); // Changed
File.Delete(fileBPath); // Deleted
File.Move(fileCPath, Path.Combine(MonitorFolder, "NewCCC.xml")); // Rename
File.Create(fileDPath).Close(); // Created
Thread.Sleep(waitMS); // wait task finish saving
codeSources = CodeIndexSearcher.SearchCode(Config.LuceneIndexForCode, new MatchAllDocsQuery(), 100);
Assert.Multiple(() =>
{
CollectionAssert.AreEquivalent(new[] { "AAA.cs", "NewCCC.xml", "DDD.txt" }, codeSources.Select(u => u.FileName));
CollectionAssert.AreEquivalent(new[] { "1234556789", "this is a content for test", string.Empty }, codeSources.Select(u => u.Content));
CollectionAssert.AreEquivalent(new[] { fileAPath, Path.Combine(MonitorFolder, "NewCCC.xml"), fileDPath }, codeSources.Select(u => u.FilePath));
});
maintainer.Dispose();
}
public void InitializeIndex(CodeIndexConfiguration config, out List <FileInfo> failedIndexFiles, bool forceDeleteAllIndex = false)
{
config.RequireNotNull(nameof(config));
log?.Info($"Initialize start for {config.LuceneIndex}");
var allFiles = FilesFetcher.FetchAllFiles(config.MonitorFolder, config.ExcludedExtensionsArray, config.ExcludedPathsArray, includedExtensions: config.IncludedExtensionsArray, isInLinux: config.IsInLinux).ToList();
List <FileInfo> needToBuildIndex = null;
var firstInitialize = true;
CodeIndexBuilder.InitIndexFolderIfNeeded(config, log);
if (CodeIndexBuilder.IndexExists(config.LuceneIndexForCode))
{
if (forceDeleteAllIndex)
{
log?.Info("Delete exist index");
CodeIndexBuilder.DeleteAllIndex(config);
}
else
{
firstInitialize = false;
log?.Info("Compare index difference");
var allCodeSource = CodeIndexBuilder.GetAllIndexedCodeSource(config.LuceneIndexForCode);
needToBuildIndex = new List <FileInfo>();
foreach (var codeSource in allCodeSource)
{
var fileInfo = allFiles.FirstOrDefault(u => u.FullName == codeSource.FilePath);
if (fileInfo != null)
{
if (fileInfo.LastWriteTimeUtc != codeSource.LastWriteTimeUtc)
{
log?.Info($"File {fileInfo.FullName} modified");
CodeIndexBuilder.DeleteIndex(config.LuceneIndexForCode, CodeFilesIndexMaintainer.GetNoneTokenizeFieldTerm(nameof(CodeSource.FilePath), codeSource.FilePath));
needToBuildIndex.Add(fileInfo);
}
allFiles.Remove(fileInfo);
}
else
{
log?.Info($"File {codeSource.FilePath} deleted");
CodeIndexBuilder.DeleteIndex(config.LuceneIndexForCode, CodeFilesIndexMaintainer.GetNoneTokenizeFieldTerm(nameof(CodeSource.FilePath), codeSource.FilePath));
}
}
foreach (var needToCreateFiles in allFiles)
{
log?.Info($"Found new file {needToCreateFiles.FullName}");
needToBuildIndex.Add(needToCreateFiles);
}
}
}
CodeIndexBuilder.BuildIndexByBatch(config, true, true, true, needToBuildIndex ?? allFiles, false, log, out failedIndexFiles);
LucenePool.SaveResultsAndClearLucenePool(config.LuceneIndexForCode);
WordsHintBuilder.BuildIndexByBatch(config, true, true, true, log, firstInitialize);
log?.Info($"Initialize finished for {config.LuceneIndex}");
}
void OnShutdown()
{
maintainer?.Dispose();
LucenePool.SaveResultsAndClearLucenePool(config);
}