public void TestUpdateIndex()
{
BuildIndex();
LucenePool.SaveResultsAndClearLucenePool(Config.LuceneIndexForCode);
var result = CodeIndexSearcher.Search(Config.LuceneIndexForCode, new TermQuery(new Term(nameof(CodeSource.FilePath) + Constants.NoneTokenizeFieldSuffix, @"D:\DDDD\A new Name.cs")), 10);
Assert.That(result.Length, Is.EqualTo(1));
CodeIndexBuilder.UpdateIndex(Config.LuceneIndexForCode, new Term(nameof(CodeSource.FilePath) + Constants.NoneTokenizeFieldSuffix, @"d:\dddd\a new name.cs"), CodeIndexBuilder.GetDocumentFromSource(new CodeSource()
{
Content = "AAA",
FileExtension = "CCC",
FilePath = "BBB",
FileName = "DDD",
IndexDate = new DateTime(1999, 12, 31),
LastWriteTimeUtc = new DateTime(2000, 1, 1)
}));
LucenePool.SaveResultsAndClearLucenePool(Config.LuceneIndexForCode);
result = CodeIndexSearcher.Search(Config.LuceneIndexForCode, new TermQuery(new Term(nameof(CodeSource.FilePath) + Constants.NoneTokenizeFieldSuffix, @"d:\dddd\a new name.cs")), 10);
Assert.That(result.Length, Is.EqualTo(0));
result = CodeIndexSearcher.Search(Config.LuceneIndexForCode, new TermQuery(new Term(nameof(CodeSource.FilePath) + Constants.NoneTokenizeFieldSuffix, "BBB")), 10);
Assert.That(result.Length, Is.EqualTo(1));
Assert.AreEqual(@"DDD", result[0].Get(nameof(CodeSource.FileName)));
Assert.AreEqual(@"CCC", result[0].Get(nameof(CodeSource.FileExtension)));
Assert.AreEqual(@"BBB", result[0].Get(nameof(CodeSource.FilePath)));
Assert.AreEqual("AAA", result[0].Get(nameof(CodeSource.Content)));
Assert.AreEqual(new DateTime(1999, 12, 31).Ticks, result[0].GetField(nameof(CodeSource.IndexDate)).GetInt64Value());
Assert.AreEqual(new DateTime(2000, 1, 1).Ticks, result[0].GetField(nameof(CodeSource.LastWriteTimeUtc)).GetInt64Value());
}
public FetchResult <IEnumerable <string> > GetHints(string word)
{
FetchResult <IEnumerable <string> > result;
try
{
word.RequireNotNullOrEmpty(nameof(word));
result = new FetchResult <IEnumerable <string> >
{
Result = CodeIndexSearcher.GetHints(codeIndexConfiguration.LuceneIndexForHint, word),
Status = new Status
{
Success = true
}
};
log.Debug($"Get Hints For '{word}' successful");
}
catch (Exception ex)
{
result = new FetchResult <IEnumerable <string> >
{
Status = new Status
{
Success = false,
StatusDesc = ex.ToString()
}
};
log.Error(ex.ToString());
}
return(result);
}
public void TestBuildIndex()
{
BuildIndex();
LucenePool.SaveResultsAndClearLucenePool(Config.LuceneIndexForCode);
var result1 = CodeIndexSearcher.Search(Config.LuceneIndexForCode, Generator.GetQueryFromStr($"{nameof(CodeSource.FileName)}:\"Dummy File\""), 10);
Assert.That(result1.Length, Is.EqualTo(1));
Assert.AreEqual(@"Dummy File", result1[0].Get(nameof(CodeSource.FileName)));
Assert.AreEqual(@"cs", result1[0].Get(nameof(CodeSource.FileExtension)));
Assert.AreEqual(@"C:\Dummy File.cs", result1[0].Get(nameof(CodeSource.FilePath)));
Assert.AreEqual(@"C:\Dummy File.cs", result1[0].Get(nameof(CodeSource.FilePath) + Constants.NoneTokenizeFieldSuffix));
Assert.AreEqual("Test Content" + Environment.NewLine + "A New Line For Test", result1[0].Get(nameof(CodeSource.Content)));
Assert.AreEqual(new DateTime(2020, 1, 1).Ticks, result1[0].GetField(nameof(CodeSource.IndexDate)).GetInt64Value());
var generator = new QueryGenerator();
var result2 = CodeIndexSearcher.Search(Config.LuceneIndexForCode, generator.GetQueryFromStr("FFFF test"), 10);
Assert.That(result2.Length, Is.EqualTo(2));
Assert.IsTrue(result2.Any(u => u.Get(nameof(CodeSource.FileName)) == "Dummy File"));
Assert.IsTrue(result2.Any(u => u.Get(nameof(CodeSource.FileName)) == "A new File"));
var result3 = CodeIndexSearcher.Search(Config.LuceneIndexForCode, generator.GetQueryFromStr("FFFF"), 10);
Assert.That(result3.Length, Is.EqualTo(1));
Assert.IsTrue(result3.Any(u => u.Get(nameof(CodeSource.FileName)) == "A new File"));
}
public void TestSearch_ReaderFromWriter()
{
CodeIndexBuilder.BuildIndex(Config, true, true, true, new[] { new CodeSource
{
FileName = "Dummy File 1",
FileExtension = "cs",
FilePath = @"C:\Dummy File 1.cs",
Content = "Test Content" + Environment.NewLine + "A New Line For Test"
},
new CodeSource
{
FileName = "Dummy File 2",
FileExtension = "cs",
FilePath = @"C:\Dummy File 2.cs",
Content = "Test Content" + Environment.NewLine + "A New Line For Test"
},
new CodeSource
{
FileName = "Dummy File 2",
FileExtension = "xml",
FilePath = @"C:\Dummy File.xml",
Content = "Test Content" + Environment.NewLine + "A New Line For Test"
} });
var results = CodeIndexSearcher.Search(Config.LuceneIndexForCode, new TermQuery(new Term(nameof(CodeSource.FileExtension), "xml")), 10);
Assert.That(results.Length, Is.EqualTo(1));
}
public FetchResult <IEnumerable <CodeSource> > GetCodeSources(string searchQuery, bool preview, string contentQuery = "", int?showResults = 0)
{
FetchResult <IEnumerable <CodeSource> > result;
try
{
searchQuery.RequireNotNullOrEmpty(nameof(searchQuery));
var showResultsValue = showResults.HasValue && showResults.Value <= 100 && showResults.Value > 0 ? showResults.Value : 1000;
result = new FetchResult <IEnumerable <CodeSource> >
{
Result = SearchCodeSource(searchQuery, out var query, showResultsValue),
Status = new Status
{
Success = true
}
};
var queryForContent = string.IsNullOrWhiteSpace(contentQuery) ? null : generator.GetQueryFromStr(contentQuery);
var maxContentHighlightLength = codeIndexConfiguration.MaxContentHighlightLength;
if (maxContentHighlightLength <= 0)
{
maxContentHighlightLength = Constants.DefaultMaxContentHighlightLength;
}
if (preview)
{
foreach (var item in result.Result)
{
item.Content = CodeIndexSearcher.GenerateHtmlPreviewText(queryForContent, item.Content, 30, generator.Analyzer, maxContentHighlightLength: maxContentHighlightLength);
}
}
else if (!preview)
{
foreach (var item in result.Result)
{
item.Content = CodeIndexSearcher.GenerateHtmlPreviewText(queryForContent, item.Content, int.MaxValue, generator.Analyzer, returnRawContentWhenResultIsEmpty: true, maxContentHighlightLength: maxContentHighlightLength);
}
}
log.Debug($"Request: '{searchQuery}' successful");
}
catch (Exception ex)
{
result = new FetchResult <IEnumerable <CodeSource> >
{
Status = new Status
{
Success = false,
StatusDesc = ex.ToString()
}
};
log.Error(ex.ToString());
}
return(result);
}
public void TestGenerateHtmlPreviewText_ContentTooLong()
{
var generator = new QueryGenerator();
var content = "My ABC\r\nIs A ABC CONTENT\r\nIt's abc in lowercase\r\nIt's Abc in mix\r\nNot AB with C";
var result = CodeIndexSearcher.GenerateHtmlPreviewText(generator.GetQueryFromStr("ABC"), content, int.MaxValue, LucenePool.GetAnalyzer(), maxContentHighlightLength: 20);
Assert.AreEqual(@"Content is too long to highlight", result);
}
public void TestGetHints()
{
WordsHintBuilder.Words.Add("Abcd");
WordsHintBuilder.BuildIndexByBatch(Config, true, true, true, null, true, 1000);
CollectionAssert.AreEquivalent(new[] { "Abcd" }, CodeIndexSearcher.GetHints(Config.LuceneIndexForHint, "abc", 20, false));
CollectionAssert.IsEmpty(CodeIndexSearcher.GetHints(Config.LuceneIndexForHint, "abc", 20, true));
CollectionAssert.AreEquivalent(new[] { "Abcd" }, CodeIndexSearcher.GetHints(Config.LuceneIndexForHint, "Abc", 20, true));
}
public void TestDeleteAllIndex()
{
BuildIndex();
LucenePool.SaveResultsAndClearLucenePool(Config.LuceneIndexForCode);
Assert.AreEqual(1, CodeIndexSearcher.Search(Config.LuceneIndexForCode, Generator.GetQueryFromStr($"{nameof(CodeSource.FileName)}:\"Dummy File\""), 10).Length);
Assert.AreEqual(1, CodeIndexSearcher.Search(Config.LuceneIndexForCode, Generator.GetQueryFromStr($"{nameof(CodeSource.FileName)}:\"A new File\""), 10).Length);
CodeIndexBuilder.DeleteAllIndex(Config);
Assert.AreEqual(0, CodeIndexSearcher.Search(Config.LuceneIndexForCode, Generator.GetQueryFromStr($"{nameof(CodeSource.FileName)}:\"Dummy File\""), 10).Length);
Assert.AreEqual(0, CodeIndexSearcher.Search(Config.LuceneIndexForCode, Generator.GetQueryFromStr($"{nameof(CodeSource.FileName)}:\"A new File\""), 10).Length);
}
public void TestGenerateHtmlPreviewText_ReturnRawContent()
{
var generator = new QueryGenerator();
var content = "My ABC\r\nIs A ABC CONTENT\r\nIt's abc in lowercase\r\nIt's Abc in mix\r\nNot AB with C";
var result = CodeIndexSearcher.GenerateHtmlPreviewText(generator.GetQueryFromStr("NotExistWord"), content, int.MaxValue, LucenePool.GetAnalyzer());
Assert.IsEmpty(result);
result = CodeIndexSearcher.GenerateHtmlPreviewText(generator.GetQueryFromStr("NotExistWord"), content, 10, LucenePool.GetAnalyzer(), returnRawContentWhenResultIsEmpty: true);
Assert.AreEqual(HttpUtility.HtmlEncode(content), result);
result = CodeIndexSearcher.GenerateHtmlPreviewText(null, content, 10, LucenePool.GetAnalyzer(), returnRawContentWhenResultIsEmpty: true);
Assert.AreEqual(HttpUtility.HtmlEncode(content), result);
}
public void TestGenerateHtmlPreviewText()
{
var generator = new QueryGenerator();
var content = "My ABC\r\nIs A ABC CONTENT\r\nIt's abc in lowercase\r\nIt's Abc in mix\r\nNot AB with C";
var result = CodeIndexSearcher.GenerateHtmlPreviewText(generator.GetQueryFromStr("ABC"), content, int.MaxValue, LucenePool.GetAnalyzer());
Assert.AreEqual(@"My <label class='highlight'>ABC</label>
Is A <label class='highlight'>ABC</label> CONTENT
It's <label class='highlight'>abc</label> in lowercase
It's <label class='highlight'>Abc</label> in mix
Not AB with C", result);
result = CodeIndexSearcher.GenerateHtmlPreviewText(generator.GetQueryFromStr("ABC"), content, 10, LucenePool.GetAnalyzer());
Assert.AreEqual(@"My <label class='highlight'>ABC</label>
Is A <label class='highlight'>ABC</label>...
It's <label class='highlight'>Abc</label>", result);
}
public void TestMaintainerIndex_RetryFailed()
{
var waitMS = 1500;
Directory.CreateDirectory(MonitorFolder);
var fileAPath = Path.Combine(MonitorFolder, "AAA.cs");
File.Create(fileAPath).Close();
File.AppendAllText(fileAPath, "12345");
Config.SaveIntervalSeconds = 1;
Config.ExcludedExtensions = ".dll";
using var maintainer = new CodeFilesIndexMaintainerForTest(Config, new DummyLog());
maintainer.StartWatch();
maintainer.SetInitalizeFinishedToTrue();
maintainer.PendingRetryCodeSources.Enqueue(new PendingRetrySource()
{
ChangesType = WatcherChangeTypes.Created,
FilePath = fileAPath,
LastRetryUTCDate = DateTime.Now.AddDays(-1)
});
var retryTime = 3;
var codeSources = Array.Empty <CodeSource>();
while (retryTime > 0)
{
Thread.Sleep(waitMS); // wait task finish saving
retryTime--;
codeSources = CodeIndexSearcher.SearchCode(Config.LuceneIndexForCode, new MatchAllDocsQuery(), 100);
if (codeSources.Length > 0)
{
break;
}
}
Assert.AreEqual(1, codeSources.Length);
Assert.AreEqual("AAA.cs", codeSources[0].FileName);
}
public void TestDeleteIndex()
{
BuildIndex();
LucenePool.SaveResultsAndClearLucenePool(Config.LuceneIndexForCode);
var generator = new QueryGenerator();
var result = CodeIndexSearcher.Search(Config.LuceneIndexForCode, generator.GetQueryFromStr("FFFF test"), 10);
Assert.That(result.Length, Is.EqualTo(2));
CodeIndexBuilder.DeleteIndex(Config.LuceneIndexForCode, new Term(nameof(CodeSource.FileExtension), "xml"));
LucenePool.SaveResultsAndClearLucenePool(Config.LuceneIndexForCode);
result = CodeIndexSearcher.Search(Config.LuceneIndexForCode, generator.GetQueryFromStr("FFFF test"), 10);
Assert.That(result.Length, Is.EqualTo(1));
CodeIndexBuilder.DeleteIndex(Config.LuceneIndexForCode, generator.GetQueryFromStr("Test"));
LucenePool.SaveResultsAndClearLucenePool(Config.LuceneIndexForCode);
result = CodeIndexSearcher.Search(Config.LuceneIndexForCode, generator.GetQueryFromStr("FFFF test"), 10);
Assert.That(result.Length, Is.EqualTo(0));
}
void FileRenamed(string oldFullPath, string fullPath, PendingRetrySource pendingRetrySource = null)
{
try
{
if (IsFile(fullPath))
{
var fileInfo = new FileInfo(fullPath);
try
{
if (fileInfo.Exists)
{
var content = FilesContentHelper.ReadAllText(fullPath);
var document = CodeIndexBuilder.GetDocumentFromSource(CodeSource.GetCodeSource(fileInfo, content));
CodeIndexBuilder.UpdateIndex(config.LuceneIndexForCode, GetNoneTokenizeFieldTerm(nameof(CodeSource.FilePath), oldFullPath), document);
pendingChanges++;
}
}
catch (IOException)
{
AddFileChangesToRetrySouce(fullPath, WatcherChangeTypes.Renamed, pendingRetrySource, oldFullPath);
}
}
else if (IsDirectory(fullPath))
{
// Rebuild All Sub Directory Index File, rename the index path
var term = new PrefixQuery(GetNoneTokenizeFieldTerm(nameof(CodeSource.FilePath), oldFullPath));
var docs = CodeIndexSearcher.Search(config.LuceneIndexForCode, term, int.MaxValue);
foreach (var doc in docs)
{
CodeIndexBuilder.UpdateCodeFilePath(doc, oldFullPath, fullPath);
CodeIndexBuilder.UpdateIndex(config.LuceneIndexForCode, new Term(nameof(CodeSource.CodePK), doc.Get(nameof(CodeSource.CodePK))), doc);
pendingChanges++;
}
}
}
catch (Exception ex)
{
log?.Error(ex.ToString());
}
}
public void TestBuildIndex_DeleteOldIndexWithSamePath()
{
BuildIndex();
LucenePool.SaveResultsAndClearLucenePool(Config.LuceneIndexForCode);
var result = CodeIndexSearcher.Search(Config.LuceneIndexForCode, Generator.GetQueryFromStr($"{nameof(CodeSource.FileName)}:\"Dummy File\""), 10);
Assert.AreEqual("Test Content" + Environment.NewLine + "A New Line For Test", result.Single().Get(nameof(CodeSource.Content)));
CodeIndexBuilder.BuildIndex(Config, true, true, true, new[] { new CodeSource
{
FileName = "Dummy File New",
FileExtension = "cs",
FilePath = @"C:\Dummy File.cs",
Content = "ABC",
IndexDate = new DateTime(2020, 1, 1)
} });
LucenePool.SaveResultsAndClearLucenePool(Config.LuceneIndexForCode);
result = CodeIndexSearcher.Search(Config.LuceneIndexForCode, Generator.GetQueryFromStr($"{nameof(CodeSource.FileName)}:\"Dummy File New\""), 10);
Assert.AreEqual("ABC", result.Single().Get(nameof(CodeSource.Content)));
}
public void TestMaintainerIndex()
{
Config.ExcludedExtensions = ".dll";
Config.SaveIntervalSeconds = 1;
var waitMS = 1500;
Directory.CreateDirectory(MonitorFolder);
Directory.CreateDirectory(Path.Combine(MonitorFolder, "FolderA"));
Directory.CreateDirectory(Path.Combine(MonitorFolder, "FolderB"));
var fileAPath = Path.Combine(MonitorFolder, "FolderA", "AAA.cs");
File.Create(fileAPath).Close();
File.AppendAllText(fileAPath, "12345");
var fileBPath = Path.Combine(MonitorFolder, "FolderB", "BBB.xml");
File.Create(fileBPath).Close();
File.AppendAllText(fileBPath, "this is a content for test, that's it\r\na new line;");
var fileCPath = Path.Combine(MonitorFolder, "CCC.xml");
File.Create(fileCPath).Close();
File.AppendAllText(fileCPath, "this is a content for test");
var fileDPath = Path.Combine(MonitorFolder, "DDD.txt");
CodeIndexBuilder.BuildIndex(Config, true, true, true,
new[]
{
CodeSource.GetCodeSource(new FileInfo(fileAPath), "12345"),
CodeSource.GetCodeSource(new FileInfo(fileBPath), "this is a content for test, that's it\r\na new line;"),
CodeSource.GetCodeSource(new FileInfo(fileCPath), "this is a content for test")
});
LucenePool.SaveResultsAndClearLucenePool(Config.LuceneIndexForCode);
var codeSources = CodeIndexSearcher.SearchCode(Config.LuceneIndexForCode, new MatchAllDocsQuery(), 100);
CollectionAssert.AreEquivalent(new[] { "AAA.cs", "BBB.xml", "CCC.xml" }, codeSources.Select(u => u.FileName));
using var maintainer = new CodeFilesIndexMaintainer(Config, new DummyLog());
maintainer.StartWatch();
maintainer.SetInitalizeFinishedToTrue();
File.AppendAllText(fileAPath, "56789"); // Changed
File.Delete(fileBPath); // Deleted
File.Move(fileCPath, Path.Combine(MonitorFolder, "NewCCC.xml")); // Rename
File.Create(fileDPath).Close(); // Created
Thread.Sleep(waitMS); // wait task finish saving
codeSources = CodeIndexSearcher.SearchCode(Config.LuceneIndexForCode, new MatchAllDocsQuery(), 100);
Assert.Multiple(() =>
{
CollectionAssert.AreEquivalent(new[] { "AAA.cs", "NewCCC.xml", "DDD.txt" }, codeSources.Select(u => u.FileName));
CollectionAssert.AreEquivalent(new[] { "1234556789", "this is a content for test", string.Empty }, codeSources.Select(u => u.Content));
CollectionAssert.AreEquivalent(new[] { fileAPath, Path.Combine(MonitorFolder, "NewCCC.xml"), fileDPath }, codeSources.Select(u => u.FilePath));
});
maintainer.Dispose();
}
CodeSource[] SearchCodeSource(string searchStr, out Query query, int showResults = 100)
{
query = generator.GetQueryFromStr(searchStr);
return(CodeIndexSearcher.SearchCode(codeIndexConfiguration.LuceneIndexForCode, query, showResults > 100 ? 100 : showResults));
}
