public void TestUpdateIndex() { BuildIndex(); LucenePool.SaveResultsAndClearLucenePool(Config.LuceneIndexForCode); var result = CodeIndexSearcher.Search(Config.LuceneIndexForCode, new TermQuery(new Term(nameof(CodeSource.FilePath) + Constants.NoneTokenizeFieldSuffix, @"D:\DDDD\A new Name.cs")), 10); Assert.That(result.Length, Is.EqualTo(1)); CodeIndexBuilder.UpdateIndex(Config.LuceneIndexForCode, new Term(nameof(CodeSource.FilePath) + Constants.NoneTokenizeFieldSuffix, @"d:\dddd\a new name.cs"), CodeIndexBuilder.GetDocumentFromSource(new CodeSource() { Content = "AAA", FileExtension = "CCC", FilePath = "BBB", FileName = "DDD", IndexDate = new DateTime(1999, 12, 31), LastWriteTimeUtc = new DateTime(2000, 1, 1) })); LucenePool.SaveResultsAndClearLucenePool(Config.LuceneIndexForCode); result = CodeIndexSearcher.Search(Config.LuceneIndexForCode, new TermQuery(new Term(nameof(CodeSource.FilePath) + Constants.NoneTokenizeFieldSuffix, @"d:\dddd\a new name.cs")), 10); Assert.That(result.Length, Is.EqualTo(0)); result = CodeIndexSearcher.Search(Config.LuceneIndexForCode, new TermQuery(new Term(nameof(CodeSource.FilePath) + Constants.NoneTokenizeFieldSuffix, "BBB")), 10); Assert.That(result.Length, Is.EqualTo(1)); Assert.AreEqual(@"DDD", result[0].Get(nameof(CodeSource.FileName))); Assert.AreEqual(@"CCC", result[0].Get(nameof(CodeSource.FileExtension))); Assert.AreEqual(@"BBB", result[0].Get(nameof(CodeSource.FilePath))); Assert.AreEqual("AAA", result[0].Get(nameof(CodeSource.Content))); Assert.AreEqual(new DateTime(1999, 12, 31).Ticks, result[0].GetField(nameof(CodeSource.IndexDate)).GetInt64Value()); Assert.AreEqual(new DateTime(2000, 1, 1).Ticks, result[0].GetField(nameof(CodeSource.LastWriteTimeUtc)).GetInt64Value()); }
public FetchResult <IEnumerable <string> > GetHints(string word) { FetchResult <IEnumerable <string> > result; try { word.RequireNotNullOrEmpty(nameof(word)); result = new FetchResult <IEnumerable <string> > { Result = CodeIndexSearcher.GetHints(codeIndexConfiguration.LuceneIndexForHint, word), Status = new Status { Success = true } }; log.Debug($"Get Hints For '{word}' successful"); } catch (Exception ex) { result = new FetchResult <IEnumerable <string> > { Status = new Status { Success = false, StatusDesc = ex.ToString() } }; log.Error(ex.ToString()); } return(result); }
public void TestBuildIndex() { BuildIndex(); LucenePool.SaveResultsAndClearLucenePool(Config.LuceneIndexForCode); var result1 = CodeIndexSearcher.Search(Config.LuceneIndexForCode, Generator.GetQueryFromStr($"{nameof(CodeSource.FileName)}:\"Dummy File\""), 10); Assert.That(result1.Length, Is.EqualTo(1)); Assert.AreEqual(@"Dummy File", result1[0].Get(nameof(CodeSource.FileName))); Assert.AreEqual(@"cs", result1[0].Get(nameof(CodeSource.FileExtension))); Assert.AreEqual(@"C:\Dummy File.cs", result1[0].Get(nameof(CodeSource.FilePath))); Assert.AreEqual(@"C:\Dummy File.cs", result1[0].Get(nameof(CodeSource.FilePath) + Constants.NoneTokenizeFieldSuffix)); Assert.AreEqual("Test Content" + Environment.NewLine + "A New Line For Test", result1[0].Get(nameof(CodeSource.Content))); Assert.AreEqual(new DateTime(2020, 1, 1).Ticks, result1[0].GetField(nameof(CodeSource.IndexDate)).GetInt64Value()); var generator = new QueryGenerator(); var result2 = CodeIndexSearcher.Search(Config.LuceneIndexForCode, generator.GetQueryFromStr("FFFF test"), 10); Assert.That(result2.Length, Is.EqualTo(2)); Assert.IsTrue(result2.Any(u => u.Get(nameof(CodeSource.FileName)) == "Dummy File")); Assert.IsTrue(result2.Any(u => u.Get(nameof(CodeSource.FileName)) == "A new File")); var result3 = CodeIndexSearcher.Search(Config.LuceneIndexForCode, generator.GetQueryFromStr("FFFF"), 10); Assert.That(result3.Length, Is.EqualTo(1)); Assert.IsTrue(result3.Any(u => u.Get(nameof(CodeSource.FileName)) == "A new File")); }
public void TestSearch_ReaderFromWriter() { CodeIndexBuilder.BuildIndex(Config, true, true, true, new[] { new CodeSource { FileName = "Dummy File 1", FileExtension = "cs", FilePath = @"C:\Dummy File 1.cs", Content = "Test Content" + Environment.NewLine + "A New Line For Test" }, new CodeSource { FileName = "Dummy File 2", FileExtension = "cs", FilePath = @"C:\Dummy File 2.cs", Content = "Test Content" + Environment.NewLine + "A New Line For Test" }, new CodeSource { FileName = "Dummy File 2", FileExtension = "xml", FilePath = @"C:\Dummy File.xml", Content = "Test Content" + Environment.NewLine + "A New Line For Test" } }); var results = CodeIndexSearcher.Search(Config.LuceneIndexForCode, new TermQuery(new Term(nameof(CodeSource.FileExtension), "xml")), 10); Assert.That(results.Length, Is.EqualTo(1)); }
public FetchResult <IEnumerable <CodeSource> > GetCodeSources(string searchQuery, bool preview, string contentQuery = "", int?showResults = 0) { FetchResult <IEnumerable <CodeSource> > result; try { searchQuery.RequireNotNullOrEmpty(nameof(searchQuery)); var showResultsValue = showResults.HasValue && showResults.Value <= 100 && showResults.Value > 0 ? showResults.Value : 1000; result = new FetchResult <IEnumerable <CodeSource> > { Result = SearchCodeSource(searchQuery, out var query, showResultsValue), Status = new Status { Success = true } }; var queryForContent = string.IsNullOrWhiteSpace(contentQuery) ? null : generator.GetQueryFromStr(contentQuery); var maxContentHighlightLength = codeIndexConfiguration.MaxContentHighlightLength; if (maxContentHighlightLength <= 0) { maxContentHighlightLength = Constants.DefaultMaxContentHighlightLength; } if (preview) { foreach (var item in result.Result) { item.Content = CodeIndexSearcher.GenerateHtmlPreviewText(queryForContent, item.Content, 30, generator.Analyzer, maxContentHighlightLength: maxContentHighlightLength); } } else if (!preview) { foreach (var item in result.Result) { item.Content = CodeIndexSearcher.GenerateHtmlPreviewText(queryForContent, item.Content, int.MaxValue, generator.Analyzer, returnRawContentWhenResultIsEmpty: true, maxContentHighlightLength: maxContentHighlightLength); } } log.Debug($"Request: '{searchQuery}' successful"); } catch (Exception ex) { result = new FetchResult <IEnumerable <CodeSource> > { Status = new Status { Success = false, StatusDesc = ex.ToString() } }; log.Error(ex.ToString()); } return(result); }
public void TestGenerateHtmlPreviewText_ContentTooLong() { var generator = new QueryGenerator(); var content = "My ABC\r\nIs A ABC CONTENT\r\nIt's abc in lowercase\r\nIt's Abc in mix\r\nNot AB with C"; var result = CodeIndexSearcher.GenerateHtmlPreviewText(generator.GetQueryFromStr("ABC"), content, int.MaxValue, LucenePool.GetAnalyzer(), maxContentHighlightLength: 20); Assert.AreEqual(@"Content is too long to highlight", result); }
public void TestGetHints() { WordsHintBuilder.Words.Add("Abcd"); WordsHintBuilder.BuildIndexByBatch(Config, true, true, true, null, true, 1000); CollectionAssert.AreEquivalent(new[] { "Abcd" }, CodeIndexSearcher.GetHints(Config.LuceneIndexForHint, "abc", 20, false)); CollectionAssert.IsEmpty(CodeIndexSearcher.GetHints(Config.LuceneIndexForHint, "abc", 20, true)); CollectionAssert.AreEquivalent(new[] { "Abcd" }, CodeIndexSearcher.GetHints(Config.LuceneIndexForHint, "Abc", 20, true)); }
public void TestDeleteAllIndex() { BuildIndex(); LucenePool.SaveResultsAndClearLucenePool(Config.LuceneIndexForCode); Assert.AreEqual(1, CodeIndexSearcher.Search(Config.LuceneIndexForCode, Generator.GetQueryFromStr($"{nameof(CodeSource.FileName)}:\"Dummy File\""), 10).Length); Assert.AreEqual(1, CodeIndexSearcher.Search(Config.LuceneIndexForCode, Generator.GetQueryFromStr($"{nameof(CodeSource.FileName)}:\"A new File\""), 10).Length); CodeIndexBuilder.DeleteAllIndex(Config); Assert.AreEqual(0, CodeIndexSearcher.Search(Config.LuceneIndexForCode, Generator.GetQueryFromStr($"{nameof(CodeSource.FileName)}:\"Dummy File\""), 10).Length); Assert.AreEqual(0, CodeIndexSearcher.Search(Config.LuceneIndexForCode, Generator.GetQueryFromStr($"{nameof(CodeSource.FileName)}:\"A new File\""), 10).Length); }
public void TestGenerateHtmlPreviewText_ReturnRawContent() { var generator = new QueryGenerator(); var content = "My ABC\r\nIs A ABC CONTENT\r\nIt's abc in lowercase\r\nIt's Abc in mix\r\nNot AB with C"; var result = CodeIndexSearcher.GenerateHtmlPreviewText(generator.GetQueryFromStr("NotExistWord"), content, int.MaxValue, LucenePool.GetAnalyzer()); Assert.IsEmpty(result); result = CodeIndexSearcher.GenerateHtmlPreviewText(generator.GetQueryFromStr("NotExistWord"), content, 10, LucenePool.GetAnalyzer(), returnRawContentWhenResultIsEmpty: true); Assert.AreEqual(HttpUtility.HtmlEncode(content), result); result = CodeIndexSearcher.GenerateHtmlPreviewText(null, content, 10, LucenePool.GetAnalyzer(), returnRawContentWhenResultIsEmpty: true); Assert.AreEqual(HttpUtility.HtmlEncode(content), result); }
public void TestGenerateHtmlPreviewText() { var generator = new QueryGenerator(); var content = "My ABC\r\nIs A ABC CONTENT\r\nIt's abc in lowercase\r\nIt's Abc in mix\r\nNot AB with C"; var result = CodeIndexSearcher.GenerateHtmlPreviewText(generator.GetQueryFromStr("ABC"), content, int.MaxValue, LucenePool.GetAnalyzer()); Assert.AreEqual(@"My <label class='highlight'>ABC</label> Is A <label class='highlight'>ABC</label> CONTENT It's <label class='highlight'>abc</label> in lowercase It's <label class='highlight'>Abc</label> in mix Not AB with C", result); result = CodeIndexSearcher.GenerateHtmlPreviewText(generator.GetQueryFromStr("ABC"), content, 10, LucenePool.GetAnalyzer()); Assert.AreEqual(@"My <label class='highlight'>ABC</label> Is A <label class='highlight'>ABC</label>... It's <label class='highlight'>Abc</label>", result); }
public void TestMaintainerIndex_RetryFailed() { var waitMS = 1500; Directory.CreateDirectory(MonitorFolder); var fileAPath = Path.Combine(MonitorFolder, "AAA.cs"); File.Create(fileAPath).Close(); File.AppendAllText(fileAPath, "12345"); Config.SaveIntervalSeconds = 1; Config.ExcludedExtensions = ".dll"; using var maintainer = new CodeFilesIndexMaintainerForTest(Config, new DummyLog()); maintainer.StartWatch(); maintainer.SetInitalizeFinishedToTrue(); maintainer.PendingRetryCodeSources.Enqueue(new PendingRetrySource() { ChangesType = WatcherChangeTypes.Created, FilePath = fileAPath, LastRetryUTCDate = DateTime.Now.AddDays(-1) }); var retryTime = 3; var codeSources = Array.Empty <CodeSource>(); while (retryTime > 0) { Thread.Sleep(waitMS); // wait task finish saving retryTime--; codeSources = CodeIndexSearcher.SearchCode(Config.LuceneIndexForCode, new MatchAllDocsQuery(), 100); if (codeSources.Length > 0) { break; } } Assert.AreEqual(1, codeSources.Length); Assert.AreEqual("AAA.cs", codeSources[0].FileName); }
public void TestDeleteIndex() { BuildIndex(); LucenePool.SaveResultsAndClearLucenePool(Config.LuceneIndexForCode); var generator = new QueryGenerator(); var result = CodeIndexSearcher.Search(Config.LuceneIndexForCode, generator.GetQueryFromStr("FFFF test"), 10); Assert.That(result.Length, Is.EqualTo(2)); CodeIndexBuilder.DeleteIndex(Config.LuceneIndexForCode, new Term(nameof(CodeSource.FileExtension), "xml")); LucenePool.SaveResultsAndClearLucenePool(Config.LuceneIndexForCode); result = CodeIndexSearcher.Search(Config.LuceneIndexForCode, generator.GetQueryFromStr("FFFF test"), 10); Assert.That(result.Length, Is.EqualTo(1)); CodeIndexBuilder.DeleteIndex(Config.LuceneIndexForCode, generator.GetQueryFromStr("Test")); LucenePool.SaveResultsAndClearLucenePool(Config.LuceneIndexForCode); result = CodeIndexSearcher.Search(Config.LuceneIndexForCode, generator.GetQueryFromStr("FFFF test"), 10); Assert.That(result.Length, Is.EqualTo(0)); }
void FileRenamed(string oldFullPath, string fullPath, PendingRetrySource pendingRetrySource = null) { try { if (IsFile(fullPath)) { var fileInfo = new FileInfo(fullPath); try { if (fileInfo.Exists) { var content = FilesContentHelper.ReadAllText(fullPath); var document = CodeIndexBuilder.GetDocumentFromSource(CodeSource.GetCodeSource(fileInfo, content)); CodeIndexBuilder.UpdateIndex(config.LuceneIndexForCode, GetNoneTokenizeFieldTerm(nameof(CodeSource.FilePath), oldFullPath), document); pendingChanges++; } } catch (IOException) { AddFileChangesToRetrySouce(fullPath, WatcherChangeTypes.Renamed, pendingRetrySource, oldFullPath); } } else if (IsDirectory(fullPath)) { // Rebuild All Sub Directory Index File, rename the index path var term = new PrefixQuery(GetNoneTokenizeFieldTerm(nameof(CodeSource.FilePath), oldFullPath)); var docs = CodeIndexSearcher.Search(config.LuceneIndexForCode, term, int.MaxValue); foreach (var doc in docs) { CodeIndexBuilder.UpdateCodeFilePath(doc, oldFullPath, fullPath); CodeIndexBuilder.UpdateIndex(config.LuceneIndexForCode, new Term(nameof(CodeSource.CodePK), doc.Get(nameof(CodeSource.CodePK))), doc); pendingChanges++; } } } catch (Exception ex) { log?.Error(ex.ToString()); } }
public void TestBuildIndex_DeleteOldIndexWithSamePath() { BuildIndex(); LucenePool.SaveResultsAndClearLucenePool(Config.LuceneIndexForCode); var result = CodeIndexSearcher.Search(Config.LuceneIndexForCode, Generator.GetQueryFromStr($"{nameof(CodeSource.FileName)}:\"Dummy File\""), 10); Assert.AreEqual("Test Content" + Environment.NewLine + "A New Line For Test", result.Single().Get(nameof(CodeSource.Content))); CodeIndexBuilder.BuildIndex(Config, true, true, true, new[] { new CodeSource { FileName = "Dummy File New", FileExtension = "cs", FilePath = @"C:\Dummy File.cs", Content = "ABC", IndexDate = new DateTime(2020, 1, 1) } }); LucenePool.SaveResultsAndClearLucenePool(Config.LuceneIndexForCode); result = CodeIndexSearcher.Search(Config.LuceneIndexForCode, Generator.GetQueryFromStr($"{nameof(CodeSource.FileName)}:\"Dummy File New\""), 10); Assert.AreEqual("ABC", result.Single().Get(nameof(CodeSource.Content))); }
public void TestMaintainerIndex() { Config.ExcludedExtensions = ".dll"; Config.SaveIntervalSeconds = 1; var waitMS = 1500; Directory.CreateDirectory(MonitorFolder); Directory.CreateDirectory(Path.Combine(MonitorFolder, "FolderA")); Directory.CreateDirectory(Path.Combine(MonitorFolder, "FolderB")); var fileAPath = Path.Combine(MonitorFolder, "FolderA", "AAA.cs"); File.Create(fileAPath).Close(); File.AppendAllText(fileAPath, "12345"); var fileBPath = Path.Combine(MonitorFolder, "FolderB", "BBB.xml"); File.Create(fileBPath).Close(); File.AppendAllText(fileBPath, "this is a content for test, that's it\r\na new line;"); var fileCPath = Path.Combine(MonitorFolder, "CCC.xml"); File.Create(fileCPath).Close(); File.AppendAllText(fileCPath, "this is a content for test"); var fileDPath = Path.Combine(MonitorFolder, "DDD.txt"); CodeIndexBuilder.BuildIndex(Config, true, true, true, new[] { CodeSource.GetCodeSource(new FileInfo(fileAPath), "12345"), CodeSource.GetCodeSource(new FileInfo(fileBPath), "this is a content for test, that's it\r\na new line;"), CodeSource.GetCodeSource(new FileInfo(fileCPath), "this is a content for test") }); LucenePool.SaveResultsAndClearLucenePool(Config.LuceneIndexForCode); var codeSources = CodeIndexSearcher.SearchCode(Config.LuceneIndexForCode, new MatchAllDocsQuery(), 100); CollectionAssert.AreEquivalent(new[] { "AAA.cs", "BBB.xml", "CCC.xml" }, codeSources.Select(u => u.FileName)); using var maintainer = new CodeFilesIndexMaintainer(Config, new DummyLog()); maintainer.StartWatch(); maintainer.SetInitalizeFinishedToTrue(); File.AppendAllText(fileAPath, "56789"); // Changed File.Delete(fileBPath); // Deleted File.Move(fileCPath, Path.Combine(MonitorFolder, "NewCCC.xml")); // Rename File.Create(fileDPath).Close(); // Created Thread.Sleep(waitMS); // wait task finish saving codeSources = CodeIndexSearcher.SearchCode(Config.LuceneIndexForCode, new MatchAllDocsQuery(), 100); Assert.Multiple(() => { CollectionAssert.AreEquivalent(new[] { "AAA.cs", "NewCCC.xml", "DDD.txt" }, codeSources.Select(u => u.FileName)); CollectionAssert.AreEquivalent(new[] { "1234556789", "this is a content for test", string.Empty }, codeSources.Select(u => u.Content)); CollectionAssert.AreEquivalent(new[] { fileAPath, Path.Combine(MonitorFolder, "NewCCC.xml"), fileDPath }, codeSources.Select(u => u.FilePath)); }); maintainer.Dispose(); }
CodeSource[] SearchCodeSource(string searchStr, out Query query, int showResults = 100) { query = generator.GetQueryFromStr(searchStr); return(CodeIndexSearcher.SearchCode(codeIndexConfiguration.LuceneIndexForCode, query, showResults > 100 ? 100 : showResults)); }