public void TestUpdateIndex()
{
BuildIndex();
LucenePool.SaveResultsAndClearLucenePool(Config.LuceneIndexForCode);
var result = CodeIndexSearcher.Search(Config.LuceneIndexForCode, new TermQuery(new Term(nameof(CodeSource.FilePath) + Constants.NoneTokenizeFieldSuffix, @"D:\DDDD\A new Name.cs")), 10);
Assert.That(result.Length, Is.EqualTo(1));
CodeIndexBuilder.UpdateIndex(Config.LuceneIndexForCode, new Term(nameof(CodeSource.FilePath) + Constants.NoneTokenizeFieldSuffix, @"d:\dddd\a new name.cs"), CodeIndexBuilder.GetDocumentFromSource(new CodeSource()
{
Content = "AAA",
FileExtension = "CCC",
FilePath = "BBB",
FileName = "DDD",
IndexDate = new DateTime(1999, 12, 31),
LastWriteTimeUtc = new DateTime(2000, 1, 1)
}));
LucenePool.SaveResultsAndClearLucenePool(Config.LuceneIndexForCode);
result = CodeIndexSearcher.Search(Config.LuceneIndexForCode, new TermQuery(new Term(nameof(CodeSource.FilePath) + Constants.NoneTokenizeFieldSuffix, @"d:\dddd\a new name.cs")), 10);
Assert.That(result.Length, Is.EqualTo(0));
result = CodeIndexSearcher.Search(Config.LuceneIndexForCode, new TermQuery(new Term(nameof(CodeSource.FilePath) + Constants.NoneTokenizeFieldSuffix, "BBB")), 10);
Assert.That(result.Length, Is.EqualTo(1));
Assert.AreEqual(@"DDD", result[0].Get(nameof(CodeSource.FileName)));
Assert.AreEqual(@"CCC", result[0].Get(nameof(CodeSource.FileExtension)));
Assert.AreEqual(@"BBB", result[0].Get(nameof(CodeSource.FilePath)));
Assert.AreEqual("AAA", result[0].Get(nameof(CodeSource.Content)));
Assert.AreEqual(new DateTime(1999, 12, 31).Ticks, result[0].GetField(nameof(CodeSource.IndexDate)).GetInt64Value());
Assert.AreEqual(new DateTime(2000, 1, 1).Ticks, result[0].GetField(nameof(CodeSource.LastWriteTimeUtc)).GetInt64Value());
}
void UpdateIndex(string fullPath, PendingRetrySource pendingRetrySource = null)
{
if (IsFile(fullPath))
{
var fileInfo = new FileInfo(fullPath);
try
{
Thread.Sleep(WaitMilliseconds); // Wait to let file finished write to disk
if (fileInfo.Exists)
{
var content = FilesContentHelper.ReadAllText(fullPath);
var document = CodeIndexBuilder.GetDocumentFromSource(CodeSource.GetCodeSource(fileInfo, content));
CodeIndexBuilder.UpdateIndex(config.LuceneIndexForCode, GetNoneTokenizeFieldTerm(nameof(CodeSource.FilePath), fullPath), document);
WordsHintBuilder.UpdateWordsHint(config, WordSegmenter.GetWords(content), log);
pendingChanges++;
}
}
catch (IOException)
{
AddFileChangesToRetrySouce(fullPath, WatcherChangeTypes.Changed, pendingRetrySource);
}
catch (Exception ex)
{
log?.Error(ex.ToString());
}
}
}
public void TestUpdateIndex()
{
using var indexBuilder = new CodeIndexBuilder("ABC", new LucenePoolLight(TempCodeIndexDir), new LucenePoolLight(TempHintIndexDir), Log);
var fileName = Path.Combine(MonitorFolder, "A.txt");
File.AppendAllText(fileName, "ABCD Eeee EEEE");
Assert.AreEqual(IndexBuildResults.Successful, indexBuilder.CreateIndex(new FileInfo(fileName)));
Assert.AreEqual("ABCD Eeee EEEE", indexBuilder.CodeIndexPool.SearchCode(new MatchAllDocsQuery()).First().Content);
CollectionAssert.AreEquivalent(new[] { "ABCD", "Eeee", "EEEE" }, indexBuilder.HintIndexPool.SearchWord(new MatchAllDocsQuery()).Select(u => u.Word));
File.Delete(fileName);
File.AppendAllText(fileName, "WOWO IT IS NEW CONTENT APPLY ABCD EEEE");
Assert.AreEqual(IndexBuildResults.Successful, indexBuilder.UpdateIndex(new FileInfo(fileName), CancellationToken.None));
Assert.AreEqual("WOWO IT IS NEW CONTENT APPLY ABCD EEEE", indexBuilder.CodeIndexPool.SearchCode(new MatchAllDocsQuery()).First().Content);
CollectionAssert.AreEquivalent(new[] { "WOWO", "ABCD", "CONTENT", "APPLY", "EEEE" }, indexBuilder.HintIndexPool.SearchWord(new MatchAllDocsQuery()).Select(u => u.Word), "Words Been Added And Removed If Needed");
var fileName2 = Path.Combine(MonitorFolder, "B.txt");
File.AppendAllText(fileName2, "ABCD Eeee");
Assert.AreEqual(IndexBuildResults.Successful, indexBuilder.CreateIndex(new FileInfo(fileName2)));
CollectionAssert.AreEquivalent(new[] { "WOWO", "ABCD", "CONTENT", "APPLY", "Eeee", "EEEE" }, indexBuilder.HintIndexPool.SearchWord(new MatchAllDocsQuery()).Select(u => u.Word), "Words Been Added And Removed If Needed");
File.Delete(fileName2);
File.AppendAllText(fileName2, "Eeee");
CollectionAssert.AreEquivalent(new[] { "WOWO", "ABCD", "CONTENT", "APPLY", "Eeee", "EEEE" }, indexBuilder.HintIndexPool.SearchWord(new MatchAllDocsQuery()).Select(u => u.Word), "Words Been Added And Removed If Needed");
}
void FileRenamed(string oldFullPath, string fullPath, PendingRetrySource pendingRetrySource = null)
{
try
{
if (IsFile(fullPath))
{
var fileInfo = new FileInfo(fullPath);
try
{
if (fileInfo.Exists)
{
var content = FilesContentHelper.ReadAllText(fullPath);
var document = CodeIndexBuilder.GetDocumentFromSource(CodeSource.GetCodeSource(fileInfo, content));
CodeIndexBuilder.UpdateIndex(config.LuceneIndexForCode, GetNoneTokenizeFieldTerm(nameof(CodeSource.FilePath), oldFullPath), document);
pendingChanges++;
}
}
catch (IOException)
{
AddFileChangesToRetrySouce(fullPath, WatcherChangeTypes.Renamed, pendingRetrySource, oldFullPath);
}
}
else if (IsDirectory(fullPath))
{
// Rebuild All Sub Directory Index File, rename the index path
var term = new PrefixQuery(GetNoneTokenizeFieldTerm(nameof(CodeSource.FilePath), oldFullPath));
var docs = CodeIndexSearcher.Search(config.LuceneIndexForCode, term, int.MaxValue);
foreach (var doc in docs)
{
CodeIndexBuilder.UpdateCodeFilePath(doc, oldFullPath, fullPath);
CodeIndexBuilder.UpdateIndex(config.LuceneIndexForCode, new Term(nameof(CodeSource.CodePK), doc.Get(nameof(CodeSource.CodePK))), doc);
pendingChanges++;
}
}
}
catch (Exception ex)
{
log?.Error(ex.ToString());
}
}
void InitializeIndexCore(bool forceRebuild)
{
var folders = IndexConfig.GetFolders(CodeIndexConfiguration.LuceneIndex);
IndexBuilder = new CodeIndexBuilder(
IndexConfig.IndexName,
new LucenePoolLight(folders.CodeIndexFolder),
new LucenePoolLight(folders.HintIndexFolder),
Log);
IndexBuilder.InitIndexFolderIfNeeded();
Status = IndexStatus.Initializing_ComponentInitializeFinished;
ChangedSources = new ConcurrentQueue <ChangedSource>();
PendingRetryCodeSources = new ConcurrentQueue <PendingRetrySource>();
FilesWatcher = FilesWatcherHelper.StartWatch(IndexConfig.MonitorFolder, OnChange, OnRename);
Log.LogInformation($"{IndexConfig.IndexName}: Start Watch files change");
var allFiles = FilesFetcher.FetchAllFiles(IndexConfig.MonitorFolder, IndexConfig.ExcludedExtensionsArray, IndexConfig.ExcludedPathsArray, includedExtensions: IndexConfig.IncludedExtensionsArray, isInLinux: CodeIndexConfiguration.IsInLinux).ToList();
Log.LogInformation($"{IndexConfig.IndexName}: Fetching {allFiles.Count} files need to indexing");
List <FileInfo> needToBuildIndex = null;
var failedUpdateOrDeleteFiles = new List <string>();
var brandNewBuild = false;
if (IndexBuilderHelper.IndexExists(IndexBuilder.CodeIndexPool.LuceneIndex))
{
if (forceRebuild)
{
brandNewBuild = true;
Log.LogInformation($"{IndexConfig.IndexName}: Force rebuild all indexes");
IndexBuilder.DeleteAllIndex();
}
else
{
Log.LogInformation($"{IndexConfig.IndexName}: Compare index difference");
var allCodeSource = IndexBuilder.GetAllIndexedCodeSource();
GC.Collect(); // Run GC after fetching massive documents from index
needToBuildIndex = new List <FileInfo>();
var allFilesDictionary = allFiles.ToDictionary(u => u.FullName);
foreach (var codeSource in allCodeSource)
{
TokenSource.Token.ThrowIfCancellationRequested();
if (allFilesDictionary.TryGetValue(codeSource.FilePath, out var fileInfo))
{
if (fileInfo.LastWriteTimeUtc != codeSource.LastWriteTimeUtc)
{
Log.LogInformation($"{IndexConfig.IndexName}: File {fileInfo.FullName} modified");
if (IndexBuilder.UpdateIndex(fileInfo, TokenSource.Token) != IndexBuildResults.Successful)
{
failedUpdateOrDeleteFiles.Add(codeSource.FilePath);
}
}
allFilesDictionary.Remove(codeSource.FilePath);
}
else
{
Log.LogInformation($"{IndexConfig.IndexName}: File {codeSource.FilePath} deleted");
if (!IndexBuilder.DeleteIndex(codeSource.FilePath))
{
failedUpdateOrDeleteFiles.Add(codeSource.FilePath);
}
}
}
foreach (var needToCreateFiles in allFilesDictionary)
{
Log.LogInformation($"{IndexConfig.IndexName}: Found new file {needToCreateFiles.Value.FullName}");
needToBuildIndex.Add(needToCreateFiles.Value);
}
}
}
else
{
brandNewBuild = true;
}
AddNewIndexFiles(needToBuildIndex ?? allFiles, out var failedIndexFiles, brandNewBuild);
GC.Collect(); // Run GC to save the memory
IndexBuilder.Commit();
if (failedIndexFiles.Count > 0 || failedUpdateOrDeleteFiles.Count > 0)
{
Log.LogError($"{IndexConfig.IndexName}: Initialize finished for {IndexConfig.MonitorFolder}, failed with these file(s): {string.Join(", ", failedIndexFiles.Select(u => u.FullName).Concat(failedUpdateOrDeleteFiles))}");
}
else
{
Log.LogInformation($"{IndexConfig.IndexName}: Initialize finished for {IndexConfig.MonitorFolder}");
}
}