void DeleteIndex(string fullPath)
{
try
{
CodeIndexBuilder.DeleteIndex(config.LuceneIndexForCode, GetNoneTokenizeFieldTerm(nameof(CodeSource.FilePath), fullPath));
pendingChanges++;
}
catch (Exception ex)
{
log?.Error(ex.ToString());
}
}
void DeleteAllDocumentsIndexUnder(string forderOldFullPath)
{
try
{
var term = new PrefixQuery(GetNoneTokenizeFieldTerm(nameof(CodeSource.FilePath), forderOldFullPath));
CodeIndexBuilder.DeleteIndex(config.LuceneIndexForCode, term);
pendingChanges++;
}
catch (Exception ex)
{
log?.Error(ex.ToString());
}
}
public void TestDeleteIndex()
{
using var indexBuilder = new CodeIndexBuilder("ABC", new LucenePoolLight(TempCodeIndexDir), new LucenePoolLight(TempHintIndexDir), Log);
File.AppendAllText(Path.Combine(MonitorFolder, "A.txt"), "ABCD EEEE dddd");
File.AppendAllText(Path.Combine(MonitorFolder, "B.txt"), "ABCD DDDD eeEE");
Assert.AreEqual(IndexBuildResults.Successful, indexBuilder.CreateIndex(new FileInfo(Path.Combine(MonitorFolder, "A.txt"))));
Assert.AreEqual(IndexBuildResults.Successful, indexBuilder.CreateIndex(new FileInfo(Path.Combine(MonitorFolder, "B.txt"))));
Assert.AreEqual(2, indexBuilder.CodeIndexPool.SearchCode(new MatchAllDocsQuery()).Length);
CollectionAssert.AreEquivalent(new[] { "ABCD", "EEEE", "DDDD", "dddd", "eeEE" }, indexBuilder.HintIndexPool.SearchWord(new MatchAllDocsQuery()).Select(u => u.Word));
Assert.IsTrue(indexBuilder.DeleteIndex(Path.Combine(MonitorFolder, "A.txt")));
Assert.AreEqual(1, indexBuilder.CodeIndexPool.SearchCode(new MatchAllDocsQuery()).Length);
CollectionAssert.AreEquivalent(new[] { "ABCD", "DDDD", "eeEE" }, indexBuilder.HintIndexPool.SearchWord(new MatchAllDocsQuery()).Select(u => u.Word), "Words Been Removed If Needed");
}
public void TestDeleteIndex()
{
BuildIndex();
LucenePool.SaveResultsAndClearLucenePool(Config.LuceneIndexForCode);
var generator = new QueryGenerator();
var result = CodeIndexSearcher.Search(Config.LuceneIndexForCode, generator.GetQueryFromStr("FFFF test"), 10);
Assert.That(result.Length, Is.EqualTo(2));
CodeIndexBuilder.DeleteIndex(Config.LuceneIndexForCode, new Term(nameof(CodeSource.FileExtension), "xml"));
LucenePool.SaveResultsAndClearLucenePool(Config.LuceneIndexForCode);
result = CodeIndexSearcher.Search(Config.LuceneIndexForCode, generator.GetQueryFromStr("FFFF test"), 10);
Assert.That(result.Length, Is.EqualTo(1));
CodeIndexBuilder.DeleteIndex(Config.LuceneIndexForCode, generator.GetQueryFromStr("Test"));
LucenePool.SaveResultsAndClearLucenePool(Config.LuceneIndexForCode);
result = CodeIndexSearcher.Search(Config.LuceneIndexForCode, generator.GetQueryFromStr("FFFF test"), 10);
Assert.That(result.Length, Is.EqualTo(0));
}
public void TestDeleteIndex_FolderDelete()
{
using var indexBuilder = new CodeIndexBuilder("ABC", new LucenePoolLight(TempCodeIndexDir), new LucenePoolLight(TempHintIndexDir), Log);
Directory.CreateDirectory(Path.Combine(MonitorFolder, "SubDir"));
File.AppendAllText(Path.Combine(MonitorFolder, "SubDir.txt"), "1234");
File.AppendAllText(Path.Combine(MonitorFolder, "SubDir", "A.txt"), "5678");
File.AppendAllText(Path.Combine(MonitorFolder, "SubDir", "B.txt"), "5678");
Assert.AreEqual(IndexBuildResults.Successful, indexBuilder.CreateIndex(new FileInfo(Path.Combine(MonitorFolder, "SubDir.txt"))));
Assert.AreEqual(IndexBuildResults.Successful, indexBuilder.CreateIndex(new FileInfo(Path.Combine(MonitorFolder, "SubDir", "A.txt"))));
Assert.AreEqual(IndexBuildResults.Successful, indexBuilder.CreateIndex(new FileInfo(Path.Combine(MonitorFolder, "SubDir", "B.txt"))));
Assert.AreEqual(3, indexBuilder.CodeIndexPool.SearchCode(new MatchAllDocsQuery()).Length);
Assert.IsTrue(indexBuilder.DeleteIndex(Path.Combine(MonitorFolder, "SubDir")));
var results = indexBuilder.CodeIndexPool.SearchCode(new MatchAllDocsQuery());
Assert.AreEqual(1, results.Length, "Should not delete the index of SubDir.txt");
Assert.AreEqual("SubDir.txt", results[0].FileName);
}
void InitializeIndexCore(bool forceRebuild)
{
var folders = IndexConfig.GetFolders(CodeIndexConfiguration.LuceneIndex);
IndexBuilder = new CodeIndexBuilder(
IndexConfig.IndexName,
new LucenePoolLight(folders.CodeIndexFolder),
new LucenePoolLight(folders.HintIndexFolder),
Log);
IndexBuilder.InitIndexFolderIfNeeded();
Status = IndexStatus.Initializing_ComponentInitializeFinished;
ChangedSources = new ConcurrentQueue <ChangedSource>();
PendingRetryCodeSources = new ConcurrentQueue <PendingRetrySource>();
FilesWatcher = FilesWatcherHelper.StartWatch(IndexConfig.MonitorFolder, OnChange, OnRename);
Log.LogInformation($"{IndexConfig.IndexName}: Start Watch files change");
var allFiles = FilesFetcher.FetchAllFiles(IndexConfig.MonitorFolder, IndexConfig.ExcludedExtensionsArray, IndexConfig.ExcludedPathsArray, includedExtensions: IndexConfig.IncludedExtensionsArray, isInLinux: CodeIndexConfiguration.IsInLinux).ToList();
Log.LogInformation($"{IndexConfig.IndexName}: Fetching {allFiles.Count} files need to indexing");
List <FileInfo> needToBuildIndex = null;
var failedUpdateOrDeleteFiles = new List <string>();
var brandNewBuild = false;
if (IndexBuilderHelper.IndexExists(IndexBuilder.CodeIndexPool.LuceneIndex))
{
if (forceRebuild)
{
brandNewBuild = true;
Log.LogInformation($"{IndexConfig.IndexName}: Force rebuild all indexes");
IndexBuilder.DeleteAllIndex();
}
else
{
Log.LogInformation($"{IndexConfig.IndexName}: Compare index difference");
var allCodeSource = IndexBuilder.GetAllIndexedCodeSource();
GC.Collect(); // Run GC after fetching massive documents from index
needToBuildIndex = new List <FileInfo>();
var allFilesDictionary = allFiles.ToDictionary(u => u.FullName);
foreach (var codeSource in allCodeSource)
{
TokenSource.Token.ThrowIfCancellationRequested();
if (allFilesDictionary.TryGetValue(codeSource.FilePath, out var fileInfo))
{
if (fileInfo.LastWriteTimeUtc != codeSource.LastWriteTimeUtc)
{
Log.LogInformation($"{IndexConfig.IndexName}: File {fileInfo.FullName} modified");
if (IndexBuilder.UpdateIndex(fileInfo, TokenSource.Token) != IndexBuildResults.Successful)
{
failedUpdateOrDeleteFiles.Add(codeSource.FilePath);
}
}
allFilesDictionary.Remove(codeSource.FilePath);
}
else
{
Log.LogInformation($"{IndexConfig.IndexName}: File {codeSource.FilePath} deleted");
if (!IndexBuilder.DeleteIndex(codeSource.FilePath))
{
failedUpdateOrDeleteFiles.Add(codeSource.FilePath);
}
}
}
foreach (var needToCreateFiles in allFilesDictionary)
{
Log.LogInformation($"{IndexConfig.IndexName}: Found new file {needToCreateFiles.Value.FullName}");
needToBuildIndex.Add(needToCreateFiles.Value);
}
}
}
else
{
brandNewBuild = true;
}
AddNewIndexFiles(needToBuildIndex ?? allFiles, out var failedIndexFiles, brandNewBuild);
GC.Collect(); // Run GC to save the memory
IndexBuilder.Commit();
if (failedIndexFiles.Count > 0 || failedUpdateOrDeleteFiles.Count > 0)
{
Log.LogError($"{IndexConfig.IndexName}: Initialize finished for {IndexConfig.MonitorFolder}, failed with these file(s): {string.Join(", ", failedIndexFiles.Select(u => u.FullName).Concat(failedUpdateOrDeleteFiles))}");
}
else
{
Log.LogInformation($"{IndexConfig.IndexName}: Initialize finished for {IndexConfig.MonitorFolder}");
}
}
public void InitializeIndex(CodeIndexConfiguration config, out List <FileInfo> failedIndexFiles, bool forceDeleteAllIndex = false)
{
config.RequireNotNull(nameof(config));
log?.Info($"Initialize start for {config.LuceneIndex}");
var allFiles = FilesFetcher.FetchAllFiles(config.MonitorFolder, config.ExcludedExtensionsArray, config.ExcludedPathsArray, includedExtensions: config.IncludedExtensionsArray, isInLinux: config.IsInLinux).ToList();
List <FileInfo> needToBuildIndex = null;
var firstInitialize = true;
CodeIndexBuilder.InitIndexFolderIfNeeded(config, log);
if (CodeIndexBuilder.IndexExists(config.LuceneIndexForCode))
{
if (forceDeleteAllIndex)
{
log?.Info("Delete exist index");
CodeIndexBuilder.DeleteAllIndex(config);
}
else
{
firstInitialize = false;
log?.Info("Compare index difference");
var allCodeSource = CodeIndexBuilder.GetAllIndexedCodeSource(config.LuceneIndexForCode);
needToBuildIndex = new List <FileInfo>();
foreach (var codeSource in allCodeSource)
{
var fileInfo = allFiles.FirstOrDefault(u => u.FullName == codeSource.FilePath);
if (fileInfo != null)
{
if (fileInfo.LastWriteTimeUtc != codeSource.LastWriteTimeUtc)
{
log?.Info($"File {fileInfo.FullName} modified");
CodeIndexBuilder.DeleteIndex(config.LuceneIndexForCode, CodeFilesIndexMaintainer.GetNoneTokenizeFieldTerm(nameof(CodeSource.FilePath), codeSource.FilePath));
needToBuildIndex.Add(fileInfo);
}
allFiles.Remove(fileInfo);
}
else
{
log?.Info($"File {codeSource.FilePath} deleted");
CodeIndexBuilder.DeleteIndex(config.LuceneIndexForCode, CodeFilesIndexMaintainer.GetNoneTokenizeFieldTerm(nameof(CodeSource.FilePath), codeSource.FilePath));
}
}
foreach (var needToCreateFiles in allFiles)
{
log?.Info($"Found new file {needToCreateFiles.FullName}");
needToBuildIndex.Add(needToCreateFiles);
}
}
}
CodeIndexBuilder.BuildIndexByBatch(config, true, true, true, needToBuildIndex ?? allFiles, false, log, out failedIndexFiles);
LucenePool.SaveResultsAndClearLucenePool(config.LuceneIndexForCode);
WordsHintBuilder.BuildIndexByBatch(config, true, true, true, log, firstInitialize);
log?.Info($"Initialize finished for {config.LuceneIndex}");
}