private void StartWatchingLocations()
{
var allLocations = this.locationsOperationsService.GetAllWatchedLocations();
foreach (var watchedLocationDto in allLocations)
{
FilesWatcherHelper.StartFilesWatcher(watchedLocationDto);
}
}
public void StartWatch()
{
FileSystemWatcher = FilesWatcherHelper.StartWatch(config.MonitorFolder, OnFileChange, RenamedEventHandler);
Task.Run(() =>
{
RetryAllFailed(tokenSource.Token);
}, tokenSource.Token);
Task.Run(() =>
{
SaveLuceneResultsWhenNeeded(tokenSource.Token);
}, tokenSource.Token);
log?.Info($"Start monitoring files change on {config.MonitorFolder}");
}
void InitializeIndexCore(bool forceRebuild)
{
var folders = IndexConfig.GetFolders(CodeIndexConfiguration.LuceneIndex);
IndexBuilder = new CodeIndexBuilder(
IndexConfig.IndexName,
new LucenePoolLight(folders.CodeIndexFolder),
new LucenePoolLight(folders.HintIndexFolder),
Log);
IndexBuilder.InitIndexFolderIfNeeded();
Status = IndexStatus.Initializing_ComponentInitializeFinished;
ChangedSources = new ConcurrentQueue <ChangedSource>();
PendingRetryCodeSources = new ConcurrentQueue <PendingRetrySource>();
FilesWatcher = FilesWatcherHelper.StartWatch(IndexConfig.MonitorFolder, OnChange, OnRename);
Log.LogInformation($"{IndexConfig.IndexName}: Start Watch files change");
var allFiles = FilesFetcher.FetchAllFiles(IndexConfig.MonitorFolder, IndexConfig.ExcludedExtensionsArray, IndexConfig.ExcludedPathsArray, includedExtensions: IndexConfig.IncludedExtensionsArray, isInLinux: CodeIndexConfiguration.IsInLinux).ToList();
Log.LogInformation($"{IndexConfig.IndexName}: Fetching {allFiles.Count} files need to indexing");
List <FileInfo> needToBuildIndex = null;
var failedUpdateOrDeleteFiles = new List <string>();
var brandNewBuild = false;
if (IndexBuilderHelper.IndexExists(IndexBuilder.CodeIndexPool.LuceneIndex))
{
if (forceRebuild)
{
brandNewBuild = true;
Log.LogInformation($"{IndexConfig.IndexName}: Force rebuild all indexes");
IndexBuilder.DeleteAllIndex();
}
else
{
Log.LogInformation($"{IndexConfig.IndexName}: Compare index difference");
var allCodeSource = IndexBuilder.GetAllIndexedCodeSource();
GC.Collect(); // Run GC after fetching massive documents from index
needToBuildIndex = new List <FileInfo>();
var allFilesDictionary = allFiles.ToDictionary(u => u.FullName);
foreach (var codeSource in allCodeSource)
{
TokenSource.Token.ThrowIfCancellationRequested();
if (allFilesDictionary.TryGetValue(codeSource.FilePath, out var fileInfo))
{
if (fileInfo.LastWriteTimeUtc != codeSource.LastWriteTimeUtc)
{
Log.LogInformation($"{IndexConfig.IndexName}: File {fileInfo.FullName} modified");
if (IndexBuilder.UpdateIndex(fileInfo, TokenSource.Token) != IndexBuildResults.Successful)
{
failedUpdateOrDeleteFiles.Add(codeSource.FilePath);
}
}
allFilesDictionary.Remove(codeSource.FilePath);
}
else
{
Log.LogInformation($"{IndexConfig.IndexName}: File {codeSource.FilePath} deleted");
if (!IndexBuilder.DeleteIndex(codeSource.FilePath))
{
failedUpdateOrDeleteFiles.Add(codeSource.FilePath);
}
}
}
foreach (var needToCreateFiles in allFilesDictionary)
{
Log.LogInformation($"{IndexConfig.IndexName}: Found new file {needToCreateFiles.Value.FullName}");
needToBuildIndex.Add(needToCreateFiles.Value);
}
}
}
else
{
brandNewBuild = true;
}
AddNewIndexFiles(needToBuildIndex ?? allFiles, out var failedIndexFiles, brandNewBuild);
GC.Collect(); // Run GC to save the memory
IndexBuilder.Commit();
if (failedIndexFiles.Count > 0 || failedUpdateOrDeleteFiles.Count > 0)
{
Log.LogError($"{IndexConfig.IndexName}: Initialize finished for {IndexConfig.MonitorFolder}, failed with these file(s): {string.Join(", ", failedIndexFiles.Select(u => u.FullName).Concat(failedUpdateOrDeleteFiles))}");
}
else
{
Log.LogInformation($"{IndexConfig.IndexName}: Initialize finished for {IndexConfig.MonitorFolder}");
}
}
public void TestStartWatch()
{
var renameHit = 0;
var changeHit = 0;
var waitMS = 10;
Directory.CreateDirectory(Path.Combine(TempDir, "SubDir"));
using (var watcher = FilesWatcherHelper.StartWatch(TempDir, OnChangedHandler, OnRenameHandler))
{
File.Create(Path.Combine(TempDir, "AAA.cs")).Close();
Thread.Sleep(waitMS);
Assert.AreEqual(1, changeHit);
Assert.AreEqual(0, renameHit);
File.AppendAllText(Path.Combine(TempDir, "AAA.cs"), "12345");
Thread.Sleep(waitMS);
Assert.AreEqual(2, changeHit);
Assert.AreEqual(0, renameHit);
File.Move(Path.Combine(TempDir, "AAA.cs"), Path.Combine(TempDir, "BBB.cs"));
Thread.Sleep(waitMS);
Assert.AreEqual(2, changeHit);
Assert.AreEqual(1, renameHit);
File.Delete(Path.Combine(TempDir, "BBB.cs"));
Thread.Sleep(waitMS);
Assert.AreEqual(3, changeHit);
Assert.AreEqual(1, renameHit);
File.Create(Path.Combine(TempDir, "SubDir", "AAA.cs")).Close();
Thread.Sleep(waitMS);
Assert.AreEqual(4, changeHit);
Assert.AreEqual(1, renameHit);
File.AppendAllText(Path.Combine(TempDir, "SubDir", "AAA.cs"), "AA BB");
Thread.Sleep(waitMS);
Assert.AreEqual(6, changeHit, "One for folder, one for file");
Assert.AreEqual(1, renameHit);
Directory.Move(Path.Combine(TempDir, "SubDir"), Path.Combine(TempDir, "SubDir2"));
Thread.Sleep(waitMS);
Assert.AreEqual(6, changeHit);
Assert.AreEqual(2, renameHit);
Directory.CreateDirectory(Path.Combine(TempDir, "SubDir3"));
Thread.Sleep(waitMS);
Assert.AreEqual(7, changeHit);
Assert.AreEqual(2, renameHit);
File.Create(Path.Combine(TempDir, "CCCC")).Close();
Thread.Sleep(waitMS);
Assert.AreEqual(8, changeHit);
Assert.AreEqual(2, renameHit);
File.SetLastAccessTime(Path.Combine(TempDir, "CCCC"), DateTime.Now.AddDays(1));
Thread.Sleep(waitMS);
Assert.AreEqual(8, changeHit, "Do not watch last access time for file");
Assert.AreEqual(2, renameHit);
}
void OnRenameHandler(object sender, RenamedEventArgs e)
{
renameHit++;
}
void OnChangedHandler(object sender, FileSystemEventArgs e)
{
changeHit++;
}
}
public void TestStartWatch()
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
var renameHit = 0;
var changeHit = 0;
var waitMS = 100;
Directory.CreateDirectory(Path.Combine(TempDir, "SubDir"));
using var watcher = FilesWatcherHelper.StartWatch(TempDir, OnChangedHandler, OnRenameHandler);
File.Create(Path.Combine(TempDir, "AAA.cs")).Close();
Thread.Sleep(waitMS);
Assert.AreEqual(1, changeHit);
Assert.AreEqual(0, renameHit);
File.AppendAllText(Path.Combine(TempDir, "AAA.cs"), "12345");
Thread.Sleep(waitMS);
Assert.AreEqual(2, changeHit);
Assert.AreEqual(0, renameHit);
File.Move(Path.Combine(TempDir, "AAA.cs"), Path.Combine(TempDir, "BBB.cs"));
Thread.Sleep(waitMS);
Assert.AreEqual(2, changeHit);
Assert.AreEqual(1, renameHit);
File.Delete(Path.Combine(TempDir, "BBB.cs"));
Thread.Sleep(waitMS);
Assert.AreEqual(3, changeHit);
Assert.AreEqual(1, renameHit);
File.Create(Path.Combine(TempDir, "SubDir", "AAA.cs")).Close();
Thread.Sleep(waitMS);
Assert.IsTrue(changeHit == 4 || changeHit == 5, "Different behavior under different machines, not important due to logic doesn't care about the folder change events");
Assert.AreEqual(1, renameHit);
File.AppendAllText(Path.Combine(TempDir, "SubDir", "AAA.cs"), "AA BB");
Thread.Sleep(waitMS);
Assert.AreEqual(6, changeHit, "One for folder, one for file");
Assert.AreEqual(1, renameHit);
Directory.Move(Path.Combine(TempDir, "SubDir"), Path.Combine(TempDir, "SubDir2"));
Thread.Sleep(waitMS);
Assert.AreEqual(6, changeHit);
Assert.AreEqual(2, renameHit);
Directory.CreateDirectory(Path.Combine(TempDir, "SubDir3"));
Thread.Sleep(waitMS);
Assert.AreEqual(7, changeHit);
Assert.AreEqual(2, renameHit);
File.Create(Path.Combine(TempDir, "CCCC")).Close();
Thread.Sleep(waitMS);
Assert.AreEqual(8, changeHit);
Assert.AreEqual(2, renameHit);
File.SetLastAccessTime(Path.Combine(TempDir, "CCCC"), DateTime.Now.AddDays(1));
Thread.Sleep(waitMS);
Assert.AreEqual(8, changeHit, "Do not watch last access time for file");
Assert.AreEqual(2, renameHit);
void OnRenameHandler(object sender, RenamedEventArgs e)
{
renameHit++;
}
void OnChangedHandler(object sender, FileSystemEventArgs e)
{
changeHit++;
}
}
else
{
Assert.Pass();
}
}
