private void ComputeNewState(FileSystemTreeSnapshot newSnapshot)
{
_fileLoadingOperationProcessor.Execute(new OperationInfo <OperationResultEventArgs> {
OnBeforeExecute = args => OnFilesLoading(args),
OnAfterExecute = args => OnFilesLoaded(args),
Execute = _ => {
Logger.Log("Computing new state of file database from file system tree.");
var sw = Stopwatch.StartNew();
var oldState = _currentFileDatabase;
var newState = _fileDatabaseFactory.CreateIncremental(oldState, newSnapshot);
sw.Stop();
Logger.Log(">>>>>>>> Done computing new state of file database from file system tree in {0:n0} msec.",
sw.ElapsedMilliseconds);
Logger.LogMemoryStats();
// Swap states
lock (_lock) {
_currentFileDatabase = newState;
}
return(new OperationResultEventArgs());
}
});
}
private void UpdateFileContents(IEnumerable <ProjectFileName> files)
{
OperationInfo operationInfo = null;
_operationProcessor.Execute(new OperationHandlers {
OnBeforeExecute = info =>
operationInfo = info,
OnError = (info, error) => {
OnFilesLoading(info);
OnFilesLoaded(new FilesLoadedResult {
OperationInfo = info,
Error = error,
TreeVersion = _currentTreeVersion,
});
},
Execute = info => {
_currentFileDatabase = _fileDatabaseFactory.CreateWithChangedFiles(
_currentFileDatabase,
files,
onLoading: () => OnFilesLoading(operationInfo),
onLoaded: () => OnFilesLoaded(new FilesLoadedResult {
OperationInfo = operationInfo,
TreeVersion = _currentTreeVersion,
}));
}
});
}
public FileCrawler(IFileDatabase fileDatabase, IFileDescriptorProvider descriptorProvider, ISignatureGenerator signatureGenerator, IProgressTracker progressTracker)
{
this.fileDatabase = fileDatabase;
this.descriptorProvider = descriptorProvider;
this.signatureGenerator = signatureGenerator;
this.progressTracker = progressTracker;
}
/// <summary>
/// Atomically updates the file contents of <paramref name="changedFiles"/>
/// with the new file contents on disk. This method violates the "pure
/// snapshot" semantics but enables efficient updates for the most common
/// type of file change events.
/// </summary>
public IFileDatabase CreateWithChangedFiles(
IFileDatabase previousFileDatabase,
IEnumerable<ProjectFileName> changedFiles,
Action onLoading,
Action onLoaded) {
return new FileDatabaseBuilder(_fileSystem, _fileContentsFactory, _progressTrackerFactory)
.BuildWithChangedFiles(previousFileDatabase, changedFiles, onLoading, onLoaded);
}
public IEnumerable<Duplicate> Find(IFileDatabase databaseA, IFileDatabase databaseB)
{
var allADescriptors = databaseA.GetAll();
var allBDescriptors = databaseB.GetAll();
var joinedDescriptors = allADescriptors.Join(allBDescriptors, x=>x, x => x, (fda, fdb) => new Duplicate(new [] {fda, fdb}), duplicateComparer);
return joinedDescriptors.ToList();
}
/// <summary>
/// Atomically updates the file contents of <paramref name="changedFiles"/>
/// with the new file contents on disk. This method violates the "pure
/// snapshot" semantics but enables efficient updates for the most common
/// type of file change events.
/// </summary>
public IFileDatabase CreateWithChangedFiles(
IFileDatabase previousFileDatabase,
IEnumerable <ProjectFileName> changedFiles,
Action onLoading,
Action onLoaded)
{
return(new FileDatabaseBuilder(_fileSystem, _fileContentsFactory, _progressTrackerFactory)
.BuildWithChangedFiles(previousFileDatabase, changedFiles, onLoading, onLoaded));
}
public IFileDatabase CreateIncremental(
IFileDatabase previousFileDatabase,
FileSystemTreeSnapshot previousSnapshot,
FileSystemTreeSnapshot newSnapshot,
FullPathChanges fullPathChanges,
Action <IFileDatabase> onIntermadiateResult)
{
return(new FileDatabaseBuilder(_fileSystem, _fileContentsFactory, _progressTrackerFactory)
.Build(previousFileDatabase, newSnapshot, fullPathChanges, onIntermadiateResult));
}
public IFileDatabase CreateIncremental(
IFileDatabase previousFileDatabase,
FileSystemTreeSnapshot previousSnapshot,
FileSystemTreeSnapshot newSnapshot,
FullPathChanges fullPathChanges,
Action<IFileDatabase> onIntermadiateResult) {
return new FileDatabaseBuilder(_fileSystem, _fileContentsFactory, _progressTrackerFactory)
.Build(previousFileDatabase, newSnapshot, fullPathChanges, onIntermadiateResult);
}
public UploadFilesHandler(
EFAppContext context,
IFileDatabase fileDatabase,
IThumbnailMaker thumbnailMaker,
ISha256Generator sha256Generator,
IActivityLogger logger)
{
this.context = context;
this.fileDatabase = fileDatabase;
this.thumbnailMaker = thumbnailMaker;
this.sha256Generator = sha256Generator;
this.logger = logger;
}
public IFileDatabase Build(
IFileDatabase previousFileDatabase,
FileSystemTreeSnapshot newSnapshot,
FullPathChanges fullPathChanges,
Action<IFileDatabase> onIntermadiateResult)
{
using (var logger = new TimeElapsedLogger("Building file database from previous one and file system tree snapshot")) {
var fileDatabase = (FileDatabase)previousFileDatabase;
// Compute list of files from tree
ComputeFileCollection(newSnapshot);
var unchangedProjects = newSnapshot
.ProjectRoots.Where(x =>
fileDatabase.ProjectHashes.ContainsKey(x.Project.RootPath) &&
fileDatabase.ProjectHashes[x.Project.RootPath] == x.Project.VersionHash)
.Select(x => x.Project);
var unchangedProjectSet = new HashSet<IProject>(unchangedProjects,
// Use reference equality for IProject is safe, as we keep this
// dictionary only for the duration of this "Build" call.
new ReferenceEqualityComparer<IProject>());
// Don't use file memoization for now, as benefit is dubvious.
//IFileContentsMemoization fileContentsMemoization = new FileContentsMemoization();
IFileContentsMemoization fileContentsMemoization = new NullFileContentsMemoization();
var loadingInfo = new FileContentsLoadingInfo {
FileContentsMemoization = fileContentsMemoization,
FullPathChanges = fullPathChanges,
LoadedTextFileCount = 0,
OldFileDatabase = fileDatabase,
UnchangedProjects = unchangedProjectSet,
PartialProgressReporter = new PartialProgressReporter(
TimeSpan.FromSeconds(5.0),
() => {
Logger.LogInfo("Creating intermedidate file database");
var database = this.CreateFileDatabse();
onIntermadiateResult(database);
})
};
// Merge old state in new state and load all missing files
LoadFileContents(loadingInfo);
return CreateFileDatabse();
}
}
public IFileDatabase Build(
IFileDatabase previousFileDatabase,
FileSystemTreeSnapshot newSnapshot,
FullPathChanges fullPathChanges,
Action <IFileDatabase> onIntermadiateResult)
{
using (var logger = new TimeElapsedLogger("Building file database from previous one and file system tree snapshot")) {
var fileDatabase = (FileDatabase)previousFileDatabase;
// Compute list of files from tree
ComputeFileCollection(newSnapshot);
var unchangedProjects = newSnapshot
.ProjectRoots.Where(x =>
fileDatabase.ProjectHashes.ContainsKey(x.Project.RootPath) &&
fileDatabase.ProjectHashes[x.Project.RootPath] == x.Project.VersionHash)
.Select(x => x.Project);
var unchangedProjectSet = new HashSet <IProject>(unchangedProjects,
// Use reference equality for IProject is safe, as we keep this
// dictionary only for the duration of this "Build" call.
new ReferenceEqualityComparer <IProject>());
// Don't use file memoization for now, as benefit is dubvious.
//IFileContentsMemoization fileContentsMemoization = new FileContentsMemoization();
IFileContentsMemoization fileContentsMemoization = new NullFileContentsMemoization();
var loadingInfo = new FileContentsLoadingInfo {
FileContentsMemoization = fileContentsMemoization,
FullPathChanges = fullPathChanges,
LoadedTextFileCount = 0,
OldFileDatabase = fileDatabase,
UnchangedProjects = unchangedProjectSet,
PartialProgressReporter = new PartialProgressReporter(
TimeSpan.FromSeconds(5.0),
() => {
Logger.LogInfo("Creating intermedidate file database");
var database = this.CreateFileDatabse();
onIntermadiateResult(database);
})
};
// Merge old state in new state and load all missing files
LoadFileContents(loadingInfo);
return(CreateFileDatabse());
}
}
private void ComputeNewState(
FileSystemTreeSnapshot previousSnapshot,
FileSystemTreeSnapshot newSnapshot,
FullPathChanges fullPathChanges)
{
_operationProcessor.Execute(new OperationHandlers {
OnBeforeExecute = info =>
OnFilesLoading(info),
OnError = (info, error) => {
_currentTreeVersion = newSnapshot.Version;
OnFilesLoaded(new FilesLoadedResult {
OperationInfo = info,
Error = error,
TreeVersion = newSnapshot.Version
});
},
Execute = info => {
using (new TimeElapsedLogger("Computing new state of file database")) {
var oldState = _currentFileDatabase;
var newState = _fileDatabaseFactory.CreateIncremental(
oldState,
previousSnapshot,
newSnapshot,
fullPathChanges,
fileDatabase => {
// Store and activate intermediate new state (atomic operation).
_currentFileDatabase = fileDatabase;
OnFilesLoadingProgress(info);
});
// Store and activate final new state (atomic operation).
_currentFileDatabase = newState;
}
_currentTreeVersion = newSnapshot.Version;
OnFilesLoaded(new FilesLoadedResult {
OperationInfo = info,
TreeVersion = newSnapshot.Version
});
}
});
}
public IFileDatabase BuildWithChangedFiles(
IFileDatabase previousFileDatabase,
IEnumerable <ProjectFileName> changedFiles,
Action onLoading,
Action onLoaded)
{
using (new TimeElapsedLogger("Building file database from previous one and list of changed files")) {
var fileDatabase = (FileDatabase)previousFileDatabase;
// Update file contents of file data entries of changed files.
var filesToRead = changedFiles
.Where(x => x.Project.IsFileSearchable(x.FileName) && fileDatabase.Files.ContainsKey(x.FileName))
.ToList();
if (filesToRead.Count == 0)
{
return(previousFileDatabase);
}
// Read file contents.
onLoading();
filesToRead.ForAll(x => {
var newContents = _fileContentsFactory.ReadFileContents(x.FileName.FullPath);
fileDatabase.Files[x.FileName].UpdateContents(newContents);
});
onLoaded();
// Return new file database with updated file contents.
var filesWithContents = FilterFilesWithContents(fileDatabase.Files.Values);
return(new FileDatabase(
fileDatabase.ProjectHashes,
fileDatabase.Files,
fileDatabase.FileNames,
fileDatabase.Directories,
CreateFilePieces(filesWithContents),
filesWithContents.Count));
}
}
public SearchEngine(
IFileSystemProcessor fileSystemProcessor,
IFileSystemNameFactory fileSystemNameFactory,
ITaskQueueFactory taskQueueFactory,
IFileDatabaseFactory fileDatabaseFactory,
IProjectDiscovery projectDiscovery,
ICompiledTextSearchDataFactory compiledTextSearchDataFactory,
IOperationProcessor operationProcessor)
{
_fileSystemNameFactory = fileSystemNameFactory;
_taskQueue = taskQueueFactory.CreateQueue("SearchEngine Task Queue");
_fileDatabaseFactory = fileDatabaseFactory;
_projectDiscovery = projectDiscovery;
_compiledTextSearchDataFactory = compiledTextSearchDataFactory;
_operationProcessor = operationProcessor;
// Create a "Null" state
_currentFileDatabase = _fileDatabaseFactory.CreateEmpty();
// Setup computing a new state everytime a new tree is computed.
fileSystemProcessor.SnapshotComputed += FileSystemProcessorOnSnapshotComputed;
fileSystemProcessor.FilesChanged += FileSystemProcessorOnFilesChanged;
}
public SearchEngine(
IFileSystemProcessor fileSystemProcessor,
IFileSystemNameFactory fileSystemNameFactory,
ICustomThreadPool customThreadPool,
IFileDatabaseFactory fileDatabaseFactory,
IProjectDiscovery projectDiscovery,
ISearchStringParser searchStringParser,
IOperationProcessor <OperationResultEventArgs> fileLoadingOperationProcessor)
{
_fileSystemNameFactory = fileSystemNameFactory;
_customThreadPool = customThreadPool;
_fileDatabaseFactory = fileDatabaseFactory;
_projectDiscovery = projectDiscovery;
_searchStringParser = searchStringParser;
_fileLoadingOperationProcessor = fileLoadingOperationProcessor;
// Create a "Null" state
_currentFileDatabase = _fileDatabaseFactory.CreateEmpty();
// Setup computing a new state everytime a new tree is computed.
fileSystemProcessor.SnapshotComputed += FileSystemProcessorOnSnapshotComputed;
fileSystemProcessor.FilesChanged += FileSystemProcessorOnFilesChanged;
}
public DeleteFileModel(IFileDatabase fileContext, IProjectDatabase projectContext)
{
_fileContext = fileContext;
_projectContext = projectContext;
}
public IFileDatabase CreateIncremental(IFileDatabase previousFileDatabase, FileSystemTreeSnapshot newSnapshot)
{
return new FileDatabaseBuilder(_fileContentsFactory, _progressTrackerFactory).Build(previousFileDatabase, newSnapshot);
}
public UploadModel(IProjectDatabase projectContext, IFileDatabase fileContext, FileUploadService uploadService)
{
_projectContext = projectContext;
_fileContext = fileContext;
_uploadService = uploadService;
}
public FileController(IFileDatabase fileDatabase, IProjectDatabase projectDatabase, FileUploadService uploadService)
{
_uploadService = uploadService;
_fileDatabase = fileDatabase;
_projectDatabase = projectDatabase;
}
public FileManager(IOperationManager operationManager, ILogger <FileManager> logger, IFileDatabase fileDatabase)
{
_operationManager = operationManager;
_logger = logger;
_fileDatabase = fileDatabase;
}
public IFileDatabase Build(IFileDatabase previousFileDatabase, FileSystemTreeSnapshot newSnapshot)
{
return(ComputeFileDatabase((FileDatabase)previousFileDatabase, newSnapshot));
}
public IFileDatabase CreateIncremental(IFileDatabase previousFileDatabase, FileSystemTreeSnapshot newSnapshot)
{
return(new FileDatabaseBuilder(_fileContentsFactory, _progressTrackerFactory).Build(previousFileDatabase, newSnapshot));
}
public DetailsModel(IProjectDatabase projectContext, ITaskDatabase taskContext, IFileDatabase fileContext)
{
_projectContext = projectContext;
_taskContext = taskContext;
_fileContext = fileContext;
}
public ProjectController(IProjectDatabase projectDatabase, IFileDatabase fileDatabase, ITaskDatabase taskDatabase)
{
_projectDatabase = projectDatabase;
_fileDatabase = fileDatabase;
_taskDatabase = taskDatabase;
}
public IFileDatabase BuildWithChangedFiles(
IFileDatabase previousFileDatabase,
IEnumerable<ProjectFileName> changedFiles,
Action onLoading,
Action onLoaded)
{
using (new TimeElapsedLogger("Building file database from previous one and list of changed files")) {
var fileDatabase = (FileDatabase)previousFileDatabase;
// Update file contents of file data entries of changed files.
var filesToRead = changedFiles
.Where(x => x.Project.IsFileSearchable(x.FileName) && fileDatabase.Files.ContainsKey(x.FileName))
.ToList();
if (filesToRead.Count == 0)
return previousFileDatabase;
// Read file contents.
onLoading();
filesToRead.ForAll(x => {
var newContents = _fileContentsFactory.GetFileContents(x.FileName.FullPath);
fileDatabase.Files[x.FileName].UpdateContents(newContents);
});
onLoaded();
// Return new file database with updated file contents.
var filesWithContents = FilterFilesWithContents(fileDatabase.Files.Values);
return new FileDatabase(
fileDatabase.ProjectHashes,
fileDatabase.Files,
fileDatabase.FileNames,
fileDatabase.Directories,
CreateFilePieces(filesWithContents),
filesWithContents.Count);
}
}
public GenericRepository()
{
_db = new FileDatabase <T>();
}
public IdeaDatabase(IFileDatabase iFileDatabase)
{
_connection = iFileDatabase.GetConnection();
_connection.CreateTable <Idea>();
}
