public async Task <SynchronizeIndexResponse> InternalExecute(SynchronizeIndexRequest request, CancellationToken cancellationToken = default) { var directory = request.Directory; await using var dataContext = directory.GetDataContext(); var operations = new ConcurrentBag <FileOperation>(); State.Status = SynchronizeIndexStatus.Scanning; // get all files from the repository var indexedFiles = await dataContext.FileRepository.GetAllReadOnlyBySpecs(new IncludeFileLocationsSpec()); var indexedFileInfos = indexedFiles.SelectMany(x => x.ToFileInfos(directory)); // get all files from the actual directory var localFiles = directory.EnumerateFiles().WithCancellation(cancellationToken).ToList(); State.Status = SynchronizeIndexStatus.Synchronizing; // get changes var(newFiles, removedFiles) = CollectionDiff.Create(indexedFileInfos, localFiles, new FileInfoComparer()); // files that are completely removed from the directory var completelyRemovedFiles = new List <IFileInfo>(); // remove files from index foreach (var removedFile in removedFiles) { var action = _serviceProvider.GetRequiredService <IRemoveFileFromIndexUseCase>(); var response = await action.Handle(new RemoveFileFromIndexRequest(removedFile.RelativeFilename !, directory)); if (action.HasError) { State.Errors.Add(removedFile.RelativeFilename !, action.Error !); } if (response !.IsCompletelyRemoved) { completelyRemovedFiles.Add(removedFile); } } IImmutableList <FileInformation> removedFileInformation = removedFiles .Select(x => GetFileInformationFromPath(x.RelativeFilename !, indexedFiles, directory)) .ToImmutableList(); var formerlyDeletedFiles = directory.MemoryManager.DirectoryMemory.DeletedFiles; var deletedFilesLock = new object(); State.Status = SynchronizeIndexStatus.IndexingNewFiles; State.TotalFiles = newFiles.Count; var processedFilesCount = 0; var removedFilesLock = new object(); var stateLock = new object(); await TaskCombinators.ThrottledCatchErrorsAsync(newFiles, async (newFile, _) => { var(action, response) = await IndexFile(newFile.Filename, directory); if (action.HasError) { lock (stateLock) { State.Errors.Add(newFile.Filename, action.Error !); } return; } var(indexedFile, fileLocation) = response !; // remove from formerly deleted files if (formerlyDeletedFiles.ContainsKey(indexedFile.Hash)) { lock (deletedFilesLock) { formerlyDeletedFiles = formerlyDeletedFiles.Remove(indexedFile.Hash); } } FileOperation fileOperation; // get operation var removedFile = removedFileInformation.FirstOrDefault(x => _fileContentComparer.Equals(x, indexedFile)); if (removedFile != null) { lock (removedFilesLock) { removedFileInformation = removedFileInformation.Remove(removedFile); } if (directory.PathComparer.Equals(fileLocation.RelativeFilename, removedFile.RelativeFilename !)) { fileOperation = FileOperation.FileChanged(fileLocation, ToFileReference(removedFile)); } else { fileOperation = FileOperation.FileMoved(fileLocation, ToFileReference(removedFile)); } } else { fileOperation = FileOperation.NewFile(fileLocation); } await using (var context = directory.GetDataContext()) { await context.OperationRepository.Add(fileOperation); operations.Add(fileOperation); } var processedFiles = Interlocked.Increment(ref processedFilesCount); State.ProcessedFiles = processedFiles; State.Progress = (double)processedFiles / newFiles.Count; }, CancellationToken.None, 8); // do not use cancellation token here as a cancellation would destroy all move/change operations as all files were already removed foreach (var removedFile in removedFileInformation) { var operation = FileOperation.FileRemoved(ToFileReference(removedFile)); await dataContext.OperationRepository.Add(operation); operations.Add(operation); // add the file to deleted files, if it was completely removed from index // WARN: if a file changes, the previous file is not marked as deleted. Idk if that is actually desired if (completelyRemovedFiles.Any(x => x.Filename == removedFile.Filename)) { formerlyDeletedFiles = formerlyDeletedFiles.Add(removedFile.Hash.ToString(), new DeletedFileInfo(removedFile.RelativeFilename !, removedFile.Length, removedFile.Hash, removedFile.PhotoProperties, removedFile.FileCreatedOn, DateTimeOffset.UtcNow)); } } await directory.MemoryManager.Update(directory.MemoryManager.DirectoryMemory.SetDeletedFiles(formerlyDeletedFiles)); return(new SynchronizeIndexResponse(operations.ToList())); }
public Task <SynchronizeIndexResponse> Execute(SynchronizeIndexRequest request, CancellationToken cancellationToken = default) { return(Task.Run(() => InternalExecute(request, cancellationToken))); }