public async Task <ExecuteOperationsResponse> Execute(ExecuteOperationsRequest request, CancellationToken cancellationToken = default)
{
var deletedDirectories = new ConcurrentDictionary <string, byte>(); // hashset
State.TotalOperations = request.Operations.Count;
var result = await TaskCombinators.ThrottledCatchErrorsAsync(request.Operations, (operation, _) =>
{
try
{
if (operation is DeleteFileOperation deleteFileOperation)
{
if (operation.File.RelativeFilename == null && !request.RemoveFilesFromOutside)
{
return(new ValueTask());
}
_fileSystem.File.Delete(deleteFileOperation.File.Filename);
deletedDirectories.TryAdd(_fileSystem.Path.GetDirectoryName(operation.File.Filename), default);
}
else if (operation is MoveFileOperation moveFileOperation)
{
var targetPath = _fileSystem.Path.Combine(request.PhotoDirectoryRoot, moveFileOperation.TargetPath);
_fileSystem.Directory.CreateDirectory(_fileSystem.Path.GetDirectoryName(targetPath));
if (operation.File.RelativeFilename == null && !request.RemoveFilesFromOutside)
{
_fileSystem.File.Copy(moveFileOperation.File.Filename, targetPath, false);
}
else
{
_fileSystem.File.Move(moveFileOperation.File.Filename, targetPath);
deletedDirectories.TryAdd(_fileSystem.Path.GetDirectoryName(operation.File.Filename), default);
}
}
}
finally
{
State.OnOperationProcessed();
}
return(new ValueTask());
}, cancellationToken);
foreach (var deletedDirectory in deletedDirectories.Keys)
{
if (!_fileSystem.Directory.EnumerateFiles(deletedDirectory, "*", SearchOption.AllDirectories).Any())
{
_fileSystem.Directory.Delete(deletedDirectory, true);
}
}
return(new ExecuteOperationsResponse(result));
}
public async Task Invoke(TContext context, CancellationToken cancellationToken)
{
var exceptions =
await TaskCombinators.ThrottledCatchErrorsAsync(_actions, (action, token) => action.Execute(context), cancellationToken);
foreach (var keyValuePair in exceptions)
{
Logger.Warn(keyValuePair.Value, "The action '{actionName}' threw an error on execution.",
keyValuePair.Key.GetType().FullName);
}
}
public async Task <IActionResult> GetFileProperties([FromQuery] string path, [FromServices] IEnumerable <IFilePropertyValueProvider> propertyValueProviders)
{
var result = new FilePropertiesDto();
var fileInfo = new FileInfo(path);
var properties = new ConcurrentBag <FileProperty>();
await TaskCombinators.ThrottledCatchErrorsAsync(propertyValueProviders, (provider, token) => Task.Run(() =>
{
foreach (var fileProperty in provider.ProvideValues(fileInfo, result).ToList())
{
properties.Add(fileProperty);
}
}), CancellationToken.None);
result.Properties = properties.ToList();
return(Ok(result));
}
public async Task <IActionResult> GetSystemInfo([FromServices] IEnumerable <ISystemInfoProvider> systemInfoProviders)
{
var info = new ConcurrentBag <SystemInfoDto>();
var result = await TaskCombinators.ThrottledCatchErrorsAsync(systemInfoProviders, (provider, token) => Task.Run(() =>
{
foreach (var systemInfoDto in provider.FetchInformation())
{
info.Add(systemInfoDto);
}
}), MazeContext.RequestAborted);
foreach (var error in result)
{
_logger.LogDebug(error.Value, "Exception occurrred when invoking {service}", error.Key.GetType().FullName);
}
return(Ok(info));
}
public async Task <SynchronizeIndexResponse> InternalExecute(SynchronizeIndexRequest request, CancellationToken cancellationToken = default)
{
var directory = request.Directory;
await using var dataContext = directory.GetDataContext();
var operations = new ConcurrentBag <FileOperation>();
State.Status = SynchronizeIndexStatus.Scanning;
// get all files from the repository
var indexedFiles = await dataContext.FileRepository.GetAllReadOnlyBySpecs(new IncludeFileLocationsSpec());
var indexedFileInfos = indexedFiles.SelectMany(x => x.ToFileInfos(directory));
// get all files from the actual directory
var localFiles = directory.EnumerateFiles().WithCancellation(cancellationToken).ToList();
State.Status = SynchronizeIndexStatus.Synchronizing;
// get changes
var(newFiles, removedFiles) = CollectionDiff.Create(indexedFileInfos, localFiles, new FileInfoComparer());
// files that are completely removed from the directory
var completelyRemovedFiles = new List <IFileInfo>();
// remove files from index
foreach (var removedFile in removedFiles)
{
var action = _serviceProvider.GetRequiredService <IRemoveFileFromIndexUseCase>();
var response = await action.Handle(new RemoveFileFromIndexRequest(removedFile.RelativeFilename !, directory));
if (action.HasError)
{
State.Errors.Add(removedFile.RelativeFilename !, action.Error !);
}
if (response !.IsCompletelyRemoved)
{
completelyRemovedFiles.Add(removedFile);
}
}
IImmutableList <FileInformation> removedFileInformation = removedFiles
.Select(x => GetFileInformationFromPath(x.RelativeFilename !, indexedFiles, directory))
.ToImmutableList();
var formerlyDeletedFiles = directory.MemoryManager.DirectoryMemory.DeletedFiles;
var deletedFilesLock = new object();
State.Status = SynchronizeIndexStatus.IndexingNewFiles;
State.TotalFiles = newFiles.Count;
var processedFilesCount = 0;
var removedFilesLock = new object();
var stateLock = new object();
await TaskCombinators.ThrottledCatchErrorsAsync(newFiles, async (newFile, _) =>
{
var(action, response) = await IndexFile(newFile.Filename, directory);
if (action.HasError)
{
lock (stateLock)
{
State.Errors.Add(newFile.Filename, action.Error !);
}
return;
}
var(indexedFile, fileLocation) = response !;
// remove from formerly deleted files
if (formerlyDeletedFiles.ContainsKey(indexedFile.Hash))
{
lock (deletedFilesLock)
{
formerlyDeletedFiles = formerlyDeletedFiles.Remove(indexedFile.Hash);
}
}
FileOperation fileOperation;
// get operation
var removedFile = removedFileInformation.FirstOrDefault(x => _fileContentComparer.Equals(x, indexedFile));
if (removedFile != null)
{
lock (removedFilesLock)
{
removedFileInformation = removedFileInformation.Remove(removedFile);
}
if (directory.PathComparer.Equals(fileLocation.RelativeFilename, removedFile.RelativeFilename !))
{
fileOperation = FileOperation.FileChanged(fileLocation, ToFileReference(removedFile));
}
else
{
fileOperation = FileOperation.FileMoved(fileLocation, ToFileReference(removedFile));
}
}
else
{
fileOperation = FileOperation.NewFile(fileLocation);
}
await using (var context = directory.GetDataContext())
{
await context.OperationRepository.Add(fileOperation);
operations.Add(fileOperation);
}
var processedFiles = Interlocked.Increment(ref processedFilesCount);
State.ProcessedFiles = processedFiles;
State.Progress = (double)processedFiles / newFiles.Count;
}, CancellationToken.None, 8); // do not use cancellation token here as a cancellation would destroy all move/change operations as all files were already removed
foreach (var removedFile in removedFileInformation)
{
var operation = FileOperation.FileRemoved(ToFileReference(removedFile));
await dataContext.OperationRepository.Add(operation);
operations.Add(operation);
// add the file to deleted files, if it was completely removed from index
// WARN: if a file changes, the previous file is not marked as deleted. Idk if that is actually desired
if (completelyRemovedFiles.Any(x => x.Filename == removedFile.Filename))
{
formerlyDeletedFiles = formerlyDeletedFiles.Add(removedFile.Hash.ToString(), new DeletedFileInfo(removedFile.RelativeFilename !, removedFile.Length, removedFile.Hash, removedFile.PhotoProperties, removedFile.FileCreatedOn, DateTimeOffset.UtcNow));
}
}
await directory.MemoryManager.Update(directory.MemoryManager.DirectoryMemory.SetDeletedFiles(formerlyDeletedFiles));
return(new SynchronizeIndexResponse(operations.ToList()));
}
