public async Task <ExecuteOperationsResponse> Execute(ExecuteOperationsRequest request, CancellationToken cancellationToken = default)
        {
            var deletedDirectories = new ConcurrentDictionary <string, byte>(); // hashset

            State.TotalOperations = request.Operations.Count;

            var result = await TaskCombinators.ThrottledCatchErrorsAsync(request.Operations, (operation, _) =>
            {
                try
                {
                    if (operation is DeleteFileOperation deleteFileOperation)
                    {
                        if (operation.File.RelativeFilename == null && !request.RemoveFilesFromOutside)
                        {
                            return(new ValueTask());
                        }

                        _fileSystem.File.Delete(deleteFileOperation.File.Filename);
                        deletedDirectories.TryAdd(_fileSystem.Path.GetDirectoryName(operation.File.Filename), default);
                    }
                    else if (operation is MoveFileOperation moveFileOperation)
                    {
                        var targetPath = _fileSystem.Path.Combine(request.PhotoDirectoryRoot, moveFileOperation.TargetPath);

                        _fileSystem.Directory.CreateDirectory(_fileSystem.Path.GetDirectoryName(targetPath));

                        if (operation.File.RelativeFilename == null && !request.RemoveFilesFromOutside)
                        {
                            _fileSystem.File.Copy(moveFileOperation.File.Filename, targetPath, false);
                        }
                        else
                        {
                            _fileSystem.File.Move(moveFileOperation.File.Filename, targetPath);
                            deletedDirectories.TryAdd(_fileSystem.Path.GetDirectoryName(operation.File.Filename), default);
                        }
                    }
                }
                finally
                {
                    State.OnOperationProcessed();
                }

                return(new ValueTask());
            }, cancellationToken);

            foreach (var deletedDirectory in deletedDirectories.Keys)
            {
                if (!_fileSystem.Directory.EnumerateFiles(deletedDirectory, "*", SearchOption.AllDirectories).Any())
                {
                    _fileSystem.Directory.Delete(deletedDirectory, true);
                }
            }

            return(new ExecuteOperationsResponse(result));
        }
Example #2
0
        public async Task Invoke(TContext context, CancellationToken cancellationToken)
        {
            var exceptions =
                await TaskCombinators.ThrottledCatchErrorsAsync(_actions, (action, token) => action.Execute(context), cancellationToken);

            foreach (var keyValuePair in exceptions)
            {
                Logger.Warn(keyValuePair.Value, "The action '{actionName}' threw an error on execution.",
                            keyValuePair.Key.GetType().FullName);
            }
        }
Example #3
0
        public async Task <IActionResult> GetFileProperties([FromQuery] string path, [FromServices] IEnumerable <IFilePropertyValueProvider> propertyValueProviders)
        {
            var result     = new FilePropertiesDto();
            var fileInfo   = new FileInfo(path);
            var properties = new ConcurrentBag <FileProperty>();

            await TaskCombinators.ThrottledCatchErrorsAsync(propertyValueProviders, (provider, token) => Task.Run(() =>
            {
                foreach (var fileProperty in provider.ProvideValues(fileInfo, result).ToList())
                {
                    properties.Add(fileProperty);
                }
            }), CancellationToken.None);

            result.Properties = properties.ToList();
            return(Ok(result));
        }
Example #4
0
        public async Task <IActionResult> GetSystemInfo([FromServices] IEnumerable <ISystemInfoProvider> systemInfoProviders)
        {
            var info   = new ConcurrentBag <SystemInfoDto>();
            var result = await TaskCombinators.ThrottledCatchErrorsAsync(systemInfoProviders, (provider, token) => Task.Run(() =>
            {
                foreach (var systemInfoDto in provider.FetchInformation())
                {
                    info.Add(systemInfoDto);
                }
            }), MazeContext.RequestAborted);

            foreach (var error in result)
            {
                _logger.LogDebug(error.Value, "Exception occurrred when invoking {service}", error.Key.GetType().FullName);
            }

            return(Ok(info));
        }
Example #5
0
        public async Task <SynchronizeIndexResponse> InternalExecute(SynchronizeIndexRequest request, CancellationToken cancellationToken = default)
        {
            var directory = request.Directory;

            await using var dataContext = directory.GetDataContext();
            var operations = new ConcurrentBag <FileOperation>();

            State.Status = SynchronizeIndexStatus.Scanning;

            // get all files from the repository
            var indexedFiles = await dataContext.FileRepository.GetAllReadOnlyBySpecs(new IncludeFileLocationsSpec());

            var indexedFileInfos = indexedFiles.SelectMany(x => x.ToFileInfos(directory));

            // get all files from the actual directory
            var localFiles = directory.EnumerateFiles().WithCancellation(cancellationToken).ToList();

            State.Status = SynchronizeIndexStatus.Synchronizing;

            // get changes
            var(newFiles, removedFiles) = CollectionDiff.Create(indexedFileInfos, localFiles, new FileInfoComparer());

            // files that are completely removed from the directory
            var completelyRemovedFiles = new List <IFileInfo>();

            // remove files from index
            foreach (var removedFile in removedFiles)
            {
                var action   = _serviceProvider.GetRequiredService <IRemoveFileFromIndexUseCase>();
                var response = await action.Handle(new RemoveFileFromIndexRequest(removedFile.RelativeFilename !, directory));

                if (action.HasError)
                {
                    State.Errors.Add(removedFile.RelativeFilename !, action.Error !);
                }

                if (response !.IsCompletelyRemoved)
                {
                    completelyRemovedFiles.Add(removedFile);
                }
            }

            IImmutableList <FileInformation> removedFileInformation = removedFiles
                                                                      .Select(x => GetFileInformationFromPath(x.RelativeFilename !, indexedFiles, directory))
                                                                      .ToImmutableList();

            var formerlyDeletedFiles = directory.MemoryManager.DirectoryMemory.DeletedFiles;
            var deletedFilesLock     = new object();

            State.Status     = SynchronizeIndexStatus.IndexingNewFiles;
            State.TotalFiles = newFiles.Count;

            var processedFilesCount = 0;
            var removedFilesLock    = new object();
            var stateLock           = new object();

            await TaskCombinators.ThrottledCatchErrorsAsync(newFiles, async (newFile, _) =>
            {
                var(action, response) = await IndexFile(newFile.Filename, directory);

                if (action.HasError)
                {
                    lock (stateLock)
                    {
                        State.Errors.Add(newFile.Filename, action.Error !);
                    }
                    return;
                }

                var(indexedFile, fileLocation) = response !;

                // remove from formerly deleted files
                if (formerlyDeletedFiles.ContainsKey(indexedFile.Hash))
                {
                    lock (deletedFilesLock)
                    {
                        formerlyDeletedFiles = formerlyDeletedFiles.Remove(indexedFile.Hash);
                    }
                }

                FileOperation fileOperation;

                // get operation
                var removedFile = removedFileInformation.FirstOrDefault(x => _fileContentComparer.Equals(x, indexedFile));
                if (removedFile != null)
                {
                    lock (removedFilesLock)
                    {
                        removedFileInformation = removedFileInformation.Remove(removedFile);
                    }

                    if (directory.PathComparer.Equals(fileLocation.RelativeFilename, removedFile.RelativeFilename !))
                    {
                        fileOperation = FileOperation.FileChanged(fileLocation, ToFileReference(removedFile));
                    }
                    else
                    {
                        fileOperation = FileOperation.FileMoved(fileLocation, ToFileReference(removedFile));
                    }
                }
                else
                {
                    fileOperation = FileOperation.NewFile(fileLocation);
                }

                await using (var context = directory.GetDataContext())
                {
                    await context.OperationRepository.Add(fileOperation);
                    operations.Add(fileOperation);
                }

                var processedFiles   = Interlocked.Increment(ref processedFilesCount);
                State.ProcessedFiles = processedFiles;
                State.Progress       = (double)processedFiles / newFiles.Count;
            }, CancellationToken.None, 8); // do not use cancellation token here as a cancellation would destroy all move/change operations as all files were already removed

            foreach (var removedFile in removedFileInformation)
            {
                var operation = FileOperation.FileRemoved(ToFileReference(removedFile));
                await dataContext.OperationRepository.Add(operation);

                operations.Add(operation);

                // add the file to deleted files, if it was completely removed from index
                // WARN: if a file changes, the previous file is not marked as deleted. Idk if that is actually desired
                if (completelyRemovedFiles.Any(x => x.Filename == removedFile.Filename))
                {
                    formerlyDeletedFiles = formerlyDeletedFiles.Add(removedFile.Hash.ToString(), new DeletedFileInfo(removedFile.RelativeFilename !, removedFile.Length, removedFile.Hash, removedFile.PhotoProperties, removedFile.FileCreatedOn, DateTimeOffset.UtcNow));
                }
            }

            await directory.MemoryManager.Update(directory.MemoryManager.DirectoryMemory.SetDeletedFiles(formerlyDeletedFiles));

            return(new SynchronizeIndexResponse(operations.ToList()));
        }