private List <KeyValuePair <string, bool> > AppendDirectoryFilePaths(List <string> fullFilePathsList, ImportSettingsModel importSettings) { var includedDirectoryFilePaths = new List <KeyValuePair <string, bool> >(); foreach (var fullFilePath in fullFilePathsList) { if (_filesystemStorage.ExistFolder(fullFilePath) && importSettings.RecursiveDirectory) { // recursive includedDirectoryFilePaths.AddRange(_filesystemStorage. GetAllFilesInDirectoryRecursive(fullFilePath) .Where(ExtensionRolesHelper.IsExtensionSyncSupported) .Select(syncedFiles => new KeyValuePair <string, bool>(syncedFiles, true))); continue; } if (_filesystemStorage.ExistFolder(fullFilePath) && !importSettings.RecursiveDirectory) { // non-recursive includedDirectoryFilePaths.AddRange(_filesystemStorage.GetAllFilesInDirectory(fullFilePath) .Where(ExtensionRolesHelper.IsExtensionSyncSupported) .Select(syncedFiles => new KeyValuePair <string, bool>(syncedFiles, true))); continue; } includedDirectoryFilePaths.Add( new KeyValuePair <string, bool>(fullFilePath, _filesystemStorage.ExistFile(fullFilePath)) ); } return(includedDirectoryFilePaths); }
[ProducesResponseType(typeof(List <ImportIndexItem>), 415)] // Wrong input (e.g. wrong extenstion type) public async Task <IActionResult> IndexPost() { var tempImportPaths = await Request.StreamFile(_appSettings, _selectorStorage); var importSettings = new ImportSettingsModel(Request); var fileIndexResultsList = await _import.Preflight(tempImportPaths, importSettings); // Import files > _bgTaskQueue.QueueBackgroundWorkItem(async token => { await ImportPostBackgroundTask(tempImportPaths, importSettings, _appSettings.IsVerbose()); }); // When all items are already imported if (importSettings.IndexMode && fileIndexResultsList.All(p => p.Status != ImportStatus.Ok)) { Response.StatusCode = 206; } // Wrong input (extension is not allowed) if (fileIndexResultsList.All(p => p.Status == ImportStatus.FileError)) { Response.StatusCode = 415; } return(Json(fileIndexResultsList)); }
private void WriteOutputStatus(ImportSettingsModel importSettings, List <ImportIndexItem> result, Stopwatch stopWatch) { if (importSettings.IsConsoleOutputModeDefault()) { var okCount = result.Count(p => p.Status == ImportStatus.Ok); _console.WriteLine($"\nDone Importing {okCount}"); if (okCount != 0) { _console.WriteLine($"Time: {Math.Round(stopWatch.Elapsed.TotalSeconds, 1)} " + $"sec. or {Math.Round(stopWatch.Elapsed.TotalMinutes, 1)} min."); } _console.WriteLine($"Failed: {result.Count(p => p.Status != ImportStatus.Ok)}"); } if (importSettings.ConsoleOutputMode != ConsoleOutputMode.Csv) { return; } _console.WriteLine($"Id;Status;SourceFullFilePath;SubPath;FileHash"); foreach (var item in result) { var filePath = item.Status == ImportStatus.Ok ? item.FilePath : ""; _console.WriteLine($"{item.Id};{item.Status};" + $"{item.SourceFullFilePath};" + $"{filePath};" + $"{item.GetFileHashWithUpdate()}"); } }
internal async Task AddToQueryAndImportDatabaseAsync( ImportIndexItem importIndexItem, ImportSettingsModel importSettings) { if (!importSettings.IndexMode || _importQuery?.TestConnection() != true) { if (_appSettings.IsVerbose()) { _logger.LogInformation(" AddToQueryAndImportDatabaseAsync Ignored - " + $"IndexMode {importSettings.IndexMode} " + $"TestConnection {_importQuery?.TestConnection()}"); } return; } // Add to Normal File Index database var query = new QueryFactory(new SetupDatabaseTypes(_appSettings), _query, _memoryCache, _appSettings, _logger).Query(); await query !.AddItemAsync(importIndexItem.FileIndexItem); // Add to check db, to avoid duplicate input var importQuery = new ImportQueryFactory(new SetupDatabaseTypes(_appSettings), _importQuery, _console, _logger).ImportQuery(); await importQuery.AddAsync(importIndexItem, importSettings.IsConsoleOutputModeDefault()); await query.DisposeAsync(); }
public async Task <List <ImportIndexItem> > Preflight(List <string> fullFilePathsList, ImportSettingsModel importSettings) { var includedDirectoryFilePaths = AppendDirectoryFilePaths( fullFilePathsList, importSettings).ToList(); // When Directory is Empty if (!includedDirectoryFilePaths.Any()) { return(new List <ImportIndexItem>()); } var importIndexItemsIEnumerable = await includedDirectoryFilePaths .ForEachAsync( async (includedFilePath) => await PreflightPerFile(includedFilePath, importSettings), _appSettings.MaxDegreesOfParallelism); var importIndexItemsList = importIndexItemsIEnumerable.ToList(); var directoriesContent = ParentFoldersDictionary(importIndexItemsList); importIndexItemsList = CheckForDuplicateNaming(importIndexItemsList.ToList(), directoriesContent); return(importIndexItemsList); }
public async Task <List <ImportIndexItem> > Importer(IEnumerable <string> inputFullPathList, ImportSettingsModel importSettings) { var preflightItemList = await Preflight(inputFullPathList.ToList(), importSettings); // When directory is empty if (!preflightItemList.Any()) { return(new List <ImportIndexItem>()); } var directoriesContent = ParentFoldersDictionary(preflightItemList); if (importSettings.IndexMode) { await CreateParentFolders(directoriesContent); } var importIndexItemsList = (await preflightItemList.AsEnumerable() .ForEachAsync( async(preflightItem) => await Importer(preflightItem, importSettings), _appSettings.MaxDegreesOfParallelism)).ToList(); await CreateMataThumbnail(importIndexItemsList, importSettings); return(importIndexItemsList); }
public void ImportFileSettingsModel_DefaultsToIgnore_Test() { var importSettings = new ImportSettingsModel { ColorClass = 999 }; Assert.AreEqual(-1, importSettings.ColorClass); }
public void ImportSettingsModel_toDefaults_Test() { var context = new DefaultHttpContext(); var importSettings = new ImportSettingsModel(context.Request); Assert.AreEqual(string.Empty, importSettings.Structure); }
public void ImportIndexItem_CtorRequest_ColorClass() { var context = new DefaultHttpContext(); context.Request.Headers["ColorClass"] = "1"; var model = new ImportSettingsModel(context.Request); Assert.AreEqual(1, model.ColorClass); }
public List <string> Import(string inputFullPathList, ImportSettingsModel importSettings) { HistoryList = Preflight(new List <string> { inputFullPathList }, importSettings).Result; return(new List <string> { inputFullPathList }); }
private void DeleteFileAfter(ImportSettingsModel importSettings, ImportIndexItem importIndexItem) { // to move files if (!importSettings.DeleteAfter) { return; } if (_appSettings.IsVerbose()) { _console.WriteLine($"🚮 Delete file: {importIndexItem.SourceFullFilePath}"); } _filesystemStorage.FileDelete(importIndexItem.SourceFullFilePath); }
/// <summary> /// Command line importer to Database and update disk /// </summary> /// <param name="args">arguments provided by command line app</param> /// <returns>Void Task</returns> public async Task Importer(string[] args) { Console.WriteLine("run importer"); _appSettings.Verbose = ArgsHelper.NeedVerbose(args); await _exifToolDownload.DownloadExifTool(_appSettings.IsWindows); _appSettings.ApplicationType = AppSettings.StarskyAppType.Importer; if (new ArgsHelper().NeedHelp(args) || new ArgsHelper(_appSettings) .GetPathFormArgs(args, false).Length <= 1) { new ArgsHelper(_appSettings, _console).NeedHelpShowDialog(); return; } var inputPathListFormArgs = new ArgsHelper(_appSettings).GetPathListFormArgs(args); if (_appSettings.IsVerbose()) { foreach (var inputPath in inputPathListFormArgs) { _console.WriteLine($">> import: {inputPath}"); } } var importSettings = new ImportSettingsModel { DeleteAfter = ArgsHelper.GetMove(args), RecursiveDirectory = ArgsHelper.NeedRecursive(args), IndexMode = new ArgsHelper().GetIndexMode(args), ColorClass = ArgsHelper.GetColorClass(args), ConsoleOutputMode = ArgsHelper.GetConsoleOutputMode(args) }; if (_appSettings.IsVerbose()) { _console.WriteLine($"Options: DeleteAfter: {importSettings.DeleteAfter}, " + $"RecursiveDirectory {importSettings.RecursiveDirectory}, " + $"ColorClass (overwrite) {importSettings.ColorClass}, " + $"Structure {_appSettings.Structure}, " + $"IndexMode {importSettings.IndexMode}"); } var stopWatch = Stopwatch.StartNew(); var result = await _importService.Importer(inputPathListFormArgs, importSettings); WriteOutputStatus(importSettings, result, stopWatch); Console.WriteLine("done import"); }
public void ImportSettingsModel_IndexMode_Test() { var context = new DefaultHttpContext(); // false context.Request.Headers["IndexMode"] = "false"; var model = new ImportSettingsModel(context.Request); Assert.AreEqual(false, model.IndexMode); // now true context.Request.Headers["IndexMode"] = "true"; model = new ImportSettingsModel(context.Request); Assert.AreEqual(true, model.IndexMode); }
internal async Task <bool> CreateMataThumbnail(IEnumerable <ImportIndexItem> importIndexItemsList, ImportSettingsModel importSettings) { if (_appSettings.MetaThumbnailOnImport == false || !importSettings.IndexMode) { return(false); } var items = importIndexItemsList .Where(p => p.Status == ImportStatus.Ok) .Select(p => (p.FilePath, p.FileIndexItem.FileHash)).ToList(); if (!items.Any()) { return(false); } return(await _metaExifThumbnailService.AddMetaThumbnail(items)); }
internal async Task <List <ImportIndexItem> > ImportPostBackgroundTask(List <string> tempImportPaths, ImportSettingsModel importSettings, bool isVerbose = false) { List <ImportIndexItem> importedFiles; using (var scope = _scopeFactory.CreateScope()) { var selectorStorage = scope.ServiceProvider.GetRequiredService <ISelectorStorage>(); var importQuery = scope.ServiceProvider.GetRequiredService <IImportQuery>(); var exifTool = scope.ServiceProvider.GetRequiredService <IExifTool>(); var query = scope.ServiceProvider.GetRequiredService <IQuery>(); var console = scope.ServiceProvider.GetRequiredService <IConsole>(); var metaExifThumbnailService = scope.ServiceProvider.GetRequiredService <IMetaExifThumbnailService>(); var memoryCache = scope.ServiceProvider.GetRequiredService <IMemoryCache>(); // use of IImport direct does not work importedFiles = await new Import(selectorStorage, _appSettings, importQuery, exifTool, query, console, metaExifThumbnailService, _logger, memoryCache).Importer(tempImportPaths, importSettings); } if (isVerbose) { foreach (var file in importedFiles) { _logger.LogInformation( $"[ImportPostBackgroundTask] import {file.Status} " + $"=> {file.FilePath} ~ {file.FileIndexItem?.FilePath}"); } } // Remove source files foreach (var toDelPath in tempImportPaths) { RemoveTempAndParentStreamFolder(toDelPath); } return(importedFiles); }
public async Task <IActionResult> FromUrl(string fileUrl, string filename, string structure) { if (filename == null) { filename = Base32.Encode(FileHash.GenerateRandomBytes(8)) + ".unknown"; } // I/O function calls should not be vulnerable to path injection attacks if (!Regex.IsMatch(filename, "^[a-zA-Z0-9_\\s\\.]+$") || !FilenamesHelper.IsValidFileName(filename)) { return(BadRequest()); } var tempImportFullPath = Path.Combine(_appSettings.TempFolder, filename); var importSettings = new ImportSettingsModel(Request) { Structure = structure }; var isDownloaded = await _httpClientHelper.Download(fileUrl, tempImportFullPath); if (!isDownloaded) { return(NotFound("'file url' not found or domain not allowed " + fileUrl)); } var importedFiles = await _import.Importer(new List <string> { tempImportFullPath }, importSettings); RemoveTempAndParentStreamFolder(tempImportFullPath); if (importedFiles.Count == 0) { Response.StatusCode = 206; } return(Json(importedFiles)); }
public Task <List <ImportIndexItem> > Preflight(List <string> inputFileFullPaths, ImportSettingsModel importSettings) { var results = new List <ImportIndexItem>(); foreach (var inputFileFullPath in inputFileFullPaths) { // if the item fails var importIndexFileError = new ImportIndexItem { FilePath = "/" + FilenamesHelper.GetFileName(inputFileFullPath), SourceFullFilePath = "~/temp/test", FileHash = "FAKE", MakeModel = "added if the item fails", Status = ImportStatus.FileError }; // Check if extension is correct if (!ExtensionRolesHelper.IsExtensionSyncSupported(inputFileFullPath)) { results.Add(importIndexFileError); } // Check if the file is correct var imageFormat = ExtensionRolesHelper.GetImageFormat( _selectorStorage.Get(SelectorStorage.StorageServices.HostFilesystem) .ReadStream(inputFileFullPath, 160)); if (!ExtensionRolesHelper.ExtensionSyncSupportedList.Contains($"{imageFormat}")) { results.Add(importIndexFileError); } results.Add(new ImportIndexItem { Id = 4, SourceFullFilePath = inputFileFullPath, FilePath = inputFileFullPath, Status = ImportStatus.Ok, FileHash = "FAKE", MakeModel = "added okay", FileIndexItem = new FileIndexItem() { FileHash = "FAKE_OK", FilePath = inputFileFullPath } }); } PreflightList.AddRange(results); return(Task.FromResult(results)); }
public List <string> ImportTo(string inputFullPathList, ImportSettingsModel importSettings) { throw new System.NotImplementedException(); }
internal async Task <ImportIndexItem> Importer(ImportIndexItem importIndexItem, ImportSettingsModel importSettings) { if (importIndexItem.Status != ImportStatus.Ok) { return(importIndexItem); } // True when exist and file type is raw var xmpExistForThisFileType = ExistXmpSidecarForThisFileType(importIndexItem); if (xmpExistForThisFileType || (_appSettings.ExifToolImportXmpCreate && ExtensionRolesHelper.IsExtensionForceXmp(importIndexItem.FilePath))) { // When a xmp file already exist (only for raws) // AND when this created afterwards with the ExifToolImportXmpCreate setting (only for raws) importIndexItem.FileIndexItem.AddSidecarExtension("xmp"); } // Add item to database await AddToQueryAndImportDatabaseAsync(importIndexItem, importSettings); // Copy if (_appSettings.IsVerbose()) { _logger.LogInformation("[Import] Next Action = Copy" + $" {importIndexItem.SourceFullFilePath} {importIndexItem.FilePath}"); } using (var sourceStream = _filesystemStorage.ReadStream(importIndexItem.SourceFullFilePath)) await _subPathStorage.WriteStreamAsync(sourceStream, importIndexItem.FilePath); // Copy the sidecar file if (xmpExistForThisFileType) { var xmpSourceFullFilePath = ExtensionRolesHelper.ReplaceExtensionWithXmp(importIndexItem.SourceFullFilePath); var destinationXmpFullPath = ExtensionRolesHelper.ReplaceExtensionWithXmp(importIndexItem.FilePath); _filesystemStorage.FileCopy(xmpSourceFullFilePath, destinationXmpFullPath); } await CreateSideCarFile(importIndexItem, xmpExistForThisFileType); // Run Exiftool to Update for example colorClass UpdateImportTransformations.QueryUpdateDelegate?updateItemAsync = null; if (importSettings.IndexMode) { updateItemAsync = new QueryFactory( new SetupDatabaseTypes(_appSettings), _query, _memoryCache, _appSettings, _logger).Query() !.UpdateItemAsync; } importIndexItem.FileIndexItem = await _updateImportTransformations.UpdateTransformations(updateItemAsync, importIndexItem.FileIndexItem, importSettings.ColorClass, importIndexItem.DateTimeFromFileName, importSettings.IndexMode); DeleteFileAfter(importSettings, importIndexItem); if (_appSettings.IsVerbose()) { _console.Write("+"); } return(importIndexItem); }
internal async Task <ImportIndexItem> PreflightPerFile(KeyValuePair <string, bool> inputFileFullPath, ImportSettingsModel importSettings) { if (_appSettings.ImportIgnore.Any(p => inputFileFullPath.Key.Contains(p))) { if (_appSettings.IsVerbose()) { _console.WriteLine($"❌ skip due rules: {inputFileFullPath.Key} "); } return(new ImportIndexItem { Status = ImportStatus.Ignore, FilePath = inputFileFullPath.Key, SourceFullFilePath = inputFileFullPath.Key, AddToDatabase = DateTime.UtcNow }); } if (!inputFileFullPath.Value || !_filesystemStorage.ExistFile(inputFileFullPath.Key)) { if (_appSettings.IsVerbose()) { _console.WriteLine($"❌ not found: {inputFileFullPath.Key}"); } return(new ImportIndexItem { Status = ImportStatus.NotFound, FilePath = inputFileFullPath.Key, SourceFullFilePath = inputFileFullPath.Key, AddToDatabase = DateTime.UtcNow }); } var imageFormat = ExtensionRolesHelper.GetImageFormat( _filesystemStorage.ReadStream(inputFileFullPath.Key, 160)); // Check if extension is correct && Check if the file is correct if (!ExtensionRolesHelper.IsExtensionSyncSupported(inputFileFullPath.Key) || !ExtensionRolesHelper.IsExtensionSyncSupported($".{imageFormat}")) { if (_appSettings.IsVerbose()) { _console.WriteLine($"❌ extension not supported: {inputFileFullPath.Key}"); } return(new ImportIndexItem { Status = ImportStatus.FileError, FilePath = inputFileFullPath.Key, SourceFullFilePath = inputFileFullPath.Key }); } var hashList = await new FileHash(_filesystemStorage).GetHashCodeAsync(inputFileFullPath.Key); if (!hashList.Value) { if (_appSettings.IsVerbose()) { _console.WriteLine($"❌ FileHash error {inputFileFullPath.Key}"); } return(new ImportIndexItem { Status = ImportStatus.FileError, FilePath = inputFileFullPath.Key, SourceFullFilePath = inputFileFullPath.Key }); } if (importSettings.IndexMode && await _importQuery !.IsHashInImportDbAsync(hashList.Key)) { if (_appSettings.IsVerbose()) { _console.WriteLine($"🤷 Ignored, exist already {inputFileFullPath.Key}"); } return(new ImportIndexItem { Status = ImportStatus.IgnoredAlreadyImported, FilePath = inputFileFullPath.Key, FileHash = hashList.Key, AddToDatabase = DateTime.UtcNow, SourceFullFilePath = inputFileFullPath.Key }); } // Only accept files with correct meta data // Check if there is a xmp file that contains data var fileIndexItem = _readMetaHost.ReadExifAndXmpFromFile(inputFileFullPath.Key); // Parse the filename and create a new importIndexItem object var importIndexItem = ObjectCreateIndexItem(inputFileFullPath.Key, imageFormat, hashList.Key, fileIndexItem, importSettings.ColorClass, _filesystemStorage.Info(inputFileFullPath.Key).Size); // Update the parent and filenames importIndexItem = ApplyStructure(importIndexItem, importSettings.Structure); return(importIndexItem); }
public async Task <List <ImportIndexItem> > Importer(IEnumerable <string> inputFullPathList, ImportSettingsModel importSettings) { var preflight = await Preflight(inputFullPathList.ToList(), importSettings); HistoryList.AddRange(preflight); return(preflight); }