private List <KeyValuePair <string, bool> > AppendDirectoryFilePaths(List <string> fullFilePathsList,
ImportSettingsModel importSettings)
{
var includedDirectoryFilePaths = new List <KeyValuePair <string, bool> >();
foreach (var fullFilePath in fullFilePathsList)
{
if (_filesystemStorage.ExistFolder(fullFilePath) && importSettings.RecursiveDirectory)
{
// recursive
includedDirectoryFilePaths.AddRange(_filesystemStorage.
GetAllFilesInDirectoryRecursive(fullFilePath)
.Where(ExtensionRolesHelper.IsExtensionSyncSupported)
.Select(syncedFiles => new KeyValuePair <string, bool>(syncedFiles, true)));
continue;
}
if (_filesystemStorage.ExistFolder(fullFilePath) && !importSettings.RecursiveDirectory)
{
// non-recursive
includedDirectoryFilePaths.AddRange(_filesystemStorage.GetAllFilesInDirectory(fullFilePath)
.Where(ExtensionRolesHelper.IsExtensionSyncSupported)
.Select(syncedFiles => new KeyValuePair <string, bool>(syncedFiles, true)));
continue;
}
includedDirectoryFilePaths.Add(
new KeyValuePair <string, bool>(fullFilePath, _filesystemStorage.ExistFile(fullFilePath))
);
}
return(includedDirectoryFilePaths);
}
[ProducesResponseType(typeof(List <ImportIndexItem>), 415)] // Wrong input (e.g. wrong extenstion type)
public async Task <IActionResult> IndexPost()
{
var tempImportPaths = await Request.StreamFile(_appSettings, _selectorStorage);
var importSettings = new ImportSettingsModel(Request);
var fileIndexResultsList = await _import.Preflight(tempImportPaths, importSettings);
// Import files >
_bgTaskQueue.QueueBackgroundWorkItem(async token =>
{
await ImportPostBackgroundTask(tempImportPaths, importSettings, _appSettings.IsVerbose());
});
// When all items are already imported
if (importSettings.IndexMode &&
fileIndexResultsList.All(p => p.Status != ImportStatus.Ok))
{
Response.StatusCode = 206;
}
// Wrong input (extension is not allowed)
if (fileIndexResultsList.All(p => p.Status == ImportStatus.FileError))
{
Response.StatusCode = 415;
}
return(Json(fileIndexResultsList));
}
private void WriteOutputStatus(ImportSettingsModel importSettings, List <ImportIndexItem> result, Stopwatch stopWatch)
{
if (importSettings.IsConsoleOutputModeDefault())
{
var okCount = result.Count(p => p.Status == ImportStatus.Ok);
_console.WriteLine($"\nDone Importing {okCount}");
if (okCount != 0)
{
_console.WriteLine($"Time: {Math.Round(stopWatch.Elapsed.TotalSeconds, 1)} " +
$"sec. or {Math.Round(stopWatch.Elapsed.TotalMinutes, 1)} min.");
}
_console.WriteLine($"Failed: {result.Count(p => p.Status != ImportStatus.Ok)}");
}
if (importSettings.ConsoleOutputMode != ConsoleOutputMode.Csv)
{
return;
}
_console.WriteLine($"Id;Status;SourceFullFilePath;SubPath;FileHash");
foreach (var item in result)
{
var filePath = item.Status == ImportStatus.Ok
? item.FilePath : "";
_console.WriteLine($"{item.Id};{item.Status};" +
$"{item.SourceFullFilePath};" +
$"{filePath};" +
$"{item.GetFileHashWithUpdate()}");
}
}
internal async Task AddToQueryAndImportDatabaseAsync(
ImportIndexItem importIndexItem,
ImportSettingsModel importSettings)
{
if (!importSettings.IndexMode || _importQuery?.TestConnection() != true)
{
if (_appSettings.IsVerbose())
{
_logger.LogInformation(" AddToQueryAndImportDatabaseAsync Ignored - " +
$"IndexMode {importSettings.IndexMode} " +
$"TestConnection {_importQuery?.TestConnection()}");
}
return;
}
// Add to Normal File Index database
var query = new QueryFactory(new SetupDatabaseTypes(_appSettings), _query,
_memoryCache, _appSettings, _logger).Query();
await query !.AddItemAsync(importIndexItem.FileIndexItem);
// Add to check db, to avoid duplicate input
var importQuery = new ImportQueryFactory(new SetupDatabaseTypes(_appSettings), _importQuery, _console, _logger).ImportQuery();
await importQuery.AddAsync(importIndexItem, importSettings.IsConsoleOutputModeDefault());
await query.DisposeAsync();
}
public async Task <List <ImportIndexItem> > Preflight(List <string> fullFilePathsList,
ImportSettingsModel importSettings)
{
var includedDirectoryFilePaths = AppendDirectoryFilePaths(
fullFilePathsList,
importSettings).ToList();
// When Directory is Empty
if (!includedDirectoryFilePaths.Any())
{
return(new List <ImportIndexItem>());
}
var importIndexItemsIEnumerable = await includedDirectoryFilePaths
.ForEachAsync(
async (includedFilePath)
=> await PreflightPerFile(includedFilePath, importSettings),
_appSettings.MaxDegreesOfParallelism);
var importIndexItemsList = importIndexItemsIEnumerable.ToList();
var directoriesContent = ParentFoldersDictionary(importIndexItemsList);
importIndexItemsList = CheckForDuplicateNaming(importIndexItemsList.ToList(), directoriesContent);
return(importIndexItemsList);
}
public async Task <List <ImportIndexItem> > Importer(IEnumerable <string> inputFullPathList,
ImportSettingsModel importSettings)
{
var preflightItemList = await Preflight(inputFullPathList.ToList(), importSettings);
// When directory is empty
if (!preflightItemList.Any())
{
return(new List <ImportIndexItem>());
}
var directoriesContent = ParentFoldersDictionary(preflightItemList);
if (importSettings.IndexMode)
{
await CreateParentFolders(directoriesContent);
}
var importIndexItemsList = (await preflightItemList.AsEnumerable()
.ForEachAsync(
async(preflightItem)
=> await Importer(preflightItem, importSettings),
_appSettings.MaxDegreesOfParallelism)).ToList();
await CreateMataThumbnail(importIndexItemsList, importSettings);
return(importIndexItemsList);
}
public void ImportFileSettingsModel_DefaultsToIgnore_Test()
{
var importSettings = new ImportSettingsModel {
ColorClass = 999
};
Assert.AreEqual(-1, importSettings.ColorClass);
}
public void ImportSettingsModel_toDefaults_Test()
{
var context = new DefaultHttpContext();
var importSettings = new ImportSettingsModel(context.Request);
Assert.AreEqual(string.Empty, importSettings.Structure);
}
public void ImportIndexItem_CtorRequest_ColorClass()
{
var context = new DefaultHttpContext();
context.Request.Headers["ColorClass"] = "1";
var model = new ImportSettingsModel(context.Request);
Assert.AreEqual(1, model.ColorClass);
}
public List <string> Import(string inputFullPathList, ImportSettingsModel importSettings)
{
HistoryList = Preflight(new List <string> {
inputFullPathList
}, importSettings).Result;
return(new List <string> {
inputFullPathList
});
}
private void DeleteFileAfter(ImportSettingsModel importSettings,
ImportIndexItem importIndexItem)
{
// to move files
if (!importSettings.DeleteAfter)
{
return;
}
if (_appSettings.IsVerbose())
{
_console.WriteLine($"🚮 Delete file: {importIndexItem.SourceFullFilePath}");
}
_filesystemStorage.FileDelete(importIndexItem.SourceFullFilePath);
}
/// <summary>
/// Command line importer to Database and update disk
/// </summary>
/// <param name="args">arguments provided by command line app</param>
/// <returns>Void Task</returns>
public async Task Importer(string[] args)
{
Console.WriteLine("run importer");
_appSettings.Verbose = ArgsHelper.NeedVerbose(args);
await _exifToolDownload.DownloadExifTool(_appSettings.IsWindows);
_appSettings.ApplicationType = AppSettings.StarskyAppType.Importer;
if (new ArgsHelper().NeedHelp(args) || new ArgsHelper(_appSettings)
.GetPathFormArgs(args, false).Length <= 1)
{
new ArgsHelper(_appSettings, _console).NeedHelpShowDialog();
return;
}
var inputPathListFormArgs = new ArgsHelper(_appSettings).GetPathListFormArgs(args);
if (_appSettings.IsVerbose())
{
foreach (var inputPath in inputPathListFormArgs)
{
_console.WriteLine($">> import: {inputPath}");
}
}
var importSettings = new ImportSettingsModel {
DeleteAfter = ArgsHelper.GetMove(args),
RecursiveDirectory = ArgsHelper.NeedRecursive(args),
IndexMode = new ArgsHelper().GetIndexMode(args),
ColorClass = ArgsHelper.GetColorClass(args),
ConsoleOutputMode = ArgsHelper.GetConsoleOutputMode(args)
};
if (_appSettings.IsVerbose())
{
_console.WriteLine($"Options: DeleteAfter: {importSettings.DeleteAfter}, " +
$"RecursiveDirectory {importSettings.RecursiveDirectory}, " +
$"ColorClass (overwrite) {importSettings.ColorClass}, " +
$"Structure {_appSettings.Structure}, " +
$"IndexMode {importSettings.IndexMode}");
}
var stopWatch = Stopwatch.StartNew();
var result = await _importService.Importer(inputPathListFormArgs, importSettings);
WriteOutputStatus(importSettings, result, stopWatch);
Console.WriteLine("done import");
}
public void ImportSettingsModel_IndexMode_Test()
{
var context = new DefaultHttpContext();
// false
context.Request.Headers["IndexMode"] = "false";
var model = new ImportSettingsModel(context.Request);
Assert.AreEqual(false, model.IndexMode);
// now true
context.Request.Headers["IndexMode"] = "true";
model = new ImportSettingsModel(context.Request);
Assert.AreEqual(true, model.IndexMode);
}
internal async Task <bool> CreateMataThumbnail(IEnumerable <ImportIndexItem>
importIndexItemsList, ImportSettingsModel importSettings)
{
if (_appSettings.MetaThumbnailOnImport == false || !importSettings.IndexMode)
{
return(false);
}
var items = importIndexItemsList
.Where(p => p.Status == ImportStatus.Ok)
.Select(p => (p.FilePath, p.FileIndexItem.FileHash)).ToList();
if (!items.Any())
{
return(false);
}
return(await _metaExifThumbnailService.AddMetaThumbnail(items));
}
internal async Task <List <ImportIndexItem> > ImportPostBackgroundTask(List <string> tempImportPaths,
ImportSettingsModel importSettings, bool isVerbose = false)
{
List <ImportIndexItem> importedFiles;
using (var scope = _scopeFactory.CreateScope())
{
var selectorStorage = scope.ServiceProvider.GetRequiredService <ISelectorStorage>();
var importQuery = scope.ServiceProvider.GetRequiredService <IImportQuery>();
var exifTool = scope.ServiceProvider.GetRequiredService <IExifTool>();
var query = scope.ServiceProvider.GetRequiredService <IQuery>();
var console = scope.ServiceProvider.GetRequiredService <IConsole>();
var metaExifThumbnailService = scope.ServiceProvider.GetRequiredService <IMetaExifThumbnailService>();
var memoryCache = scope.ServiceProvider.GetRequiredService <IMemoryCache>();
// use of IImport direct does not work
importedFiles = await new Import(selectorStorage, _appSettings,
importQuery, exifTool, query, console,
metaExifThumbnailService, _logger, memoryCache).Importer(tempImportPaths, importSettings);
}
if (isVerbose)
{
foreach (var file in importedFiles)
{
_logger.LogInformation(
$"[ImportPostBackgroundTask] import {file.Status} " +
$"=> {file.FilePath} ~ {file.FileIndexItem?.FilePath}");
}
}
// Remove source files
foreach (var toDelPath in tempImportPaths)
{
RemoveTempAndParentStreamFolder(toDelPath);
}
return(importedFiles);
}
public async Task <IActionResult> FromUrl(string fileUrl, string filename, string structure)
{
if (filename == null)
{
filename = Base32.Encode(FileHash.GenerateRandomBytes(8)) + ".unknown";
}
// I/O function calls should not be vulnerable to path injection attacks
if (!Regex.IsMatch(filename, "^[a-zA-Z0-9_\\s\\.]+$") || !FilenamesHelper.IsValidFileName(filename))
{
return(BadRequest());
}
var tempImportFullPath = Path.Combine(_appSettings.TempFolder, filename);
var importSettings = new ImportSettingsModel(Request)
{
Structure = structure
};
var isDownloaded = await _httpClientHelper.Download(fileUrl, tempImportFullPath);
if (!isDownloaded)
{
return(NotFound("'file url' not found or domain not allowed " + fileUrl));
}
var importedFiles = await _import.Importer(new List <string> {
tempImportFullPath
}, importSettings);
RemoveTempAndParentStreamFolder(tempImportFullPath);
if (importedFiles.Count == 0)
{
Response.StatusCode = 206;
}
return(Json(importedFiles));
}
public Task <List <ImportIndexItem> > Preflight(List <string> inputFileFullPaths, ImportSettingsModel importSettings)
{
var results = new List <ImportIndexItem>();
foreach (var inputFileFullPath in inputFileFullPaths)
{
// if the item fails
var importIndexFileError = new ImportIndexItem {
FilePath = "/" + FilenamesHelper.GetFileName(inputFileFullPath),
SourceFullFilePath = "~/temp/test",
FileHash = "FAKE",
MakeModel = "added if the item fails",
Status = ImportStatus.FileError
};
// Check if extension is correct
if (!ExtensionRolesHelper.IsExtensionSyncSupported(inputFileFullPath))
{
results.Add(importIndexFileError);
}
// Check if the file is correct
var imageFormat = ExtensionRolesHelper.GetImageFormat(
_selectorStorage.Get(SelectorStorage.StorageServices.HostFilesystem)
.ReadStream(inputFileFullPath, 160));
if (!ExtensionRolesHelper.ExtensionSyncSupportedList.Contains($"{imageFormat}"))
{
results.Add(importIndexFileError);
}
results.Add(new ImportIndexItem
{
Id = 4,
SourceFullFilePath = inputFileFullPath,
FilePath = inputFileFullPath,
Status = ImportStatus.Ok,
FileHash = "FAKE",
MakeModel = "added okay",
FileIndexItem = new FileIndexItem()
{
FileHash = "FAKE_OK", FilePath = inputFileFullPath
}
});
}
PreflightList.AddRange(results);
return(Task.FromResult(results));
}
public List <string> ImportTo(string inputFullPathList, ImportSettingsModel importSettings)
{
throw new System.NotImplementedException();
}
internal async Task <ImportIndexItem> Importer(ImportIndexItem importIndexItem,
ImportSettingsModel importSettings)
{
if (importIndexItem.Status != ImportStatus.Ok)
{
return(importIndexItem);
}
// True when exist and file type is raw
var xmpExistForThisFileType = ExistXmpSidecarForThisFileType(importIndexItem);
if (xmpExistForThisFileType || (_appSettings.ExifToolImportXmpCreate &&
ExtensionRolesHelper.IsExtensionForceXmp(importIndexItem.FilePath)))
{
// When a xmp file already exist (only for raws)
// AND when this created afterwards with the ExifToolImportXmpCreate setting (only for raws)
importIndexItem.FileIndexItem.AddSidecarExtension("xmp");
}
// Add item to database
await AddToQueryAndImportDatabaseAsync(importIndexItem, importSettings);
// Copy
if (_appSettings.IsVerbose())
{
_logger.LogInformation("[Import] Next Action = Copy" +
$" {importIndexItem.SourceFullFilePath} {importIndexItem.FilePath}");
}
using (var sourceStream = _filesystemStorage.ReadStream(importIndexItem.SourceFullFilePath))
await _subPathStorage.WriteStreamAsync(sourceStream, importIndexItem.FilePath);
// Copy the sidecar file
if (xmpExistForThisFileType)
{
var xmpSourceFullFilePath = ExtensionRolesHelper.ReplaceExtensionWithXmp(importIndexItem.SourceFullFilePath);
var destinationXmpFullPath = ExtensionRolesHelper.ReplaceExtensionWithXmp(importIndexItem.FilePath);
_filesystemStorage.FileCopy(xmpSourceFullFilePath, destinationXmpFullPath);
}
await CreateSideCarFile(importIndexItem, xmpExistForThisFileType);
// Run Exiftool to Update for example colorClass
UpdateImportTransformations.QueryUpdateDelegate?updateItemAsync = null;
if (importSettings.IndexMode)
{
updateItemAsync = new QueryFactory(
new SetupDatabaseTypes(_appSettings), _query,
_memoryCache, _appSettings, _logger).Query() !.UpdateItemAsync;
}
importIndexItem.FileIndexItem = await _updateImportTransformations.UpdateTransformations(updateItemAsync, importIndexItem.FileIndexItem,
importSettings.ColorClass, importIndexItem.DateTimeFromFileName, importSettings.IndexMode);
DeleteFileAfter(importSettings, importIndexItem);
if (_appSettings.IsVerbose())
{
_console.Write("+");
}
return(importIndexItem);
}
internal async Task <ImportIndexItem> PreflightPerFile(KeyValuePair <string, bool> inputFileFullPath,
ImportSettingsModel importSettings)
{
if (_appSettings.ImportIgnore.Any(p => inputFileFullPath.Key.Contains(p)))
{
if (_appSettings.IsVerbose())
{
_console.WriteLine($"❌ skip due rules: {inputFileFullPath.Key} ");
}
return(new ImportIndexItem {
Status = ImportStatus.Ignore,
FilePath = inputFileFullPath.Key,
SourceFullFilePath = inputFileFullPath.Key,
AddToDatabase = DateTime.UtcNow
});
}
if (!inputFileFullPath.Value || !_filesystemStorage.ExistFile(inputFileFullPath.Key))
{
if (_appSettings.IsVerbose())
{
_console.WriteLine($"❌ not found: {inputFileFullPath.Key}");
}
return(new ImportIndexItem {
Status = ImportStatus.NotFound,
FilePath = inputFileFullPath.Key,
SourceFullFilePath = inputFileFullPath.Key,
AddToDatabase = DateTime.UtcNow
});
}
var imageFormat = ExtensionRolesHelper.GetImageFormat(
_filesystemStorage.ReadStream(inputFileFullPath.Key,
160));
// Check if extension is correct && Check if the file is correct
if (!ExtensionRolesHelper.IsExtensionSyncSupported(inputFileFullPath.Key) ||
!ExtensionRolesHelper.IsExtensionSyncSupported($".{imageFormat}"))
{
if (_appSettings.IsVerbose())
{
_console.WriteLine($"❌ extension not supported: {inputFileFullPath.Key}");
}
return(new ImportIndexItem {
Status = ImportStatus.FileError, FilePath = inputFileFullPath.Key, SourceFullFilePath = inputFileFullPath.Key
});
}
var hashList = await
new FileHash(_filesystemStorage).GetHashCodeAsync(inputFileFullPath.Key);
if (!hashList.Value)
{
if (_appSettings.IsVerbose())
{
_console.WriteLine($"❌ FileHash error {inputFileFullPath.Key}");
}
return(new ImportIndexItem {
Status = ImportStatus.FileError, FilePath = inputFileFullPath.Key, SourceFullFilePath = inputFileFullPath.Key
});
}
if (importSettings.IndexMode && await _importQuery !.IsHashInImportDbAsync(hashList.Key))
{
if (_appSettings.IsVerbose())
{
_console.WriteLine($"🤷 Ignored, exist already {inputFileFullPath.Key}");
}
return(new ImportIndexItem
{
Status = ImportStatus.IgnoredAlreadyImported,
FilePath = inputFileFullPath.Key,
FileHash = hashList.Key,
AddToDatabase = DateTime.UtcNow,
SourceFullFilePath = inputFileFullPath.Key
});
}
// Only accept files with correct meta data
// Check if there is a xmp file that contains data
var fileIndexItem = _readMetaHost.ReadExifAndXmpFromFile(inputFileFullPath.Key);
// Parse the filename and create a new importIndexItem object
var importIndexItem = ObjectCreateIndexItem(inputFileFullPath.Key, imageFormat,
hashList.Key, fileIndexItem, importSettings.ColorClass,
_filesystemStorage.Info(inputFileFullPath.Key).Size);
// Update the parent and filenames
importIndexItem = ApplyStructure(importIndexItem, importSettings.Structure);
return(importIndexItem);
}
public async Task <List <ImportIndexItem> > Importer(IEnumerable <string> inputFullPathList, ImportSettingsModel importSettings)
{
var preflight = await Preflight(inputFullPathList.ToList(), importSettings);
HistoryList.AddRange(preflight);
return(preflight);
}
