public void GetImageFormat_QuickTimeMp4_Test() { var newImage = CreateAnQuickTimeMp4.Bytes.Take(15).ToArray(); var result = ExtensionRolesHelper.GetImageFormat(newImage); Assert.AreEqual(ExtensionRolesHelper.ImageFormat.mp4, result); }
public void GetImageFormat_Jpeg_Test() { var newImage = CreateAnImage.Bytes.Take(15).ToArray(); var result = ExtensionRolesHelper.GetImageFormat(newImage); Assert.AreEqual(ExtensionRolesHelper.ImageFormat.jpg, result); }
public void Files_GetImageFormat_xmp_Test() { byte[] bmBytes = Encoding.ASCII.GetBytes("<x:xmpmeta"); var fileType = ExtensionRolesHelper.GetImageFormat(bmBytes); Assert.AreEqual(fileType, ExtensionRolesHelper.ImageFormat.xmp); }
public void Files_ExtensionThumbSupportedList_TiffMp4MovXMPCheck() { Assert.AreEqual(false, ExtensionRolesHelper.IsExtensionThumbnailSupported("file.tiff")); Assert.AreEqual(false, ExtensionRolesHelper.IsExtensionThumbnailSupported("file.mp4")); Assert.AreEqual(false, ExtensionRolesHelper.IsExtensionThumbnailSupported("file.mov")); Assert.AreEqual(false, ExtensionRolesHelper.IsExtensionThumbnailSupported("file.xmp")); }
public FileIndexItem XmpGetSidecarFile(FileIndexItem databaseItem) { if (databaseItem == null) { databaseItem = new FileIndexItem(); } // Parse an xmp file for this location var xmpSubPath = ExtensionRolesHelper.ReplaceExtensionWithXmp(databaseItem.FilePath); // also add when the file is a jpeg, we are not writing to it then if (_iStorage.ExistFile(xmpSubPath)) { databaseItem.AddSidecarExtension("xmp"); } // Read content from sidecar xmp file if (!ExtensionRolesHelper.IsExtensionForceXmp(databaseItem.FilePath) || !_iStorage.ExistFile(xmpSubPath)) { return(databaseItem); } // Read the text-content of the xmp file. var xmp = new PlainTextFileHelper().StreamToString(_iStorage.ReadStream(xmpSubPath)); // Get the data from the xmp databaseItem = GetDataFromString(xmp, databaseItem); return(databaseItem); }
public void Files_GetImageFormat_gif_Test() { byte[] bmBytes = Encoding.ASCII.GetBytes("GIF"); var fileType = ExtensionRolesHelper.GetImageFormat(bmBytes); Assert.AreEqual(fileType, ExtensionRolesHelper.ImageFormat.gif); }
private FileIndexItem ReadExifAndXmpFromFileDirect(string subPath) { if (_iStorage.ExistFile(subPath) && ExtensionRolesHelper.IsExtensionForceGpx(subPath)) { return(ReadMetaGpx.ReadGpxFromFileReturnAfterFirstField(_iStorage.ReadStream(subPath), subPath)); } var fileIndexItemWithPath = new FileIndexItem(subPath); // Read first the sidecar file var xmpFileIndexItem = _readXmp.XmpGetSidecarFile(fileIndexItemWithPath.Clone()); if (xmpFileIndexItem.IsoSpeed == 0 || string.IsNullOrEmpty(xmpFileIndexItem.Make) || xmpFileIndexItem.DateTime.Year == 0 || xmpFileIndexItem.ImageHeight == 0) { // so the sidecar file is not used var fileExifItemFile = _readExif.ReadExifFromFile(subPath, fileIndexItemWithPath); // overwrite content with incomplete sidecar file (this file can contain tags) FileIndexCompareHelper.Compare(fileExifItemFile, xmpFileIndexItem); return(fileExifItemFile); } return(xmpFileIndexItem); }
private async Task LoopOverSidecarFiles(List <string> subPaths) { var parentDirectories = new HashSet <string>(); var xmpSubPaths = subPaths .Where(ExtensionRolesHelper.IsExtensionSidecar).ToList(); foreach (var xmpPath in xmpSubPaths) { parentDirectories.Add(FilenamesHelper.GetParentPath(xmpPath)); } var itemsInDirectories = new HashSet <FileIndexItem>( await _query.GetAllFilesAsync(parentDirectories.ToList())); // that is an filepath without extension var collectionPath = xmpSubPaths.Select(singlePath => $"{FilenamesHelper.GetParentPath(singlePath)}/" + FilenamesHelper.GetFileNameWithoutExtension(singlePath)).ToList(); foreach (var item in itemsInDirectories) { foreach (var singleCollectionPath in collectionPath) { if (item.FilePath.StartsWith(singleCollectionPath) && !ExtensionRolesHelper.IsExtensionSidecar(item.FilePath)) { item.RemoveSidecarExtension("xmp"); await _query.UpdateItemAsync(item); } } } Console.WriteLine(); }
// used by the html generator public List <FileIndexItem> ReadExifAndXmpFromFileAddFilePathHash(List <string> subPathList, List <string> fileHashes = null) { var fileIndexList = new List <FileIndexItem>(); for (int i = 0; i < subPathList.Count; i++) { var subPath = subPathList[i]; var returnItem = ReadExifAndXmpFromFile(subPath); var imageFormat = ExtensionRolesHelper.GetImageFormat(_iStorage.ReadStream(subPath, 50)); returnItem.ImageFormat = imageFormat; returnItem.FileName = Path.GetFileName(subPath); returnItem.IsDirectory = false; returnItem.Status = FileIndexItem.ExifStatus.Ok; returnItem.ParentDirectory = FilenamesHelper.GetParentPath(subPath); if (fileHashes == null || fileHashes.Count <= i) { returnItem.FileHash = new FileHash(_iStorage).GetHashCode(subPath).Key; } else { returnItem.FileHash = fileHashes[i]; } fileIndexList.Add(returnItem); } return(fileIndexList); }
/// <summary> /// Create a XMP file when it not exist /// </summary> /// <param name="updateModel">model</param> /// <param name="inputSubPaths">list of paths</param> /// <returns>void</returns> internal async Task CreateXmpFileIsNotExist(FileIndexItem updateModel, List <string> inputSubPaths) { foreach (var subPath in inputSubPaths) { // only for raw files if (!ExtensionRolesHelper.IsExtensionForceXmp(subPath)) { return; } var withXmp = ExtensionRolesHelper.ReplaceExtensionWithXmp(subPath); if (_iStorage.IsFolderOrFile(withXmp) != FolderOrFileModel.FolderOrFileTypeList.Deleted) { continue; } new ExifCopy(_iStorage, _thumbnailStorage, _exifTool, _readMeta).XmpCreate(withXmp); var comparedNames = FileIndexCompareHelper.Compare(new FileIndexItem(), updateModel); var command = ExifToolCommandLineArgs(updateModel, comparedNames, true); await _exifTool.WriteTagsAsync(withXmp, command); } }
private IActionResult ReturnThumbnailResult(string f, bool json, ThumbnailSize size) { Response.Headers.Add("x-image-size", new StringValues(size.ToString())); var stream = _thumbnailStorage.ReadStream(ThumbnailNameHelper.Combine(f, size), 50); var imageFormat = ExtensionRolesHelper.GetImageFormat(stream); if (imageFormat == ExtensionRolesHelper.ImageFormat.unknown) { SetExpiresResponseHeadersToZero(); return(NoContent()); // 204 } // When using the api to check using javascript // use the cached version of imageFormat, otherwise you have to check if it deleted if (json) { return(Json("OK")); } stream = _thumbnailStorage.ReadStream( ThumbnailNameHelper.Combine(f, size)); // thumbs are always in jpeg Response.Headers.Add("x-filename", new StringValues(FilenamesHelper.GetFileName(f + ".jpg"))); return(File(stream, "image/jpeg")); }
/// <summary> /// Move sidecar files when those exist /// </summary> /// <param name="inputFileSubPath">from path</param> /// <param name="toFileSubPath">to path</param> private void MoveSidecarFile(string inputFileSubPath, string toFileSubPath) { // json sidecar move var jsonInputFileSubPathSidecarFile = JsonSidecarLocation .JsonLocation(inputFileSubPath); var jsonSidecarFile = JsonSidecarLocation.JsonLocation(toFileSubPath); if (_iStorage.ExistFile(jsonInputFileSubPathSidecarFile)) { _iStorage.FileMove(jsonInputFileSubPathSidecarFile, jsonSidecarFile); } // xmp sidecar file move if (!ExtensionRolesHelper.IsExtensionForceXmp(inputFileSubPath)) { return; } var xmpInputFileSubPathSidecarFile = ExtensionRolesHelper .ReplaceExtensionWithXmp(inputFileSubPath); var xmpSidecarFile = ExtensionRolesHelper .ReplaceExtensionWithXmp(toFileSubPath); if (_iStorage.ExistFile(xmpInputFileSubPathSidecarFile)) { _iStorage.FileMove(xmpInputFileSubPathSidecarFile, xmpSidecarFile); } }
/// <summary> /// When the file is not supported or does not exist return status /// </summary> /// <param name="subPath">relative path</param> /// <returns>item with status</returns> private FileIndexItem CheckForStatusNotOk(string subPath) { var statusItem = new FileIndexItem(subPath) { Status = FileIndexItem.ExifStatus.Ok }; // File extension is not supported if (!ExtensionRolesHelper.IsExtensionSyncSupported(subPath)) { statusItem.Status = FileIndexItem.ExifStatus.OperationNotSupported; return(statusItem); } // File check if jpg #not corrupt var imageFormat = ExtensionRolesHelper.GetImageFormat(_subPathStorage.ReadStream(subPath, 160)); if (imageFormat == ExtensionRolesHelper.ImageFormat.notfound) { statusItem.Status = FileIndexItem.ExifStatus.NotFoundSourceMissing; return(statusItem); } // ReSharper disable once InvertIf if (!ExtensionRolesHelper.ExtensionSyncSupportedList.Contains(imageFormat.ToString())) { statusItem.Status = FileIndexItem.ExifStatus.OperationNotSupported; return(statusItem); } return(statusItem); }
public void Gpx_WithXmlNoPrefix() { var gpxExample = Encoding.ASCII.GetBytes("<gpx xmlns=\"http://www.topografix.com/GPX/1/1\" xmlns:gpxx=\"h"); var result = ExtensionRolesHelper.GetImageFormat(gpxExample); Assert.AreEqual(ExtensionRolesHelper.ImageFormat.gpx, result); }
public void Gpx_Stream_WithXmlPrefix() { var gpxExample = Encoding.ASCII.GetBytes( "<?xml version=\"1.0\" encoding=\"UTF-8\" ?><gpx version=\"1.1\" creator=\"Trails 4.06 - https://www.trails.io\""); var ms = new MemoryStream(gpxExample); var result = ExtensionRolesHelper.GetImageFormat(ms); Assert.AreEqual(ExtensionRolesHelper.ImageFormat.gpx, result); }
private void AddOrRemoveXmpSidecarFileToDatabase(FileIndexItem metaDataItem) { if (_iStorage.ExistFile(ExtensionRolesHelper.ReplaceExtensionWithXmp(metaDataItem .FilePath))) { metaDataItem.AddSidecarExtension("xmp"); return; } metaDataItem.RemoveSidecarExtension("xmp"); }
public IActionResult AllowedTypesThumb(string f) { var result = ExtensionRolesHelper.IsExtensionThumbnailSupported(f); if (!result) { Response.StatusCode = 415; } return(Json(result)); }
private void RemoveXmpSideCarFile(DetailView detailViewItem) { // remove the sidecar file (if exist) if (ExtensionRolesHelper.IsExtensionForceXmp(detailViewItem.FileIndexItem .FileName)) { _iStorage.FileDelete( ExtensionRolesHelper.ReplaceExtensionWithXmp(detailViewItem .FileIndexItem.FilePath)); } }
// Define the event handlers. /// <summary> /// Specify what is done when a file is changed. e.FullPath /// </summary> /// <param name="source"></param> /// <param name="e"></param> internal void OnChanged(object source, FileSystemEventArgs e) { if (e.FullPath.EndsWith(".tmp") || !ExtensionRolesHelper.IsExtensionSyncSupported(e.FullPath)) { return; } _webLogger.LogDebug($"[DiskWatcher] " + $"{e.FullPath} OnChanged ChangeType is: {e.ChangeType} " + DateTimeDebug()); _queueProcessor.QueueInput(e.FullPath, null, e.ChangeType); // Specify what is done when a file is changed, created, or deleted. }
/// <summary> /// This list will be included in the zip /// </summary> /// <param name="fileIndexResultsList">the items</param> /// <param name="thumbnail">add the thumbnail or the source image</param> /// <returns>list of file paths</returns> public async Task <List <string> > CreateListToExport(List <FileIndexItem> fileIndexResultsList, bool thumbnail) { var filePaths = new List <string>(); foreach (var item in fileIndexResultsList.Where(p => p.Status == FileIndexItem.ExifStatus.Ok).ToList()) { if (thumbnail) { var sourceThumb = Path.Combine(_appSettings.ThumbnailTempFolder, ThumbnailNameHelper.Combine(item.FileHash, ThumbnailSize.Large, true)); await new Thumbnail(_iStorage, _thumbnailStorage, _logger) .CreateThumb(item.FilePath, item.FileHash, true); filePaths.Add(sourceThumb); continue; } var sourceFile = _appSettings.DatabasePathToFilePath(item.FilePath, false); if (!_hostFileSystemStorage.ExistFile(sourceFile)) { continue; } // the jpeg file for example filePaths.Add(sourceFile); // when there is .xmp sidecar file (but only when file is a RAW file, ignored when for example jpeg) if (!ExtensionRolesHelper.IsExtensionForceXmp(item.FilePath) || !_iStorage.ExistFile( ExtensionRolesHelper.ReplaceExtensionWithXmp( item.FilePath))) { continue; } var xmpFileFullPath = _appSettings.DatabasePathToFilePath( ExtensionRolesHelper.ReplaceExtensionWithXmp( item.FilePath), false); if (!_hostFileSystemStorage.ExistFile(xmpFileFullPath)) { continue; } filePaths.Add(xmpFileFullPath); } return(filePaths); }
private static List <FileIndexItem> StackCollections(List <FileIndexItem> databaseSubFolderList) { // Get a list of duplicate items var stackItemsByFileCollectionName = databaseSubFolderList .GroupBy(item => item.FileCollectionName) .SelectMany(grp => grp.Skip(1).Take(1)).ToList(); // databaseSubFolderList.ToList() > Collection was modified; enumeration operation may not execute. // duplicateItemsByFilePath > // If you have 3 item with the same name it will include 1 name // So we do a linq query to search simalar items // We keep the first item // And Delete duplicate items var querySubFolderList = new List <FileIndexItem>(); // Do not remove it from: databaseSubFolderList otherwise it will be deleted from cache foreach (var stackItemByName in stackItemsByFileCollectionName) { var duplicateItems = databaseSubFolderList.Where(p => p.FileCollectionName == stackItemByName.FileCollectionName).ToList(); // The idea to pick thumbnail based images first, followed by non-thumb supported // when not pick alphabetaly > todo implement this for (int i = 0; i < duplicateItems.Count; i++) { if (ExtensionRolesHelper.IsExtensionThumbnailSupported(duplicateItems[i].FileName)) { querySubFolderList.Add(duplicateItems[i]); } } } // Then add the items that are non duplicate back to the list // ReSharper disable once ForeachCanBeConvertedToQueryUsingAnotherGetEnumerator foreach (var dbItem in databaseSubFolderList.ToList()) { // check if any item is duplicate if (stackItemsByFileCollectionName.All(p => p.FileCollectionName != dbItem.FileCollectionName)) { querySubFolderList.Add(dbItem); } } return(querySubFolderList.OrderBy(p => p.FileName).ToList()); }
public Task <List <ImportIndexItem> > Preflight(List <string> inputFileFullPaths, ImportSettingsModel importSettings) { var results = new List <ImportIndexItem>(); foreach (var inputFileFullPath in inputFileFullPaths) { // if the item fails var importIndexFileError = new ImportIndexItem { FilePath = "/" + FilenamesHelper.GetFileName(inputFileFullPath), SourceFullFilePath = "~/temp/test", FileHash = "FAKE", MakeModel = "added if the item fails", Status = ImportStatus.FileError }; // Check if extension is correct if (!ExtensionRolesHelper.IsExtensionSyncSupported(inputFileFullPath)) { results.Add(importIndexFileError); } // Check if the file is correct var imageFormat = ExtensionRolesHelper.GetImageFormat( _selectorStorage.Get(SelectorStorage.StorageServices.HostFilesystem) .ReadStream(inputFileFullPath, 160)); if (!ExtensionRolesHelper.ExtensionSyncSupportedList.Contains($"{imageFormat}")) { results.Add(importIndexFileError); } results.Add(new ImportIndexItem { Id = 4, SourceFullFilePath = inputFileFullPath, FilePath = inputFileFullPath, Status = ImportStatus.Ok, FileHash = "FAKE", MakeModel = "added okay", FileIndexItem = new FileIndexItem() { FileHash = "FAKE_OK", FilePath = inputFileFullPath } }); } PreflightList.AddRange(results); return(Task.FromResult(results)); }
internal bool ExistXmpSidecarForThisFileType(ImportIndexItem importIndexItem) { if (string.IsNullOrEmpty(importIndexItem.SourceFullFilePath)) { return(false); } // Support for include sidecar files var xmpSourceFullFilePath = ExtensionRolesHelper.ReplaceExtensionWithXmp(importIndexItem .SourceFullFilePath); return(ExtensionRolesHelper.IsExtensionForceXmp(importIndexItem .SourceFullFilePath) && _filesystemStorage.ExistFile(xmpSourceFullFilePath)); }
/// <summary> /// For Raw files us an external .xmp sidecar file, and add this to the PathsList /// </summary> /// <param name="inputSubPaths">list of files to update</param> /// <returns>list of files, where needed for raw-files there are .xmp used</returns> private List <string> PathsListTagsFromFile(List <string> inputSubPaths) { var pathsList = new List <string>(); foreach (var subPath in inputSubPaths) { if (ExtensionRolesHelper.IsExtensionForceXmp(subPath)) { var xmpPath = ExtensionRolesHelper.ReplaceExtensionWithXmp(subPath); pathsList.Add(xmpPath); continue; } pathsList.Add(subPath); } return(pathsList); }
[ProducesResponseType(404)] // not found public IActionResult DownloadSidecar(string f) { if (!ExtensionRolesHelper.IsExtensionSidecar(f)) { return(NotFound("FileName is not a sidecar")); } if (!_iStorage.ExistFile(f)) { return(NotFound($"source image missing {f}")); } var fs = _iStorage.ReadStream(f); return(File(fs, MimeHelper.GetMimeTypeByFileName(f))); }
/// <summary> /// this updates the main database item for a sidecar file /// </summary> /// <param name="xmpSubPath">sidecar file</param> /// <param name="directoryWithFileIndexItems">directory where the sidecar is located</param> /// <returns>completed task</returns> private async Task UpdateSidecarFile(string xmpSubPath, List <FileIndexItem> directoryWithFileIndexItems) { if (!ExtensionRolesHelper.IsExtensionSidecar(xmpSubPath)) { return; } var sidecarExt = FilenamesHelper.GetFileExtensionWithoutDot(xmpSubPath); foreach (var item in directoryWithFileIndexItems.Where(item => !item.SidecarExtensionsList.Contains(sidecarExt))) { item.AddSidecarExtension(sidecarExt); await _query.UpdateItemAsync(item); } }
/// <summary> /// Check if the image has the right first bytes, if not remove /// </summary> /// <param name="fileHash">the fileHash file</param> /// <param name="thumbnailToSourceSize">size of output thumbnail Large/ExtraLarge</param> internal bool RemoveCorruptImage(string fileHash, ThumbnailSize thumbnailToSourceSize) { if (!_thumbnailStorage.ExistFile(ThumbnailNameHelper.Combine(fileHash, thumbnailToSourceSize))) { return(false); } var imageFormat = ExtensionRolesHelper.GetImageFormat(_thumbnailStorage.ReadStream( ThumbnailNameHelper.Combine(fileHash, thumbnailToSourceSize), 160)); if (imageFormat != ExtensionRolesHelper.ImageFormat.unknown) { return(false); } _thumbnailStorage.FileDelete(ThumbnailNameHelper.Combine(fileHash, thumbnailToSourceSize)); return(true); }
/// <summary> /// Sidecar files don't have an own item, but there referenced by file items /// in the method xmp files are added to the AddSidecarExtension list. /// </summary> /// <param name="xmpSubPath">sidecar item</param> /// <returns>completed task</returns> public async Task UpdateSidecarFile(string xmpSubPath) { if (!ExtensionRolesHelper.IsExtensionSidecar(xmpSubPath)) { return; } var parentPath = FilenamesHelper.GetParentPath(xmpSubPath); var fileNameWithoutExtension = FilenamesHelper.GetFileNameWithoutExtension(xmpSubPath); var directoryWithFileIndexItems = (await _query.GetAllFilesAsync(parentPath)).Where( p => p.ParentDirectory == parentPath && p.FileCollectionName == fileNameWithoutExtension).ToList(); await UpdateSidecarFile(xmpSubPath, directoryWithFileIndexItems); }
/// <summary> /// Prepare an new item (no update in db) /// </summary> /// <param name="filePath">path of file</param> /// <param name="fileHash">optional could be null</param> /// <param name="parentDirectory">parent directory name</param> /// <param name="fileName">name without path</param> /// <returns></returns> public async Task <FileIndexItem> NewFileItem(string filePath, string fileHash, string parentDirectory, string fileName) { var updatedDatabaseItem = _readMeta.ReadExifAndXmpFromFile(filePath); updatedDatabaseItem.ImageFormat = ExtensionRolesHelper .GetImageFormat(_subPathStorage.ReadStream(filePath, 50)); // future: read json sidecar await SetFileHashStatus(filePath, fileHash, updatedDatabaseItem); updatedDatabaseItem.SetAddToDatabase(); updatedDatabaseItem.SetLastEdited(); updatedDatabaseItem.IsDirectory = false; updatedDatabaseItem.Size = _subPathStorage.Info(filePath).Size; updatedDatabaseItem.ParentDirectory = parentDirectory; updatedDatabaseItem.FileName = fileName; return(updatedDatabaseItem); }
[ProducesResponseType(404)] // not found public IActionResult ListSizesByHash(string f) { // For serving jpeg files f = FilenamesHelper.GetFileNameWithoutExtension(f); // Restrict the fileHash to letters and digits only // I/O function calls should not be vulnerable to path injection attacks if (!Regex.IsMatch(f, "^[a-zA-Z0-9_-]+$")) { return(BadRequest()); } var data = new ThumbnailSizesExistStatusModel { TinyMeta = _thumbnailStorage.ExistFile(ThumbnailNameHelper.Combine(f, ThumbnailSize.TinyMeta)), Small = _thumbnailStorage.ExistFile(ThumbnailNameHelper.Combine(f, ThumbnailSize.Small)), Large = _thumbnailStorage.ExistFile(ThumbnailNameHelper.Combine(f, ThumbnailSize.Large)), ExtraLarge = _thumbnailStorage.ExistFile(ThumbnailNameHelper.Combine(f, ThumbnailSize.ExtraLarge)) }; // Success has all items (except tinyMeta) if (data.Small && data.Large && data.ExtraLarge) { return(Json(data)); } var sourcePath = _query.GetSubPathByHash(f); var isThumbnailSupported = ExtensionRolesHelper.IsExtensionThumbnailSupported(sourcePath); switch (isThumbnailSupported) { case true when !string.IsNullOrEmpty(sourcePath): Response.StatusCode = 202; return(Json(data)); case false when !string.IsNullOrEmpty(sourcePath): Response.StatusCode = 210; // A conflict, that the thumb is not generated yet return(Json("Thumbnail is not supported; for example you try to view a raw or video file")); default: return(NotFound("not in index")); } }