public async Task <KeyValuePair <bool, string> > ReadString(string sourceHttpUrl)
{
Uri sourceUri = new Uri(sourceHttpUrl);
_logger.LogInformation("[ReadString] HttpClientHelper > " + sourceUri.Host + " ~ " + sourceHttpUrl);
// allow whitelist and https only
if (!_allowedDomains.Contains(sourceUri.Host) || sourceUri.Scheme != "https")
{
return
(new KeyValuePair <bool, string>(false, string.Empty));
}
try
{
using (HttpResponseMessage response = await _httpProvider.GetAsync(sourceHttpUrl))
using (Stream streamToReadFrom = await response.Content.ReadAsStreamAsync())
{
var reader = new StreamReader(streamToReadFrom, Encoding.UTF8);
var result = await reader.ReadToEndAsync();
return(new KeyValuePair <bool, string>(response.StatusCode == HttpStatusCode.OK, result));
}
}
catch (HttpRequestException exception)
{
return(new KeyValuePair <bool, string>(false, exception.Message));
}
}
public async Task <bool> DownloadExifTool(bool isWindows)
{
if (_appSettings.AddSwaggerExport == true && _appSettings.AddSwaggerExportExitAfter == true)
{
_logger.LogInformation("[DownloadExifTool] Skipped due AddSwaggerExportExitAfter setting");
return(false);
}
if (isWindows &&
!_hostFileSystemStorage.ExistFile(ExeExifToolWindowsFullFilePath()))
{
return(await StartDownloadForWindows());
}
if (!isWindows &&
!_hostFileSystemStorage.ExistFile(ExeExifToolUnixFullFilePath()))
{
return(await StartDownloadForUnix());
}
if (_appSettings.IsVerbose())
{
var debugPath = isWindows ? ExeExifToolWindowsFullFilePath()
: ExeExifToolUnixFullFilePath();
_logger.LogInformation($"[DownloadExifTool] {debugPath}");
}
// When running deploy scripts rights might reset (only for unix)
if (isWindows)
{
return(true);
}
return(await RunChmodOnExifToolUnixExe());
}
public async Task <List <string> > CleanAllUnusedFilesAsync(int chunkSize = 50)
{
if (!_thumbnailStorage.ExistFolder("/"))
{
throw new DirectoryNotFoundException("Thumbnail folder not found");
}
var allThumbnailFiles = _thumbnailStorage
.GetAllFilesInDirectory(null).ToList();
_logger.LogDebug($"Total files in thumb dir: {allThumbnailFiles.Count}");
var deletedFileHashes = new List <string>();
foreach (var fileNamesInChunk in allThumbnailFiles.ChunkyEnumerable(chunkSize))
{
var itemsInChunk = GetFileNamesWithExtension(fileNamesInChunk.ToList());
try
{
await LoopThoughChunk(itemsInChunk, deletedFileHashes);
}
catch (Microsoft.EntityFrameworkCore.Storage.RetryLimitExceededException exception)
{
_logger.LogInformation($"[CleanAllUnusedFiles] catch-ed and " +
$"skip {string.Join(",", itemsInChunk.ToList())} ~ {exception.Message}", exception);
}
}
return(deletedFileHashes);
}
public async Task <FileIndexItem.ExifStatus> ManualSync(string subPath,
string operationId = null)
{
var fileIndexItem = await _query.GetObjectByFilePathAsync(subPath);
// on a new database ->
if (subPath == "/" && fileIndexItem == null)
{
fileIndexItem = new FileIndexItem();
}
if (fileIndexItem == null)
{
_logger.LogInformation($"[ManualSync] NotFoundNotInIndex skip for: {subPath}");
return(FileIndexItem.ExifStatus.NotFoundNotInIndex);
}
if (_cache.TryGetValue(ManualSyncCacheName + subPath, out _))
{
// also used in removeCache
_query.RemoveCacheParentItem(subPath);
_logger.LogInformation($"[ManualSync] Cache hit skip for: {subPath}");
return(FileIndexItem.ExifStatus.OperationNotSupported);
}
_cache.Set(ManualSyncCacheName + subPath, true,
new TimeSpan(0, 1, 0));
_bgTaskQueue.QueueBackgroundWorkItem(async _ =>
{
await BackgroundTask(fileIndexItem.FilePath, operationId);
});
return(FileIndexItem.ExifStatus.Ok);
}
public bool FolderDelete(string path)
{
foreach (string directory in Directory.GetDirectories(path))
{
FolderDelete(directory);
}
try
{
Directory.Delete(path, true);
}
catch (IOException exception)
{
_logger?.LogInformation(exception,
"[FolderDelete] catch-ed IOException");
Directory.Delete(path, true);
}
catch (UnauthorizedAccessException exception)
{
_logger?.LogInformation(exception,
"[FolderDelete] catch-ed UnauthorizedAccessException");
Directory.Delete(path, true);
}
return(true);
}
internal (ExifThumbnailDirectory, int, int, FileIndexItem.Rotation) ParseMetaThumbnail(List <Directory> allExifItems,
ExifThumbnailDirectory exifThumbnailDir, string reference = null)
{
if (exifThumbnailDir == null)
{
return(null, 0, 0, FileIndexItem.Rotation.DoNotChange);
}
var jpegTags = allExifItems.FirstOrDefault(p =>
p.Name == "JPEG")?.Tags;
var rotation = ReadMetaExif.GetOrientationFromExifItem(
allExifItems.FirstOrDefault(p => p.Name == "Exif IFD0"));
int.TryParse(
jpegTags?.FirstOrDefault(p => p.Name == "Image Height")?
.Description.Replace(" pixels", string.Empty), out var height);
int.TryParse(
jpegTags?.FirstOrDefault(p => p.Name == "Image Width")?
.Description.Replace(" pixels", string.Empty), out var width);
if (height == 0 || width == 0)
{
_logger.LogInformation($"[] ${reference} has no height or width {width}x{height} ");
}
return(exifThumbnailDir, width, height, rotation);
}
/// <summary>
/// @see: https://docs.microsoft.com/en-us/dotnet/api/system.io.filesystemwatcher?view=netcore-3.1
/// </summary>
public void Watcher(string fullFilePath)
{
_webLogger.LogInformation($"[DiskWatcher] started {fullFilePath}" +
$"{DateTimeDebug()}");
// Create a new FileSystemWatcher and set its properties.
_fileSystemWatcherWrapper.Path = fullFilePath;
_fileSystemWatcherWrapper.Filter = "*";
_fileSystemWatcherWrapper.IncludeSubdirectories = true;
_fileSystemWatcherWrapper.NotifyFilter = NotifyFilters.FileName
| NotifyFilters.DirectoryName
| NotifyFilters.Attributes
| NotifyFilters.Size
| NotifyFilters.LastWrite
// NotifyFilters.LastAccess is removed
| NotifyFilters.CreationTime
| NotifyFilters.Security;
// Watch for changes in LastAccess and LastWrite times, and
// the renaming of files or directories.
// Add event handlers.
_fileSystemWatcherWrapper.Created += OnChanged;
_fileSystemWatcherWrapper.Changed += OnChanged;
_fileSystemWatcherWrapper.Deleted += OnChanged;
_fileSystemWatcherWrapper.Renamed += OnRenamed;
_fileSystemWatcherWrapper.Error += OnError;
// Begin watching.
_fileSystemWatcherWrapper.EnableRaisingEvents = true;
}
[ProducesResponseType(typeof(string), 200)] // "Not found"
public IActionResult GeoSyncFolder(
string f = "/",
bool index = true,
bool overwriteLocationNames = false
)
{
if (_iStorage.IsFolderOrFile(f) == FolderOrFileModel.FolderOrFileTypeList.Deleted)
{
return(NotFound("Folder location is not found"));
}
var operationId = HttpContext.GetOperationId();
_bgTaskQueue.QueueBackgroundWorkItem(async token =>
{
_logger.LogInformation($"{nameof(GeoSyncFolder)} started {f} {DateTime.UtcNow.ToShortTimeString()}");
var operationHolder = RequestTelemetryHelper.GetOperationHolder(_serviceScopeFactory,
nameof(GeoSyncFolder), operationId);
var geoBackgroundTask = _serviceScopeFactory.CreateScope().ServiceProvider
.GetService <IGeoBackgroundTask>();
var result = await geoBackgroundTask.GeoBackgroundTaskAsync(f, index,
overwriteLocationNames);
operationHolder.SetData(_serviceScopeFactory, result);
_logger.LogInformation($"{nameof(GeoSyncFolder)} end {f} {operationHolder.Telemetry?.Duration}");
});
return(Json("job started"));
}
internal async Task <List <string> > AddParentCacheIfNotExistAsync(IEnumerable <string> updatedPaths)
{
var parentDirectoryList = new HashSet <string>();
foreach (var path in updatedPaths)
{
parentDirectoryList.Add(FilenamesHelper.GetParentPath(path));
}
var shouldAddParentDirectoriesToCache = parentDirectoryList.Where(parentDirectory =>
!_query.CacheGetParentFolder(parentDirectory).Item1).ToList();
if (!shouldAddParentDirectoriesToCache.Any())
{
return(new List <string>());
}
var databaseQueryResult = await _query.GetAllObjectsAsync(shouldAddParentDirectoriesToCache);
_logger.LogInformation("[AddParentCacheIfNotExist] files added to cache " +
string.Join(",", shouldAddParentDirectoriesToCache));
foreach (var directory in shouldAddParentDirectoriesToCache)
{
var byDirectory = databaseQueryResult.Where(p => p.ParentDirectory == directory).ToList();
_query.AddCacheParentItem(directory, byDirectory);
}
return(shouldAddParentDirectoriesToCache);
}
/// <summary>
/// Export Values to Storage
/// </summary>
/// <param name="appSettings">App Settings</param>
/// <param name="selectorStorage">Storage Provider</param>
/// <param name="swaggerProvider">Swagger</param>
/// <exception cref="ArgumentNullException">swaggerJsonText = null</exception>
public bool Add03AppExport(AppSettings appSettings, ISelectorStorage selectorStorage, ISwaggerProvider swaggerProvider)
{
if (appSettings.AddSwagger != true || appSettings.AddSwaggerExport != true)
{
return(false);
}
var swaggerJsonText = GenerateSwagger(swaggerProvider, appSettings.Name);
if (string.IsNullOrEmpty(swaggerJsonText))
{
throw new ArgumentException("swaggerJsonText = null", nameof(swaggerProvider));
}
var swaggerJsonFullPath =
Path.Join(appSettings.TempFolder, appSettings.Name.ToLowerInvariant() + ".json");
var storage = selectorStorage.Get(SelectorStorage.StorageServices.HostFilesystem);
storage.FileDelete(swaggerJsonFullPath);
storage.WriteStream(new PlainTextFileHelper().StringToStream(swaggerJsonText),
swaggerJsonFullPath);
_logger?.LogInformation($"app__addSwaggerExport {swaggerJsonFullPath}");
return(true);
}
/// <summary>
/// Returns a database object file or folder
/// </summary>
/// <param name="filePath">relative database path</param>
/// <param name="cacheTime">time to have the cache present</param>
/// <returns>FileIndex-objects with database data</returns>
public async Task <FileIndexItem> GetObjectByFilePathAsync(
string filePath, TimeSpan?cacheTime = null)
{
// cache code:
if (cacheTime != null &&
_appSettings?.AddMemoryCache == true &&
_cache != null &&
_cache.TryGetValue(
GetObjectByFilePathAsyncCacheName(filePath), out var data))
{
_logger.LogInformation("Get from cache " + GetObjectByFilePathAsyncCacheName(filePath));
if (!(data is FileIndexItem fileIndexItem))
{
return(null);
}
fileIndexItem.Status = FileIndexItem.ExifStatus.OkAndSame;
return(fileIndexItem);
}
// end cache
var result = (await GetObjectByFilePathQueryAsync(filePath));
// cache code:
if (cacheTime == null || _appSettings?.AddMemoryCache != true || result == null)
{
return(result);
}
SetGetObjectByFilePathCache(filePath, result.Clone(), cacheTime);
return(result);
}
public void StopUpdateReplaceStopWatch(string name, string f, bool collections, Stopwatch stopwatch)
{
// for debug
stopwatch.Stop();
_logger.LogInformation($"[{name}] f: {f} Stopwatch response collections: " +
$"{collections} {DateTime.UtcNow} duration: {stopwatch.Elapsed.TotalMilliseconds} ms or:" +
$" {stopwatch.Elapsed.TotalSeconds} sec");
}
/// <summary>
/// For Checking single items without querying the database
/// </summary>
/// <param name="subPath">path</param>
/// <param name="dbItem">current item, can be null</param>
/// <param name="updateDelegate">push updates realtime to the user and avoid waiting</param>
/// <returns>updated item with status</returns>
internal async Task <FileIndexItem> SingleFile(string subPath,
FileIndexItem dbItem,
ISynchronize.SocketUpdateDelegate updateDelegate = null)
{
// when item does not exist in db
if (dbItem == null)
{
return(await SingleFile(subPath));
}
// Route without database check
if (_appSettings.ApplicationType == AppSettings.StarskyAppType.WebController)
{
_logger.LogInformation($"[SingleFile/no-db] {subPath} - {DateTime.UtcNow.ToShortTimeString()}");
}
// Sidecar files are updated but ignored by the process
await UpdateSidecarFile(subPath);
var statusItem = CheckForStatusNotOk(subPath);
if (statusItem.Status != FileIndexItem.ExifStatus.Ok)
{
_logger.LogInformation($"[SingleFile/no-db] status {statusItem.Status} for {subPath}");
return(statusItem);
}
var(isSame, updatedDbItem) = await SizeFileHashIsTheSame(dbItem);
if (!isSame)
{
if (updateDelegate != null)
{
new Thread(() => updateDelegate(new List <FileIndexItem> {
dbItem
})).Start();
}
return(await UpdateItem(dbItem, updatedDbItem.Size, subPath));
}
// to avoid reSync
updatedDbItem.Status = FileIndexItem.ExifStatus.OkAndSame;
AddDeleteStatus(statusItem, FileIndexItem.ExifStatus.DeletedAndSame);
return(updatedDbItem);
}
internal TelemetryClient GetTelemetryClient()
{
var client = _serviceProvider != null?_serviceProvider.GetService <TelemetryClient>() : _app.ApplicationServices.GetService <TelemetryClient>();
if (client == null && _appSettings != null && !string.IsNullOrEmpty(_appSettings.ApplicationInsightsInstrumentationKey))
{
_logger.LogInformation("TelemetryClient is null on exit");
}
return(client);
}
/// <summary>
/// Update ExifTool, Thumbnail, Database and if needed rotateClock
/// </summary>
/// <param name="fileIndexItem">output database object</param>
/// <param name="comparedNamesList">name of fields updated by exifTool</param>
/// <param name="rotateClock">rotation value (if needed)</param>
private async Task UpdateWriteDiskDatabase(FileIndexItem fileIndexItem, List <string> comparedNamesList, int rotateClock = 0)
{
// do rotation on thumbs
await RotationThumbnailExecute(rotateClock, fileIndexItem);
if (fileIndexItem.IsDirectory != true &&
ExtensionRolesHelper.IsExtensionExifToolSupported(fileIndexItem.FileName))
{
// feature to exif update
var exifUpdateFilePaths = new List <string>
{
fileIndexItem.FilePath
};
var exifTool = new ExifToolCmdHelper(_exifTool, _iStorage, _thumbnailStorage, _readMeta);
// to avoid diskWatcher catch up
_query.SetGetObjectByFilePathCache(fileIndexItem.FilePath, fileIndexItem, TimeSpan.FromSeconds(10));
// Do an Exif Sync for all files, including thumbnails
var(exifResult, newFileHashes) = await exifTool.UpdateAsync(fileIndexItem,
exifUpdateFilePaths, comparedNamesList, true, true);
await ApplyOrGenerateUpdatedFileHash(newFileHashes, fileIndexItem);
_logger.LogInformation(string.IsNullOrEmpty(exifResult)
? $"[UpdateWriteDiskDatabase] ExifTool result is Nothing or " +
$"Null for: path:{fileIndexItem.FilePath} {DateTime.UtcNow.ToShortTimeString()}"
: $"[UpdateWriteDiskDatabase] ExifTool result: {exifResult} path:{fileIndexItem.FilePath}");
}
else
{
await new FileIndexItemJsonParser(_iStorage).WriteAsync(fileIndexItem);
}
// Do a database sync + cache sync
// Clone to avoid reference when cache exist
await _query.UpdateItemAsync(fileIndexItem.Clone());
// > async > force you to read the file again
// do not include thumbs in MetaCache
// only the full path url of the source image
_readMeta.RemoveReadMetaCache(fileIndexItem.FilePath);
}
public static TelemetryClient?InitTelemetryClient(string appInsightsConnectionString, string roleName, IWebLogger?logger, TelemetryClient?telemetryClient)
{
TelemetryClient?Clean(Exception exception)
{
logger?.LogInformation($"catch-ed exception; {exception.Message} ", exception);
logger?.LogInformation("run GC.Collect next -->");
GC.Collect();
return(null);
}
try
{
// Should skip to avoid memory issues
if (telemetryClient == null)
{
var telemetryConfiguration =
CreateTelemetryConfiguration(appInsightsConnectionString);
if (telemetryConfiguration == null)
{
return(null);
}
telemetryClient =
new TelemetryClient(telemetryConfiguration);
telemetryClient.Context.Cloud.RoleName = roleName;
telemetryClient.Context.Cloud.RoleInstance =
Environment.MachineName;
logger?.LogInformation("Added TelemetryClient [should avoid due memory issues]");
}
var module = CreateDatabaseDependencyTrackingTelemetryModule();
module.Initialize(telemetryClient.TelemetryConfiguration);
return(telemetryClient);
}
catch (OutOfMemoryException exception)
{
return(Clean(exception));
}
catch (System.Threading.Tasks.TaskSchedulerException exception)
{
return(Clean(exception));
}
}
/// <summary>
/// Run Transformation on Import to the files in the database && Update fileHash in database
/// </summary>
/// <param name="queryUpdateDelegate"></param>
/// <param name="fileIndexItem">information</param>
/// <param name="colorClassTransformation">change colorClass</param>
/// <param name="dateTimeParsedFromFileName">is date time parsed from fileName</param>
/// <param name="indexMode">should update database</param>
internal async Task <FileIndexItem> UpdateTransformations(QueryUpdateDelegate queryUpdateDelegate, FileIndexItem fileIndexItem,
int colorClassTransformation, bool dateTimeParsedFromFileName, bool indexMode)
{
if (!ExtensionRolesHelper.IsExtensionExifToolSupported(fileIndexItem.FileName))
{
return(fileIndexItem);
}
var comparedNamesList = new List <string>();
if (dateTimeParsedFromFileName)
{
_logger.LogInformation($"[Import] DateTimeParsedFromFileName ExifTool Sync {fileIndexItem.FilePath}");
comparedNamesList = DateTimeParsedComparedNamesList();
}
if (colorClassTransformation >= 0)
{
_logger.LogInformation($"[Import] ColorClassComparedNamesList ExifTool Sync {fileIndexItem.FilePath}");
comparedNamesList = ColorClassComparedNamesList(comparedNamesList);
}
if (!comparedNamesList.Any())
{
return(fileIndexItem);
}
await new ExifToolCmdHelper(_exifTool, _subPathStorage, _thumbnailStorage,
new ReadMeta(_subPathStorage, _appSettings)).UpdateAsync(fileIndexItem, comparedNamesList);
// Only update database when indexMode is true
if (!indexMode || queryUpdateDelegate == null)
{
return(fileIndexItem);
}
// Hash is changed after transformation
fileIndexItem.FileHash = (await new FileHash(_subPathStorage).GetHashCodeAsync(fileIndexItem.FilePath)).Key;
await queryUpdateDelegate(fileIndexItem);
return(fileIndexItem.Clone());
}
public static async Task ProcessTaskQueueAsync(IBaseBackgroundTaskQueue taskQueue, IWebLogger logger, CancellationToken stoppingToken)
{
logger.LogInformation($"Queued Hosted Service {taskQueue.GetType().Name} is " +
$"starting on {Environment.MachineName}");
while (!stoppingToken.IsCancellationRequested)
{
var workItem = await taskQueue.DequeueAsync(stoppingToken);
try
{
await workItem(stoppingToken);
}
catch (Exception exception)
{
logger.LogError(exception,
$"Error occurred executing workItem ", nameof(workItem));
}
}
logger.LogInformation("Queued Hosted Service has stopped");
}
internal async Task WorkItem(string subPath, IStorage subPathStorage,
IStorage thumbnailStorage)
{
try
{
_logger.LogInformation($"[ThumbnailGenerationController] start {subPath}");
var thumbnail = new Thumbnail(subPathStorage,
thumbnailStorage, _logger);
var thumbs = await thumbnail.CreateThumb(subPath);
var getAllFilesAsync = await _query.GetAllFilesAsync(subPath);
var result = new List <FileIndexItem>();
foreach (var item in
getAllFilesAsync.Where(item => thumbs.FirstOrDefault(p => p.Item1 == item.FilePath).Item2))
{
if (item.Tags.Contains("!delete!"))
{
continue;
}
item.SetLastEdited();
result.Add(item);
}
if (!result.Any())
{
return;
}
var webSocketResponse =
new ApiNotificationResponseModel <List <FileIndexItem> >(result, ApiNotificationType.ThumbnailGeneration);
await _connectionsService.SendToAllAsync(webSocketResponse, CancellationToken.None);
_logger.LogInformation($"[ThumbnailGenerationController] done {subPath}");
}
catch (UnauthorizedAccessException e)
{
_logger.LogError($"[ThumbnailGenerationController] catch-ed exception {e.Message}", e);
}
}
public async Task <bool> WriteAndCropFile(string fileHash,
OffsetModel offsetData, int sourceWidth,
int sourceHeight, FileIndexItem.Rotation rotation,
string reference = null)
{
try
{
using (var thumbnailStream = new MemoryStream(offsetData.Data, offsetData.Index, offsetData.Count))
using (var smallImage = await Image.LoadAsync(thumbnailStream))
using (var outputStream = new MemoryStream())
{
var smallImageWidth = smallImage.Width;
var smallImageHeight = smallImage.Height;
var result = NewImageSize.NewImageSizeCalc(smallImageWidth,
smallImageHeight, sourceWidth, sourceHeight);
smallImage.Mutate(
i => i.Resize(smallImageWidth, smallImageHeight, KnownResamplers.Lanczos3)
.Crop(new Rectangle(result.DestX, result.DestY, result.DestWidth, result.DestHeight)));
var larger = (int)Math.Round(result.DestWidth * 1.2, 0);
smallImage.Mutate(
i => i.Resize(larger, 0, KnownResamplers.Lanczos3));
var rotate = RotateEnumToDegrees(rotation);
smallImage.Mutate(
i => i.Rotate(rotate));
await smallImage.SaveAsJpegAsync(outputStream);
await _thumbnailStorage.WriteStreamAsync(outputStream, ThumbnailNameHelper.Combine(fileHash, ThumbnailSize.TinyMeta));
if (_appSettings.ApplicationType == AppSettings.StarskyAppType.WebController)
{
_logger.LogInformation($"[WriteAndCropFile] fileHash: {fileHash} is written");
}
}
return(true);
}
catch (Exception ex)
{
var message = ex.Message;
if (message.StartsWith("Image cannot be loaded"))
{
message = "Image cannot be loaded";
}
_logger.LogError($"[WriteFile@meta] Exception {reference} {message}", ex);
return(false);
}
}
/// <summary>
/// This method is triggered by BackgroundService
/// </summary>
/// <param name="stoppingToken">ignored but required</param>
/// <returns>Task/nothing</returns>
protected override Task ExecuteAsync(CancellationToken stoppingToken)
{
_logger.LogInformation((_appSettings.UseDiskWatcher != false ? $"UseDiskWatcher is enabled"
: "UseDiskWatcher is disabled") + $" on {Environment.MachineName}");
if (_appSettings.UseDiskWatcher == false &&
_appSettings.ApplicationType == AppSettings.StarskyAppType.WebController)
{
return(Task.CompletedTask);
}
_diskWatcher.Watcher(_appSettings.StorageFolder);
return(Task.CompletedTask);
}
internal async Task <List <ImportIndexItem> > ImportPostBackgroundTask(List <string> tempImportPaths,
ImportSettingsModel importSettings, bool isVerbose = false)
{
List <ImportIndexItem> importedFiles;
using (var scope = _scopeFactory.CreateScope())
{
var selectorStorage = scope.ServiceProvider.GetRequiredService <ISelectorStorage>();
var importQuery = scope.ServiceProvider.GetRequiredService <IImportQuery>();
var exifTool = scope.ServiceProvider.GetRequiredService <IExifTool>();
var query = scope.ServiceProvider.GetRequiredService <IQuery>();
var console = scope.ServiceProvider.GetRequiredService <IConsole>();
var metaExifThumbnailService = scope.ServiceProvider.GetRequiredService <IMetaExifThumbnailService>();
var memoryCache = scope.ServiceProvider.GetRequiredService <IMemoryCache>();
// use of IImport direct does not work
importedFiles = await new Import(selectorStorage, _appSettings,
importQuery, exifTool, query, console,
metaExifThumbnailService, _logger, memoryCache).Importer(tempImportPaths, importSettings);
}
if (isVerbose)
{
foreach (var file in importedFiles)
{
_logger.LogInformation(
$"[ImportPostBackgroundTask] import {file.Status} " +
$"=> {file.FilePath} ~ {file.FileIndexItem?.FilePath}");
}
}
// Remove source files
foreach (var toDelPath in tempImportPaths)
{
RemoveTempAndParentStreamFolder(toDelPath);
}
return(importedFiles);
}
public IActionResult PublishCreate(string f, string itemName,
string publishProfileName, bool force = false)
{
_webLogger.LogInformation($"[/api/publish/create] Press publish: {itemName} {f} {DateTime.UtcNow}");
var inputFilePaths = PathHelper.SplitInputFilePaths(f).ToList();
var info = _metaInfo.GetInfo(inputFilePaths, false);
if (info.All(p => p.Status != FileIndexItem.ExifStatus.Ok && p.Status != FileIndexItem.ExifStatus.ReadOnly))
{
return(NotFound(info));
}
var slugItemName = _appSettings.GenerateSlug(itemName, true);
var location = Path.Combine(_appSettings.TempFolder, slugItemName);
if (CheckIfNameExist(slugItemName))
{
if (!force)
{
return(Conflict($"name {slugItemName} exist"));
}
ForceCleanPublishFolderAndZip(location);
}
// todo: check if overlay image path: WebHtmlPublish/EmbeddedViews/default.png or something else exists
// Creating Publish is a background task
_bgTaskQueue.QueueBackgroundWorkItem(async token =>
{
var renderCopyResult = await _publishService.RenderCopy(info,
publishProfileName, itemName, location);
await _publishService.GenerateZip(_appSettings.TempFolder, itemName,
renderCopyResult, true);
_webLogger.LogInformation($"[/api/publish/create] done: {itemName} {DateTime.UtcNow}");
});
// Get the zip by [HttpGet("/export/zip/{f}.zip")]
return(Json(slugItemName));
}
public async Task <IActionResult> UpdateAsync(FileIndexItem inputModel, string f, bool append,
bool collections = true, int rotateClock = 0)
{
var stopwatch = StopWatchLogger.StartUpdateReplaceStopWatch();
var inputFilePaths = PathHelper.SplitInputFilePaths(f);
var(fileIndexResultsList, changedFileIndexItemName) = await _metaPreflight.Preflight(inputModel,
inputFilePaths, append, collections, rotateClock);
var operationId = HttpContext.GetOperationId();
// Update >
_bgTaskQueue.QueueBackgroundWorkItem(async _ =>
{
var operationHolder = RequestTelemetryHelper.GetOperationHolder(_scopeFactory,
nameof(UpdateAsync), operationId);
var metaUpdateService = _scopeFactory.CreateScope()
.ServiceProvider.GetRequiredService <IMetaUpdateService>();
var data = await metaUpdateService.UpdateAsync(
changedFileIndexItemName, fileIndexResultsList, null,
collections, append, rotateClock);
operationHolder.SetData(_scopeFactory, data);
});
// before sending not founds
new StopWatchLogger(_logger).StopUpdateReplaceStopWatch("update", f, collections, stopwatch);
// When all items are not found
if (fileIndexResultsList.All(p => p.Status != FileIndexItem.ExifStatus.Ok &&
p.Status != FileIndexItem.ExifStatus.Deleted))
{
return(NotFound(fileIndexResultsList));
}
// Clone an new item in the list to display
var returnNewResultList = fileIndexResultsList.Select(item => item.Clone()).ToList();
// when switching very fast between images the background task has not run yet
_metaUpdateService.UpdateReadMetaCache(returnNewResultList);
// Push direct to socket when update or replace to avoid undo after a second
_logger.LogInformation($"[UpdateController] send to socket {f}");
await Task.Run(async() => await TaskRun(fileIndexResultsList));
return(Json(returnNewResultList));
}
public static async Task Run(ApplicationDbContext dbContext, IWebLogger logger, AppSettings appSettings, int retryCount = 2)
{
async Task <bool> Migrate()
{
if (appSettings.DatabaseType == AppSettings.DatabaseTypeList.InMemoryDatabase)
{
return(true);
}
await dbContext.Database.MigrateAsync();
if (appSettings.DatabaseType !=
AppSettings.DatabaseTypeList.Mysql)
{
return(true);
}
var connection = new MySqlConnection(appSettings.DatabaseConnection);
var databaseFixes =
new MySqlDatabaseFixes(connection, appSettings);
await databaseFixes.OpenConnection();
var tableNames = dbContext.Model.GetEntityTypes()
.Select(t => t.GetTableName())
.Distinct()
.ToList();
await databaseFixes.FixUtf8Encoding(tableNames);
await databaseFixes.FixAutoIncrement("Notifications");
return(true);
}
try
{
await RetryHelper.DoAsync(Migrate, TimeSpan.FromSeconds(2), retryCount);
}
catch (AggregateException exception)
{
logger.LogInformation("[RunMigrations] start migration failed");
logger.LogError(exception.Message);
logger.LogError(exception.InnerException?.Message);
logger.LogError("[RunMigrations] end catch-ed");
}
}
public Task SendToAllAsync(string message, CancellationToken cancellationToken)
{
List<Task> connectionsTasks = new List<Task>();
foreach (WebSocketConnection connection in _connections.Values)
{
try
{
connectionsTasks.Add(connection.SendAsync(message, cancellationToken));
}
catch ( WebSocketException exception)
{
// if the client is closing the socket the wrong way
_logger.LogInformation(exception, "catch-ed exception socket");
}
}
return Task.WhenAll(connectionsTasks);
}
/// <summary>
/// Test the connection if this is mysql
/// </summary>
/// <param name="context">database context</param>
/// <param name="logger">logger</param>
/// <returns>bool, true if connection is there</returns>
/// <exception cref="ArgumentNullException">When AppSettings is null</exception>
public static bool TestConnection(this DbContext context, IWebLogger logger)
{
if (context?.Database == null)
{
return(false);
}
try
{
context.Database.CanConnect();
}
catch (MySqlException e)
{
logger.LogInformation($"[TestConnection] WARNING >>> \n{e}\n <<<");
return(false);
}
return(true);
}
/// <summary>
/// Based on the preflight create a Zip Export
/// </summary>
/// <param name="fileIndexResultsList">Result of Preflight</param>
/// <param name="thumbnail">isThumbnail?</param>
/// <param name="zipOutputFileName">filename of zip file (no extension)</param>
/// <returns>nothing</returns>
public async Task CreateZip(List <FileIndexItem> fileIndexResultsList, bool thumbnail,
string zipOutputFileName)
{
var filePaths = await CreateListToExport(fileIndexResultsList, thumbnail);
var fileNames = FilePathToFileName(filePaths, thumbnail);
new Zipper().CreateZip(_appSettings.TempFolder, filePaths, fileNames, zipOutputFileName);
// Write a single file to be sure that writing is ready
var doneFileFullPath = Path.Combine(_appSettings.TempFolder, zipOutputFileName) + ".done";
await _hostFileSystemStorage.
WriteStreamAsync(new PlainTextFileHelper().StringToStream("OK"), doneFileFullPath);
if (_appSettings.IsVerbose())
{
_logger.LogInformation("[CreateZip] Zip done: " + doneFileFullPath);
}
}
public void BuilderDb(string?foundationDatabaseName = "")
{
if (_services == null)
{
throw new AggregateException("services is missing");
}
if (_logger != null && _appSettings.IsVerbose())
{
_logger.LogInformation($"Database connection: {_appSettings.DatabaseConnection}");
_logger?.LogInformation($"Application Insights Database tracking is {IsDatabaseTrackingEnabled()}");
}
#if ENABLE_DEFAULT_DATABASE
// dirty hack
_services.AddDbContext <ApplicationDbContext>(options =>
options.UseSqlite(_appSettings.DatabaseConnection,
b =>
{
if (!string.IsNullOrWhiteSpace(foundationDatabaseName))
{
b.MigrationsAssembly(foundationDatabaseName);
}
}));
#endif
#if ENABLE_MYSQL_DATABASE
// dirty hack
_services.AddDbContext <ApplicationDbContext>(options =>
options.UseMySql(_appSettings.DatabaseConnection, GetServerVersionMySql(),
b =>
{
if (!string.IsNullOrWhiteSpace(foundationDatabaseName))
{
b.MigrationsAssembly(foundationDatabaseName);
}
}));
#endif
_services.AddScoped(provider => new ApplicationDbContext(BuilderDbFactorySwitch(foundationDatabaseName)));
}
public async Task <NotificationItem> AddNotification(string content)
{
var item = new NotificationItem
{
DateTime = DateTime.UtcNow,
Content = content
};
try
{
await _context.Notifications.AddAsync(item);
await _context.SaveChangesAsync();
}
catch (DbUpdateConcurrencyException concurrencyException)
{
_logger.LogInformation("[AddNotification] try to fix DbUpdateConcurrencyException", concurrencyException);
SolveConcurrency.SolveConcurrencyExceptionLoop(concurrencyException.Entries);
await _context.SaveChangesAsync();
}
return(item);
}
