private void OnError(object source, ErrorEventArgs e) { // Show that an error has been detected. _webLogger.LogError(e.GetException(), "[DiskWatcher] The FileSystemWatcher has an error (catch-ed) - next: retry " + $"{DateTimeDebug()}"); _webLogger.LogError("[DiskWatcher] (catch-ed) " + e.GetException().Message); // Give more information if the error is due to an internal buffer overflow. if (e.GetException().GetType() == typeof(InternalBufferOverflowException)) { // This can happen if Windows is reporting many file system events quickly // and internal buffer of the FileSystemWatcher is not large enough to handle this // rate of events. The InternalBufferOverflowException error informs the application // that some of the file system events are being lost. _webLogger.LogError(e.GetException(), "[DiskWatcher] The file system watcher experienced an internal buffer overflow "); } // when test dont retry if (e.GetException().Message == "test") { return; } // When fail it should try it again Retry(new BufferingFileSystemWatcher(_fileSystemWatcherWrapper.Path)); }
/// <summary> /// Get All files from the directory /// </summary> /// <param name="path">fullFilePath</param> /// <returns></returns> public IEnumerable <string> GetAllFilesInDirectory(string path) { string[] allFiles; try { allFiles = Directory.GetFiles(path); } catch (UnauthorizedAccessException e) { _logger?.LogError(e, "[GetAllFilesInDirectory] catch-ed UnauthorizedAccessException"); return(new string[] {}); } var imageFilesList = new List <string>(); foreach (var file in allFiles) { // Path.GetExtension uses (.ext) // the same check in SingleFile // Recruisive >= same check // ignore Files with ._ names, this is Mac OS specific var isAppleDouble = Path.GetFileName(file).StartsWith("._"); if (!isAppleDouble) { imageFilesList.Add(file); } // to filter use: // ..etAllFilesInDirectory(subPath) // .Where(ExtensionRolesHelper.IsExtensionExifToolSupported) // OR: // .Where(ExtensionRolesHelper.IsExtensionSyncSupported } return(imageFilesList); }
/// <summary> /// Generate loop of Jpeg images with overlay image /// With Retry included /// </summary> /// <param name="profile">contains sizes</param> /// <param name="fileIndexItemsList">list of items to generate jpeg for</param> /// <param name="outputParentFullFilePathFolder">outputParentFullFilePathFolder</param> /// <param name="delay">when failed output, has default value</param> /// <returns></returns> internal async Task <Dictionary <string, bool> > GenerateJpeg(AppSettingsPublishProfiles profile, IReadOnlyCollection <FileIndexItem> fileIndexItemsList, string outputParentFullFilePathFolder, int delay = 6) { _toCreateSubfolder.Create(profile, outputParentFullFilePathFolder); foreach (var item in fileIndexItemsList) { var outputPath = _overlayImage.FilePathOverlayImage(outputParentFullFilePathFolder, item.FilePath, profile); async Task <bool> ResizerLocal() { return(await Resizer(outputPath, profile, item)); } try { await RetryHelper.DoAsync(ResizerLocal, TimeSpan.FromSeconds(delay)); } catch (AggregateException e) { _logger.LogError($"[ResizerLocal] Skip due errors: (catch-ed exception) {item.FilePath} {item.FileHash}"); foreach (var exception in e.InnerExceptions) { _logger.LogError("[ResizerLocal] " + exception.Message, exception); } } } return(fileIndexItemsList.ToDictionary(item => _overlayImage.FilePathOverlayImage(item.FilePath, profile), item => profile.Copy)); }
/// <summary> /// Used to fill the cache with an array of /// All keywords are stored lowercase /// </summary> /// <returns></returns> public async Task <List <KeyValuePair <string, int> > > Inflate() { if (_cache.TryGetValue(nameof(SearchSuggestionsService), out _)) { return(new Dictionary <string, int>().ToList()); } var allFilesList = new List <KeyValuePair <string, int> >(); try { allFilesList = await _context.FileIndex.GroupBy(i => i.Tags) // ReSharper disable once UseMethodAny.1 .Where(x => x.Count() >= 1) // .ANY is not supported by EF Core .TagWith("Inflate SearchSuggestionsService") .Select(val => new KeyValuePair <string, int>(val.Key, val.Count())).ToListAsync(); } catch (Exception exception) { if (!exception.Message.Contains("Unknown column")) { _logger.LogError(exception, "mysql search suggest exception catch-ed"); } return(allFilesList); } var suggestions = new Dictionary <string, int>(StringComparer.InvariantCultureIgnoreCase); foreach (var tag in allFilesList) { if (string.IsNullOrEmpty(tag.Key)) { continue; } var keywordsHashSet = HashSetHelper.StringToHashSet(tag.Key.Trim()); foreach (var keyword in keywordsHashSet) { if (suggestions.ContainsKey(keyword)) { suggestions[keyword] += tag.Value; } else { suggestions.Add(keyword, tag.Value); } } } var suggestionsFiltered = suggestions .Where(p => p.Value >= 10) .OrderByDescending(p => p.Value) .ToList(); _cache.Set(nameof(SearchSuggestionsService), suggestionsFiltered, new TimeSpan(100, 0, 0)); return(suggestionsFiltered); }
public static async Task Run(ApplicationDbContext dbContext, IWebLogger logger, AppSettings appSettings, int retryCount = 2) { async Task <bool> Migrate() { if (appSettings.DatabaseType == AppSettings.DatabaseTypeList.InMemoryDatabase) { return(true); } await dbContext.Database.MigrateAsync(); if (appSettings.DatabaseType != AppSettings.DatabaseTypeList.Mysql) { return(true); } var connection = new MySqlConnection(appSettings.DatabaseConnection); var databaseFixes = new MySqlDatabaseFixes(connection, appSettings); await databaseFixes.OpenConnection(); var tableNames = dbContext.Model.GetEntityTypes() .Select(t => t.GetTableName()) .Distinct() .ToList(); await databaseFixes.FixUtf8Encoding(tableNames); await databaseFixes.FixAutoIncrement("Notifications"); return(true); } try { await RetryHelper.DoAsync(Migrate, TimeSpan.FromSeconds(2), retryCount); } catch (AggregateException exception) { logger.LogInformation("[RunMigrations] start migration failed"); logger.LogError(exception.Message); logger.LogError(exception.InnerException?.Message); logger.LogError("[RunMigrations] end catch-ed"); } }
/// <summary> /// Retry when an Exception has occured /// </summary> /// <param name="updateStatusContent"></param> /// <param name="e">Exception</param> private async Task RetrySaveChangesAsync(FileIndexItem updateStatusContent, Exception e) { _logger?.LogInformation(e, "[RetrySaveChangesAsync] retry catch-ed exception "); _logger?.LogInformation("[RetrySaveChangesAsync] next retry ~>"); async Task LocalRetrySaveChangesAsyncQuery() { // InvalidOperationException: A second operation started on this context before a previous operation completed. // https://go.microsoft.com/fwlink/?linkid=2097913 await Task.Delay(5); var context = new InjectServiceScope(_scopeFactory).Context(); context.Attach(updateStatusContent).State = EntityState.Modified; await context.SaveChangesAsync(); context.Attach(updateStatusContent).State = EntityState.Detached; await context.DisposeAsync(); } try { await LocalRetrySaveChangesAsyncQuery(); } catch (MySqlException mySqlException) { _logger?.LogError(mySqlException, "[RetrySaveChangesAsync] MySqlException catch-ed and retry again"); await LocalRetrySaveChangesAsyncQuery(); } catch (DbUpdateConcurrencyException concurrencyException) { SolveConcurrency.SolveConcurrencyExceptionLoop(concurrencyException.Entries); try { _logger?.LogInformation("[RetrySaveChangesAsync] SolveConcurrencyExceptionLoop disposed item"); var context = new InjectServiceScope(_scopeFactory).Context(); await context.SaveChangesAsync(); } catch (DbUpdateConcurrencyException retry2Exception) { _logger?.LogInformation(retry2Exception, "[RetrySaveChangesAsync] save failed after DbUpdateConcurrencyException"); } } }
public async Task <bool> WriteAndCropFile(string fileHash, OffsetModel offsetData, int sourceWidth, int sourceHeight, FileIndexItem.Rotation rotation, string reference = null) { try { using (var thumbnailStream = new MemoryStream(offsetData.Data, offsetData.Index, offsetData.Count)) using (var smallImage = await Image.LoadAsync(thumbnailStream)) using (var outputStream = new MemoryStream()) { var smallImageWidth = smallImage.Width; var smallImageHeight = smallImage.Height; var result = NewImageSize.NewImageSizeCalc(smallImageWidth, smallImageHeight, sourceWidth, sourceHeight); smallImage.Mutate( i => i.Resize(smallImageWidth, smallImageHeight, KnownResamplers.Lanczos3) .Crop(new Rectangle(result.DestX, result.DestY, result.DestWidth, result.DestHeight))); var larger = (int)Math.Round(result.DestWidth * 1.2, 0); smallImage.Mutate( i => i.Resize(larger, 0, KnownResamplers.Lanczos3)); var rotate = RotateEnumToDegrees(rotation); smallImage.Mutate( i => i.Rotate(rotate)); await smallImage.SaveAsJpegAsync(outputStream); await _thumbnailStorage.WriteStreamAsync(outputStream, ThumbnailNameHelper.Combine(fileHash, ThumbnailSize.TinyMeta)); if (_appSettings.ApplicationType == AppSettings.StarskyAppType.WebController) { _logger.LogInformation($"[WriteAndCropFile] fileHash: {fileHash} is written"); } } return(true); } catch (Exception ex) { var message = ex.Message; if (message.StartsWith("Image cannot be loaded")) { message = "Image cannot be loaded"; } _logger.LogError($"[WriteFile@meta] Exception {reference} {message}", ex); return(false); } }
private void InitReverseGeoCode() { if (_reverseGeoCode != null) { return; } try { _reverseGeoCode = new ReverseGeoCode <ExtendedGeoName>( GeoFileReader.ReadExtendedGeoNames( Path.Combine(_appSettings.TempFolder, GeoFileDownload.CountryName + ".txt"))); } catch (ParserException e) { _logger?.LogError(e, "catch-ed GeoFileDownload GeoReverseLookup error"); } catch (FileNotFoundException e) { _logger?.LogError(e, "catch-ed GeoFileDownload GeoReverseLookup error"); } }
public async Task CleanOldMessagesAsync() { try { var messages = await _notificationQuery.GetOlderThan(DateTime.UtcNow.AddDays(-30)); await _notificationQuery.RemoveAsync(messages); } catch (Exception exception) { if (!exception.Message.Contains("Notifications' doesn't exist")) { _logger.LogError(exception, "[CleanOldMessagesAsync] catch-ed exception"); } } }
public IActionResult MemoryCacheDebug() { var result = new Dictionary <string, object>(); foreach (var key in _memoryCache.GetKeys <string>()) { _memoryCache.TryGetValue(key, out var data); try { result.Add(key, JsonSerializer.Serialize(data)); } catch (JsonException exception) { _logger.LogError(exception, $"[MemoryCacheDebug] Json {key} has failed (catch-ed)"); result.Add(key, "[ERROR] Has failed parsing"); } } return(Json(result)); }
public static async Task ProcessTaskQueueAsync(IBaseBackgroundTaskQueue taskQueue, IWebLogger logger, CancellationToken stoppingToken) { logger.LogInformation($"Queued Hosted Service {taskQueue.GetType().Name} is " + $"starting on {Environment.MachineName}"); while (!stoppingToken.IsCancellationRequested) { var workItem = await taskQueue.DequeueAsync(stoppingToken); try { await workItem(stoppingToken); } catch (Exception exception) { logger.LogError(exception, $"Error occurred executing workItem ", nameof(workItem)); } } logger.LogInformation("Queued Hosted Service has stopped"); }
/// <summary> /// Run Update /// </summary> /// <param name="changedFileIndexItemName">Per file stored string{fileHash}, /// List*string*{FileIndexItem.name (e.g. Tags) that are changed}</param> /// <param name="fileIndexResultsList">items stored in the database</param> /// <param name="inputModel">(only used when cache is disabled) /// This model is overwritten in the database and ExifTool</param> /// <param name="collections">enable or disable this feature</param> /// <param name="append">only for disabled cache or changedFileIndexItemName=null</param> /// <param name="rotateClock">rotation value 1 left, -1 right, 0 nothing</param> public async Task <List <FileIndexItem> > UpdateAsync( Dictionary <string, List <string> > changedFileIndexItemName, List <FileIndexItem> fileIndexResultsList, FileIndexItem inputModel, // only when changedFileIndexItemName = null bool collections, bool append, // only when changedFileIndexItemName = null int rotateClock) // <- this one is needed { if (changedFileIndexItemName == null) { changedFileIndexItemName = (await _metaPreflight.Preflight(inputModel, fileIndexResultsList.Select(p => p.FilePath).ToArray(), append, collections, rotateClock)).changedFileIndexItemName; } var updatedItems = new List <FileIndexItem>(); var fileIndexItemList = fileIndexResultsList .Where(p => p.Status == FileIndexItem.ExifStatus.Ok || p.Status == FileIndexItem.ExifStatus.Deleted).ToList(); foreach (var fileIndexItem in fileIndexItemList) { if (changedFileIndexItemName.ContainsKey(fileIndexItem.FilePath)) { // used for tracking differences, in the database/ExifTool compare var comparedNamesList = changedFileIndexItemName[fileIndexItem.FilePath]; await UpdateWriteDiskDatabase(fileIndexItem, comparedNamesList, rotateClock); updatedItems.Add(fileIndexItem); continue; } _logger.LogError($"Missing in key: {fileIndexItem.FilePath}", new InvalidDataException($"changedFileIndexItemName: " + $"{string.Join(",",changedFileIndexItemName)}")); throw new ArgumentException($"Missing in key: {fileIndexItem.FilePath}", nameof(changedFileIndexItemName)); } return(updatedItems); }
internal async Task WorkItem(string subPath, IStorage subPathStorage, IStorage thumbnailStorage) { try { _logger.LogInformation($"[ThumbnailGenerationController] start {subPath}"); var thumbnail = new Thumbnail(subPathStorage, thumbnailStorage, _logger); var thumbs = await thumbnail.CreateThumb(subPath); var getAllFilesAsync = await _query.GetAllFilesAsync(subPath); var result = new List <FileIndexItem>(); foreach (var item in getAllFilesAsync.Where(item => thumbs.FirstOrDefault(p => p.Item1 == item.FilePath).Item2)) { if (item.Tags.Contains("!delete!")) { continue; } item.SetLastEdited(); result.Add(item); } if (!result.Any()) { return; } var webSocketResponse = new ApiNotificationResponseModel <List <FileIndexItem> >(result, ApiNotificationType.ThumbnailGeneration); await _connectionsService.SendToAllAsync(webSocketResponse, CancellationToken.None); _logger.LogInformation($"[ThumbnailGenerationController] done {subPath}"); } catch (UnauthorizedAccessException e) { _logger.LogError($"[ThumbnailGenerationController] catch-ed exception {e.Message}", e); } }
public async Task <bool> Download(string sourceHttpUrl, string fullLocalPath) { if (_storage == null) { throw new EndOfStreamException("is null " + nameof(_storage)); } Uri sourceUri = new Uri(sourceHttpUrl); _logger.LogInformation("[Download] HttpClientHelper > " + sourceUri.Host + " ~ " + sourceHttpUrl); // allow whitelist and https only if (!_allowedDomains.Contains(sourceUri.Host) || sourceUri.Scheme != "https") { _logger.LogInformation("[Download] HttpClientHelper > " + "skip: domain not whitelisted " + " ~ " + sourceHttpUrl); return(false); } try { using (HttpResponseMessage response = await _httpProvider.GetAsync(sourceHttpUrl)) using (Stream streamToReadFrom = await response.Content.ReadAsStreamAsync()) { if (response.StatusCode != HttpStatusCode.OK) { _logger.LogInformation("[Download] HttpClientHelper > " + response.StatusCode + " ~ " + sourceHttpUrl); return(false); } await _storage.WriteStreamAsync(streamToReadFrom, fullLocalPath); return(true); } } catch (HttpRequestException exception) { _logger.LogError(exception, $"[Download] {exception.Message}"); return(false); } }
internal async Task <KeyValuePair <bool, string>?> DownloadCheckSums() { var checksums = await _httpClientHelper.ReadString(CheckSumLocation); if (checksums.Key) { return(checksums); } _logger.LogError($"Checksum loading failed {CheckSumLocation}, next retry from mirror"); checksums = await _httpClientHelper.ReadString(CheckSumLocationMirror); if (checksums.Key) { return(new KeyValuePair <bool, string>(false, checksums.Value)); } _logger.LogError($"Checksum loading failed {CheckSumLocationMirror}" + $", next stop; please connect to internet and restart app"); return(null); }
/// <summary> /// Private use => CreateThumb /// Create a Thumbnail file to load it faster in the UI. Use FileIndexItem or database style path, Feature used by the cli tool /// </summary> /// <param name="subPath">relative path to find the file in the storage folder</param> /// <param name="fileHash">the base32 hash of the subPath file</param> /// <param name="skipExtraLarge">skip the extra large image</param> /// <returns>true, if successful</returns> private async Task <bool> CreateThumbInternal(string subPath, string fileHash, bool skipExtraLarge = false) { // FileType=supported + subPath=exit + fileHash=NOT exist if (!ExtensionRolesHelper.IsExtensionThumbnailSupported(subPath) || !_iStorage.ExistFile(subPath)) { return(false); } // File is already tested if (_iStorage.ExistFile(GetErrorLogItemFullPath(subPath))) { return(false); } var thumbnailToSourceSize = ThumbnailSize.ExtraLarge; if (skipExtraLarge) { thumbnailToSourceSize = ThumbnailSize.Large; } var largeThumbnailHash = ThumbnailNameHelper.Combine(fileHash, thumbnailToSourceSize); if (!_thumbnailStorage.ExistFile(ThumbnailNameHelper.Combine( fileHash, thumbnailToSourceSize))) { // run resize sync var(_, resizeSuccess, resizeMessage) = (await ResizeThumbnailFromSourceImage(subPath, ThumbnailNameHelper.GetSize(thumbnailToSourceSize), largeThumbnailHash)); // check if output any good RemoveCorruptImage(fileHash, thumbnailToSourceSize); if (!resizeSuccess || !_thumbnailStorage.ExistFile( ThumbnailNameHelper.Combine(fileHash, thumbnailToSourceSize))) { _logger.LogError($"[ResizeThumbnailFromSourceImage] " + $"output is null or corrupt for subPath {subPath}"); await WriteErrorMessageToBlockLog(subPath, resizeMessage); return(false); } Console.Write("."); } var thumbnailFromThumbnailUpdateList = new List <ThumbnailSize>(); void Add(ThumbnailSize size) { if (!_thumbnailStorage.ExistFile( ThumbnailNameHelper.Combine( fileHash, size)) ) { thumbnailFromThumbnailUpdateList.Add(size); } } new List <ThumbnailSize> { ThumbnailSize.Small, ThumbnailSize.Large // <- will be false when skipExtraLarge = true }.ForEach(Add); await(thumbnailFromThumbnailUpdateList).ForEachAsync( async(size) => await ResizeThumbnailFromThumbnailImage( largeThumbnailHash, ThumbnailNameHelper.GetSize(size), ThumbnailNameHelper.Combine(fileHash, size)), 10); Console.Write("."); return(thumbnailFromThumbnailUpdateList.Any()); }