private void OnError(object source, ErrorEventArgs e)
{
// Show that an error has been detected.
_webLogger.LogError(e.GetException(), "[DiskWatcher] The FileSystemWatcher has an error (catch-ed) - next: retry " +
$"{DateTimeDebug()}");
_webLogger.LogError("[DiskWatcher] (catch-ed) " + e.GetException().Message);
// Give more information if the error is due to an internal buffer overflow.
if (e.GetException().GetType() == typeof(InternalBufferOverflowException))
{
// This can happen if Windows is reporting many file system events quickly
// and internal buffer of the FileSystemWatcher is not large enough to handle this
// rate of events. The InternalBufferOverflowException error informs the application
// that some of the file system events are being lost.
_webLogger.LogError(e.GetException(), "[DiskWatcher] The file system watcher experienced an internal buffer overflow ");
}
// when test dont retry
if (e.GetException().Message == "test")
{
return;
}
// When fail it should try it again
Retry(new BufferingFileSystemWatcher(_fileSystemWatcherWrapper.Path));
}
/// <summary>
/// Get All files from the directory
/// </summary>
/// <param name="path">fullFilePath</param>
/// <returns></returns>
public IEnumerable <string> GetAllFilesInDirectory(string path)
{
string[] allFiles;
try
{
allFiles = Directory.GetFiles(path);
}
catch (UnauthorizedAccessException e)
{
_logger?.LogError(e, "[GetAllFilesInDirectory] catch-ed UnauthorizedAccessException");
return(new string[] {});
}
var imageFilesList = new List <string>();
foreach (var file in allFiles)
{
// Path.GetExtension uses (.ext)
// the same check in SingleFile
// Recruisive >= same check
// ignore Files with ._ names, this is Mac OS specific
var isAppleDouble = Path.GetFileName(file).StartsWith("._");
if (!isAppleDouble)
{
imageFilesList.Add(file);
}
// to filter use:
// ..etAllFilesInDirectory(subPath)
// .Where(ExtensionRolesHelper.IsExtensionExifToolSupported)
// OR:
// .Where(ExtensionRolesHelper.IsExtensionSyncSupported
}
return(imageFilesList);
}
/// <summary>
/// Generate loop of Jpeg images with overlay image
/// With Retry included
/// </summary>
/// <param name="profile">contains sizes</param>
/// <param name="fileIndexItemsList">list of items to generate jpeg for</param>
/// <param name="outputParentFullFilePathFolder">outputParentFullFilePathFolder</param>
/// <param name="delay">when failed output, has default value</param>
/// <returns></returns>
internal async Task <Dictionary <string, bool> > GenerateJpeg(AppSettingsPublishProfiles profile,
IReadOnlyCollection <FileIndexItem> fileIndexItemsList, string outputParentFullFilePathFolder, int delay = 6)
{
_toCreateSubfolder.Create(profile, outputParentFullFilePathFolder);
foreach (var item in fileIndexItemsList)
{
var outputPath = _overlayImage.FilePathOverlayImage(outputParentFullFilePathFolder,
item.FilePath, profile);
async Task <bool> ResizerLocal()
{
return(await Resizer(outputPath, profile, item));
}
try
{
await RetryHelper.DoAsync(ResizerLocal, TimeSpan.FromSeconds(delay));
}
catch (AggregateException e)
{
_logger.LogError($"[ResizerLocal] Skip due errors: (catch-ed exception) {item.FilePath} {item.FileHash}");
foreach (var exception in e.InnerExceptions)
{
_logger.LogError("[ResizerLocal] " + exception.Message, exception);
}
}
}
return(fileIndexItemsList.ToDictionary(item =>
_overlayImage.FilePathOverlayImage(item.FilePath, profile),
item => profile.Copy));
}
/// <summary>
/// Used to fill the cache with an array of
/// All keywords are stored lowercase
/// </summary>
/// <returns></returns>
public async Task <List <KeyValuePair <string, int> > > Inflate()
{
if (_cache.TryGetValue(nameof(SearchSuggestionsService), out _))
{
return(new Dictionary <string, int>().ToList());
}
var allFilesList = new List <KeyValuePair <string, int> >();
try
{
allFilesList = await _context.FileIndex.GroupBy(i => i.Tags)
// ReSharper disable once UseMethodAny.1
.Where(x => x.Count() >= 1) // .ANY is not supported by EF Core
.TagWith("Inflate SearchSuggestionsService")
.Select(val => new KeyValuePair <string, int>(val.Key, val.Count())).ToListAsync();
}
catch (Exception exception)
{
if (!exception.Message.Contains("Unknown column"))
{
_logger.LogError(exception, "mysql search suggest exception catch-ed");
}
return(allFilesList);
}
var suggestions = new Dictionary <string, int>(StringComparer.InvariantCultureIgnoreCase);
foreach (var tag in allFilesList)
{
if (string.IsNullOrEmpty(tag.Key))
{
continue;
}
var keywordsHashSet = HashSetHelper.StringToHashSet(tag.Key.Trim());
foreach (var keyword in keywordsHashSet)
{
if (suggestions.ContainsKey(keyword))
{
suggestions[keyword] += tag.Value;
}
else
{
suggestions.Add(keyword, tag.Value);
}
}
}
var suggestionsFiltered = suggestions
.Where(p => p.Value >= 10)
.OrderByDescending(p => p.Value)
.ToList();
_cache.Set(nameof(SearchSuggestionsService), suggestionsFiltered,
new TimeSpan(100, 0, 0));
return(suggestionsFiltered);
}
public static async Task Run(ApplicationDbContext dbContext, IWebLogger logger, AppSettings appSettings, int retryCount = 2)
{
async Task <bool> Migrate()
{
if (appSettings.DatabaseType == AppSettings.DatabaseTypeList.InMemoryDatabase)
{
return(true);
}
await dbContext.Database.MigrateAsync();
if (appSettings.DatabaseType !=
AppSettings.DatabaseTypeList.Mysql)
{
return(true);
}
var connection = new MySqlConnection(appSettings.DatabaseConnection);
var databaseFixes =
new MySqlDatabaseFixes(connection, appSettings);
await databaseFixes.OpenConnection();
var tableNames = dbContext.Model.GetEntityTypes()
.Select(t => t.GetTableName())
.Distinct()
.ToList();
await databaseFixes.FixUtf8Encoding(tableNames);
await databaseFixes.FixAutoIncrement("Notifications");
return(true);
}
try
{
await RetryHelper.DoAsync(Migrate, TimeSpan.FromSeconds(2), retryCount);
}
catch (AggregateException exception)
{
logger.LogInformation("[RunMigrations] start migration failed");
logger.LogError(exception.Message);
logger.LogError(exception.InnerException?.Message);
logger.LogError("[RunMigrations] end catch-ed");
}
}
/// <summary>
/// Retry when an Exception has occured
/// </summary>
/// <param name="updateStatusContent"></param>
/// <param name="e">Exception</param>
private async Task RetrySaveChangesAsync(FileIndexItem updateStatusContent, Exception e)
{
_logger?.LogInformation(e, "[RetrySaveChangesAsync] retry catch-ed exception ");
_logger?.LogInformation("[RetrySaveChangesAsync] next retry ~>");
async Task LocalRetrySaveChangesAsyncQuery()
{
// InvalidOperationException: A second operation started on this context before a previous operation completed.
// https://go.microsoft.com/fwlink/?linkid=2097913
await Task.Delay(5);
var context = new InjectServiceScope(_scopeFactory).Context();
context.Attach(updateStatusContent).State = EntityState.Modified;
await context.SaveChangesAsync();
context.Attach(updateStatusContent).State = EntityState.Detached;
await context.DisposeAsync();
}
try
{
await LocalRetrySaveChangesAsyncQuery();
}
catch (MySqlException mySqlException)
{
_logger?.LogError(mySqlException, "[RetrySaveChangesAsync] MySqlException catch-ed and retry again");
await LocalRetrySaveChangesAsyncQuery();
}
catch (DbUpdateConcurrencyException concurrencyException)
{
SolveConcurrency.SolveConcurrencyExceptionLoop(concurrencyException.Entries);
try
{
_logger?.LogInformation("[RetrySaveChangesAsync] SolveConcurrencyExceptionLoop disposed item");
var context = new InjectServiceScope(_scopeFactory).Context();
await context.SaveChangesAsync();
}
catch (DbUpdateConcurrencyException retry2Exception)
{
_logger?.LogInformation(retry2Exception,
"[RetrySaveChangesAsync] save failed after DbUpdateConcurrencyException");
}
}
}
public async Task <bool> WriteAndCropFile(string fileHash,
OffsetModel offsetData, int sourceWidth,
int sourceHeight, FileIndexItem.Rotation rotation,
string reference = null)
{
try
{
using (var thumbnailStream = new MemoryStream(offsetData.Data, offsetData.Index, offsetData.Count))
using (var smallImage = await Image.LoadAsync(thumbnailStream))
using (var outputStream = new MemoryStream())
{
var smallImageWidth = smallImage.Width;
var smallImageHeight = smallImage.Height;
var result = NewImageSize.NewImageSizeCalc(smallImageWidth,
smallImageHeight, sourceWidth, sourceHeight);
smallImage.Mutate(
i => i.Resize(smallImageWidth, smallImageHeight, KnownResamplers.Lanczos3)
.Crop(new Rectangle(result.DestX, result.DestY, result.DestWidth, result.DestHeight)));
var larger = (int)Math.Round(result.DestWidth * 1.2, 0);
smallImage.Mutate(
i => i.Resize(larger, 0, KnownResamplers.Lanczos3));
var rotate = RotateEnumToDegrees(rotation);
smallImage.Mutate(
i => i.Rotate(rotate));
await smallImage.SaveAsJpegAsync(outputStream);
await _thumbnailStorage.WriteStreamAsync(outputStream, ThumbnailNameHelper.Combine(fileHash, ThumbnailSize.TinyMeta));
if (_appSettings.ApplicationType == AppSettings.StarskyAppType.WebController)
{
_logger.LogInformation($"[WriteAndCropFile] fileHash: {fileHash} is written");
}
}
return(true);
}
catch (Exception ex)
{
var message = ex.Message;
if (message.StartsWith("Image cannot be loaded"))
{
message = "Image cannot be loaded";
}
_logger.LogError($"[WriteFile@meta] Exception {reference} {message}", ex);
return(false);
}
}
private void InitReverseGeoCode()
{
if (_reverseGeoCode != null)
{
return;
}
try
{
_reverseGeoCode = new ReverseGeoCode <ExtendedGeoName>(
GeoFileReader.ReadExtendedGeoNames(
Path.Combine(_appSettings.TempFolder, GeoFileDownload.CountryName + ".txt")));
}
catch (ParserException e)
{
_logger?.LogError(e, "catch-ed GeoFileDownload GeoReverseLookup error");
}
catch (FileNotFoundException e)
{
_logger?.LogError(e, "catch-ed GeoFileDownload GeoReverseLookup error");
}
}
public async Task CleanOldMessagesAsync()
{
try
{
var messages = await _notificationQuery.GetOlderThan(DateTime.UtcNow.AddDays(-30));
await _notificationQuery.RemoveAsync(messages);
}
catch (Exception exception)
{
if (!exception.Message.Contains("Notifications' doesn't exist"))
{
_logger.LogError(exception, "[CleanOldMessagesAsync] catch-ed exception");
}
}
}
public IActionResult MemoryCacheDebug()
{
var result = new Dictionary <string, object>();
foreach (var key in _memoryCache.GetKeys <string>())
{
_memoryCache.TryGetValue(key, out var data);
try
{
result.Add(key, JsonSerializer.Serialize(data));
}
catch (JsonException exception)
{
_logger.LogError(exception, $"[MemoryCacheDebug] Json {key} has failed (catch-ed)");
result.Add(key, "[ERROR] Has failed parsing");
}
}
return(Json(result));
}
public static async Task ProcessTaskQueueAsync(IBaseBackgroundTaskQueue taskQueue, IWebLogger logger, CancellationToken stoppingToken)
{
logger.LogInformation($"Queued Hosted Service {taskQueue.GetType().Name} is " +
$"starting on {Environment.MachineName}");
while (!stoppingToken.IsCancellationRequested)
{
var workItem = await taskQueue.DequeueAsync(stoppingToken);
try
{
await workItem(stoppingToken);
}
catch (Exception exception)
{
logger.LogError(exception,
$"Error occurred executing workItem ", nameof(workItem));
}
}
logger.LogInformation("Queued Hosted Service has stopped");
}
/// <summary>
/// Run Update
/// </summary>
/// <param name="changedFileIndexItemName">Per file stored string{fileHash},
/// List*string*{FileIndexItem.name (e.g. Tags) that are changed}</param>
/// <param name="fileIndexResultsList">items stored in the database</param>
/// <param name="inputModel">(only used when cache is disabled)
/// This model is overwritten in the database and ExifTool</param>
/// <param name="collections">enable or disable this feature</param>
/// <param name="append">only for disabled cache or changedFileIndexItemName=null</param>
/// <param name="rotateClock">rotation value 1 left, -1 right, 0 nothing</param>
public async Task <List <FileIndexItem> > UpdateAsync(
Dictionary <string, List <string> > changedFileIndexItemName,
List <FileIndexItem> fileIndexResultsList,
FileIndexItem inputModel, // only when changedFileIndexItemName = null
bool collections, bool append, // only when changedFileIndexItemName = null
int rotateClock) // <- this one is needed
{
if (changedFileIndexItemName == null)
{
changedFileIndexItemName = (await _metaPreflight.Preflight(inputModel,
fileIndexResultsList.Select(p => p.FilePath).ToArray(), append, collections,
rotateClock)).changedFileIndexItemName;
}
var updatedItems = new List <FileIndexItem>();
var fileIndexItemList = fileIndexResultsList
.Where(p => p.Status == FileIndexItem.ExifStatus.Ok ||
p.Status == FileIndexItem.ExifStatus.Deleted).ToList();
foreach (var fileIndexItem in fileIndexItemList)
{
if (changedFileIndexItemName.ContainsKey(fileIndexItem.FilePath))
{
// used for tracking differences, in the database/ExifTool compare
var comparedNamesList = changedFileIndexItemName[fileIndexItem.FilePath];
await UpdateWriteDiskDatabase(fileIndexItem, comparedNamesList, rotateClock);
updatedItems.Add(fileIndexItem);
continue;
}
_logger.LogError($"Missing in key: {fileIndexItem.FilePath}",
new InvalidDataException($"changedFileIndexItemName: " +
$"{string.Join(",",changedFileIndexItemName)}"));
throw new ArgumentException($"Missing in key: {fileIndexItem.FilePath}",
nameof(changedFileIndexItemName));
}
return(updatedItems);
}
internal async Task WorkItem(string subPath, IStorage subPathStorage,
IStorage thumbnailStorage)
{
try
{
_logger.LogInformation($"[ThumbnailGenerationController] start {subPath}");
var thumbnail = new Thumbnail(subPathStorage,
thumbnailStorage, _logger);
var thumbs = await thumbnail.CreateThumb(subPath);
var getAllFilesAsync = await _query.GetAllFilesAsync(subPath);
var result = new List <FileIndexItem>();
foreach (var item in
getAllFilesAsync.Where(item => thumbs.FirstOrDefault(p => p.Item1 == item.FilePath).Item2))
{
if (item.Tags.Contains("!delete!"))
{
continue;
}
item.SetLastEdited();
result.Add(item);
}
if (!result.Any())
{
return;
}
var webSocketResponse =
new ApiNotificationResponseModel <List <FileIndexItem> >(result, ApiNotificationType.ThumbnailGeneration);
await _connectionsService.SendToAllAsync(webSocketResponse, CancellationToken.None);
_logger.LogInformation($"[ThumbnailGenerationController] done {subPath}");
}
catch (UnauthorizedAccessException e)
{
_logger.LogError($"[ThumbnailGenerationController] catch-ed exception {e.Message}", e);
}
}
public async Task <bool> Download(string sourceHttpUrl, string fullLocalPath)
{
if (_storage == null)
{
throw new EndOfStreamException("is null " + nameof(_storage));
}
Uri sourceUri = new Uri(sourceHttpUrl);
_logger.LogInformation("[Download] HttpClientHelper > " + sourceUri.Host + " ~ " + sourceHttpUrl);
// allow whitelist and https only
if (!_allowedDomains.Contains(sourceUri.Host) ||
sourceUri.Scheme != "https")
{
_logger.LogInformation("[Download] HttpClientHelper > " + "skip: domain not whitelisted " + " ~ " + sourceHttpUrl);
return(false);
}
try
{
using (HttpResponseMessage response = await _httpProvider.GetAsync(sourceHttpUrl))
using (Stream streamToReadFrom = await response.Content.ReadAsStreamAsync())
{
if (response.StatusCode != HttpStatusCode.OK)
{
_logger.LogInformation("[Download] HttpClientHelper > " + response.StatusCode + " ~ " + sourceHttpUrl);
return(false);
}
await _storage.WriteStreamAsync(streamToReadFrom, fullLocalPath);
return(true);
}
}
catch (HttpRequestException exception)
{
_logger.LogError(exception, $"[Download] {exception.Message}");
return(false);
}
}
internal async Task <KeyValuePair <bool, string>?> DownloadCheckSums()
{
var checksums = await _httpClientHelper.ReadString(CheckSumLocation);
if (checksums.Key)
{
return(checksums);
}
_logger.LogError($"Checksum loading failed {CheckSumLocation}, next retry from mirror");
checksums = await _httpClientHelper.ReadString(CheckSumLocationMirror);
if (checksums.Key)
{
return(new KeyValuePair <bool, string>(false, checksums.Value));
}
_logger.LogError($"Checksum loading failed {CheckSumLocationMirror}" +
$", next stop; please connect to internet and restart app");
return(null);
}
/// <summary>
/// Private use => CreateThumb
/// Create a Thumbnail file to load it faster in the UI. Use FileIndexItem or database style path, Feature used by the cli tool
/// </summary>
/// <param name="subPath">relative path to find the file in the storage folder</param>
/// <param name="fileHash">the base32 hash of the subPath file</param>
/// <param name="skipExtraLarge">skip the extra large image</param>
/// <returns>true, if successful</returns>
private async Task <bool> CreateThumbInternal(string subPath, string fileHash, bool skipExtraLarge = false)
{
// FileType=supported + subPath=exit + fileHash=NOT exist
if (!ExtensionRolesHelper.IsExtensionThumbnailSupported(subPath) ||
!_iStorage.ExistFile(subPath))
{
return(false);
}
// File is already tested
if (_iStorage.ExistFile(GetErrorLogItemFullPath(subPath)))
{
return(false);
}
var thumbnailToSourceSize = ThumbnailSize.ExtraLarge;
if (skipExtraLarge)
{
thumbnailToSourceSize = ThumbnailSize.Large;
}
var largeThumbnailHash = ThumbnailNameHelper.Combine(fileHash, thumbnailToSourceSize);
if (!_thumbnailStorage.ExistFile(ThumbnailNameHelper.Combine(
fileHash, thumbnailToSourceSize)))
{
// run resize sync
var(_, resizeSuccess, resizeMessage) = (await ResizeThumbnailFromSourceImage(subPath,
ThumbnailNameHelper.GetSize(thumbnailToSourceSize),
largeThumbnailHash));
// check if output any good
RemoveCorruptImage(fileHash, thumbnailToSourceSize);
if (!resizeSuccess || !_thumbnailStorage.ExistFile(
ThumbnailNameHelper.Combine(fileHash, thumbnailToSourceSize)))
{
_logger.LogError($"[ResizeThumbnailFromSourceImage] " +
$"output is null or corrupt for subPath {subPath}");
await WriteErrorMessageToBlockLog(subPath, resizeMessage);
return(false);
}
Console.Write(".");
}
var thumbnailFromThumbnailUpdateList = new List <ThumbnailSize>();
void Add(ThumbnailSize size)
{
if (!_thumbnailStorage.ExistFile(
ThumbnailNameHelper.Combine(
fileHash, size))
)
{
thumbnailFromThumbnailUpdateList.Add(size);
}
}
new List <ThumbnailSize>
{
ThumbnailSize.Small,
ThumbnailSize.Large // <- will be false when skipExtraLarge = true
}.ForEach(Add);
await(thumbnailFromThumbnailUpdateList).ForEachAsync(
async(size)
=> await ResizeThumbnailFromThumbnailImage(
largeThumbnailHash,
ThumbnailNameHelper.GetSize(size),
ThumbnailNameHelper.Combine(fileHash, size)),
10);
Console.Write(".");
return(thumbnailFromThumbnailUpdateList.Any());
}
