/// <summary>
/// Folder handler
/// </summary>
/// <param name="storage">Storage base</param>
/// <param name="globalMetas">Global Metas</param>
/// <param name="basePath">Base path</param>
public FolderHandler(FileStorage storage, ConcurrentDictionary <string, StorageItemMeta> globalMetas, string basePath)
{
BasePath = basePath;
_storage = storage;
_globalMetas = globalMetas;
_indexSerializer = storage.IndexSerializer;
_saveMetadataBuffered = ActionDelegate.Create(SaveMetadata).CreateBufferedAction(1000);
var extension = _indexSerializer.Extensions[0];
_transactionLogFilePath = Path.Combine(BasePath, TransactionLogFileName + extension);
_dataPathPattern = Path.Combine(BasePath, "$FILE$" + DataExtension);
_indexFilePath = Path.Combine(BasePath, IndexFileName + extension);
_oldTransactionLogFilePath = _transactionLogFilePath + ".old";
_oldIndexFilePath = _indexFilePath + ".old";
_pendingItems = new ConcurrentDictionary <string, SerializedObject>();
_storageWorker = new Worker <FileStorageMetaLog>(WorkerProcess);
_storageWorker.OnWorkDone += (s, e) => _saveMetadataBuffered();
Status = FolderHandlerStatus.Startup;
Core.Status.Attach(collection =>
{
var workerCount = _storageWorker.Count;
var percentWork = (double)workerCount / _storage.SlowDownWriteThreshold;
collection.Add(nameof(BasePath), BasePath);
collection.Add("Count", _metasCount, StatusItemValueStatus.Ok, true);
collection.Add("Pending Count", _pendingItemsCount, StatusItemValueStatus.Ok, true);
collection.Add("Worker Count", workerCount, percentWork < 0.8 ? StatusItemValueStatus.Ok : percentWork < 0.95 ? StatusItemValueStatus.Warning : StatusItemValueStatus.Error, true);
collection.Add("Worker Process Percent", Math.Round(percentWork * 100, 2) + "%", percentWork < 0.8 ? StatusItemValueStatus.Ok : percentWork < 0.95 ? StatusItemValueStatus.Warning : StatusItemValueStatus.Error);
collection.Add("Transaction Log Length", _currentTransactionLogLength, StatusItemValueStatus.Ok);
collection.Add("Index File", _indexFilePath, StatusItemValueStatus.Ok);
collection.Add("Transaction File", _transactionLogFilePath, StatusItemValueStatus.Ok);
}, this);
}
/// <summary>
/// Load folder
/// </summary>
/// <param name="cancellationToken">CancellationToken instance</param>
/// <returns>Folder load task</returns>
public async Task LoadAsync(CancellationToken cancellationToken = default)
{
try
{
//Ensure Directory
if (!Directory.Exists(BasePath))
{
Core.Log.InfoBasic("Creating SubFolder Directory: {0}", BasePath);
Directory.CreateDirectory(BasePath);
}
//Initialize files in case doesn't exists
if (!File.Exists(_transactionLogFilePath))
{
File.WriteAllBytes(_transactionLogFilePath, Array.Empty <byte>());
}
if (!File.Exists(_indexFilePath))
{
_indexSerializer.SerializeToFile(EmptyMetaList, _indexFilePath);
}
//Start loading
if (cancellationToken.IsCancellationRequested)
{
return;
}
#region Loading index file
var indexLoaded = await Task.Run(() => LoadIndexFile(_indexFilePath) || LoadIndexFile(_oldIndexFilePath)).ConfigureAwait(false);
if (!indexLoaded)
{
Core.Log.Warning("The index doesn't exist or couldn't be loaded. Generating new index file.");
var dateNow = Core.Now;
var eTime = dateNow.AddDays(5);
if (_storage.MaximumItemDuration.HasValue)
{
eTime = dateNow.Add(_storage.MaximumItemDuration.Value);
}
if (_storage.ItemsExpirationDateOverwrite.HasValue)
{
eTime = dateNow.Add(_storage.ItemsExpirationDateOverwrite.Value);
}
if (_storage.ItemsExpirationAbsoluteDateOverwrite.HasValue)
{
eTime = _storage.ItemsExpirationAbsoluteDateOverwrite.Value;
}
if (_metas is null)
{
_metas = new ConcurrentDictionary <string, StorageItemMeta>();
}
await Task.Run(() =>
{
var allFiles = Directory.EnumerateFiles(BasePath, "*" + DataExtension, SearchOption.AllDirectories);
var idx = 0;
foreach (var file in allFiles)
{
var cTime = File.GetCreationTime(file);
var key = Path.GetFileNameWithoutExtension(file);
var stoMeta = new StorageItemMeta
{
Key = key,
CreationDate = cTime,
ExpirationDate = eTime
};
_metas.TryAdd(key, stoMeta);
_globalMetas.TryAdd(key, stoMeta);
if (idx % 100 == 0)
{
Core.Log.InfoBasic("Number of files loaded: {0}", idx);
}
idx++;
}
}).ConfigureAwait(false);
Core.Log.InfoBasic("Index generated...");
}
#endregion
#region Loading transaction log file
var transactionLog = await LoadTransactionFileAsync(_transactionLogFilePath).ConfigureAwait(false);
if (transactionLog is null)
{
transactionLog = await LoadTransactionFileAsync(_oldTransactionLogFilePath).ConfigureAwait(false);
}
#endregion
#region Applying pending transactions
if (transactionLog?.Count > 0)
{
Core.Log.InfoBasic("Applying {0} pending transactions", transactionLog.Count);
foreach (var item in transactionLog)
{
switch (item.Type)
{
case FileStorageMetaLog.TransactionType.Add:
_globalMetas.TryRemove(item.Meta.Key, out _);
if (_metas.TryRemove(item.Meta.Key, out var oldAddedMeta) && oldAddedMeta != null)
{
oldAddedMeta.Dispose();
}
_metas.TryAdd(item.Meta.Key, item.Meta);
_globalMetas.TryAdd(item.Meta.Key, item.Meta);
break;
case FileStorageMetaLog.TransactionType.Remove:
_globalMetas.TryRemove(item.Meta.Key, out _);
if (_metas.TryRemove(item.Meta.Key, out var oldMeta) && oldMeta != null)
{
oldMeta.Dispose();
}
break;
}
}
}
#endregion
_transactionStream = File.Open(_transactionLogFilePath, FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite);
//await SaveMetadataAsync().ConfigureAwait(false);
SaveMetadata();
RemoveExpiredItems(false);
foreach (var metaItem in _metas)
{
if (metaItem.Value is null)
{
continue;
}
metaItem.Value.OnExpire = Meta_OnExpire;
}
var metasCount = _metas.Count;
Interlocked.Exchange(ref _metasCount, metasCount);
Core.Log.InfoBasic("Total item loaded in {0}: {1}", BasePath, metasCount);
Status = FolderHandlerStatus.Loaded;
}
catch (Exception ex)
{
Core.Log.Write(ex);
Status = FolderHandlerStatus.LoadFailed;
}
}
