public bool TrySet(StorageItemMeta meta, SerializedObject value)
{
if (_storageWorker.Status != WorkerStatus.Started && _storageWorker.Status != WorkerStatus.Stopped)
{
Core.Log.Warning("The storage is disposing, modifying the collection is forbidden.");
return(false);
}
_pendingItems.AddOrUpdate(meta.Key, k =>
{
Interlocked.Increment(ref _pendingItemsCount);
return(value);
}, (k, v) =>
{
return(value);
});
_globalMetas.TryRemove(meta.Key, out _);
if (_metas.TryRemove(meta.Key, out var oldMeta))
{
Interlocked.Decrement(ref _metasCount);
if (oldMeta != null && oldMeta != meta)
{
oldMeta.Dispose();
}
}
if (meta.IsExpired)
{
if (_pendingItems.TryRemove(meta.Key, out _))
{
Interlocked.Decrement(ref _pendingItemsCount);
}
return(false);
}
if (!_metas.TryAdd(meta.Key, meta))
{
return(false);
}
_globalMetas.TryAdd(meta.Key, meta);
Interlocked.Increment(ref _metasCount);
meta.OnExpire = Meta_OnExpire;
if (_storageWorker.Count >= _storage.SlowDownWriteThreshold)
{
Core.Log.Warning("The storage working has reached his maximum capacity, slowing down the collection modification.");
TaskHelper.SleepUntil(() => _storageWorker.Count < _storage.SlowDownWriteThreshold).WaitAsync();
}
var fstoItem = FileStoragePool.New();
fstoItem.Meta = meta;
fstoItem.Type = FileStorageMetaLog.TransactionType.Add;
_storageWorker.Enqueue(fstoItem);
return(true);
}
public bool TryGetMeta(string key, out StorageItemMeta value, Predicate <StorageItemMeta> condition = null)
{
value = null;
if (!_metas.TryGetValue(key, out var metaValue))
{
return(false);
}
if (metaValue != null && !metaValue.IsExpired && (condition is null || condition(metaValue)))
{
value = metaValue;
return(true);
}
return(false);
}
public bool TryRemove(string key, out StorageItemMeta removedMeta)
{
if (_storageWorker.Status != WorkerStatus.Started && _storageWorker.Status != WorkerStatus.Stopped)
{
Core.Log.Warning("The storage is disposing, modifying the collection is forbidden.");
removedMeta = null;
return(false);
}
if (_pendingItems.TryRemove(key, out _))
{
Interlocked.Decrement(ref _pendingItemsCount);
}
_globalMetas.TryRemove(key, out _);
if (!_metas.TryRemove(key, out var meta))
{
removedMeta = null;
return(false);
}
Interlocked.Decrement(ref _metasCount);
meta.Dispose();
if (_storageWorker.Count >= _storage.SlowDownWriteThreshold)
{
Core.Log.Warning("The storage working has reached his maximum capacity, slowing down the collection modification.");
TaskHelper.SleepUntil(() => _storageWorker.Count < _storage.SlowDownWriteThreshold).WaitAsync();
}
var fstoItem = FileStoragePool.New();
fstoItem.Meta = meta;
fstoItem.Type = FileStorageMetaLog.TransactionType.Remove;
_storageWorker.Enqueue(fstoItem);
removedMeta = meta;
return(true);
}
public bool Set(StorageItemMeta meta, SerializedObject data) => Invoke <StorageItemMeta, SerializedObject, bool>(meta, data);
/// <summary>
/// Load folder
/// </summary>
/// <param name="cancellationToken">CancellationToken instance</param>
/// <returns>Folder load task</returns>
public async Task LoadAsync(CancellationToken cancellationToken = default)
{
try
{
//Ensure Directory
if (!Directory.Exists(BasePath))
{
Core.Log.InfoBasic("Creating SubFolder Directory: {0}", BasePath);
Directory.CreateDirectory(BasePath);
}
//Initialize files in case doesn't exists
if (!File.Exists(_transactionLogFilePath))
{
File.WriteAllBytes(_transactionLogFilePath, Array.Empty <byte>());
}
if (!File.Exists(_indexFilePath))
{
_indexSerializer.SerializeToFile(EmptyMetaList, _indexFilePath);
}
//Start loading
if (cancellationToken.IsCancellationRequested)
{
return;
}
#region Loading index file
var indexLoaded = await Task.Run(() => LoadIndexFile(_indexFilePath) || LoadIndexFile(_oldIndexFilePath)).ConfigureAwait(false);
if (!indexLoaded)
{
Core.Log.Warning("The index doesn't exist or couldn't be loaded. Generating new index file.");
var dateNow = Core.Now;
var eTime = dateNow.AddDays(5);
if (_storage.MaximumItemDuration.HasValue)
{
eTime = dateNow.Add(_storage.MaximumItemDuration.Value);
}
if (_storage.ItemsExpirationDateOverwrite.HasValue)
{
eTime = dateNow.Add(_storage.ItemsExpirationDateOverwrite.Value);
}
if (_storage.ItemsExpirationAbsoluteDateOverwrite.HasValue)
{
eTime = _storage.ItemsExpirationAbsoluteDateOverwrite.Value;
}
if (_metas is null)
{
_metas = new ConcurrentDictionary <string, StorageItemMeta>();
}
await Task.Run(() =>
{
var allFiles = Directory.EnumerateFiles(BasePath, "*" + DataExtension, SearchOption.AllDirectories);
var idx = 0;
foreach (var file in allFiles)
{
var cTime = File.GetCreationTime(file);
var key = Path.GetFileNameWithoutExtension(file);
var stoMeta = new StorageItemMeta
{
Key = key,
CreationDate = cTime,
ExpirationDate = eTime
};
_metas.TryAdd(key, stoMeta);
_globalMetas.TryAdd(key, stoMeta);
if (idx % 100 == 0)
{
Core.Log.InfoBasic("Number of files loaded: {0}", idx);
}
idx++;
}
}).ConfigureAwait(false);
Core.Log.InfoBasic("Index generated...");
}
#endregion
#region Loading transaction log file
var transactionLog = await LoadTransactionFileAsync(_transactionLogFilePath).ConfigureAwait(false);
if (transactionLog is null)
{
transactionLog = await LoadTransactionFileAsync(_oldTransactionLogFilePath).ConfigureAwait(false);
}
#endregion
#region Applying pending transactions
if (transactionLog?.Count > 0)
{
Core.Log.InfoBasic("Applying {0} pending transactions", transactionLog.Count);
foreach (var item in transactionLog)
{
switch (item.Type)
{
case FileStorageMetaLog.TransactionType.Add:
_globalMetas.TryRemove(item.Meta.Key, out _);
if (_metas.TryRemove(item.Meta.Key, out var oldAddedMeta) && oldAddedMeta != null)
{
oldAddedMeta.Dispose();
}
_metas.TryAdd(item.Meta.Key, item.Meta);
_globalMetas.TryAdd(item.Meta.Key, item.Meta);
break;
case FileStorageMetaLog.TransactionType.Remove:
_globalMetas.TryRemove(item.Meta.Key, out _);
if (_metas.TryRemove(item.Meta.Key, out var oldMeta) && oldMeta != null)
{
oldMeta.Dispose();
}
break;
}
}
}
#endregion
_transactionStream = File.Open(_transactionLogFilePath, FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite);
//await SaveMetadataAsync().ConfigureAwait(false);
SaveMetadata();
RemoveExpiredItems(false);
foreach (var metaItem in _metas)
{
if (metaItem.Value is null)
{
continue;
}
metaItem.Value.OnExpire = Meta_OnExpire;
}
var metasCount = _metas.Count;
Interlocked.Exchange(ref _metasCount, metasCount);
Core.Log.InfoBasic("Total item loaded in {0}: {1}", BasePath, metasCount);
Status = FolderHandlerStatus.Loaded;
}
catch (Exception ex)
{
Core.Log.Write(ex);
Status = FolderHandlerStatus.LoadFailed;
}
}
