public bool TrySet(StorageItemMeta meta, SerializedObject value) { if (_storageWorker.Status != WorkerStatus.Started && _storageWorker.Status != WorkerStatus.Stopped) { Core.Log.Warning("The storage is disposing, modifying the collection is forbidden."); return(false); } _pendingItems.AddOrUpdate(meta.Key, k => { Interlocked.Increment(ref _pendingItemsCount); return(value); }, (k, v) => { return(value); }); _globalMetas.TryRemove(meta.Key, out _); if (_metas.TryRemove(meta.Key, out var oldMeta)) { Interlocked.Decrement(ref _metasCount); if (oldMeta != null && oldMeta != meta) { oldMeta.Dispose(); } } if (meta.IsExpired) { if (_pendingItems.TryRemove(meta.Key, out _)) { Interlocked.Decrement(ref _pendingItemsCount); } return(false); } if (!_metas.TryAdd(meta.Key, meta)) { return(false); } _globalMetas.TryAdd(meta.Key, meta); Interlocked.Increment(ref _metasCount); meta.OnExpire = Meta_OnExpire; if (_storageWorker.Count >= _storage.SlowDownWriteThreshold) { Core.Log.Warning("The storage working has reached his maximum capacity, slowing down the collection modification."); TaskHelper.SleepUntil(() => _storageWorker.Count < _storage.SlowDownWriteThreshold).WaitAsync(); } var fstoItem = FileStoragePool.New(); fstoItem.Meta = meta; fstoItem.Type = FileStorageMetaLog.TransactionType.Add; _storageWorker.Enqueue(fstoItem); return(true); }
public bool TryGetMeta(string key, out StorageItemMeta value, Predicate <StorageItemMeta> condition = null) { value = null; if (!_metas.TryGetValue(key, out var metaValue)) { return(false); } if (metaValue != null && !metaValue.IsExpired && (condition is null || condition(metaValue))) { value = metaValue; return(true); } return(false); }
public bool TryRemove(string key, out StorageItemMeta removedMeta) { if (_storageWorker.Status != WorkerStatus.Started && _storageWorker.Status != WorkerStatus.Stopped) { Core.Log.Warning("The storage is disposing, modifying the collection is forbidden."); removedMeta = null; return(false); } if (_pendingItems.TryRemove(key, out _)) { Interlocked.Decrement(ref _pendingItemsCount); } _globalMetas.TryRemove(key, out _); if (!_metas.TryRemove(key, out var meta)) { removedMeta = null; return(false); } Interlocked.Decrement(ref _metasCount); meta.Dispose(); if (_storageWorker.Count >= _storage.SlowDownWriteThreshold) { Core.Log.Warning("The storage working has reached his maximum capacity, slowing down the collection modification."); TaskHelper.SleepUntil(() => _storageWorker.Count < _storage.SlowDownWriteThreshold).WaitAsync(); } var fstoItem = FileStoragePool.New(); fstoItem.Meta = meta; fstoItem.Type = FileStorageMetaLog.TransactionType.Remove; _storageWorker.Enqueue(fstoItem); removedMeta = meta; return(true); }
public bool Set(StorageItemMeta meta, SerializedObject data) => Invoke <StorageItemMeta, SerializedObject, bool>(meta, data);
/// <summary> /// Load folder /// </summary> /// <param name="cancellationToken">CancellationToken instance</param> /// <returns>Folder load task</returns> public async Task LoadAsync(CancellationToken cancellationToken = default) { try { //Ensure Directory if (!Directory.Exists(BasePath)) { Core.Log.InfoBasic("Creating SubFolder Directory: {0}", BasePath); Directory.CreateDirectory(BasePath); } //Initialize files in case doesn't exists if (!File.Exists(_transactionLogFilePath)) { File.WriteAllBytes(_transactionLogFilePath, Array.Empty <byte>()); } if (!File.Exists(_indexFilePath)) { _indexSerializer.SerializeToFile(EmptyMetaList, _indexFilePath); } //Start loading if (cancellationToken.IsCancellationRequested) { return; } #region Loading index file var indexLoaded = await Task.Run(() => LoadIndexFile(_indexFilePath) || LoadIndexFile(_oldIndexFilePath)).ConfigureAwait(false); if (!indexLoaded) { Core.Log.Warning("The index doesn't exist or couldn't be loaded. Generating new index file."); var dateNow = Core.Now; var eTime = dateNow.AddDays(5); if (_storage.MaximumItemDuration.HasValue) { eTime = dateNow.Add(_storage.MaximumItemDuration.Value); } if (_storage.ItemsExpirationDateOverwrite.HasValue) { eTime = dateNow.Add(_storage.ItemsExpirationDateOverwrite.Value); } if (_storage.ItemsExpirationAbsoluteDateOverwrite.HasValue) { eTime = _storage.ItemsExpirationAbsoluteDateOverwrite.Value; } if (_metas is null) { _metas = new ConcurrentDictionary <string, StorageItemMeta>(); } await Task.Run(() => { var allFiles = Directory.EnumerateFiles(BasePath, "*" + DataExtension, SearchOption.AllDirectories); var idx = 0; foreach (var file in allFiles) { var cTime = File.GetCreationTime(file); var key = Path.GetFileNameWithoutExtension(file); var stoMeta = new StorageItemMeta { Key = key, CreationDate = cTime, ExpirationDate = eTime }; _metas.TryAdd(key, stoMeta); _globalMetas.TryAdd(key, stoMeta); if (idx % 100 == 0) { Core.Log.InfoBasic("Number of files loaded: {0}", idx); } idx++; } }).ConfigureAwait(false); Core.Log.InfoBasic("Index generated..."); } #endregion #region Loading transaction log file var transactionLog = await LoadTransactionFileAsync(_transactionLogFilePath).ConfigureAwait(false); if (transactionLog is null) { transactionLog = await LoadTransactionFileAsync(_oldTransactionLogFilePath).ConfigureAwait(false); } #endregion #region Applying pending transactions if (transactionLog?.Count > 0) { Core.Log.InfoBasic("Applying {0} pending transactions", transactionLog.Count); foreach (var item in transactionLog) { switch (item.Type) { case FileStorageMetaLog.TransactionType.Add: _globalMetas.TryRemove(item.Meta.Key, out _); if (_metas.TryRemove(item.Meta.Key, out var oldAddedMeta) && oldAddedMeta != null) { oldAddedMeta.Dispose(); } _metas.TryAdd(item.Meta.Key, item.Meta); _globalMetas.TryAdd(item.Meta.Key, item.Meta); break; case FileStorageMetaLog.TransactionType.Remove: _globalMetas.TryRemove(item.Meta.Key, out _); if (_metas.TryRemove(item.Meta.Key, out var oldMeta) && oldMeta != null) { oldMeta.Dispose(); } break; } } } #endregion _transactionStream = File.Open(_transactionLogFilePath, FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite); //await SaveMetadataAsync().ConfigureAwait(false); SaveMetadata(); RemoveExpiredItems(false); foreach (var metaItem in _metas) { if (metaItem.Value is null) { continue; } metaItem.Value.OnExpire = Meta_OnExpire; } var metasCount = _metas.Count; Interlocked.Exchange(ref _metasCount, metasCount); Core.Log.InfoBasic("Total item loaded in {0}: {1}", BasePath, metasCount); Status = FolderHandlerStatus.Loaded; } catch (Exception ex) { Core.Log.Write(ex); Status = FolderHandlerStatus.LoadFailed; } }