/// <summary> /// Folder handler /// </summary> /// <param name="storage">Storage base</param> /// <param name="globalMetas">Global Metas</param> /// <param name="basePath">Base path</param> public FolderHandler(FileStorage storage, ConcurrentDictionary <string, StorageItemMeta> globalMetas, string basePath) { BasePath = basePath; _storage = storage; _globalMetas = globalMetas; _indexSerializer = storage.IndexSerializer; _saveMetadataBuffered = ActionDelegate.Create(SaveMetadata).CreateBufferedAction(1000); var extension = _indexSerializer.Extensions[0]; _transactionLogFilePath = Path.Combine(BasePath, TransactionLogFileName + extension); _dataPathPattern = Path.Combine(BasePath, "$FILE$" + DataExtension); _indexFilePath = Path.Combine(BasePath, IndexFileName + extension); _oldTransactionLogFilePath = _transactionLogFilePath + ".old"; _oldIndexFilePath = _indexFilePath + ".old"; _pendingItems = new ConcurrentDictionary <string, SerializedObject>(); _storageWorker = new Worker <FileStorageMetaLog>(WorkerProcess); _storageWorker.OnWorkDone += (s, e) => _saveMetadataBuffered(); Status = FolderHandlerStatus.Startup; Core.Status.Attach(collection => { var workerCount = _storageWorker.Count; var percentWork = (double)workerCount / _storage.SlowDownWriteThreshold; collection.Add(nameof(BasePath), BasePath); collection.Add("Count", _metasCount, StatusItemValueStatus.Ok, true); collection.Add("Pending Count", _pendingItemsCount, StatusItemValueStatus.Ok, true); collection.Add("Worker Count", workerCount, percentWork < 0.8 ? StatusItemValueStatus.Ok : percentWork < 0.95 ? StatusItemValueStatus.Warning : StatusItemValueStatus.Error, true); collection.Add("Worker Process Percent", Math.Round(percentWork * 100, 2) + "%", percentWork < 0.8 ? StatusItemValueStatus.Ok : percentWork < 0.95 ? StatusItemValueStatus.Warning : StatusItemValueStatus.Error); collection.Add("Transaction Log Length", _currentTransactionLogLength, StatusItemValueStatus.Ok); collection.Add("Index File", _indexFilePath, StatusItemValueStatus.Ok); collection.Add("Transaction File", _transactionLogFilePath, StatusItemValueStatus.Ok); }, this); }
/// <summary> /// Load folder /// </summary> /// <param name="cancellationToken">CancellationToken instance</param> /// <returns>Folder load task</returns> public async Task LoadAsync(CancellationToken cancellationToken = default) { try { //Ensure Directory if (!Directory.Exists(BasePath)) { Core.Log.InfoBasic("Creating SubFolder Directory: {0}", BasePath); Directory.CreateDirectory(BasePath); } //Initialize files in case doesn't exists if (!File.Exists(_transactionLogFilePath)) { File.WriteAllBytes(_transactionLogFilePath, Array.Empty <byte>()); } if (!File.Exists(_indexFilePath)) { _indexSerializer.SerializeToFile(EmptyMetaList, _indexFilePath); } //Start loading if (cancellationToken.IsCancellationRequested) { return; } #region Loading index file var indexLoaded = await Task.Run(() => LoadIndexFile(_indexFilePath) || LoadIndexFile(_oldIndexFilePath)).ConfigureAwait(false); if (!indexLoaded) { Core.Log.Warning("The index doesn't exist or couldn't be loaded. Generating new index file."); var dateNow = Core.Now; var eTime = dateNow.AddDays(5); if (_storage.MaximumItemDuration.HasValue) { eTime = dateNow.Add(_storage.MaximumItemDuration.Value); } if (_storage.ItemsExpirationDateOverwrite.HasValue) { eTime = dateNow.Add(_storage.ItemsExpirationDateOverwrite.Value); } if (_storage.ItemsExpirationAbsoluteDateOverwrite.HasValue) { eTime = _storage.ItemsExpirationAbsoluteDateOverwrite.Value; } if (_metas is null) { _metas = new ConcurrentDictionary <string, StorageItemMeta>(); } await Task.Run(() => { var allFiles = Directory.EnumerateFiles(BasePath, "*" + DataExtension, SearchOption.AllDirectories); var idx = 0; foreach (var file in allFiles) { var cTime = File.GetCreationTime(file); var key = Path.GetFileNameWithoutExtension(file); var stoMeta = new StorageItemMeta { Key = key, CreationDate = cTime, ExpirationDate = eTime }; _metas.TryAdd(key, stoMeta); _globalMetas.TryAdd(key, stoMeta); if (idx % 100 == 0) { Core.Log.InfoBasic("Number of files loaded: {0}", idx); } idx++; } }).ConfigureAwait(false); Core.Log.InfoBasic("Index generated..."); } #endregion #region Loading transaction log file var transactionLog = await LoadTransactionFileAsync(_transactionLogFilePath).ConfigureAwait(false); if (transactionLog is null) { transactionLog = await LoadTransactionFileAsync(_oldTransactionLogFilePath).ConfigureAwait(false); } #endregion #region Applying pending transactions if (transactionLog?.Count > 0) { Core.Log.InfoBasic("Applying {0} pending transactions", transactionLog.Count); foreach (var item in transactionLog) { switch (item.Type) { case FileStorageMetaLog.TransactionType.Add: _globalMetas.TryRemove(item.Meta.Key, out _); if (_metas.TryRemove(item.Meta.Key, out var oldAddedMeta) && oldAddedMeta != null) { oldAddedMeta.Dispose(); } _metas.TryAdd(item.Meta.Key, item.Meta); _globalMetas.TryAdd(item.Meta.Key, item.Meta); break; case FileStorageMetaLog.TransactionType.Remove: _globalMetas.TryRemove(item.Meta.Key, out _); if (_metas.TryRemove(item.Meta.Key, out var oldMeta) && oldMeta != null) { oldMeta.Dispose(); } break; } } } #endregion _transactionStream = File.Open(_transactionLogFilePath, FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite); //await SaveMetadataAsync().ConfigureAwait(false); SaveMetadata(); RemoveExpiredItems(false); foreach (var metaItem in _metas) { if (metaItem.Value is null) { continue; } metaItem.Value.OnExpire = Meta_OnExpire; } var metasCount = _metas.Count; Interlocked.Exchange(ref _metasCount, metasCount); Core.Log.InfoBasic("Total item loaded in {0}: {1}", BasePath, metasCount); Status = FolderHandlerStatus.Loaded; } catch (Exception ex) { Core.Log.Write(ex); Status = FolderHandlerStatus.LoadFailed; } }