public static void BeforeEveryTest() { TestAssetLocalStorageLmdbCtor.RebuildLocalStorageFolder(DATABASE_FOLDER_PATH, TestStorageManager.WRITE_CACHE_FILE_PATH); _localStorage = _localStorageLmdb = new AssetLocalStorageLmdb( _chattelConfigRead, DATABASE_MAX_SIZE_BYTES ); }
public static void CleanupAfterEveryTest() { _localStorage = null; IDisposable localStorageDisposal = _localStorageLmdb; _localStorageLmdb = null; localStorageDisposal.Dispose(); TestAssetLocalStorageLmdbPartitionedLRUCtor.CleanLocalStorageFolder(DATABASE_FOLDER_PATH, TestStorageManager.WRITE_CACHE_FILE_PATH); }
public static void BeforeEveryTest() { TestAssetLocalStorageLmdbPartitionedLRUCtor.RebuildLocalStorageFolder(DATABASE_FOLDER_PATH, TestStorageManager.WRITE_CACHE_FILE_PATH); _localStorage = _localStorageLmdb = new AssetLocalStorageLmdbPartitionedLRU( _chattelConfigRead, DATABASE_MAX_SIZE_BYTES, TestAssetLocalStorageLmdbPartitionedLRUCtor.DATABASE_PARTITION_INTERVAL ); }
/// <summary> /// Purges all assets that have the "local" flag set from both disk and memory. /// Never touches the remotes. /// </summary> public void PurgeAllLocalAssets() { IChattelLocalStorage chattelStorage = _localStorage; chattelStorage.PurgeAll(new List <AssetFilter> { new AssetFilter { LocalFilter = true, } }); }
public void Init() { // Configure Log4Net XmlConfigurator.Configure(new FileInfo(Constants.LOG_CONFIG_PATH)); // Set CWD so that native libs are found. Directory.SetCurrentDirectory(TestContext.CurrentContext.TestDirectory); // Load INI stuff var configSource = new ArgvConfigSource(new string[] { }); // Configure nIni aliases and locale Thread.CurrentThread.CurrentCulture = new System.Globalization.CultureInfo("en-US", true); configSource.Alias.AddAlias("On", true); configSource.Alias.AddAlias("Off", false); configSource.Alias.AddAlias("True", true); configSource.Alias.AddAlias("False", false); configSource.Alias.AddAlias("Yes", true); configSource.Alias.AddAlias("No", false); // Read in the ini file configSource.Merge(new IniConfigSource(Constants.INI_PATH)); // Prep cache folder try { Directory.Delete(Constants.TEST_CACHE_PATH, true); } catch (DirectoryNotFoundException) { // Skip. } Directory.CreateDirectory(Constants.TEST_CACHE_PATH); // Start booting server var pidFileManager = new PIDFileManager(Constants.PID_FILE_PATH); var chattelConfigRead = new ChattelConfiguration(Constants.TEST_CACHE_PATH); LocalStorage = new AssetStorageSimpleFolderTree(chattelConfigRead); var chattelReader = new ChattelReader(chattelConfigRead, LocalStorage); _service = new F_Stop( Constants.SERVICE_URI, Constants.SERVICE_ADMIN_TOKEN, TimeSpan.FromSeconds(Constants.SERVICE_NC_LIFETIME_SECONDS), chattelReader, new List <sbyte> { 0, 12, /*18, 19,*/ 49 } ); _service.Start(); }
/// <summary> /// Initializes a new instance of the <see cref="T:ChattelReader"/> class. /// If local storage is enabled, but no local storage instance was passed in, automatically sets up and uses the AssetStorageSimpleFolderTree for local storage. /// If purgeLocalStorage is set, purges all assets in the storage. /// </summary> /// <param name="config">Instance of the configuration class.</param> /// <param name="localStorage">Instance of the IChattelLocalStorage interface. If left null, then the default AssetStorageSimpleFolderTree will be instantiated.</param> /// <param name="purgeLocalStorage">Whether or not to attempt to purge local storage.</param> public ChattelReader(ChattelConfiguration config, IChattelLocalStorage localStorage, bool purgeLocalStorage) { _config = config ?? throw new ArgumentNullException(nameof(config)); if (_config.LocalStorageEnabled) { _localStorage = localStorage ?? new AssetStorageSimpleFolderTree(config); } if (purgeLocalStorage) { _localStorage?.PurgeAll(null); } }
/// <summary> /// Initializes a new instance of the <see cref="T:ChattelWriter"/> class. /// </summary> /// <param name="config">Instance of the configuration class.</param> /// <param name="localStorage">Instance of the IChattelLocalStorage interface. If left null, then the default AssetStorageSimpleFolderTree will be instantiated.</param> /// <param name="purgeLocalStorage">Whether or not to attempt to purge local storage.</param> /// <exception cref="!:ChattelConfigurationException">Thrown if the are pending assets to be sent upstream and there are no upstream servers configured.</exception> public ChattelWriter(ChattelConfiguration config, IChattelLocalStorage localStorage, bool purgeLocalStorage) { _config = config ?? throw new ArgumentNullException(nameof(config)); if (config.LocalStorageEnabled) { _localStorage = localStorage ?? new AssetStorageSimpleFolderTree(config); } if (purgeLocalStorage) { _localStorage?.PurgeAll(null); } if (config.LocalStorageEnabled && config.WriteCacheFile != null) { _writeCache = new WriteCache(config.WriteCacheFile, config.WriteCacheRecordCount, this, localStorage); } }
public static void TestAssetLocalStorageLmdb_Ctor2_RestoresIndex() { var asset = new StratusAsset { Id = Guid.NewGuid(), }; using (var localStorage = new AssetLocalStorageLmdb( _chattelConfigRead, DATABASE_MAX_SIZE_BYTES )) { IChattelLocalStorage localStorageViaInterface = localStorage; localStorageViaInterface.StoreAsset(asset); } using (var localStorage = new AssetLocalStorageLmdb( _chattelConfigRead, DATABASE_MAX_SIZE_BYTES )) { Assert.True(localStorage.Contains(asset.Id)); } }
/// <summary> /// Purges the given asset from disk and memory. /// Never touches the remotes. /// </summary> /// <param name="assetId">Asset identifier.</param> /// <param name="resultCallback">Result callback.</param> public void PurgeAsset(Guid assetId, PurgeResultCallback resultCallback) { if (assetId == Guid.Empty) { throw new ArgumentException("Asset Id should not be empty.", nameof(assetId)); } IChattelLocalStorage chattelStorage = _localStorage; var result = PurgeResult.NOT_FOUND_LOCALLY; try { chattelStorage.Purge(assetId); result = PurgeResult.DONE; } catch (AssetNotFoundException) { // Nothing to do here. } resultCallback(result); }
/// <summary> /// Opens or creates the write cache file. If there are entries in the file that are marked as not uploaded, then /// this ctor loads those assets from the local storage and uploads them to the remotes passed in via the ChattelWriter instance. /// </summary> /// <param name="fileInfo">FileInfo instance for the path where to load or create the write cache file.</param> /// <param name="recordCount">Record count to set the write cache to.</param> /// <param name="writer">ChattelWriter instance for uploading un-finished assets to on load.</param> /// <param name="localStorage">Local storage instace to load unfinished assets from.</param> /// <exception cref="T:Chattel.ChattelConfigurationException">Thrown if there are assets marked as needing to be uploaded but the current configuration prevents uploading.</exception> public WriteCache(FileInfo fileInfo, uint recordCount, ChattelWriter writer, IChattelLocalStorage localStorage) { _fileInfo = fileInfo ?? throw new ArgumentNullException(nameof(fileInfo)); if (recordCount < 2) { throw new ArgumentOutOfRangeException(nameof(recordCount), "Having less than two record makes no sense and causes errors."); } // If the file doesn't exist, create it and zero the needed records. if (!_fileInfo.Exists) { LOG.Log(Logging.LogLevel.Info, () => $"Write cache file doesn't exist, creating and formatting file '{_fileInfo.FullName}'"); Initialize(recordCount); _fileInfo.Refresh(); LOG.Log(Logging.LogLevel.Debug, () => $"Write cache formatting complete."); } var writeCacheFileRecordCount = (uint)((_fileInfo.Length - WRITE_CACHE_MAGIC_NUMBER.Length) / WriteCacheNode.BYTE_SIZE); if (writeCacheFileRecordCount < recordCount) { // Expand the file. Expand(recordCount - writeCacheFileRecordCount); } else if (writeCacheFileRecordCount > recordCount) { // For now, use the file size. LOG.Log(Logging.LogLevel.Warn, () => $"Write cache not able to be shrunk in this version of Chattel, continuing with old value of {writeCacheFileRecordCount} records instead of requested {recordCount} records."); recordCount = writeCacheFileRecordCount; // TODO: find a way to shrink the file without losing ANY of the records that have not yet been submitted to an upstream server. // Could get difficult in the case of a full file... } LOG.Log(Logging.LogLevel.Info, () => $"Reading write cache from file '{_fileInfo.FullName}'. Expecting {recordCount} records, found {writeCacheFileRecordCount} records, choosing the larger."); _writeCacheNodes = Read(out IEnumerable <WriteCacheNode> assetsToBeSentUpstream).ToArray(); LOG.Log(Logging.LogLevel.Debug, () => $"Reading write cache complete."); if (assetsToBeSentUpstream.Any()) { if (writer == null) { throw new ChattelConfigurationException("Write cache indicates assets needing to be sent to remote servers, but there is no asset writer!"); } if (localStorage == null) { throw new ChattelConfigurationException("Write cache indicates assets needing to be sent to remote servers, but there no cache to read them from!"); } if (!writer.HasUpstream) { throw new ChattelConfigurationException("Write cache indicates assets needing to be sent to remote servers, but there are no remote servers configured!"); } } // Send the assets to the remote server. Yes do this in the startup thread: if you can't access the servers, then why continue? foreach (var assetCacheNode in assetsToBeSentUpstream) { LOG.Log(Logging.LogLevel.Debug, () => $"Attempting to remotely store {assetCacheNode.AssetId}."); if (localStorage.TryGetAsset(assetCacheNode.AssetId, out var asset)) { try { writer.PutAssetSync(asset); } catch (AssetExistsException) { // Ignore these. LOG.Log(Logging.LogLevel.Info, () => $"Remote server reports that the asset with ID {assetCacheNode.AssetId} already exists."); } ClearNode(assetCacheNode); } else { LOG.Log(Logging.LogLevel.Warn, () => $"Write cache indicates asset {assetCacheNode.AssetId} has not been sent upstream, but the cache reports that there's no such asset!."); } } // Bootstrap the system. GetNextAvailableNode(); }
/// <summary> /// Initializes a new instance of the <see cref="T:ChattelReader"/> class. /// If local storage is enabled, but no local storage instance was passed in, automatically sets up and uses the AssetStorageSimpleFolderTree for local storage. /// </summary> /// <param name="config">Instance of the configuration class.</param> /// <param name="localStorage">Instance of the IChattelLocalStorage interface. If left null, then the default AssetStorageSimpleFolderTree will be instantiated.</param> public ChattelReader(ChattelConfiguration config, IChattelLocalStorage localStorage) : this(config, localStorage, false) { }