public static void BeforeEveryTest()
{
TestAssetLocalStorageLmdbCtor.RebuildLocalStorageFolder(DATABASE_FOLDER_PATH, TestStorageManager.WRITE_CACHE_FILE_PATH);
_localStorage = _localStorageLmdb = new AssetLocalStorageLmdb(
_chattelConfigRead,
DATABASE_MAX_SIZE_BYTES
);
}
public static void CleanupAfterEveryTest()
{
_localStorage = null;
IDisposable localStorageDisposal = _localStorageLmdb;
_localStorageLmdb = null;
localStorageDisposal.Dispose();
TestAssetLocalStorageLmdbPartitionedLRUCtor.CleanLocalStorageFolder(DATABASE_FOLDER_PATH, TestStorageManager.WRITE_CACHE_FILE_PATH);
}
public static void BeforeEveryTest()
{
TestAssetLocalStorageLmdbPartitionedLRUCtor.RebuildLocalStorageFolder(DATABASE_FOLDER_PATH, TestStorageManager.WRITE_CACHE_FILE_PATH);
_localStorage = _localStorageLmdb = new AssetLocalStorageLmdbPartitionedLRU(
_chattelConfigRead,
DATABASE_MAX_SIZE_BYTES,
TestAssetLocalStorageLmdbPartitionedLRUCtor.DATABASE_PARTITION_INTERVAL
);
}
/// <summary>
/// Purges all assets that have the "local" flag set from both disk and memory.
/// Never touches the remotes.
/// </summary>
public void PurgeAllLocalAssets()
{
IChattelLocalStorage chattelStorage = _localStorage;
chattelStorage.PurgeAll(new List <AssetFilter> {
new AssetFilter {
LocalFilter = true,
}
});
}
public void Init()
{
// Configure Log4Net
XmlConfigurator.Configure(new FileInfo(Constants.LOG_CONFIG_PATH));
// Set CWD so that native libs are found.
Directory.SetCurrentDirectory(TestContext.CurrentContext.TestDirectory);
// Load INI stuff
var configSource = new ArgvConfigSource(new string[] { });
// Configure nIni aliases and locale
Thread.CurrentThread.CurrentCulture = new System.Globalization.CultureInfo("en-US", true);
configSource.Alias.AddAlias("On", true);
configSource.Alias.AddAlias("Off", false);
configSource.Alias.AddAlias("True", true);
configSource.Alias.AddAlias("False", false);
configSource.Alias.AddAlias("Yes", true);
configSource.Alias.AddAlias("No", false);
// Read in the ini file
configSource.Merge(new IniConfigSource(Constants.INI_PATH));
// Prep cache folder
try {
Directory.Delete(Constants.TEST_CACHE_PATH, true);
}
catch (DirectoryNotFoundException) {
// Skip.
}
Directory.CreateDirectory(Constants.TEST_CACHE_PATH);
// Start booting server
var pidFileManager = new PIDFileManager(Constants.PID_FILE_PATH);
var chattelConfigRead = new ChattelConfiguration(Constants.TEST_CACHE_PATH);
LocalStorage = new AssetStorageSimpleFolderTree(chattelConfigRead);
var chattelReader = new ChattelReader(chattelConfigRead, LocalStorage);
_service = new F_Stop(
Constants.SERVICE_URI,
Constants.SERVICE_ADMIN_TOKEN,
TimeSpan.FromSeconds(Constants.SERVICE_NC_LIFETIME_SECONDS),
chattelReader,
new List <sbyte> {
0, 12, /*18, 19,*/ 49
}
);
_service.Start();
}
/// <summary>
/// Initializes a new instance of the <see cref="T:ChattelReader"/> class.
/// If local storage is enabled, but no local storage instance was passed in, automatically sets up and uses the AssetStorageSimpleFolderTree for local storage.
/// If purgeLocalStorage is set, purges all assets in the storage.
/// </summary>
/// <param name="config">Instance of the configuration class.</param>
/// <param name="localStorage">Instance of the IChattelLocalStorage interface. If left null, then the default AssetStorageSimpleFolderTree will be instantiated.</param>
/// <param name="purgeLocalStorage">Whether or not to attempt to purge local storage.</param>
public ChattelReader(ChattelConfiguration config, IChattelLocalStorage localStorage, bool purgeLocalStorage)
{
_config = config ?? throw new ArgumentNullException(nameof(config));
if (_config.LocalStorageEnabled)
{
_localStorage = localStorage ?? new AssetStorageSimpleFolderTree(config);
}
if (purgeLocalStorage)
{
_localStorage?.PurgeAll(null);
}
}
/// <summary>
/// Initializes a new instance of the <see cref="T:ChattelWriter"/> class.
/// </summary>
/// <param name="config">Instance of the configuration class.</param>
/// <param name="localStorage">Instance of the IChattelLocalStorage interface. If left null, then the default AssetStorageSimpleFolderTree will be instantiated.</param>
/// <param name="purgeLocalStorage">Whether or not to attempt to purge local storage.</param>
/// <exception cref="!:ChattelConfigurationException">Thrown if the are pending assets to be sent upstream and there are no upstream servers configured.</exception>
public ChattelWriter(ChattelConfiguration config, IChattelLocalStorage localStorage, bool purgeLocalStorage)
{
_config = config ?? throw new ArgumentNullException(nameof(config));
if (config.LocalStorageEnabled)
{
_localStorage = localStorage ?? new AssetStorageSimpleFolderTree(config);
}
if (purgeLocalStorage)
{
_localStorage?.PurgeAll(null);
}
if (config.LocalStorageEnabled && config.WriteCacheFile != null)
{
_writeCache = new WriteCache(config.WriteCacheFile, config.WriteCacheRecordCount, this, localStorage);
}
}
public static void TestAssetLocalStorageLmdb_Ctor2_RestoresIndex()
{
var asset = new StratusAsset {
Id = Guid.NewGuid(),
};
using (var localStorage = new AssetLocalStorageLmdb(
_chattelConfigRead,
DATABASE_MAX_SIZE_BYTES
)) {
IChattelLocalStorage localStorageViaInterface = localStorage;
localStorageViaInterface.StoreAsset(asset);
}
using (var localStorage = new AssetLocalStorageLmdb(
_chattelConfigRead,
DATABASE_MAX_SIZE_BYTES
)) {
Assert.True(localStorage.Contains(asset.Id));
}
}
/// <summary>
/// Purges the given asset from disk and memory.
/// Never touches the remotes.
/// </summary>
/// <param name="assetId">Asset identifier.</param>
/// <param name="resultCallback">Result callback.</param>
public void PurgeAsset(Guid assetId, PurgeResultCallback resultCallback)
{
if (assetId == Guid.Empty)
{
throw new ArgumentException("Asset Id should not be empty.", nameof(assetId));
}
IChattelLocalStorage chattelStorage = _localStorage;
var result = PurgeResult.NOT_FOUND_LOCALLY;
try {
chattelStorage.Purge(assetId);
result = PurgeResult.DONE;
}
catch (AssetNotFoundException) {
// Nothing to do here.
}
resultCallback(result);
}
/// <summary>
/// Opens or creates the write cache file. If there are entries in the file that are marked as not uploaded, then
/// this ctor loads those assets from the local storage and uploads them to the remotes passed in via the ChattelWriter instance.
/// </summary>
/// <param name="fileInfo">FileInfo instance for the path where to load or create the write cache file.</param>
/// <param name="recordCount">Record count to set the write cache to.</param>
/// <param name="writer">ChattelWriter instance for uploading un-finished assets to on load.</param>
/// <param name="localStorage">Local storage instace to load unfinished assets from.</param>
/// <exception cref="T:Chattel.ChattelConfigurationException">Thrown if there are assets marked as needing to be uploaded but the current configuration prevents uploading.</exception>
public WriteCache(FileInfo fileInfo, uint recordCount, ChattelWriter writer, IChattelLocalStorage localStorage)
{
_fileInfo = fileInfo ?? throw new ArgumentNullException(nameof(fileInfo));
if (recordCount < 2)
{
throw new ArgumentOutOfRangeException(nameof(recordCount), "Having less than two record makes no sense and causes errors.");
}
// If the file doesn't exist, create it and zero the needed records.
if (!_fileInfo.Exists)
{
LOG.Log(Logging.LogLevel.Info, () => $"Write cache file doesn't exist, creating and formatting file '{_fileInfo.FullName}'");
Initialize(recordCount);
_fileInfo.Refresh();
LOG.Log(Logging.LogLevel.Debug, () => $"Write cache formatting complete.");
}
var writeCacheFileRecordCount = (uint)((_fileInfo.Length - WRITE_CACHE_MAGIC_NUMBER.Length) / WriteCacheNode.BYTE_SIZE);
if (writeCacheFileRecordCount < recordCount)
{
// Expand the file.
Expand(recordCount - writeCacheFileRecordCount);
}
else if (writeCacheFileRecordCount > recordCount)
{
// For now, use the file size.
LOG.Log(Logging.LogLevel.Warn, () => $"Write cache not able to be shrunk in this version of Chattel, continuing with old value of {writeCacheFileRecordCount} records instead of requested {recordCount} records.");
recordCount = writeCacheFileRecordCount;
// TODO: find a way to shrink the file without losing ANY of the records that have not yet been submitted to an upstream server.
// Could get difficult in the case of a full file...
}
LOG.Log(Logging.LogLevel.Info, () => $"Reading write cache from file '{_fileInfo.FullName}'. Expecting {recordCount} records, found {writeCacheFileRecordCount} records, choosing the larger.");
_writeCacheNodes = Read(out IEnumerable <WriteCacheNode> assetsToBeSentUpstream).ToArray();
LOG.Log(Logging.LogLevel.Debug, () => $"Reading write cache complete.");
if (assetsToBeSentUpstream.Any())
{
if (writer == null)
{
throw new ChattelConfigurationException("Write cache indicates assets needing to be sent to remote servers, but there is no asset writer!");
}
if (localStorage == null)
{
throw new ChattelConfigurationException("Write cache indicates assets needing to be sent to remote servers, but there no cache to read them from!");
}
if (!writer.HasUpstream)
{
throw new ChattelConfigurationException("Write cache indicates assets needing to be sent to remote servers, but there are no remote servers configured!");
}
}
// Send the assets to the remote server. Yes do this in the startup thread: if you can't access the servers, then why continue?
foreach (var assetCacheNode in assetsToBeSentUpstream)
{
LOG.Log(Logging.LogLevel.Debug, () => $"Attempting to remotely store {assetCacheNode.AssetId}.");
if (localStorage.TryGetAsset(assetCacheNode.AssetId, out var asset))
{
try {
writer.PutAssetSync(asset);
}
catch (AssetExistsException) {
// Ignore these.
LOG.Log(Logging.LogLevel.Info, () => $"Remote server reports that the asset with ID {assetCacheNode.AssetId} already exists.");
}
ClearNode(assetCacheNode);
}
else
{
LOG.Log(Logging.LogLevel.Warn, () => $"Write cache indicates asset {assetCacheNode.AssetId} has not been sent upstream, but the cache reports that there's no such asset!.");
}
}
// Bootstrap the system.
GetNextAvailableNode();
}
/// <summary>
/// Initializes a new instance of the <see cref="T:ChattelReader"/> class.
/// If local storage is enabled, but no local storage instance was passed in, automatically sets up and uses the AssetStorageSimpleFolderTree for local storage.
/// </summary>
/// <param name="config">Instance of the configuration class.</param>
/// <param name="localStorage">Instance of the IChattelLocalStorage interface. If left null, then the default AssetStorageSimpleFolderTree will be instantiated.</param>
public ChattelReader(ChattelConfiguration config, IChattelLocalStorage localStorage)
: this(config, localStorage, false)
{
}
