public static void TestAssetLocalStorageLmdbPartitionedLRU_TryGetAsset_EmpyId_ArgumentException()
{
Assert.Throws <ArgumentException>(() => _localStorage.TryGetAsset(Guid.Empty, out var assetResult));
}
/// <summary>
/// Gets the asset from the server.
/// </summary>
/// <returns>The asset.</returns>
/// <param name="assetId">Asset identifier.</param>
/// <param name="handler">Callback delegate to hand the asset to.</param>
/// <param name="cacheRule">Bitfield controlling how local storage is to be handled when used as a cache for remote servers.</param>
public void GetAssetAsync(Guid assetId, AssetHandler handler, CacheRule cacheRule)
{
// Ask for null, get null.
if (assetId == Guid.Empty)
{
handler(null);
}
// TODO: see if https://github.com/Reactive-Extensions/Rx.NET would do a better job, but they have to finish releasing 4.0 first.
// It might be beneficial to move the listener processsing to another thread, but then you potentially lose parallism across multiple asset IDs.
StratusAsset result = null;
while (true)
{
// Hit up the local storage first. If there's no upstream then ignore skipread.
if (!(cacheRule.HasFlag(CacheRule.SkipRead) && _config.SerialParallelAssetServers.Any()) && (_localStorage?.TryGetAsset(assetId, out result) ?? false))
{
handler(result);
return;
}
var listeners = new Queue <AssetHandler>();
listeners.Enqueue(handler); // Add myself to the new listeners list first thing, assuming, probably wrongly, that the following test is true. If wrong, meh: this queue gets dropped like an old potato.
if (_idsBeingFetched.TryAdd(assetId, listeners))
{
// Got to go try the servers now.
foreach (var parallelServers in _config.SerialParallelAssetServers)
{
if (parallelServers.Count() == 1) // Optimization: no need to hit up the parallel stuff if there's only 1.
{
result = parallelServers.First().RequestAssetSync(assetId);
}
else
{
result = parallelServers.AsParallel().Select(server => server.RequestAssetSync(assetId)).FirstOrDefault(a => a != null);
}
if (result != null)
{
if (!cacheRule.HasFlag(CacheRule.SkipWrite))
{
_localStorage?.StoreAsset(result);
}
break;
}
}
// Now to process the listeners.
var exceptions = new ConcurrentQueue <Exception>();
lock (listeners) { // Prevent new listeners from being added.
Parallel.ForEach(listeners, waiting_handler => {
if (waiting_handler == null)
{
LOG.Log(Logging.LogLevel.Warn, () => $"Attempted to process a handler for assetId {assetId} that was null!");
return;
}
try {
waiting_handler(result);
}
catch (Exception e) {
exceptions.Enqueue(e);
}
});
_idsBeingFetched.TryRemove(assetId, out var trash);
}
if (exceptions.Count > 0)
{
LOG.Log(Logging.LogLevel.Error, () => $"Exceptions ({exceptions.Count}) were thrown by handler(s) listening for asset {assetId}", new AggregateException(exceptions));
}
return; // We're done here.
}
// See if we can add ourselves to the listener list.
if (_idsBeingFetched.TryGetValue(assetId, out listeners))
{
// Skiplock: if the lock cannot be taken, move on to the retry because the list is already being emptied.
var lockTaken = false;
try {
Monitor.TryEnter(listeners, ref lockTaken);
if (lockTaken)
{
listeners.Enqueue(handler);
return;
}
}
finally {
if (lockTaken)
{
Monitor.Exit(listeners);
}
}
// lock was skipped, therefore that list is already being cleaned out.
}
// It's gone already, so let's try again as the asset should be in local storage or we should query the servers again.
Thread.Sleep(50);
}
}
/// <summary>
/// Opens or creates the write cache file. If there are entries in the file that are marked as not uploaded, then
/// this ctor loads those assets from the local storage and uploads them to the remotes passed in via the ChattelWriter instance.
/// </summary>
/// <param name="fileInfo">FileInfo instance for the path where to load or create the write cache file.</param>
/// <param name="recordCount">Record count to set the write cache to.</param>
/// <param name="writer">ChattelWriter instance for uploading un-finished assets to on load.</param>
/// <param name="localStorage">Local storage instace to load unfinished assets from.</param>
/// <exception cref="T:Chattel.ChattelConfigurationException">Thrown if there are assets marked as needing to be uploaded but the current configuration prevents uploading.</exception>
public WriteCache(FileInfo fileInfo, uint recordCount, ChattelWriter writer, IChattelLocalStorage localStorage)
{
_fileInfo = fileInfo ?? throw new ArgumentNullException(nameof(fileInfo));
if (recordCount < 2)
{
throw new ArgumentOutOfRangeException(nameof(recordCount), "Having less than two record makes no sense and causes errors.");
}
// If the file doesn't exist, create it and zero the needed records.
if (!_fileInfo.Exists)
{
LOG.Log(Logging.LogLevel.Info, () => $"Write cache file doesn't exist, creating and formatting file '{_fileInfo.FullName}'");
Initialize(recordCount);
_fileInfo.Refresh();
LOG.Log(Logging.LogLevel.Debug, () => $"Write cache formatting complete.");
}
var writeCacheFileRecordCount = (uint)((_fileInfo.Length - WRITE_CACHE_MAGIC_NUMBER.Length) / WriteCacheNode.BYTE_SIZE);
if (writeCacheFileRecordCount < recordCount)
{
// Expand the file.
Expand(recordCount - writeCacheFileRecordCount);
}
else if (writeCacheFileRecordCount > recordCount)
{
// For now, use the file size.
LOG.Log(Logging.LogLevel.Warn, () => $"Write cache not able to be shrunk in this version of Chattel, continuing with old value of {writeCacheFileRecordCount} records instead of requested {recordCount} records.");
recordCount = writeCacheFileRecordCount;
// TODO: find a way to shrink the file without losing ANY of the records that have not yet been submitted to an upstream server.
// Could get difficult in the case of a full file...
}
LOG.Log(Logging.LogLevel.Info, () => $"Reading write cache from file '{_fileInfo.FullName}'. Expecting {recordCount} records, found {writeCacheFileRecordCount} records, choosing the larger.");
_writeCacheNodes = Read(out IEnumerable <WriteCacheNode> assetsToBeSentUpstream).ToArray();
LOG.Log(Logging.LogLevel.Debug, () => $"Reading write cache complete.");
if (assetsToBeSentUpstream.Any())
{
if (writer == null)
{
throw new ChattelConfigurationException("Write cache indicates assets needing to be sent to remote servers, but there is no asset writer!");
}
if (localStorage == null)
{
throw new ChattelConfigurationException("Write cache indicates assets needing to be sent to remote servers, but there no cache to read them from!");
}
if (!writer.HasUpstream)
{
throw new ChattelConfigurationException("Write cache indicates assets needing to be sent to remote servers, but there are no remote servers configured!");
}
}
// Send the assets to the remote server. Yes do this in the startup thread: if you can't access the servers, then why continue?
foreach (var assetCacheNode in assetsToBeSentUpstream)
{
LOG.Log(Logging.LogLevel.Debug, () => $"Attempting to remotely store {assetCacheNode.AssetId}.");
if (localStorage.TryGetAsset(assetCacheNode.AssetId, out var asset))
{
try {
writer.PutAssetSync(asset);
}
catch (AssetExistsException) {
// Ignore these.
LOG.Log(Logging.LogLevel.Info, () => $"Remote server reports that the asset with ID {assetCacheNode.AssetId} already exists.");
}
ClearNode(assetCacheNode);
}
else
{
LOG.Log(Logging.LogLevel.Warn, () => $"Write cache indicates asset {assetCacheNode.AssetId} has not been sent upstream, but the cache reports that there's no such asset!.");
}
}
// Bootstrap the system.
GetNextAvailableNode();
}
/// <summary>
/// Sends the asset to the asset servers.
/// Throws AssetExistsException or AggregateException.
/// </summary>
/// <param name="asset">The asset to store.</param>
public void PutAssetSync(StratusAsset asset)
{
asset = asset ?? throw new ArgumentNullException(nameof(asset));
if (asset.Id == Guid.Empty)
{
throw new ArgumentException("Asset cannot have zero ID.", nameof(asset));
}
// Handle parallel calls with the same asset ID.
var firstLock = new ReaderWriterLockSlim();
try {
firstLock.EnterWriteLock();
var activeLock = _activeWriteLocks.GetOrAdd(asset.Id, firstLock);
if (firstLock != activeLock)
{
LOG.Log(Logging.LogLevel.Warn, () => $"Another thread already storing asset with ID {asset.Id}, halting this call until the first completes, then just returning.");
// There's another thread currently adding this exact ID, so we need to wait on it so that we return when it's actually ready for a GET.
activeLock.EnterReadLock();
activeLock.ExitReadLock();
return;
}
// Hit up local storage first.
if (_localStorage?.TryGetAsset(asset.Id, out var result) ?? false)
{
_activeWriteLocks.TryRemove(asset.Id, out var lockObj);
// Lock is cleared in the finally clause.
throw new AssetExistsException(asset.Id);
}
var exceptions = new List <Exception>();
var success = false;
WriteCacheNode activeNode = null;
// First step: get it in local storage.
try {
_localStorage?.StoreAsset(asset);
if (HasUpstream)
{
// Write to writecache file. In this way if we crash after this point we can recover and send the asset to the servers.
activeNode = _writeCache.WriteNode(asset);
// If that fails, it'll throw.
}
else
{
// Set success if there're no upstream servers. This supports applications that act as asset servers.
success = true;
}
}
catch (WriteCacheFullException e) {
LOG.Log(Logging.LogLevel.Warn, () => $"Local cache is full, attempting remote servers before failing.", e);
exceptions.Add(e);
}
catch (Exception e) {
exceptions.Add(e);
}
// Got to go try the servers now.
foreach (var parallelServers in _config.SerialParallelAssetServers)
{
// Remember each iteration of this loop is going through serially accessed blocks of parallel-access servers.
// Therefore any failure or problem in one of the blocks means to just continue to the next block.
try {
if (parallelServers.Count() == 1)
{
parallelServers.First().StoreAssetSync(asset);
}
else
{
var errorBag = new ConcurrentBag <Exception>();
Parallel.ForEach(parallelServers, server => {
try {
server.StoreAssetSync(asset);
}
catch (Exception e) {
errorBag.Add(e);
}
});
if (errorBag.Count >= parallelServers.Count())
{
// If all the servers choked, then pass the buck.
throw new AggregateException(errorBag);
}
}
if (activeNode != null)
{
_writeCache.ClearNode(activeNode);
}
success = true;
break; // It was successfully stored in the first bank of parallel servers, don't do the next bank.
}
catch (AssetException e) {
exceptions.Add(e);
}
catch (AggregateException e) {
// Unwind the aggregate one layer.
foreach (var ex in e.InnerExceptions)
{
exceptions.Add(ex);
}
}
catch (Exception e) {
exceptions.Add(e);
}
}
if (!success)
{
throw new AggregateException("Unable to store asset anywhere. See inner exceptions for details.", exceptions);
}
}
finally {
_activeWriteLocks.TryRemove(asset.Id, out var lockObj);
firstLock.ExitWriteLock();
}
}
