public static void TestAssetLocalStorageLmdb_Ctor2_RestoresIndex()
{
var asset = new StratusAsset {
Id = Guid.NewGuid(),
};
using (var localStorage = new AssetLocalStorageLmdb(
_chattelConfigRead,
DATABASE_MAX_SIZE_BYTES
)) {
IChattelLocalStorage localStorageViaInterface = localStorage;
localStorageViaInterface.StoreAsset(asset);
}
using (var localStorage = new AssetLocalStorageLmdb(
_chattelConfigRead,
DATABASE_MAX_SIZE_BYTES
)) {
Assert.True(localStorage.Contains(asset.Id));
}
}
public static void TestAssetLocalStorageLmdbPartitionedLRU_AssetOnDisk_KnownTwoPartitions_True()
{
var assetId1 = Guid.NewGuid();
var assetId2 = Guid.NewGuid();
_localStorage.StoreAsset(new StratusAsset {
Id = assetId1,
});
Thread.Sleep(1100);
_localStorage.StoreAsset(new StratusAsset {
Id = assetId2,
});
Assert.True(_localStorageLmdb.AssetOnDisk(assetId1));
Assert.True(_localStorageLmdb.AssetOnDisk(assetId2));
}
public static void TestAssetLocalStorageLmdbCtor2_StoreAsset_AssetOnDiskImmediately()
{
var asset = new StratusAsset {
Id = Guid.NewGuid(),
};
_localStorage.StoreAsset(asset);
Assert.True(_localStorageLmdb.AssetOnDisk(asset.Id));
}
/// <summary>
/// Gets the asset from the server.
/// </summary>
/// <returns>The asset.</returns>
/// <param name="assetId">Asset identifier.</param>
/// <param name="handler">Callback delegate to hand the asset to.</param>
/// <param name="cacheRule">Bitfield controlling how local storage is to be handled when used as a cache for remote servers.</param>
public void GetAssetAsync(Guid assetId, AssetHandler handler, CacheRule cacheRule)
{
// Ask for null, get null.
if (assetId == Guid.Empty)
{
handler(null);
}
// TODO: see if https://github.com/Reactive-Extensions/Rx.NET would do a better job, but they have to finish releasing 4.0 first.
// It might be beneficial to move the listener processsing to another thread, but then you potentially lose parallism across multiple asset IDs.
StratusAsset result = null;
while (true)
{
// Hit up the local storage first. If there's no upstream then ignore skipread.
if (!(cacheRule.HasFlag(CacheRule.SkipRead) && _config.SerialParallelAssetServers.Any()) && (_localStorage?.TryGetAsset(assetId, out result) ?? false))
{
handler(result);
return;
}
var listeners = new Queue <AssetHandler>();
listeners.Enqueue(handler); // Add myself to the new listeners list first thing, assuming, probably wrongly, that the following test is true. If wrong, meh: this queue gets dropped like an old potato.
if (_idsBeingFetched.TryAdd(assetId, listeners))
{
// Got to go try the servers now.
foreach (var parallelServers in _config.SerialParallelAssetServers)
{
if (parallelServers.Count() == 1) // Optimization: no need to hit up the parallel stuff if there's only 1.
{
result = parallelServers.First().RequestAssetSync(assetId);
}
else
{
result = parallelServers.AsParallel().Select(server => server.RequestAssetSync(assetId)).FirstOrDefault(a => a != null);
}
if (result != null)
{
if (!cacheRule.HasFlag(CacheRule.SkipWrite))
{
_localStorage?.StoreAsset(result);
}
break;
}
}
// Now to process the listeners.
var exceptions = new ConcurrentQueue <Exception>();
lock (listeners) { // Prevent new listeners from being added.
Parallel.ForEach(listeners, waiting_handler => {
if (waiting_handler == null)
{
LOG.Log(Logging.LogLevel.Warn, () => $"Attempted to process a handler for assetId {assetId} that was null!");
return;
}
try {
waiting_handler(result);
}
catch (Exception e) {
exceptions.Enqueue(e);
}
});
_idsBeingFetched.TryRemove(assetId, out var trash);
}
if (exceptions.Count > 0)
{
LOG.Log(Logging.LogLevel.Error, () => $"Exceptions ({exceptions.Count}) were thrown by handler(s) listening for asset {assetId}", new AggregateException(exceptions));
}
return; // We're done here.
}
// See if we can add ourselves to the listener list.
if (_idsBeingFetched.TryGetValue(assetId, out listeners))
{
// Skiplock: if the lock cannot be taken, move on to the retry because the list is already being emptied.
var lockTaken = false;
try {
Monitor.TryEnter(listeners, ref lockTaken);
if (lockTaken)
{
listeners.Enqueue(handler);
return;
}
}
finally {
if (lockTaken)
{
Monitor.Exit(listeners);
}
}
// lock was skipped, therefore that list is already being cleaned out.
}
// It's gone already, so let's try again as the asset should be in local storage or we should query the servers again.
Thread.Sleep(50);
}
}
/// <summary>
/// Sends the asset to the asset servers.
/// Throws AssetExistsException or AggregateException.
/// </summary>
/// <param name="asset">The asset to store.</param>
public void PutAssetSync(StratusAsset asset)
{
asset = asset ?? throw new ArgumentNullException(nameof(asset));
if (asset.Id == Guid.Empty)
{
throw new ArgumentException("Asset cannot have zero ID.", nameof(asset));
}
// Handle parallel calls with the same asset ID.
var firstLock = new ReaderWriterLockSlim();
try {
firstLock.EnterWriteLock();
var activeLock = _activeWriteLocks.GetOrAdd(asset.Id, firstLock);
if (firstLock != activeLock)
{
LOG.Log(Logging.LogLevel.Warn, () => $"Another thread already storing asset with ID {asset.Id}, halting this call until the first completes, then just returning.");
// There's another thread currently adding this exact ID, so we need to wait on it so that we return when it's actually ready for a GET.
activeLock.EnterReadLock();
activeLock.ExitReadLock();
return;
}
// Hit up local storage first.
if (_localStorage?.TryGetAsset(asset.Id, out var result) ?? false)
{
_activeWriteLocks.TryRemove(asset.Id, out var lockObj);
// Lock is cleared in the finally clause.
throw new AssetExistsException(asset.Id);
}
var exceptions = new List <Exception>();
var success = false;
WriteCacheNode activeNode = null;
// First step: get it in local storage.
try {
_localStorage?.StoreAsset(asset);
if (HasUpstream)
{
// Write to writecache file. In this way if we crash after this point we can recover and send the asset to the servers.
activeNode = _writeCache.WriteNode(asset);
// If that fails, it'll throw.
}
else
{
// Set success if there're no upstream servers. This supports applications that act as asset servers.
success = true;
}
}
catch (WriteCacheFullException e) {
LOG.Log(Logging.LogLevel.Warn, () => $"Local cache is full, attempting remote servers before failing.", e);
exceptions.Add(e);
}
catch (Exception e) {
exceptions.Add(e);
}
// Got to go try the servers now.
foreach (var parallelServers in _config.SerialParallelAssetServers)
{
// Remember each iteration of this loop is going through serially accessed blocks of parallel-access servers.
// Therefore any failure or problem in one of the blocks means to just continue to the next block.
try {
if (parallelServers.Count() == 1)
{
parallelServers.First().StoreAssetSync(asset);
}
else
{
var errorBag = new ConcurrentBag <Exception>();
Parallel.ForEach(parallelServers, server => {
try {
server.StoreAssetSync(asset);
}
catch (Exception e) {
errorBag.Add(e);
}
});
if (errorBag.Count >= parallelServers.Count())
{
// If all the servers choked, then pass the buck.
throw new AggregateException(errorBag);
}
}
if (activeNode != null)
{
_writeCache.ClearNode(activeNode);
}
success = true;
break; // It was successfully stored in the first bank of parallel servers, don't do the next bank.
}
catch (AssetException e) {
exceptions.Add(e);
}
catch (AggregateException e) {
// Unwind the aggregate one layer.
foreach (var ex in e.InnerExceptions)
{
exceptions.Add(ex);
}
}
catch (Exception e) {
exceptions.Add(e);
}
}
if (!success)
{
throw new AggregateException("Unable to store asset anywhere. See inner exceptions for details.", exceptions);
}
}
finally {
_activeWriteLocks.TryRemove(asset.Id, out var lockObj);
firstLock.ExitWriteLock();
}
}