public static void TestAssetLocalStorageLmdbPartitionedLRU_TryGetAsset_EmpyId_ArgumentException() { Assert.Throws <ArgumentException>(() => _localStorage.TryGetAsset(Guid.Empty, out var assetResult)); }
/// <summary> /// Gets the asset from the server. /// </summary> /// <returns>The asset.</returns> /// <param name="assetId">Asset identifier.</param> /// <param name="handler">Callback delegate to hand the asset to.</param> /// <param name="cacheRule">Bitfield controlling how local storage is to be handled when used as a cache for remote servers.</param> public void GetAssetAsync(Guid assetId, AssetHandler handler, CacheRule cacheRule) { // Ask for null, get null. if (assetId == Guid.Empty) { handler(null); } // TODO: see if https://github.com/Reactive-Extensions/Rx.NET would do a better job, but they have to finish releasing 4.0 first. // It might be beneficial to move the listener processsing to another thread, but then you potentially lose parallism across multiple asset IDs. StratusAsset result = null; while (true) { // Hit up the local storage first. If there's no upstream then ignore skipread. if (!(cacheRule.HasFlag(CacheRule.SkipRead) && _config.SerialParallelAssetServers.Any()) && (_localStorage?.TryGetAsset(assetId, out result) ?? false)) { handler(result); return; } var listeners = new Queue <AssetHandler>(); listeners.Enqueue(handler); // Add myself to the new listeners list first thing, assuming, probably wrongly, that the following test is true. If wrong, meh: this queue gets dropped like an old potato. if (_idsBeingFetched.TryAdd(assetId, listeners)) { // Got to go try the servers now. foreach (var parallelServers in _config.SerialParallelAssetServers) { if (parallelServers.Count() == 1) // Optimization: no need to hit up the parallel stuff if there's only 1. { result = parallelServers.First().RequestAssetSync(assetId); } else { result = parallelServers.AsParallel().Select(server => server.RequestAssetSync(assetId)).FirstOrDefault(a => a != null); } if (result != null) { if (!cacheRule.HasFlag(CacheRule.SkipWrite)) { _localStorage?.StoreAsset(result); } break; } } // Now to process the listeners. var exceptions = new ConcurrentQueue <Exception>(); lock (listeners) { // Prevent new listeners from being added. Parallel.ForEach(listeners, waiting_handler => { if (waiting_handler == null) { LOG.Log(Logging.LogLevel.Warn, () => $"Attempted to process a handler for assetId {assetId} that was null!"); return; } try { waiting_handler(result); } catch (Exception e) { exceptions.Enqueue(e); } }); _idsBeingFetched.TryRemove(assetId, out var trash); } if (exceptions.Count > 0) { LOG.Log(Logging.LogLevel.Error, () => $"Exceptions ({exceptions.Count}) were thrown by handler(s) listening for asset {assetId}", new AggregateException(exceptions)); } return; // We're done here. } // See if we can add ourselves to the listener list. if (_idsBeingFetched.TryGetValue(assetId, out listeners)) { // Skiplock: if the lock cannot be taken, move on to the retry because the list is already being emptied. var lockTaken = false; try { Monitor.TryEnter(listeners, ref lockTaken); if (lockTaken) { listeners.Enqueue(handler); return; } } finally { if (lockTaken) { Monitor.Exit(listeners); } } // lock was skipped, therefore that list is already being cleaned out. } // It's gone already, so let's try again as the asset should be in local storage or we should query the servers again. Thread.Sleep(50); } }
/// <summary> /// Opens or creates the write cache file. If there are entries in the file that are marked as not uploaded, then /// this ctor loads those assets from the local storage and uploads them to the remotes passed in via the ChattelWriter instance. /// </summary> /// <param name="fileInfo">FileInfo instance for the path where to load or create the write cache file.</param> /// <param name="recordCount">Record count to set the write cache to.</param> /// <param name="writer">ChattelWriter instance for uploading un-finished assets to on load.</param> /// <param name="localStorage">Local storage instace to load unfinished assets from.</param> /// <exception cref="T:Chattel.ChattelConfigurationException">Thrown if there are assets marked as needing to be uploaded but the current configuration prevents uploading.</exception> public WriteCache(FileInfo fileInfo, uint recordCount, ChattelWriter writer, IChattelLocalStorage localStorage) { _fileInfo = fileInfo ?? throw new ArgumentNullException(nameof(fileInfo)); if (recordCount < 2) { throw new ArgumentOutOfRangeException(nameof(recordCount), "Having less than two record makes no sense and causes errors."); } // If the file doesn't exist, create it and zero the needed records. if (!_fileInfo.Exists) { LOG.Log(Logging.LogLevel.Info, () => $"Write cache file doesn't exist, creating and formatting file '{_fileInfo.FullName}'"); Initialize(recordCount); _fileInfo.Refresh(); LOG.Log(Logging.LogLevel.Debug, () => $"Write cache formatting complete."); } var writeCacheFileRecordCount = (uint)((_fileInfo.Length - WRITE_CACHE_MAGIC_NUMBER.Length) / WriteCacheNode.BYTE_SIZE); if (writeCacheFileRecordCount < recordCount) { // Expand the file. Expand(recordCount - writeCacheFileRecordCount); } else if (writeCacheFileRecordCount > recordCount) { // For now, use the file size. LOG.Log(Logging.LogLevel.Warn, () => $"Write cache not able to be shrunk in this version of Chattel, continuing with old value of {writeCacheFileRecordCount} records instead of requested {recordCount} records."); recordCount = writeCacheFileRecordCount; // TODO: find a way to shrink the file without losing ANY of the records that have not yet been submitted to an upstream server. // Could get difficult in the case of a full file... } LOG.Log(Logging.LogLevel.Info, () => $"Reading write cache from file '{_fileInfo.FullName}'. Expecting {recordCount} records, found {writeCacheFileRecordCount} records, choosing the larger."); _writeCacheNodes = Read(out IEnumerable <WriteCacheNode> assetsToBeSentUpstream).ToArray(); LOG.Log(Logging.LogLevel.Debug, () => $"Reading write cache complete."); if (assetsToBeSentUpstream.Any()) { if (writer == null) { throw new ChattelConfigurationException("Write cache indicates assets needing to be sent to remote servers, but there is no asset writer!"); } if (localStorage == null) { throw new ChattelConfigurationException("Write cache indicates assets needing to be sent to remote servers, but there no cache to read them from!"); } if (!writer.HasUpstream) { throw new ChattelConfigurationException("Write cache indicates assets needing to be sent to remote servers, but there are no remote servers configured!"); } } // Send the assets to the remote server. Yes do this in the startup thread: if you can't access the servers, then why continue? foreach (var assetCacheNode in assetsToBeSentUpstream) { LOG.Log(Logging.LogLevel.Debug, () => $"Attempting to remotely store {assetCacheNode.AssetId}."); if (localStorage.TryGetAsset(assetCacheNode.AssetId, out var asset)) { try { writer.PutAssetSync(asset); } catch (AssetExistsException) { // Ignore these. LOG.Log(Logging.LogLevel.Info, () => $"Remote server reports that the asset with ID {assetCacheNode.AssetId} already exists."); } ClearNode(assetCacheNode); } else { LOG.Log(Logging.LogLevel.Warn, () => $"Write cache indicates asset {assetCacheNode.AssetId} has not been sent upstream, but the cache reports that there's no such asset!."); } } // Bootstrap the system. GetNextAvailableNode(); }
/// <summary> /// Sends the asset to the asset servers. /// Throws AssetExistsException or AggregateException. /// </summary> /// <param name="asset">The asset to store.</param> public void PutAssetSync(StratusAsset asset) { asset = asset ?? throw new ArgumentNullException(nameof(asset)); if (asset.Id == Guid.Empty) { throw new ArgumentException("Asset cannot have zero ID.", nameof(asset)); } // Handle parallel calls with the same asset ID. var firstLock = new ReaderWriterLockSlim(); try { firstLock.EnterWriteLock(); var activeLock = _activeWriteLocks.GetOrAdd(asset.Id, firstLock); if (firstLock != activeLock) { LOG.Log(Logging.LogLevel.Warn, () => $"Another thread already storing asset with ID {asset.Id}, halting this call until the first completes, then just returning."); // There's another thread currently adding this exact ID, so we need to wait on it so that we return when it's actually ready for a GET. activeLock.EnterReadLock(); activeLock.ExitReadLock(); return; } // Hit up local storage first. if (_localStorage?.TryGetAsset(asset.Id, out var result) ?? false) { _activeWriteLocks.TryRemove(asset.Id, out var lockObj); // Lock is cleared in the finally clause. throw new AssetExistsException(asset.Id); } var exceptions = new List <Exception>(); var success = false; WriteCacheNode activeNode = null; // First step: get it in local storage. try { _localStorage?.StoreAsset(asset); if (HasUpstream) { // Write to writecache file. In this way if we crash after this point we can recover and send the asset to the servers. activeNode = _writeCache.WriteNode(asset); // If that fails, it'll throw. } else { // Set success if there're no upstream servers. This supports applications that act as asset servers. success = true; } } catch (WriteCacheFullException e) { LOG.Log(Logging.LogLevel.Warn, () => $"Local cache is full, attempting remote servers before failing.", e); exceptions.Add(e); } catch (Exception e) { exceptions.Add(e); } // Got to go try the servers now. foreach (var parallelServers in _config.SerialParallelAssetServers) { // Remember each iteration of this loop is going through serially accessed blocks of parallel-access servers. // Therefore any failure or problem in one of the blocks means to just continue to the next block. try { if (parallelServers.Count() == 1) { parallelServers.First().StoreAssetSync(asset); } else { var errorBag = new ConcurrentBag <Exception>(); Parallel.ForEach(parallelServers, server => { try { server.StoreAssetSync(asset); } catch (Exception e) { errorBag.Add(e); } }); if (errorBag.Count >= parallelServers.Count()) { // If all the servers choked, then pass the buck. throw new AggregateException(errorBag); } } if (activeNode != null) { _writeCache.ClearNode(activeNode); } success = true; break; // It was successfully stored in the first bank of parallel servers, don't do the next bank. } catch (AssetException e) { exceptions.Add(e); } catch (AggregateException e) { // Unwind the aggregate one layer. foreach (var ex in e.InnerExceptions) { exceptions.Add(ex); } } catch (Exception e) { exceptions.Add(e); } } if (!success) { throw new AggregateException("Unable to store asset anywhere. See inner exceptions for details.", exceptions); } } finally { _activeWriteLocks.TryRemove(asset.Id, out var lockObj); firstLock.ExitWriteLock(); } }