private WriteCacheNode GetNextAvailableNode()
{
WriteCacheNode writeCacheNode;
void updateToNextNode()
{
if (_nextAvailableWriteCacheNode == null)
{
try {
_nextAvailableWriteCacheNode = _writeCacheNodes.First(node => node.IsAvailable);
}
catch (InvalidOperationException) {
// No available nodes found, which means we are out of ability to safely continue until one becomes available...
_nextAvailableWriteCacheNode = null;
throw new WriteCacheFullException("All write cache nodes are used!");
}
}
}
lock (_writeCacheNodeLock) {
// If we've not bootstrapped, do so.
updateToNextNode();
writeCacheNode = _nextAvailableWriteCacheNode;
writeCacheNode.IsAvailable = false;
_nextAvailableWriteCacheNode = null;
// Find the next one.
updateToNextNode();
}
return(writeCacheNode);
}
/// <summary>
/// Marks the passed in node as completely uploaded to a remote server and available for reuse, both in memory and on disk.
/// </summary>
/// <param name="node">Node.</param>
public void ClearNode(WriteCacheNode node)
{
node = node ?? throw new ArgumentNullException(nameof(node));
// Clear the byte on disk before clearing in memory.
using (var mmf = MemoryMappedFile.CreateFromFile(_fileInfo.FullName, FileMode.Open))
using (var accessor = mmf.CreateViewAccessor((long)node.FileOffset, WriteCacheNode.BYTE_SIZE)) {
accessor.Write(0, (byte)0);
}
node.IsAvailable = true;
}
private IEnumerable <WriteCacheNode> Read(out IEnumerable <WriteCacheNode> assetsToSend)
{
var fileRecordCount = (int)(_fileInfo.Length / WriteCacheNode.BYTE_SIZE);
var assetsToSendOut = new List <WriteCacheNode>();
using (var mmf = MemoryMappedFile.CreateFromFile(_fileInfo.FullName, FileMode.Open)) {
using (var stream = mmf.CreateViewStream()) {
var offset = 0UL;
{
var magic_number = new byte[WRITE_CACHE_MAGIC_NUMBER.Length];
stream.Read(magic_number, 0, WRITE_CACHE_MAGIC_NUMBER.Length);
if (!magic_number.SequenceEqual(WRITE_CACHE_MAGIC_NUMBER))
{
throw new InvalidDataException($"Magic number mismatch when given path: {_fileInfo.FullName}");
}
offset += (ulong)WRITE_CACHE_MAGIC_NUMBER.Length;
}
var nodes = new List <WriteCacheNode>(fileRecordCount);
var buffer = new byte[WriteCacheNode.BYTE_SIZE];
while (nodes.Count < fileRecordCount)
{
stream.Read(buffer, 0, (int)WriteCacheNode.BYTE_SIZE);
var node = new WriteCacheNode(buffer, offset);
nodes.Add(node);
offset += WriteCacheNode.BYTE_SIZE;
// If the node isn't available that means it's an ID that still needs to be written to long-term storage.
if (!node.IsAvailable)
{
assetsToSendOut.Add(node);
}
}
assetsToSend = assetsToSendOut;
return(nodes);
}
}
}
/// <summary>
/// Sends the asset to the asset servers.
/// Throws AssetExistsException or AggregateException.
/// </summary>
/// <param name="asset">The asset to store.</param>
public void PutAssetSync(StratusAsset asset)
{
asset = asset ?? throw new ArgumentNullException(nameof(asset));
if (asset.Id == Guid.Empty)
{
throw new ArgumentException("Asset cannot have zero ID.", nameof(asset));
}
// Handle parallel calls with the same asset ID.
var firstLock = new ReaderWriterLockSlim();
try {
firstLock.EnterWriteLock();
var activeLock = _activeWriteLocks.GetOrAdd(asset.Id, firstLock);
if (firstLock != activeLock)
{
LOG.Log(Logging.LogLevel.Warn, () => $"Another thread already storing asset with ID {asset.Id}, halting this call until the first completes, then just returning.");
// There's another thread currently adding this exact ID, so we need to wait on it so that we return when it's actually ready for a GET.
activeLock.EnterReadLock();
activeLock.ExitReadLock();
return;
}
// Hit up local storage first.
if (_localStorage?.TryGetAsset(asset.Id, out var result) ?? false)
{
_activeWriteLocks.TryRemove(asset.Id, out var lockObj);
// Lock is cleared in the finally clause.
throw new AssetExistsException(asset.Id);
}
var exceptions = new List <Exception>();
var success = false;
WriteCacheNode activeNode = null;
// First step: get it in local storage.
try {
_localStorage?.StoreAsset(asset);
if (HasUpstream)
{
// Write to writecache file. In this way if we crash after this point we can recover and send the asset to the servers.
activeNode = _writeCache.WriteNode(asset);
// If that fails, it'll throw.
}
else
{
// Set success if there're no upstream servers. This supports applications that act as asset servers.
success = true;
}
}
catch (WriteCacheFullException e) {
LOG.Log(Logging.LogLevel.Warn, () => $"Local cache is full, attempting remote servers before failing.", e);
exceptions.Add(e);
}
catch (Exception e) {
exceptions.Add(e);
}
// Got to go try the servers now.
foreach (var parallelServers in _config.SerialParallelAssetServers)
{
// Remember each iteration of this loop is going through serially accessed blocks of parallel-access servers.
// Therefore any failure or problem in one of the blocks means to just continue to the next block.
try {
if (parallelServers.Count() == 1)
{
parallelServers.First().StoreAssetSync(asset);
}
else
{
var errorBag = new ConcurrentBag <Exception>();
Parallel.ForEach(parallelServers, server => {
try {
server.StoreAssetSync(asset);
}
catch (Exception e) {
errorBag.Add(e);
}
});
if (errorBag.Count >= parallelServers.Count())
{
// If all the servers choked, then pass the buck.
throw new AggregateException(errorBag);
}
}
if (activeNode != null)
{
_writeCache.ClearNode(activeNode);
}
success = true;
break; // It was successfully stored in the first bank of parallel servers, don't do the next bank.
}
catch (AssetException e) {
exceptions.Add(e);
}
catch (AggregateException e) {
// Unwind the aggregate one layer.
foreach (var ex in e.InnerExceptions)
{
exceptions.Add(ex);
}
}
catch (Exception e) {
exceptions.Add(e);
}
}
if (!success)
{
throw new AggregateException("Unable to store asset anywhere. See inner exceptions for details.", exceptions);
}
}
finally {
_activeWriteLocks.TryRemove(asset.Id, out var lockObj);
firstLock.ExitWriteLock();
}
}