示例#1
0
        private WriteCacheNode GetNextAvailableNode()
        {
            WriteCacheNode writeCacheNode;

            void updateToNextNode()
            {
                if (_nextAvailableWriteCacheNode == null)
                {
                    try {
                        _nextAvailableWriteCacheNode = _writeCacheNodes.First(node => node.IsAvailable);
                    }
                    catch (InvalidOperationException) {
                        // No available nodes found, which means we are out of ability to safely continue until one becomes available...
                        _nextAvailableWriteCacheNode = null;
                        throw new WriteCacheFullException("All write cache nodes are used!");
                    }
                }
            }

            lock (_writeCacheNodeLock) {
                // If we've not bootstrapped, do so.
                updateToNextNode();

                writeCacheNode               = _nextAvailableWriteCacheNode;
                writeCacheNode.IsAvailable   = false;
                _nextAvailableWriteCacheNode = null;

                // Find the next one.
                updateToNextNode();
            }

            return(writeCacheNode);
        }
示例#2
0
        /// <summary>
        /// Marks the passed in node as completely uploaded to a remote server and available for reuse, both in memory and on disk.
        /// </summary>
        /// <param name="node">Node.</param>
        public void ClearNode(WriteCacheNode node)
        {
            node = node ?? throw new ArgumentNullException(nameof(node));

            // Clear the byte on disk before clearing in memory.
            using (var mmf = MemoryMappedFile.CreateFromFile(_fileInfo.FullName, FileMode.Open))
                using (var accessor = mmf.CreateViewAccessor((long)node.FileOffset, WriteCacheNode.BYTE_SIZE)) {
                    accessor.Write(0, (byte)0);
                }
            node.IsAvailable = true;
        }
示例#3
0
        private IEnumerable <WriteCacheNode> Read(out IEnumerable <WriteCacheNode> assetsToSend)
        {
            var fileRecordCount = (int)(_fileInfo.Length / WriteCacheNode.BYTE_SIZE);

            var assetsToSendOut = new List <WriteCacheNode>();

            using (var mmf = MemoryMappedFile.CreateFromFile(_fileInfo.FullName, FileMode.Open)) {
                using (var stream = mmf.CreateViewStream()) {
                    var offset = 0UL;
                    {
                        var magic_number = new byte[WRITE_CACHE_MAGIC_NUMBER.Length];
                        stream.Read(magic_number, 0, WRITE_CACHE_MAGIC_NUMBER.Length);
                        if (!magic_number.SequenceEqual(WRITE_CACHE_MAGIC_NUMBER))
                        {
                            throw new InvalidDataException($"Magic number mismatch when given path: {_fileInfo.FullName}");
                        }
                        offset += (ulong)WRITE_CACHE_MAGIC_NUMBER.Length;
                    }

                    var nodes = new List <WriteCacheNode>(fileRecordCount);

                    var buffer = new byte[WriteCacheNode.BYTE_SIZE];

                    while (nodes.Count < fileRecordCount)
                    {
                        stream.Read(buffer, 0, (int)WriteCacheNode.BYTE_SIZE);
                        var node = new WriteCacheNode(buffer, offset);
                        nodes.Add(node);
                        offset += WriteCacheNode.BYTE_SIZE;

                        // If the node isn't available that means it's an ID that still needs to be written to long-term storage.
                        if (!node.IsAvailable)
                        {
                            assetsToSendOut.Add(node);
                        }
                    }

                    assetsToSend = assetsToSendOut;
                    return(nodes);
                }
            }
        }
示例#4
0
        /// <summary>
        /// Sends the asset to the asset servers.
        /// Throws AssetExistsException or AggregateException.
        /// </summary>
        /// <param name="asset">The asset to store.</param>
        public void PutAssetSync(StratusAsset asset)
        {
            asset = asset ?? throw new ArgumentNullException(nameof(asset));

            if (asset.Id == Guid.Empty)
            {
                throw new ArgumentException("Asset cannot have zero ID.", nameof(asset));
            }

            // Handle parallel calls with the same asset ID.
            var firstLock = new ReaderWriterLockSlim();

            try {
                firstLock.EnterWriteLock();

                var activeLock = _activeWriteLocks.GetOrAdd(asset.Id, firstLock);
                if (firstLock != activeLock)
                {
                    LOG.Log(Logging.LogLevel.Warn, () => $"Another thread already storing asset with ID {asset.Id}, halting this call until the first completes, then just returning.");
                    // There's another thread currently adding this exact ID, so we need to wait on it so that we return when it's actually ready for a GET.
                    activeLock.EnterReadLock();
                    activeLock.ExitReadLock();
                    return;
                }

                // Hit up local storage first.
                if (_localStorage?.TryGetAsset(asset.Id, out var result) ?? false)
                {
                    _activeWriteLocks.TryRemove(asset.Id, out var lockObj);
                    // Lock is cleared in the finally clause.
                    throw new AssetExistsException(asset.Id);
                }

                var            exceptions = new List <Exception>();
                var            success    = false;
                WriteCacheNode activeNode = null;

                // First step: get it in local storage.
                try {
                    _localStorage?.StoreAsset(asset);

                    if (HasUpstream)
                    {
                        // Write to writecache file. In this way if we crash after this point we can recover and send the asset to the servers.
                        activeNode = _writeCache.WriteNode(asset);
                        // If that fails, it'll throw.
                    }
                    else
                    {
                        // Set success if there're no upstream servers. This supports applications that act as asset servers.
                        success = true;
                    }
                }
                catch (WriteCacheFullException e) {
                    LOG.Log(Logging.LogLevel.Warn, () => $"Local cache is full, attempting remote servers before failing.", e);
                    exceptions.Add(e);
                }
                catch (Exception e) {
                    exceptions.Add(e);
                }

                // Got to go try the servers now.
                foreach (var parallelServers in _config.SerialParallelAssetServers)
                {
                    // Remember each iteration of this loop is going through serially accessed blocks of parallel-access servers.
                    // Therefore any failure or problem in one of the blocks means to just continue to the next block.
                    try {
                        if (parallelServers.Count() == 1)
                        {
                            parallelServers.First().StoreAssetSync(asset);
                        }
                        else
                        {
                            var errorBag = new ConcurrentBag <Exception>();

                            Parallel.ForEach(parallelServers, server => {
                                try {
                                    server.StoreAssetSync(asset);
                                }
                                catch (Exception e) {
                                    errorBag.Add(e);
                                }
                            });

                            if (errorBag.Count >= parallelServers.Count())
                            {
                                // If all the servers choked, then pass the buck.
                                throw new AggregateException(errorBag);
                            }
                        }

                        if (activeNode != null)
                        {
                            _writeCache.ClearNode(activeNode);
                        }

                        success = true;
                        break;                         // It was successfully stored in the first bank of parallel servers, don't do the next bank.
                    }
                    catch (AssetException e) {
                        exceptions.Add(e);
                    }
                    catch (AggregateException e) {
                        // Unwind the aggregate one layer.
                        foreach (var ex in e.InnerExceptions)
                        {
                            exceptions.Add(ex);
                        }
                    }
                    catch (Exception e) {
                        exceptions.Add(e);
                    }
                }

                if (!success)
                {
                    throw new AggregateException("Unable to store asset anywhere. See inner exceptions for details.", exceptions);
                }
            }
            finally {
                _activeWriteLocks.TryRemove(asset.Id, out var lockObj);
                firstLock.ExitWriteLock();
            }
        }