Пример #1
0
        public void MessageUpload()
        {
            _callback.Invoke("----- CoreManager Message Upload Test (Start) -----");
            _callback.Invoke("");

            var random = RandomProvider.GetThreadRandom();

            _coreManager.Resize((long)1024 * 1024 * 1024 * 32).Wait();

            Metadata metadata = null;
            Hash     hash;

            using (var stream = new RecyclableMemoryStream(_bufferManager))
            {
                using (var safeBuffer = _bufferManager.CreateSafeBuffer(1024 * 4))
                {
                    for (long remain = (long)1024 * 1024 * 256; 0 < remain; remain -= safeBuffer.Value.Length)
                    {
                        int length = (int)Math.Min(remain, safeBuffer.Value.Length);

                        random.NextBytes(safeBuffer.Value);
                        stream.Write(safeBuffer.Value, 0, length);
                    }
                }

                stream.Seek(0, SeekOrigin.Begin);
                hash = new Hash(HashAlgorithm.Sha256, Sha256.Compute(new WrapperStream(stream)));

                stream.Seek(0, SeekOrigin.Begin);
                metadata = _coreManager.VolatileSetStream(stream, new TimeSpan(1, 0, 0, 0), CancellationToken.None).Result;
            }

            using (var stream = metadata.Export(_bufferManager))
                using (var safeBuffer = _bufferManager.CreateSafeBuffer((int)stream.Length))
                {
                    stream.Read(safeBuffer.Value, 0, (int)stream.Length);
                    Console.WriteLine(NetworkConverter.ToBase64UrlString(safeBuffer.Value, 0, (int)stream.Length));
                }

            using (var stream = hash.Export(_bufferManager))
                using (var safeBuffer = _bufferManager.CreateSafeBuffer((int)stream.Length))
                {
                    stream.Read(safeBuffer.Value, 0, (int)stream.Length);
                    Console.WriteLine(NetworkConverter.ToBase64UrlString(safeBuffer.Value, 0, (int)stream.Length));
                }

            _callback.Invoke("----- CoreManager Message Upload Test (End) -----");
        }
Пример #2
0
        public static BasicHeader Upgrade(RecyclableMemoryStream target)
        {
            var type = (PacketType)target.ReadByte();

            target.Seek(-1, SeekOrigin.Current);
            BasicHeader upgraded;

            switch (type)
            {
            case PacketType.ContentPacket:
                upgraded = new ContentHeader();
                break;

            case PacketType.RawData:
                upgraded = new RawDataHeader();
                break;

            case PacketType.Trackable:
                upgraded = new TrackableHeader();
                break;

            default:
                upgraded = new BasicHeader();
                break;
            }

            return(upgraded.Deserialize(target));
        }
Пример #3
0
    private async ValueTask <IMemoryOwner <byte> > GetMovieImageAsync(string path, int width, int height, ThumbnailResizeType resizeType, ThumbnailFormatType formatType, CancellationToken cancellationToken = default)
    {
        try
        {
            var arguments = $"-loglevel error -i \"{path}\" -vf thumbnail=30 -frames:v 1 -f image2 pipe:1";

            using var process = Process.Start(new ProcessStartInfo("ffmpeg", arguments)
            {
                CreateNoWindow         = true,
                UseShellExecute        = false,
                RedirectStandardOutput = true,
                RedirectStandardError  = false,
            });

            using var baseStream = process !.StandardOutput.BaseStream;

            using var inStream  = new RecyclableMemoryStream(_bytesPool);
            using var outStream = new RecyclableMemoryStream(_bytesPool);

            await baseStream.CopyToAsync(inStream, cancellationToken);

            inStream.Seek(0, SeekOrigin.Begin);
            await process.WaitForExitAsync(cancellationToken);

            this.ConvertImage(inStream, outStream, width, height, resizeType, formatType);

            return(outStream.ToMemoryOwner());
        }
        catch (Exception e)
        {
            _logger.Warn(e);
            throw;
        }
    }
        public async Task <UpsertOutcome> UpsertAsync(ResourceWrapper resource, WeakETag weakETag, bool allowCreate, bool keepHistory, CancellationToken cancellationToken)
        {
            int?eTag = weakETag == null
                ? (int?)null
                : (int.TryParse(weakETag.VersionId, out var parsedETag) ? parsedETag : -1); // Set the etag to a sentinel value to enable expected failure paths when updating with both existing and nonexistent resources.

            var resourceMetadata = new ResourceMetadata(
                resource.CompartmentIndices,
                resource.SearchIndices?.ToLookup(e => _searchParameterTypeMap.GetSearchValueType(e)),
                resource.LastModifiedClaims);

            using (SqlConnectionWrapper sqlConnectionWrapper = await _sqlConnectionWrapperFactory.ObtainSqlConnectionWrapperAsync(cancellationToken, true))
                using (SqlCommandWrapper sqlCommandWrapper = sqlConnectionWrapper.CreateSqlCommand())
                    using (var stream = new RecyclableMemoryStream(_memoryStreamManager))
                    {
                        CompressedRawResourceConverter.WriteCompressedRawResource(stream, resource.RawResource.Data);

                        stream.Seek(0, 0);

                        PopulateUpsertResourceCommand(sqlCommandWrapper, resource, resourceMetadata, allowCreate, keepHistory, eTag, stream);

                        try
                        {
                            var newVersion = (int?)await sqlCommandWrapper.ExecuteScalarAsync(cancellationToken);

                            if (newVersion == null)
                            {
                                // indicates a redundant delete
                                return(null);
                            }

                            resource.Version = newVersion.ToString();

                            return(new UpsertOutcome(resource, newVersion == 1 ? SaveOutcomeType.Created : SaveOutcomeType.Updated));
                        }
                        catch (SqlException e)
                        {
                            switch (e.Number)
                            {
                            case SqlErrorCodes.PreconditionFailed:
                                throw new PreconditionFailedException(string.Format(Core.Resources.ResourceVersionConflict, weakETag?.VersionId));

                            case SqlErrorCodes.NotFound:
                                if (weakETag != null)
                                {
                                    throw new ResourceNotFoundException(string.Format(Core.Resources.ResourceNotFoundByIdAndVersion, resource.ResourceTypeName, resource.ResourceId, weakETag.VersionId));
                                }

                                goto default;

                            case SqlErrorCodes.MethodNotAllowed:
                                throw new MethodNotAllowedException(Core.Resources.ResourceCreationNotAllowed);

                            default:
                                _logger.LogError(e, "Error from SQL database on upsert");
                                throw;
                            }
                        }
                    }
        }
        // Decompress into MemoryStream
        private void Decompress(Stream stream, int decompressedSize, int compressedSize)
        {
            // thanks to ShinAli (https://bitbucket.org/alisci01/xnbdecompressor)
            // default window size for XNB encoded files is 64Kb (need 16 bits to represent it)
            _decompressedStream = RecyclableMemoryManager.Default.GetMemoryStream(decompressedSize);
            long startPos = stream.Position;
            long pos      = startPos;

            while (pos - startPos < compressedSize)
            {
                // the compressed stream is seperated into blocks that will decompress
                // into 32Kb or some other size if specified.
                // normal, 32Kb output blocks will have a short indicating the size
                // of the block before the block starts
                // blocks that have a defined output will be preceded by a byte of value
                // 0xFF (255), then a short indicating the output size and another
                // for the block size
                // all shorts for these cases are encoded in big endian order
                int hi         = stream.ReadByte();
                int lo         = stream.ReadByte();
                int block_size = (hi << 8) | lo;
                int frame_size = 0x8000; // frame size is 32Kb by default
                                         // does this block define a frame size?
                if (hi == 0xFF)
                {
                    hi         = lo;
                    lo         = (byte)stream.ReadByte();
                    frame_size = (hi << 8) | lo;
                    hi         = (byte)stream.ReadByte();
                    lo         = (byte)stream.ReadByte();
                    block_size = (hi << 8) | lo;
                    pos       += 5;
                }
                else
                {
                    pos += 2;
                }

                // either says there is nothing to decode
                if (block_size == 0 || frame_size == 0)
                {
                    break;
                }

                _decoder.Decompress(stream, block_size, _decompressedStream, frame_size);
                pos += block_size;

                // reset the position of the input just incase the bit buffer
                // read in some unused bytes
                stream.Seek(pos, SeekOrigin.Begin);
            }

            if (_decompressedStream.Position != decompressedSize)
            {
                throw new ContentLoadException("Decompression failed.");
            }

            _decompressedStream.Seek(0, SeekOrigin.Begin);
        }
Пример #6
0
        public static T Clone <T>(T item)
        {
            using (var stream = new RecyclableMemoryStream(BufferManager.Instance))
            {
                JsonUtils.Save(stream, item);
                stream.Seek(0, SeekOrigin.Begin);

                return(JsonUtils.Load <T>(stream));
            }
        }
Пример #7
0
        private void Cancel(int id)
        {
            var messageStream = new RecyclableMemoryStream(_bufferManager);
            var writer        = new MessageStreamWriter(messageStream, _bufferManager);

            writer.Write((ulong)AmoebaFunctionType.Cancel);
            writer.Write((ulong)id);

            messageStream.Seek(0, SeekOrigin.Begin);
            _messagingManager.Send(messageStream);
        }
        /// <inheritdoc/>
        public async Task <Stream> OpenReadAsync()
        {
            // Copy to a MemoryStream first because RetriableStreamImpl
            // doesn't support Position.
            Stream blobStream   = (await this.blob.DownloadAsync()).Value.Content;
            var    memoryStream = new RecyclableMemoryStream(this.memoryStreamManager);

            await blobStream.CopyToAsync(memoryStream);

            memoryStream.Seek(0, SeekOrigin.Begin);

            return(memoryStream);
        }
Пример #9
0
        private void EventLoop(CancellationToken token)
        {
            try
            {
                for (; ;)
                {
                    token.ThrowIfCancellationRequested();

                    long length = (long)Varint.GetUInt64(_stream);
                    if (length == 0)
                    {
                        continue;
                    }

                    Stream resultSteram = null;

                    try
                    {
                        resultSteram = new RecyclableMemoryStream(_bufferManager);

                        using (var safeBuffer = _bufferManager.CreateSafeBuffer(1024 * 32))
                        {
                            long remain = length;

                            while (remain > 0)
                            {
                                int readLength = _stream.Read(safeBuffer.Value, 0, (int)Math.Min(remain, safeBuffer.Value.Length));
                                resultSteram.Write(safeBuffer.Value, 0, readLength);

                                remain -= readLength;
                            }
                        }

                        resultSteram.Seek(0, SeekOrigin.Begin);

                        this.OnReceive(resultSteram);
                    }
                    catch (Exception)
                    {
                        if (resultSteram != null)
                        {
                            resultSteram.Dispose();
                            resultSteram = null;
                        }
                    }
                }
            }
            catch (Exception)
            {
            }
        }
Пример #10
0
    private async ValueTask <ThumbnailGeneratorGetThumbnailResult> GetPictureThumbnailAsync(NestedPath filePath, ThumbnailGeneratorGetThumbnailOptions options, CancellationToken cancellationToken = default)
    {
        var ext = filePath.GetExtension().ToLower();

        if (!_pictureTypeExtensionList.Contains(ext))
        {
            return(new ThumbnailGeneratorGetThumbnailResult(ThumbnailGeneratorGetThumbnailResultStatus.Failed));
        }

        try
        {
            var fileLength = await _fileSystem.GetFileSizeAsync(filePath, cancellationToken);

            var fileLastWriteTime = await _fileSystem.GetFileLastWriteTimeAsync(filePath, cancellationToken);

            using (var inStream = await _fileSystem.GetFileStreamAsync(filePath, cancellationToken))
                using (var outStream = new RecyclableMemoryStream(_bytesPool))
                {
                    this.ConvertImage(inStream, outStream, options.Width, options.Height, options.ResizeType, options.FormatType);
                    outStream.Seek(0, SeekOrigin.Begin);

                    var image = outStream.ToMemoryOwner();

                    var fileMeta      = new FileMeta(filePath, (ulong)fileLength, Timestamp.FromDateTime(fileLastWriteTime));
                    var thumbnailMeta = new ThumbnailMeta(options.ResizeType, options.FormatType, (uint)options.Width, (uint)options.Height);
                    var content       = new ThumbnailContent(image);
                    var cache         = new ThumbnailCache(fileMeta, thumbnailMeta, new[] { content });

                    await _thumbnailGeneratorRepository.ThumbnailCaches.InsertAsync(cache);

                    return(new ThumbnailGeneratorGetThumbnailResult(ThumbnailGeneratorGetThumbnailResultStatus.Succeeded, cache.Contents));
                }
        }
        catch (NotSupportedException e)
        {
            _logger.Warn(e);
        }
        catch (OperationCanceledException e)
        {
            _logger.Debug(e);
        }
        catch (Exception e)
        {
            _logger.Error(e);
            throw;
        }

        return(new ThumbnailGeneratorGetThumbnailResult(ThumbnailGeneratorGetThumbnailResultStatus.Failed));
    }
Пример #11
0
    private void ConvertImage(Stream inStream, Stream outStream, int width, int height, ThumbnailResizeType resizeType, ThumbnailFormatType formatType)
    {
        try
        {
            this.InternalImageSharpConvertImage(inStream, outStream, width, height, resizeType, formatType);
        }
        catch (SixLabors.ImageSharp.UnknownImageFormatException)
        {
            using (var bitmapStream = new RecyclableMemoryStream(_bytesPool))
            {
                this.InternalMagickImageConvertImage(inStream, bitmapStream);
                bitmapStream.Seek(0, SeekOrigin.Begin);

                this.InternalImageSharpConvertImage(bitmapStream, outStream, width, height, resizeType, formatType);
            }
        }
    }
Пример #12
0
        public Stream GetCashStream(string signature)
        {
            var bufferManager = BufferManager.Instance;

            var signatureStream = new RecyclableMemoryStream(bufferManager);
            {
                var writer = new MessageStreamWriter(signatureStream, bufferManager);
                writer.Write((ulong)5);
                writer.Write(signature);

                signatureStream.Seek(0, SeekOrigin.Begin);
            }

            var target = new MulticastMetadata(this.Type, this.Tag, this.CreationTime, this.Metadata, (Cash)null, (Certificate)null);

            return(new UniteStream(target.Export(bufferManager), signatureStream));
        }
Пример #13
0
            private Stream GetStream(string url)
            {
                var recyclableMemoryStream = new RecyclableMemoryStream(_bufferManager);

                try
                {
                    using (var client = new HttpClient())
                    {
                        using (var stream = client.GetStreamAsync(url).Result)
                        {
                            if (stream.Length > 1024 * 1024 * 32)
                            {
                                throw new Exception("too large");
                            }

                            using (var safeBuffer = _bufferManager.CreateSafeBuffer(1024 * 4))
                            {
                                int length;

                                while ((length = stream.Read(safeBuffer.Value, 0, safeBuffer.Value.Length)) > 0)
                                {
                                    recyclableMemoryStream.Write(safeBuffer.Value, 0, length);

                                    if (recyclableMemoryStream.Length > 1024 * 1024 * 32)
                                    {
                                        throw new Exception("too large");
                                    }
                                }
                            }

                            recyclableMemoryStream.Seek(0, SeekOrigin.Begin);
                            return(recyclableMemoryStream);
                        }
                    }
                }
                catch (Exception)
                {
                    recyclableMemoryStream.Dispose();

                    throw;
                }
            }
Пример #14
0
        private (int, WaitQueue <ResponseInfo>) Send <TArgument>(AmoebaFunctionType type, TArgument argument)
        {
            int id    = this.CreateId();
            var queue = new WaitQueue <ResponseInfo>();

            _queueMap.Add(id, queue);

            var messageStream = new RecyclableMemoryStream(_bufferManager);
            var writer        = new MessageStreamWriter(messageStream, _bufferManager);

            writer.Write((ulong)type);
            writer.Write((ulong)id);

            Stream valueStream = null;

            if (argument != null)
            {
                try
                {
                    valueStream = new RecyclableMemoryStream(_bufferManager);
                    JsonUtils.Save(valueStream, argument);
                }
                catch (Exception)
                {
                    if (valueStream != null)
                    {
                        valueStream.Dispose();
                        valueStream = null;
                    }

                    throw;
                }
            }

            messageStream.Seek(0, SeekOrigin.Begin);
            _messagingManager.Send(new UniteStream(messageStream, valueStream));

            return(id, queue);
        }
Пример #15
0
            private static Stream Compress(Stream stream)
            {
                if (stream == null)
                {
                    throw new ArgumentNullException(nameof(stream));
                }

                try
                {
                    var dic = new Dictionary <byte, Stream>();

                    try
                    {
                        stream.Seek(0, SeekOrigin.Begin);

                        RecyclableMemoryStream deflateBufferStream = null;

                        try
                        {
                            deflateBufferStream = new RecyclableMemoryStream(_bufferManager);

                            using (var deflateStream = new DeflateStream(deflateBufferStream, CompressionMode.Compress, true))
                                using (var safeBuffer = _bufferManager.CreateSafeBuffer(1024 * 4))
                                {
                                    int length;

                                    while ((length = stream.Read(safeBuffer.Value, 0, safeBuffer.Value.Length)) > 0)
                                    {
                                        deflateStream.Write(safeBuffer.Value, 0, length);
                                    }
                                }

                            deflateBufferStream.Seek(0, SeekOrigin.Begin);

                            dic.Add((byte)ConvertCompressionAlgorithm.Deflate, deflateBufferStream);
                        }
                        catch (Exception)
                        {
                            if (deflateBufferStream != null)
                            {
                                deflateBufferStream.Dispose();
                            }
                        }
                    }
                    catch (Exception)
                    {
                    }

                    dic.Add((byte)ConvertCompressionAlgorithm.None, stream);

                    var list = dic.ToList();

                    list.Sort((x, y) =>
                    {
                        int c = x.Value.Length.CompareTo(y.Value.Length);
                        if (c != 0)
                        {
                            return(c);
                        }

                        return(x.Key.CompareTo(y.Key));
                    });

                    for (int i = 1; i < list.Count; i++)
                    {
                        list[i].Value.Dispose();
                    }

                    var headerStream = new RecyclableMemoryStream(_bufferManager);
                    Varint.SetUInt64(headerStream, list[0].Key);

                    return(new UniteStream(headerStream, list[0].Value));
                }
                catch (Exception ex)
                {
                    if (stream != null)
                    {
                        stream.Dispose();
                    }

                    throw new ArgumentException(ex.Message, ex);
                }
            }
Пример #16
0
            private static Stream Decrypt(Stream stream, ExchangePrivateKey privateKey)
            {
                if (stream == null)
                {
                    throw new ArgumentNullException(nameof(stream));
                }
                if (privateKey == null)
                {
                    throw new ArgumentNullException(nameof(privateKey));
                }

                try
                {
                    int type = (int)Varint.GetUInt64(stream);

                    if (type == (int)ConvertCryptoAlgorithm.Aes256)
                    {
                        byte[] cryptoKey;

                        {
                            int length = (int)Varint.GetUInt64(stream);

                            var encryptedBuffer = new byte[length];
                            if (stream.Read(encryptedBuffer, 0, encryptedBuffer.Length) != encryptedBuffer.Length)
                            {
                                throw new ArgumentException();
                            }

                            cryptoKey = Exchange.Decrypt(privateKey, encryptedBuffer);
                        }

                        var iv = new byte[32];
                        stream.Read(iv, 0, iv.Length);

                        RecyclableMemoryStream outStream = null;

                        try
                        {
                            outStream = new RecyclableMemoryStream(_bufferManager);

                            using (var aes = Aes.Create())
                            {
                                aes.KeySize = 256;
                                aes.Mode    = CipherMode.CBC;
                                aes.Padding = PaddingMode.PKCS7;

                                using (var inStream = new RangeStream(stream, stream.Position, stream.Length - stream.Position, true))
                                    using (var cs = new CryptoStream(inStream, aes.CreateDecryptor(cryptoKey, iv), CryptoStreamMode.Read))
                                        using (var safeBuffer = _bufferManager.CreateSafeBuffer(1024 * 4))
                                        {
                                            int length;

                                            while ((length = cs.Read(safeBuffer.Value, 0, safeBuffer.Value.Length)) > 0)
                                            {
                                                outStream.Write(safeBuffer.Value, 0, length);
                                            }
                                        }
                            }

                            outStream.Seek(0, SeekOrigin.Begin);
                        }
                        catch (Exception)
                        {
                            if (outStream != null)
                            {
                                outStream.Dispose();
                            }

                            throw;
                        }

                        return(outStream);
                    }
                    else
                    {
                        throw new NotSupportedException();
                    }
                }
                catch (Exception e)
                {
                    throw new ArgumentException(e.Message, e);
                }
                finally
                {
                    if (stream != null)
                    {
                        stream.Dispose();
                    }
                }
            }
Пример #17
0
            private static Stream Encrypt(Stream stream, ExchangePublicKey publicKey)
            {
                if (stream == null)
                {
                    throw new ArgumentNullException(nameof(stream));
                }
                if (publicKey == null)
                {
                    throw new ArgumentNullException(nameof(publicKey));
                }

                try
                {
                    RecyclableMemoryStream outStream = null;

                    try
                    {
                        outStream = new RecyclableMemoryStream(_bufferManager);
                        Varint.SetUInt64(outStream, (uint)ConvertCryptoAlgorithm.Aes256);

                        var cryptoKey = new byte[32];
                        var iv        = new byte[32];

                        using (var random = RandomNumberGenerator.Create())
                        {
                            random.GetBytes(cryptoKey);
                            random.GetBytes(iv);
                        }

                        {
                            var encryptedBuffer = Exchange.Encrypt(publicKey, cryptoKey);
                            Varint.SetUInt64(outStream, (uint)encryptedBuffer.Length);
                            outStream.Write(encryptedBuffer, 0, encryptedBuffer.Length);
                        }

                        outStream.Write(iv, 0, iv.Length);

                        using (var aes = Aes.Create())
                        {
                            aes.KeySize = 256;
                            aes.Mode    = CipherMode.CBC;
                            aes.Padding = PaddingMode.PKCS7;

                            using (var inStream = new WrapperStream(stream))
                                using (var cs = new CryptoStream(inStream, aes.CreateEncryptor(cryptoKey, iv), CryptoStreamMode.Read))
                                    using (var safeBuffer = _bufferManager.CreateSafeBuffer(1024 * 4))
                                    {
                                        int length;

                                        while ((length = cs.Read(safeBuffer.Value, 0, safeBuffer.Value.Length)) > 0)
                                        {
                                            outStream.Write(safeBuffer.Value, 0, length);
                                        }
                                    }
                        }

                        outStream.Seek(0, SeekOrigin.Begin);
                    }
                    catch (Exception)
                    {
                        if (outStream != null)
                        {
                            outStream.Dispose();
                        }

                        throw;
                    }

                    return(outStream);
                }
                catch (Exception e)
                {
                    throw new ArgumentException(e.Message, e);
                }
                finally
                {
                    if (stream != null)
                    {
                        stream.Dispose();
                    }
                }
            }
Пример #18
0
            private static Stream Decompress(Stream stream)
            {
                if (stream == null)
                {
                    throw new ArgumentNullException(nameof(stream));
                }

                try
                {
                    stream.Seek(0, SeekOrigin.Begin);

                    int type = (int)Varint.GetUInt64(stream);

                    if (type == (int)ConvertCompressionAlgorithm.None)
                    {
                        return(new RangeStream(stream));
                    }
                    else if (type == (int)ConvertCompressionAlgorithm.Deflate)
                    {
                        RecyclableMemoryStream deflateBufferStream = null;

                        try
                        {
                            deflateBufferStream = new RecyclableMemoryStream(_bufferManager);

                            using (var deflateStream = new DeflateStream(stream, CompressionMode.Decompress))
                                using (var safeBuffer = _bufferManager.CreateSafeBuffer(1024 * 4))
                                {
                                    int length;

                                    while ((length = deflateStream.Read(safeBuffer.Value, 0, safeBuffer.Value.Length)) > 0)
                                    {
                                        deflateBufferStream.Write(safeBuffer.Value, 0, length);

                                        if (deflateBufferStream.Length > 1024 * 1024 * 256)
                                        {
                                            throw new Exception("too large");
                                        }
                                    }
                                }

                            deflateBufferStream.Seek(0, SeekOrigin.Begin);

                            return(deflateBufferStream);
                        }
                        catch (Exception)
                        {
                            if (deflateBufferStream != null)
                            {
                                deflateBufferStream.Dispose();
                            }

                            throw;
                        }
                    }
                    else
                    {
                        throw new ArgumentException("ArgumentException");
                    }
                }
                catch (Exception e)
                {
                    if (stream != null)
                    {
                        stream.Dispose();
                    }

                    throw new ArgumentException(e.Message, e);
                }
            }
Пример #19
0
        private static T FromStream <T>(int version, Stream stream)
            where T : MessageBase <T>
        {
            try
            {
                stream.Seek(0, SeekOrigin.Begin);

                // Check
                {
                    var verifyCrc  = Crc32_Castagnoli.Compute(new RangeStream(stream, 0, stream.Length - 4, true));
                    var orignalCrc = new byte[4];

                    using (var crcStream = new RangeStream(stream, stream.Length - 4, 4, true))
                    {
                        crcStream.Read(orignalCrc, 0, orignalCrc.Length);
                    }

                    if (!Unsafe.Equals(verifyCrc, orignalCrc))
                    {
                        throw new ArgumentException("Crc Error");
                    }
                }

                stream.Seek(0, SeekOrigin.Begin);

                if (version != (int)Varint.GetUInt64(stream))
                {
                    throw new ArgumentException("version");
                }
                int type = (int)Varint.GetUInt64(stream);

                using (var dataStream = new RangeStream(stream, stream.Position, stream.Length - stream.Position - 4, true))
                {
                    if (type == (int)ConvertCompressionAlgorithm.None)
                    {
                        return(MessageBase <T> .Import(dataStream, _bufferManager));
                    }
                    else if (type == (int)ConvertCompressionAlgorithm.Deflate)
                    {
                        using (var deflateBufferStream = new RecyclableMemoryStream(_bufferManager))
                        {
                            using (var deflateStream = new DeflateStream(dataStream, CompressionMode.Decompress, true))
                                using (var safeBuffer = _bufferManager.CreateSafeBuffer(1024 * 4))
                                {
                                    int length;

                                    while ((length = deflateStream.Read(safeBuffer.Value, 0, safeBuffer.Value.Length)) > 0)
                                    {
                                        deflateBufferStream.Write(safeBuffer.Value, 0, length);

                                        if (deflateBufferStream.Length > 1024 * 1024 * 32)
                                        {
                                            throw new Exception("too large");
                                        }
                                    }
                                }

                            deflateBufferStream.Seek(0, SeekOrigin.Begin);

                            return(MessageBase <T> .Import(deflateBufferStream, _bufferManager));
                        }
                    }
                    else
                    {
                        throw new ArgumentException("ArgumentException");
                    }
                }
            }
            catch (Exception e)
            {
                throw new ArgumentException(e.Message, e);
            }
            finally
            {
                if (stream != null)
                {
                    stream.Dispose();
                }
            }
        }
Пример #20
0
        public async Task <UpsertOutcome> UpsertAsync(ResourceWrapper resource, WeakETag weakETag, bool allowCreate, bool keepHistory, CancellationToken cancellationToken)
        {
            await _model.EnsureInitialized();

            int etag = 0;

            if (weakETag != null && !int.TryParse(weakETag.VersionId, out etag))
            {
                throw new ResourceConflictException(weakETag);
            }

            var resourceMetadata = new ResourceMetadata(
                resource.CompartmentIndices,
                resource.SearchIndices?.ToLookup(e => _searchParameterTypeMap.GetSearchValueType(e)),
                resource.LastModifiedClaims);

            using (var connection = new SqlConnection(_configuration.ConnectionString))
            {
                await connection.OpenAsync(cancellationToken);

                using (var command = connection.CreateCommand())
                    using (var stream = new RecyclableMemoryStream(_memoryStreamManager))
                        using (var gzipStream = new GZipStream(stream, CompressionMode.Compress))
                            using (var writer = new StreamWriter(gzipStream, ResourceEncoding))
                            {
                                writer.Write(resource.RawResource.Data);
                                writer.Flush();

                                stream.Seek(0, 0);

                                V1.UpsertResource.PopulateCommand(
                                    command,
                                    resourceTypeId: _model.GetResourceTypeId(resource.ResourceTypeName),
                                    resourceId: resource.ResourceId,
                                    eTag: weakETag == null ? null : (int?)etag,
                                    allowCreate: allowCreate,
                                    isDeleted: resource.IsDeleted,
                                    updatedDateTime: resource.LastModified,
                                    keepHistory: keepHistory,
                                    requestMethod: resource.Request.Method,
                                    rawResource: stream,
                                    tableValuedParameters: _upsertResourceTvpGenerator.Generate(resourceMetadata));

                                try
                                {
                                    var newVersion = (int?)await command.ExecuteScalarAsync(cancellationToken);

                                    if (newVersion == null)
                                    {
                                        // indicates a redundant delete
                                        return(null);
                                    }

                                    resource.Version = newVersion.ToString();

                                    return(new UpsertOutcome(resource, newVersion == 1 ? SaveOutcomeType.Created : SaveOutcomeType.Updated));
                                }
                                catch (SqlException e)
                                {
                                    switch (e.Number)
                                    {
                                    case SqlErrorCodes.NotFound:
                                        throw new MethodNotAllowedException(Resources.ResourceCreationNotAllowed);

                                    case SqlErrorCodes.PreconditionFailed:
                                        throw new ResourceConflictException(weakETag);

                                    default:
                                        _logger.LogError(e, "Error from SQL database on upsert");
                                        throw;
                                    }
                                }
                            }
            }
        }
        public async Task <UpsertOutcome> UpsertAsync(ResourceWrapper resource, WeakETag weakETag, bool allowCreate, bool keepHistory, CancellationToken cancellationToken)
        {
            await _model.EnsureInitialized();

            int etag = 0;

            if (weakETag != null && !int.TryParse(weakETag.VersionId, out etag))
            {
                throw new ResourceConflictException(weakETag);
            }

            var resourceMetadata = new ResourceMetadata(
                resource.CompartmentIndices,
                resource.SearchIndices?.ToLookup(e => _searchParameterTypeMap.GetSearchValueType(e)),
                resource.LastModifiedClaims);

            // Insert an Observation record to CDS
            DynamicsCrmFhirDataStore crmFhirDataStoreObj = new DynamicsCrmFhirDataStore();

            try
            {
                // Dynamics Crm code
                // write to Cds
                crmFhirDataStoreObj.PutCdsObservationData(resource.RawResource.Data);
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex);
                return(null);
            }

            // Gets all data(top 20) from CDS
            var cdsObservationData = crmFhirDataStoreObj.GetCdsObservationData("all");

            // Store above inserted data to blob
            string storageConnection = "DefaultEndpointsProtocol=https;AccountName=clusterstudiostorage;AccountKey=0WY56Pft4WN3GIUfjhGGpGMewvtUO55AMULDOiCj/s1IviuEd4kMbHNYi83mgnZm/N6ZGShdpot0i44uDMJgNA==;EndpointSuffix=core.windows.net";
            CloudStorageAccount cloudStorageAccount = CloudStorageAccount.Parse(storageConnection);
            CloudBlobClient     blobClient          = cloudStorageAccount.CreateCloudBlobClient();
            CloudBlobContainer  blobContainer       = blobClient.GetContainerReference("database");

            blobContainer.CreateIfNotExists();
            string errorBlobName = "CDSData/" + System.DateTime.Now.ToString("yyyy-MM-dd-hh-mm") + "_a.Json";
            var    exceptionblob = blobContainer.GetBlockBlobReference(errorBlobName);

            exceptionblob.UploadTextAsync(cdsObservationData.ToString()).GetAwaiter().GetResult();

            // search with a particulara record for example records with cold
            var cdsObservationSearchData = crmFhirDataStoreObj.GetCdsObservationData("cold");

            // Store above searched data to blob
            blobContainer.CreateIfNotExists();
            string searchBlobName = "CDSData/" + System.DateTime.Now.ToString("yyyy-MM-dd-hh-mm") + "_s.Json";
            var    searchBlob     = blobContainer.GetBlockBlobReference(searchBlobName);

            searchBlob.UploadTextAsync(cdsObservationSearchData.ToString()).GetAwaiter().GetResult();
            using (var connection = new SqlConnection(_configuration.ConnectionString))
            {
                await connection.OpenAsync(cancellationToken);

                using (var command = connection.CreateCommand())
                    using (var stream = new RecyclableMemoryStream(_memoryStreamManager))
                        using (var gzipStream = new GZipStream(stream, CompressionMode.Compress))
                            using (var writer = new StreamWriter(gzipStream, ResourceEncoding))
                            {
                                writer.Write(resource.RawResource.Data);
                                writer.Flush();

                                stream.Seek(0, 0);

                                V1.UpsertResource.PopulateCommand(
                                    command,
                                    baseResourceSurrogateId: ResourceSurrogateIdHelper.LastUpdatedToResourceSurrogateId(resource.LastModified.UtcDateTime),
                                    resourceTypeId: _model.GetResourceTypeId(resource.ResourceTypeName),
                                    resourceId: resource.ResourceId,
                                    eTag: weakETag == null ? null : (int?)etag,
                                    allowCreate: allowCreate,
                                    isDeleted: resource.IsDeleted,
                                    keepHistory: keepHistory,
                                    requestMethod: resource.Request.Method,
                                    rawResource: stream,
                                    tableValuedParameters: _upsertResourceTvpGenerator.Generate(resourceMetadata));

                                try
                                {
                                    var newVersion = (int?)await command.ExecuteScalarAsync(cancellationToken);

                                    if (newVersion == null)
                                    {
                                        // indicates a redundant delete
                                        return(null);
                                    }

                                    resource.Version = newVersion.ToString();

                                    return(new UpsertOutcome(resource, newVersion == 1 ? SaveOutcomeType.Created : SaveOutcomeType.Updated));
                                }
                                catch (SqlException e)
                                {
                                    switch (e.Number)
                                    {
                                    case SqlErrorCodes.NotFound:
                                        throw new MethodNotAllowedException(Core.Resources.ResourceCreationNotAllowed);

                                    case SqlErrorCodes.PreconditionFailed:
                                        throw new ResourceConflictException(weakETag);

                                    default:
                                        _logger.LogError(e, "Error from SQL database on upsert");
                                        throw;
                                    }
                                }
                            }
            }
        }
        public async Task <UpsertOutcome> UpsertAsync(ResourceWrapper resource, WeakETag weakETag, bool allowCreate, bool keepHistory, CancellationToken cancellationToken)
        {
            int etag = 0;

            if (weakETag != null && !int.TryParse(weakETag.VersionId, out etag))
            {
                // Set the etag to a sentinel value to enable expected failure paths when updating with both existing and nonexistent resources.
                etag = -1;
            }

            var resourceMetadata = new ResourceMetadata(
                resource.CompartmentIndices,
                resource.SearchIndices?.ToLookup(e => _searchParameterTypeMap.GetSearchValueType(e)),
                resource.LastModifiedClaims);

            using (SqlConnectionWrapper sqlConnectionWrapper = _sqlConnectionWrapperFactory.ObtainSqlConnectionWrapper(true))
                using (SqlCommandWrapper sqlCommandWrapper = sqlConnectionWrapper.CreateSqlCommand())
                    using (var stream = new RecyclableMemoryStream(_memoryStreamManager))
                        using (var gzipStream = new GZipStream(stream, CompressionMode.Compress))
                            using (var writer = new StreamWriter(gzipStream, ResourceEncoding))
                            {
                                writer.Write(resource.RawResource.Data);
                                writer.Flush();

                                stream.Seek(0, 0);

                                VLatest.UpsertResource.PopulateCommand(
                                    sqlCommandWrapper,
                                    baseResourceSurrogateId: ResourceSurrogateIdHelper.LastUpdatedToResourceSurrogateId(resource.LastModified.UtcDateTime),
                                    resourceTypeId: _model.GetResourceTypeId(resource.ResourceTypeName),
                                    resourceId: resource.ResourceId,
                                    eTag: weakETag == null ? null : (int?)etag,
                                    allowCreate: allowCreate,
                                    isDeleted: resource.IsDeleted,
                                    keepHistory: keepHistory,
                                    requestMethod: resource.Request.Method,
                                    rawResource: stream,
                                    tableValuedParameters: _upsertResourceTvpGeneratorVLatest.Generate(resourceMetadata));

                                try
                                {
                                    var newVersion = (int?)await sqlCommandWrapper.ExecuteScalarAsync(cancellationToken);

                                    if (newVersion == null)
                                    {
                                        // indicates a redundant delete
                                        return(null);
                                    }

                                    resource.Version = newVersion.ToString();

                                    return(new UpsertOutcome(resource, newVersion == 1 ? SaveOutcomeType.Created : SaveOutcomeType.Updated));
                                }
                                catch (SqlException e)
                                {
                                    switch (e.Number)
                                    {
                                    case SqlErrorCodes.PreconditionFailed:
                                        throw new PreconditionFailedException(string.Format(Core.Resources.ResourceVersionConflict, weakETag?.VersionId));

                                    case SqlErrorCodes.NotFound:
                                        if (weakETag != null)
                                        {
                                            throw new ResourceNotFoundException(string.Format(Core.Resources.ResourceNotFoundByIdAndVersion, resource.ResourceTypeName, resource.ResourceId, weakETag.VersionId));
                                        }

                                        goto default;

                                    case SqlErrorCodes.MethodNotAllowed:
                                        throw new MethodNotAllowedException(Core.Resources.ResourceCreationNotAllowed);

                                    default:
                                        _logger.LogError(e, "Error from SQL database on upsert");
                                        throw;
                                    }
                                }
                            }
        }
Пример #23
0
        public async Task <UpsertOutcome> UpsertAsync(
            ResourceWrapper resource,
            WeakETag weakETag,
            bool allowCreate,
            bool keepHistory,
            CancellationToken cancellationToken,
            bool requireETagOnUpdate = false)
        {
            int?eTag = weakETag == null
                ? null
                : (int.TryParse(weakETag.VersionId, out var parsedETag) ? parsedETag : -1); // Set the etag to a sentinel value to enable expected failure paths when updating with both existing and nonexistent resources.

            var resourceMetadata = new ResourceMetadata(
                resource.CompartmentIndices,
                resource.SearchIndices?.ToLookup(e => _searchParameterTypeMap.GetSearchValueType(e)),
                resource.LastModifiedClaims);

            using (SqlConnectionWrapper sqlConnectionWrapper = await _sqlConnectionWrapperFactory.ObtainSqlConnectionWrapperAsync(cancellationToken, true))
                using (SqlCommandWrapper sqlCommandWrapper = sqlConnectionWrapper.CreateSqlCommand())
                    using (var stream = new RecyclableMemoryStream(_memoryStreamManager))
                    {
                        _compressedRawResourceConverter.WriteCompressedRawResource(stream, resource.RawResource.Data);

                        stream.Seek(0, 0);

                        PopulateUpsertResourceCommand(sqlCommandWrapper, resource, resourceMetadata, allowCreate, keepHistory, requireETagOnUpdate, eTag, stream, _coreFeatures.SupportsResourceChangeCapture);

                        try
                        {
                            var newVersion = (int?)await sqlCommandWrapper.ExecuteScalarAsync(cancellationToken);

                            if (newVersion == null)
                            {
                                // indicates a redundant delete
                                return(null);
                            }

                            resource.Version = newVersion.ToString();

                            SaveOutcomeType saveOutcomeType;
                            if (newVersion == 1)
                            {
                                saveOutcomeType = SaveOutcomeType.Created;
                            }
                            else
                            {
                                saveOutcomeType = SaveOutcomeType.Updated;
                                resource.RawResource.IsMetaSet = false;
                            }

                            return(new UpsertOutcome(resource, saveOutcomeType));
                        }
                        catch (SqlException e)
                        {
                            switch (e.Number)
                            {
                            case SqlErrorCodes.PreconditionFailed:
                                if (weakETag != null)
                                {
                                    // The backwards compatibility behavior of Stu3 is to return 409 Conflict instead of a 412 Precondition Failed
                                    if (_modelInfoProvider.Version == FhirSpecification.Stu3)
                                    {
                                        throw new ResourceConflictException(weakETag);
                                    }

                                    throw new PreconditionFailedException(string.Format(Core.Resources.ResourceVersionConflict, weakETag.VersionId));
                                }

                                goto default;

                            case SqlErrorCodes.NotFound:
                                if (weakETag != null)
                                {
                                    throw new ResourceNotFoundException(string.Format(Core.Resources.ResourceNotFoundByIdAndVersion, resource.ResourceTypeName, resource.ResourceId, weakETag.VersionId));
                                }

                                goto default;

                            case SqlErrorCodes.MethodNotAllowed:
                                throw new MethodNotAllowedException(Core.Resources.ResourceCreationNotAllowed);

                            case SqlErrorCodes.TimeoutExpired:
                                throw new RequestTimeoutException(Resources.ExecutionTimeoutExpired);

                            case 50400: // TODO: Add this to SQL error codes in AB#88286
                                // The backwards compatibility behavior of Stu3 is to return 412 Precondition Failed instead of a 400 Bad Request
                                if (_modelInfoProvider.Version == FhirSpecification.Stu3)
                                {
                                    throw new PreconditionFailedException(string.Format(Core.Resources.IfMatchHeaderRequiredForResource, resource.ResourceTypeName));
                                }

                                throw new BadRequestException(string.Format(Core.Resources.IfMatchHeaderRequiredForResource, resource.ResourceTypeName));

                            default:
                                _logger.LogError(e, "Error from SQL database on upsert");
                                throw;
                            }
                        }
                    }
        }
Пример #24
0
        public async ValueTask <XeusClue> Import(string path, CancellationToken token = default)
        {
            if (path == null)
            {
                throw new ArgumentNullException(nameof(path));
            }

            return(await Task.Run(async() =>
            {
                // Check
                lock (_lockObject)
                {
                    var info = _contentMetadataStorage.GetFileContentMetadata(path);
                    if (info != null)
                    {
                        return info.Clue;
                    }
                }

                XeusClue?clue = null;
                var lockedHashes = new HashSet <OmniHash>();
                SharedBlocksMetadata?sharedBlocksInfo = null;

                {
                    const int blockLength = 1024 * 1024;
                    const OmniHashAlgorithmType hashAlgorithmType = OmniHashAlgorithmType.Sha2_256;
                    const CorrectionAlgorithmType correctionAlgorithmType = CorrectionAlgorithmType.ReedSolomon8;

                    byte depth = 0;

                    var merkleTreeSectionList = new List <MerkleTreeSection>();

                    // File
                    using (var stream = new UnbufferedFileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, FileOptions.None, _bufferPool))
                    {
                        if (stream.Length <= blockLength)
                        {
                            OmniHash hash;

                            using (var bufferMemoryOwner = _bufferPool.Rent((int)stream.Length))
                            {
                                stream.Read(bufferMemoryOwner.Memory.Span);

                                if (hashAlgorithmType == OmniHashAlgorithmType.Sha2_256)
                                {
                                    hash = new OmniHash(OmniHashAlgorithmType.Sha2_256, Sha2_256.ComputeHash(bufferMemoryOwner.Memory.Span));
                                }
                            }

                            sharedBlocksInfo = new SharedBlocksMetadata(path, (ulong)stream.Length, (uint)stream.Length, new OmniHash[] { hash });
                            clue = new XeusClue(hash, depth);
                        }
                        else
                        {
                            var sharedHashes = new List <OmniHash>();

                            for (; ;)
                            {
                                var targetHashes = new List <OmniHash>();
                                var targetMemoryOwners = new List <IMemoryOwner <byte> >();
                                ulong sumLength = 0;

                                try
                                {
                                    for (int i = 0; stream.Position < stream.Length; i++)
                                    {
                                        token.ThrowIfCancellationRequested();

                                        uint length = (uint)Math.Min(stream.Length - stream.Position, blockLength);
                                        var bufferMemoryOwner = _bufferPool.Rent((int)length);

                                        try
                                        {
                                            stream.Read(bufferMemoryOwner.Memory.Span);

                                            sumLength += length;
                                        }
                                        catch (Exception e)
                                        {
                                            bufferMemoryOwner.Dispose();

                                            throw e;
                                        }

                                        OmniHash hash;

                                        if (hashAlgorithmType == OmniHashAlgorithmType.Sha2_256)
                                        {
                                            hash = new OmniHash(OmniHashAlgorithmType.Sha2_256, Sha2_256.ComputeHash(bufferMemoryOwner.Memory.Span));
                                        }

                                        sharedHashes.Add(hash);

                                        targetHashes.Add(hash);
                                        targetMemoryOwners.Add(bufferMemoryOwner);

                                        if (targetMemoryOwners.Count >= 128)
                                        {
                                            break;
                                        }
                                    }

                                    var parityHashes = await this.ParityEncode(targetMemoryOwners.Select(n => n.Memory), hashAlgorithmType, correctionAlgorithmType, token);
                                    lockedHashes.UnionWith(parityHashes);

                                    merkleTreeSectionList.Add(new MerkleTreeSection(correctionAlgorithmType, sumLength, CollectionHelper.Unite(targetHashes, parityHashes).ToArray()));
                                }
                                finally
                                {
                                    foreach (var memoryOwner in targetMemoryOwners)
                                    {
                                        memoryOwner.Dispose();
                                    }
                                }

                                if (stream.Position == stream.Length)
                                {
                                    break;
                                }
                            }

                            sharedBlocksInfo = new SharedBlocksMetadata(path, (ulong)stream.Length, blockLength, sharedHashes.ToArray());

                            depth++;
                        }
                    }

                    while (merkleTreeSectionList.Count > 0)
                    {
                        // Index
                        using (var stream = new RecyclableMemoryStream(_bufferPool))
                        {
                            RocketPackHelper.MessageToStream(new MerkleTreeNode(merkleTreeSectionList.ToArray()), stream);
                            stream.Seek(0, SeekOrigin.Begin);

                            merkleTreeSectionList.Clear();

                            if (stream.Length <= blockLength)
                            {
                                OmniHash hash;

                                using (var bufferMemoryOwner = _bufferPool.Rent((int)stream.Length))
                                {
                                    stream.Read(bufferMemoryOwner.Memory.Span);

                                    if (hashAlgorithmType == OmniHashAlgorithmType.Sha2_256)
                                    {
                                        hash = new OmniHash(OmniHashAlgorithmType.Sha2_256, Sha2_256.ComputeHash(bufferMemoryOwner.Memory.Span));
                                    }

                                    _blockStorage.Lock(hash);

                                    bool result = _blockStorage.TrySet(hash, bufferMemoryOwner.Memory.Span);

                                    if (!result)
                                    {
                                        throw new ImportFailed("Failed to save Block.");
                                    }

                                    lockedHashes.Add(hash);
                                }

                                clue = new XeusClue(hash, depth);
                            }
                            else
                            {
                                for (; ;)
                                {
                                    var targetHashes = new List <OmniHash>();
                                    var targetMemoryOwners = new List <IMemoryOwner <byte> >();
                                    ulong sumLength = 0;

                                    try
                                    {
                                        for (int i = 0; stream.Position < stream.Length; i++)
                                        {
                                            token.ThrowIfCancellationRequested();

                                            uint length = (uint)Math.Min(stream.Length - stream.Position, blockLength);
                                            var bufferMemoryOwner = _bufferPool.Rent((int)length);

                                            try
                                            {
                                                stream.Read(bufferMemoryOwner.Memory.Span);

                                                sumLength += length;
                                            }
                                            catch (Exception e)
                                            {
                                                bufferMemoryOwner.Dispose();

                                                throw e;
                                            }

                                            OmniHash hash;

                                            if (hashAlgorithmType == OmniHashAlgorithmType.Sha2_256)
                                            {
                                                hash = new OmniHash(OmniHashAlgorithmType.Sha2_256, Sha2_256.ComputeHash(bufferMemoryOwner.Memory.Span));
                                            }

                                            _blockStorage.Lock(hash);

                                            bool result = _blockStorage.TrySet(hash, bufferMemoryOwner.Memory.Span);

                                            if (!result)
                                            {
                                                throw new ImportFailed("Failed to save Block.");
                                            }

                                            lockedHashes.Add(hash);

                                            targetHashes.Add(hash);
                                            targetMemoryOwners.Add(bufferMemoryOwner);

                                            if (targetMemoryOwners.Count >= 128)
                                            {
                                                break;
                                            }
                                        }

                                        var parityHashes = await this.ParityEncode(targetMemoryOwners.Select(n => n.Memory), hashAlgorithmType, correctionAlgorithmType, token);
                                        lockedHashes.UnionWith(parityHashes);

                                        merkleTreeSectionList.Add(new MerkleTreeSection(correctionAlgorithmType, sumLength, CollectionHelper.Unite(targetHashes, parityHashes).ToArray()));
                                    }
                                    finally
                                    {
                                        foreach (var memoryOwner in targetMemoryOwners)
                                        {
                                            memoryOwner.Dispose();
                                        }
                                    }

                                    if (stream.Position == stream.Length)
                                    {
                                        break;
                                    }
                                }

                                depth++;
                            }
                        }
                    }
                }

                if (clue == null)
                {
                    throw new ImportFailed("clue is null");
                }

                lock (_lockObject)
                {
                    if (!_contentMetadataStorage.ContainsFileContentMetadata(path))
                    {
                        _contentMetadataStorage.Add(new ContentMetadata(clue, lockedHashes.ToArray(), sharedBlocksInfo));

                        foreach (var hash in lockedHashes)
                        {
                            _blockStorage.Lock(hash);
                        }
                    }
                }

                return clue;
            }, token));
        }
        private void _messagingManager_ReceiveEvent(Stream requestStream)
        {
            var reader = new MessageStreamReader(new WrapperStream(requestStream), _bufferManager);

            {
                var type = (AmoebaFunctionType)reader.GetUInt64();
                int id   = (int)reader.GetUInt64();

                if (type == AmoebaFunctionType.Exit)
                {
                    SendResponse(AmoebaFunctionResponseType.Result, id, (object)null);
                    _tokenSource.Cancel();
                }
                else if (type == AmoebaFunctionType.Cancel)
                {
                    if (_tasks.TryGetValue(id, out var responseTask))
                    {
                        responseTask.Stop();
                    }
                }
                else
                {
                    var responseTask = ResponseTask.Create((token) =>
                    {
                        try
                        {
                            switch (type)
                            {
                            case AmoebaFunctionType.GetReport:
                                {
                                    SendResponse(AmoebaFunctionResponseType.Result, id, _serviceManager.Report);
                                    break;
                                }

                            case AmoebaFunctionType.GetNetworkConnectionReports:
                                {
                                    SendResponse(AmoebaFunctionResponseType.Result, id, _serviceManager.GetNetworkConnectionReports());
                                    break;
                                }

                            case AmoebaFunctionType.GetCacheContentReports:
                                {
                                    SendResponse(AmoebaFunctionResponseType.Result, id, _serviceManager.GetCacheContentReports());
                                    break;
                                }

                            case AmoebaFunctionType.GetDownloadContentReports:
                                {
                                    SendResponse(AmoebaFunctionResponseType.Result, id, _serviceManager.GetDownloadContentReports());
                                    break;
                                }

                            case AmoebaFunctionType.GetConfig:
                                {
                                    SendResponse(AmoebaFunctionResponseType.Result, id, _serviceManager.Config);
                                    break;
                                }

                            case AmoebaFunctionType.SetConfig:
                                {
                                    var config = JsonUtils.Load <ServiceConfig>(requestStream);
                                    _serviceManager.SetConfig(config);
                                    SendResponse(AmoebaFunctionResponseType.Result, id, (object)null);
                                    break;
                                }

                            case AmoebaFunctionType.SetCloudLocations:
                                {
                                    var cloudLocations = JsonUtils.Load <Location[]>(requestStream);
                                    _serviceManager.SetCloudLocations(cloudLocations);
                                    SendResponse(AmoebaFunctionResponseType.Result, id, (object)null);
                                    break;
                                }

                            case AmoebaFunctionType.GetSize:
                                {
                                    SendResponse(AmoebaFunctionResponseType.Result, id, _serviceManager.Size);
                                    break;
                                }

                            case AmoebaFunctionType.Resize:
                                {
                                    long size = JsonUtils.Load <long>(requestStream);
                                    _serviceManager.Resize(size);
                                    SendResponse(AmoebaFunctionResponseType.Result, id, (object)null);
                                    break;
                                }

                            case AmoebaFunctionType.CheckBlocks:
                                {
                                    _serviceManager.CheckBlocks(new Action <CheckBlocksProgressReport>((report) =>
                                    {
                                        SendResponse(AmoebaFunctionResponseType.Output, id, report);
                                    }), token).Wait();

                                    SendResponse(AmoebaFunctionResponseType.Result, id, (object)null);
                                    break;
                                }

                            case AmoebaFunctionType.AddContent:
                                {
                                    var arguments = JsonUtils.Load <(string, DateTime)>(requestStream);
                                    var result    = _serviceManager.AddContent(arguments.Item1, arguments.Item2, token).Result;
                                    SendResponse(AmoebaFunctionResponseType.Result, id, result);
                                    break;
                                }

                            case AmoebaFunctionType.RemoveContent:
                                {
                                    string path = JsonUtils.Load <string>(requestStream);
                                    _serviceManager.RemoveContent(path);
                                    SendResponse(AmoebaFunctionResponseType.Result, id, (object)null);
                                    break;
                                }

                            case AmoebaFunctionType.Diffusion:
                                {
                                    string path = JsonUtils.Load <string>(requestStream);
                                    _serviceManager.DiffuseContent(path);
                                    SendResponse(AmoebaFunctionResponseType.Result, id, (object)null);
                                    break;
                                }

                            case AmoebaFunctionType.AddDownload:
                                {
                                    var arguments = JsonUtils.Load <(Metadata, string, long)>(requestStream);
                                    _serviceManager.AddDownload(arguments.Item1, arguments.Item2, arguments.Item3);
                                    SendResponse(AmoebaFunctionResponseType.Result, id, (object)null);
                                    break;
                                }

                            case AmoebaFunctionType.RemoveDownload:
                                {
                                    var arguments = JsonUtils.Load <(Metadata, string)>(requestStream);
                                    _serviceManager.RemoveDownload(arguments.Item1, arguments.Item2);
                                    SendResponse(AmoebaFunctionResponseType.Result, id, (object)null);
                                    break;
                                }

                            case AmoebaFunctionType.ResetDownload:
                                {
                                    var arguments = JsonUtils.Load <(Metadata, string)>(requestStream);
                                    _serviceManager.ResetDownload(arguments.Item1, arguments.Item2);
                                    SendResponse(AmoebaFunctionResponseType.Result, id, (object)null);
                                    break;
                                }

                            case AmoebaFunctionType.SetProfile:
                                {
                                    var arguments = JsonUtils.Load <(ProfileContent, DigitalSignature)>(requestStream);
                                    _serviceManager.SetProfile(arguments.Item1, arguments.Item2, token).Wait();
                                    SendResponse(AmoebaFunctionResponseType.Result, id, (object)null);
                                    break;
                                }

                            case AmoebaFunctionType.SetStore:
                                {
                                    var arguments = JsonUtils.Load <(StoreContent, DigitalSignature)>(requestStream);
                                    _serviceManager.SetStore(arguments.Item1, arguments.Item2, token).Wait();
                                    SendResponse(AmoebaFunctionResponseType.Result, id, (object)null);
                                    break;
                                }

                            case AmoebaFunctionType.SetUnicastCommentMessage:
                                {
                                    var arguments = JsonUtils.Load <(Signature, CommentContent, AgreementPublicKey, DigitalSignature)>(requestStream);
                                    _serviceManager.SetUnicastCommentMessage(arguments.Item1, arguments.Item2, arguments.Item3, arguments.Item4, token).Wait();
                                    SendResponse(AmoebaFunctionResponseType.Result, id, (object)null);
                                    break;
                                }

                            case AmoebaFunctionType.SetMulticastCommentMessage:
                                {
                                    var arguments = JsonUtils.Load <(Tag, CommentContent, DigitalSignature, TimeSpan)>(requestStream);
                                    _serviceManager.SetMulticastCommentMessage(arguments.Item1, arguments.Item2, arguments.Item3, arguments.Item4, token).Wait();
                                    SendResponse(AmoebaFunctionResponseType.Result, id, (object)null);
                                    break;
                                }

                            case AmoebaFunctionType.GetProfile:
                                {
                                    var signature = JsonUtils.Load <Signature>(requestStream);
                                    var result    = _serviceManager.GetProfile(signature, token).Result;
                                    SendResponse(AmoebaFunctionResponseType.Result, id, result);
                                    break;
                                }

                            case AmoebaFunctionType.GetStore:
                                {
                                    var signature = JsonUtils.Load <Signature>(requestStream);
                                    var result    = _serviceManager.GetStore(signature, token).Result;
                                    SendResponse(AmoebaFunctionResponseType.Result, id, result);
                                    break;
                                }

                            case AmoebaFunctionType.GetUnicastCommentMessages:
                                {
                                    var arguments = JsonUtils.Load <(Signature, AgreementPrivateKey)>(requestStream);
                                    var result    = _serviceManager.GetUnicastCommentMessages(arguments.Item1, arguments.Item2, token).Result;
                                    SendResponse(AmoebaFunctionResponseType.Result, id, result);
                                    break;
                                }

                            case AmoebaFunctionType.GetMulticastCommentMessages:
                                {
                                    var tag    = JsonUtils.Load <Tag>(requestStream);
                                    var result = _serviceManager.GetMulticastCommentMessages(tag, token).Result;
                                    SendResponse(AmoebaFunctionResponseType.Result, id, result);
                                    break;
                                }

                            case AmoebaFunctionType.GetState:
                                {
                                    SendResponse(AmoebaFunctionResponseType.Result, id, _serviceManager.State);
                                    break;
                                }

                            case AmoebaFunctionType.Start:
                                {
                                    _serviceManager.Start();
                                    SendResponse(AmoebaFunctionResponseType.Result, id, (object)null);
                                    break;
                                }

                            case AmoebaFunctionType.Stop:
                                {
                                    _serviceManager.Stop();
                                    SendResponse(AmoebaFunctionResponseType.Result, id, (object)null);
                                    break;
                                }

                            case AmoebaFunctionType.Load:
                                {
                                    _serviceManager.Load();
                                    SendResponse(AmoebaFunctionResponseType.Result, id, (object)null);
                                    break;
                                }

                            case AmoebaFunctionType.Save:
                                {
                                    _serviceManager.Save();
                                    SendResponse(AmoebaFunctionResponseType.Result, id, (object)null);
                                    break;
                                }
                            }
                        }
                        catch (OperationCanceledException)
                        {
                            SendResponse(AmoebaFunctionResponseType.Cancel, id, (object)null);
                        }
                        catch (Exception e)
                        {
                            Log.Error(string.Format("Rpc Error: {0}", type.ToString()));
                            Log.Error(e);

                            var argument = new AmoebaErrorMessage(e.GetType().ToString(), e.Message, e.StackTrace?.ToString());
                            SendResponse(AmoebaFunctionResponseType.Error, id, argument);
                        }
                        finally
                        {
                            requestStream.Dispose();
                            _tasks.Remove(id);
                        }
                    });

                    _tasks.Add(id, responseTask);
                    responseTask.Start();
                }
            }

            void SendResponse <T>(AmoebaFunctionResponseType type, int id, T value)
            {
                var messageStream = new RecyclableMemoryStream(_bufferManager);
                var writer        = new MessageStreamWriter(messageStream, _bufferManager);

                writer.Write((ulong)type);
                writer.Write((ulong)id);

                Stream valueStream = null;

                if (value != null)
                {
                    try
                    {
                        valueStream = new RecyclableMemoryStream(_bufferManager);
                        JsonUtils.Save(valueStream, value);
                    }
                    catch (Exception)
                    {
                        if (valueStream != null)
                        {
                            valueStream.Dispose();
                            valueStream = null;
                        }

                        return;
                    }
                }

                messageStream.Seek(0, SeekOrigin.Begin);
                _messagingManager.Send(new UniteStream(messageStream, valueStream));
            }
        }