private async Task <(long Chunks, long Bytes)> StreamContentWithCompressionAsync(Stream reader, byte[] buffer, IServerStreamWriter <CopyFileResponse> responseStream, CancellationToken ct = default(CancellationToken)) { Debug.Assert(!(reader is null)); Debug.Assert(!(responseStream is null)); long bytes = 0L; long chunks = 0L; using (Stream grpcStream = new BufferedWriteStream( buffer, async(byte[] bf, int offset, int count) => { ByteString content = ByteString.CopyFrom(bf, offset, count); CopyFileResponse response = new CopyFileResponse() { Content = content, Index = chunks }; await responseStream.WriteAsync(response); bytes += count; chunks++; } )) { using (Stream compressionStream = new GZipStream(grpcStream, System.IO.Compression.CompressionLevel.Fastest, true)) { await reader.CopyToAsync(compressionStream, buffer.Length, ct).ConfigureAwait(false); await compressionStream.FlushAsync().ConfigureAwait(false); } await grpcStream.FlushAsync().ConfigureAwait(false); } return(chunks, bytes); }
private void PutCopyFile(string storageName, string srcPath, string destStorageName, string destPath, string versionId = null) { var request = new PutCopyRequest(); request.Path = srcPath; request.Storage = storageName; request.VersionId = versionId; request.Newdest = destPath; request.DestStorage = destStorageName; CopyFileResponse response = null; try { response = StorageApi.PutCopy(request); } catch (Exception ex) { var msg = new string[] { "Invalid version id specified", //AmazonS3 "Invalid snapshot time", //Windows Azure "Value should match pattern", //DropBox }; Assert.IsTrue(msg.Where(x => ex.Message.Contains(x)).Any(), ex.Message); return; } Assert.AreEqual(200, response.Code); }
private async Task <(long Chunks, long Bytes)> StreamContentAsync(Stream reader, byte[] buffer, IServerStreamWriter <CopyFileResponse> responseStream, CancellationToken ct = default(CancellationToken)) { Debug.Assert(!(reader is null)); Debug.Assert(!(responseStream is null)); long chunks = 0L; long bytes = 0L; while (true) { ct.ThrowIfCancellationRequested(); int chunkSize = await reader.ReadAsync(buffer, 0, buffer.Length, ct).ConfigureAwait(false); if (chunkSize == 0) { break; } ByteString content = ByteString.CopyFrom(buffer, 0, chunkSize); CopyFileResponse response = new CopyFileResponse() { Content = content, Index = chunks }; await responseStream.WriteAsync(response).ConfigureAwait(false); bytes += chunkSize; chunks++; Console.WriteLine(chunks); } return(chunks, bytes); }
private async Task <Result <(long Chunks, long Bytes)> > StreamContentWithCompressionAsync(Stream input, byte[] buffer, byte[] secondaryBuffer, IServerStreamWriter <CopyFileResponse> responseStream, CancellationToken ct) { long bytes = 0L; long chunks = 0L; using (Stream grpcStream = new BufferedWriteStream( buffer, async(byte[] bf, int offset, int count) => { ByteString content = ByteString.CopyFrom(bf, offset, count); CopyFileResponse response = new CopyFileResponse() { Content = content, Index = chunks }; await responseStream.WriteAsync(response); bytes += count; chunks++; } )) { using (Stream compressionStream = new GZipStream(grpcStream, System.IO.Compression.CompressionLevel.Fastest, true)) { await input.CopyToAsync(compressionStream, buffer.Length, ct); await compressionStream.FlushAsync(ct); } await grpcStream.FlushAsync(ct); } return(chunks, bytes); }
private async Task StreamContentAsync(OpenStreamResult openStreamResult, IServerStreamWriter <CopyFileResponse> responseStream, DateTime startTime) { using (var stream = openStreamResult.Stream) { byte[] buffer = new byte[ContentStore.Grpc.Utils.DefaultBufferSize]; long chunks = 0; while (true) { int chunkSize = await stream.ReadAsync(buffer, 0, buffer.Length); if (chunkSize == 0) { break; } ByteString content = ByteString.CopyFrom(buffer, 0, chunkSize); CopyFileResponse reply = new CopyFileResponse() { // TODO: Check on this code Header = new ResponseHeader(startTime, openStreamResult.Succeeded, (int)openStreamResult.Code, openStreamResult.ErrorMessage, openStreamResult.Diagnostics), Content = content, Index = chunks }; await responseStream.WriteAsync(reply); chunks++; } } }
private async Task <long> StreamContentAsync(Stream fileStream, IAsyncStreamReader <CopyFileResponse> replyStream) { long bytesReceived = 0L; while (await replyStream.MoveNext(CancellationToken.None)) { CopyFileResponse oneOfManyReply = replyStream.Current; bytesReceived += oneOfManyReply.Content.Length; oneOfManyReply.Content.WriteTo(fileStream); } return(bytesReceived); }
private async Task <(long Chunks, long Bytes)> StreamContentAsync(Stream targetStream, IAsyncStreamReader <CopyFileResponse> replyStream, CancellationToken ct = default(CancellationToken)) { long chunks = 0L; long bytes = 0L; while (await replyStream.MoveNext(ct).ConfigureAwait(false)) { chunks++; CopyFileResponse reply = replyStream.Current; bytes += reply.Content.Length; reply.Content.WriteTo(targetStream); } return(chunks, bytes); }
private async Task <(long chunks, long bytes)> StreamContentAsync(Stream targetStream, IAsyncStreamReader <CopyFileResponse> replyStream, CopyToOptions?options, CancellationToken ct) { long chunks = 0L; long bytes = 0L; while (await replyStream.MoveNext(ct)) { chunks++; CopyFileResponse reply = replyStream.Current; bytes += reply.Content.Length; reply.Content.WriteTo(targetStream); options?.UpdateTotalBytesCopied(bytes); } return(chunks, bytes); }
public void SerializingCopyFileResponse() { var response = new CopyFileResponse() { Path = "a" }; var bytes = Google.Protobuf.MessageExtensions.ToByteArray(response); var expected = new byte[] { 10, 1, 97 }; Assert.Equal(expected, bytes); var deserialized = CopyFileResponse.Parser.ParseFrom(bytes); Assert.Equal("a", deserialized.Path); }
private async Task <(long Chunks, long Bytes)> StreamContentAsync(Stream targetStream, IAsyncStreamReader <CopyFileResponse> replyStream, CancellationToken ct) { Contract.Requires(targetStream != null); Contract.Requires(replyStream != null); long chunks = 0L; long bytes = 0L; while (await replyStream.MoveNext(ct)) { chunks++; CopyFileResponse reply = replyStream.Current; bytes += reply.Content.Length; reply.Content.WriteTo(targetStream); } return(chunks, bytes); }
private async Task <Result <(long Chunks, long Bytes)> > StreamContentAsync(Stream input, byte[] buffer, byte[] secondaryBuffer, IServerStreamWriter <CopyFileResponse> responseStream, CancellationToken ct) { long chunks = 0L; long bytes = 0L; byte[] inputBuffer = buffer; // Pre-fill buffer with the file's first chunk int chunkSize = await readNextChunk(input, inputBuffer, ct); while (true) { ct.ThrowIfCancellationRequested(); if (chunkSize == 0) { break; } // Created ByteString can reuse the buffer. // To avoid double writes we need two buffers: // * One buffer is used as the output buffer // * And another buffer is used as the input buffer. (ByteString content, bool bufferReused) = CreateByteStringForStreamContent(inputBuffer, chunkSize); CopyFileResponse response = new CopyFileResponse() { Content = content, Index = chunks }; if (bufferReused) { // If the buffer is reused we need to swap the input buffer with the secondary buffer. inputBuffer = inputBuffer == buffer ? secondaryBuffer : buffer; } bytes += chunkSize; chunks++; // Read the next chunk while waiting for the response var readNextChunkTask = readNextChunk(input, inputBuffer, ct); await Task.WhenAll(readNextChunkTask, responseStream.WriteAsync(response)); chunkSize = await readNextChunkTask; } return(chunks, bytes);
private async Task <Result <(long Chunks, long Bytes)> > StreamContentAsync(Stream input, byte[] buffer, IServerStreamWriter <CopyFileResponse> responseStream, CancellationToken ct) { Contract.Requires(!(input is null)); Contract.Requires(!(responseStream is null)); int chunkSize = 0; long chunks = 0L; long bytes = 0L; // Pre-fill buffer with the file's first chunk await readNextChunk(); while (true) { ct.ThrowIfCancellationRequested(); if (chunkSize == 0) { break; } ByteString content = ByteString.CopyFrom(buffer, 0, chunkSize); CopyFileResponse response = new CopyFileResponse() { Content = content, Index = chunks }; bytes += chunkSize; chunks++; // Read the next chunk while waiting for the response await Task.WhenAll(readNextChunk(), responseStream.WriteAsync(response)); } return(chunks, bytes); async Task <int> readNextChunk() { chunkSize = await input.ReadAsync(buffer, 0, buffer.Length, ct); return(chunkSize); } }
/// <summary> /// Copies file to stream /// </summary> public async Task <CopyFileResult> CopyFileAsync(Context context, ContentHash contentHash, AbsolutePath destinationPath, long fileSize = -1, bool enableCompression = false) { // TODO: Pipe through flag for compression type CopyCompression compression = enableCompression ? CopyCompression.Gzip : CopyCompression.None; AsyncServerStreamingCall <CopyFileResponse> response = _client.CopyFile(new CopyFileRequest() { TraceId = context.Id.ToString(), HashType = (int)contentHash.HashType, ContentHash = contentHash.ToByteString(), Drive = destinationPath.DriveLetter.ToString(), Offset = 0, Compression = compression }); IAsyncStreamReader <CopyFileResponse> replyStream = response.ResponseStream; long bytesReceived = 0L; using (var stream = new FileStream(destinationPath.Path, FileMode.Create, FileAccess.Write, FileShare.None, ContentStore.Grpc.Utils.DefaultBufferSize, FileOptions.Asynchronous | FileOptions.SequentialScan)) { bytesReceived = await StreamContentAsync(stream, response.ResponseStream); while (await replyStream.MoveNext(CancellationToken.None)) { CopyFileResponse oneOfManyReply = replyStream.Current; bytesReceived += oneOfManyReply.Content.Length; oneOfManyReply.Content.WriteTo(stream); } } if (fileSize >= 0 && bytesReceived != fileSize) { return(new CopyFileResult(CopyFileResult.ResultCode.InvalidHash, $"Received {bytesReceived} bytes for {contentHash}, expected {fileSize}")); } return(CopyFileResult.Success); }
private void CopyFileFinished(object sender, copyCompletedEventArgs e) { object[] state = (object[]) e.UserState; OperationFinished<CopyFileResponse> copyFileCompleted = (OperationFinished<CopyFileResponse>) state[0]; CopyFileResponse copyFileResponse; if (e.Error != null) { copyFileResponse = new CopyFileResponse { Error = e.Error, Status = CopyObjectStatus.Failed, UserState = state[1] }; } else { copyFileResponse = new CopyFileResponse { Status = StatusMessageParser.ParseCopyObjectStatus(e.Result), UserState = state[1] }; copyFileResponse.Error = copyFileResponse.Status == CopyObjectStatus.Unknown ? new UnknownOperationStatusException(e.Result) : null; } copyFileCompleted(copyFileResponse); }