private async Task <(long Chunks, long Bytes)> StreamContentWithCompressionAsync(Stream reader, byte[] buffer, IServerStreamWriter <CopyFileResponse> responseStream, CancellationToken ct = default(CancellationToken))
{
Debug.Assert(!(reader is null));
Debug.Assert(!(responseStream is null));
long bytes = 0L;
long chunks = 0L;
using (Stream grpcStream = new BufferedWriteStream(
buffer,
async(byte[] bf, int offset, int count) =>
{
ByteString content = ByteString.CopyFrom(bf, offset, count);
CopyFileResponse response = new CopyFileResponse()
{
Content = content, Index = chunks
};
await responseStream.WriteAsync(response);
bytes += count;
chunks++;
}
))
{
using (Stream compressionStream = new GZipStream(grpcStream, System.IO.Compression.CompressionLevel.Fastest, true))
{
await reader.CopyToAsync(compressionStream, buffer.Length, ct).ConfigureAwait(false);
await compressionStream.FlushAsync().ConfigureAwait(false);
}
await grpcStream.FlushAsync().ConfigureAwait(false);
}
return(chunks, bytes);
}
private void PutCopyFile(string storageName, string srcPath, string destStorageName, string destPath, string versionId = null)
{
var request = new PutCopyRequest();
request.Path = srcPath;
request.Storage = storageName;
request.VersionId = versionId;
request.Newdest = destPath;
request.DestStorage = destStorageName;
CopyFileResponse response = null;
try
{
response = StorageApi.PutCopy(request);
}
catch (Exception ex)
{
var msg = new string[]
{
"Invalid version id specified", //AmazonS3
"Invalid snapshot time", //Windows Azure
"Value should match pattern", //DropBox
};
Assert.IsTrue(msg.Where(x => ex.Message.Contains(x)).Any(), ex.Message);
return;
}
Assert.AreEqual(200, response.Code);
}
private async Task <(long Chunks, long Bytes)> StreamContentAsync(Stream reader, byte[] buffer, IServerStreamWriter <CopyFileResponse> responseStream, CancellationToken ct = default(CancellationToken))
{
Debug.Assert(!(reader is null));
Debug.Assert(!(responseStream is null));
long chunks = 0L;
long bytes = 0L;
while (true)
{
ct.ThrowIfCancellationRequested();
int chunkSize = await reader.ReadAsync(buffer, 0, buffer.Length, ct).ConfigureAwait(false);
if (chunkSize == 0)
{
break;
}
ByteString content = ByteString.CopyFrom(buffer, 0, chunkSize);
CopyFileResponse response = new CopyFileResponse()
{
Content = content, Index = chunks
};
await responseStream.WriteAsync(response).ConfigureAwait(false);
bytes += chunkSize;
chunks++;
Console.WriteLine(chunks);
}
return(chunks, bytes);
}
private async Task <Result <(long Chunks, long Bytes)> > StreamContentWithCompressionAsync(Stream input, byte[] buffer, byte[] secondaryBuffer, IServerStreamWriter <CopyFileResponse> responseStream, CancellationToken ct)
{
long bytes = 0L;
long chunks = 0L;
using (Stream grpcStream = new BufferedWriteStream(
buffer,
async(byte[] bf, int offset, int count) =>
{
ByteString content = ByteString.CopyFrom(bf, offset, count);
CopyFileResponse response = new CopyFileResponse()
{
Content = content, Index = chunks
};
await responseStream.WriteAsync(response);
bytes += count;
chunks++;
}
))
{
using (Stream compressionStream = new GZipStream(grpcStream, System.IO.Compression.CompressionLevel.Fastest, true))
{
await input.CopyToAsync(compressionStream, buffer.Length, ct);
await compressionStream.FlushAsync(ct);
}
await grpcStream.FlushAsync(ct);
}
return(chunks, bytes);
}
private async Task StreamContentAsync(OpenStreamResult openStreamResult, IServerStreamWriter <CopyFileResponse> responseStream, DateTime startTime)
{
using (var stream = openStreamResult.Stream)
{
byte[] buffer = new byte[ContentStore.Grpc.Utils.DefaultBufferSize];
long chunks = 0;
while (true)
{
int chunkSize = await stream.ReadAsync(buffer, 0, buffer.Length);
if (chunkSize == 0)
{
break;
}
ByteString content = ByteString.CopyFrom(buffer, 0, chunkSize);
CopyFileResponse reply = new CopyFileResponse()
{
// TODO: Check on this code
Header = new ResponseHeader(startTime, openStreamResult.Succeeded, (int)openStreamResult.Code, openStreamResult.ErrorMessage, openStreamResult.Diagnostics),
Content = content,
Index = chunks
};
await responseStream.WriteAsync(reply);
chunks++;
}
}
}
private async Task <long> StreamContentAsync(Stream fileStream, IAsyncStreamReader <CopyFileResponse> replyStream)
{
long bytesReceived = 0L;
while (await replyStream.MoveNext(CancellationToken.None))
{
CopyFileResponse oneOfManyReply = replyStream.Current;
bytesReceived += oneOfManyReply.Content.Length;
oneOfManyReply.Content.WriteTo(fileStream);
}
return(bytesReceived);
}
private async Task <(long Chunks, long Bytes)> StreamContentAsync(Stream targetStream, IAsyncStreamReader <CopyFileResponse> replyStream, CancellationToken ct = default(CancellationToken))
{
long chunks = 0L;
long bytes = 0L;
while (await replyStream.MoveNext(ct).ConfigureAwait(false))
{
chunks++;
CopyFileResponse reply = replyStream.Current;
bytes += reply.Content.Length;
reply.Content.WriteTo(targetStream);
}
return(chunks, bytes);
}
private async Task <(long chunks, long bytes)> StreamContentAsync(Stream targetStream, IAsyncStreamReader <CopyFileResponse> replyStream, CopyToOptions?options, CancellationToken ct)
{
long chunks = 0L;
long bytes = 0L;
while (await replyStream.MoveNext(ct))
{
chunks++;
CopyFileResponse reply = replyStream.Current;
bytes += reply.Content.Length;
reply.Content.WriteTo(targetStream);
options?.UpdateTotalBytesCopied(bytes);
}
return(chunks, bytes);
}
public void SerializingCopyFileResponse()
{
var response = new CopyFileResponse()
{
Path = "a"
};
var bytes = Google.Protobuf.MessageExtensions.ToByteArray(response);
var expected = new byte[] { 10, 1, 97 };
Assert.Equal(expected, bytes);
var deserialized = CopyFileResponse.Parser.ParseFrom(bytes);
Assert.Equal("a", deserialized.Path);
}
private async Task <(long Chunks, long Bytes)> StreamContentAsync(Stream targetStream, IAsyncStreamReader <CopyFileResponse> replyStream, CancellationToken ct)
{
Contract.Requires(targetStream != null);
Contract.Requires(replyStream != null);
long chunks = 0L;
long bytes = 0L;
while (await replyStream.MoveNext(ct))
{
chunks++;
CopyFileResponse reply = replyStream.Current;
bytes += reply.Content.Length;
reply.Content.WriteTo(targetStream);
}
return(chunks, bytes);
}
private async Task <Result <(long Chunks, long Bytes)> > StreamContentAsync(Stream input, byte[] buffer, byte[] secondaryBuffer, IServerStreamWriter <CopyFileResponse> responseStream, CancellationToken ct)
{
long chunks = 0L;
long bytes = 0L;
byte[] inputBuffer = buffer;
// Pre-fill buffer with the file's first chunk
int chunkSize = await readNextChunk(input, inputBuffer, ct);
while (true)
{
ct.ThrowIfCancellationRequested();
if (chunkSize == 0)
{
break;
}
// Created ByteString can reuse the buffer.
// To avoid double writes we need two buffers:
// * One buffer is used as the output buffer
// * And another buffer is used as the input buffer.
(ByteString content, bool bufferReused) = CreateByteStringForStreamContent(inputBuffer, chunkSize);
CopyFileResponse response = new CopyFileResponse()
{
Content = content, Index = chunks
};
if (bufferReused)
{
// If the buffer is reused we need to swap the input buffer with the secondary buffer.
inputBuffer = inputBuffer == buffer ? secondaryBuffer : buffer;
}
bytes += chunkSize;
chunks++;
// Read the next chunk while waiting for the response
var readNextChunkTask = readNextChunk(input, inputBuffer, ct);
await Task.WhenAll(readNextChunkTask, responseStream.WriteAsync(response));
chunkSize = await readNextChunkTask;
}
return(chunks, bytes);
private async Task <Result <(long Chunks, long Bytes)> > StreamContentAsync(Stream input, byte[] buffer, IServerStreamWriter <CopyFileResponse> responseStream, CancellationToken ct)
{
Contract.Requires(!(input is null));
Contract.Requires(!(responseStream is null));
int chunkSize = 0;
long chunks = 0L;
long bytes = 0L;
// Pre-fill buffer with the file's first chunk
await readNextChunk();
while (true)
{
ct.ThrowIfCancellationRequested();
if (chunkSize == 0)
{
break;
}
ByteString content = ByteString.CopyFrom(buffer, 0, chunkSize);
CopyFileResponse response = new CopyFileResponse()
{
Content = content, Index = chunks
};
bytes += chunkSize;
chunks++;
// Read the next chunk while waiting for the response
await Task.WhenAll(readNextChunk(), responseStream.WriteAsync(response));
}
return(chunks, bytes);
async Task <int> readNextChunk()
{
chunkSize = await input.ReadAsync(buffer, 0, buffer.Length, ct); return(chunkSize);
}
}
/// <summary>
/// Copies file to stream
/// </summary>
public async Task <CopyFileResult> CopyFileAsync(Context context, ContentHash contentHash, AbsolutePath destinationPath, long fileSize = -1, bool enableCompression = false)
{
// TODO: Pipe through flag for compression type
CopyCompression compression = enableCompression ? CopyCompression.Gzip : CopyCompression.None;
AsyncServerStreamingCall <CopyFileResponse> response = _client.CopyFile(new CopyFileRequest()
{
TraceId = context.Id.ToString(),
HashType = (int)contentHash.HashType,
ContentHash = contentHash.ToByteString(),
Drive = destinationPath.DriveLetter.ToString(),
Offset = 0,
Compression = compression
});
IAsyncStreamReader <CopyFileResponse> replyStream = response.ResponseStream;
long bytesReceived = 0L;
using (var stream = new
FileStream(destinationPath.Path, FileMode.Create, FileAccess.Write, FileShare.None, ContentStore.Grpc.Utils.DefaultBufferSize, FileOptions.Asynchronous | FileOptions.SequentialScan))
{
bytesReceived = await StreamContentAsync(stream, response.ResponseStream);
while (await replyStream.MoveNext(CancellationToken.None))
{
CopyFileResponse oneOfManyReply = replyStream.Current;
bytesReceived += oneOfManyReply.Content.Length;
oneOfManyReply.Content.WriteTo(stream);
}
}
if (fileSize >= 0 && bytesReceived != fileSize)
{
return(new CopyFileResult(CopyFileResult.ResultCode.InvalidHash, $"Received {bytesReceived} bytes for {contentHash}, expected {fileSize}"));
}
return(CopyFileResult.Success);
}
private void CopyFileFinished(object sender, copyCompletedEventArgs e)
{
object[] state = (object[]) e.UserState;
OperationFinished<CopyFileResponse> copyFileCompleted = (OperationFinished<CopyFileResponse>) state[0];
CopyFileResponse copyFileResponse;
if (e.Error != null)
{
copyFileResponse = new CopyFileResponse
{
Error = e.Error,
Status = CopyObjectStatus.Failed,
UserState = state[1]
};
}
else
{
copyFileResponse = new CopyFileResponse
{
Status = StatusMessageParser.ParseCopyObjectStatus(e.Result),
UserState = state[1]
};
copyFileResponse.Error = copyFileResponse.Status == CopyObjectStatus.Unknown
?
new UnknownOperationStatusException(e.Result)
:
null;
}
copyFileCompleted(copyFileResponse);
}
