public async Task <IActionResult> GetAsync(string path, long length, long bytesPerChunk)
{
// 0. Wait
await Task.Delay(Constants.FILESTITCH_DELAY);
// 1. Determine which chunk is next up
var chunkFolder = $"{path}.{Constants.PARTIAL}";
FileInfo fileinfo = null;
try
{
policyFile.Execute(() => fileinfo = new FileInfo(path));
}
catch (Exception ex) when(ex is SecurityException || ex is UnauthorizedAccessException)
{
return(Forbid());
}
catch (Exception ex) when(ex is FileNotFoundException || ex is DirectoryNotFoundException)
{
return(NotFound());
}
var fileLength = fileinfo.Length;
if (fileLength == length)
{
try
{
policyFile.Execute(() => Directory.Delete(chunkFolder));
return(Ok());
}
catch (Exception ex) when(ex is SecurityException || ex is UnauthorizedAccessException)
{
return(Forbid());
}
catch (Exception ex) when(ex is FileNotFoundException || ex is DirectoryNotFoundException)
{
return(NotFound());
}
}
long nextChunk = fileLength / bytesPerChunk;
// 2. Determine if file is ready for next chunk
bool fileReady = fileLength % bytesPerChunk == 0;
// 3. Determine if next-up chunk is ready to go
var chunkPath = $"{chunkFolder}/{nextChunk.ToString(Constants.ID_FORMAT)}";
var chunkinfo = new FileInfo(chunkPath);
var lastChunk = length % bytesPerChunk != 0 ? length / bytesPerChunk + 1 : length / bytesPerChunk;
var lastChunkLength = length - lastChunk * bytesPerChunk;
bool chunkReady = chunkinfo.Length == bytesPerChunk || (nextChunk == lastChunk && chunkinfo.Length == lastChunkLength);
// 4. If chunk is ready to be stitched, stitch
if (chunkReady)
{
try
{
await policy.ExecuteAsync(async() =>
{
using (var chunkStream = chunkinfo.Open(FileMode.Open, FileAccess.Read, FileShare.None))
using (var fileStream = fileinfo.Open(FileMode.Append, FileAccess.Write, FileShare.None))
{
await chunkStream.CopyToAsync(fileStream);
}
chunkinfo.Delete();
});
}
catch (Exception ex) when(ex is SecurityException || ex is UnauthorizedAccessException)
{
return(Forbid());
}
catch (Exception ex) when(ex is FileNotFoundException || ex is DirectoryNotFoundException)
{
return(NotFound());
}
}
// 5. Let someone else do the rest of the work
var builder = new HttpRequestBuilder();
var stitchRequest = builder.SetPath("/api/filestitch")
.AddQuery("path", path)
.AddQuery("length", length.ToString())
.AddQuery("bytesPerChunk", bytesPerChunk.ToString())
.Build();
var stitchMessage = httpMessageFactory.CreateMessage(HttpContext.Request.Headers, stitchRequest);
await policy.ExecuteAsync(() => topic.SendAsync(stitchMessage));
return(Ok());
}
public async Task <IActionResult> GetAsync(string path)
{
long bytesPerMessage = config.GetBytesPerMessage();
// 1. Check the file exists
var fileName = Uri.UnescapeDataString(path);
var partial = $"{fileName}.{Constants.PARTIAL}";
try
{
FileInfo file = null;
policyFile.Execute(() => file = new FileInfo(fileName));
if (file.Exists)
{
return(Conflict());
}
policyFile.Execute(() => Directory.CreateDirectory(partial));
}
catch (Exception ex) when(ex is SecurityException || ex is UnauthorizedAccessException)
{
return(Forbid());
}
catch (Exception ex) when(ex is FileNotFoundException || ex is DirectoryNotFoundException)
{
return(NotFound());
}
// 2. Check the blob exists
var blob = container.GetBlockBlobReference(path);
if (!await policy.ExecuteAsync(blob.ExistsAsync))
{
return(NotFound());
}
// 2. calculate number of message queue messages to create
await policy.ExecuteAsync(blob.FetchAttributesAsync);
var blobLength = blob.Properties.Length;
long numMessages = blobLength % bytesPerMessage != 0 ? blobLength / bytesPerMessage + 1 : blobLength / bytesPerMessage;
// 3. setup reusable objects to minimize `new` memory
int messagesPerSend = config.GetMessagesPerSend();
var builder = new HttpRequestBuilder()
.SetHost("/")
.SetMethod("get")
.SetPath("/api/filechunk");
foreach (var header in HttpContext.Request.Headers)
{
builder.AddHeader(header.Key, header.Value);
}
var httpRequestArray = Enumerable.Range(0, messagesPerSend).Select(i => builder.Build()).ToArray();
var messageArray = Enumerable.Range(0, messagesPerSend).Select(i => messageFactory.CreateMessage(HttpContext.Request.Headers)).ToArray();
var messageList = new List <Message>(messagesPerSend);
// 4. bin messages into bulk groups and send by bulk: outer loop defines bulk bin, inner loop populates bulk bin
for (long i = 0; i < numMessages; i += messagesPerSend)
{
messageList.Clear();
for (long j = i; j < numMessages && j < messagesPerSend; j++)
{
var request = httpRequestArray[j % messagesPerSend];
request.Query = $"path={path}&length={bytesPerMessage}&startindex = {j * bytesPerMessage}";
var message = messageArray[j % messagesPerSend];
message.Body = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(request));
messageList.Add(message);
}
// 5. send messages in bulk to topic
await policy.ExecuteAsync(() => topic.SendAsync(messageList));
}
// 6. send a message to a controller that can stitch chunks together
var stitchRequest = builder.SetPath("/api/filestitch")
.AddQuery("path", path)
.AddQuery("length", blobLength.ToString())
.AddQuery("bytesPerChunk", bytesPerMessage.ToString())
.Build();
var stitchMessage = messageFactory.CreateMessage(HttpContext.Request.Headers, stitchRequest);
await policy.ExecuteAsync(async() => await topic.SendAsync(stitchMessage));
return(Ok());
}