public async Task <UploadContextModel> UploadSendChunkAsync(
UploadContextModel uploadContext,
int chunkIndex,
long rangeFrom,
long rangeTo,
Stream stream,
CancellationToken cancellationToken = default(CancellationToken))
{
using (var request = await Connection.RequestAsync(new HttpMethod("PATCH"), APIEndpoint.FileUploadChunk.Endpoint, new
{
uploadContext,
chunkIndex,
}))
{
// todo: hmm... will this let us reuse a stream at current offset? no good for retries.
request.Content = new StreamContent(stream);
request.Content.Headers.Add("Content-Range",
new ContentRangeHeaderValue(
rangeFrom,
rangeTo,
uploadContext.FileLength
).ToString()
);
request.Headers.TransferEncodingChunked = true;
using (var response = await Connection.SendAsync(request))
{
await Connection.DocumentsEnsureSuccessStatus(response);
return(await Connection.ReadAsAsync <UploadContextModel>(response));
}
}
}
public async Task <UploadContextModel> PatchRange(
UploadContextModel uploadContext,
int chunkIndex,
CancellationToken cancellationToken = default(CancellationToken)
)
{
if (Request.Headers["Content-Range"].Count == 0)
{
throw new Exception("Missing required Content-Range header");
}
var range = ContentRangeHeaderValue.Parse(Request.Headers["Content-Range"].ToString());
if (range.From == null)
{
throw new ArgumentNullException("Range Header From must be specified");
}
if (range.To == null)
{
throw new ArgumentNullException("Range Header To must be specified");
}
if ((long)range.Length != uploadContext.FileLength)
{
throw new ArgumentNullException("Range Header Length does not match UploadContext.FileLength");
}
return(await FileContentsService.UploadChunkAsync(
uploadContext, (long)range.From, (long)range.To, Request.Body, chunkIndex, cancellationToken));
}
public async Task <UploadContextModel> UploadChunkAsync(
UploadContextModel uploadContext,
long from,
long to,
Stream inputStream,
int chunkIndex,
CancellationToken cancellationToken = default(CancellationToken)
)
{
var uploadStateToken = UploadTokenModel.Parse(uploadContext.UploadToken);
var fileLocator = await FileStore.FileLocatorGetAsync(uploadStateToken.Identifier);
var uploadChunk = new UploadChunkModel
{
Identifier = new UploadChunkIdentifier(uploadStateToken.Identifier, fileLocator + '.' + chunkIndex.ToString()),
ChunkIndex = chunkIndex,
PositionFrom = from,
PositionTo = to
};
uploadContext.SequentialState = await BackendClient.UploadChunkAsync(
await LoadConfigurationAsync(uploadStateToken.Identifier as OrganizationIdentifier),
uploadStateToken.Identifier.UploadKey,
fileLocator,
uploadChunk.Identifier.UploadChunkKey,
chunkIndex,
uploadContext.TotalChunks,
uploadContext.SequentialState,
uploadChunk.PositionFrom,
uploadChunk.PositionTo,
uploadContext.FileLength,
inputStream,
cancellationToken
);
uploadChunk.State = uploadContext.SequentialState;
uploadChunk.Success = true;
await UploadChunkStore.InsertAsync(uploadChunk);
return(uploadContext);
}
public async Task <FileModel> UploadCompleteAsync(
UploadContextModel uploadContext,
CancellationToken cancellationToken = default(CancellationToken)
)
{
var uploadStateToken = UploadTokenModel.Parse(uploadContext.UploadToken);
var upload = await UploadStore.GetOneAsync(uploadStateToken.Identifier, new[]
{
new PopulationDirective
{
Name = nameof(UploadModel.Chunks)
}
});
var chunkStates = upload.Chunks?.Rows
.OrderBy(c => c.ChunkIndex)
.Select(c => new ChunkedStatusModel
{
ChunkIndex = c.ChunkIndex,
UploadChunkKey = c.Identifier.UploadChunkKey,
State = c.State,
Success = true
})
.ToArray();
var fileLocator = await FileStore.FileLocatorGetAsync(uploadStateToken.Identifier);
var returnData = await BackendClient.CompleteChunkedUploadAsync(
await LoadConfigurationAsync(uploadStateToken.Identifier as OrganizationIdentifier),
uploadStateToken.Identifier.UploadKey,
fileLocator,
chunkStates
);
var fileModel = await FileStore.GetOneAsync(uploadStateToken.Identifier);
await FileStore.UpdateAsync(fileModel);
if (returnData != null)
{
await FileStore.HashSetAsync(
fileModel.Identifier,
GetHash(returnData, "md5"),
GetHash(returnData, "sha1"),
GetHash(returnData, "sha256")
);
}
await FileStore.UploadingStatusSetAsync(fileModel.Identifier, false);
await UploadStore.Cleanup(upload.Identifier);
var evt = new FileContentsUploadCompleteEvent
{
FileIdentifier = fileModel.Identifier
};
evt.Populate(fileModel);
await EventSender.SendAsync(evt);
return(fileModel);
}
public Task <FileModel> UploadEndAsync(UploadContextModel uploadContext, CancellationToken cancellationToken = default(CancellationToken)) => Connection.APICallAsync <FileModel>(HttpMethod.Post, APIEndpoint.FileUploadEnd, bodyContent: uploadContext, cancellationToken: cancellationToken);
public async Task <ActionResult> UploadChunk(
PathIdentifier pathIdentifier,
BrowserFileInformation fileInformation,
string token,
CancellationToken cancellationToken = default(CancellationToken)
)
{
if (!Request.ContentLength.HasValue)
{
throw new Exception("Missing Content-Length header");
}
if (Request.Headers["Content-Range"].Count == 0)
{
throw new Exception("Missing Content-Range header");
}
if (Request.Headers["Content-Disposition"].Count == 0)
{
throw new Exception("Missing Content-Disposition header");
}
var range = ContentRangeHeaderValue.Parse(Request.Headers["Content-Range"]);
if (!range.HasLength)
{
throw new Exception("Content-Range header does not include total length");
}
long from = (long)range.From;
long to = (long)range.To;
long fileLength = (long)range.Length;
var fileName = fileInformation?.Name;
if (fileName == null)
{
var contentDisposition = ContentDispositionHeaderValue.Parse(Request.Headers["Content-Disposition"]);
fileName = contentDisposition?.FileName;
if (fileName == null)
{
throw new Exception("Filename is not specified in either Content-Disposition header or fileInformation");
}
// for some dumb reason, the ContentDispositionHeaderValue parser doesn't finish parsing file names
// it leaves them quoted
if (fileName.StartsWith('"') && fileName.EndsWith('"') && fileName.Length > 1)
{
fileName = WebUtility.UrlDecode(fileName.Substring(1, fileName.Length - 2));
}
}
Stream stream = Request.Body;
UploadContextModel tokenState = null;
// test retries
//if ((new Random()).NextDouble() < .3)
// throw new Exception("Chaos Monkey");
bool isFirstChunk = from == 0;
bool isLastChunk = to == (fileLength - 1);
FileIdentifier fileIdentifier = null;
var folderModel = await Connection.Folder.GetOrThrowAsync(pathIdentifier);
var modules = ModuleConfigurator.GetActiveModules(folderModel);
// === Step 1: Begin Upload
if (isFirstChunk)
{
var fileModel = new FileModel
{
Identifier = new FileIdentifier(pathIdentifier, null),
Name = fileName,
Modified = fileInformation?.LastModified != null
? epoch.AddMilliseconds(fileInformation.LastModified.Value)
: DateTime.UtcNow,
Length = fileLength,
MimeType = fileInformation?.Type ?? Request.ContentType ?? "application/octet-stream",
}.InitializeEmptyMetadata();
fileModel.Created = new DateTime(Math.Min(DateTime.UtcNow.Ticks, fileModel.Modified.Ticks), DateTimeKind.Utc);
// some browsers will send us the relative path of a file during a folder upload
var relativePath = fileInformation?.FullPath;
if (!string.IsNullOrWhiteSpace(relativePath) && relativePath != "/")
{
if (relativePath.StartsWith("/"))
{
relativePath = relativePath.Substring(1);
}
pathIdentifier = pathIdentifier.CreateChild(relativePath);
}
fileModel.MetaPathIdentifierWrite(pathIdentifier);
foreach (var module in modules)
{
await module.PreUploadAsync(folderModel, fileModel);
}
tokenState = await Connection.File.UploadBeginAsync(fileModel, cancellationToken : cancellationToken);
}
else
{
if (token == null)
{
throw new Exception("Uploaded secondary chunk without token");
}
tokenState = JsonConvert.DeserializeObject <UploadContextModel>(token);
}
// === Step 2: Send this Chunk
using (stream)
{
if (!isLastChunk)
{
if (to - from != tokenState.ChunkSize - 1)
{
throw new Exception($"Chunk Size Mismatch: received ({to - from}) expected ({tokenState.ChunkSize})");
}
}
int chunkIndex = (int)(from / tokenState.ChunkSize);
tokenState = await Connection.File.UploadSendChunkAsync(
tokenState,
chunkIndex,
from,
to,
stream,
cancellationToken : cancellationToken
);
}
// === Step 3: End the Upload
if (isLastChunk) // if file is one chunk, all three steps happen in a single call
{
var fileModel = await Connection.File.UploadEndAsync(tokenState, cancellationToken : cancellationToken);
fileIdentifier = fileModel.Identifier;
foreach (var module in modules)
{
await module.PostUploadAsync(folderModel, fileModel);
}
}
return(Json(new APIResponse <UploadedFileResponse>
{
Response = new UploadedFileResponse
{
FileIdentifier = fileIdentifier,
Token = JsonConvert.SerializeObject(tokenState)
}
}));
}
public async Task <FileModel> PostEnd([FromBody] UploadContextModel uploadContext)
{
return(await FileContentsService.UploadCompleteAsync(uploadContext));
}
