/// <inheritdoc />
public async Task <Stream> DownloadFileAsync(
GitHost host,
string repositoryName,
ObjectId objectId,
BatchObjectAction action)
{
using (var httpClient = new HttpClient())
{
SetClientHeaders(action, httpClient);
this.logger.LogInformation(
$"Download from {action.HRef} with repository name {repositoryName}, request:{objectId.Hash.Substring(0, 10)}/{objectId.Size}");
HttpResponseMessage result = await httpClient.GetAsync(action.HRef, HttpCompletionOption.ResponseHeadersRead).ConfigureAwait(false);
if (!result.IsSuccessStatusCode)
{
await this.HandleErrorAsync(result).ConfigureAwait(false);
}
this.logger.LogInformation(
$"Now saving to a file {repositoryName}, request:{objectId.Hash.Substring(0, 10)}/{objectId.Size}\"");
return(this.fileManager.SaveFile(
out string fileName,
repositoryName,
objectId,
FileLocation.Permanent,
await result.Content.ReadAsStreamAsync().ConfigureAwait(false)));
}
}
public async Task <IActionResult> UploadFile(int hostId, string repositoryName, string objectId, long size)
{
var fileObjectId = new ObjectId(objectId, size);
try
{
GitHost host = await this.context.GitHost.FindAsync(hostId);
if (host == null)
{
this.logger.LogWarning("Not a valid host id");
return(this.NotFound(new ErrorResponse {
Message = "Not a valid host id"
}));
}
this.logger.LogInformation($"Saving the file to disk for {fileObjectId}");
this.Response.Headers.Remove("transfer-encoding");
if (!this.fileManager.IsFileStored(
repositoryName,
fileObjectId,
FileLocation.Metadata,
false,
"upload"))
{
this.fileManager.DeleteFile(repositoryName, fileObjectId, FileLocation.Metadata, "upload");
return(this.Ok());
}
BatchObjectAction action = this.transferSerialiser.ObjectActionFromString(await this.fileManager.GetFileContentsAsync(repositoryName, fileObjectId, FileLocation.Metadata, "upload"));
this.logger.LogInformation($"Starting file upload for {fileObjectId}");
await this.lfsClient.UploadFile(action, this.Request.Body);
this.logger.LogInformation($"Finished file upload for {fileObjectId}");
this.fileManager.DeleteFile(repositoryName, fileObjectId, FileLocation.Metadata, "upload");
return(this.Ok());
}
catch (ErrorResponseException ex)
{
this.logger.LogWarning(null, ex, $"Received a error response with message {ex.Message} for {fileObjectId}");
this.fileManager.DeleteFile(repositoryName, fileObjectId, FileLocation.Temporary);
return(this.StatusCode(ex.StatusCode.Value, ex.ErrorResponse));
}
catch (StatusCodeException ex)
{
this.logger.LogWarning(null, ex, $"Received a status code error with message {ex.Message} for {fileObjectId}");
this.fileManager.DeleteFile(repositoryName, fileObjectId, FileLocation.Temporary);
return(this.StatusCode(ex.StatusCode.Value, new ErrorResponse {
Message = ex.Message
}));
}
}
private static void SetClientHeaders(BatchObjectAction action, HttpClient downloadHttpClient)
{
if (action.Headers != null)
{
foreach (BatchHeader header in action.Headers)
{
downloadHttpClient.DefaultRequestHeaders.TryAddWithoutValidation(header.Key, header.Value);
}
}
downloadHttpClient.DefaultRequestHeaders.Add("User-Agent", "glennawatson");
}
public void DeserialiseObjectActionTest()
{
var testData = "{\"download\": { \"href\": \"https://github-cloud.s3.amazonaws.com/alambic/media/148182185/0a/f2/0af22f9dadcc067af162661e1b302481eefef87b36d29b75499d0f788092d807?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIMWPLRQEC4XCWWPA%2F20170522%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20170522T051527Z&X-Amz-Expires=86400&X-Amz-Signature=f6c916650fe11c8ec33ed73c727a9dcfd71beac8569bc3819ca4634055ba8a38&X-Amz-SignedHeaders=host&actor_id=28613580&token=1\", \"expires_in\": 3600, \"expires_at\": \"2017-05-23T05:15:27Z\"}}";
var action = new BatchObjectAction()
{
HRef = "https://github-cloud.s3.amazonaws.com/alambic/media/148182185/0a/f2/0af22f9dadcc067af162661e1b302481eefef87b36d29b75499d0f788092d807?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIMWPLRQEC4XCWWPA%2F20170522%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20170522T051527Z&X-Amz-Expires=86400&X-Amz-Signature=f6c916650fe11c8ec33ed73c727a9dcfd71beac8569bc3819ca4634055ba8a38&X-Amz-SignedHeaders=host&actor_id=28613580&token=1",
ExpiresIn = 3600,
ExpiresAt = DateTime.Parse("2017-05-23T05:15:27Z", CultureInfo.InvariantCulture)
};
var serialiser = new JsonBatchTransferSerialiser();
var shouldBeJson = serialiser.ToString(action);
BatchObjectAction testAction = serialiser.ObjectActionFromString(testData);
}
public async Task <IActionResult> Verify(int hostId, string repositoryName, string objectId, long size)
{
var fileObjectId = new ObjectId(objectId, size);
try
{
GitHost host = await this.context.GitHost.FindAsync(hostId);
if (host == null)
{
return(this.NotFound(new ErrorResponse {
Message = "Not a valid host id."
}));
}
this.fileManager.MoveFile(repositoryName, fileObjectId, FileLocation.Temporary, FileLocation.Permenant);
if (this.fileManager.IsFileStored(repositoryName, fileObjectId, FileLocation.Metadata, false, "verify"))
{
this.logger.LogInformation($"Starting verify for {fileObjectId}");
BatchObjectAction action = this.transferSerialiser.ObjectActionFromString(await this.fileManager.GetFileContentsAsync(repositoryName, fileObjectId, FileLocation.Metadata, "verify"));
await this.lfsClient.Verify(host, repositoryName, fileObjectId, action);
this.logger.LogInformation($"Ending verify for {fileObjectId}");
}
this.fileManager.DeleteFile(repositoryName, fileObjectId, FileLocation.Metadata, "verify");
return(this.Ok());
}
catch (ErrorResponseException ex)
{
this.logger.LogWarning(null, ex, $"Received a error response with message {ex.Message} for {fileObjectId}");
return(this.StatusCode(ex.StatusCode.Value, ex.ErrorResponse));
}
catch (StatusCodeException ex)
{
this.logger.LogWarning(null, ex, $"Received a status code error with message {ex.Message} for {fileObjectId}");
return(this.StatusCode(ex.StatusCode.Value, new ErrorResponse {
Message = ex.Message
}));
}
}
/// <inheritdoc />
public async Task UploadFileAsync(BatchObjectAction action, Stream stream)
{
using (var httpClient = new HttpClient())
{
SetClientHeaders(action, httpClient);
var content = new StreamContent(stream, 2000);
content.Headers.ContentType = MediaTypeHeaderValue.Parse("application/octet-stream");
this.logger.LogInformation($"Uploading to {action.HRef}");
HttpResponseMessage result = await httpClient.PutAsync(action.HRef, content).ConfigureAwait(false);
if (!result.IsSuccessStatusCode)
{
this.logger.LogInformation($"Failed uploading to {action.HRef}");
await this.HandleErrorAsync(result).ConfigureAwait(false);
}
else
{
this.logger.LogInformation(
$"Success uploading to {action.HRef}");
}
}
}
/// <inheritdoc />
public async Task VerifyAsync(GitHost host, string repositoryName, ObjectId objectId, BatchObjectAction action)
{
using (var httpClient = new HttpClient())
{
SetClientHeaders(action, httpClient);
using (var content = new StringContent(
this.verifySerialiser.ToString(objectId),
null,
"application/vnd.git-lfs+json"))
{
this.logger.LogInformation(
$"Verify from {action.HRef} with repository name {repositoryName}, request:{objectId.Hash.Substring(0, 10)}/{objectId.Size}");
HttpResponseMessage result = await httpClient.PostAsync(action.HRef, content).ConfigureAwait(false);
if (!result.IsSuccessStatusCode)
{
await this.HandleErrorAsync(result).ConfigureAwait(false);
}
}
}
}
public async Task <IActionResult> DownloadFile(int hostId, string repositoryName, string objectId, long size)
{
var fileObjectId = new ObjectId(objectId, size);
try
{
GitHost host = await this.context.GitHost.FindAsync(hostId).ConfigureAwait(false);
if (host == null)
{
return(this.NotFound(new ErrorResponse {
Message = "Not a valid host id."
}));
}
Stream stream = this.fileManager.GetFileStream(repositoryName, fileObjectId, FileLocation.Permanent);
if (stream != null)
{
this.logger.LogInformation($"Found local cache file: {fileObjectId}");
return(this.File(stream, "application/octet-stream"));
}
var actionStream = await this.fileManager.GetFileContentsAsync(repositoryName, fileObjectId, FileLocation.Metadata, "download").ConfigureAwait(false);
if (actionStream == null)
{
this.logger.LogWarning($"No valid batch request for {fileObjectId}");
return(this.NotFound(new ErrorResponse {
Message = $"No valid batch request for {fileObjectId}"
}));
}
BatchObjectAction action = this.transferSerialiser.ObjectActionFromString(actionStream);
if (action == null)
{
this.logger.LogWarning($"Unable to find file {objectId}");
return(this.NotFound(new ErrorResponse {
Message = $"Unable to find file {objectId}"
}));
}
if (action.Mode != BatchActionMode.Download)
{
this.logger.LogWarning($"No download action associated with request: {objectId}");
return(this.StatusCode(
422,
new ErrorResponse {
Message = $"No download action associated with request: {objectId}"
}));
}
this.fileManager.DeleteFile(repositoryName, fileObjectId, FileLocation.Metadata, "download");
return(this.File(
await this.lfsClient.DownloadFileAsync(
host,
repositoryName,
fileObjectId,
action).ConfigureAwait(false),
"application/octet-stream"));
}
catch (ErrorResponseException ex)
{
this.logger.LogWarning(null, ex, $"Received a error response with message {ex.Message} for {fileObjectId}");
return(this.StatusCode(ex.StatusCode ?? DefaultErrorCode, ex.ErrorResponse));
}
catch (StatusCodeException ex)
{
this.logger.LogWarning(null, ex, $"Received a status code error with message {ex.Message}");
return(this.StatusCode(ex.StatusCode ?? DefaultErrorCode, new ErrorResponse {
Message = ex.Message
}));
}
}
public async Task <IActionResult> HandleBatchRequest(
int hostId,
string repositoryName,
[FromBody] BatchRequest request)
{
GitHost host = await this.context.GitHost.FindAsync(hostId).ConfigureAwait(false);
if (host == null)
{
return(this.NotFound(new ErrorResponse {
Message = "Not a valid host id."
}));
}
var serverBatchRequest = new BatchRequest
{
Objects = new List <ObjectId>(),
Operation = request.Operation,
Transfers = request.Transfers
};
foreach (ObjectId objectId in request.Objects)
{
if (request.Operation == BatchRequestMode.Download &&
this.fileManager.IsFileStored(repositoryName, objectId, FileLocation.Permanent))
{
continue;
}
serverBatchRequest.Objects.Add(objectId);
}
try
{
IDictionary <ObjectId, IBatchObject> batchObjects = new Dictionary <ObjectId, IBatchObject>();
if (serverBatchRequest.Objects.Count > 0)
{
BatchTransfer serverResults =
await this.lfsClient.RequestBatchAsync(host, repositoryName, serverBatchRequest).ConfigureAwait(false);
foreach (IBatchObject serverResult in serverResults.Objects)
{
batchObjects.Add(serverResult.Id, serverResult);
if (serverResult is BatchObject batchObject)
{
foreach (BatchObjectAction action in batchObject.Actions)
{
await this.fileManager.SaveFileAsync(
repositoryName,
serverResult.Id,
FileLocation.Metadata,
this.transferSerialiser.ToString(action),
action.Mode.ToString().ToLowerInvariant()).ConfigureAwait(false);
}
}
}
}
var returnResult = new BatchTransfer();
returnResult.Mode = TransferMode.Basic;
returnResult.Objects = new List <IBatchObject>();
foreach (ObjectId pendingObjectId in request.Objects)
{
batchObjects.TryGetValue(pendingObjectId, out IBatchObject batchObjectBase);
var batchObject = batchObjectBase as BatchObject;
if (batchObjectBase is BatchObjectError errorResult)
{
returnResult.Objects.Add(errorResult);
}
else
{
switch (request.Operation)
{
case BatchRequestMode.Upload:
returnResult.Objects.Add(batchObject);
break;
case BatchRequestMode.Download:
{
var returnBatchObject = new BatchObject {
Id = pendingObjectId, Actions = new List <BatchObjectAction>()
};
var action = new BatchObjectAction
{
Mode = BatchActionMode.Download,
HRef =
$"{this.Request.Scheme}://{this.Request.Host}/api/{hostId}/{repositoryName}/info/lfs/{pendingObjectId.Hash}/{pendingObjectId.Size}"
};
returnBatchObject.Actions.Add(action);
returnResult.Objects.Add(returnBatchObject);
break;
}
}
}
}
return(this.Ok(returnResult));
}
catch (ErrorResponseException ex)
{
this.logger.LogWarning(null, ex, $"Received a error response with message {ex.Message}");
return(this.StatusCode(ex.StatusCode ?? DefaultErrorCode, ex.ErrorResponse));
}
catch (StatusCodeException ex)
{
this.logger.LogWarning(null, ex, $"Received a status code error with message {ex.Message}");
return(this.StatusCode(ex.StatusCode ?? DefaultErrorCode, new ErrorResponse {
Message = ex.Message
}));
}
}
