public async Task Create_ShallReturnAJob()
{
var mockClient = new Mock <IJobsClient>();
var mockLogger = new Mock <ILogger <ClaraJobsApi> >();
JobId.TryParse("jobid", out JobId jobId);
PayloadId.TryParse("payloadid", out PayloadId payloadId);
PipelineId.TryParse("pipelineid", out PipelineId pipelineId);
mockClient.Setup(p => p.CreateJob(It.IsAny <PipelineId>(), It.IsAny <string>(), It.IsAny <JobPriority>()))
.ReturnsAsync(new JobInfo
{
Name = "bla bla job",
JobId = jobId,
PayloadId = payloadId,
PipelineId = pipelineId
});
var service = new ClaraJobsApi(
mockClient.Object, mockLogger.Object);
var job = await service.Create(Guid.NewGuid().ToString(), "bla bla", JobPriority.Higher);
Assert.Equal(jobId.ToString(), job.JobId);
Assert.Equal(payloadId.ToString(), job.PayloadId);
mockClient.Verify(
p => p.CreateJob(It.IsAny <PipelineId>(), It.IsAny <string>(), JobPriority.Higher),
Times.Exactly(1));
mockLogger.VerifyLogging(LogLevel.Information, Times.Once());
mockLogger.VerifyLogging(LogLevel.Error, Times.Never());
}
public async Task <PayloadFile> Download(string payload, string name)
{
Guard.Against.NullOrWhiteSpace(payload, nameof(payload));
Guard.Against.NullOrWhiteSpace(name, nameof(name));
if (!PayloadId.TryParse(payload, out PayloadId payloadId))
{
throw new ArgumentException($"Invalid Payload ID received: {payload}");
}
return(await Policy <PayloadFile>
.Handle <Exception>()
.WaitAndRetryAsync(3, (r) => TimeSpan.FromSeconds(r * 1.5f), (data, retryCount, context) =>
{
_logger.Log(LogLevel.Error, "Failed to download file {0} from Payloads Service {1}: {2}", name, payload, data.Exception);
})
.ExecuteAsync(async() =>
{
PayloadFile file = new PayloadFile();
using (var ms = new MemoryStream())
{
var details = await _payloadsClient.DownloadFrom(payloadId, name, ms);
file.Data = ms.ToArray();
file.Name = details.Name;
}
_logger.Log(LogLevel.Information, "File {0} successfully downloaded from {1}.", name, payloadId);
return file;
}).ConfigureAwait(false));
}
public async Task Upload(string payload, string name, string filePath)
{
Guard.Against.NullOrWhiteSpace(payload, nameof(payload));
Guard.Against.NullOrWhiteSpace(name, nameof(name));
Guard.Against.NullOrWhiteSpace(filePath, nameof(filePath));
if (!PayloadId.TryParse(payload, out var payloadId))
{
throw new ArgumentException($"Invalid Payload ID received: {{{payload}}}.", nameof(payload));
}
using var loggerScope = _logger.BeginScope(new LogginDataDictionary <string, object> { { "PayloadId", payload }, { "Name", name }, { "File", filePath } });
await Policy.Handle <Exception>()
.WaitAndRetryAsync(3,
retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)),
(exception, retryCount, context) => _logger.Log(LogLevel.Error, "Exception while uploading file(s) to {{{0}}}: {exception}.", payload, exception))
.ExecuteAsync(async() =>
{
Stream stream;
try
{
stream = _fileSystem.File.OpenRead(filePath);
}
catch (System.Exception ex)
{
_logger.Log(LogLevel.Error, ex, "Error reading file.");
throw;
}
try
{
await _payloadsClient.UploadTo(payloadId, 0, name, stream);
_logger.Log(LogLevel.Debug, "File uploaded successfully.");
}
catch (PayloadUploadFailedException ex)
{
_logger.Log(LogLevel.Error, ex, "Error uploading file.");
throw;
}
finally
{
stream?.Dispose();
}
}).ConfigureAwait(false);
}
public async Task Upload(string payload, string basePath, IEnumerable <string> filePaths)
{
if (payload is null)
{
throw new ArgumentNullException(nameof(payload));
}
if (filePaths is null)
{
throw new ArgumentNullException(nameof(filePaths));
}
if (!PayloadId.TryParse(payload, out var payloadId))
{
throw new ArgumentException($"Invalid Payload ID received: {{{payload}}}.", nameof(payload));
}
var queue = new Queue <string>(filePaths);
var completedFiles = new List <PayloadFileDetails>();
basePath = EnsureBasePathEndsWithSlash(basePath);
await Policy.Handle <Exception>()
.WaitAndRetryAsync(3,
retryAttempt
=> TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)),
(exception, retryCount, context)
=> _logger.Log(LogLevel.Error, "Exception while uploading file(s) to {{{0}}}: {exception}.", payload, exception))
.ExecuteAsync(async() =>
{
var list = new List <(uint mode, string name, Stream stream)>();
var temp = new List <string>();
try
{
while (queue.Count > 0)
{
var filePath = queue.Dequeue();
temp.Add(filePath);
var filename = filePath.Replace(basePath, "");
if (System.Linq.Enumerable.Any(completedFiles, (PayloadFileDetails pfd) => OrdinalIgnoreCase.Equals(filename, pfd.Name)))
{
continue;
}
try
{
var stream = _fileSystem.File.OpenRead(filePath);
list.Add((0, filename, stream));
_logger.Log(LogLevel.Debug, "Ready to upload file \"{0}\" to PayloadId {1}.", filename, payloadId);
}
catch (System.Exception ex)
{
_logger.Log(LogLevel.Error, "Failed to open/read file {0}: {1}", filename, ex);
throw;
}
}
try
{
var uploadedFiles = await _payloadsClient.UploadTo(payloadId, list);
completedFiles.AddRange(uploadedFiles);
_logger.Log(LogLevel.Information, "{0} files uploaded to PayloadId {1}", completedFiles.Count, payloadId);
}
catch (PayloadUploadFailedException ex)
{
completedFiles.AddRange(ex.CompletedFiles);
temp.ForEach(file => queue.Enqueue(file));
if (completedFiles.Count != filePaths.Count())
{
throw;
}
_logger.Log(LogLevel.Information, "{0} files uploaded to PayloadId {1}", completedFiles.Count, payloadId);
}
catch
{
temp.ForEach(file => queue.Enqueue(file));
throw;
}
}
finally
{
for (var i = 0; i < list.Count; i += 1)
{
try
{
list[i].stream?.Dispose();
}
catch (Exception exception)
{
_logger.Log(LogLevel.Error, exception, $"Failed to dispose \"{list[i].name}\" stream.");
}
}
}
}).ConfigureAwait(false);
}