public async Task <PayloadFile> Download(string payload, string name)
{
Guard.Against.NullOrWhiteSpace(payload, nameof(payload));
Guard.Against.NullOrWhiteSpace(name, nameof(name));
if (!PayloadId.TryParse(payload, out PayloadId payloadId))
{
throw new ArgumentException($"Invalid Payload ID received: {payload}");
}
return(await Policy <PayloadFile>
.Handle <Exception>()
.WaitAndRetryAsync(3, (r) => TimeSpan.FromSeconds(r * 1.5f), (data, retryCount, context) =>
{
_logger.Log(LogLevel.Error, "Failed to download file {0} from Payloads Service {1}: {2}", name, payload, data.Exception);
})
.ExecuteAsync(async() =>
{
PayloadFile file = new PayloadFile();
using (var ms = new MemoryStream())
{
var details = await _payloadsClient.DownloadFrom(payloadId, name, ms);
file.Data = ms.ToArray();
file.Name = details.Name;
}
_logger.Log(LogLevel.Information, "File {0} successfully downloaded from {1}.", name, payloadId);
return file;
}).ConfigureAwait(false));
}
public async Task Create_ShallReturnAJob()
{
var mockClient = new Mock <IJobsClient>();
var mockLogger = new Mock <ILogger <ClaraJobsApi> >();
JobId.TryParse("jobid", out JobId jobId);
PayloadId.TryParse("payloadid", out PayloadId payloadId);
PipelineId.TryParse("pipelineid", out PipelineId pipelineId);
mockClient.Setup(p => p.CreateJob(It.IsAny <PipelineId>(), It.IsAny <string>(), It.IsAny <JobPriority>()))
.ReturnsAsync(new JobInfo
{
Name = "bla bla job",
JobId = jobId,
PayloadId = payloadId,
PipelineId = pipelineId
});
var service = new ClaraJobsApi(
mockClient.Object, mockLogger.Object);
var job = await service.Create(Guid.NewGuid().ToString(), "bla bla", JobPriority.Higher);
Assert.Equal(jobId.ToString(), job.JobId);
Assert.Equal(payloadId.ToString(), job.PayloadId);
mockClient.Verify(
p => p.CreateJob(It.IsAny <PipelineId>(), It.IsAny <string>(), JobPriority.Higher),
Times.Exactly(1));
mockLogger.VerifyLogging(LogLevel.Information, Times.Once());
mockLogger.VerifyLogging(LogLevel.Error, Times.Never());
}
/// <summary>
/// Serves as the default hash function.
/// </summary>
/// <returns>A hash code for the current object.</returns>
public override int GetHashCode()
{
unchecked
{
int hashCode = PayloadId != null?PayloadId.GetHashCode() : 0;
hashCode = (hashCode * 397) ^ (DownloadUrl != null ? DownloadUrl.GetHashCode() : 0);
hashCode = (hashCode * 397) ^ (int)Status;
return(hashCode);
}
}
public IPayloadComponentId RegisterPayloadComponent(IPayloadDefinition definition)
{
IPayloadComponentId id = new PayloadId(payloads.Count);
var internalDefinition = new InternalPayloadDefinition(definition, id);
if (payloadHashAlgorithm != null)
{
internalDefinition.IncrementalHash(payloadHashAlgorithm);
}
payloads.Add(internalDefinition);
return(id);
}
public async Task Upload(string payload, string name, string filePath)
{
Guard.Against.NullOrWhiteSpace(payload, nameof(payload));
Guard.Against.NullOrWhiteSpace(name, nameof(name));
Guard.Against.NullOrWhiteSpace(filePath, nameof(filePath));
if (!PayloadId.TryParse(payload, out var payloadId))
{
throw new ArgumentException($"Invalid Payload ID received: {{{payload}}}.", nameof(payload));
}
using var loggerScope = _logger.BeginScope(new LogginDataDictionary <string, object> { { "PayloadId", payload }, { "Name", name }, { "File", filePath } });
await Policy.Handle <Exception>()
.WaitAndRetryAsync(3,
retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)),
(exception, retryCount, context) => _logger.Log(LogLevel.Error, "Exception while uploading file(s) to {{{0}}}: {exception}.", payload, exception))
.ExecuteAsync(async() =>
{
Stream stream;
try
{
stream = _fileSystem.File.OpenRead(filePath);
}
catch (System.Exception ex)
{
_logger.Log(LogLevel.Error, ex, "Error reading file.");
throw;
}
try
{
await _payloadsClient.UploadTo(payloadId, 0, name, stream);
_logger.Log(LogLevel.Debug, "File uploaded successfully.");
}
catch (PayloadUploadFailedException ex)
{
_logger.Log(LogLevel.Error, ex, "Error uploading file.");
throw;
}
finally
{
stream?.Dispose();
}
}).ConfigureAwait(false);
}
public override Task <DeepestPayload> GetDeepestPayload(PayloadId request, ServerCallContext context)
{
DeepestPayload dp = _db.GetDeepestPayload(request.Id);
return(Task.FromResult(dp));
}
public override Task <LargePayload> GetLargePayload(PayloadId request, ServerCallContext context)
{
LargePayload lp = _db.GetLargePayload(request.Id);
return(Task.FromResult(lp));
}
public override Task <MediumPayload> GetMediumPayload(PayloadId request, ServerCallContext context)
{
MediumPayload mp = _db.GetMediumPayload(request.Id);
return(Task.FromResult(mp));
}
//Get (id)
public override Task <SmallPayload> GetSmallPayload(PayloadId request, ServerCallContext context)
{
SmallPayload sp = _db.GetSmallPayload(request.Id);
return(Task.FromResult(sp));
}
public async Task Upload(string payload, string basePath, IEnumerable <string> filePaths)
{
if (payload is null)
{
throw new ArgumentNullException(nameof(payload));
}
if (filePaths is null)
{
throw new ArgumentNullException(nameof(filePaths));
}
if (!PayloadId.TryParse(payload, out var payloadId))
{
throw new ArgumentException($"Invalid Payload ID received: {{{payload}}}.", nameof(payload));
}
var queue = new Queue <string>(filePaths);
var completedFiles = new List <PayloadFileDetails>();
basePath = EnsureBasePathEndsWithSlash(basePath);
await Policy.Handle <Exception>()
.WaitAndRetryAsync(3,
retryAttempt
=> TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)),
(exception, retryCount, context)
=> _logger.Log(LogLevel.Error, "Exception while uploading file(s) to {{{0}}}: {exception}.", payload, exception))
.ExecuteAsync(async() =>
{
var list = new List <(uint mode, string name, Stream stream)>();
var temp = new List <string>();
try
{
while (queue.Count > 0)
{
var filePath = queue.Dequeue();
temp.Add(filePath);
var filename = filePath.Replace(basePath, "");
if (System.Linq.Enumerable.Any(completedFiles, (PayloadFileDetails pfd) => OrdinalIgnoreCase.Equals(filename, pfd.Name)))
{
continue;
}
try
{
var stream = _fileSystem.File.OpenRead(filePath);
list.Add((0, filename, stream));
_logger.Log(LogLevel.Debug, "Ready to upload file \"{0}\" to PayloadId {1}.", filename, payloadId);
}
catch (System.Exception ex)
{
_logger.Log(LogLevel.Error, "Failed to open/read file {0}: {1}", filename, ex);
throw;
}
}
try
{
var uploadedFiles = await _payloadsClient.UploadTo(payloadId, list);
completedFiles.AddRange(uploadedFiles);
_logger.Log(LogLevel.Information, "{0} files uploaded to PayloadId {1}", completedFiles.Count, payloadId);
}
catch (PayloadUploadFailedException ex)
{
completedFiles.AddRange(ex.CompletedFiles);
temp.ForEach(file => queue.Enqueue(file));
if (completedFiles.Count != filePaths.Count())
{
throw;
}
_logger.Log(LogLevel.Information, "{0} files uploaded to PayloadId {1}", completedFiles.Count, payloadId);
}
catch
{
temp.ForEach(file => queue.Enqueue(file));
throw;
}
}
finally
{
for (var i = 0; i < list.Count; i += 1)
{
try
{
list[i].stream?.Dispose();
}
catch (Exception exception)
{
_logger.Log(LogLevel.Error, exception, $"Failed to dispose \"{list[i].name}\" stream.");
}
}
}
}).ConfigureAwait(false);
}
static async Task Main(string[] args)
{
Random rng = new Random();
Stopwatch stopwatch = new Stopwatch();
SmallPayload sp;
MediumPayload mp;
LargePayload lp;
DeepPayload dp;
DeeperPayload dper;
DeepestPayload dpst;
TimeSpan ts;
TimeSpan dts;
string elapsedTime;
var httpClientHandler = new HttpClientHandler();
// Return `true` to allow certificates that are untrusted/invalid
httpClientHandler.ServerCertificateCustomValidationCallback =
HttpClientHandler.DangerousAcceptAnyServerCertificateValidator;
var httpClient = new HttpClient(httpClientHandler);
var channel = GrpcChannel.ForAddress("https://80.198.94.195:5001",
new GrpcChannelOptions {
HttpClient = httpClient
});
var payloadClient = new PayloadService.PayloadServiceClient(channel);
//Verify payload objects
PayloadId payloadId = new PayloadId {
Id = 1
};
SmallPayload smallPayload = await payloadClient.GetSmallPayloadAsync(payloadId);
MediumPayload mediumPayload = await payloadClient.GetMediumPayloadAsync(payloadId);
LargePayload largePyload = await payloadClient.GetLargePayloadAsync(payloadId);
DeepPayload deepPayload = await payloadClient.GetDeepPayloadAsync(payloadId);
DeeperPayload deeperPayload = await payloadClient.GetDeeperPayloadAsync(payloadId);
DeepestPayload deepestPayload = await payloadClient.GetDeepestPayloadAsync(payloadId);
//GetSmallPayload
Console.WriteLine("Calling 'GetSmallPayload(Payload id)' 100 times");
for (int i = 0; i < 100; i++)
{
stopwatch.Start();
await payloadClient.GetSmallPayloadAsync(new PayloadId { Id = i });
stopwatch.Stop();
}
ts = stopwatch.Elapsed;
elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}",
ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds);
Console.WriteLine("Total run time (100 Calls): " + elapsedTime);
dts = ts.Divide(100);
elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}",
dts.Hours, dts.Minutes, dts.Seconds, dts.Milliseconds);
Console.WriteLine("Average run time: " + elapsedTime);
stopwatch.Reset();
//GetMediumPayload
Console.WriteLine("Calling 'GetMediumPayload(Payload id)' 100 times");
for (int i = 0; i < 100; i++)
{
stopwatch.Start();
await payloadClient.GetMediumPayloadAsync(new PayloadId { Id = i });
stopwatch.Stop();
}
ts = stopwatch.Elapsed;
elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}",
ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds);
Console.WriteLine("Total run time (100 Calls): " + elapsedTime);
dts = ts.Divide(100);
elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}",
dts.Hours, dts.Minutes, dts.Seconds, dts.Milliseconds);
Console.WriteLine("Average run time: " + elapsedTime);
stopwatch.Reset();
//GetLargePayload
Console.WriteLine("Calling 'GetLargePayload(Payload id)' 100 times");
for (int i = 0; i < 100; i++)
{
stopwatch.Start();
await payloadClient.GetLargePayloadAsync(new PayloadId { Id = i });
stopwatch.Stop();
}
ts = stopwatch.Elapsed;
elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}",
ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds);
Console.WriteLine("Total run time (100 Calls): " + elapsedTime);
dts = ts.Divide(100);
elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}",
dts.Hours, dts.Minutes, dts.Seconds, dts.Milliseconds);
Console.WriteLine("Average run time: " + elapsedTime);
stopwatch.Reset();
//GetDeepPayload
Console.WriteLine("Calling 'GetDeepPayload(Payload id)' 100 times");
for (int i = 0; i < 100; i++)
{
stopwatch.Start();
await payloadClient.GetDeepPayloadAsync(new PayloadId { Id = i });
stopwatch.Stop();
}
ts = stopwatch.Elapsed;
elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}",
ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds);
Console.WriteLine("Total run time (100 Calls): " + elapsedTime);
dts = ts.Divide(100);
elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}",
dts.Hours, dts.Minutes, dts.Seconds, dts.Milliseconds);
Console.WriteLine("Average run time: " + elapsedTime);
stopwatch.Reset();
//GetDeeperPayload
Console.WriteLine("Calling 'GetDeeperPayload(Payload id)' 100 times");
for (int i = 0; i < 100; i++)
{
stopwatch.Start();
await payloadClient.GetDeeperPayloadAsync(new PayloadId { Id = i });
stopwatch.Stop();
}
ts = stopwatch.Elapsed;
elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}",
ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds);
Console.WriteLine("Total run time (100 Calls): " + elapsedTime);
dts = ts.Divide(100);
elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}",
dts.Hours, dts.Minutes, dts.Seconds, dts.Milliseconds);
Console.WriteLine("Average run time: " + elapsedTime);
stopwatch.Reset();
//GetDeepestPayload
Console.WriteLine("Calling 'GetDeepestPayload(Payload id)' 100 times");
for (int i = 0; i < 100; i++)
{
stopwatch.Start();
await payloadClient.GetDeepestPayloadAsync(new PayloadId { Id = i });
stopwatch.Stop();
}
ts = stopwatch.Elapsed;
elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}",
ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds);
Console.WriteLine("Total run time (100 Calls): " + elapsedTime);
dts = ts.Divide(100);
elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}",
dts.Hours, dts.Minutes, dts.Seconds, dts.Milliseconds);
Console.WriteLine("Average run time: " + elapsedTime);
stopwatch.Reset();
//GetAllSmallPayloads
Console.WriteLine("Calling 'GetAllSmallPayloads()' 100 times");
for (int i = 0; i < 100; i++)
{
stopwatch.Start();
using (var requestAllSmallPayloads = payloadClient.GetAllSmallPayloads(new EmptyRequest()))
{
while (await requestAllSmallPayloads.ResponseStream.MoveNext())
{
sp = requestAllSmallPayloads.ResponseStream.Current;
}
}
stopwatch.Stop();
}
ts = stopwatch.Elapsed;
elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}",
ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds);
Console.WriteLine("Total run time (100 Calls): " + elapsedTime);
dts = ts.Divide(100);
elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}",
dts.Hours, dts.Minutes, dts.Seconds, dts.Milliseconds);
Console.WriteLine("Average run time: " + elapsedTime);
stopwatch.Reset();
//GetAllMediumPayloads
Console.WriteLine("Calling 'GetAllMediumPayloads()' 100 times");
for (int i = 0; i < 100; i++)
{
stopwatch.Start();
using (var requestAllMediumPayloads = payloadClient.GetAllMediumPayloads(new EmptyRequest()))
{
while (await requestAllMediumPayloads.ResponseStream.MoveNext())
{
mp = requestAllMediumPayloads.ResponseStream.Current;
}
}
stopwatch.Stop();
}
ts = stopwatch.Elapsed;
elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}",
ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds);
Console.WriteLine("Total run time (100 Calls): " + elapsedTime);
dts = ts.Divide(100);
elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}",
dts.Hours, dts.Minutes, dts.Seconds, dts.Milliseconds);
Console.WriteLine("Average run time: " + elapsedTime);
stopwatch.Reset();
//GetAllLargePayloads
Console.WriteLine("Calling 'GetAllLargePayloads()' 100 times");
for (int i = 0; i < 100; i++)
{
stopwatch.Start();
using (var requestAllLargePayloads = payloadClient.GetAllLargePayloads(new EmptyRequest()))
{
while (await requestAllLargePayloads.ResponseStream.MoveNext())
{
lp = requestAllLargePayloads.ResponseStream.Current;
}
}
stopwatch.Stop();
}
ts = stopwatch.Elapsed;
elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}",
ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds);
Console.WriteLine("Total run time (100 Calls): " + elapsedTime);
dts = ts.Divide(100);
elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}",
dts.Hours, dts.Minutes, dts.Seconds, dts.Milliseconds);
Console.WriteLine("Average run time: " + elapsedTime);
stopwatch.Reset();
//GetAllDeepPayloads
Console.WriteLine("Calling 'GetAllDeepPayloads()' 100 times");
for (int i = 0; i < 100; i++)
{
stopwatch.Start();
using (var requestAllDeepPayloads = payloadClient.GetAllDeepPayloads(new EmptyRequest()))
{
while (await requestAllDeepPayloads.ResponseStream.MoveNext())
{
dp = requestAllDeepPayloads.ResponseStream.Current;
}
}
stopwatch.Stop();
}
ts = stopwatch.Elapsed;
elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}",
ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds);
Console.WriteLine("Total run time (100 Calls): " + elapsedTime);
dts = ts.Divide(100);
elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}",
dts.Hours, dts.Minutes, dts.Seconds, dts.Milliseconds);
Console.WriteLine("Average run time: " + elapsedTime);
stopwatch.Reset();
//GetAllDeeperPayloads
Console.WriteLine("Calling 'GetAllDeeperPayloads()' 100 times");
for (int i = 0; i < 100; i++)
{
stopwatch.Start();
using (var requestAllDeeperPayloads = payloadClient.GetAllDeeperPayloads(new EmptyRequest()))
{
while (await requestAllDeeperPayloads.ResponseStream.MoveNext())
{
dper = requestAllDeeperPayloads.ResponseStream.Current;
}
}
stopwatch.Stop();
}
ts = stopwatch.Elapsed;
elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}",
ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds);
Console.WriteLine("Total run time (100 Calls): " + elapsedTime);
dts = ts.Divide(100);
elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}",
dts.Hours, dts.Minutes, dts.Seconds, dts.Milliseconds);
Console.WriteLine("Average run time: " + elapsedTime);
stopwatch.Reset();
//GetAllDeepestPayloads
Console.WriteLine("Calling 'GetAllDeepestPayloads()' 100 times");
for (int i = 0; i < 100; i++)
{
stopwatch.Start();
using (var requestAllDeepestPayloads = payloadClient.GetAllDeepestPayloads(new EmptyRequest()))
{
while (await requestAllDeepestPayloads.ResponseStream.MoveNext())
{
dpst = requestAllDeepestPayloads.ResponseStream.Current;
}
}
stopwatch.Stop();
}
ts = stopwatch.Elapsed;
elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}",
ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds);
Console.WriteLine("Total run time (100 Calls): " + elapsedTime);
dts = ts.Divide(100);
elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}",
dts.Hours, dts.Minutes, dts.Seconds, dts.Milliseconds);
Console.WriteLine("Average run time: " + elapsedTime);
stopwatch.Reset();
Console.ReadLine();
}
