public async Task <PayloadFile> Download(string payload, string name) { Guard.Against.NullOrWhiteSpace(payload, nameof(payload)); Guard.Against.NullOrWhiteSpace(name, nameof(name)); if (!PayloadId.TryParse(payload, out PayloadId payloadId)) { throw new ArgumentException($"Invalid Payload ID received: {payload}"); } return(await Policy <PayloadFile> .Handle <Exception>() .WaitAndRetryAsync(3, (r) => TimeSpan.FromSeconds(r * 1.5f), (data, retryCount, context) => { _logger.Log(LogLevel.Error, "Failed to download file {0} from Payloads Service {1}: {2}", name, payload, data.Exception); }) .ExecuteAsync(async() => { PayloadFile file = new PayloadFile(); using (var ms = new MemoryStream()) { var details = await _payloadsClient.DownloadFrom(payloadId, name, ms); file.Data = ms.ToArray(); file.Name = details.Name; } _logger.Log(LogLevel.Information, "File {0} successfully downloaded from {1}.", name, payloadId); return file; }).ConfigureAwait(false)); }
public async Task Create_ShallReturnAJob() { var mockClient = new Mock <IJobsClient>(); var mockLogger = new Mock <ILogger <ClaraJobsApi> >(); JobId.TryParse("jobid", out JobId jobId); PayloadId.TryParse("payloadid", out PayloadId payloadId); PipelineId.TryParse("pipelineid", out PipelineId pipelineId); mockClient.Setup(p => p.CreateJob(It.IsAny <PipelineId>(), It.IsAny <string>(), It.IsAny <JobPriority>())) .ReturnsAsync(new JobInfo { Name = "bla bla job", JobId = jobId, PayloadId = payloadId, PipelineId = pipelineId }); var service = new ClaraJobsApi( mockClient.Object, mockLogger.Object); var job = await service.Create(Guid.NewGuid().ToString(), "bla bla", JobPriority.Higher); Assert.Equal(jobId.ToString(), job.JobId); Assert.Equal(payloadId.ToString(), job.PayloadId); mockClient.Verify( p => p.CreateJob(It.IsAny <PipelineId>(), It.IsAny <string>(), JobPriority.Higher), Times.Exactly(1)); mockLogger.VerifyLogging(LogLevel.Information, Times.Once()); mockLogger.VerifyLogging(LogLevel.Error, Times.Never()); }
/// <summary> /// Serves as the default hash function. /// </summary> /// <returns>A hash code for the current object.</returns> public override int GetHashCode() { unchecked { int hashCode = PayloadId != null?PayloadId.GetHashCode() : 0; hashCode = (hashCode * 397) ^ (DownloadUrl != null ? DownloadUrl.GetHashCode() : 0); hashCode = (hashCode * 397) ^ (int)Status; return(hashCode); } }
public IPayloadComponentId RegisterPayloadComponent(IPayloadDefinition definition) { IPayloadComponentId id = new PayloadId(payloads.Count); var internalDefinition = new InternalPayloadDefinition(definition, id); if (payloadHashAlgorithm != null) { internalDefinition.IncrementalHash(payloadHashAlgorithm); } payloads.Add(internalDefinition); return(id); }
public async Task Upload(string payload, string name, string filePath) { Guard.Against.NullOrWhiteSpace(payload, nameof(payload)); Guard.Against.NullOrWhiteSpace(name, nameof(name)); Guard.Against.NullOrWhiteSpace(filePath, nameof(filePath)); if (!PayloadId.TryParse(payload, out var payloadId)) { throw new ArgumentException($"Invalid Payload ID received: {{{payload}}}.", nameof(payload)); } using var loggerScope = _logger.BeginScope(new LogginDataDictionary <string, object> { { "PayloadId", payload }, { "Name", name }, { "File", filePath } }); await Policy.Handle <Exception>() .WaitAndRetryAsync(3, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)), (exception, retryCount, context) => _logger.Log(LogLevel.Error, "Exception while uploading file(s) to {{{0}}}: {exception}.", payload, exception)) .ExecuteAsync(async() => { Stream stream; try { stream = _fileSystem.File.OpenRead(filePath); } catch (System.Exception ex) { _logger.Log(LogLevel.Error, ex, "Error reading file."); throw; } try { await _payloadsClient.UploadTo(payloadId, 0, name, stream); _logger.Log(LogLevel.Debug, "File uploaded successfully."); } catch (PayloadUploadFailedException ex) { _logger.Log(LogLevel.Error, ex, "Error uploading file."); throw; } finally { stream?.Dispose(); } }).ConfigureAwait(false); }
public override Task <DeepestPayload> GetDeepestPayload(PayloadId request, ServerCallContext context) { DeepestPayload dp = _db.GetDeepestPayload(request.Id); return(Task.FromResult(dp)); }
public override Task <LargePayload> GetLargePayload(PayloadId request, ServerCallContext context) { LargePayload lp = _db.GetLargePayload(request.Id); return(Task.FromResult(lp)); }
public override Task <MediumPayload> GetMediumPayload(PayloadId request, ServerCallContext context) { MediumPayload mp = _db.GetMediumPayload(request.Id); return(Task.FromResult(mp)); }
//Get (id) public override Task <SmallPayload> GetSmallPayload(PayloadId request, ServerCallContext context) { SmallPayload sp = _db.GetSmallPayload(request.Id); return(Task.FromResult(sp)); }
public async Task Upload(string payload, string basePath, IEnumerable <string> filePaths) { if (payload is null) { throw new ArgumentNullException(nameof(payload)); } if (filePaths is null) { throw new ArgumentNullException(nameof(filePaths)); } if (!PayloadId.TryParse(payload, out var payloadId)) { throw new ArgumentException($"Invalid Payload ID received: {{{payload}}}.", nameof(payload)); } var queue = new Queue <string>(filePaths); var completedFiles = new List <PayloadFileDetails>(); basePath = EnsureBasePathEndsWithSlash(basePath); await Policy.Handle <Exception>() .WaitAndRetryAsync(3, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)), (exception, retryCount, context) => _logger.Log(LogLevel.Error, "Exception while uploading file(s) to {{{0}}}: {exception}.", payload, exception)) .ExecuteAsync(async() => { var list = new List <(uint mode, string name, Stream stream)>(); var temp = new List <string>(); try { while (queue.Count > 0) { var filePath = queue.Dequeue(); temp.Add(filePath); var filename = filePath.Replace(basePath, ""); if (System.Linq.Enumerable.Any(completedFiles, (PayloadFileDetails pfd) => OrdinalIgnoreCase.Equals(filename, pfd.Name))) { continue; } try { var stream = _fileSystem.File.OpenRead(filePath); list.Add((0, filename, stream)); _logger.Log(LogLevel.Debug, "Ready to upload file \"{0}\" to PayloadId {1}.", filename, payloadId); } catch (System.Exception ex) { _logger.Log(LogLevel.Error, "Failed to open/read file {0}: {1}", filename, ex); throw; } } try { var uploadedFiles = await _payloadsClient.UploadTo(payloadId, list); completedFiles.AddRange(uploadedFiles); _logger.Log(LogLevel.Information, "{0} files uploaded to PayloadId {1}", completedFiles.Count, payloadId); } catch (PayloadUploadFailedException ex) { completedFiles.AddRange(ex.CompletedFiles); temp.ForEach(file => queue.Enqueue(file)); if (completedFiles.Count != filePaths.Count()) { throw; } _logger.Log(LogLevel.Information, "{0} files uploaded to PayloadId {1}", completedFiles.Count, payloadId); } catch { temp.ForEach(file => queue.Enqueue(file)); throw; } } finally { for (var i = 0; i < list.Count; i += 1) { try { list[i].stream?.Dispose(); } catch (Exception exception) { _logger.Log(LogLevel.Error, exception, $"Failed to dispose \"{list[i].name}\" stream."); } } } }).ConfigureAwait(false); }
static async Task Main(string[] args) { Random rng = new Random(); Stopwatch stopwatch = new Stopwatch(); SmallPayload sp; MediumPayload mp; LargePayload lp; DeepPayload dp; DeeperPayload dper; DeepestPayload dpst; TimeSpan ts; TimeSpan dts; string elapsedTime; var httpClientHandler = new HttpClientHandler(); // Return `true` to allow certificates that are untrusted/invalid httpClientHandler.ServerCertificateCustomValidationCallback = HttpClientHandler.DangerousAcceptAnyServerCertificateValidator; var httpClient = new HttpClient(httpClientHandler); var channel = GrpcChannel.ForAddress("https://80.198.94.195:5001", new GrpcChannelOptions { HttpClient = httpClient }); var payloadClient = new PayloadService.PayloadServiceClient(channel); //Verify payload objects PayloadId payloadId = new PayloadId { Id = 1 }; SmallPayload smallPayload = await payloadClient.GetSmallPayloadAsync(payloadId); MediumPayload mediumPayload = await payloadClient.GetMediumPayloadAsync(payloadId); LargePayload largePyload = await payloadClient.GetLargePayloadAsync(payloadId); DeepPayload deepPayload = await payloadClient.GetDeepPayloadAsync(payloadId); DeeperPayload deeperPayload = await payloadClient.GetDeeperPayloadAsync(payloadId); DeepestPayload deepestPayload = await payloadClient.GetDeepestPayloadAsync(payloadId); //GetSmallPayload Console.WriteLine("Calling 'GetSmallPayload(Payload id)' 100 times"); for (int i = 0; i < 100; i++) { stopwatch.Start(); await payloadClient.GetSmallPayloadAsync(new PayloadId { Id = i }); stopwatch.Stop(); } ts = stopwatch.Elapsed; elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds); Console.WriteLine("Total run time (100 Calls): " + elapsedTime); dts = ts.Divide(100); elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", dts.Hours, dts.Minutes, dts.Seconds, dts.Milliseconds); Console.WriteLine("Average run time: " + elapsedTime); stopwatch.Reset(); //GetMediumPayload Console.WriteLine("Calling 'GetMediumPayload(Payload id)' 100 times"); for (int i = 0; i < 100; i++) { stopwatch.Start(); await payloadClient.GetMediumPayloadAsync(new PayloadId { Id = i }); stopwatch.Stop(); } ts = stopwatch.Elapsed; elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds); Console.WriteLine("Total run time (100 Calls): " + elapsedTime); dts = ts.Divide(100); elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", dts.Hours, dts.Minutes, dts.Seconds, dts.Milliseconds); Console.WriteLine("Average run time: " + elapsedTime); stopwatch.Reset(); //GetLargePayload Console.WriteLine("Calling 'GetLargePayload(Payload id)' 100 times"); for (int i = 0; i < 100; i++) { stopwatch.Start(); await payloadClient.GetLargePayloadAsync(new PayloadId { Id = i }); stopwatch.Stop(); } ts = stopwatch.Elapsed; elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds); Console.WriteLine("Total run time (100 Calls): " + elapsedTime); dts = ts.Divide(100); elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", dts.Hours, dts.Minutes, dts.Seconds, dts.Milliseconds); Console.WriteLine("Average run time: " + elapsedTime); stopwatch.Reset(); //GetDeepPayload Console.WriteLine("Calling 'GetDeepPayload(Payload id)' 100 times"); for (int i = 0; i < 100; i++) { stopwatch.Start(); await payloadClient.GetDeepPayloadAsync(new PayloadId { Id = i }); stopwatch.Stop(); } ts = stopwatch.Elapsed; elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds); Console.WriteLine("Total run time (100 Calls): " + elapsedTime); dts = ts.Divide(100); elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", dts.Hours, dts.Minutes, dts.Seconds, dts.Milliseconds); Console.WriteLine("Average run time: " + elapsedTime); stopwatch.Reset(); //GetDeeperPayload Console.WriteLine("Calling 'GetDeeperPayload(Payload id)' 100 times"); for (int i = 0; i < 100; i++) { stopwatch.Start(); await payloadClient.GetDeeperPayloadAsync(new PayloadId { Id = i }); stopwatch.Stop(); } ts = stopwatch.Elapsed; elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds); Console.WriteLine("Total run time (100 Calls): " + elapsedTime); dts = ts.Divide(100); elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", dts.Hours, dts.Minutes, dts.Seconds, dts.Milliseconds); Console.WriteLine("Average run time: " + elapsedTime); stopwatch.Reset(); //GetDeepestPayload Console.WriteLine("Calling 'GetDeepestPayload(Payload id)' 100 times"); for (int i = 0; i < 100; i++) { stopwatch.Start(); await payloadClient.GetDeepestPayloadAsync(new PayloadId { Id = i }); stopwatch.Stop(); } ts = stopwatch.Elapsed; elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds); Console.WriteLine("Total run time (100 Calls): " + elapsedTime); dts = ts.Divide(100); elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", dts.Hours, dts.Minutes, dts.Seconds, dts.Milliseconds); Console.WriteLine("Average run time: " + elapsedTime); stopwatch.Reset(); //GetAllSmallPayloads Console.WriteLine("Calling 'GetAllSmallPayloads()' 100 times"); for (int i = 0; i < 100; i++) { stopwatch.Start(); using (var requestAllSmallPayloads = payloadClient.GetAllSmallPayloads(new EmptyRequest())) { while (await requestAllSmallPayloads.ResponseStream.MoveNext()) { sp = requestAllSmallPayloads.ResponseStream.Current; } } stopwatch.Stop(); } ts = stopwatch.Elapsed; elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds); Console.WriteLine("Total run time (100 Calls): " + elapsedTime); dts = ts.Divide(100); elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", dts.Hours, dts.Minutes, dts.Seconds, dts.Milliseconds); Console.WriteLine("Average run time: " + elapsedTime); stopwatch.Reset(); //GetAllMediumPayloads Console.WriteLine("Calling 'GetAllMediumPayloads()' 100 times"); for (int i = 0; i < 100; i++) { stopwatch.Start(); using (var requestAllMediumPayloads = payloadClient.GetAllMediumPayloads(new EmptyRequest())) { while (await requestAllMediumPayloads.ResponseStream.MoveNext()) { mp = requestAllMediumPayloads.ResponseStream.Current; } } stopwatch.Stop(); } ts = stopwatch.Elapsed; elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds); Console.WriteLine("Total run time (100 Calls): " + elapsedTime); dts = ts.Divide(100); elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", dts.Hours, dts.Minutes, dts.Seconds, dts.Milliseconds); Console.WriteLine("Average run time: " + elapsedTime); stopwatch.Reset(); //GetAllLargePayloads Console.WriteLine("Calling 'GetAllLargePayloads()' 100 times"); for (int i = 0; i < 100; i++) { stopwatch.Start(); using (var requestAllLargePayloads = payloadClient.GetAllLargePayloads(new EmptyRequest())) { while (await requestAllLargePayloads.ResponseStream.MoveNext()) { lp = requestAllLargePayloads.ResponseStream.Current; } } stopwatch.Stop(); } ts = stopwatch.Elapsed; elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds); Console.WriteLine("Total run time (100 Calls): " + elapsedTime); dts = ts.Divide(100); elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", dts.Hours, dts.Minutes, dts.Seconds, dts.Milliseconds); Console.WriteLine("Average run time: " + elapsedTime); stopwatch.Reset(); //GetAllDeepPayloads Console.WriteLine("Calling 'GetAllDeepPayloads()' 100 times"); for (int i = 0; i < 100; i++) { stopwatch.Start(); using (var requestAllDeepPayloads = payloadClient.GetAllDeepPayloads(new EmptyRequest())) { while (await requestAllDeepPayloads.ResponseStream.MoveNext()) { dp = requestAllDeepPayloads.ResponseStream.Current; } } stopwatch.Stop(); } ts = stopwatch.Elapsed; elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds); Console.WriteLine("Total run time (100 Calls): " + elapsedTime); dts = ts.Divide(100); elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", dts.Hours, dts.Minutes, dts.Seconds, dts.Milliseconds); Console.WriteLine("Average run time: " + elapsedTime); stopwatch.Reset(); //GetAllDeeperPayloads Console.WriteLine("Calling 'GetAllDeeperPayloads()' 100 times"); for (int i = 0; i < 100; i++) { stopwatch.Start(); using (var requestAllDeeperPayloads = payloadClient.GetAllDeeperPayloads(new EmptyRequest())) { while (await requestAllDeeperPayloads.ResponseStream.MoveNext()) { dper = requestAllDeeperPayloads.ResponseStream.Current; } } stopwatch.Stop(); } ts = stopwatch.Elapsed; elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds); Console.WriteLine("Total run time (100 Calls): " + elapsedTime); dts = ts.Divide(100); elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", dts.Hours, dts.Minutes, dts.Seconds, dts.Milliseconds); Console.WriteLine("Average run time: " + elapsedTime); stopwatch.Reset(); //GetAllDeepestPayloads Console.WriteLine("Calling 'GetAllDeepestPayloads()' 100 times"); for (int i = 0; i < 100; i++) { stopwatch.Start(); using (var requestAllDeepestPayloads = payloadClient.GetAllDeepestPayloads(new EmptyRequest())) { while (await requestAllDeepestPayloads.ResponseStream.MoveNext()) { dpst = requestAllDeepestPayloads.ResponseStream.Current; } } stopwatch.Stop(); } ts = stopwatch.Elapsed; elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds); Console.WriteLine("Total run time (100 Calls): " + elapsedTime); dts = ts.Divide(100); elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", dts.Hours, dts.Minutes, dts.Seconds, dts.Milliseconds); Console.WriteLine("Average run time: " + elapsedTime); stopwatch.Reset(); Console.ReadLine(); }