public async Task <bool> UpdateUploadConfiguration(UploadConfiguration newItem, UploadConfigurationType type) { var item = await this.GetUploadConfiguration(type); if (item == null) { return(false); } item.SourceTypeId = newItem.SourceTypeId; item.SqlDataSourceId = newItem.SqlDataSourceId; item.FileDataSourceId = newItem.FileDataSourceId; item.FileName = newItem.FileName; foreach (var field in item.UploadConfigurationFields) { var newField = newItem.UploadConfigurationFields?.FirstOrDefault(x => x.Id == field.Id); if (newField != null) { field.SqlDataSourceFieldId = item.SqlDataSourceId.HasValue ? newField.SqlDataSourceField?.Id : null; field.FileDataSourceFieldId = item.FileDataSourceId.HasValue ? newField.FileDataSourceField?.Id : null; } } this.unitOfWork.IncidentUploadConfigurationRepository.Edit(item); return(true); }
public UploadService( IServiceProvider services, SHA512Managed sha512Managed, ILogger <UploadService> logger, EntityConfigurationProvider <UploadConfiguration> uploadConfigurationProvider) { _services = services; _sha512Managed = sha512Managed; _logger = logger; _uploadConfiguration = uploadConfigurationProvider.GetConfiguration(); }
public UploadConfiguration ToDomain(UploadRequest request) { var configuration = new UploadConfiguration { ReportSuccessForRecords = request.ReportSuccessForRecords, ContainsHeader = request.ContainsHeader, MaxRecordsInResponse = request.MaxRecordsInResponse }; return(configuration); }
public FilesController( UserManager <ApplicationUser> userManager, IOptions <UploadConfiguration> configUploadOptions, DefaultDbContext context, IActivityLogService activityLogService) { _userManager = userManager; _configUploadOptions = configUploadOptions.Value; _context = context; _activityLogService = activityLogService; }
public async Task <UploadResult> Upload(Stream stream, UploadConfiguration uploadConfiguration) { var result = new UploadResult(); var buffer = new List <RecordProcessingResult>(); DateTime start = DateTime.Now; try { var records = _csvProcessor.Process(stream, uploadConfiguration.ContainsHeader).ConfigureAwait(false); await foreach (var record in records) { await ValidateRecord(record).ConfigureAwait(false); buffer.Add(record); if (buffer.Count >= _settings.BufferSize) { await Store(buffer).ConfigureAwait(false); MoveBufferToResult(result, buffer, uploadConfiguration); if (result.Records.Count >= uploadConfiguration.MaxRecordsInResponse) { break; } } } await Store(buffer).ConfigureAwait(false); MoveBufferToResult(result, buffer, uploadConfiguration); } // For handling file validation exceptions etc., which we want to report to end user. // There are currently no business expections thrown, so this is just for show catch (BusinessException e) { _logger.LogWarning(e, "Upload failed with a business exception"); result.Issue = new Error(e.Message); } // For handling other exceptions, i.e. storage failures. We don't want to propagate the exception message to user in this case catch (Exception e) { _logger.LogError(e, "Upload failed with an unexpected exception"); result.Issue = new Error("Upload failed with an unexpected exception"); } _logger.LogInformation($"Returning {result.Records.Count} records. Last processed line: {result.LastProcessedLine}. Time ellapsed: {(DateTime.Now - start).TotalSeconds} seconds"); return(result); }
/// <summary> /// Configure upload information /// </summary> /// <param name="uploadConfiguration">Upload configuration</param> public static void Configure(UploadConfiguration uploadConfiguration) { if (uploadConfiguration == null) { return; } if (uploadConfiguration.Default != null) { ConfigureDefault(uploadConfiguration.Default); } if (!uploadConfiguration.UploadObjects.IsNullOrEmpty()) { ConfigureUploadObject(uploadConfiguration.UploadObjects.ToArray()); } }
public async Task <IActionResult> EditUploadConfiguration(int type, [FromBody] UploadConfiguration item) { if (item == null) { return(this.BadRequest()); } var result = await this.compensationServices.UpdateUploadConfiguration(item, (UploadConfigurationType)type); if (!result) { return(this.NotFound("Configuration not Found")); } return(this.Ok(item)); }
public static async Task MainUploadManager() { logger.Info("Starting Upload Manager Example"); var provider = new ConfigFileAuthenticationDetailsProvider("DEFAULT"); var osClient = new ObjectStorageClient(provider, new ClientConfiguration() { TimeoutMillis = 1000 * 1000 // 1000 secs. Setting large timeouts is required for uploading large objects to avoid httpclient timeout exceptions. }); var putObjectRequest = new PutObjectRequest() { BucketName = Environment.GetEnvironmentVariable("BUCKET_NAME"), NamespaceName = Environment.GetEnvironmentVariable("NAMESPACE_NAME"), ObjectName = Environment.GetEnvironmentVariable("OBJECT_NAME"), PutObjectBody = System.IO.File.OpenRead(Environment.GetEnvironmentVariable("FILE_TO_UPLOAD")), ContentLength = Convert.ToInt64(Environment.GetEnvironmentVariable("CONTENT_LENGTH")), ContentType = Environment.GetEnvironmentVariable("CONTENT_TYPE") }; var uploadConfiguration = new UploadConfiguration() { AllowMultipartUploads = true }; var uploadManager = new UploadManager(osClient, uploadConfiguration); var uploadRequest = new UploadManager.UploadRequest(putObjectRequest) { AllowOverwrite = true, OnProgress = OnProgres }; var uploadResponse = await uploadManager.Upload(uploadRequest); logger.Info(uploadResponse); var headObject = await osClient.HeadObject(new HeadObjectRequest() { BucketName = Environment.GetEnvironmentVariable("BUCKET_NAME"), NamespaceName = Environment.GetEnvironmentVariable("NAMESPACE_NAME"), ObjectName = Environment.GetEnvironmentVariable("OBJECT_NAME") }); logger.Info($"{headObject.ETag}"); osClient.Dispose(); }
public async Task EmptyFile() { var config = new UploadConfiguration { ReportSuccessForRecords = true, MaxRecordsInResponse = 1000 }; MockCsvProcessorReturn(new RecordProcessingResult[0]); var actual = await _service.Upload(null, config).ConfigureAwait(false); Assert.IsTrue(actual.Success); Assert.IsNull(actual.Issue); Assert.AreEqual(0, actual.LastProcessedLine); Assert.IsNotNull(actual.Records); Assert.AreEqual(0, actual.Records.Count); }
public async Task InvalidRecord() { var config = new UploadConfiguration { ReportSuccessForRecords = true, MaxRecordsInResponse = 1000 }; var failedRecord = MockRecordProcessingResult(success: false); MockCsvProcessorReturn(new[] { failedRecord }); var actual = await _service.Upload(null, config).ConfigureAwait(false); Assert.IsTrue(actual.Success); Assert.IsNull(actual.Issue); Assert.AreEqual(1, actual.LastProcessedLine); Assert.IsNotNull(actual.Records); Assert.AreEqual(1, actual.Records.Count); Assert.IsFalse(actual.Records[0].Success); }
public async Task BusinessFailure() { var config = new UploadConfiguration { ReportSuccessForRecords = false, MaxRecordsInResponse = 5 }; MockCsvProcessorReturn(new RecordProcessingResult[0]); var exception = new BusinessException("Exception message with business reason"); _db.Setup(x => x.BulkUpsert(It.IsAny <IEnumerable <Article> >())).ThrowsAsync(exception); var actual = await _service.Upload(null, config).ConfigureAwait(false); Assert.IsFalse(actual.Success); Assert.IsNotNull(actual.Issue); Assert.IsNotEmpty(actual.Issue.Text); Assert.IsTrue(actual.Issue.Text.Contains("business reason")); }
public async Task RespectMaxRecordsInResponse() { var config = new UploadConfiguration { ReportSuccessForRecords = true, MaxRecordsInResponse = 5 }; var records = Enumerable .Range(0, 100) .Select(i => MockRecordProcessingResult(success: true, line: i + 1)) .ToArray(); MockCsvProcessorReturn(records); var actual = await _service.Upload(null, config).ConfigureAwait(false); Assert.IsTrue(actual.Success); Assert.IsNull(actual.Issue); Assert.IsNotNull(actual.Records); Assert.AreEqual(5, actual.Records.Count); }
protected override void ProcessRecord() { base.ProcessRecord(); PutObjectRequest request; UploadManager uploadManager; UploadManager.UploadRequest uploadRequest; UploadConfiguration uploadConfiguration; if (ParameterSetName.Equals(FromFileSet)) { PutObjectBody = System.IO.File.OpenRead(GetAbsoluteFilePath(PutObjectBodyFromFile)); } try { request = new PutObjectRequest { NamespaceName = NamespaceName, BucketName = BucketName, ObjectName = ObjectName, ContentLength = ContentLength, PutObjectBody = PutObjectBody, IfMatch = IfMatch, IfNoneMatch = IfNoneMatch, OpcClientRequestId = OpcClientRequestId, Expect = Expect, ContentMD5 = ContentMD5, ContentType = ContentType, ContentLanguage = ContentLanguage, ContentEncoding = ContentEncoding, ContentDisposition = ContentDisposition, CacheControl = CacheControl, OpcSseCustomerAlgorithm = OpcSseCustomerAlgorithm, OpcSseCustomerKey = OpcSseCustomerKey, OpcSseCustomerKeySha256 = OpcSseCustomerKeySha256, StorageTier = StorageTier, OpcMeta = OpcMeta }; uploadConfiguration = new UploadConfiguration { LengthPerUploadPartInMiB = LengthPerUploadPartInMiB, AllowMultipartUploads = AllowMultipartUploads, ParallelUploadCount = ParallelUploadCount, DisableAutoAbort = DisableAutoAbort, EnforceMd5MultipartUpload = EnforceMd5MultipartUpload, EnforceMd5Upload = EnforceMd5Upload }; uploadManager = new UploadManager(client, uploadConfiguration); uploadRequest = new UploadManager.UploadRequest(request); uploadResponse = uploadManager.Upload(uploadRequest).GetAwaiter().GetResult(); WriteOutput(uploadResponse); FinishProcessing(uploadResponse); } catch (Exception ex) { TerminatingErrorDuringExecution(ex); } }
private void MoveBufferToResult(UploadResult result, List <RecordProcessingResult> buffer, UploadConfiguration uploadConfiguration) { foreach (var r in buffer) { r.Record = null; result.LastProcessedLine = r.Line; if (uploadConfiguration.ReportSuccessForRecords || !r.Success) { if (result.Records.Count < uploadConfiguration.MaxRecordsInResponse) { result.Records.Add(r); } } } buffer.Clear(); }