public async Task <bool> UpdateUploadConfiguration(UploadConfiguration newItem, UploadConfigurationType type)
{
var item = await this.GetUploadConfiguration(type);
if (item == null)
{
return(false);
}
item.SourceTypeId = newItem.SourceTypeId;
item.SqlDataSourceId = newItem.SqlDataSourceId;
item.FileDataSourceId = newItem.FileDataSourceId;
item.FileName = newItem.FileName;
foreach (var field in item.UploadConfigurationFields)
{
var newField = newItem.UploadConfigurationFields?.FirstOrDefault(x => x.Id == field.Id);
if (newField != null)
{
field.SqlDataSourceFieldId = item.SqlDataSourceId.HasValue ? newField.SqlDataSourceField?.Id : null;
field.FileDataSourceFieldId = item.FileDataSourceId.HasValue ? newField.FileDataSourceField?.Id : null;
}
}
this.unitOfWork.IncidentUploadConfigurationRepository.Edit(item);
return(true);
}
public UploadService(
IServiceProvider services,
SHA512Managed sha512Managed,
ILogger <UploadService> logger,
EntityConfigurationProvider <UploadConfiguration> uploadConfigurationProvider)
{
_services = services;
_sha512Managed = sha512Managed;
_logger = logger;
_uploadConfiguration = uploadConfigurationProvider.GetConfiguration();
}
public UploadConfiguration ToDomain(UploadRequest request)
{
var configuration = new UploadConfiguration
{
ReportSuccessForRecords = request.ReportSuccessForRecords,
ContainsHeader = request.ContainsHeader,
MaxRecordsInResponse = request.MaxRecordsInResponse
};
return(configuration);
}
public FilesController(
UserManager <ApplicationUser> userManager,
IOptions <UploadConfiguration> configUploadOptions,
DefaultDbContext context,
IActivityLogService activityLogService)
{
_userManager = userManager;
_configUploadOptions = configUploadOptions.Value;
_context = context;
_activityLogService = activityLogService;
}
public async Task <UploadResult> Upload(Stream stream, UploadConfiguration uploadConfiguration)
{
var result = new UploadResult();
var buffer = new List <RecordProcessingResult>();
DateTime start = DateTime.Now;
try
{
var records = _csvProcessor.Process(stream, uploadConfiguration.ContainsHeader).ConfigureAwait(false);
await foreach (var record in records)
{
await ValidateRecord(record).ConfigureAwait(false);
buffer.Add(record);
if (buffer.Count >= _settings.BufferSize)
{
await Store(buffer).ConfigureAwait(false);
MoveBufferToResult(result, buffer, uploadConfiguration);
if (result.Records.Count >= uploadConfiguration.MaxRecordsInResponse)
{
break;
}
}
}
await Store(buffer).ConfigureAwait(false);
MoveBufferToResult(result, buffer, uploadConfiguration);
}
// For handling file validation exceptions etc., which we want to report to end user.
// There are currently no business expections thrown, so this is just for show
catch (BusinessException e)
{
_logger.LogWarning(e, "Upload failed with a business exception");
result.Issue = new Error(e.Message);
}
// For handling other exceptions, i.e. storage failures. We don't want to propagate the exception message to user in this case
catch (Exception e)
{
_logger.LogError(e, "Upload failed with an unexpected exception");
result.Issue = new Error("Upload failed with an unexpected exception");
}
_logger.LogInformation($"Returning {result.Records.Count} records. Last processed line: {result.LastProcessedLine}. Time ellapsed: {(DateTime.Now - start).TotalSeconds} seconds");
return(result);
}
/// <summary>
/// Configure upload information
/// </summary>
/// <param name="uploadConfiguration">Upload configuration</param>
public static void Configure(UploadConfiguration uploadConfiguration)
{
if (uploadConfiguration == null)
{
return;
}
if (uploadConfiguration.Default != null)
{
ConfigureDefault(uploadConfiguration.Default);
}
if (!uploadConfiguration.UploadObjects.IsNullOrEmpty())
{
ConfigureUploadObject(uploadConfiguration.UploadObjects.ToArray());
}
}
public async Task <IActionResult> EditUploadConfiguration(int type, [FromBody] UploadConfiguration item)
{
if (item == null)
{
return(this.BadRequest());
}
var result = await this.compensationServices.UpdateUploadConfiguration(item, (UploadConfigurationType)type);
if (!result)
{
return(this.NotFound("Configuration not Found"));
}
return(this.Ok(item));
}
public static async Task MainUploadManager()
{
logger.Info("Starting Upload Manager Example");
var provider = new ConfigFileAuthenticationDetailsProvider("DEFAULT");
var osClient = new ObjectStorageClient(provider, new ClientConfiguration()
{
TimeoutMillis = 1000 * 1000 // 1000 secs. Setting large timeouts is required for uploading large objects to avoid httpclient timeout exceptions.
});
var putObjectRequest = new PutObjectRequest()
{
BucketName = Environment.GetEnvironmentVariable("BUCKET_NAME"),
NamespaceName = Environment.GetEnvironmentVariable("NAMESPACE_NAME"),
ObjectName = Environment.GetEnvironmentVariable("OBJECT_NAME"),
PutObjectBody = System.IO.File.OpenRead(Environment.GetEnvironmentVariable("FILE_TO_UPLOAD")),
ContentLength = Convert.ToInt64(Environment.GetEnvironmentVariable("CONTENT_LENGTH")),
ContentType = Environment.GetEnvironmentVariable("CONTENT_TYPE")
};
var uploadConfiguration = new UploadConfiguration()
{
AllowMultipartUploads = true
};
var uploadManager = new UploadManager(osClient, uploadConfiguration);
var uploadRequest = new UploadManager.UploadRequest(putObjectRequest)
{
AllowOverwrite = true,
OnProgress = OnProgres
};
var uploadResponse = await uploadManager.Upload(uploadRequest);
logger.Info(uploadResponse);
var headObject = await osClient.HeadObject(new HeadObjectRequest()
{
BucketName = Environment.GetEnvironmentVariable("BUCKET_NAME"),
NamespaceName = Environment.GetEnvironmentVariable("NAMESPACE_NAME"),
ObjectName = Environment.GetEnvironmentVariable("OBJECT_NAME")
});
logger.Info($"{headObject.ETag}");
osClient.Dispose();
}
public async Task EmptyFile()
{
var config = new UploadConfiguration
{
ReportSuccessForRecords = true,
MaxRecordsInResponse = 1000
};
MockCsvProcessorReturn(new RecordProcessingResult[0]);
var actual = await _service.Upload(null, config).ConfigureAwait(false);
Assert.IsTrue(actual.Success);
Assert.IsNull(actual.Issue);
Assert.AreEqual(0, actual.LastProcessedLine);
Assert.IsNotNull(actual.Records);
Assert.AreEqual(0, actual.Records.Count);
}
public async Task InvalidRecord()
{
var config = new UploadConfiguration
{
ReportSuccessForRecords = true,
MaxRecordsInResponse = 1000
};
var failedRecord = MockRecordProcessingResult(success: false);
MockCsvProcessorReturn(new[] { failedRecord });
var actual = await _service.Upload(null, config).ConfigureAwait(false);
Assert.IsTrue(actual.Success);
Assert.IsNull(actual.Issue);
Assert.AreEqual(1, actual.LastProcessedLine);
Assert.IsNotNull(actual.Records);
Assert.AreEqual(1, actual.Records.Count);
Assert.IsFalse(actual.Records[0].Success);
}
public async Task BusinessFailure()
{
var config = new UploadConfiguration
{
ReportSuccessForRecords = false,
MaxRecordsInResponse = 5
};
MockCsvProcessorReturn(new RecordProcessingResult[0]);
var exception = new BusinessException("Exception message with business reason");
_db.Setup(x => x.BulkUpsert(It.IsAny <IEnumerable <Article> >())).ThrowsAsync(exception);
var actual = await _service.Upload(null, config).ConfigureAwait(false);
Assert.IsFalse(actual.Success);
Assert.IsNotNull(actual.Issue);
Assert.IsNotEmpty(actual.Issue.Text);
Assert.IsTrue(actual.Issue.Text.Contains("business reason"));
}
public async Task RespectMaxRecordsInResponse()
{
var config = new UploadConfiguration
{
ReportSuccessForRecords = true,
MaxRecordsInResponse = 5
};
var records = Enumerable
.Range(0, 100)
.Select(i => MockRecordProcessingResult(success: true, line: i + 1))
.ToArray();
MockCsvProcessorReturn(records);
var actual = await _service.Upload(null, config).ConfigureAwait(false);
Assert.IsTrue(actual.Success);
Assert.IsNull(actual.Issue);
Assert.IsNotNull(actual.Records);
Assert.AreEqual(5, actual.Records.Count);
}
protected override void ProcessRecord()
{
base.ProcessRecord();
PutObjectRequest request;
UploadManager uploadManager;
UploadManager.UploadRequest uploadRequest;
UploadConfiguration uploadConfiguration;
if (ParameterSetName.Equals(FromFileSet))
{
PutObjectBody = System.IO.File.OpenRead(GetAbsoluteFilePath(PutObjectBodyFromFile));
}
try
{
request = new PutObjectRequest
{
NamespaceName = NamespaceName,
BucketName = BucketName,
ObjectName = ObjectName,
ContentLength = ContentLength,
PutObjectBody = PutObjectBody,
IfMatch = IfMatch,
IfNoneMatch = IfNoneMatch,
OpcClientRequestId = OpcClientRequestId,
Expect = Expect,
ContentMD5 = ContentMD5,
ContentType = ContentType,
ContentLanguage = ContentLanguage,
ContentEncoding = ContentEncoding,
ContentDisposition = ContentDisposition,
CacheControl = CacheControl,
OpcSseCustomerAlgorithm = OpcSseCustomerAlgorithm,
OpcSseCustomerKey = OpcSseCustomerKey,
OpcSseCustomerKeySha256 = OpcSseCustomerKeySha256,
StorageTier = StorageTier,
OpcMeta = OpcMeta
};
uploadConfiguration = new UploadConfiguration
{
LengthPerUploadPartInMiB = LengthPerUploadPartInMiB,
AllowMultipartUploads = AllowMultipartUploads,
ParallelUploadCount = ParallelUploadCount,
DisableAutoAbort = DisableAutoAbort,
EnforceMd5MultipartUpload = EnforceMd5MultipartUpload,
EnforceMd5Upload = EnforceMd5Upload
};
uploadManager = new UploadManager(client, uploadConfiguration);
uploadRequest = new UploadManager.UploadRequest(request);
uploadResponse = uploadManager.Upload(uploadRequest).GetAwaiter().GetResult();
WriteOutput(uploadResponse);
FinishProcessing(uploadResponse);
}
catch (Exception ex)
{
TerminatingErrorDuringExecution(ex);
}
}
private void MoveBufferToResult(UploadResult result, List <RecordProcessingResult> buffer, UploadConfiguration uploadConfiguration)
{
foreach (var r in buffer)
{
r.Record = null;
result.LastProcessedLine = r.Line;
if (uploadConfiguration.ReportSuccessForRecords || !r.Success)
{
if (result.Records.Count < uploadConfiguration.MaxRecordsInResponse)
{
result.Records.Add(r);
}
}
}
buffer.Clear();
}
