private async Task ProcessEtlFile(FileUploadMessageDto dto, string ftpDirectoryPath, string uploadDirectoryPath)
{
bool ProcessFile(string x)
{
return(this._uploadEtlFileMonitor.ProcessFile(x));
}
await this.ProcessS3File(dto, ftpDirectoryPath, uploadDirectoryPath, ProcessFile);
}
private async Task ProcessFile(FileUploadMessageDto dto, int retries, string ftpDirectoryPath)
{
if (dto == null)
{
this._logger.LogError("File upload dto was null, exiting process file");
return;
}
var versionId = string.IsNullOrWhiteSpace(dto?.VersionId) ? string.Empty : $"{dto?.VersionId}-";
var fileName = Path.GetFileName(dto.FileName) ?? string.Empty;
if (string.IsNullOrWhiteSpace(fileName))
{
this._logger.LogInformation(
$"S3 File Upload Monitoring Process had a null or empty file path when reading from S3 message. File ({dto}).");
return;
}
this._logger.LogInformation($"S3 Processor about to process file ({dto}).");
fileName = $"{versionId}{fileName}";
var destinationFileName = Path.Combine(ftpDirectoryPath, fileName);
var result = await this._s3Client.RetrieveFile(
dto.Bucket,
dto.FileName,
dto.VersionId,
destinationFileName);
if (retries <= 0)
{
this._logger.LogInformation($"S3 Process File ran out of retries for processing file {dto.FileName}");
}
if (result)
{
this._logger.LogInformation(
$"S3 Processor successfully retrieved file ({dto}) to {destinationFileName}");
return;
}
if (retries <= 0)
{
this._logger.LogError($"S3 Processor ran out of retries trying to fetch file ({dto}) to {destinationFileName}");
return;
}
await this.ProcessFile(dto, retries - 1, ftpDirectoryPath);
}
private async Task ProcessS3File(
FileUploadMessageDto dto,
string ftpDirectoryPath,
string uploadDirectoryPath,
Func <string, bool> processFileDelegate)
{
await this.ProcessFile(dto, 3, ftpDirectoryPath);
var files = Directory.EnumerateFiles(ftpDirectoryPath).ToList();
var fileCount = files.Count;
this._logger.LogInformation(
$"Found {fileCount} files in the local ftp folder. Moving to the processing folder.");
foreach (var file in files)
{
try
{
this._logger.LogInformation($"S3 processing file {file}");
var result = processFileDelegate(file);
if (!result)
{
this._logger.LogError($"S3 processing file {file} was unsuccessful");
}
if (File.Exists(file))
{
this._logger.LogInformation($"S3 completed processing {file}. Now deleting {file}.");
File.Delete(file);
}
else
{
this._logger.LogInformation($"S3 Processor could not find file {file}");
}
}
catch (Exception e)
{
this._logger.LogError(e, $"S3 File Upload Monitoring Process moving process trade file {file} to {uploadDirectoryPath}");
}
}
this._logger.LogInformation($"Moved all {fileCount}.");
}