private async Task ProcessEtlFile(FileUploadMessageDto dto, string ftpDirectoryPath, string uploadDirectoryPath) { bool ProcessFile(string x) { return(this._uploadEtlFileMonitor.ProcessFile(x)); } await this.ProcessS3File(dto, ftpDirectoryPath, uploadDirectoryPath, ProcessFile); }
private async Task ProcessFile(FileUploadMessageDto dto, int retries, string ftpDirectoryPath) { if (dto == null) { this._logger.LogError("File upload dto was null, exiting process file"); return; } var versionId = string.IsNullOrWhiteSpace(dto?.VersionId) ? string.Empty : $"{dto?.VersionId}-"; var fileName = Path.GetFileName(dto.FileName) ?? string.Empty; if (string.IsNullOrWhiteSpace(fileName)) { this._logger.LogInformation( $"S3 File Upload Monitoring Process had a null or empty file path when reading from S3 message. File ({dto})."); return; } this._logger.LogInformation($"S3 Processor about to process file ({dto})."); fileName = $"{versionId}{fileName}"; var destinationFileName = Path.Combine(ftpDirectoryPath, fileName); var result = await this._s3Client.RetrieveFile( dto.Bucket, dto.FileName, dto.VersionId, destinationFileName); if (retries <= 0) { this._logger.LogInformation($"S3 Process File ran out of retries for processing file {dto.FileName}"); } if (result) { this._logger.LogInformation( $"S3 Processor successfully retrieved file ({dto}) to {destinationFileName}"); return; } if (retries <= 0) { this._logger.LogError($"S3 Processor ran out of retries trying to fetch file ({dto}) to {destinationFileName}"); return; } await this.ProcessFile(dto, retries - 1, ftpDirectoryPath); }
private async Task ProcessS3File( FileUploadMessageDto dto, string ftpDirectoryPath, string uploadDirectoryPath, Func <string, bool> processFileDelegate) { await this.ProcessFile(dto, 3, ftpDirectoryPath); var files = Directory.EnumerateFiles(ftpDirectoryPath).ToList(); var fileCount = files.Count; this._logger.LogInformation( $"Found {fileCount} files in the local ftp folder. Moving to the processing folder."); foreach (var file in files) { try { this._logger.LogInformation($"S3 processing file {file}"); var result = processFileDelegate(file); if (!result) { this._logger.LogError($"S3 processing file {file} was unsuccessful"); } if (File.Exists(file)) { this._logger.LogInformation($"S3 completed processing {file}. Now deleting {file}."); File.Delete(file); } else { this._logger.LogInformation($"S3 Processor could not find file {file}"); } } catch (Exception e) { this._logger.LogError(e, $"S3 File Upload Monitoring Process moving process trade file {file} to {uploadDirectoryPath}"); } } this._logger.LogInformation($"Moved all {fileCount}."); }