Esempio n. 1
0
        private async Task ProcessEtlFile(FileUploadMessageDto dto, string ftpDirectoryPath, string uploadDirectoryPath)
        {
            bool ProcessFile(string x)
            {
                return(this._uploadEtlFileMonitor.ProcessFile(x));
            }

            await this.ProcessS3File(dto, ftpDirectoryPath, uploadDirectoryPath, ProcessFile);
        }
Esempio n. 2
0
        private async Task ProcessFile(FileUploadMessageDto dto, int retries, string ftpDirectoryPath)
        {
            if (dto == null)
            {
                this._logger.LogError("File upload dto was null, exiting process file");
                return;
            }

            var versionId = string.IsNullOrWhiteSpace(dto?.VersionId) ? string.Empty : $"{dto?.VersionId}-";
            var fileName  = Path.GetFileName(dto.FileName) ?? string.Empty;

            if (string.IsNullOrWhiteSpace(fileName))
            {
                this._logger.LogInformation(
                    $"S3 File Upload Monitoring Process had a null or empty file path when reading from S3 message.  File ({dto}).");
                return;
            }

            this._logger.LogInformation($"S3 Processor about to process file ({dto}).");

            fileName = $"{versionId}{fileName}";
            var destinationFileName = Path.Combine(ftpDirectoryPath, fileName);
            var result = await this._s3Client.RetrieveFile(
                dto.Bucket,
                dto.FileName,
                dto.VersionId,
                destinationFileName);

            if (retries <= 0)
            {
                this._logger.LogInformation($"S3 Process File ran out of retries for processing file {dto.FileName}");
            }

            if (result)
            {
                this._logger.LogInformation(
                    $"S3 Processor successfully retrieved file ({dto}) to {destinationFileName}");
                return;
            }

            if (retries <= 0)
            {
                this._logger.LogError($"S3 Processor ran out of retries trying to fetch file ({dto}) to {destinationFileName}");
                return;
            }

            await this.ProcessFile(dto, retries - 1, ftpDirectoryPath);
        }
Esempio n. 3
0
        private async Task ProcessS3File(
            FileUploadMessageDto dto,
            string ftpDirectoryPath,
            string uploadDirectoryPath,
            Func <string, bool> processFileDelegate)
        {
            await this.ProcessFile(dto, 3, ftpDirectoryPath);

            var files     = Directory.EnumerateFiles(ftpDirectoryPath).ToList();
            var fileCount = files.Count;

            this._logger.LogInformation(
                $"Found {fileCount} files in the local ftp folder. Moving to the processing folder.");

            foreach (var file in files)
            {
                try
                {
                    this._logger.LogInformation($"S3 processing file {file}");
                    var result = processFileDelegate(file);

                    if (!result)
                    {
                        this._logger.LogError($"S3 processing file {file} was unsuccessful");
                    }

                    if (File.Exists(file))
                    {
                        this._logger.LogInformation($"S3 completed processing {file}. Now deleting {file}.");
                        File.Delete(file);
                    }
                    else
                    {
                        this._logger.LogInformation($"S3 Processor could not find file {file}");
                    }
                }
                catch (Exception e)
                {
                    this._logger.LogError(e, $"S3 File Upload Monitoring Process moving process trade file {file} to {uploadDirectoryPath}");
                }
            }

            this._logger.LogInformation($"Moved all {fileCount}.");
        }