public async Task <bool> DownloadToLocalAsync(string localPath, string Path, string Filename)
{
try
{
if (await _ftpClient.DirectoryExistsAsync(Path))
{
_ftpClient.RetryAttempts = 10;
string fullpath = Path + Filename;
using (MemoryStream ms = new MemoryStream())
{
_log.Info($"{FtpTemplateMessage("DownloadLocal", Filename, "Downloading")}", fullpath);
return(await _ftpClient.DownloadFileAsync(localPath, fullpath, FtpLocalExists.Overwrite));
}
}
else
{
_log.Info($"{FtpTemplateMessage("DownloadLocal", Filename, "Not Exist")}", Path + Filename);
throw new Exception("File Not Exist");
}
}
catch (Exception ex)
{
_log.Error($"{FtpTemplateMessage("DownloadLocal", Filename, "throw exception")}", ex);
throw ex;
}
}
public async Task <FtpFile <TPayload> > GetResource(FtpMetadata metadata, IResourceTrackedState lastState, CancellationToken ctk = default)
{
var contents = await Policy
.Handle <Exception>()
.RetryAsync(3)
.ExecuteAsync(async ct =>
{
using (var cts1 = new CancellationTokenSource(_config.DownloadTimeout))
using (var cts2 = CancellationTokenSource.CreateLinkedTokenSource(cts1.Token, ct))
{
return(await _ftpClient.DownloadFileAsync(metadata.Entry.FullPath, ctk: cts2.Token).ConfigureAwait(false));
}
}, ctk).ConfigureAwait(false);
var checksum = _computeChecksum(contents);
if (lastState?.CheckSum == checksum)
{
return(null);
}
var file = new FtpFile <TPayload>(metadata)
{
CheckSum = checksum,
RetrievedAt = SystemClock.Instance.GetCurrentInstant(),
ParsedData = _parser.Parse(metadata, contents)
};
return(file);
}
/// <summary>
/// Process MasterCard rebate confirmation file job execution.
/// </summary>
/// <param name="details">
/// Details of the job to be executed.
/// </param>
/// <param name="log">
/// Log within which to log status of job processing.
/// </param>
/// <returns>
/// A task to execute the job.
/// </returns>
/// <remarks>
/// Once complete, this job will schedule a corresponding MasterCardProcessRebateConfirmationJob.
/// </remarks>
public async Task Execute(ScheduledJobDetails details,
CommerceLog log)
{
Log = log;
Log.Verbose("Starting execution of job.\r\nDetails {0}", details);
MasterCardRebateConfirmationBlobClient blobClient = MasterCardBlobClientFactory.MasterCardRebateConfirmationBlobClient(log);
// Download files from MasterCard and upload them to the blob store.
IFtpClient ftpClient = MasterCardFtpClientFactory.RebateConfirmationFtpClient(Log);
string[] files = await ftpClient.DirectoryListAsync();
if (files != null)
{
foreach (string fileName in files)
{
using (MemoryStream memStream = new MemoryStream())
{
// Download the file from MasterCard.
await ftpClient.DownloadFileAsync(fileName, memStream).ConfigureAwait(false);
// Upload the file to the blob store.
memStream.Position = 0;
await blobClient.UploadAsync(memStream, fileName).ConfigureAwait(false);
}
}
}
// Process all pending rebate confirmation files in the blob store.
ICollection <string> fileNames = blobClient.RetrieveNamesOfPendingFiles();
if (fileNames != null)
{
foreach (string fileName in fileNames)
{
using (MemoryStream memoryStream = new MemoryStream())
{
// Download the file from the blob store.
memoryStream.Position = 0;
await blobClient.DownloadAsync(memoryStream, fileName).ConfigureAwait(false);
// Process the file.
memoryStream.Position = 0;
ISettlementFileProcessor rebateConfirmationProcessor = MasterCardFileProcessorFactory.MasterCardRebateConfirmationProcessor(memoryStream, fileName);
await rebateConfirmationProcessor.Process().ConfigureAwait(false);
}
// Mark the file as having been processed.
await blobClient.MarkAsCompleteAsync(fileName).ConfigureAwait(false);
}
}
Log.Verbose("Execution of job {0} complete ", details.JobId);
}
/// <summary>
/// Executa download de arquivo no caminho remoto para o local específicado
/// </summary>
/// <param name="arquivo">Caminho local onde será baixado</param>
/// <param name="destino">Caminho no local (contar a partir da raiz do servidor</param>
/// <returns></returns>
public async Task Download(string arquivo, string destino)
{
try
{
await _cliente.DownloadFileAsync(arquivo, destino, FtpLocalExists.Overwrite);
}
catch (Exception ex)
{
throw ex;
}
}
public static async Task DownloadFolderRecursive(this IFtpClient client, string source, string destination)
{
var entries = await client.GetListingAsync(source);
foreach (var item in entries.Where(x => x.Type == FtpFileSystemObjectType.File))
{
try
{
var downloadFile = true;
var file = $"{destination}\\{item.Name}";
if (File.Exists(file))
{
var fileInfo = new FileInfo(file);
downloadFile = fileInfo.LastWriteTimeUtc < item.Modified;
}
if (!downloadFile)
{
continue;
}
var result = await client.DownloadFileAsync(file, item.FullName);
if (!result)
{
throw new FtpException($"Error downloading file {item.FullName}");
}
}
catch (FtpException exc)
{
if (exc.InnerException is FtpCommandException ftpCommandException)
{
throw new FtpException($"Error downloading file {item.FullName}. Response type {ftpCommandException.ResponseType}", ftpCommandException);
}
throw new FtpException($"Error downloading file {item.FullName}", exc);
}
}
foreach (var item in entries.Where(x => x.Type == FtpFileSystemObjectType.Directory))
{
var newDestination = $@"{destination}\{item.Name}";
if (!Directory.Exists(newDestination))
{
Directory.CreateDirectory(newDestination);
}
await client.DownloadFolderRecursive($"{source}/{item.Name}", newDestination);
}
}
/// <summary>
/// 1. Process the FDC Extract file.
/// 2. Schedule the Process Pts Job
/// </summary>
/// <param name="details">
/// Details of the job we are executing here.
/// </param>
/// <param name="logger">
/// Handle to the logger
/// </param>
public async Task Execute(ScheduledJobDetails details, CommerceLog logger)
{
logger.Verbose("Starting execution of job \r\n " +
"Details {0}", details);
string connectionString = CloudConfigurationManager.GetSetting("Lomo.Commerce.Fdc.Blob.ConnectionString");
IFtpClient ftpClient = FirstDataFtpClientFactory.FirstDataExtractFtpClient(logger);
FirstDataExtractBlobClient blobClient = FirstDataBlobClientFactory.FirstDataExtractBlobClient(connectionString, logger);
string[] files = await ftpClient.DirectoryListAsync();
if (files != null)
{
foreach (string fileName in files)
{
MemoryStream memStream = new MemoryStream();
await ftpClient.DownloadFileAsync(fileName, memStream).ConfigureAwait(false);
// lets upload it to blob
memStream.Position = 0;
await blobClient.UploadAsync(memStream, fileName).ConfigureAwait(false);
}
}
// Now try to run all the pending files in the blob
ICollection <string> listOfFiles = blobClient.RetrieveFilesToProcess();
if (listOfFiles != null)
{
foreach (string fileName in listOfFiles)
{
MemoryStream memStream = new MemoryStream();
memStream.Position = 0;
await blobClient.DownloadAsync(memStream, fileName).ConfigureAwait(false);
memStream.Position = 0;
ISettlementFileProcessor extractProcessor = FirstDataFileProcessorFactory.FirstDataExtractProcessor(fileName, memStream);
await extractProcessor.Process().ConfigureAwait(false);
await blobClient.MarkAsProcessedAsync(fileName).ConfigureAwait(false);
}
}
logger.Verbose("Execution of job {0} complete ", details.JobId);
}
