/// <summary>
/// Process MasterCard rebate confirmation file job execution.
/// </summary>
/// <param name="details">
/// Details of the job to be executed.
/// </param>
/// <param name="log">
/// Log within which to log status of job processing.
/// </param>
/// <returns>
/// A task to execute the job.
/// </returns>
/// <remarks>
/// Once complete, this job will schedule a corresponding MasterCardProcessRebateConfirmationJob.
/// </remarks>
public async Task Execute(ScheduledJobDetails details,
CommerceLog log)
{
Log = log;
Log.Verbose("Starting execution of job.\r\nDetails {0}", details);
MasterCardRebateConfirmationBlobClient blobClient = MasterCardBlobClientFactory.MasterCardRebateConfirmationBlobClient(log);
// Download files from MasterCard and upload them to the blob store.
IFtpClient ftpClient = MasterCardFtpClientFactory.RebateConfirmationFtpClient(Log);
string[] files = await ftpClient.DirectoryListAsync();
if (files != null)
{
foreach (string fileName in files)
{
using (MemoryStream memStream = new MemoryStream())
{
// Download the file from MasterCard.
await ftpClient.DownloadFileAsync(fileName, memStream).ConfigureAwait(false);
// Upload the file to the blob store.
memStream.Position = 0;
await blobClient.UploadAsync(memStream, fileName).ConfigureAwait(false);
}
}
}
// Process all pending rebate confirmation files in the blob store.
ICollection <string> fileNames = blobClient.RetrieveNamesOfPendingFiles();
if (fileNames != null)
{
foreach (string fileName in fileNames)
{
using (MemoryStream memoryStream = new MemoryStream())
{
// Download the file from the blob store.
memoryStream.Position = 0;
await blobClient.DownloadAsync(memoryStream, fileName).ConfigureAwait(false);
// Process the file.
memoryStream.Position = 0;
ISettlementFileProcessor rebateConfirmationProcessor = MasterCardFileProcessorFactory.MasterCardRebateConfirmationProcessor(memoryStream, fileName);
await rebateConfirmationProcessor.Process().ConfigureAwait(false);
}
// Mark the file as having been processed.
await blobClient.MarkAsCompleteAsync(fileName).ConfigureAwait(false);
}
}
Log.Verbose("Execution of job {0} complete ", details.JobId);
}
public async Task Execute(ScheduledJobDetails details, CommerceLog log)
{
Log = log;
Log.Verbose("Starting execution of job.\r\nDetails {0}", details);
// Process rebate job for Visa.
var visaProcessor = VisaSettlementProcessorFactory.VisaRebateProcessor();
await visaProcessor.Process().ConfigureAwait(false);
Log.Verbose("Exeuction of job {0} complete ", details.JobId);
}
/// <summary>
/// Process Pts File Job Execution
/// </summary>
/// <param name="details">
/// Details of the job we are executing here.
/// </param>
/// <param name="logger">
/// Handle to the logger
/// </param>
public async Task Execute(ScheduledJobDetails details, CommerceLog logger)
{
Logger = logger;
Logger.Verbose("Starting execution of job \r\n " +
"Details {0}", details);
// Process PTS Job for FDC
ISettlementFileProcessor ptsProcessor = FirstDataFileProcessorFactory.FirstDataPtsProcessor(OnPtsBuild);
await ptsProcessor.Process().ConfigureAwait(false);
Logger.Verbose("Exeuction of job {0} complete ", details.JobId);
}
/// <summary>
/// Process MasterCard rebate file job execution.
/// </summary>
/// <param name="details">
/// Details of the job to be executed.
/// </param>
/// <param name="log">
/// Log within which to log status of job processing.
/// </param>
/// <returns>
/// A task to execute the job.
/// </returns>
public async Task Execute(ScheduledJobDetails details,
CommerceLog log)
{
Log = log;
Log.Verbose("Starting execution of job.\r\nDetails {0}", details);
// Process rebate job for MasterCard.
ISettlementFileProcessor masterCardProcessor = MasterCardFileProcessorFactory.MasterCardRebateProcessor(UploadRebateFile);
await masterCardProcessor.Process().ConfigureAwait(false);
Log.Verbose("Exeuction of job {0} complete ", details.JobId);
}
/// <summary>
/// Process Amex Statement Credi File Job Execution
/// </summary>
/// <param name="details">
/// Details of the job we are executing here.
/// </param>
/// <param name="logger">
/// Handle to the logger
/// </param>
public async Task Execute(ScheduledJobDetails details, CommerceLog logger)
{
Logger = logger;
Logger.Verbose("Starting execution of job \r\n Details {0}", details);
// Process Statement Credit Acknowledgment files
await ProcessStatementCreditResponse();
// Process Statement Credit files
StatementCreditFileBuilder builder = new StatementCreditFileBuilder();
await builder.Build(OnStmtCreditFileBuild).ConfigureAwait(false);
Logger.Verbose("Execution of job {0} complete ", details.JobId);
}
/// <summary>
/// 1. Process the FDC Extract file.
/// 2. Schedule the Process Pts Job
/// </summary>
/// <param name="details">
/// Details of the job we are executing here.
/// </param>
/// <param name="logger">
/// Handle to the logger
/// </param>
public async Task Execute(ScheduledJobDetails details, CommerceLog logger)
{
logger.Verbose("Starting execution of job \r\n " +
"Details {0}", details);
string connectionString = CloudConfigurationManager.GetSetting("Lomo.Commerce.Fdc.Blob.ConnectionString");
IFtpClient ftpClient = FirstDataFtpClientFactory.FirstDataExtractFtpClient(logger);
FirstDataExtractBlobClient blobClient = FirstDataBlobClientFactory.FirstDataExtractBlobClient(connectionString, logger);
string[] files = await ftpClient.DirectoryListAsync();
if (files != null)
{
foreach (string fileName in files)
{
MemoryStream memStream = new MemoryStream();
await ftpClient.DownloadFileAsync(fileName, memStream).ConfigureAwait(false);
// lets upload it to blob
memStream.Position = 0;
await blobClient.UploadAsync(memStream, fileName).ConfigureAwait(false);
}
}
// Now try to run all the pending files in the blob
ICollection <string> listOfFiles = blobClient.RetrieveFilesToProcess();
if (listOfFiles != null)
{
foreach (string fileName in listOfFiles)
{
MemoryStream memStream = new MemoryStream();
memStream.Position = 0;
await blobClient.DownloadAsync(memStream, fileName).ConfigureAwait(false);
memStream.Position = 0;
ISettlementFileProcessor extractProcessor = FirstDataFileProcessorFactory.FirstDataExtractProcessor(fileName, memStream);
await extractProcessor.Process().ConfigureAwait(false);
await blobClient.MarkAsProcessedAsync(fileName).ConfigureAwait(false);
}
}
logger.Verbose("Execution of job {0} complete ", details.JobId);
}
/// <summary>
/// Process Amex Transaction Log File Job Execution
/// </summary>
/// <param name="details">
/// Details of the job we are executing here.
/// </param>
/// <param name="logger">
/// Handle to the logger
/// </param>
public async Task Execute(ScheduledJobDetails details, CommerceLog logger)
{
logger.Verbose("Starting execution of job \r\n Details {0}", details);
string connectionString = CloudConfigurationManager.GetSetting("Lomo.Commerce.Fdc.Blob.ConnectionString");
AmexTransactionLogSftpClient ftpClient = new AmexTransactionLogSftpClient(logger);
AmexTransactionLogFileBlobClient blobClient = AmexBlobFactory.TransactionLogBlobClient(connectionString, logger);
string[] files = await ftpClient.DirectoryListAsync("AXP_MSF_TLOG", "outbox");
if (files != null)
{
foreach (string fileName in files)
{
MemoryStream memStream = new MemoryStream();
await ftpClient.DownloadFileAsync(fileName, memStream, "outbox").ConfigureAwait(false);
// lets upload it to blob
memStream.Position = 0;
await blobClient.UploadAsync(memStream, fileName).ConfigureAwait(false);
}
}
ICollection <string> listOfFiles = blobClient.RetrieveFilesToProcess();
if (listOfFiles != null)
{
foreach (string fileName in listOfFiles)
{
MemoryStream memStream = new MemoryStream();
memStream.Position = 0;
await blobClient.DownloadAsync(memStream, fileName).ConfigureAwait(false);
memStream.Position = 0;
TransactionLogFileProcessor transactionLogFileProcessor = new TransactionLogFileProcessor()
{
TransactionLogFileName = fileName,
TransactionLogFileStream = memStream
};
await transactionLogFileProcessor.Process().ConfigureAwait(false);
await blobClient.MarkAsProcessedAsync(fileName).ConfigureAwait(false);
}
}
logger.Verbose("Execution of job {0} complete ", details.JobId);
}
/// <summary>
/// Exponentially backoff the next run time of a job after every failure (Non-Terminal) upto a defined MAX.
/// Currently, only one time jobs are exponentially backed off, and recurring jobs are just scheduled for next occurence.
/// </summary>
/// <param name="scheduler">
/// Instance of the Scheduler
/// </param>
/// <param name="jobDetails">
/// Scheduled Job Details.
/// </param>
/// <returns>
/// Async Task Wrapper
/// </returns>
public static async Task ExponentiallyBackoffAsync(this IScheduler scheduler, ScheduledJobDetails jobDetails, CommerceLog log)
{
// this branch should not happen once we schedule the job once.
if (jobDetails != null && jobDetails.Recurrence != null)
{
// if job is scheduled to run only once
if (jobDetails.Recurrence.Count == 1)
{
// initialize retry count if it does not exist
if (jobDetails.Payload == null)
{
jobDetails.Payload = new Dictionary <string, string>();
jobDetails.Payload["RetryCount"] = "0";
}
else if (!jobDetails.Payload.ContainsKey("RetryCount"))
{
jobDetails.Payload["RetryCount"] = "0";
}
int retryCount;
if (!int.TryParse(jobDetails.Payload["RetryCount"], out retryCount))
{
retryCount = 0;
}
//Important: Since the job is a run once job, so recurrence for the next retry is solely
// dependent on the retry interval. Past recurrence is immaterial.
jobDetails.Recurrence = new Recurrence()
{
Frequency = RecurrenceFrequency.Second,
Count = 1,
Interval = GetWaitTimeInSeconds(retryCount)
};
log.Verbose("Job Id {0} has been retried {1} times, back off to try the next time after {2} seconds",
jobDetails.JobId,
retryCount,
jobDetails.Recurrence.Interval);
// increment retry count in payload
jobDetails.Payload["RetryCount"] = (retryCount + 1).ToString(CultureInfo.InvariantCulture);
// schedule it to run later
await scheduler.UpdateJobAsync(jobDetails).ConfigureAwait(false);
}
else // recurring job
{
// just mark current iteration as done. We will try again next time
await scheduler.CompleteJobIterationAsync(jobDetails).ConfigureAwait(false);
}
}
else
{
log.Warning("After first run of job, job or recurrence should not be null.");
await Task.Factory.StartNew(() => { }).ConfigureAwait(false);
}
}
private async Task <AmexCardResponse> AddOrRemoveCardAsync <T>(T requestPayload, string requestUri, AmexOAuthResponse amexOAuthResponse)
{
if (amexOAuthResponse != null)
{
string timestamp = ((int)(DateTime.UtcNow.Subtract(new DateTime(1970, 1, 1))).TotalSeconds).ToString();
string nonce = string.Concat(timestamp, ":AMEX");
UriBuilder uri = new UriBuilder(requestUri);
//Format = timestamp \n timestamp:AMEX \n method \n path \n port \n\n
string message = string.Format("{0}\n{1}\nPOST\n{2}\n{3}\n{4}\n\n", timestamp, nonce, Regex.Replace(HttpUtility.UrlEncode(uri.Path.ToLowerInvariant()), @"%[a-f0-9]{2}", c => c.Value.ToUpper()), uri.Host.ToLowerInvariant(), uri.Port);
using (HMACSHA256 hmac = new HMACSHA256(Encoding.UTF8.GetBytes(amexOAuthResponse.MacKey)))
{
string mac = Convert.ToBase64String(hmac.ComputeHash(Encoding.UTF8.GetBytes(message)));
// Format -> MAC id="access_token",ts="timestamp",nonce="timestamp:AMEX",mac="HMACSHA256 hash"
string authorizationHeader = string.Format("MAC id=\"{0}\",ts=\"{1}\",nonce=\"{2}\",mac=\"{3}\"", amexOAuthResponse.AccessToken, timestamp, nonce, mac);
Stopwatch timer = Stopwatch.StartNew();
try
{
using (HttpClient httpClient = new HttpClient())
{
httpClient.DefaultRequestHeaders.Accept.Clear();
//httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("MAC", authorizationHeader.ToString());
httpClient.DefaultRequestHeaders.Add("Authorization", authorizationHeader.ToString());
httpClient.DefaultRequestHeaders.Add("X-AMEX-API-KEY", ConfigurationManager.AppSettings[AmexConstants.AmexOAuthClientId]);
httpClient.DefaultRequestHeaders.Add("X-AMEX-MSG-ID", CommerceLog.ActivityId.ToString("N"));
HttpResponseMessage response = await httpClient
.PostAsJsonAsync(requestUri, requestPayload)
.ConfigureAwait(false);
string content = await response.Content.ReadAsStringAsync();
if (response.IsSuccessStatusCode)
{
CommerceLog.Verbose("Received Amex AddOrRemoveCardAsync response. \r\nUri: {0} \r\nContent: {1}", requestUri, content);
return(JsonConvert.DeserializeObject <AmexCardResponse>(content));
}
else
{
CommerceLog.Critical("Unable to get Amex AddOrRemoveCardAsync response. \r\nUri: {0} \r\nStatus: {1} \r\nContent: {2}", null, requestUri, response.StatusCode, content);
}
}
}
finally
{
timer.Stop();
PerformanceInformation.Add(requestUri, string.Format("{0} ms", timer.ElapsedMilliseconds));
}
}
}
return(null);
}
private async Task <AmexOAuthResponse> GetOAuthTokenAsync()
{
// unix epoch format
string timestamp = ((int)(DateTime.UtcNow.Subtract(new DateTime(1970, 1, 1))).TotalSeconds).ToString();
string nonce = string.Concat(timestamp, ":AMEX");
string clientId = ConfigurationManager.AppSettings[AmexConstants.AmexOAuthClientId];
// Format -> client_id \n timestamp \n timestamp:AMEX \n grant
string message = string.Format("{0}\n{1}\n{2}\nclient_credentials\n", clientId, timestamp, nonce);
using (HMACSHA256 hmac = new HMACSHA256(Encoding.UTF8.GetBytes(ConfigurationManager.AppSettings[AmexConstants.AmexOAuthClientSecret])))
{
string mac = Convert.ToBase64String(hmac.ComputeHash(Encoding.UTF8.GetBytes(message)));
// Format -> MAC id="client_id",ts="timestamp",nonce="timestamp:AMEX",mac="HMACSHA256 hash"
string authenticationHeader = string.Format("MAC id=\"{0}\",ts=\"{1}\",nonce=\"{2}\",mac=\"{3}\"", clientId, timestamp, nonce, mac);
Stopwatch timer = Stopwatch.StartNew();
try
{
using (HttpClient httpClient = new HttpClient())
{
httpClient.DefaultRequestHeaders.Accept.Clear();
httpClient.DefaultRequestHeaders.Add("Authentication", authenticationHeader);
httpClient.DefaultRequestHeaders.Add("X-AMEX-API-KEY", clientId);
HttpContent httpContent = new StringContent("grant_type=client_credentials&app_spec_info=Apigee&guid_type=privateguid");
httpContent.Headers.ContentType = new MediaTypeHeaderValue("application/x-www-form-urlencoded");
HttpResponseMessage response = await httpClient
.PostAsync(ConfigurationManager.AppSettings[AmexConstants.AmexOAuthUri], httpContent)
.ConfigureAwait(false);
string content = await response.Content.ReadAsStringAsync();
if (response.IsSuccessStatusCode)
{
CommerceLog.Verbose("Received Amex OAuth token. \r\nContent: {0}", content);
return(JsonConvert.DeserializeObject <AmexOAuthResponse>(content));
}
else
{
CommerceLog.Critical("Unable to get Amex OAuth token. \r\nStatus: {0} \r\nContent: {1}", null, response.StatusCode, content);
}
}
}
finally
{
timer.Stop();
PerformanceInformation.Add("Amex OAuth API", string.Format("{0} ms", timer.ElapsedMilliseconds));
}
}
return(null);
}
/// <summary>
/// When the worker role starts, we do the following:
/// 1. Initialize Logger to be used
/// 2. Schedule Extract Processing job on startup if configured for.
/// 3. Schedule Ping Job
/// </summary>
/// <remarks>
/// If we schedule Extract Processing job at the same time everyday, we might get
/// duplicate jobs to execute at the same time (which might be a problem when we have
/// more than one instance of the role). We can revisit this later, but for now we will
/// schedule it at the start.
/// </remarks>
/// <returns>
/// boolean status about startup
/// </returns>
public override bool OnStart()
{
// Set the maximum number of concurrent connections
ServicePointManager.DefaultConnectionLimit = 100 * Environment.ProcessorCount;
// Use only for debugging
// TelemetryConfiguration.Active.TelemetryChannel.DeveloperMode = true;
//TelemetryConfiguration.Active.InstrumentationKey = CloudConfigurationManager.GetSetting("APPINSIGHTS_INSTRUMENTATIONKEY");
// Create a CommerceLog instance to funnel log entries to the log.
LogInitializer.CreateLogInstance(CommerceWorkerConfig.Instance.LogVerbosity,
CommerceWorkerConfig.Instance.ForceEventLog, CommerceLogSource,
CommerceWorkerConfig.Instance);
Log = new CommerceLog(Guid.NewGuid(), CommerceWorkerConfig.Instance.LogVerbosity, CommerceLogSource);
ConfigChangeHandler = new ConfigChangeHandler(Log, ExemptConfigurationItems);
// turn off processing jobs at start
ProcessJobs = Convert.ToBoolean(CloudConfigurationManager.GetSetting(ProcessJobsPropertyKey));
// role should not exit even after processing jobs stop
ExitRole = false;
// event handlers
RoleEnvironment.Changing += RoleEnvironmentChanging;
RoleEnvironment.Changed += RoleEnvironmentChanged;
if (!string.IsNullOrEmpty(ConcurrencyMonitorConnectionString))
{
Log.Verbose("Initializing Jobs.");
using (ConcurrencyMonitor monitor = new ConcurrencyMonitor(ConcurrencyMonitorConnectionString))
{
monitor.InvokeWithLease(() => PartnerJobInitializer.InitializeJobs(Scheduler));
}
Log.Verbose("Initialized Jobs.");
}
return(base.OnStart());
}
