protected async Task <T> ExecuteQueryAsync <T>(string uri, Func <HttpResponseMessage, Task <T> > handler)
{
Polly.Retry.AsyncRetryPolicy <HttpResponseMessage> httpRetryPolicy = Policy
.HandleResult <HttpResponseMessage>(r => r.StatusCode == (HttpStatusCode)429)
.WaitAndRetryAsync(new[]
{
TimeSpan.FromSeconds(30),
TimeSpan.FromSeconds(60),
TimeSpan.FromSeconds(90)
},
onRetry: (outcome, timespan, retryAttempt, context) =>
{
//included for debug to see what the response is
});
var response = await httpRetryPolicy.ExecuteAndCaptureAsync(() => _httpClient.GetAsync(uri));
if (response.Outcome == OutcomeType.Successful)
{
if (response.Result.IsSuccessStatusCode)
{
return(await handler(response.Result));
}
throw new HttpRequestException($"There was an error while executing the HTTP query. Reason: {response.Result.ReasonPhrase}");
}
else
{
var reason = response.FinalHandledResult != null ? response.FinalHandledResult.ReasonPhrase : response.FinalException.Message;
throw new HttpRequestException($"There was an error while executing the HTTP query. Reason: {reason}");
}
}
private static IAsyncPolicy<HttpResponseMessage> GetFinalPolicy(Polly.Retry.AsyncRetryPolicy<HttpResponseMessage> retryPolicy, Func<HttpResponseMessage, Task<bool>> onResultFuncAsync = null)
{
if (onResultFuncAsync == null)
{
return retryPolicy;
}
var fallbackPolicy = BuildFallbackPolicy(onResultFuncAsync);
return retryPolicy.WrapAsync(fallbackPolicy);
}
public override async Task RetrieveResources(SessionHostsStartInfo sessionHostsStartInfo)
{
string registryWithImageName = $"{sessionHostsStartInfo.ImageDetails.Registry}/{sessionHostsStartInfo.ImageDetails.ImageName}";
string imageTag = sessionHostsStartInfo.ImageDetails.ImageTag;
string username = sessionHostsStartInfo.ImageDetails.Username;
string password = sessionHostsStartInfo.ImageDetails.Password;
if (string.IsNullOrEmpty(imageTag))
{
imageTag = "latest";
}
_logger.LogInformation($"Starting image pull for: {registryWithImageName}:{imageTag}.");
LogReporter logReporter = new LogReporter(_logger);
Polly.Retry.AsyncRetryPolicy retryPolicy = Policy
.Handle <Exception>((Exception e) =>
{
_logger.LogError($"Exception encountered when creating image: {e.ToString()}");
return(true);
})
.WaitAndRetryAsync(_maxRetryAttempts, i => TimeSpan.FromMinutes(_createImageRetryTimeMins / _maxRetryAttempts));
await retryPolicy.ExecuteAsync(async() =>
{
await _dockerClient.Images.CreateImageAsync(
new ImagesCreateParameters {
FromImage = registryWithImageName, Tag = imageTag
},
new AuthConfig()
{
Username = username, Password = password
},
logReporter);
// Making sure that the image was actually downloaded properly
// We have seen some cases where Docker Registry API returns 'success' on pull while the image has not been properly downloaded
IEnumerable <ImagesListResponse> images = await _dockerClient.Images.ListImagesAsync(new ImagesListParameters {
All = true
});
if (images.All(image => !image.RepoTags.Contains($"{registryWithImageName}:{imageTag}")))
{
throw new ApplicationException("CreateImageAsync is completed but the image doesn't exist");
}
});
_logger.LogEvent(MetricConstants.PullImage, null, new Dictionary <string, double>
{
{ MetricConstants.DownloadDurationInMilliseconds, logReporter.DownloadSummary?.DurationInMilliseconds ?? 0d },
{ MetricConstants.ExtractDurationInMilliseconds, logReporter.ExtractionSummary?.DurationInMilliseconds ?? 0d },
{ MetricConstants.SizeInBytes, logReporter.DownloadSummary?.TotalSizeInBytes ?? 0d }
}
);
}
public static async Task <HttpResponseMessage> CheckConnection(string uri)
{
Polly.Retry.AsyncRetryPolicy retryPolly = Policy
.Handle <HttpRequestException>()
.WaitAndRetryAsync(MaxTry, i => TimeSpan.FromSeconds(Seconds));
var client = new HttpClient();
var response = await retryPolly
.ExecuteAsync(() => client.GetAsync(uri));
return(response);
}
public static IAsyncPolicy <HttpResponseMessage> Retry(int retryAttempts)
{
Polly.Retry.AsyncRetryPolicy <HttpResponseMessage> retryPolicy = HttpPolicyExtensions
.HandleTransientHttpError()
.OrResult(msg => msg.StatusCode == HttpStatusCode.NotFound)
.WaitAndRetryAsync(retryAttempts, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)));
Polly.Fallback.AsyncFallbackPolicy <HttpResponseMessage> fallbackPolicy = HttpPolicyExtensions.HandleTransientHttpError().FallbackAsync(fallbackAction: (result, context, token) => Task.FromResult(new HttpResponseMessage(HttpStatusCode.RequestTimeout)
{
Content = new StringContent(result.Exception.Message)
}),
(result, context) => Task.CompletedTask);
return(Policy.WrapAsync(retryPolicy, fallbackPolicy));
}
/// <summary>
/// Initializes a new instance of the <see cref="HttpClientHelper"/> class.
/// </summary>
/// <param name="jobSettings">Job settings</param>
/// <param name="retryPolicy">Retry policy</param>
public HttpClientHelper(Settings jobSettings, Polly.Retry.AsyncRetryPolicy retryPolicy)
{
_settings = jobSettings;
_retryPolicy = retryPolicy;
//Use Tls1.2 as default transport layer
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
var httpClientHandler = new HttpClientHandler {
AllowAutoRedirect = false,
UseCookies = false,
AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate
};
_httpClient = new HttpClient(httpClientHandler)
{
Timeout = TimeSpan.FromMinutes(60) //Timeout for large uploads or downloads
};
_authenticationHelper = new AuthenticationHelper(_settings, _retryPolicy);
}
/// <summary>
/// Called by the <see cref="T:Quartz.IScheduler" /> when a <see cref="T:Quartz.ITrigger" />
/// fires that is associated with the <see cref="T:Quartz.IJob" />.
/// </summary>
/// <param name="context">The execution context.</param>
/// <exception cref="JobExecutionException">false</exception>
/// <remarks>
/// The implementation may wish to set a result object on the
/// JobExecutionContext before this method exits. The result itself
/// is meaningless to Quartz, but may be informative to
/// <see cref="T:Quartz.IJobListener" />s or
/// <see cref="T:Quartz.ITriggerListener" />s that are watching the job's
/// execution.
/// </remarks>
public async Task Execute(IJobExecutionContext context)
{
try
{
log4net.Config.XmlConfigurator.Configure();
_context = context;
_settings.Initialize(context);
if (_settings.IndefinitePause)
{
await context.Scheduler.PauseJob(context.JobDetail.Key);
Log.InfoFormat(CultureInfo.InvariantCulture,
string.Format(Resources.Job_0_was_paused_indefinitely, _context.JobDetail.Key));
return;
}
_retryPolicyForIo = Policy.Handle <IOException>().WaitAndRetry(
retryCount: _settings.RetryCount,
sleepDurationProvider: attempt => TimeSpan.FromSeconds(_settings.RetryDelay),
onRetry: (exception, calculatedWaitDuration) =>
{
Log.WarnFormat(CultureInfo.InvariantCulture, string.Format(Resources.Job_0_Retrying_IO_operation_Exception_1, _context.JobDetail.Key, exception.Message));
});
_retryPolicyForHttp = Policy.Handle <HttpRequestException>().WaitAndRetryAsync(
retryCount: _settings.RetryCount,
sleepDurationProvider: attempt => TimeSpan.FromSeconds(_settings.RetryDelay),
onRetry: (exception, calculatedWaitDuration) =>
{
Log.WarnFormat(CultureInfo.InvariantCulture, string.Format(Resources.Job_0_Retrying_Http_operation_Exception_1, _context.JobDetail.Key, exception.Message));
});
if (Log.IsDebugEnabled)
{
Log.DebugFormat(CultureInfo.InvariantCulture, string.Format(Resources.Job_0_starting, _context.JobDetail.Key));
}
await Process();
if (Log.IsDebugEnabled)
{
Log.DebugFormat(CultureInfo.InvariantCulture, string.Format(Resources.Job_0_ended, _context.JobDetail.Key));
}
}
catch (Exception ex)
{
if (_settings.PauseJobOnException)
{
await context.Scheduler.PauseJob(context.JobDetail.Key);
Log.WarnFormat(CultureInfo.InvariantCulture,
string.Format(Resources.Job_0_was_paused_because_of_error, _context.JobDetail.Key));
}
if (Log.IsDebugEnabled)
{
if (!string.IsNullOrEmpty(ex.Message))
{
Log.Error(ex.Message, ex);
}
else
{
Log.Error("Uknown exception", ex);
}
while (ex.InnerException != null)
{
if (!string.IsNullOrEmpty(ex.InnerException.Message))
{
Log.Error(ex.InnerException.Message, ex.InnerException);
}
ex = ex.InnerException;
}
}
if (context.Scheduler.SchedulerName != "Private")
{
throw new JobExecutionException(string.Format(Resources.Upload_job_0_failed, _context.JobDetail.Key), ex, false);
}
if (!Log.IsDebugEnabled)
{
Log.Error(string.Format(Resources.Job_0_thrown_an_error_1, _context.JobDetail.Key, ex.Message));
}
}
}
/// <summary>
/// Initializes a new instance of the <see cref="AuthenticationHelper"/> class.
/// </summary>
/// <param name="jobSettings">Job settings</param>
/// <param name="retryPolicy">Retry policy</param>
public AuthenticationHelper(Settings jobSettings, Polly.Retry.AsyncRetryPolicy retryPolicy)
{
_settings = jobSettings;
_retryPolicy = retryPolicy;
}
public Rev(
Config config
)
{
_config = config;
if (string.IsNullOrWhiteSpace(_config?.jwt))
{
throw new Exception("invalid config.JWT");
}
if (string.IsNullOrWhiteSpace(_config.workspace))
{
throw new Exception("invalid config.workspace");
}
if (string.IsNullOrWhiteSpace(_config.revUrl))
{
throw new Exception("invalid config.revUrl");
}
var trimmed = _config.revUrl.TrimEnd('/');
completedRevUrl = (str) => $"{trimmed}{str}";
_httpClient = new HttpClient();
_httpClient.DefaultRequestHeaders.Add("workspace", _config.workspace);
_httpClient.DefaultRequestHeaders.Authorization
= new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", _config.jwt);
_retryPolicy = Policy
.HandleResult <HttpResponseMessage>(message =>
{
var ret = !message.IsSuccessStatusCode &&
message.StatusCode != HttpStatusCode.NotImplemented &&
message.StatusCode != HttpStatusCode.NotFound &&
message.StatusCode != HttpStatusCode.Unauthorized
;
return(ret);
})
.WaitAndRetryAsync(8, i => {
if (i == 1)
{
return(TimeSpan.FromMilliseconds(1));
}
if (i > 4)
{
return(TimeSpan.FromSeconds(10 * (i - 4)));
}
return(TimeSpan.FromSeconds(5));
}, (result, timeSpan, retryCount, context) =>
{
//var t = result;
if (1 == retryCount)
{
return;
}
Console.WriteLine($"Request failed with {result.Result.StatusCode}. Waiting {timeSpan} before next retry. Retry attempt {retryCount}");
});
}
/// <summary>
/// Called by the <see cref="T:Quartz.IScheduler" /> when a <see cref="T:Quartz.ITrigger" />
/// fires that is associated with the <see cref="T:Quartz.IJob" />.
/// </summary>
/// <param name="context">The execution context.</param>
/// <exception cref="JobExecutionException">false</exception>
/// <remarks>
/// The implementation may wish to set a result object on the
/// JobExecutionContext before this method exits. The result itself
/// is meaningless to Quartz, but may be informative to
/// <see cref="T:Quartz.IJobListener" />s or
/// <see cref="T:Quartz.ITriggerListener" />s that are watching the job's
/// execution.
/// </remarks>
public async Task Execute(IJobExecutionContext context)
{
var dataMap = context.JobDetail.JobDataMap;
// Save the orig Values for later
string inputDir = dataMap.GetString(SettingsConstants.InputDir);
string uploadSuccessDir = dataMap.GetString(SettingsConstants.UploadSuccessDir);
string uploadErrorsDir = dataMap.GetString(SettingsConstants.UploadErrorsDir);
string origCompany = dataMap.GetString(SettingsConstants.Company);
try
{
log4net.Config.XmlConfigurator.Configure();
_context = context;
string placeHolder = "???";
foreach (string company in FileHelper.getCompanysFromPath(inputDir, placeHolder, origCompany))
{
// Set the folderspecific company values
dataMap.Put(SettingsConstants.InputDir, inputDir.Replace(placeHolder, company));
dataMap.Put(SettingsConstants.UploadSuccessDir, uploadSuccessDir.Replace(placeHolder, company));
dataMap.Put(SettingsConstants.UploadErrorsDir, uploadErrorsDir.Replace(placeHolder, company));
dataMap.Put(SettingsConstants.Company, company);
_settings.Initialize(context);
// Reset to orig value
dataMap.Put(SettingsConstants.InputDir, inputDir);
dataMap.Put(SettingsConstants.UploadSuccessDir, uploadSuccessDir);
dataMap.Put(SettingsConstants.UploadErrorsDir, uploadErrorsDir);
dataMap.Put(SettingsConstants.Company, origCompany);
if (_settings.IndefinitePause)
{
await context.Scheduler.PauseJob(context.JobDetail.Key);
Log.InfoFormat(CultureInfo.InvariantCulture,
string.Format(Resources.Job_0_was_paused_indefinitely, _context.JobDetail.Key));
return;
}
_retryPolicyForIo = Policy.Handle <IOException>().WaitAndRetry(
retryCount: _settings.RetryCount,
sleepDurationProvider: attempt => TimeSpan.FromSeconds(_settings.RetryDelay),
onRetry: (exception, calculatedWaitDuration) =>
{
Log.WarnFormat(CultureInfo.InvariantCulture, string.Format(Resources.Job_0_Retrying_IO_operation_Exception_1, _context.JobDetail.Key, exception.Message));
});
_retryPolicyForHttp = Policy.Handle <HttpRequestException>().WaitAndRetryAsync(
retryCount: _settings.RetryCount,
sleepDurationProvider: attempt => TimeSpan.FromSeconds(_settings.RetryDelay),
onRetry: (exception, calculatedWaitDuration) =>
{
Log.WarnFormat(CultureInfo.InvariantCulture, string.Format(Resources.Job_0_Retrying_Http_operation_Exception_1, _context.JobDetail.Key, exception.Message));
});
if (Log.IsDebugEnabled)
{
Log.DebugFormat(CultureInfo.InvariantCulture, string.Format(Resources.Job_0_starting, _context.JobDetail.Key));
}
await Process();
if (Log.IsDebugEnabled)
{
Log.DebugFormat(CultureInfo.InvariantCulture, string.Format(Resources.Job_0_ended, _context.JobDetail.Key));
}
}
}
catch (Exception ex)
{
if (dataMap != null)
{
// Reset to orig values if an error occured
dataMap.Put(SettingsConstants.InputDir, inputDir);
dataMap.Put(SettingsConstants.UploadSuccessDir, uploadSuccessDir);
dataMap.Put(SettingsConstants.UploadErrorsDir, uploadErrorsDir);
dataMap.Put(SettingsConstants.Company, origCompany);
}
if (_settings.PauseJobOnException)
{
await context.Scheduler.PauseJob(context.JobDetail.Key);
Log.WarnFormat(CultureInfo.InvariantCulture,
string.Format(Resources.Job_0_was_paused_because_of_error, _context.JobDetail.Key));
}
if (Log.IsDebugEnabled)
{
if (!string.IsNullOrEmpty(ex.Message))
{
Log.Error(ex.Message, ex);
}
else
{
Log.Error("Uknown exception", ex);
}
while (ex.InnerException != null)
{
if (!string.IsNullOrEmpty(ex.InnerException.Message))
{
Log.Error(ex.InnerException.Message, ex.InnerException);
}
ex = ex.InnerException;
}
}
if (context.Scheduler.SchedulerName != "Private")
{
throw new JobExecutionException(string.Format(Resources.Upload_job_0_failed, _context.JobDetail.Key), ex, false);
}
if (!Log.IsDebugEnabled)
{
Log.Error(string.Format(Resources.Job_0_thrown_an_error_1, _context.JobDetail.Key, ex.Message));
}
}
}
