public void AttemptsFiveTimesOnError()
{
var packageId = "FakePackageId";
var version = new NuGetVersion(1, 2, 3);
var feedUri = new Uri("http://www.myget.org");
var feedCredentials = new CredentialCache();
var targetFilePath = "FakeTargetFilePath";
var calledCount = 0;
Assert.Throws <Exception>(() =>
{
//total attempts is initial attempt + 4 retries
var retryTracker = new RetryTracker(maxRetries: 4,
timeLimit: null,
retryInterval: new RetryInterval(100, 150, 1));
var downloader = new NuGetPackageDownloader(retryTracker);
downloader.DownloadPackage(packageId, version, feedUri, feedCredentials, targetFilePath,
(arg1, arg2, arg3, arg4, arg5) =>
{
calledCount++;
throw new ApplicationException("Expected exception from test");
});
});
Assert.That(calledCount, Is.EqualTo(5));
}
public void DownloadPackage(
string packageId,
IVersion version,
Uri feedUri,
ICredentials feedCredentials,
string targetFilePath,
int maxDownloadAttempts,
TimeSpan downloadAttemptBackoff,
Action <string, IVersion, Uri, ICredentials, string> action)
{
if (maxDownloadAttempts <= 0)
{
throw new ArgumentException($"The number of download attempts should be greater than zero, but was {maxDownloadAttempts}", nameof(maxDownloadAttempts));
}
var tempTargetFilePath = targetFilePath + Download.NuGetPackageDownloader.DownloadingExtension;
// The RetryTracker is a bit finicky to set up...
var numberOfRetriesOnFailure = maxDownloadAttempts - 1;
var retry = new RetryTracker(numberOfRetriesOnFailure, timeLimit: null, retryInterval: new LinearRetryInterval(downloadAttemptBackoff));
while (retry.Try())
{
Log.Verbose($"Downloading package (attempt {retry.CurrentTry} of {maxDownloadAttempts})");
try
{
action(packageId, version, feedUri, feedCredentials, tempTargetFilePath);
fileSystem.MoveFile(tempTargetFilePath, targetFilePath);
return;
}
catch (Exception ex)
{
if (ex is WebException webException &&
webException.Response is HttpWebResponse response &&
response.StatusCode == HttpStatusCode.Unauthorized)
{
throw new Exception($"Unable to download package: {webException.Message}", ex);
}
Log.Verbose($"Attempt {retry.CurrentTry} of {maxDownloadAttempts}: {ex.Message}");
fileSystem.DeleteFile(tempTargetFilePath, FailureOptions.IgnoreFailure);
fileSystem.DeleteFile(targetFilePath, FailureOptions.IgnoreFailure);
if (retry.CanRetry())
{
var wait = TimeSpan.FromMilliseconds(retry.Sleep());
Log.Verbose($"Going to wait {wait.TotalSeconds}s before attempting the download from the external feed again.");
Thread.Sleep(wait);
}
else
{
var helpfulFailure = $"The package {packageId} version {version} could not be downloaded from the external feed '{feedUri}' after making {maxDownloadAttempts} attempts over a total of {Math.Floor(retry.TotalElapsed.TotalSeconds)}s. Make sure the package is pushed to the external feed and try the deployment again. For a detailed troubleshooting guide go to http://g.octopushq.com/TroubleshootMissingPackages";
helpfulFailure += $"{Environment.NewLine}{ex}";
throw new Exception(helpfulFailure, ex);
}
}
}
}
public void ShouldTryToLimitAfterMaxIsNotReachedAndResetting()
{
var cnt = 0;
var subject = new RetryTracker(20, null, new LinearRetryInterval(TimeSpan.Zero));
subject.Try();
subject.Reset();
cnt = 0;
while (subject.Try())
{
cnt++;
}
cnt.Should().Be(21);
}
static bool TerraformFileAvailable(string downloadBaseUrl, RetryTracker retry, string fileName)
{
try
{
var request = WebRequest.Create($"{downloadBaseUrl}{fileName}");
request.Method = "HEAD";
using (request.GetResponse())
{
return(true);
}
}
catch (Exception ex)
{
Log.Error($"There was an error accessing the terraform cli on try #{retry.CurrentTry}. Falling back to default. {ex.Message}");
return(false);
}
}
void DeleteFile(string path, FailureOptions options, RetryTracker retry, CancellationToken cancel)
{
if (string.IsNullOrWhiteSpace(path))
{
return;
}
retry.Reset();
while (retry.Try())
{
cancel.ThrowIfCancellationRequested();
try
{
if (File.Exists(path))
{
if (retry.IsNotFirstAttempt)
{
File.SetAttributes(path, FileAttributes.Normal);
}
File.Delete(path);
}
break;
}
catch (Exception ex)
{
if (retry.CanRetry())
{
if (retry.ShouldLogWarning())
{
Log.VerboseFormat("Retry #{0} on delete file '{1}'. Exception: {2}", retry.CurrentTry, path, ex.Message);
}
Thread.Sleep(retry.Sleep());
}
else
{
if (options == FailureOptions.ThrowOnFailure)
{
throw;
}
}
}
}
}
public void ShouldThrowOnceRetriesExceeded()
{
const int retries = 100;
var subject = new RetryTracker(100, null, new LimitedExponentialRetryInterval(100, 200, 2));
Exception caught = null;
var retried = 0;
try
{
while (subject.Try())
{
try
{
throw new Exception("Blah");
}
catch
{
if (subject.CanRetry())
{
//swallow exception
retried++;
}
else
{
throw;
}
}
}
}
catch (Exception ex)
{
caught = ex;
}
Assert.NotNull(caught);
Assert.AreEqual(retries, retried);
}
public async Task InstallTools()
{
async Task DownloadCli(string destination)
{
Console.WriteLine("Downloading terraform cli...");
var retry = new RetryTracker(3, TimeSpan.MaxValue, new LimitedExponentialRetryInterval(1000, 30000, 2));
while (retry.Try())
{
try
{
using (var client = new HttpClient())
{
var json = await client.GetStringAsync("https://checkpoint-api.hashicorp.com/v1/check/terraform");
var parsedJson = JObject.Parse(json);
var downloadBaseUrl = parsedJson["current_download_url"].Value <string>();
var currentVersion = parsedJson["current_version"].Value <string>();
var fileName = GetTerraformFileName(currentVersion);
if (!TerraformFileAvailable(downloadBaseUrl, retry, fileName))
{
// At times Terraform's API has been unreliable. This is a fallback
// for a version we know exists.
downloadBaseUrl = "https://releases.hashicorp.com/terraform/0.12.19/";
currentVersion = "0.12.19";
fileName = GetTerraformFileName(currentVersion);
}
await DownloadTerraform(fileName, client, downloadBaseUrl, destination);
}
customTerraformExecutable = Directory.EnumerateFiles(destination).FirstOrDefault();
Console.WriteLine($"Downloaded terraform to {customTerraformExecutable}");
break;
}
catch
{
if (!retry.CanRetry())
{
throw;
}
await Task.Delay(retry.Sleep());
}
}
}
var destinationDirectoryName = TestEnvironment.GetTestPath("TerraformCLIPath");
if (Directory.Exists(destinationDirectoryName))
{
var path = Directory.EnumerateFiles(destinationDirectoryName).FirstOrDefault();
if (path != null)
{
customTerraformExecutable = path;
Console.WriteLine($"Using existing terraform located in {customTerraformExecutable}");
return;
}
}
await DownloadCli(destinationDirectoryName);
}
public NuGetPackageDownloader(RetryTracker retry)
{
this.retry = retry;
}
public async Task InstallTools()
{
async Task DownloadCli(string destination)
{
Console.WriteLine("Downloading terraform cli...");
var retry = new RetryTracker(3, TimeSpan.MaxValue, new LimitedExponentialRetryInterval(1000, 30000, 2));
while (retry.Try())
{
try
{
using (var client = new HttpClient())
{
var json = await client.GetStringAsync("https://checkpoint-api.hashicorp.com/v1/check/terraform");
var parsedJson = JObject.Parse(json);
var downloadBaseUrl = parsedJson["current_download_url"].Value <string>();
var currentVersion = parsedJson["current_version"].Value <string>();
var fileName = $"terraform_{currentVersion}_windows_amd64.zip";
if (CalamariEnvironment.IsRunningOnNix)
{
fileName = $"terraform_{currentVersion}_linux_amd64.zip";
}
var zipPath = Path.Combine(Path.GetTempPath(), fileName);
using (new TemporaryFile(zipPath))
{
using (var fileStream =
new FileStream(zipPath, FileMode.Create, FileAccess.Write, FileShare.None))
using (var stream = await client.GetStreamAsync($"{downloadBaseUrl}{fileName}"))
{
await stream.CopyToAsync(fileStream);
}
ZipFile.ExtractToDirectory(zipPath, destination);
}
}
customTerraformExecutable = Directory.EnumerateFiles(destination).FirstOrDefault();
Console.WriteLine($"Downloaded terraform to {customTerraformExecutable}");
break;
}
catch
{
if (!retry.CanRetry())
{
throw;
}
await Task.Delay(retry.Sleep());
}
}
}
var destinationDirectoryName = TestEnvironment.GetTestPath("TerraformCLIPath");
if (Directory.Exists(destinationDirectoryName))
{
var path = Directory.EnumerateFiles(destinationDirectoryName).FirstOrDefault();
if (path != null)
{
customTerraformExecutable = path;
Console.WriteLine($"Using existing terraform located in {customTerraformExecutable}");
return;
}
}
await DownloadCli(destinationDirectoryName);
}
