public async Task Timeout()
{
var policy = new ExponentialRetryPolicy(TransientDetector, initialRetryInterval: TimeSpan.FromSeconds(0.5), maxRetryInterval: TimeSpan.FromSeconds(4), timeout: TimeSpan.FromSeconds(1.5));
var times = new List <DateTime>();
Assert.Equal(int.MaxValue, policy.MaxAttempts);
Assert.Equal(TimeSpan.FromSeconds(0.5), policy.InitialRetryInterval);
Assert.Equal(TimeSpan.FromSeconds(4), policy.MaxRetryInterval);
Assert.Equal(TimeSpan.FromSeconds(1.5), policy.Timeout);
await Assert.ThrowsAsync <TransientException>(
async() =>
{
await policy.InvokeAsync(
async() =>
{
times.Add(DateTime.UtcNow);
await Task.CompletedTask;
throw new TransientException();
});
});
Assert.Equal(3, times.Count);
// Additional test to verify this serious problem is fixed:
//
// https://github.com/nforgeio/neonKUBE/issues/762
//
// We'll wait a bit longer to enure that any (incorrect) deadline computed
// by the policy when constructed above does not impact a subsequent run.
await Task.Delay(TimeSpan.FromSeconds(4));
times.Clear();
Assert.Equal(TimeSpan.FromSeconds(0.5), policy.InitialRetryInterval);
Assert.Equal(TimeSpan.FromSeconds(4), policy.MaxRetryInterval);
Assert.Equal(TimeSpan.FromSeconds(1.5), policy.Timeout);
await Assert.ThrowsAsync <TransientException>(
async() =>
{
await policy.InvokeAsync(
async() =>
{
times.Add(DateTime.UtcNow);
await Task.CompletedTask;
throw new TransientException();
});
});
Assert.Equal(3, times.Count);
}
public async Task SuccessCustom_Result()
{
var policy = new ExponentialRetryPolicy(TransientDetector, maxAttempts: 6, initialRetryInterval: TimeSpan.FromSeconds(0.5), maxRetryInterval: TimeSpan.FromSeconds(4));
var times = new List <DateTime>();
Assert.Equal(6, policy.MaxAttempts);
Assert.Equal(TimeSpan.FromSeconds(0.5), policy.InitialRetryInterval);
Assert.Equal(TimeSpan.FromSeconds(4), policy.MaxRetryInterval);
var success = await policy.InvokeAsync(
async() =>
{
times.Add(DateTime.UtcNow);
await Task.Delay(0);
if (times.Count < policy.MaxAttempts)
{
throw new TransientException();
}
return("WOOHOO!");
});
Assert.Equal("WOOHOO!", success);
Assert.Equal(policy.MaxAttempts, times.Count);
VerifyIntervals(times, policy);
}
public async Task SuccessDelayedAggregateArray()
{
var policy = new ExponentialRetryPolicy(new Type[] { typeof(NotReadyException), typeof(KeyNotFoundException) });
var times = new List <DateTime>();
var success = false;
await policy.InvokeAsync(
async() =>
{
times.Add(DateTime.UtcNow);
await Task.Delay(0);
if (times.Count < policy.MaxAttempts)
{
if (times.Count % 1 == 0)
{
throw new AggregateException(new NotReadyException());
}
else
{
throw new AggregateException(new KeyNotFoundException());
}
}
success = true;
});
Assert.True(success);
Assert.Equal(policy.MaxAttempts, times.Count);
VerifyIntervals(times, policy);
}
public async Task FailDelayed_Result()
{
var policy = new ExponentialRetryPolicy(TransientDetector);
var times = new List <DateTime>();
await Assert.ThrowsAsync <NotImplementedException>(
async() =>
{
await policy.InvokeAsync <string>(
async() =>
{
times.Add(DateTime.UtcNow);
await Task.Delay(0);
if (times.Count < 2)
{
throw new TransientException();
}
else
{
throw new NotImplementedException();
}
});
});
Assert.Equal(2, times.Count);
VerifyIntervals(times, policy);
}
public async Task Timeout()
{
var policy = new ExponentialRetryPolicy(TransientDetector, maxAttempts: 6, initialRetryInterval: TimeSpan.FromSeconds(0.5), maxRetryInterval: TimeSpan.FromSeconds(4), timeout: TimeSpan.FromSeconds(1.5));
var times = new List <DateTime>();
Assert.Equal(6, policy.MaxAttempts);
Assert.Equal(TimeSpan.FromSeconds(0.5), policy.InitialRetryInterval);
Assert.Equal(TimeSpan.FromSeconds(4), policy.MaxRetryInterval);
Assert.Equal(TimeSpan.FromSeconds(1.5), policy.Timeout);
await Assert.ThrowsAsync <TransientException>(
async() =>
{
await policy.InvokeAsync(
async() =>
{
times.Add(DateTime.UtcNow);
await Task.CompletedTask;
throw new TransientException();
});
});
Assert.True(times.Count < 6);
}
public async Task SuccessCustom()
{
var policy = new ExponentialRetryPolicy(TransientDetector, maxAttempts: 6, initialRetryInterval: TimeSpan.FromSeconds(0.5), maxRetryInterval: TimeSpan.FromSeconds(4));
var times = new List <DateTime>();
var success = false;
Assert.Equal(6, policy.MaxAttempts);
Assert.Equal(TimeSpan.FromSeconds(0.5), policy.InitialRetryInterval);
Assert.Equal(TimeSpan.FromSeconds(4), policy.MaxRetryInterval);
await policy.InvokeAsync(
async() =>
{
times.Add(DateTime.UtcNow);
await Task.CompletedTask;
if (times.Count < policy.MaxAttempts)
{
throw new TransientException();
}
success = true;
});
Assert.True(success);
Assert.Equal(policy.MaxAttempts, times.Count);
VerifyIntervals(times, policy);
}
/// <summary>
/// Implements the service as a <see cref="Task"/>.
/// </summary>
/// <returns>The <see cref="Task"/>.</returns>
private static async Task RunAsync()
{
try
{
var settings =
new CouchbaseSettings()
{
Servers = new List <Uri>()
{
new Uri("couchbase://10.0.0.90"),
new Uri("couchbase://10.0.0.91"),
new Uri("couchbase://10.0.0.92")
},
Bucket = "stoke"
};
var credentials =
new Credentials()
{
Username = Environment.GetEnvironmentVariable("TS_COUCHBASE_USERNAME"),
Password = Environment.GetEnvironmentVariable("TS_COUCHBASE_PASSWORD")
};
using (var bucket = settings.OpenBucket(credentials))
{
var retry = new ExponentialRetryPolicy(CouchbaseTransientDetector.IsTransient);
for (int i = 0; i < 500000; i++)
{
var key = bucket.GenKey();
await retry.InvokeAsync(async() => await bucket.InsertSafeAsync(key, new Document <Item>()
{
Id = key, Content = new Item()
{
Name = "Jeff", Age = 56
}
}));
var exists = await bucket.ExistsAsync(key);
var result2 = await bucket.GetAsync <Document <Item> >(key);
result2.EnsureSuccess();
}
}
}
catch (OperationCanceledException)
{
return;
}
catch (Exception e)
{
log.LogError(e);
}
finally
{
terminator.ReadyToExit();
}
}
public async Task SuccessImmediate_Result()
{
var policy = new ExponentialRetryPolicy(TransientDetector);
var times = new List <DateTime>();
var success = await policy.InvokeAsync(
async() =>
{
times.Add(DateTime.UtcNow);
await Task.Delay(0);
return("WOOHOO!");
});
Assert.Single(times);
Assert.Equal("WOOHOO!", success);
}
public async Task SuccessImmediate()
{
var policy = new ExponentialRetryPolicy(TransientDetector);
var times = new List <DateTime>();
var success = false;
await policy.InvokeAsync(
async() =>
{
times.Add(DateTime.UtcNow);
await Task.Delay(0);
success = true;
});
Assert.Single(times);
Assert.True(success);
}
public async Task FailImmediate_Result()
{
var policy = new ExponentialRetryPolicy(TransientDetector);
var times = new List <DateTime>();
await Assert.ThrowsAsync <NotImplementedException>(
async() =>
{
await policy.InvokeAsync <string>(
async() =>
{
times.Add(DateTime.UtcNow);
await Task.Delay(0);
throw new NotImplementedException();
});
});
Assert.Single(times);
}
public async Task FailAll_Result()
{
var policy = new ExponentialRetryPolicy(TransientDetector);
var times = new List <DateTime>();
await Assert.ThrowsAsync <TransientException>(
async() =>
{
await policy.InvokeAsync <string>(
async() =>
{
times.Add(DateTime.UtcNow);
await Task.Delay(0);
throw new TransientException();
});
});
Assert.Equal(policy.MaxAttempts, times.Count);
VerifyIntervals(times, policy);
}
public async Task SuccessDelayed_Result()
{
var policy = new ExponentialRetryPolicy(TransientDetector);
var times = new List <DateTime>();
var success = await policy.InvokeAsync(
async() =>
{
times.Add(DateTime.UtcNow);
await Task.Delay(0);
if (times.Count < policy.MaxAttempts)
{
throw new TransientException();
}
return("WOOHOO!");
});
Assert.Equal("WOOHOO!", success);
Assert.Equal(policy.MaxAttempts, times.Count);
VerifyIntervals(times, policy);
}
public async Task SuccessDelayedByType()
{
var policy = new ExponentialRetryPolicy(typeof(NotReadyException));
var times = new List <DateTime>();
var success = false;
await policy.InvokeAsync(
async() =>
{
times.Add(DateTime.UtcNow);
await Task.Delay(0);
if (times.Count < policy.MaxAttempts)
{
throw new NotReadyException();
}
success = true;
});
Assert.True(success);
Assert.Equal(policy.MaxAttempts, times.Count);
VerifyIntervals(times, policy);
}
public async Task SuccessDelayed()
{
var policy = new ExponentialRetryPolicy(TransientDetector);
var times = new List <DateTime>();
var success = false;
await policy.InvokeAsync(
async() =>
{
times.Add(DateTime.UtcNow);
await Task.CompletedTask;
if (times.Count < policy.MaxAttempts)
{
throw new TransientException();
}
success = true;
});
Assert.True(success);
Assert.Equal(policy.MaxAttempts, times.Count);
VerifyIntervals(times, policy);
}
/// <summary>
/// <para>
/// Deletes workflow runs from a GitHub repo.
/// </para>
/// <note>
/// Only completed runs will be deleted.
/// </note>
/// </summary>
/// <param name="repo">Identifies the target repository.</param>
/// <param name="workflowName">
/// Optionally specifies the workflow whose runs are to be deleted otherwise
/// runs from all workflows in the repo will be deleted.
/// </param>
/// <param name="maxAge">
/// Optionally specifies the maximum age for retained workflow runs. This
/// defaults to <see cref="TimeSpan.Zero"/> which deletes all runs.
/// </param>
/// <returns>The number of runs deleted.</returns>
public async Task <int> DeleteRunsAsync(string repo, string workflowName = null, TimeSpan maxAge = default)
{
await SyncContext.Clear;
GitHub.GetCredentials();
var repoPath = GitHubRepoPath.Parse(repo);
var deleteCount = 0;
using (var client = new HttpClient())
{
var retry = new ExponentialRetryPolicy(TransientDetector.NetworkOrHttp, 5);
client.BaseAddress = new Uri("https://api.github.com");
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", GitHub.AccessToken);
client.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("neonforge.com", "0"));
client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/vnd.github.v3+json"));
// List all of the workflow runs for the repo, paging to get all of them.
//
// https://docs.github.com/en/rest/reference/actions#list-workflow-runs-for-a-repository
var runs = new List <RunInfo>();
var page = 1;
while (true)
{
var response = await retry.InvokeAsync(
async() =>
{
var request = new HttpRequestMessage(HttpMethod.Get, $"/repos/{repoPath.Owner}/{repoPath.Repo}/actions/runs?page={page}");
// We're seeing some 502 Bad Gateway responses from GHCR.io. We're going to
// treat these as transients.
var response = await client.SendAsync(request);
if (response.StatusCode == HttpStatusCode.BadGateway)
{
throw new TransientException("503 (Bad Gateway)");
}
return(response);
});
response.EnsureSuccessStatusCode();
var json = response.Content.ReadAsStringAsync().Result;
var result = JsonConvert.DeserializeObject <dynamic>(json);
var workflowRuns = result.workflow_runs;
if (workflowRuns.Count == 0)
{
// We've seen all of the runs.
break;
}
foreach (var run in workflowRuns)
{
runs.Add(
new RunInfo()
{
Id = run.id,
Name = run.name,
Status = run.status,
UpdatedAtUtc = run.updated_at
});
}
page++;
}
// Here's the reference for deleting runs:
//
// https://docs.github.com/en/rest/reference/actions#delete-a-workflow-run
var minDate = DateTime.UtcNow - maxAge;
var selectedRuns = runs.Where(run => run.UpdatedAtUtc < minDate && run.Status == "completed");
if (!string.IsNullOrEmpty(workflowName))
{
selectedRuns = selectedRuns.Where(run => run.Name.Equals(workflowName, StringComparison.InvariantCultureIgnoreCase));
}
foreach (var run in selectedRuns)
{
var response = await retry.InvokeAsync(
async() =>
{
var request = new HttpRequestMessage(HttpMethod.Delete, $"/repos/{repoPath.Owner}/{repoPath.Repo}/actions/runs/{run.Id}");
return(await client.SendAsync(request));
});
// We're also seeing some 500s but I'm not sure why. We'll ignore these
// for now.
if (response.StatusCode == HttpStatusCode.InternalServerError)
{
Task.Delay(TimeSpan.FromSeconds(2)).WaitWithoutAggregate(); // Pause in case this is a rate-limit thing
continue;
}
// We're seeing 403s for some runs, so we'll ignore those too.
if (response.StatusCode != HttpStatusCode.Forbidden)
{
response.EnsureSuccessStatusCode();
}
deleteCount++;
}
}
return(deleteCount);
}
