internal void RemoveFromTracker(
[DefaultValue((uint)7089), Description("GRPC port")] uint grpcPort)
{
Initialize();
// We need to initialize this here because there's no codepath otherwise.
GrpcEnvironment.Initialize();
var context = new Context(_logger);
var retryPolicy = RetryPolicyFactory.GetLinearPolicy(ex => ex is ClientCanRetryException, (int)_retryCount, TimeSpan.FromSeconds(_retryIntervalSeconds));
_logger.Debug("Begin repair handling...");
// This action is synchronous to make sure the calling application doesn't exit before the method returns.
using (var rpcClient = new GrpcRepairClient(grpcPort))
{
var removeFromTrackerResult = retryPolicy.ExecuteAsync(() => rpcClient.RemoveFromTrackerAsync(context), _cancellationToken).Result;
if (!removeFromTrackerResult.Succeeded)
{
throw new CacheException(removeFromTrackerResult.ErrorMessage);
}
else
{
_logger.Debug($"Repair handling succeeded. Removed {removeFromTrackerResult.Data} hashes from the content tracker.");
}
var shutdownResult = rpcClient.ShutdownAsync(context).Result;
if (!shutdownResult.Succeeded)
{
throw new CacheException(shutdownResult.ErrorMessage);
}
}
}
public async Task TestTracingRetryPolicy()
{
var context = new OperationContext(new Context(TestGlobal.Logger));
const int RetryCount = 3;
var retryPolicy = RetryPolicyFactory.GetLinearPolicy(shouldRetry: _ => true, retries: RetryCount, retryInterval: TimeSpan.FromMilliseconds(1));
int callBackCallCount = 0;
try
{
await retryPolicy.ExecuteAsync(
context.TracingContext,
async() =>
{
callBackCallCount++;
await Task.Yield();
throw new ApplicationException("1");
},
CancellationToken.None,
databaseName : string.Empty);
Assert.True(false, "ExecuteAsync should fail.");
}
catch (ApplicationException)
{
var fullOutput = GetFullOutput();
fullOutput.Should().Contain($"Redis operation '{nameof(TestTracingRetryPolicy)}' failed");
callBackCallCount.Should().Be(RetryCount + 1); // +1 because we have: original try + the number of retries.
}
}
public async Task Test(bool usePolly)
{
RetryPolicyFactory.UsePolly = usePolly;
var policy = RetryPolicyFactory.GetLinearPolicy(shouldRetry: _ => true, retries: 3, retryInterval: TimeSpan.FromMilliseconds(1));
var c = 0;
(await policy.ExecuteAsync(() => c++ == 3 ? Task.FromResult(true) : throw new Exception(), CancellationToken.None)).Should().BeTrue();
}
internal void CopyFile(
[Required, Description("Machine to copy from")] string host,
[Required, Description("Expected content hash")] string hashString,
[Required, Description("Path to destination file")] string destinationPath,
[Description("Whether or not GZip is enabled"), DefaultValue(false)] bool useCompressionForCopies,
[Description("File name where the GRPC port can be found when using cache service. 'CASaaS GRPC port' if not specified")] string grpcPortFileName,
[Description("The GRPC port"), DefaultValue(0)] int grpcPort)
{
Initialize();
var context = new Context(_logger);
var retryPolicy = RetryPolicyFactory.GetLinearPolicy(ex => ex is ClientCanRetryException, (int)_retryCount, TimeSpan.FromSeconds(_retryIntervalSeconds));
if (grpcPort == 0)
{
grpcPort = Helpers.GetGrpcPortFromFile(_logger, grpcPortFileName);
}
if (!ContentHash.TryParse(hashString, out var hash))
{
throw new CacheException($"Invalid content hash string provided: {hashString}");
}
try
{
var config = new GrpcCopyClientConfiguration();
using var clientCache = new GrpcCopyClientCache(context, new GrpcCopyClientCacheConfiguration()
{
GrpcCopyClientConfiguration = config
});
var finalPath = new AbsolutePath(destinationPath);
var copyFileResult = clientCache.UseAsync(new OperationContext(context), host, grpcPort, (nestedContext, rpcClient) =>
{
return(retryPolicy.ExecuteAsync(
() => rpcClient.CopyFileAsync(nestedContext, hash, finalPath, new CopyOptions(bandwidthConfiguration: null)
{
CompressionHint = useCompressionForCopies ? CopyCompression.Gzip : CopyCompression.None,
}),
_cancellationToken));
}).GetAwaiter().GetResult();
if (!copyFileResult.Succeeded)
{
throw new CacheException(copyFileResult.ErrorMessage);
}
else
{
_logger.Debug($"Copy of {hashString} to {finalPath} was successful");
}
}
catch (Exception ex)
{
throw new CacheException(ex.ToString());
}
}
internal void CopyFileTo(
[Required, Description("Machine to copy to")] string host,
[Required, Description("Path to source file")] string sourcePath,
[Description("File name where the GRPC port can be found when using cache service. 'CASaaS GRPC port' if not specified")] string grpcPortFileName,
[Description("The GRPC port"), DefaultValue(0)] int grpcPort)
{
Initialize();
var context = new Context(_logger);
var operationContext = new OperationContext(context, CancellationToken.None);
var retryPolicy = RetryPolicyFactory.GetLinearPolicy(ex => ex is ClientCanRetryException, (int)_retryCount, TimeSpan.FromSeconds(_retryIntervalSeconds));
if (grpcPort == 0)
{
grpcPort = Helpers.GetGrpcPortFromFile(_logger, grpcPortFileName);
}
var hasher = ContentHashers.Get(HashType.MD5);
var bytes = File.ReadAllBytes(sourcePath);
var hash = hasher.GetContentHash(bytes);
try
{
var path = new AbsolutePath(sourcePath);
using Stream stream = File.OpenRead(path.Path);
var config = GrpcCopyClientConfiguration.WithGzipCompression(false);
config.BandwidthCheckerConfiguration = BandwidthChecker.Configuration.Disabled;
using var clientCache = new GrpcCopyClientCache(context, new GrpcCopyClientCacheConfiguration()
{
GrpcCopyClientConfiguration = config
});
var copyFileResult = clientCache.UseAsync(operationContext, host, grpcPort, (nestedContext, rpcClient) =>
{
return(retryPolicy.ExecuteAsync(
() => rpcClient.PushFileAsync(nestedContext, hash, stream, new CopyOptions(bandwidthConfiguration: null)),
_cancellationToken));
}).GetAwaiter().GetResult();
if (!copyFileResult.Succeeded)
{
_tracer.Error(context, $"{copyFileResult}");
throw new CacheException(copyFileResult.ErrorMessage);
}
else
{
_tracer.Info(context, $"Copy of {sourcePath} was successful");
}
}
catch (Exception ex)
{
throw new CacheException(ex.ToString());
}
}
public async Task RetryPolicyStopsOnCancellation(bool usePolly)
{
RetryPolicyFactory.UsePolly = usePolly;
// This test shows that if a cancellation token provided to 'RetryPolicy' is triggered
// and at least one error already occurred, then the operation will fail with the last exception
// If using Polly, TaskCancelledException is thrown.
var cts = new CancellationTokenSource();
var context = new OperationContext(new Context(TestGlobal.Logger), cts.Token);
const int RetryCount = 4;
var retryPolicy = RetryPolicyFactory.GetLinearPolicy(shouldRetry: _ => true, retries: RetryCount, retryInterval: TimeSpan.FromMilliseconds(1));
int callBackCallCount = 0;
try
{
await retryPolicy.ExecuteAsync(
context.TracingContext,
async() =>
{
callBackCallCount++;
if (callBackCallCount == 2)
{
cts.Cancel();
}
await Task.Yield();
throw new ApplicationException($"{callBackCallCount}");
},
context.Token,
databaseName : string.Empty);
Assert.True(false, "ExecuteAsync should fail.");
}
catch (ApplicationException e)
{
usePolly.Should().BeFalse();
callBackCallCount.Should().Be(2);
e.Message.Should().Be("2");
}
catch (TaskCanceledException)
{
usePolly.Should().BeTrue();
callBackCallCount.Should().Be(2);
}
}
public IRetryPolicy CreateRetryPolicy(Action <Exception> onRedisException)
{
var policy = new RedisDatabaseAdapter.RedisRetryPolicy(onRedisException, TreatObjectDisposedExceptionAsTransient);
if (_retryCount != null)
{
return(RetryPolicyFactory.GetLinearPolicy(policy.IsTransient, _retryCount.Value));
}
else if (ExponentialBackoffConfiguration != null)
{
var config = ExponentialBackoffConfiguration;
return(RetryPolicyFactory.GetExponentialPolicy(policy.IsTransient, config.RetryCount, config.MinBackoff, config.MaxBackoff, config.DeltaBackoff));
}
else
{
return(RetryPolicyFactory.GetExponentialPolicy(policy.IsTransient));
}
}
/// <nodoc />
public ServiceClientContentStoreConfiguration(
string cacheName,
ServiceClientRpcConfiguration rpcConfiguration,
string?scenario = null)
{
Contract.RequiresNotNullOrEmpty(cacheName);
CacheName = cacheName;
RpcConfiguration = rpcConfiguration;
Scenario = scenario;
_retryPolicy = new Lazy <IRetryPolicy>(
() =>
{
var strategy = new TransientErrorDetectionStrategy();
return(RetryPolicyFactory.GetLinearPolicy(
shouldRetry: e => strategy.IsTransient(e),
(int)RetryCount,
TimeSpan.FromSeconds(RetryIntervalSeconds)));
});
}