public ProactiveCopyResult(PushFileResult ringCopyResult, PushFileResult outsideRingCopyResult, int retries, ContentLocationEntry?entry = null)
{
RingCopyResult = ringCopyResult;
OutsideRingCopyResult = outsideRingCopyResult;
Retries = retries;
Entry = entry ?? ContentLocationEntry.Missing;
var results = new[] { ringCopyResult, outsideRingCopyResult };
if (results.Any(r => r.Succeeded))
{
Status = ProactiveCopyStatus.Success;
}
else if (results.Any(r => r.Status.IsRejection()))
{
Status = ProactiveCopyStatus.Rejected;
}
else if (results.All(r => r.Status == CopyResultCode.Disabled))
{
Status = ProactiveCopyStatus.Skipped;
}
else
{
Status = ProactiveCopyStatus.Error;
}
if (!IsSuccessfulStatus(Status))
{
var error = GetErrorMessage(ringCopyResult, outsideRingCopyResult);
var diagnostics = GetDiagnostics(ringCopyResult, outsideRingCopyResult);
_error = Error.FromErrorMessage(error !, diagnostics);
}
}
public ProactiveCopyResult(PushFileResult ringCopyResult, PushFileResult outsideRingCopyResult, ContentLocationEntry?entry = null)
: base(GetErrorMessage(ringCopyResult, outsideRingCopyResult), GetDiagnostics(ringCopyResult, outsideRingCopyResult))
{
RingCopyResult = ringCopyResult;
OutsideRingCopyResult = outsideRingCopyResult;
Entry = entry ?? ContentLocationEntry.Missing;
var results = new[] { ringCopyResult, outsideRingCopyResult };
if (results.Any(r => r.Succeeded))
{
Status = ProactiveCopyStatus.Success;
}
else if (results.Any(r => r.Status == RejectedByServer))
{
Status = ProactiveCopyStatus.Rejected;
}
else if (results.All(r => r.Status == Disabled))
{
Status = ProactiveCopyStatus.Skipped;
}
else
{
Status = ProactiveCopyStatus.Error;
}
}
public void Success()
{
var result = PushFileResult.PushSucceeded();
result.Succeeded.Should().BeTrue();
result.ToString().Should().Contain("Success");
}
public void Disabled()
{
var result = PushFileResult.Disabled();
result.Succeeded.Should().BeFalse();
result.ToString().Should().Contain("Disabled");
}
public void Rejected()
{
var result = PushFileResult.Rejected();
result.Succeeded.Should().BeFalse();
result.ToString().Should().Contain("Rejected");
}
public void Unavailable()
{
var result = PushFileResult.ServerUnavailable();
result.Succeeded.Should().BeFalse();
result.ToString().Should().Contain("Unavailable");
}
public void Reject_Succeeded_Is_Succeeded()
{
var ringCopyResult = PushFileResult.Rejected(RejectionReason.OlderThanLastEvictedContent);
var outsideRingCopyResult = PushFileResult.PushSucceeded();
var result = new ProactiveCopyResult(ringCopyResult, outsideRingCopyResult, retries: 0);
// Even if one of the operations is successful, the overall operation is successful.
result.Succeeded.Should().BeTrue();
result.ToString().Should().Contain(ringCopyResult.Status.ToString());
}
private static string?GetDiagnostics(PushFileResult ringCopyResult, PushFileResult outsideRingCopyResult)
{
if (!ringCopyResult.Status.IsSuccess() || !outsideRingCopyResult.Status.IsSuccess())
{
return
($"RingMachineResult=[{ringCopyResult.GetStatusOrDiagnostics()}] " +
$"OutsideRingMachineResult=[{outsideRingCopyResult.GetStatusOrDiagnostics()}] ");
}
return(null);
}
private static string?GetDiagnostics(PushFileResult ringCopyResult, PushFileResult outsideRingCopyResult)
{
if (ringCopyResult.Status == Error || outsideRingCopyResult.Status == Error)
{
return
($"RingMachineResult=[{ringCopyResult.GetStatusOrDiagnostics()}] " +
$"OutsideRingMachineResult=[{outsideRingCopyResult.GetStatusOrDiagnostics()}] ");
}
return(null);
}
public void Unavailable_Success_Is_Succeeded()
{
var ringCopyResult = PushFileResult.ServerUnavailable();
var outsideRingCopyResult = PushFileResult.PushSucceeded();
var result = new ProactiveCopyResult(ringCopyResult, outsideRingCopyResult, retries: 0);
result.Succeeded.Should().BeTrue();
ringCopyResult.Succeeded.Should().BeFalse();
outsideRingCopyResult.Succeeded.Should().BeTrue();
result.ToString().Should().Contain("Success");
result.ToString().Should().Contain("Unavailable");
}
public void Reject_Reject_Is_Rejected()
{
var ringCopyResult = PushFileResult.Rejected(RejectionReason.OlderThanLastEvictedContent);
var outsideRingCopyResult = PushFileResult.Rejected(RejectionReason.OlderThanLastEvictedContent);
var result = new ProactiveCopyResult(ringCopyResult, outsideRingCopyResult, retries: 0);
result.Succeeded.Should().BeFalse();
result.Status.Should().Be(ProactiveCopyStatus.Rejected);
ringCopyResult.Succeeded.Should().BeFalse();
result.ToString().Should().Contain(ringCopyResult.Status.ToString());
}
public void Reject_Reject_Is_Rejected()
{
var ringCopyResult = PushFileResult.Rejected();
var outsideRingCopyResult = PushFileResult.Rejected();
var result = new ProactiveCopyResult(ringCopyResult, outsideRingCopyResult);
result.Succeeded.Should().BeFalse();
result.Status.Should().Be(ProactiveCopyStatus.Rejected);
ringCopyResult.Succeeded.Should().BeFalse();
result.ToString().Should().Contain(ringCopyResult.Status.ToString());
}
private static string?GetErrorMessage(PushFileResult ringCopyResult, PushFileResult outsideRingCopyResult)
{
if (ringCopyResult.Status == Error || outsideRingCopyResult.Status == Error)
{
return
($"Success count: {(ringCopyResult.Succeeded ^ outsideRingCopyResult.Succeeded ? 1 : 0)} " +
$"RingMachineResult=[{ringCopyResult.GetStatusOrErrorMessage()}] " +
$"OutsideRingMachineResult=[{outsideRingCopyResult.GetStatusOrErrorMessage()}] ");
}
return(null);
}
private static string?GetErrorMessage(PushFileResult ringCopyResult, PushFileResult outsideRingCopyResult)
{
if (!ringCopyResult.Status.IsSuccess() || !outsideRingCopyResult.Status.IsSuccess())
{
return
($"Success count: {(ringCopyResult.Succeeded ^ outsideRingCopyResult.Succeeded ? 1 : 0)} " +
$"Ring=[{ringCopyResult.GetStatusOrErrorMessage()}], " +
$"OutsideRing=[{outsideRingCopyResult.GetStatusOrErrorMessage()}] ");
}
return(null);
}
public void Unavailable_Error_Is_Error()
{
var ringCopyResult = PushFileResult.ServerUnavailable();
var error = "my error";
var outsideRingCopyResult = new PushFileResult(error);
var result = new ProactiveCopyResult(ringCopyResult, outsideRingCopyResult, retries: 0);
result.Succeeded.Should().BeFalse();
result.Status.Should().Be(ProactiveCopyStatus.Error);
result.ToString().Should().Contain(ringCopyResult.Status.ToString());
result.ToString().Should().Contain(outsideRingCopyResult.Status.ToString());
}
public void Diabled_Succeeded_Is_Succeeded()
{
var ringCopyResult = PushFileResult.Disabled();
var outsideRingCopyResult = PushFileResult.PushSucceeded();
var result = new ProactiveCopyResult(ringCopyResult, outsideRingCopyResult, retries: 0);
result.Succeeded.Should().BeTrue();
ringCopyResult.Succeeded.Should().BeFalse();
outsideRingCopyResult.Succeeded.Should().BeTrue();
result.ToString().Should().Contain("Success");
result.ToString().Should().Contain("Disabled");
}
public void Error_Succeeded_Is_Succeeded()
{
var myDiagnostics = "My diagnostics";
var myErrorMessage = "My error message";
var ringCopyResult = new PushFileResult(myErrorMessage, myDiagnostics);
var outsideRingCopyResult = PushFileResult.PushSucceeded();
var result = new ProactiveCopyResult(ringCopyResult, outsideRingCopyResult, retries: 0);
// Even if one of the operations is successful, the overall operation is successful.
result.Succeeded.Should().BeTrue();
result.ToString().Should().NotContain(myDiagnostics, "The final error should not have an error message. Only error status");
result.ToString().Should().NotContain(myErrorMessage, "The final error should not have an error message. Only error status");
result.ToString().Should().Contain(ringCopyResult.Status.ToString());
}
public void Error_Reject_Is_Rejected()
{
string error = "my error";
var ringCopyResult = new PushFileResult(error);
var outsideRingCopyResult = PushFileResult.Rejected();
var result = new ProactiveCopyResult(ringCopyResult, outsideRingCopyResult);
result.Succeeded.Should().BeFalse();
result.Status.Should().Be(ProactiveCopyStatus.Rejected);
result.ToString().Should().NotContain("Success");
result.ToString().Should().NotContain(error);
result.ToString().Should().Contain(ringCopyResult.Status.ToString());
}
public virtual async Task <PushFileResult> PushFileAsync(OperationContext context, ContentHash hash, Stream stream, MachineLocation targetMachine, CopyOptions options)
{
var tempFile = AbsolutePath.CreateRandomFileName(WorkingDirectory);
using (var file = File.OpenWrite(tempFile.Path))
{
await stream.CopyToAsync(file);
}
var result = await PushHandlersByLocation[targetMachine].HandlePushFileAsync(context, hash, tempFile, CancellationToken.None);
File.Delete(tempFile.Path);
return(result ? PushFileResult.PushSucceeded(result.ContentSize) : new PushFileResult(result));
}
public async Task <PushFileResult> PushFileAsync(OperationContext context, ContentHash hash, Func <Task <Result <Stream> > > source, MachineLocation targetMachine)
{
var tempFile = AbsolutePath.CreateRandomFileName(WorkingDirectory);
var streamResult = await source();
using var stream = streamResult.Value;
using (var file = File.OpenWrite(tempFile.Path))
{
await stream.CopyToAsync(file);
}
var result = await PushHandlersByLocation[targetMachine].HandlePushFileAsync(context, hash, tempFile, CancellationToken.None);
File.Delete(tempFile.Path);
return(result ? PushFileResult.PushSucceeded() : new PushFileResult(result));
}
public Task ProactiveCopyInsideRingRetriesTest()
{
EnableProactiveCopy = true;
ProactiveCopyMode = ProactiveCopyMode.InsideRing; // Disable outside-ring proactive copies.
ProactiveCopyOnPuts = true;
ProactiveCopyOnPins = true;
ProactiveCopyLocationThreshold = 4; // Large enough that we 'always' try to push.
ProactiveCopyRetries = 2;
var contentHashes = new List <ContentHash>();
int machineCount = 3;
ConfigureWithOneMaster();
var buildId = Guid.NewGuid().ToString();
return(RunTestAsync(
machineCount,
async context =>
{
var masterStore = context.GetMaster();
var defaultFileSize = (Config.MaxSizeQuota.Hard / 4) + 1;
var sessions = context.GetDistributedSessions();
// Insert random file #1 into worker #1
var putResult = await sessions[0].PutRandomAsync(context, HashType.Vso0, false, defaultFileSize, Token).ShouldBeSuccess();
var hash = putResult.ContentHash;
var getBulkResult = await masterStore.GetBulkAsync(context, hash, GetBulkOrigin.Global).ShouldBeSuccess();
// Proactive copy should have replicated the content.
getBulkResult.ContentHashesInfo[0].Locations.Count.Should().Be(2);
var counters = sessions[0].GetCounters().ToDictionaryIntegral();
counters["ProactiveCopyInsideRingRetries.Count"].Should().Be(ProactiveCopyRetries);
counters["ProactiveCopyRetries.Count"].Should().Be(ProactiveCopyRetries);
},
implicitPin: ImplicitPin.None,
buildId: buildId,
testCopier: new ErrorReturningTestFileCopier(errorsToReturn: ProactiveCopyRetries, failingResult: PushFileResult.ServerUnavailable())));
}
public Task ProactiveCopyRetryTest()
{
EnableProactiveCopy = true;
ProactiveCopyRetries = 2;
var contentHashes = new List <ContentHash>();
int machineCount = 2;
ConfigureWithOneMaster();
return(RunTestAsync(
machineCount,
async context =>
{
var masterStore = context.GetMaster();
var defaultFileSize = (Config.MaxSizeQuota.Hard / 4) + 1;
var sessions = context.EnumerateWorkersIndices().Select(i => context.GetDistributedSession(i)).ToArray();
// Insert random file #1 into worker #1
var putResult1 = await sessions[0].PutRandomAsync(context, HashType.Vso0, false, defaultFileSize, Token).ShouldBeSuccess();
var hash1 = putResult1.ContentHash;
var getBulkResult1 = await masterStore.GetBulkAsync(context, hash1, GetBulkOrigin.Global).ShouldBeSuccess();
// Proactive copy should have replicated the content.
getBulkResult1.ContentHashesInfo[0].Locations.Count.Should().Be(2);
var counters = sessions[0].GetCounters().ToDictionaryIntegral();
counters["ProactiveCopyRetries.Count"].Should().Be(ProactiveCopyRetries);
counters["ProactiveCopyOutsideRingRetries.Count"].Should().Be(ProactiveCopyRetries);
},
testCopier: new ErrorReturningTestFileCopier(errorsToReturn: ProactiveCopyRetries, failingResult: PushFileResult.ServerUnavailable()),
implicitPin: ImplicitPin.None));
}
public virtual async Task <PushFileResult> PushFileAsync(OperationContext context, ContentHash hash, Stream stream, MachineLocation targetMachine, CopyOptions options)
{
var result = await PushHandlersByLocation[targetMachine].HandlePushFileAsync(context, hash, new FileSource(stream), CancellationToken.None);
return(result ? PushFileResult.PushSucceeded(result.ContentSize) : new PushFileResult(result));
}
