public virtual GitRefs QueryInfoRefs(string branch)
{
Uri infoRefsEndpoint;
try
{
infoRefsEndpoint = new Uri(this.enlistment.RepoUrl + GVFSConstants.InfoRefsEndpointSuffix);
}
catch (UriFormatException)
{
return(null);
}
RetryWrapper <GitRefs> retrier = new RetryWrapper <GitRefs>(this.MaxRetries);
retrier.OnFailure += RetryWrapper <GitRefs> .StandardErrorHandler(this.Tracer, "QueryInfoRefs");
RetryWrapper <GitRefs> .InvocationResult output = retrier.Invoke(
tryCount =>
{
GitEndPointResponseData response = this.SendRequest(infoRefsEndpoint, HttpMethod.Get, null);
if (response.HasErrors)
{
return(new RetryWrapper <GitRefs> .CallbackResult(response.Error, response.ShouldRetry));
}
using (StreamReader reader = new StreamReader(response.Stream))
{
List <string> infoRefsResponse = reader.RetryableReadAllLines();
return(new RetryWrapper <GitRefs> .CallbackResult(new GitRefs(infoRefsResponse, branch)));
}
});
return(output.Result);
}
private RetryWrapper <GitObjectTaskResult> .InvocationResult GetSingleObject(
string objectId,
Func <int, GitEndPointResponseData, RetryWrapper <GitObjectTaskResult> .CallbackResult> onSuccess,
Action <RetryWrapper <GitObjectTaskResult> .ErrorEventArgs> onFailure)
{
if (this.shaContents.ContainsKey(objectId))
{
using (GitEndPointResponseData response = new GitEndPointResponseData(
HttpStatusCode.OK,
GVFSConstants.MediaTypes.LooseObjectMediaType,
new ReusableMemoryStream(this.shaContents[objectId]),
message: null,
onResponseDisposed: null))
{
RetryWrapper <GitObjectTaskResult> .CallbackResult result = onSuccess(1, response);
return(new RetryWrapper <GitObjectTaskResult> .InvocationResult(1, true, result.Result));
}
}
if (onFailure != null)
{
onFailure(new RetryWrapper <GitObjectTaskResult> .ErrorEventArgs(new Exception("Could not find mock object: " + objectId), 1, false));
}
return(new RetryWrapper <GitObjectTaskResult> .InvocationResult(1, new Exception("Mock failure in TryDownloadObjectsAsync")));
}
public override bool TryDownloadAndSaveCommits(IEnumerable <string> objectShas, int commitDepth)
{
bool output = true;
foreach (string sha in objectShas)
{
RetryWrapper <HttpGitObjects.GitObjectTaskResult> .InvocationResult result = this.GitObjectRequestor.TryDownloadObjects(
new[] { sha },
commitDepth,
onSuccess: (tryCount, response) =>
{
// Add the contents to the mock repo
using (StreamReader reader = new StreamReader(response.Stream))
{
((MockGitRepo)this.Context.Repository).AddBlob(sha, "DownloadedFile", reader.ReadToEnd());
}
return(new RetryWrapper <HttpGitObjects.GitObjectTaskResult> .CallbackResult(new HttpGitObjects.GitObjectTaskResult(true)));
},
onFailure: null,
preferBatchedLooseObjects: false);
return(result.Succeeded && result.Result.Success);
}
return(output);
}
public void WillRetryWhenRequested()
{
const int MaxTries = 5;
const int ExpectedFailures = 5;
const int ExpectedTries = 5;
RetryWrapper <bool> dut = new RetryWrapper <bool>(MaxTries, exponentialBackoffBase: 0);
int actualFailures = 0;
dut.OnFailure += errorArgs => actualFailures++;
int actualTries = 0;
RetryWrapper <bool> .InvocationResult output = dut.InvokeAsync(
tryCount =>
{
actualTries++;
return(Task.Run(() => new RetryWrapper <bool> .CallbackResult(new Exception("Test"), true)));
}).Result;
output.Succeeded.ShouldEqual(false);
output.Result.ShouldEqual(false);
actualTries.ShouldEqual(ExpectedTries);
actualFailures.ShouldEqual(ExpectedFailures);
}
public void WillNotRetryForWhenCanceledDuringAttempts()
{
const int MaxTries = 5;
int actualTries = 0;
int expectedTries = 3;
using (CancellationTokenSource tokenSource = new CancellationTokenSource())
{
RetryWrapper <bool> dut = new RetryWrapper <bool>(MaxTries, tokenSource.Token, exponentialBackoffBase: 0);
Assert.Throws <OperationCanceledException>(
() =>
{
RetryWrapper <bool> .InvocationResult output = dut.Invoke(tryCount =>
{
++actualTries;
if (actualTries == expectedTries)
{
tokenSource.Cancel();
}
return(new RetryWrapper <bool> .CallbackResult(new Exception("Test"), shouldRetry: true));
});
});
actualTries.ShouldEqual(expectedTries);
}
}
public void WillNotRetryWhenCancelledDuringBackoff()
{
const int MaxTries = 5;
int actualTries = 0;
int expectedTries = 2; // 2 because RetryWrapper does not wait after the first failure
using (CancellationTokenSource tokenSource = new CancellationTokenSource())
{
RetryWrapper <bool> dut = new RetryWrapper <bool>(MaxTries, tokenSource.Token, exponentialBackoffBase: 300);
Task.Run(() =>
{
// Wait 3 seconds and cancel
Thread.Sleep(1000 * 3);
tokenSource.Cancel();
});
Assert.Throws <OperationCanceledException>(
() =>
{
RetryWrapper <bool> .InvocationResult output = dut.Invoke(tryCount =>
{
++actualTries;
return(new RetryWrapper <bool> .CallbackResult(new Exception("Test"), shouldRetry: true));
});
});
actualTries.ShouldEqual(expectedTries);
}
}
public void WillNotRetryWhenNotRequested()
{
const int MaxTries = 5;
const int ExpectedFailures = 1;
const int ExpectedTries = 1;
RetryWrapper <bool> dut = new RetryWrapper <bool>(MaxTries, CancellationToken.None, exponentialBackoffBase: 0);
int actualFailures = 0;
dut.OnFailure += errorArgs => actualFailures++;
int actualTries = 0;
RetryWrapper <bool> .InvocationResult output = dut.Invoke(
tryCount =>
{
actualTries++;
return(new RetryWrapper <bool> .CallbackResult(new Exception("Test"), shouldRetry: false));
});
output.Succeeded.ShouldEqual(false);
output.Result.ShouldEqual(false);
actualTries.ShouldEqual(ExpectedTries);
actualFailures.ShouldEqual(ExpectedFailures);
}
protected virtual DownloadAndSaveObjectResult TryDownloadAndSaveObject(string objectSha)
{
if (objectSha == GVFSConstants.AllZeroSha)
{
return(DownloadAndSaveObjectResult.Error);
}
RetryWrapper <HttpGitObjects.GitObjectTaskResult> .InvocationResult output = this.GitObjectRequestor.TryDownloadLooseObject(
objectSha,
onSuccess: (tryCount, response) =>
{
this.WriteLooseObject(this.Enlistment.WorkingDirectoryRoot, response.Stream, objectSha);
return(new RetryWrapper <HttpGitObjects.GitObjectTaskResult> .CallbackResult(new HttpGitObjects.GitObjectTaskResult(true)));
},
onFailure: this.HandleDownloadAndSaveObjectError);
if (output.Succeeded && output.Result.Success)
{
return(DownloadAndSaveObjectResult.Success);
}
if (output.Result != null && output.Result.HttpStatusCodeResult == HttpStatusCode.NotFound)
{
return(DownloadAndSaveObjectResult.ObjectNotOnServer);
}
return(DownloadAndSaveObjectResult.Error);
}
public void OnSuccessIsOnlyCalledOnce()
{
const int MaxTries = 5;
const int ExpectedFailures = 0;
const int ExpectedTries = 1;
RetryWrapper <bool> dut = new RetryWrapper <bool>(MaxTries, exponentialBackoffBase: 0);
int actualFailures = 0;
dut.OnFailure += errorArgs => actualFailures++;
int actualTries = 0;
RetryWrapper <bool> .InvocationResult output = dut.InvokeAsync(
tryCount =>
{
actualTries++;
return(Task.Run(() => new RetryWrapper <bool> .CallbackResult(true)));
}).Result;
output.Succeeded.ShouldEqual(true);
output.Result.ShouldEqual(true);
actualTries.ShouldEqual(ExpectedTries);
actualFailures.ShouldEqual(ExpectedFailures);
}
protected virtual DownloadAndSaveObjectResult TryDownloadAndSaveObject(string objectSha)
{
if (objectSha == GVFSConstants.AllZeroSha)
{
return(DownloadAndSaveObjectResult.Error);
}
// To reduce allocations, reuse the same buffer when writing objects in this batch
byte[] bufToCopyWith = new byte[StreamUtil.DefaultCopyBufferSize];
RetryWrapper <HttpGitObjects.GitObjectTaskResult> .InvocationResult output = this.GitObjectRequestor.TryDownloadLooseObject(
objectSha,
onSuccess: (tryCount, response) =>
{
this.WriteLooseObject(this.Enlistment.WorkingDirectoryRoot, response.Stream, objectSha, bufToCopyWith);
return(new RetryWrapper <HttpGitObjects.GitObjectTaskResult> .CallbackResult(new HttpGitObjects.GitObjectTaskResult(true)));
},
onFailure: this.HandleDownloadAndSaveObjectError);
if (output.Succeeded && output.Result.Success)
{
return(DownloadAndSaveObjectResult.Success);
}
if (output.Result != null && output.Result.HttpStatusCodeResult == HttpStatusCode.NotFound)
{
return(DownloadAndSaveObjectResult.ObjectNotOnServer);
}
return(DownloadAndSaveObjectResult.Error);
}
public virtual RetryWrapper <GitObjectTaskResult> .InvocationResult TrySendProtocolRequest(
Func <int, GitEndPointResponseData, RetryWrapper <GitObjectTaskResult> .CallbackResult> onSuccess,
Action <RetryWrapper <GitObjectTaskResult> .ErrorEventArgs> onFailure,
HttpMethod method,
Func <Uri> endPointGenerator,
Func <string> requestBodyGenerator,
MediaTypeWithQualityHeaderValue acceptType = null)
{
RetryWrapper <GitObjectTaskResult> retrier = new RetryWrapper <GitObjectTaskResult>(this.MaxRetries);
if (onFailure != null)
{
retrier.OnFailure += onFailure;
}
return(retrier.Invoke(
tryCount =>
{
GitEndPointResponseData response = this.SendRequest(
endPointGenerator(),
method,
requestBodyGenerator(),
acceptType);
if (response.HasErrors)
{
return new RetryWrapper <GitObjectTaskResult> .CallbackResult(response.Error, response.ShouldRetry, new GitObjectTaskResult(response.StatusCode));
}
using (Stream responseStream = response.Stream)
{
return onSuccess(tryCount, response);
}
}));
}
public virtual bool TryDownloadPrefetchPacks(GitProcess gitProcess, long latestTimestamp, out List <string> packIndexes)
{
EventMetadata metadata = CreateEventMetadata();
metadata.Add("latestTimestamp", latestTimestamp);
using (ITracer activity = this.Tracer.StartActivity("TryDownloadPrefetchPacks", EventLevel.Informational, Keywords.Telemetry, metadata))
{
long bytesDownloaded = 0;
long requestId = HttpRequestor.GetNewRequestId();
List <string> innerPackIndexes = null;
RetryWrapper <GitObjectsHttpRequestor.GitObjectTaskResult> .InvocationResult result = this.GitObjectRequestor.TrySendProtocolRequest(
requestId: requestId,
onSuccess: (tryCount, response) => this.DeserializePrefetchPacks(response, ref latestTimestamp, ref bytesDownloaded, ref innerPackIndexes, gitProcess),
onFailure: RetryWrapper <GitObjectsHttpRequestor.GitObjectTaskResult> .StandardErrorHandler(activity, requestId, "TryDownloadPrefetchPacks"),
method: HttpMethod.Get,
endPointGenerator: () => new Uri(
string.Format(
"{0}?lastPackTimestamp={1}",
this.GitObjectRequestor.CacheServer.PrefetchEndpointUrl,
latestTimestamp)),
requestBodyGenerator: () => null,
cancellationToken: CancellationToken.None,
acceptType: new MediaTypeWithQualityHeaderValue(GVFSConstants.MediaTypes.PrefetchPackFilesAndIndexesMediaType));
packIndexes = innerPackIndexes;
if (!result.Succeeded)
{
if (result.Result != null && result.Result.HttpStatusCodeResult == HttpStatusCode.NotFound)
{
EventMetadata warning = CreateEventMetadata();
warning.Add(TracingConstants.MessageKey.WarningMessage, "The server does not support " + GVFSConstants.Endpoints.GVFSPrefetch);
warning.Add(nameof(this.GitObjectRequestor.CacheServer.PrefetchEndpointUrl), this.GitObjectRequestor.CacheServer.PrefetchEndpointUrl);
activity.RelatedEvent(EventLevel.Warning, "CommandNotSupported", warning);
}
else
{
EventMetadata error = CreateEventMetadata();
error.Add("latestTimestamp", latestTimestamp);
error.Add("Exception", result.Error);
error.Add(nameof(this.GitObjectRequestor.CacheServer.PrefetchEndpointUrl), this.GitObjectRequestor.CacheServer.PrefetchEndpointUrl);
activity.RelatedWarning(error, "DownloadPrefetchPacks failed.", Keywords.Telemetry);
}
}
activity.Stop(new EventMetadata
{
{ "Area", EtwArea },
{ "Success", result.Succeeded },
{ "Attempts", result.Attempts },
{ "BytesDownloaded", bytesDownloaded },
});
return(result.Succeeded);
}
}
public virtual bool TryCopyBlobContentStream(
string sha,
CancellationToken cancellationToken,
RequestSource requestSource,
Action <Stream, long> writeAction)
{
RetryWrapper <bool> retrier = new RetryWrapper <bool>(this.GitObjectRequestor.RetryConfig.MaxAttempts, cancellationToken);
retrier.OnFailure +=
errorArgs =>
{
EventMetadata metadata = new EventMetadata();
metadata.Add("sha", sha);
metadata.Add("AttemptNumber", errorArgs.TryCount);
metadata.Add("WillRetry", errorArgs.WillRetry);
if (errorArgs.Error != null)
{
metadata.Add("Exception", errorArgs.Error.ToString());
}
string message = "TryCopyBlobContentStream: Failed to provide blob contents";
if (errorArgs.WillRetry)
{
this.Tracer.RelatedWarning(metadata, message, Keywords.Telemetry);
}
else
{
this.Tracer.RelatedError(metadata, message);
}
};
RetryWrapper <bool> .InvocationResult invokeResult = retrier.Invoke(
tryCount =>
{
bool success = this.Context.Repository.TryCopyBlobContentStream(sha, writeAction);
if (success)
{
return(new RetryWrapper <bool> .CallbackResult(true));
}
else
{
// Pass in false for retryOnFailure because the retrier in this method manages multiple attempts
if (this.TryDownloadAndSaveObject(sha, cancellationToken, requestSource, retryOnFailure: false) == DownloadAndSaveObjectResult.Success)
{
if (this.Context.Repository.TryCopyBlobContentStream(sha, writeAction))
{
return(new RetryWrapper <bool> .CallbackResult(true));
}
}
return(new RetryWrapper <bool> .CallbackResult(error: null, shouldRetry: true));
}
});
return(invokeResult.Result);
}
private RetryWrapper <GitObjectTaskResult> .InvocationResult StreamObjects(
IEnumerable <string> objectIds,
Func <int, GitEndPointResponseData, RetryWrapper <GitObjectTaskResult> .CallbackResult> onSuccess,
Action <RetryWrapper <GitObjectTaskResult> .ErrorEventArgs> onFailure)
{
for (int i = 0; i < this.RetryConfig.MaxAttempts; ++i)
{
try
{
using (ReusableMemoryStream mem = new ReusableMemoryStream(string.Empty))
using (BinaryWriter writer = new BinaryWriter(mem))
{
writer.Write(new byte[] { (byte)'G', (byte)'V', (byte)'F', (byte)'S', (byte)' ', 1 });
foreach (string objectId in objectIds)
{
string contents = this.objectResolver(objectId);
if (!string.IsNullOrEmpty(contents))
{
writer.Write(this.SHA1BytesFromString(objectId));
byte[] bytes = Encoding.UTF8.GetBytes(contents);
writer.Write((long)bytes.Length);
writer.Write(bytes);
}
else
{
writer.Write(new byte[20]);
writer.Write(0L);
}
}
writer.Write(new byte[20]);
writer.Flush();
mem.Seek(0, SeekOrigin.Begin);
using (GitEndPointResponseData response = new GitEndPointResponseData(
HttpStatusCode.OK,
GSDConstants.MediaTypes.CustomLooseObjectsMediaType,
mem,
message: null,
onResponseDisposed: null))
{
RetryWrapper <GitObjectTaskResult> .CallbackResult result = onSuccess(1, response);
return(new RetryWrapper <GitObjectTaskResult> .InvocationResult(1, true, result.Result));
}
}
}
catch
{
continue;
}
}
return(new RetryWrapper <GitObjectTaskResult> .InvocationResult(this.RetryConfig.MaxAttempts, null));
}
protected override void DoWork()
{
BlobDownloadRequest request;
while (this.inputQueue.TryTake(out request))
{
Interlocked.Increment(ref this.activeDownloadCount);
EventMetadata metadata = new EventMetadata();
metadata.Add("PackId", request.PackId);
metadata.Add("ActiveDownloads", this.activeDownloadCount);
metadata.Add("NumberOfObjects", request.ObjectIds.Count);
using (ITracer activity = this.tracer.StartActivity(DownloadAreaPath, EventLevel.Informational, metadata))
{
try
{
RetryWrapper <HttpGitObjects.GitObjectTaskResult> .InvocationResult result;
if (request.ObjectIds.Count == 1)
{
result = this.httpGitObjects.TryDownloadLooseObject(
request.ObjectIds[0],
onSuccess: (tryCount, response) => this.WriteObjectOrPackAsync(request, tryCount, response),
onFailure: RetryWrapper <HttpGitObjects.GitObjectTaskResult> .StandardErrorHandler(activity, DownloadAreaPath));
}
else
{
HashSet <string> successfulDownloads = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
result = this.httpGitObjects.TryDownloadObjects(
() => request.ObjectIds.Except(successfulDownloads),
commitDepth: 1,
onSuccess: (tryCount, response) => this.WriteObjectOrPackAsync(request, tryCount, response, successfulDownloads),
onFailure: RetryWrapper <HttpGitObjects.GitObjectTaskResult> .StandardErrorHandler(activity, DownloadAreaPath),
preferBatchedLooseObjects: true);
}
if (!result.Succeeded)
{
this.HasFailures = true;
}
metadata.Add("Success", result.Succeeded);
metadata.Add("AttemptNumber", result.Attempts);
metadata["ActiveDownloads"] = this.activeDownloadCount - 1;
activity.Stop(metadata);
}
finally
{
Interlocked.Decrement(ref this.activeDownloadCount);
}
}
}
}
public void WillNotRetryForGenericExceptions()
{
const int MaxTries = 5;
RetryWrapper <bool> dut = new RetryWrapper <bool>(MaxTries, CancellationToken.None, exponentialBackoffBase: 0);
Assert.Throws <Exception>(
() =>
{
RetryWrapper <bool> .InvocationResult output = dut.Invoke(tryCount => { throw new Exception(); });
});
}
private void HandleDownloadAndSaveObjectError(RetryWrapper <HttpGitObjects.GitObjectTaskResult> .ErrorEventArgs errorArgs)
{
// Silence logging 404's for object downloads. They are far more likely to be git checking for the
// previous existence of a new object than a truly missing object.
HttpGitObjects.HttpGitObjectsException ex = errorArgs.Error as HttpGitObjects.HttpGitObjectsException;
if (ex != null && ex.StatusCode == HttpStatusCode.NotFound)
{
return;
}
RetryWrapper <HttpGitObjects.GitObjectTaskResult> .StandardErrorHandler(this.Tracer, nameof(this.TryDownloadAndSaveObject))(errorArgs);
}
public void WillNotRetryForGenericExceptions()
{
const int MaxTries = 5;
RetryWrapper <bool> dut = new RetryWrapper <bool>(MaxTries, exponentialBackoffBase: 0);
Assert.Throws <AggregateException>(
() =>
{
RetryWrapper <bool> .InvocationResult output = dut.InvokeAsync(tryCount => { throw new Exception(); }).Result;
});
}
public GVFSConfig QueryGVFSConfig()
{
Uri gvfsConfigEndpoint;
string gvfsConfigEndpointString = this.repoUrl + GVFSConstants.Endpoints.GVFSConfig;
try
{
gvfsConfigEndpoint = new Uri(gvfsConfigEndpointString);
}
catch (UriFormatException e)
{
EventMetadata metadata = new EventMetadata();
metadata.Add("Method", nameof(this.QueryGVFSConfig));
metadata.Add("ErrorMessage", e);
metadata.Add("Url", gvfsConfigEndpointString);
this.Tracer.RelatedError(metadata, Keywords.Network);
return(null);
}
long requestId = HttpRequestor.GetNewRequestId();
RetryWrapper <GVFSConfig> retrier = new RetryWrapper <GVFSConfig>(this.RetryConfig.MaxAttempts);
retrier.OnFailure += RetryWrapper <GVFSConfig> .StandardErrorHandler(this.Tracer, requestId, "QueryGvfsConfig");
RetryWrapper <GVFSConfig> .InvocationResult output = retrier.Invoke(
tryCount =>
{
GitEndPointResponseData response = this.SendRequest(requestId, gvfsConfigEndpoint, HttpMethod.Get, null);
if (response.HasErrors)
{
return(new RetryWrapper <GVFSConfig> .CallbackResult(response.Error, response.ShouldRetry));
}
try
{
using (StreamReader reader = new StreamReader(response.Stream))
{
string configString = reader.RetryableReadToEnd();
GVFSConfig config = JsonConvert.DeserializeObject <GVFSConfig>(configString);
return(new RetryWrapper <GVFSConfig> .CallbackResult(config));
}
}
catch (JsonReaderException e)
{
return(new RetryWrapper <GVFSConfig> .CallbackResult(e, false));
}
});
return(output.Result);
}
public GVFSConfigResponse QueryGVFSConfig()
{
Uri gvfsConfigEndpoint;
string gvfsConfigEndpointString = this.enlistment.RepoUrl + GVFSConstants.GVFSConfigEndpointSuffix;
try
{
gvfsConfigEndpoint = new Uri(gvfsConfigEndpointString);
}
catch (UriFormatException e)
{
EventMetadata metadata = new EventMetadata();
metadata.Add("Method", nameof(this.QueryGVFSConfig));
metadata.Add("ErrorMessage", e);
metadata.Add("Url", gvfsConfigEndpointString);
this.tracer.RelatedError(metadata, Keywords.Network);
return(null);
}
RetryWrapper <GVFSConfigResponse> retrier = new RetryWrapper <GVFSConfigResponse>(this.MaxRetries);
retrier.OnFailure += RetryWrapper <GVFSConfigResponse> .StandardErrorHandler(this.tracer, "QueryGvfsConfig");
RetryWrapper <GVFSConfigResponse> .InvocationResult output = retrier.Invoke(
tryCount =>
{
GitEndPointResponseData response = this.SendRequest(gvfsConfigEndpoint, HttpMethod.Get, null);
if (response.HasErrors)
{
return(new RetryWrapper <GVFSConfigResponse> .CallbackResult(response.Error, response.ShouldRetry));
}
using (Stream responseStream = response.Stream)
using (StreamReader reader = new StreamReader(responseStream))
{
try
{
return(new RetryWrapper <GVFSConfigResponse> .CallbackResult(
JsonConvert.DeserializeObject <GVFSConfigResponse>(reader.ReadToEnd())));
}
catch (JsonReaderException e)
{
return(new RetryWrapper <GVFSConfigResponse> .CallbackResult(e, false));
}
}
});
return(output.Result);
}
private void HandleDownloadAndSaveObjectError(bool retryOnFailure, long requestId, RetryWrapper <GitObjectsHttpRequestor.GitObjectTaskResult> .ErrorEventArgs errorArgs)
{
// Silence logging 404's for object downloads. They are far more likely to be git checking for the
// previous existence of a new object than a truly missing object.
GitObjectsHttpException ex = errorArgs.Error as GitObjectsHttpException;
if (ex != null && ex.StatusCode == HttpStatusCode.NotFound)
{
return;
}
// If the caller has requested that we not retry on failure, caller must handle logging errors
bool forceLogAsWarning = !retryOnFailure;
RetryWrapper <GitObjectsHttpRequestor.GitObjectTaskResult> .StandardErrorHandler(this.Tracer, requestId, nameof(this.TryDownloadLooseObject), forceLogAsWarning)(errorArgs);
}
public virtual List <GitObjectSize> QueryForFileSizes(IEnumerable <string> objectIds)
{
long requestId = HttpRequestor.GetNewRequestId();
string objectIdsJson = ToJsonList(objectIds);
Uri gvfsEndpoint = new Uri(this.enlistment.RepoUrl + GVFSConstants.Endpoints.GVFSSizes);
EventMetadata metadata = new EventMetadata();
metadata.Add("RequestId", requestId);
int objectIdCount = objectIds.Count();
if (objectIdCount > 10)
{
metadata.Add("ObjectIdCount", objectIdCount);
}
else
{
metadata.Add("ObjectIdJson", objectIdsJson);
}
this.Tracer.RelatedEvent(EventLevel.Informational, "QueryFileSizes", metadata, Keywords.Network);
RetryWrapper <List <GitObjectSize> > retrier = new RetryWrapper <List <GitObjectSize> >(this.RetryConfig.MaxAttempts);
retrier.OnFailure += RetryWrapper <List <GitObjectSize> > .StandardErrorHandler(this.Tracer, requestId, "QueryFileSizes");
RetryWrapper <List <GitObjectSize> > .InvocationResult requestTask = retrier.Invoke(
tryCount =>
{
GitEndPointResponseData response = this.SendRequest(requestId, gvfsEndpoint, HttpMethod.Post, objectIdsJson);
if (response.HasErrors)
{
return(new RetryWrapper <List <GitObjectSize> > .CallbackResult(response.Error, response.ShouldRetry));
}
using (StreamReader reader = new StreamReader(response.Stream))
{
string objectSizesString = reader.RetryableReadToEnd();
List <GitObjectSize> objectSizes = JsonConvert.DeserializeObject <List <GitObjectSize> >(objectSizesString);
return(new RetryWrapper <List <GitObjectSize> > .CallbackResult(objectSizes));
}
});
return(requestTask.Result ?? new List <GitObjectSize>(0));
}
public bool TryDownloadPrefetchPacks(long latestTimestamp)
{
EventMetadata metadata = new EventMetadata();
metadata.Add("latestTimestamp", latestTimestamp);
using (ITracer activity = this.Tracer.StartActivity(nameof(this.TryDownloadPrefetchPacks), EventLevel.Informational, metadata))
{
long requestId = HttpRequestor.GetNewRequestId();
RetryWrapper <GitObjectsHttpRequestor.GitObjectTaskResult> .InvocationResult result = this.GitObjectRequestor.TrySendProtocolRequest(
requestId: requestId,
onSuccess: (tryCount, response) => this.DeserializePrefetchPacks(response, ref latestTimestamp),
onFailure: RetryWrapper <GitObjectsHttpRequestor.GitObjectTaskResult> .StandardErrorHandler(activity, requestId, nameof(this.TryDownloadPrefetchPacks)),
method: HttpMethod.Get,
endPointGenerator: () => new Uri(
string.Format(
"{0}?lastPackTimestamp={1}",
this.GitObjectRequestor.CacheServer.PrefetchEndpointUrl,
latestTimestamp)),
requestBodyGenerator: () => null,
acceptType: new MediaTypeWithQualityHeaderValue(GVFSConstants.MediaTypes.PrefetchPackFilesAndIndexesMediaType));
if (!result.Succeeded)
{
if (result.Result != null && result.Result.HttpStatusCodeResult == HttpStatusCode.NotFound)
{
EventMetadata warning = new EventMetadata();
warning.Add("ErrorMessage", "The server does not support " + GVFSConstants.Endpoints.GVFSPrefetch);
warning.Add(nameof(this.GitObjectRequestor.CacheServer.PrefetchEndpointUrl), this.GitObjectRequestor.CacheServer.PrefetchEndpointUrl);
activity.RelatedEvent(EventLevel.Warning, "CommandNotSupported", warning);
}
else
{
EventMetadata error = new EventMetadata();
error.Add("latestTimestamp", latestTimestamp);
error.Add("Exception", result.Error);
error.Add("ErrorMessage", "DownloadPrefetchPacks failed.");
error.Add(nameof(this.GitObjectRequestor.CacheServer.PrefetchEndpointUrl), this.GitObjectRequestor.CacheServer.PrefetchEndpointUrl);
activity.RelatedError(error);
}
}
return(result.Succeeded);
}
}
private bool TryDownloadAndSaveObjects(IEnumerable <string> objectIds, int commitDepth, bool preferLooseObjects)
{
RetryWrapper <HttpGitObjects.GitObjectTaskResult> .InvocationResult output = this.GitObjectRequestor.TryDownloadObjects(
objectIds,
commitDepth,
onSuccess: (tryCount, response) => this.TrySavePackOrLooseObject(objectIds, preferLooseObjects, response),
onFailure: (eArgs) =>
{
EventMetadata metadata = new EventMetadata();
metadata.Add("Operation", "DownloadAndSaveObjects");
metadata.Add("WillRetry", eArgs.WillRetry);
metadata.Add("ErrorMessage", eArgs.Error.ToString());
this.Tracer.RelatedError(metadata, Keywords.Network);
},
preferBatchedLooseObjects: preferLooseObjects);
return(output.Succeeded && output.Result.Success);
}
public void WillRetryOnIOException()
{
const int ExpectedTries = 5;
RetryWrapper <bool> dut = new RetryWrapper <bool>(ExpectedTries, exponentialBackoffBase: 0);
int actualTries = 0;
RetryWrapper <bool> .InvocationResult output = dut.InvokeAsync(
tryCount =>
{
actualTries++;
throw new IOException();
}).Result;
output.Succeeded.ShouldEqual(false);
actualTries.ShouldEqual(ExpectedTries);
}
public virtual GitRefs QueryInfoRefs(string branch)
{
long requestId = HttpRequestor.GetNewRequestId();
Uri infoRefsEndpoint;
try
{
infoRefsEndpoint = new Uri(this.enlistment.RepoUrl + GVFSConstants.Endpoints.InfoRefs);
}
catch (UriFormatException)
{
return(null);
}
CancellationToken neverCanceledToken = new CancellationToken(canceled: false);
RetryWrapper <GitRefs> retrier = new RetryWrapper <GitRefs>(this.RetryConfig.MaxAttempts, neverCanceledToken);
retrier.OnFailure += RetryWrapper <GitRefs> .StandardErrorHandler(this.Tracer, requestId, "QueryInfoRefs");
RetryWrapper <GitRefs> .InvocationResult output = retrier.Invoke(
tryCount =>
{
GitEndPointResponseData response = this.SendRequest(
requestId,
infoRefsEndpoint,
HttpMethod.Get,
requestContent: null,
cancellationToken: neverCanceledToken);
if (response.HasErrors)
{
return(new RetryWrapper <GitRefs> .CallbackResult(response.Error, response.ShouldRetry));
}
using (StreamReader reader = new StreamReader(response.Stream))
{
List <string> infoRefsResponse = reader.RetryableReadAllLines();
return(new RetryWrapper <GitRefs> .CallbackResult(new GitRefs(infoRefsResponse, branch)));
}
});
return(output.Result);
}
public void WillNotMakeAnyAttemptWhenInitiallyCanceled()
{
const int MaxTries = 5;
int actualTries = 0;
RetryWrapper <bool> dut = new RetryWrapper <bool>(MaxTries, new CancellationToken(canceled: true), exponentialBackoffBase: 0);
Assert.Throws <OperationCanceledException>(
() =>
{
RetryWrapper <bool> .InvocationResult output = dut.Invoke(tryCount =>
{
++actualTries;
return(new RetryWrapper <bool> .CallbackResult(true));
});
});
actualTries.ShouldEqual(0);
}
public bool TryCopyBlobContentStream(string sha, Action <Stream, long> writeAction)
{
RetryWrapper <bool> retrier = new RetryWrapper <bool>(this.GitObjectRequestor.RetryConfig.MaxAttempts);
retrier.OnFailure +=
errorArgs =>
{
EventMetadata metadata = new EventMetadata();
metadata.Add("sha", sha);
metadata.Add("AttemptNumber", errorArgs.TryCount);
metadata.Add("WillRetry", errorArgs.WillRetry);
metadata.Add("ErrorMessage", "TryCopyBlobContentStream: Failed to provide blob contents");
this.Tracer.RelatedError(metadata);
};
string firstTwoShaDigits = sha.Substring(0, 2);
string remainingShaDigits = sha.Substring(2);
RetryWrapper <bool> .InvocationResult invokeResult = retrier.Invoke(
tryCount =>
{
bool success = this.Context.Repository.TryCopyBlobContentStream(sha, writeAction);
if (success)
{
return(new RetryWrapper <bool> .CallbackResult(true));
}
else
{
// Pass in 1 for maxAttempts because the retrier in this method manages multiple attempts
if (this.TryDownloadAndSaveObject(firstTwoShaDigits, remainingShaDigits, maxAttempts: 1))
{
if (this.Context.Repository.TryCopyBlobContentStream(sha, writeAction))
{
return(new RetryWrapper <bool> .CallbackResult(true));
}
}
return(new RetryWrapper <bool> .CallbackResult(error: null, shouldRetry: true));
}
});
return(invokeResult.Result);
}
public void OnFailureIsCalledWhenEventHandlerAttached()
{
const int MaxTries = 5;
const int ExpectedFailures = 5;
RetryWrapper <bool> dut = new RetryWrapper <bool>(MaxTries, exponentialBackoffBase: 0);
int actualFailures = 0;
dut.OnFailure += errorArgs => actualFailures++;
RetryWrapper <bool> .InvocationResult output = dut.InvokeAsync(
tryCount =>
{
throw new IOException();
}).Result;
output.Succeeded.ShouldEqual(false);
actualFailures.ShouldEqual(ExpectedFailures);
}
/// <summary>
/// Get the <see cref="Uri"/>s to download and store in the pack directory for bootstrapping
/// </summary>
public IList <Uri> TryGetBootstrapPackSources(Uri bootstrapSource, string branchName)
{
IList <Uri> packUris = null;
RetryWrapper <GitObjectTaskResult> .InvocationResult output = this.TrySendProtocolRequest(
onSuccess: (tryCount, response) =>
{
using (StreamReader reader = new StreamReader(response.Stream))
{
string packUriString = reader.RetryableReadToEnd();
packUris = JsonConvert.DeserializeObject <BootstrapResponse>(packUriString).PackUris;
return(new RetryWrapper <GitObjectTaskResult> .CallbackResult(new GitObjectTaskResult(true)));
}
},
onFailure: RetryWrapper <GitObjectTaskResult> .StandardErrorHandler(this.tracer, nameof(this.TryGetBootstrapPackSources)),
method: HttpMethod.Post,
endPoint: bootstrapSource,
requestBody: JsonConvert.SerializeObject(new { branchName = branchName }));
return(packUris);
}
