/// <summary>
/// Uses a <see cref="PrefetchPacksDeserializer"/> to read the packs from the stream.
/// </summary>
private RetryWrapper <GitObjectsHttpRequestor.GitObjectTaskResult> .CallbackResult DeserializePrefetchPacks(
GitEndPointResponseData response,
ref long latestTimestamp,
ref long bytesDownloaded,
ref List <string> packIndexes,
GitProcess gitProcess)
{
if (packIndexes == null)
{
packIndexes = new List <string>();
}
using (ITracer activity = this.Tracer.StartActivity("DeserializePrefetchPacks", EventLevel.Informational))
{
PrefetchPacksDeserializer deserializer = new PrefetchPacksDeserializer(response.Stream);
string tempPackFolderPath = Path.Combine(this.Enlistment.GitPackRoot, TempPackFolder);
this.fileSystem.CreateDirectory(tempPackFolderPath);
List <TempPrefetchPackAndIdx> tempPacks = new List <TempPrefetchPackAndIdx>();
foreach (PrefetchPacksDeserializer.PackAndIndex pack in deserializer.EnumeratePacks())
{
// The advertised size may not match the actual, on-disk size.
long indexLength = 0;
long packLength;
// Write the temp and index to a temp folder to avoid putting corrupt files in the pack folder
// Once the files are validated and flushed they can be moved to the pack folder
string packName = string.Format("{0}-{1}-{2}.pack", GVFSConstants.PrefetchPackPrefix, pack.Timestamp, pack.UniqueId);
string packTempPath = Path.Combine(tempPackFolderPath, packName);
string idxName = string.Format("{0}-{1}-{2}.idx", GVFSConstants.PrefetchPackPrefix, pack.Timestamp, pack.UniqueId);
string idxTempPath = Path.Combine(tempPackFolderPath, idxName);
EventMetadata data = CreateEventMetadata();
data["timestamp"] = pack.Timestamp.ToString();
data["uniqueId"] = pack.UniqueId;
activity.RelatedEvent(EventLevel.Informational, "Receiving Pack/Index", data);
// Write the pack
// If it fails, TryWriteTempFile cleans up the file and we retry the prefetch
Task packFlushTask;
if (!this.TryWriteTempFile(activity, pack.PackStream, packTempPath, out packLength, out packFlushTask))
{
bytesDownloaded += packLength;
return(new RetryWrapper <GitObjectsHttpRequestor.GitObjectTaskResult> .CallbackResult(null, true));
}
bytesDownloaded += packLength;
// We will try to build an index if the server does not send one
if (pack.IndexStream == null)
{
GitProcess.Result result;
if (!this.TryBuildIndex(activity, packTempPath, out result, gitProcess))
{
if (packFlushTask != null)
{
packFlushTask.Wait();
}
// Move whatever has been successfully downloaded so far
Exception moveException;
this.TryFlushAndMoveTempPacks(tempPacks, ref latestTimestamp, out moveException);
return(new RetryWrapper <GitObjectsHttpRequestor.GitObjectTaskResult> .CallbackResult(null, true));
}
tempPacks.Add(new TempPrefetchPackAndIdx(pack.Timestamp, packName, packTempPath, packFlushTask, idxName, idxTempPath, idxFlushTask: null));
}
else
{
Task indexFlushTask;
if (this.TryWriteTempFile(activity, pack.IndexStream, idxTempPath, out indexLength, out indexFlushTask))
{
tempPacks.Add(new TempPrefetchPackAndIdx(pack.Timestamp, packName, packTempPath, packFlushTask, idxName, idxTempPath, indexFlushTask));
}
else
{
bytesDownloaded += indexLength;
// Try to build the index manually, then retry the prefetch
GitProcess.Result result;
if (this.TryBuildIndex(activity, packTempPath, out result, gitProcess))
{
// If we were able to recreate the failed index
// we can start the prefetch at the next timestamp
tempPacks.Add(new TempPrefetchPackAndIdx(pack.Timestamp, packName, packTempPath, packFlushTask, idxName, idxTempPath, idxFlushTask: null));
}
else
{
if (packFlushTask != null)
{
packFlushTask.Wait();
}
}
// Move whatever has been successfully downloaded so far
Exception moveException;
this.TryFlushAndMoveTempPacks(tempPacks, ref latestTimestamp, out moveException);
// The download stream will not be in a good state if the index download fails.
// So we have to restart the prefetch
return(new RetryWrapper <GitObjectsHttpRequestor.GitObjectTaskResult> .CallbackResult(null, true));
}
}
bytesDownloaded += indexLength;
}
Exception exception = null;
if (!this.TryFlushAndMoveTempPacks(tempPacks, ref latestTimestamp, out exception))
{
return(new RetryWrapper <GitObjectsHttpRequestor.GitObjectTaskResult> .CallbackResult(exception, true));
}
foreach (TempPrefetchPackAndIdx tempPack in tempPacks)
{
packIndexes.Add(tempPack.IdxName);
}
return(new RetryWrapper <GitObjectsHttpRequestor.GitObjectTaskResult> .CallbackResult(
new GitObjectsHttpRequestor.GitObjectTaskResult(success: true)));
}
}
private RetryWrapper <GitObjectsHttpRequestor.GitObjectTaskResult> .CallbackResult TrySavePackOrLooseObject(IEnumerable <string> objectShas, bool unpackObjects, GitEndPointResponseData responseData)
{
if (responseData.ContentType == GitObjectContentType.LooseObject)
{
List <string> objectShaList = objectShas.Distinct().ToList();
if (objectShaList.Count != 1)
{
return(new RetryWrapper <GitObjectsHttpRequestor.GitObjectTaskResult> .CallbackResult(new InvalidOperationException("Received loose object when multiple objects were requested."), shouldRetry : false));
}
// To reduce allocations, reuse the same buffer when writing objects in this batch
byte[] bufToCopyWith = new byte[StreamUtil.DefaultCopyBufferSize];
this.WriteLooseObject(this.Enlistment.WorkingDirectoryRoot, responseData.Stream, objectShaList[0], bufToCopyWith);
}
else if (responseData.ContentType == GitObjectContentType.BatchedLooseObjects)
{
// To reduce allocations, reuse the same buffer when writing objects in this batch
byte[] bufToCopyWith = new byte[StreamUtil.DefaultCopyBufferSize];
BatchedLooseObjectDeserializer deserializer = new BatchedLooseObjectDeserializer(
responseData.Stream,
(stream, sha) => this.WriteLooseObject(this.Enlistment.WorkingDirectoryRoot, stream, sha, bufToCopyWith));
deserializer.ProcessObjects();
}
else
{
GitProcess.Result result = this.TryAddPackFile(responseData.Stream, unpackObjects);
if (result.HasErrors)
{
return(new RetryWrapper <GitObjectsHttpRequestor.GitObjectTaskResult> .CallbackResult(new InvalidOperationException("Could not add pack file: " + result.Errors), shouldRetry : false));
}
}
return(new RetryWrapper <GitObjectsHttpRequestor.GitObjectTaskResult> .CallbackResult(new GitObjectsHttpRequestor.GitObjectTaskResult(true)));
}
private RetryWrapper <GitObjectsHttpRequestor.GitObjectTaskResult> .CallbackResult WriteObjectOrPack(
BlobDownloadRequest request,
int tryCount,
GitEndPointResponseData response,
HashSet <string> successfulDownloads = null)
{
// To reduce allocations, reuse the same buffer when writing objects in this batch
byte[] bufToCopyWith = new byte[StreamUtil.DefaultCopyBufferSize];
string fileName = null;
switch (response.ContentType)
{
case GitObjectContentType.LooseObject:
string sha = request.ObjectIds.First();
fileName = this.gitObjects.WriteLooseObject(
response.Stream,
sha,
overwriteExistingObject: false,
bufToCopyWith: bufToCopyWith);
this.AvailableObjects.Add(sha);
break;
case GitObjectContentType.PackFile:
fileName = this.gitObjects.WriteTempPackFile(response.Stream);
this.AvailablePacks.Add(new IndexPackRequest(fileName, request));
break;
case GitObjectContentType.BatchedLooseObjects:
BatchedLooseObjectDeserializer.OnLooseObject onLooseObject = (objectStream, sha1) =>
{
this.gitObjects.WriteLooseObject(
objectStream,
sha1,
overwriteExistingObject: false,
bufToCopyWith: bufToCopyWith);
this.AvailableObjects.Add(sha1);
if (successfulDownloads != null)
{
successfulDownloads.Add(sha1);
}
// This isn't strictly correct because we don't add object header bytes,
// just the actual compressed content length, but we expect the amount of
// header data to be negligible compared to the objects themselves.
Interlocked.Add(ref this.bytesDownloaded, objectStream.Length);
};
new BatchedLooseObjectDeserializer(response.Stream, onLooseObject).ProcessObjects();
break;
}
if (fileName != null)
{
// NOTE: If we are writing a file as part of this method, the only case
// where it's not expected to exist is when running unit tests
FileInfo info = new FileInfo(fileName);
if (info.Exists)
{
Interlocked.Add(ref this.bytesDownloaded, info.Length);
}
else
{
return(new RetryWrapper <GitObjectsHttpRequestor.GitObjectTaskResult> .CallbackResult(
new GitObjectsHttpRequestor.GitObjectTaskResult(false)));
}
}
return(new RetryWrapper <GitObjectsHttpRequestor.GitObjectTaskResult> .CallbackResult(
new GitObjectsHttpRequestor.GitObjectTaskResult(true)));
}
