public static BlobIdentifierWithBlocks Deserialize(string serialized)
{
// Marked "new"
string[] tokens = serialized.Split(':');
return(new BlobIdentifierWithBlocks(
BlobIdentifier.Deserialize(tokens[0]),
tokens[1].Split(SplitCharacters, StringSplitOptions.RemoveEmptyEntries).Select(idString => new BlobBlockHash(idString)).ToList()));
}
/// <summary>
/// Bulk adds download URIs to the cache.
/// </summary>
public void BulkAddDownloadUris(IDictionary <string, Uri> blobDownloadUris)
{
if (blobDownloadUris == null)
{
return;
}
foreach (var blobDownloadUri in blobDownloadUris)
{
AddDownloadUri(
BlobIdentifier.Deserialize(blobDownloadUri.Key).ToContentHash(),
new PreauthenticatedUri(blobDownloadUri.Value, EdgeType.Unknown)); // EdgeType value shouldn't matter because we don't use it.
}
}
/// <summary>
/// Converts a ContentStore ContentHash to a BlobIdentifier.
/// </summary>
public static BlobIdentifier ToBlobIdentifier(this ContentHash contentHash)
{
switch (contentHash.HashType)
{
case HashType.Vso0:
case HashType.Dedup64K:
case HashType.Dedup1024K:
case HashType.Murmur:
return(BlobIdentifier.Deserialize(contentHash.ToHex()));
default:
throw new ArgumentException($"ContentHash has unsupported type when converting to BlobIdentifier: {contentHash.HashType}");
}
}
private async Task <ObjectResult <ContentHashListWithCacheMetadata> > UnpackBlobContentHashListAsync(Context context, BlobContentHashListWithCacheMetadata blobCacheMetadata)
{
Contract.Assert(blobCacheMetadata != null);
if (blobCacheMetadata.ContentHashListWithDeterminism.BlobIdentifier == null)
{
return(new ObjectResult <ContentHashListWithCacheMetadata>(
new ContentHashListWithCacheMetadata(
new ContentHashListWithDeterminism(null, blobCacheMetadata.Determinism),
blobCacheMetadata.GetRawExpirationTimeUtc(),
blobCacheMetadata.ContentGuarantee,
blobCacheMetadata.HashOfExistingContentHashList)));
}
BlobIdentifier blobId = blobCacheMetadata.ContentHashListWithDeterminism.BlobIdentifier;
Func <ContentHash, CancellationToken, Task <ObjectResult <Stream> > > openStreamFunc = async(hash, cts) =>
{
OpenStreamResult openStreamResult = await _blobContentSession.OpenStreamAsync(context, hash, cts);
if (openStreamResult.Succeeded)
{
return(new ObjectResult <Stream>(openStreamResult.Stream));
}
return(new ObjectResult <Stream>(openStreamResult));
};
StructResult <ContentHashListWithDeterminism> contentHashListResult =
await BlobContentHashListExtensions.UnpackFromBlob(
openStreamFunc,
blobId);
if (contentHashListResult.Succeeded)
{
var contentHashListWithCacheMetadata = new ContentHashListWithCacheMetadata(
contentHashListResult.Data,
blobCacheMetadata.GetRawExpirationTimeUtc(),
blobCacheMetadata.ContentGuarantee,
blobCacheMetadata.HashOfExistingContentHashList);
return(new ObjectResult <ContentHashListWithCacheMetadata>(contentHashListWithCacheMetadata));
}
else
{
return(new ObjectResult <ContentHashListWithCacheMetadata>(contentHashListResult));
}
}
private async Task <Stream> GetStreamInternalAsync(Context context, ContentHash contentHash, int?overrideStreamMinimumReadSizeInBytes, CancellationToken cts)
{
if (_downloadBlobsThroughBlobStore)
{
return(await ArtifactHttpClientErrorDetectionStrategy.ExecuteWithTimeoutAsync(
context,
"GetStreamInternalThroughBlobStore",
innerCts => BlobStoreHttpClient.GetBlobAsync(ToVstsBlobIdentifier(contentHash.ToBlobIdentifier()), cancellationToken: innerCts),
cts).ConfigureAwait(false));
}
else
{
PreauthenticatedUri uri;
if (!DownloadUriCache.Instance.TryGetDownloadUri(contentHash, out uri))
{
Tracer.RecordDownloadUriFetchedFromRemote();
BlobIdentifier blobId = contentHash.ToBlobIdentifier();
IDictionary <VstsBlobIdentifier, PreauthenticatedUri> mappings = await ArtifactHttpClientErrorDetectionStrategy.ExecuteWithTimeoutAsync(
context,
"GetStreamInternal",
innerCts => BlobStoreHttpClient.GetDownloadUrisAsync(
new[] { ToVstsBlobIdentifier(blobId) },
EdgeCache.NotAllowed,
cancellationToken: innerCts),
cts).ConfigureAwait(false);
if (mappings == null || !mappings.TryGetValue(ToVstsBlobIdentifier(blobId), out uri))
{
return(null);
}
DownloadUriCache.Instance.AddDownloadUri(contentHash, uri);
}
else
{
Tracer.RecordDownloadUriFetchedFromCache();
}
return(await GetStreamThroughAzureBlobs(
uri.NotNullUri,
overrideStreamMinimumReadSizeInBytes,
_parallelSegmentDownloadConfig.SegmentDownloadTimeout,
cts).ConfigureAwait(false));
}
}
private void Validate()
{
if (BlobId == null)
{
throw new ArgumentNullException(nameof(BlobId));
}
if (BlockHashes == null)
{
throw new ArgumentNullException(nameof(BlockHashes));
}
BlobIdentifier computedBlobId = ComputeIdentifierBasedOnBlocks(BlockHashes);
if (!BlobId.Equals(computedBlobId))
{
throw new InvalidDataException(string.Format(
CultureInfo.InvariantCulture,
"Computed id '{0}' does not match given one '{1}'.",
computedBlobId,
BlobId));
}
}
/// <summary>
/// Converts a BlobIdentifier to its corresponding ContentHash.
/// </summary>
public static ContentHash ToContentHash(this BlobIdentifier blobId)
{
switch (blobId.AlgorithmId)
{
case VsoHash.VsoAlgorithmId:
return(new ContentHash(HashType.Vso0, blobId.Bytes));
case ChunkDedupIdentifier.ChunkAlgorithmId:
return(new ContentHash(HashType.Dedup64K, blobId.Bytes)); // TODO: Chunk size optimization
case (byte)NodeAlgorithmId.Node64K:
return(new ContentHash(HashType.Dedup64K, blobId.Bytes));
case (byte)NodeAlgorithmId.Node1024K:
return(new ContentHash(HashType.Dedup1024K, blobId.Bytes));
case MurmurHashInfo.MurmurAlgorithmId:
return(new ContentHash(HashType.Murmur, blobId.Bytes));
default:
throw new ArgumentException($"BlobIdentifier has an unrecognized AlgorithmId: {blobId.AlgorithmId}");
}
}
public BlobIdentifierWithBlocks(BlobIdentifier blobId, IEnumerable <BlobBlockHash> blockIdentifiers)
{
BlockHashes = blockIdentifiers.ToList();
BlobId = blobId;
Validate();
}
/// <summary>
/// Converts a ContentStore BlobId to an Artifact BlobId
/// </summary>
protected static VstsBlobIdentifier ToVstsBlobIdentifier(BlobIdentifier blobIdentifier)
{
return(new VstsBlobIdentifier(blobIdentifier.Bytes));
}
/// <summary> /// Converts a ContentStore BlobId to an Artifact BlobId /// </summary> protected static VstsBlobIdentifier ToVstsBlobIdentifier(BlobIdentifier blobIdentifier) => new VstsBlobIdentifier(blobIdentifier.Bytes);