public static BlobIdentifierWithBlocks Deserialize(string serialized) { // Marked "new" string[] tokens = serialized.Split(':'); return(new BlobIdentifierWithBlocks( BlobIdentifier.Deserialize(tokens[0]), tokens[1].Split(SplitCharacters, StringSplitOptions.RemoveEmptyEntries).Select(idString => new BlobBlockHash(idString)).ToList())); }
/// <summary> /// Bulk adds download URIs to the cache. /// </summary> public void BulkAddDownloadUris(IDictionary <string, Uri> blobDownloadUris) { if (blobDownloadUris == null) { return; } foreach (var blobDownloadUri in blobDownloadUris) { AddDownloadUri( BlobIdentifier.Deserialize(blobDownloadUri.Key).ToContentHash(), new PreauthenticatedUri(blobDownloadUri.Value, EdgeType.Unknown)); // EdgeType value shouldn't matter because we don't use it. } }
/// <summary> /// Converts a ContentStore ContentHash to a BlobIdentifier. /// </summary> public static BlobIdentifier ToBlobIdentifier(this ContentHash contentHash) { switch (contentHash.HashType) { case HashType.Vso0: case HashType.Dedup64K: case HashType.Dedup1024K: case HashType.Murmur: return(BlobIdentifier.Deserialize(contentHash.ToHex())); default: throw new ArgumentException($"ContentHash has unsupported type when converting to BlobIdentifier: {contentHash.HashType}"); } }
private async Task <ObjectResult <ContentHashListWithCacheMetadata> > UnpackBlobContentHashListAsync(Context context, BlobContentHashListWithCacheMetadata blobCacheMetadata) { Contract.Assert(blobCacheMetadata != null); if (blobCacheMetadata.ContentHashListWithDeterminism.BlobIdentifier == null) { return(new ObjectResult <ContentHashListWithCacheMetadata>( new ContentHashListWithCacheMetadata( new ContentHashListWithDeterminism(null, blobCacheMetadata.Determinism), blobCacheMetadata.GetRawExpirationTimeUtc(), blobCacheMetadata.ContentGuarantee, blobCacheMetadata.HashOfExistingContentHashList))); } BlobIdentifier blobId = blobCacheMetadata.ContentHashListWithDeterminism.BlobIdentifier; Func <ContentHash, CancellationToken, Task <ObjectResult <Stream> > > openStreamFunc = async(hash, cts) => { OpenStreamResult openStreamResult = await _blobContentSession.OpenStreamAsync(context, hash, cts); if (openStreamResult.Succeeded) { return(new ObjectResult <Stream>(openStreamResult.Stream)); } return(new ObjectResult <Stream>(openStreamResult)); }; StructResult <ContentHashListWithDeterminism> contentHashListResult = await BlobContentHashListExtensions.UnpackFromBlob( openStreamFunc, blobId); if (contentHashListResult.Succeeded) { var contentHashListWithCacheMetadata = new ContentHashListWithCacheMetadata( contentHashListResult.Data, blobCacheMetadata.GetRawExpirationTimeUtc(), blobCacheMetadata.ContentGuarantee, blobCacheMetadata.HashOfExistingContentHashList); return(new ObjectResult <ContentHashListWithCacheMetadata>(contentHashListWithCacheMetadata)); } else { return(new ObjectResult <ContentHashListWithCacheMetadata>(contentHashListResult)); } }
private async Task <Stream> GetStreamInternalAsync(Context context, ContentHash contentHash, int?overrideStreamMinimumReadSizeInBytes, CancellationToken cts) { if (_downloadBlobsThroughBlobStore) { return(await ArtifactHttpClientErrorDetectionStrategy.ExecuteWithTimeoutAsync( context, "GetStreamInternalThroughBlobStore", innerCts => BlobStoreHttpClient.GetBlobAsync(ToVstsBlobIdentifier(contentHash.ToBlobIdentifier()), cancellationToken: innerCts), cts).ConfigureAwait(false)); } else { PreauthenticatedUri uri; if (!DownloadUriCache.Instance.TryGetDownloadUri(contentHash, out uri)) { Tracer.RecordDownloadUriFetchedFromRemote(); BlobIdentifier blobId = contentHash.ToBlobIdentifier(); IDictionary <VstsBlobIdentifier, PreauthenticatedUri> mappings = await ArtifactHttpClientErrorDetectionStrategy.ExecuteWithTimeoutAsync( context, "GetStreamInternal", innerCts => BlobStoreHttpClient.GetDownloadUrisAsync( new[] { ToVstsBlobIdentifier(blobId) }, EdgeCache.NotAllowed, cancellationToken: innerCts), cts).ConfigureAwait(false); if (mappings == null || !mappings.TryGetValue(ToVstsBlobIdentifier(blobId), out uri)) { return(null); } DownloadUriCache.Instance.AddDownloadUri(contentHash, uri); } else { Tracer.RecordDownloadUriFetchedFromCache(); } return(await GetStreamThroughAzureBlobs( uri.NotNullUri, overrideStreamMinimumReadSizeInBytes, _parallelSegmentDownloadConfig.SegmentDownloadTimeout, cts).ConfigureAwait(false)); } }
private void Validate() { if (BlobId == null) { throw new ArgumentNullException(nameof(BlobId)); } if (BlockHashes == null) { throw new ArgumentNullException(nameof(BlockHashes)); } BlobIdentifier computedBlobId = ComputeIdentifierBasedOnBlocks(BlockHashes); if (!BlobId.Equals(computedBlobId)) { throw new InvalidDataException(string.Format( CultureInfo.InvariantCulture, "Computed id '{0}' does not match given one '{1}'.", computedBlobId, BlobId)); } }
/// <summary> /// Converts a BlobIdentifier to its corresponding ContentHash. /// </summary> public static ContentHash ToContentHash(this BlobIdentifier blobId) { switch (blobId.AlgorithmId) { case VsoHash.VsoAlgorithmId: return(new ContentHash(HashType.Vso0, blobId.Bytes)); case ChunkDedupIdentifier.ChunkAlgorithmId: return(new ContentHash(HashType.Dedup64K, blobId.Bytes)); // TODO: Chunk size optimization case (byte)NodeAlgorithmId.Node64K: return(new ContentHash(HashType.Dedup64K, blobId.Bytes)); case (byte)NodeAlgorithmId.Node1024K: return(new ContentHash(HashType.Dedup1024K, blobId.Bytes)); case MurmurHashInfo.MurmurAlgorithmId: return(new ContentHash(HashType.Murmur, blobId.Bytes)); default: throw new ArgumentException($"BlobIdentifier has an unrecognized AlgorithmId: {blobId.AlgorithmId}"); } }
public BlobIdentifierWithBlocks(BlobIdentifier blobId, IEnumerable <BlobBlockHash> blockIdentifiers) { BlockHashes = blockIdentifiers.ToList(); BlobId = blobId; Validate(); }
/// <summary> /// Converts a ContentStore BlobId to an Artifact BlobId /// </summary> protected static VstsBlobIdentifier ToVstsBlobIdentifier(BlobIdentifier blobIdentifier) { return(new VstsBlobIdentifier(blobIdentifier.Bytes)); }
/// <summary> /// Converts a ContentStore BlobId to an Artifact BlobId /// </summary> protected static VstsBlobIdentifier ToVstsBlobIdentifier(BlobIdentifier blobIdentifier) => new VstsBlobIdentifier(blobIdentifier.Bytes);