Exemplo n.º 1
0
 public static BlobIdentifierWithBlocks Deserialize(string serialized)
 {
     // Marked "new"
     string[] tokens = serialized.Split(':');
     return(new BlobIdentifierWithBlocks(
                BlobIdentifier.Deserialize(tokens[0]),
                tokens[1].Split(SplitCharacters, StringSplitOptions.RemoveEmptyEntries).Select(idString => new BlobBlockHash(idString)).ToList()));
 }
Exemplo n.º 2
0
        /// <summary>
        /// Bulk adds download URIs to the cache.
        /// </summary>
        public void BulkAddDownloadUris(IDictionary <string, Uri> blobDownloadUris)
        {
            if (blobDownloadUris == null)
            {
                return;
            }

            foreach (var blobDownloadUri in blobDownloadUris)
            {
                AddDownloadUri(
                    BlobIdentifier.Deserialize(blobDownloadUri.Key).ToContentHash(),
                    new PreauthenticatedUri(blobDownloadUri.Value, EdgeType.Unknown)); // EdgeType value shouldn't matter because we don't use it.
            }
        }
Exemplo n.º 3
0
        /// <summary>
        /// Converts a ContentStore ContentHash to a BlobIdentifier.
        /// </summary>
        public static BlobIdentifier ToBlobIdentifier(this ContentHash contentHash)
        {
            switch (contentHash.HashType)
            {
            case HashType.Vso0:
            case HashType.Dedup64K:
            case HashType.Dedup1024K:
            case HashType.Murmur:
                return(BlobIdentifier.Deserialize(contentHash.ToHex()));

            default:
                throw new ArgumentException($"ContentHash has unsupported type when converting to BlobIdentifier: {contentHash.HashType}");
            }
        }
        private async Task <ObjectResult <ContentHashListWithCacheMetadata> > UnpackBlobContentHashListAsync(Context context, BlobContentHashListWithCacheMetadata blobCacheMetadata)
        {
            Contract.Assert(blobCacheMetadata != null);
            if (blobCacheMetadata.ContentHashListWithDeterminism.BlobIdentifier == null)
            {
                return(new ObjectResult <ContentHashListWithCacheMetadata>(
                           new ContentHashListWithCacheMetadata(
                               new ContentHashListWithDeterminism(null, blobCacheMetadata.Determinism),
                               blobCacheMetadata.GetRawExpirationTimeUtc(),
                               blobCacheMetadata.ContentGuarantee,
                               blobCacheMetadata.HashOfExistingContentHashList)));
            }

            BlobIdentifier blobId = blobCacheMetadata.ContentHashListWithDeterminism.BlobIdentifier;

            Func <ContentHash, CancellationToken, Task <ObjectResult <Stream> > > openStreamFunc = async(hash, cts) =>
            {
                OpenStreamResult openStreamResult = await _blobContentSession.OpenStreamAsync(context, hash, cts);

                if (openStreamResult.Succeeded)
                {
                    return(new ObjectResult <Stream>(openStreamResult.Stream));
                }

                return(new ObjectResult <Stream>(openStreamResult));
            };

            StructResult <ContentHashListWithDeterminism> contentHashListResult =
                await BlobContentHashListExtensions.UnpackFromBlob(
                    openStreamFunc,
                    blobId);

            if (contentHashListResult.Succeeded)
            {
                var contentHashListWithCacheMetadata = new ContentHashListWithCacheMetadata(
                    contentHashListResult.Data,
                    blobCacheMetadata.GetRawExpirationTimeUtc(),
                    blobCacheMetadata.ContentGuarantee,
                    blobCacheMetadata.HashOfExistingContentHashList);
                return(new ObjectResult <ContentHashListWithCacheMetadata>(contentHashListWithCacheMetadata));
            }
            else
            {
                return(new ObjectResult <ContentHashListWithCacheMetadata>(contentHashListResult));
            }
        }
Exemplo n.º 5
0
        private async Task <Stream> GetStreamInternalAsync(Context context, ContentHash contentHash, int?overrideStreamMinimumReadSizeInBytes, CancellationToken cts)
        {
            if (_downloadBlobsThroughBlobStore)
            {
                return(await ArtifactHttpClientErrorDetectionStrategy.ExecuteWithTimeoutAsync(
                           context,
                           "GetStreamInternalThroughBlobStore",
                           innerCts => BlobStoreHttpClient.GetBlobAsync(ToVstsBlobIdentifier(contentHash.ToBlobIdentifier()), cancellationToken: innerCts),
                           cts).ConfigureAwait(false));
            }
            else
            {
                PreauthenticatedUri uri;
                if (!DownloadUriCache.Instance.TryGetDownloadUri(contentHash, out uri))
                {
                    Tracer.RecordDownloadUriFetchedFromRemote();
                    BlobIdentifier blobId = contentHash.ToBlobIdentifier();

                    IDictionary <VstsBlobIdentifier, PreauthenticatedUri> mappings = await ArtifactHttpClientErrorDetectionStrategy.ExecuteWithTimeoutAsync(
                        context,
                        "GetStreamInternal",
                        innerCts => BlobStoreHttpClient.GetDownloadUrisAsync(
                            new[] { ToVstsBlobIdentifier(blobId) },
                            EdgeCache.NotAllowed,
                            cancellationToken: innerCts),
                        cts).ConfigureAwait(false);

                    if (mappings == null || !mappings.TryGetValue(ToVstsBlobIdentifier(blobId), out uri))
                    {
                        return(null);
                    }

                    DownloadUriCache.Instance.AddDownloadUri(contentHash, uri);
                }
                else
                {
                    Tracer.RecordDownloadUriFetchedFromCache();
                }

                return(await GetStreamThroughAzureBlobs(
                           uri.NotNullUri,
                           overrideStreamMinimumReadSizeInBytes,
                           _parallelSegmentDownloadConfig.SegmentDownloadTimeout,
                           cts).ConfigureAwait(false));
            }
        }
Exemplo n.º 6
0
        private void Validate()
        {
            if (BlobId == null)
            {
                throw new ArgumentNullException(nameof(BlobId));
            }

            if (BlockHashes == null)
            {
                throw new ArgumentNullException(nameof(BlockHashes));
            }

            BlobIdentifier computedBlobId = ComputeIdentifierBasedOnBlocks(BlockHashes);

            if (!BlobId.Equals(computedBlobId))
            {
                throw new InvalidDataException(string.Format(
                                                   CultureInfo.InvariantCulture,
                                                   "Computed id '{0}' does not match given one '{1}'.",
                                                   computedBlobId,
                                                   BlobId));
            }
        }
Exemplo n.º 7
0
        /// <summary>
        /// Converts a BlobIdentifier to its corresponding ContentHash.
        /// </summary>
        public static ContentHash ToContentHash(this BlobIdentifier blobId)
        {
            switch (blobId.AlgorithmId)
            {
            case VsoHash.VsoAlgorithmId:
                return(new ContentHash(HashType.Vso0, blobId.Bytes));

            case ChunkDedupIdentifier.ChunkAlgorithmId:
                return(new ContentHash(HashType.Dedup64K, blobId.Bytes));    // TODO: Chunk size optimization

            case (byte)NodeAlgorithmId.Node64K:
                return(new ContentHash(HashType.Dedup64K, blobId.Bytes));

            case (byte)NodeAlgorithmId.Node1024K:
                return(new ContentHash(HashType.Dedup1024K, blobId.Bytes));

            case MurmurHashInfo.MurmurAlgorithmId:
                return(new ContentHash(HashType.Murmur, blobId.Bytes));

            default:
                throw new ArgumentException($"BlobIdentifier has an unrecognized AlgorithmId: {blobId.AlgorithmId}");
            }
        }
Exemplo n.º 8
0
 public BlobIdentifierWithBlocks(BlobIdentifier blobId, IEnumerable <BlobBlockHash> blockIdentifiers)
 {
     BlockHashes = blockIdentifiers.ToList();
     BlobId      = blobId;
     Validate();
 }
Exemplo n.º 9
0
 /// <summary>
 /// Converts a ContentStore BlobId to an Artifact BlobId
 /// </summary>
 protected static VstsBlobIdentifier ToVstsBlobIdentifier(BlobIdentifier blobIdentifier)
 {
     return(new VstsBlobIdentifier(blobIdentifier.Bytes));
 }
Exemplo n.º 10
0
 /// <summary>
 /// Converts a ContentStore BlobId to an Artifact BlobId
 /// </summary>
 protected static VstsBlobIdentifier ToVstsBlobIdentifier(BlobIdentifier blobIdentifier) => new VstsBlobIdentifier(blobIdentifier.Bytes);