Exemplo n.º 1
0
        internal SymbolFile(
            Func <string, bool> symlinkTester,
            Client bxlClient,
            string filePath,
            string fileId,
            ContentHash hash,
            IEnumerable <IDebugEntryData> debugEntries)
        {
            Contract.Requires(symlinkTester != null);
            Contract.Requires(bxlClient != null);
            Contract.Requires(!string.IsNullOrEmpty(filePath));
            Contract.Requires(!string.IsNullOrEmpty(fileId));
            Contract.Requires(hash.IsValid);
            // It's not clear whether the symbol endpoint can play nicely with dedup hashes, so locking it down to VSO0 for now.
            Contract.Requires(hash.HashType == HashType.Vso0, "support only VSO0 hashes (for now)");

            if (debugEntries != null)
            {
                var blobIdentifier = new BlobIdentifier(hash.ToHashByteArray());
                Contract.Assert(debugEntries.All(e => e.BlobIdentifier == blobIdentifier));
                m_debugEntries = new List <IDebugEntryData>(debugEntries);
            }

            m_symlinkTester = symlinkTester;
            m_bxlClient     = bxlClient;
            FullFilePath    = Path.GetFullPath(filePath);
            m_file          = FileId.Parse(fileId);
            Hash            = hash;
        }
        private async Task <BoolResult> GetFileWithDedupAsync(OperationContext context, ContentHash contentHash, string path)
        {
            BlobIdentifier  blobId  = contentHash.ToBlobIdentifier();
            DedupIdentifier dedupId = blobId.ToDedupIdentifier();

            try
            {
                await TryGatedArtifactOperationAsync <object>(
                    context,
                    contentHash.ToString(),
                    "DownloadToFileAsync",
                    async innerCts =>
                {
                    await DedupStoreClient.DownloadToFileAsync(dedupId, path, null, null, EdgeCache.Allowed, innerCts);
                    return(null);
                });
            }
            catch (NullReferenceException) // Null reference thrown when DedupIdentifier doesn't exist in VSTS.
            {
                return(new BoolResult("DedupIdentifier not found."));
            }
            catch (Exception ex)
            {
                return(new BoolResult(ex));
            }

            return(BoolResult.Success);
        }
Exemplo n.º 3
0
        /// <summary>
        ///     Constructor.
        /// </summary>
        /// <param name="fileFullPath">
        ///     Path to the target file which is to be added to drop.  The file must exist on disk at the time of invocation.
        /// </param>
        /// <param name="relativeDropPath">
        ///     Relative path under which to associate the file in the target drop.  If null, file name of the file is used.
        /// </param>
        /// <param name="fileContentInfo">
        ///     Expected content hash and file length. May be left null.
        /// </param>
        public DropItemForFile(string fileFullPath, string relativeDropPath = null, FileContentInfo?fileContentInfo = null)
        {
            Contract.Requires(fileFullPath != null);

            FullFilePath     = Path.GetFullPath(fileFullPath);
            RelativeDropPath = relativeDropPath ?? Path.GetFileName(FullFilePath);
            if (fileContentInfo != null)
            {
                var contentInfo = fileContentInfo.Value;

                if (contentInfo.Hash.HashType.Equals(BuildXL.Cache.ContentStore.Hashing.HashType.DedupChunk))
                {
                    BlobIdentifier = new ChunkDedupIdentifier(contentInfo.Hash.ToHashByteArray()).ToBlobIdentifier();
                }
                else
                {
                    BlobIdentifier = new BlobIdentifier(contentInfo.Hash.ToHashByteArray());
                }

                FileLength = contentInfo.HasKnownLength ? contentInfo.Length : UnknownFileLength;
            }
            else
            {
                BlobIdentifier = null;
                FileLength     = UnknownFileLength;
            }
        }
Exemplo n.º 4
0
        public void CanBeSerializedThenDeserialized_NewtonsoftJson()
        {
            BlobIdentifier original           = BlobIdentifier.MaxValue;
            string         serialized         = JsonConvert.SerializeObject(original, Formatting.None);
            BlobIdentifier deserializedObject = JsonConvert.DeserializeObject <BlobIdentifier>(serialized);

            Assert.Equal(original, deserializedObject);
        }
Exemplo n.º 5
0
        /// <nodoc />
        public DropFile(string relativePath, long?fileSize, BlobIdentifier blobId)
        {
            Contract.Requires(!string.IsNullOrEmpty(relativePath));
            Contract.Requires(blobId != null);

            BlobIdentifier = blobId;
            FileSize       = fileSize;
            RelativePath   = relativePath;
        }
Exemplo n.º 6
0
        /// <nodoc/>
        public void SetDebugEntries(List <DebugEntryData> entries)
        {
            Contract.Requires(entries != null);

            // check that either all entries are missing the blobId, or all the entries have the same blobId and that blobId matches this file
            var blobIdentifier = new BlobIdentifier(Hash.ToHashByteArray());

            Contract.Assert(entries.All(e => e.BlobIdentifier == null) || entries.All(e => e.BlobIdentifier == blobIdentifier));

            // ensure that BlobIdentifier is not null
            // here we 'trust' that the debug entries are from the current symbol file
            entries.ForEach(entry => entry.BlobIdentifier = blobIdentifier);
            m_debugEntries = new List <IDebugEntryData>(entries);
        }
Exemplo n.º 7
0
        public async Task <BlobIdentifierWithBlocks> UploadLogToBlobstorageService(Stream blob, string hubName, Guid planId, int logId)
        {
            CheckConnection();

            BlobIdentifier blobId = VsoHash.CalculateBlobIdentifierWithBlocks(blob).BlobId;

            // Since we read this while calculating the hash, the position needs to be reset before we send this
            blob.Position = 0;

            using (var blobClient = CreateArtifactsClient(_connection, default(CancellationToken)))
            {
                return(await blobClient.UploadBlocksForBlobAsync(blobId, blob, default(CancellationToken)));
            }
        }
Exemplo n.º 8
0
        /// <summary>
        /// Converts a VSO BlobIdentifier to a ContentHash.
        /// </summary>
        public static ContentHash ToContentHash(this BlobIdentifier blobId)
        {
            switch (blobId.AlgorithmId)
            {
            case VsoHash.VsoAlgorithmId:
                return(new ContentHash(HashType.Vso0, blobId.Bytes));

            case BuildXL.Cache.ContentStore.Hashing.NodeDedupIdentifier.NodeAlgorithmId:
            case ChunkDedupIdentifier.ChunkAlgorithmId:
                return(new ContentHash(HashType.DedupNodeOrChunk, blobId.Bytes));

            default:
                throw new ArgumentException($"BlobIdentifier has an unrecognized AlgorithmId: {blobId.AlgorithmId}");
            }
        }
Exemplo n.º 9
0
        public void SerializationFormatIsCorrect_NewtonsoftJson()
        {
            BlobIdentifier original   = BlobIdentifier.CreateFromAlgorithmResult(HashIdentifier, 0xF);
            string         serialized = JsonConvert.SerializeObject(original);
            JObject        jObject    = JObject.Parse(serialized);

            Assert.Equal(1, jObject.Count);

            var identifierProperty = jObject["identifierValue"];

            Assert.NotNull(identifierProperty);

            var bytes = identifierProperty.ToObject <byte[]>();

            Assert.Equal(bytes, original.Bytes);
        }
Exemplo n.º 10
0
        /// <summary>
        /// Extracts the debug entry for a javascript sourcemap file.
        /// It will try to extract the client key from the sourcemap, so that
        /// the tool that writes the sourcemap has control over they key.
        /// It will fall back to the sha256 of the sourcemap as the client key
        /// when it can't be found.
        /// </summary>
        internal static async Task <DebugEntryData[]> GetJsMapDebugEntryAsync(FileInfo file, bool calculateBlobId = false)
        {
            var    fileName  = file.FullName;
            string clientKey = TryGetSymbolClientKeyFromJsMap(fileName);

            if (clientKey == null)
            {
                // If the .js.map file file does not contain the proper info, use content hash as a fallback
                try
                {
                    using (var fileStream = FileStreamUtility.OpenFileStreamForAsync(fileName, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete))
                    {
                        var hash = await HashInfoLookup.GetContentHasher(HashType.SHA256).GetContentHashAsync(fileStream);

                        var clientId = hash.ToHex().ToLowerInvariant();
                        clientKey = CreateClientKey(clientId, Path.GetFileName(fileName));
                    }
                }
                catch (IOException)
                {
                    return(new DebugEntryData[0]);
                }
                catch (UnauthorizedAccessException)
                {
                    return(new DebugEntryData[0]);
                }
            }

            BlobIdentifier blobId = null;

            if (calculateBlobId)
            {
                var blobDescriptor = await FileBlobDescriptor.CalculateAsync(file.DirectoryName, chunkDedup : false, file.Name, FileBlobType.File, CancellationToken.None);

                blobId = blobDescriptor.BlobIdentifier;
            }

            return(new[] {
                new DebugEntryData()
                {
                    BlobIdentifier = blobId,
                    ClientKey = clientKey,
                    InformationLevel = DebugInformationLevel.Private
                }
            });
        }
Exemplo n.º 11
0
        /// <summary>
        /// Gets a single hash representing the ContentHashList
        /// </summary>
        public static byte[] GetHashOfHashes(this ContentHashList contentHashList)
        {
            var            rollingBlobIdentifier = new VsoHash.RollingBlobIdentifier();
            BlobIdentifier blobIdOfContentHashes = VsoHash.OfNothing.BlobId;

            for (int i = 0; i < contentHashList.Hashes.Count; i++)
            {
                BlobIdentifier blobId = BlobIdentifier.Deserialize(contentHashList.Hashes[i].ToHex());

                if (i != contentHashList.Hashes.Count - 1)
                {
                    rollingBlobIdentifier.Update(VsoHash.HashBlock(blobId.Bytes, blobId.Bytes.Length));
                }
                else
                {
                    blobIdOfContentHashes = rollingBlobIdentifier.Finalize(VsoHash.HashBlock(blobId.Bytes, blobId.Bytes.Length));
                }
            }

            return(blobIdOfContentHashes.Bytes);
        }
Exemplo n.º 12
0
        public void HashAlgorithmForCacheIsTheSame()
        {
            using (var hashAlgorithm = new VsoHashAlgorithm())
            {
                var blobSizes = new[]
                {
                    0, 1,
                    VsoHash.BlockSize - 1, VsoHash.BlockSize, VsoHash.BlockSize + 1,
                    (2 * VsoHash.BlockSize) - 1, 2 * VsoHash.BlockSize, (2 * VsoHash.BlockSize) + 1,
                };

                foreach (int blobSize in blobSizes)
                {
                    var content = ThreadSafeRandom.GetBytes(blobSize);
                    hashAlgorithm.Initialize();

                    byte[]         hashAlgoBytes = hashAlgorithm.ComputeHash(content);
                    BlobIdentifier blobId        = VsoHash.CalculateBlobIdentifier(content);
                    Assert.True(hashAlgoBytes.SequenceEqual(blobId.Bytes));
                }
            }
        }
        /// <summary>
        /// Converts a VSO BlobIdentifier to a ContentHash.
        /// </summary>
        public static ContentHash ToContentHash(this BlobIdentifier blobId)
        {
            switch (blobId.AlgorithmId)
            {
            case VsoHash.VsoAlgorithmId:
                return(new ContentHash(HashType.Vso0, blobId.Bytes));

            case ChunkDedupIdentifier.ChunkAlgorithmId:
                return(new ContentHash(HashType.Dedup64K, blobId.Bytes));    // TODO: Chunk size optimization

            case (byte)NodeAlgorithmId.Node64K:
                return(new ContentHash(HashType.Dedup64K, blobId.Bytes));

            case (byte)NodeAlgorithmId.Node1024K:
                return(new ContentHash(HashType.Dedup1024K, blobId.Bytes));

            case MurmurHashInfo.MurmurAlgorithmId:
                return(new ContentHash(HashType.Murmur, blobId.Bytes));

            default:
                throw new ArgumentException($"BlobIdentifier has an unrecognized AlgorithmId: {blobId.AlgorithmId}");
            }
        }
Exemplo n.º 14
0
 /// <summary>
 /// Initializes a new instance of the <see cref="BlobContentHashListWithDeterminism"/> struct.
 /// </summary>
 public BlobContentHashListWithDeterminism(Guid value, BlobIdentifier blobId, Uri downloadUri)
 {
     ContentHashListMetadataBlobId = blobId?.ValueString;
     ContentHashListMetadataBlobDownloadUriString = downloadUri?.AbsoluteUri;
     Guid = value.ToString();
 }
Exemplo n.º 15
0
 /// <summary>
 /// Initializes a new instance of the <see cref="BlobContentHashListWithDeterminism"/> struct.
 /// </summary>
 public BlobContentHashListWithDeterminism(Guid value, BlobIdentifier blobId)
     : this(value, blobId, null)
 {
 }
        private async Task <PinResult> PinCoreImplAsync(
            OperationContext context, ContentHash contentHash, DateTime keepUntil)
        {
            if (!contentHash.HashType.IsValidDedup())
            {
                return(new PinResult($"DedupStore client requires a HashType that supports dedup. Given hash type: {contentHash.HashType}."));
            }

            var pinResult = CheckPinInMemory(contentHash, keepUntil);

            if (pinResult.Succeeded)
            {
                return(pinResult);
            }

            BlobIdentifier  blobId  = contentHash.ToBlobIdentifier();
            DedupIdentifier dedupId = blobId.ToDedupIdentifier();

            if (dedupId.AlgorithmId == Hashing.ChunkDedupIdentifier.ChunkAlgorithmId)
            {
                // No need to optimize since pinning a chunk is always a fast operation.
                return(await PinImplAsync(context, contentHash, keepUntil));
            }

            // Since pinning the whole tree can be an expensive operation, we have optimized how we call it. Depending on the current
            //  keepUntil of the root node, which is unexpensive to check, the operation will behave differently:
            //      The pin operation will be ignored if it is greater than ignorePinThreshold, to reduce amount of calls
            //      The pin operation will be inlined if it is lower than pinInlineThreshold, to make sure that we don't try to use
            //          content that we pin in the background but has expired before we could complete the pin.
            //      The pin operation will be done asynchronously and will return success otherwise. Most calls should follow this
            //          behavior, to avoid waiting on a potentially long operation. We're confident returning a success because we
            //          know that the content is there even though we still have to extend it's keepUntil

            var keepUntilResult = await CheckNodeKeepUntilAsync(context, dedupId);

            if (!keepUntilResult.Succeeded)
            {
                // Returned a service error. Fail fast.
                return(new PinResult(keepUntilResult));
            }
            else if (!keepUntilResult.Value.HasValue)
            {
                // Content not found.
                return(new PinResult(PinResult.ResultCode.ContentNotFound));
            }

            var timeLeft = keepUntilResult.Value.Value - DateTime.UtcNow;


            // Make sure to only trigger this optimization for normal pins and not for pins for incorporate
            if (keepUntil == EndDateTime && timeLeft > _ignorePinThreshold)
            {
                Tracer.Debug(context, $"Pin was skipped because keepUntil has remaining time [{timeLeft}] that is greater than ignorePinThreshold=[{_ignorePinThreshold}]");
                _dedupCounters[Counters.PinIgnored].Increment();
                return(PinResult.Success);
            }

            var pinTask = PinImplAsync(context, contentHash, keepUntil);

            if (timeLeft < _pinInlineThreshold)
            {
                Tracer.Debug(context, $"Pin inlined because keepUntil has remaining time [{timeLeft}] that is less than pinInlineThreshold=[{_pinInlineThreshold}]");
                _dedupCounters[Counters.PinInlined].Increment();
                return(await pinTask);
            }

            pinTask.FireAndForget(context);

            return(PinResult.Success);
        }
Exemplo n.º 17
0
        private BlobIdentifier CreateBlobIdentifier()
        {
            const string HashIdentifier = "54CE418A2A89A74B42CC39630167795DED5F3B16A75FF32A01B2B01C59697784";

            return(BlobIdentifier.CreateFromAlgorithmResult(HashIdentifier.ToUpperInvariant()));
        }