public async Task ThrowInParallelCallback()
{
try
{
using (var content = new MemoryStream(Enumerable.Range(0, 4 * VsoHash.BlockSize).Select(i => (byte)(i % 256)).ToArray()))
{
using (var semaphore = new SemaphoreSlim(1, 1))
{
try
{
await VsoHash.WalkBlocksAsync(
content,
semaphore,
true,
singleBlockCallback : (block, blockLength, blockHash) => Task.FromResult(0),
multipleBlockCallback : (block, blockLength, blockHash, isFinalBlock) => Task.Run(() =>
{
throw new FileNotFoundException();
}),
multipleBlockSealCallback : blobIdWithBlocks => Task.FromResult(0));
}
catch (AggregateException aggEx)
{
// ReSharper disable once PossibleNullReferenceException
throw aggEx.InnerException;
}
}
}
Assert.True(false, "should have thrown");
}
catch (FileNotFoundException)
{
}
}
public async Task WalkAllBlocks()
{
var random = new Random(0);
await TryWithDifferentSizesAsync(async testSize =>
{
var bytes = new byte[testSize];
random.NextBytes(bytes);
using (var byteStream = new MemoryStream(bytes))
{
var blobIdWithBlocks = await VsoHash.WalkAllBlobBlocksAsync(
byteStream,
null,
true,
multipleBlockCallback: (block, blockLength, blockHash, isFinalBlock) => Task.FromResult(0));
byteStream.Position = 0;
Assert.Equal(await VsoHash.CalculateBlobIdentifierWithBlocksAsync(byteStream), blobIdWithBlocks);
byteStream.Position = 0;
Assert.Equal(await VsoHash.CalculateBlobIdentifierWithBlocksAsync(byteStream), blobIdWithBlocks);
}
});
}
public Task <BlobIdentifierWithBlocks> UploadLogToBlobstorageService(Stream blob, string hubName, Guid planId, int logId)
{
var blockBlobId = VsoHash.CalculateBlobIdentifierWithBlocks(blob);
blob.Position = 0;
using (var reader = new StreamReader(blob))
{
var text = reader.ReadToEnd();
var lines = text.Split("\n");
UploadedLogBlobs.Add(blockBlobId, lines);
}
return(Task.FromResult(blockBlobId));
}
public void CalculatedIdentifiersForIdenticalByteArrayBlocksMatchExactly()
{
byte[] content = MockBuilder.GetContent();
string contentId1 = VsoHash.CalculateBlobIdentifier(content).ValueString;
string contentId2 = VsoHash.CalculateBlobIdentifier(content).ValueString;
if (contentId1 == null || contentId2 == null)
{
Assert.True(false, "inconclusive");
}
Assert.Equal(contentId1, contentId2);
}
public async Task <BlobIdentifierWithBlocks> UploadLogToBlobstorageService(Stream blob, string hubName, Guid planId, int logId)
{
CheckConnection();
BlobIdentifier blobId = VsoHash.CalculateBlobIdentifierWithBlocks(blob).BlobId;
// Since we read this while calculating the hash, the position needs to be reset before we send this
blob.Position = 0;
using (var blobClient = CreateArtifactsClient(_connection, default(CancellationToken)))
{
return(await blobClient.UploadBlocksForBlobAsync(blobId, blob, default(CancellationToken)));
}
}
public void CalculateBlobIdentifierFromArrayAndStreamUsingBlocksMatchEachOther()
{
using (Stream contentStream = MockBuilder.GetContentStream(VsoHash.BlockSize))
{
byte[] content = MockBuilder.GetContent(VsoHash.BlockSize);
string contentId1 = VsoHash.CalculateBlobIdentifierWithBlocks(contentStream).BlobId.ValueString;
string contentId2 = VsoHash.CalculateBlobIdentifier(content).ValueString;
if (contentId1 == null || contentId2 == null)
{
Assert.True(false, "inconclusive");
}
Assert.Equal(contentId1, contentId2);
}
}
public void CalculateBlobIdentifierEntireBytesBuiltInBlocksForIdenticalArraysMatchExactly()
{
using (MockBuilder.GetContentStream(VsoHash.BlockSize))
{
byte[] content = MockBuilder.GetContent(VsoHash.BlockSize);
string contentId1 = VsoHash.CalculateBlobIdentifier(content).ValueString;
string contentId2 = VsoHash.CalculateBlobIdentifier(content).ValueString;
if (contentId1 == null || contentId2 == null)
{
Assert.True(false, "inconclusive");
}
Assert.Equal(contentId1, contentId2);
}
}
public void ChunkHashesMatchAndRollupToIdentifier()
{
var random = new Random();
TryWithDifferentSizes(testSize =>
{
var bytes = new byte[testSize];
random.NextBytes(bytes);
var blocks = new List <BlobBlockHash>();
var rollingId = new VsoHash.RollingBlobIdentifierWithBlocks();
BlobIdentifierWithBlocks blobIdentifierWithBlocks = null;
for (int i = 0; i < testSize;)
{
int blockSize = Math.Min(testSize - i, VsoHash.BlockSize);
var block = new byte[blockSize];
Array.Copy(bytes, i, block, 0, blockSize);
BlobBlockHash blockHash = VsoHash.HashBlock(block, blockSize);
blocks.Add(blockHash);
i += blockSize;
if (i < testSize)
{
rollingId.Update(blockHash);
}
else
{
blobIdentifierWithBlocks = rollingId.Finalize(blockHash);
}
}
if (testSize == 0)
{
BlobBlockHash blockHash = VsoHash.HashBlock(new byte[] { }, 0);
blocks.Add(blockHash);
blobIdentifierWithBlocks = rollingId.Finalize(blockHash);
}
using (var byteStream = new MemoryStream(bytes))
{
BlobIdentifierWithBlocks identifierWithBlocks = VsoHash.CalculateBlobIdentifierWithBlocks(byteStream);
Assert.True(identifierWithBlocks.BlockHashes.SequenceEqual(blocks));
Assert.Equal(identifierWithBlocks, blobIdentifierWithBlocks);
}
});
}
public void CalculateBlobIdentifierFromArrayAndStreamMatchEachOther()
{
byte[] content = MockBuilder.GetContent();
string contentId1;
using (Stream contentStream = MockBuilder.GetContentStream())
{
contentId1 = VsoHash.CalculateBlobIdentifier(contentStream).ValueString;
}
var contentId2 = VsoHash.CalculateBlobIdentifier(content).ValueString;
if (contentId1 == null || contentId2 == null)
{
Assert.True(false, "inconclusive");
}
Assert.Equal(contentId1, contentId2);
}
public void BlockHashesDoNotChange()
{
var knownValues = new Dictionary <IEnumerable <int>, string>()
{
{ Enumerable.Empty <int>(), "E3B0C44298FC1C149AFBF4C8996FB92427AE41E4649B934CA495991B7852B855" },
{ Enumerable.Range(0, 1), "1406E05881E299367766D313E26C05564EC91BF721D31726BD6E46E60689539A" },
{ Enumerable.Range(0, VsoHash.PageSize - 1), "12078762B8EDA8A5499C46E9B5C7F8D37BAB3A684571AEFC2A7D3ABCD56C093E" },
{ Enumerable.Range(0, VsoHash.PageSize), "B00BE365B41949CF3571F69F8F5AD95514F6AFDFE094ABA614ECD34BD828272B" },
{ Enumerable.Range(0, VsoHash.PageSize + 1), "4A3E85BABDD4243495A3617E9316BDF9CDC4526F97AA0E435A47226876C3D167" },
{ Enumerable.Range(0, VsoHash.BlockSize - 1), "3492B19DDCD76EA1ED5C07090A021705ADC7E7D5C5AAD4FF619FD12FAECEB197" },
{ Enumerable.Range(0, VsoHash.BlockSize), "E8DEEF25ED53357D2A738D7156067E69892A7BDC190818CD2AD698A3A1F95E03" },
};
foreach (var knownValue in knownValues)
{
byte[] bytes = knownValue.Key.Select(i => (byte)(i & 0xFF)).ToArray();
Assert.Equal(
knownValue.Value,
VsoHash.HashBlock(bytes, bytes.Length).HashString);
}
}
/// <summary>
/// Gets a single hash representing the ContentHashList
/// </summary>
public static byte[] GetHashOfHashes(this ContentHashList contentHashList)
{
var rollingBlobIdentifier = new VsoHash.RollingBlobIdentifier();
BlobIdentifier blobIdOfContentHashes = VsoHash.OfNothing.BlobId;
for (int i = 0; i < contentHashList.Hashes.Count; i++)
{
BlobIdentifier blobId = BlobIdentifier.Deserialize(contentHashList.Hashes[i].ToHex());
if (i != contentHashList.Hashes.Count - 1)
{
rollingBlobIdentifier.Update(VsoHash.HashBlock(blobId.Bytes, blobId.Bytes.Length));
}
else
{
blobIdOfContentHashes = rollingBlobIdentifier.Finalize(VsoHash.HashBlock(blobId.Bytes, blobId.Bytes.Length));
}
}
return(blobIdOfContentHashes.Bytes);
}
public void HashAlgorithmForCacheIsTheSame()
{
using (var hashAlgorithm = new VsoHashAlgorithm())
{
var blobSizes = new[]
{
0, 1,
VsoHash.BlockSize - 1, VsoHash.BlockSize, VsoHash.BlockSize + 1,
(2 * VsoHash.BlockSize) - 1, 2 * VsoHash.BlockSize, (2 * VsoHash.BlockSize) + 1,
};
foreach (int blobSize in blobSizes)
{
var content = ThreadSafeRandom.GetBytes(blobSize);
hashAlgorithm.Initialize();
byte[] hashAlgoBytes = hashAlgorithm.ComputeHash(content);
BlobIdentifier blobId = VsoHash.CalculateBlobIdentifier(content);
Assert.True(hashAlgoBytes.SequenceEqual(blobId.Bytes));
}
}
}
public void BlobIdsDoNotChange()
{
var knownValues = new Dictionary <IEnumerable <int>, string>()
{
{ Enumerable.Empty <int>(), "1E57CF2792A900D06C1CDFB3C453F35BC86F72788AA9724C96C929D1CC6B456A00" },
{ Enumerable.Range(0, 1), "3DA32150B5E69B54E7AD1765D9573BC5E6E05D3B6529556C1B4A436A76A511F400" },
{ Enumerable.Range(0, VsoHash.PageSize - 1), "4AE1AD6462D75D117A5DAFCF98167981371A4B21E1CEE49D0B982DE2CE01032300" },
{ Enumerable.Range(0, VsoHash.PageSize), "85840E1CB7CBFD78B464921C54C96F68C19066F20860EFA8CCE671B40BA5162300" },
{ Enumerable.Range(0, VsoHash.PageSize + 1), "D92A37C547F9D5B6B7B791A24F587DA8189CCA14EBC8511D2482E7448763E2BD00" },
{ Enumerable.Range(0, VsoHash.BlockSize - 1), "1C3C73F7E829E84A5BA05631195105FB49E033FA23BDA6D379B3E46B5D73EF3700" },
{ Enumerable.Range(0, VsoHash.BlockSize), "6DAE3ED3E623AED293297C289C3D20A53083529138B7631E99920EF0D93AF3CD00" },
{ Enumerable.Range(0, VsoHash.BlockSize + 1), "1F9F3C008EA37ECB65BC5FB14A420CEBB3CA72A9601EC056709A6B431F91807100" },
{ Enumerable.Range(0, (2 * VsoHash.BlockSize) - 1), "DF0E0DB15E866592DBFA9BCA74E6D547D67789F7EB088839FC1A5CEFA862353700" },
{ Enumerable.Range(0, 2 * VsoHash.BlockSize), "5E3A80B2ACB2284CD21A08979C49CBB80874E1377940699B07A8ABEE9175113200" },
{ Enumerable.Range(0, (2 * VsoHash.BlockSize) + 1), "B9A44A420593FA18453B3BE7B63922DF43C93FF52D88F2CAB26FE1FADBA7003100" },
};
foreach (var knownValue in knownValues)
{
Assert.Equal(
knownValue.Value,
VsoHash.CalculateBlobIdentifier(knownValue.Key.Select(i => (byte)(i & 0xFF)).ToArray()).ValueString);
}
}
/// <summary>
/// Alternative for UploadAndReferenceBlobAsync that uses WalkBlocksAsync instead of the synchronous WalkBlocks.
/// Also utilizes the AsyncHttpRetryHelper to mitigate transient exceptions.
/// </summary>
public static Task UploadAndReferenceBlobWithRetriesAsync(
this IBlobStoreHttpClient client,
BlobIdentifier blobId,
Stream stream,
BlobReference reference,
Context context,
CancellationToken cts)
{
Contract.Requires(stream != null);
var attempt = 0;
return(AsyncHttpRetryHelper.InvokeVoidAsync(
async() =>
{
bool blobUploaded = false;
stream.Position = 0;
attempt++;
await VsoHash.WalkBlocksAsync(
stream,
blockActionSemaphore: null,
multiBlocksInParallel: true,
singleBlockCallback:
async(block, blockLength, blockHash) =>
{
await client.PutSingleBlockBlobAndReferenceAsync(
blobId, block, blockLength, reference, cts).ConfigureAwait(false);
blobUploaded = true;
},
multiBlockCallback:
(block, blockLength, blockHash, isFinalBlock) =>
client.PutBlobBlockAsync(blobId, block, blockLength, cts),
multiBlockSealCallback: async blobIdWithBlocks =>
{
var failedRefs = await client.TryReferenceWithBlocksAsync(
new Dictionary <BlobIdentifierWithBlocks, IEnumerable <BlobReference> >
{
{ blobIdWithBlocks, new[] { reference } }
},
cancellationToken: cts).ConfigureAwait(false);
blobUploaded = !failedRefs.Any();
}
);
if (!blobUploaded)
{
throw new AsyncHttpRetryHelper.RetryableException($"Could not upload blob on attempt {attempt}.");
}
},
maxRetries: 5,
tracer: new AppTraceSourceContextAdapter(context, "BlobStoreHttpClient", SourceLevels.All),
canRetryDelegate: exception =>
{
// HACK HACK: This is an additional layer of retries specifically to catch the SSL exceptions seen in DM.
// Newer versions of Artifact packages have this retry automatically, but the packages deployed with the M119.0 release do not.
// Once the next release is completed, these retries can be removed.
if (exception is HttpRequestException && exception.InnerException is WebException)
{
return true;
}
while (exception != null)
{
if (exception is TimeoutException)
{
return true;
}
exception = exception.InnerException;
}
return false;
},
cancellationToken: cts,
continueOnCapturedContext: false,
context: context.Id.ToString()));
}
public void CalculateBlobIdentifierEntireBytesMethodThrowsWhenAByteArrayIsNotProvided()
{
Assert.Throws <ArgumentNullException>(() => VsoHash.CalculateBlobIdentifier((byte[])null));
}
public void CalculateBlobIdentifierMethodThrowsWhenAStreamIsNotProvided()
{
Assert.Throws <ArgumentNullException>(() => VsoHash.CalculateBlobIdentifier((Stream)null));
}
