[Trait("Category", "Integration2")] // Using a different category name to allow all integration tests to run in parallel.
public void Can1024KChunkReallyLargeFiles()
{
// We want to make sure this goes past uint.MaxValue == 4GB
HashType defaultHashType = HashType.Dedup1024K;
int blockSize = (DedupNode.MaxDirectChildrenPerNode * 2) * defaultHashType.GetAvgChunkSize(); // ~1GB => 1024 * 1024K
int blockCount = (int)((uint.MaxValue / (uint)blockSize) + 1); // 3-4 blocks?
Assert.True(((long)blockSize * (long)blockCount) > (long)uint.MaxValue);
var node = CanChunkLargeFilesHelper(
defaultHashType,
blockSize,
blockCount,
"CE6299176DC223B083363ED4DF81646198DD1E4423C676B82196B7FA17031A42");
var chunks = node.EnumerateChunkLeafsInOrder().ToList();
var nodes = node.EnumerateInnerNodesDepthFirst().ToList();
int minExpectedChunks = DedupNode.MaxDirectChildrenPerNode * blockCount;
Assert.True(chunks.Count > minExpectedChunks, $"Expecting at least '{minExpectedChunks}' chunks but got '{chunks.Count}'.");
int thisLevel = chunks.Count;
int expectedNodes = 0;
while (thisLevel > 1)
{
int parentNodesForThisLevel = (thisLevel + DedupNode.MaxDirectChildrenPerNode - 1) / DedupNode.MaxDirectChildrenPerNode;
expectedNodes += parentNodesForThisLevel;
thisLevel = parentNodesForThisLevel;
}
Assert.True(nodes.Count <= expectedNodes);
}
public void Can64KChunkReallyLargeFiles()
{
// We want to make sure this goes past uint.MaxValue == 4GB
HashType defaultHashType = HashType.Dedup64K;
int blockSize = 2 * DedupNode.MaxDirectChildrenPerNode * defaultHashType.GetAvgChunkSize(); // ~64MB
int blockCount = (int)((uint.MaxValue / (uint)blockSize) + 1);
Assert.True(((long)blockSize * (long)blockCount) > (long)uint.MaxValue);
var node = CanChunkLargeFilesHelper(
defaultHashType,
blockSize,
blockCount,
"A09C8CB4C1B23022C571E75CA143040F9ED8D9A593A7FEECDE2B98725A19E3F5");
var chunks = node.EnumerateChunkLeafsInOrder().ToList();
var nodes = node.EnumerateInnerNodesDepthFirst().ToList();
int minExpectedChunks = DedupNode.MaxDirectChildrenPerNode * blockCount;
Assert.True(chunks.Count > minExpectedChunks, $"Expecting at least '{minExpectedChunks}' chunks but got '{chunks.Count}'.");
int thisLevel = chunks.Count;
int expectedNodes = 0;
while (thisLevel > 1)
{
int parentNodesForThisLevel = (thisLevel + DedupNode.MaxDirectChildrenPerNode - 1) / DedupNode.MaxDirectChildrenPerNode;
expectedNodes += parentNodesForThisLevel;
thisLevel = parentNodesForThisLevel;
}
Assert.True(nodes.Count <= expectedNodes);
}
/// <summary>
/// Initializes a new instance of the <see cref="Dedup1024KHashAlgorithm"/> class.
/// </summary>
public Dedup1024KHashAlgorithm(IChunker chunker)
: base(chunker)
{
int expectedAvgChunkSize = TargetHashType.GetAvgChunkSize();
Contract.Check(chunker.Configuration.AvgChunkSize == expectedAvgChunkSize)?.Assert($"Invalid average chunk size (in bytes) specified: {chunker.Configuration.AvgChunkSize} expected: {expectedAvgChunkSize}");
}
[InlineData(HashType.Dedup1024K, DedupNode.MaxDirectChildrenPerNode / 8, 64, 16)] // 4MB | 64 * 4 * 1024K 256MB
public void HashOfChunksInNodeMatchesChunkHashAlgorithmNegative(HashType hashType, int expectedChunkCount, int multiplier, int divider)
{
Assert.True(hashType.IsValidDedup(), $"Hash type: {hashType} is not a valid dedup.");
var config = new ChunkerConfiguration((multiplier * hashType.GetAvgChunkSize()) / divider);
Assert.Throws <NotImplementedException>(() => HashOfChunksInNodeMatchesChunkHashAlgorithmInner(expectedChunkCount, config, new ManagedChunker(config)));
if (Chunker.IsComChunkerSupported &&
config.AvgChunkSize == ChunkerConfiguration.SupportedComChunkerConfiguration.AvgChunkSize &&
hashType == HashType.Dedup64K) // No COMchunker support for any other chunk sizes.
{
Assert.Throws <NotImplementedException>(() => HashOfChunksInNodeMatchesChunkHashAlgorithmInner(expectedChunkCount, config, new ComChunker(config)));
}
}
/// <nodoc />
public static ChunkerConfiguration GetChunkerConfiguration(this HashType hashType)
{
return(hashType.IsValidDedup() ?
new ChunkerConfiguration(hashType.GetAvgChunkSize()) :
throw new NotImplementedException($"Unsupported enum {hashType} of type {nameof(HashType)} encountered."));
}
