public void CanEnumerateChunksInChunk()
{
DedupNode rootFromhash;
using (var hasher = new DedupNodeHashAlgorithm())
{
hasher.ComputeHash(new byte[1]);
rootFromhash = hasher.GetNode().ChildNodes.Single();
}
Assert.Equal(rootFromhash.HashString, rootFromhash.EnumerateChunkLeafsInOrder().Single().HashString);
}
public void ChunkerIsStable(int byteCount, string expectedHash)
{
var bytes = new byte[byteCount];
FillBufferWithTestContent(seed: 0, bytes);
using (var hasher = new DedupNodeHashAlgorithm())
{
hasher.ComputeHash(bytes, 0, bytes.Length);
var node = hasher.GetNode();
Assert.Equal <string>(expectedHash, node.Hash.ToHex());
}
}
public void DedupHashFile
(
[Required] string[] path,
[DefaultValue(false)] bool chunks,
[DefaultValue(false)] bool childNodes,
[DefaultValue(false)] bool rollingHash,
[DefaultValue(FileSystemConstants.FileIOBufferSize)] int bufferSize,
[DefaultValue((long)0)] long startOffset
)
{
Initialize();
_displayChunks = chunks;
_displayChildNodes = childNodes;
var paths = new List <AbsolutePath>();
foreach (AbsolutePath root in path.Select(p => new AbsolutePath(p)))
{
if (_fileSystem.DirectoryExists(root))
{
paths.AddRange(_fileSystem.EnumerateFiles(root, EnumerateOptions.Recurse).Select(fileInfo => fileInfo.FullPath));
}
else if (_fileSystem.FileExists(root))
{
paths.Add(root);
}
else
{
throw new ArgumentException("given path is not an existing file or directory");
}
}
var buffer = new byte[bufferSize];
using (var hasher = new DedupNodeHashAlgorithm(rollingHash ? DedupNodeTree.Algorithm.RollingHash : DedupNodeTree.Algorithm.MaximallyPacked))
{
foreach (var p in paths)
{
hasher.Initialize();
TaskSafetyHelpers.SyncResultOnThreadPool(async() =>
{
using (var fs = await _fileSystem.OpenReadOnlySafeAsync(p, FileShare.Read | FileShare.Delete))
{
fs.Position = startOffset;
int bytesRead;
while ((bytesRead = await fs.ReadAsync(buffer, 0, buffer.Length)) > 0)
{
hasher.TransformBlock(buffer, 0, bytesRead, null, 0);
}
hasher.TransformFinalBlock(new byte[0], 0, 0);
DedupNode root = hasher.GetNode();
ulong offset = 0;
LogNode(true, string.Empty, root, p, ref offset);
}
return(0);
});
}
}
_logger.Always("Totals:");
_logger.Always($"Bytes: Unique={_uniqueBytes:N0} Total={_totalBytes:N0}");
_logger.Always($"Chunks: Unique={_allChunks.Count:N0} Total={_totalChunks:N0}");
_logger.Always($"Nodes: Unique={_allNodes.Count:N0} Total={_totalNodes:N0}");
}
private void ChunksEnumeratedAsFileIsRead(Func <IChunker> chunkerFactory)
{
var chunks = new List <ChunkInfo>();
byte[] bytes = new byte[4 * Chunker.MinPushBufferSize];
var r = new Random(Seed: 0);
r.NextBytes(bytes);
using (var chunker = DedupNodeHashAlgorithm.CreateChunker())
using (var session = chunker.BeginChunking(chunk =>
{
chunks.Add(chunk);
}))
{
int pushSize = 2 * (int)Chunker.MinPushBufferSize;
int lastChunkCount = 0;
for (int i = 0; i < bytes.Length; i += pushSize)
{
session.PushBuffer(bytes, i, Math.Min(pushSize, bytes.Length - i));
Assert.True(chunks.Count > lastChunkCount);
lastChunkCount = chunks.Count;
}
}
string[] expectedChunkHashes = chunks.Select(c => c.Hash.ToHex()).ToArray();
DedupNode rootFromhash;
string[] actualChunkHashes;
using (var hasher = new DedupNodeHashAlgorithm())
{
hasher.ComputeHash(bytes);
rootFromhash = hasher.GetNode();
actualChunkHashes = rootFromhash.EnumerateChunkLeafsInOrder().Select(c => c.Hash.ToHex()).ToArray();
Assert.Equal(expectedChunkHashes, actualChunkHashes);
}
var seenNodes = new HashSet <byte[]>(chunks.Select(c => c.Hash), ByteArrayComparer.Instance);
DedupNode?root = null;
foreach (var node in PackedDedupNodeTree.EnumerateTree(chunks)
.Where(n => n.Type != DedupNode.NodeType.ChunkLeaf))
{
foreach (var child in node.ChildNodes)
{
Assert.True(seenNodes.Contains(child.Hash));
}
Assert.True(seenNodes.Add(node.Hash));
root = node;
}
Assert.True(root.HasValue);
// ReSharper disable once PossibleInvalidOperationException
Assert.Equal(rootFromhash, root.Value);
actualChunkHashes = root.Value.EnumerateChunkLeafsInOrder().Select(c => c.Hash.ToHex()).ToArray();
Assert.Equal(expectedChunkHashes, actualChunkHashes);
}
