public void HashFile([Required] string path, [Description(HashTypeDescription)] string hashType)
{
Initialize();
var ht = GetHashTypeByNameOrDefault(hashType);
var absolutePath = new AbsolutePath(Path.GetFullPath(path));
var paths = new List <AbsolutePath>();
if (_fileSystem.DirectoryExists(absolutePath))
{
paths.AddRange(_fileSystem.EnumerateFiles(absolutePath, EnumerateOptions.Recurse).Select(fileInfo => fileInfo.FullPath));
}
else if (_fileSystem.FileExists(absolutePath))
{
paths.Add(absolutePath);
}
else
{
throw new ArgumentException("given path is not an existing file or directory");
}
foreach (var p in paths)
{
var contentHash = TaskSafetyHelpers.SyncResultOnThreadPool(() => _fileSystem.CalculateHashAsync(p, ht));
_logger.Always($"{contentHash} {p}");
}
}
// ReSharper disable once UnusedParameter.Local
static int Main(string[] args)
{
int resultCode;
using (var log = new ConsoleLog())
using (var logger = new Logger(log))
{
try
{
using (var fileSystem = new PassThroughFileSystem())
using (var directory = new DisposableDirectory(fileSystem))
using (var store = new FileSystemContentStore(fileSystem, logger, SystemClock.Instance, directory.Path))
{
var context = new Context(logger);
// ReSharper disable once AccessToDisposedClosure
resultCode = TaskSafetyHelpers.SyncResultOnThreadPool(() => RunStore(context, store));
}
}
catch (Exception exception)
{
logger.Error($"Unexpected error: {exception}");
resultCode = 1;
}
}
return(resultCode);
}
public void DedupHashFile
(
[Required] string[] path,
[DefaultValue(false)] bool chunks,
[DefaultValue(false)] bool childNodes,
[DefaultValue(false)] bool rollingHash,
[DefaultValue(FileSystemConstants.FileIOBufferSize)] int bufferSize,
[DefaultValue((long)0)] long startOffset
)
{
Initialize();
_displayChunks = chunks;
_displayChildNodes = childNodes;
var paths = new List <AbsolutePath>();
foreach (AbsolutePath root in path.Select(p => new AbsolutePath(p)))
{
if (_fileSystem.DirectoryExists(root))
{
paths.AddRange(_fileSystem.EnumerateFiles(root, EnumerateOptions.Recurse).Select(fileInfo => fileInfo.FullPath));
}
else if (_fileSystem.FileExists(root))
{
paths.Add(root);
}
else
{
throw new ArgumentException("given path is not an existing file or directory");
}
}
var buffer = new byte[bufferSize];
using (var hasher = new DedupNodeHashAlgorithm(rollingHash ? DedupNodeTree.Algorithm.RollingHash : DedupNodeTree.Algorithm.MaximallyPacked))
{
foreach (var p in paths)
{
hasher.Initialize();
TaskSafetyHelpers.SyncResultOnThreadPool(async() =>
{
using (var fs = await _fileSystem.OpenReadOnlySafeAsync(p, FileShare.Read | FileShare.Delete))
{
fs.Position = startOffset;
int bytesRead;
while ((bytesRead = await fs.ReadAsync(buffer, 0, buffer.Length)) > 0)
{
hasher.TransformBlock(buffer, 0, bytesRead, null, 0);
}
hasher.TransformFinalBlock(new byte[0], 0, 0);
DedupNode root = hasher.GetNode();
ulong offset = 0;
LogNode(true, string.Empty, root, p, ref offset);
}
return(0);
});
}
}
_logger.Always("Totals:");
_logger.Always($"Bytes: Unique={_uniqueBytes:N0} Total={_totalBytes:N0}");
_logger.Always($"Chunks: Unique={_allChunks.Count:N0} Total={_totalChunks:N0}");
_logger.Always($"Nodes: Unique={_allNodes.Count:N0} Total={_totalNodes:N0}");
}
public void DedupHashFile
(
[Required] string[] path,
[Required] string hashType,
[DefaultValue(false)] bool chunks,
[DefaultValue(false)] bool childNodes,
[DefaultValue(FileSystemConstants.FileIOBufferSize)] int bufferSize,
[DefaultValue((long)0)] long startOffset
)
{
Initialize();
_displayChunks = chunks;
_displayChildNodes = childNodes;
if (!Enum.TryParse(hashType, out HashType dedupHashType))
{
throw new ArgumentException($"HashType couldn't be inferred - {hashType}. Valid HashType is required.");
}
var paths = new List <AbsolutePath>();
foreach (AbsolutePath root in path.Select(p => new AbsolutePath(Path.GetFullPath(p))))
{
if (_fileSystem.DirectoryExists(root))
{
paths.AddRange(_fileSystem.EnumerateFiles(root, EnumerateOptions.Recurse).Select(fileInfo => fileInfo.FullPath));
}
else if (_fileSystem.FileExists(root))
{
paths.Add(root);
}
else
{
throw new ArgumentException("given path is not an existing file or directory");
}
}
var buffer = new byte[bufferSize];
using (var contentHasher = new DedupNodeOrChunkHashAlgorithm(new ManagedChunker(dedupHashType.GetChunkerConfiguration())))
{
foreach (var p in paths)
{
contentHasher.Initialize();
TaskSafetyHelpers.SyncResultOnThreadPool(async() =>
{
using (Stream fs = _fileSystem.OpenReadOnly(p, FileShare.Read | FileShare.Delete))
{
fs.Position = startOffset;
int bytesRead;
while ((bytesRead = await fs.ReadAsync(buffer, 0, buffer.Length)) > 0)
{
contentHasher.TransformBlock(buffer, 0, bytesRead, null, 0);
}
contentHasher.TransformFinalBlock(new byte[0], 0, 0);
DedupNode root = contentHasher.GetNode();
ulong offset = 0;
LogNode(true, string.Empty, root, p, ref offset);
}
return(0);
});
}
}
_logger.Always("Totals:");
_logger.Always($"Bytes: Unique={_uniqueBytes:N0} Total={_totalBytes:N0}");
_logger.Always($"Chunks: Unique={_allChunks.Count:N0} Total={_totalChunks:N0}");
_logger.Always($"Nodes: Unique={_allNodes.Count:N0} Total={_totalNodes:N0}");
}
