public void HashFile([Required] string path, [Description(HashTypeDescription)] string hashType) { Initialize(); var ht = GetHashTypeByNameOrDefault(hashType); var absolutePath = new AbsolutePath(Path.GetFullPath(path)); var paths = new List <AbsolutePath>(); if (_fileSystem.DirectoryExists(absolutePath)) { paths.AddRange(_fileSystem.EnumerateFiles(absolutePath, EnumerateOptions.Recurse).Select(fileInfo => fileInfo.FullPath)); } else if (_fileSystem.FileExists(absolutePath)) { paths.Add(absolutePath); } else { throw new ArgumentException("given path is not an existing file or directory"); } foreach (var p in paths) { var contentHash = TaskSafetyHelpers.SyncResultOnThreadPool(() => _fileSystem.CalculateHashAsync(p, ht)); _logger.Always($"{contentHash} {p}"); } }
// ReSharper disable once UnusedParameter.Local static int Main(string[] args) { int resultCode; using (var log = new ConsoleLog()) using (var logger = new Logger(log)) { try { using (var fileSystem = new PassThroughFileSystem()) using (var directory = new DisposableDirectory(fileSystem)) using (var store = new FileSystemContentStore(fileSystem, logger, SystemClock.Instance, directory.Path)) { var context = new Context(logger); // ReSharper disable once AccessToDisposedClosure resultCode = TaskSafetyHelpers.SyncResultOnThreadPool(() => RunStore(context, store)); } } catch (Exception exception) { logger.Error($"Unexpected error: {exception}"); resultCode = 1; } } return(resultCode); }
public void DedupHashFile ( [Required] string[] path, [DefaultValue(false)] bool chunks, [DefaultValue(false)] bool childNodes, [DefaultValue(false)] bool rollingHash, [DefaultValue(FileSystemConstants.FileIOBufferSize)] int bufferSize, [DefaultValue((long)0)] long startOffset ) { Initialize(); _displayChunks = chunks; _displayChildNodes = childNodes; var paths = new List <AbsolutePath>(); foreach (AbsolutePath root in path.Select(p => new AbsolutePath(p))) { if (_fileSystem.DirectoryExists(root)) { paths.AddRange(_fileSystem.EnumerateFiles(root, EnumerateOptions.Recurse).Select(fileInfo => fileInfo.FullPath)); } else if (_fileSystem.FileExists(root)) { paths.Add(root); } else { throw new ArgumentException("given path is not an existing file or directory"); } } var buffer = new byte[bufferSize]; using (var hasher = new DedupNodeHashAlgorithm(rollingHash ? DedupNodeTree.Algorithm.RollingHash : DedupNodeTree.Algorithm.MaximallyPacked)) { foreach (var p in paths) { hasher.Initialize(); TaskSafetyHelpers.SyncResultOnThreadPool(async() => { using (var fs = await _fileSystem.OpenReadOnlySafeAsync(p, FileShare.Read | FileShare.Delete)) { fs.Position = startOffset; int bytesRead; while ((bytesRead = await fs.ReadAsync(buffer, 0, buffer.Length)) > 0) { hasher.TransformBlock(buffer, 0, bytesRead, null, 0); } hasher.TransformFinalBlock(new byte[0], 0, 0); DedupNode root = hasher.GetNode(); ulong offset = 0; LogNode(true, string.Empty, root, p, ref offset); } return(0); }); } } _logger.Always("Totals:"); _logger.Always($"Bytes: Unique={_uniqueBytes:N0} Total={_totalBytes:N0}"); _logger.Always($"Chunks: Unique={_allChunks.Count:N0} Total={_totalChunks:N0}"); _logger.Always($"Nodes: Unique={_allNodes.Count:N0} Total={_totalNodes:N0}"); }
public void DedupHashFile ( [Required] string[] path, [Required] string hashType, [DefaultValue(false)] bool chunks, [DefaultValue(false)] bool childNodes, [DefaultValue(FileSystemConstants.FileIOBufferSize)] int bufferSize, [DefaultValue((long)0)] long startOffset ) { Initialize(); _displayChunks = chunks; _displayChildNodes = childNodes; if (!Enum.TryParse(hashType, out HashType dedupHashType)) { throw new ArgumentException($"HashType couldn't be inferred - {hashType}. Valid HashType is required."); } var paths = new List <AbsolutePath>(); foreach (AbsolutePath root in path.Select(p => new AbsolutePath(Path.GetFullPath(p)))) { if (_fileSystem.DirectoryExists(root)) { paths.AddRange(_fileSystem.EnumerateFiles(root, EnumerateOptions.Recurse).Select(fileInfo => fileInfo.FullPath)); } else if (_fileSystem.FileExists(root)) { paths.Add(root); } else { throw new ArgumentException("given path is not an existing file or directory"); } } var buffer = new byte[bufferSize]; using (var contentHasher = new DedupNodeOrChunkHashAlgorithm(new ManagedChunker(dedupHashType.GetChunkerConfiguration()))) { foreach (var p in paths) { contentHasher.Initialize(); TaskSafetyHelpers.SyncResultOnThreadPool(async() => { using (Stream fs = _fileSystem.OpenReadOnly(p, FileShare.Read | FileShare.Delete)) { fs.Position = startOffset; int bytesRead; while ((bytesRead = await fs.ReadAsync(buffer, 0, buffer.Length)) > 0) { contentHasher.TransformBlock(buffer, 0, bytesRead, null, 0); } contentHasher.TransformFinalBlock(new byte[0], 0, 0); DedupNode root = contentHasher.GetNode(); ulong offset = 0; LogNode(true, string.Empty, root, p, ref offset); } return(0); }); } } _logger.Always("Totals:"); _logger.Always($"Bytes: Unique={_uniqueBytes:N0} Total={_totalBytes:N0}"); _logger.Always($"Chunks: Unique={_allChunks.Count:N0} Total={_totalChunks:N0}"); _logger.Always($"Nodes: Unique={_allNodes.Count:N0} Total={_totalNodes:N0}"); }