public static void DoTest(long size, string password, string salt) { FileInfo source = null; FileInfo pending = null; FileInfo target = null; FileInfo restored = null; try { source = CreateRandomFile(size); string sourceHash = KeepHasher.GetHash(source); pending = new FileInfo(source.FullName + ".pending"); target = new FileInfo(source.FullName + ".target"); restored = new FileInfo(source.FullName + ".restored"); IFile testFile = new TestFile() { Name = source.Name, Sha256 = sourceHash, SizeBytes = size }; FileJob job = new FileJob(testFile, source.FullName, pending.FullName, target.FullName, FileJob.ModeEnum.CompressEncrypt, password, salt); job.Execute(); FileJob job2 = new FileJob(testFile, target.FullName, pending.FullName, restored.FullName, FileJob.ModeEnum.DecryptDecompress, password, salt); job2.Execute(); string restoredHash = KeepHasher.GetHash(restored); if (sourceHash != restoredHash) { throw new Exception(); } } finally { if (source != null && source.Exists) { source.Delete(); } if (pending != null && pending.Exists) { pending.Delete(); } if (target != null && target.Exists) { target.Delete(); } if (restored != null && restored.Exists) { restored.Delete(); } } }
public override void Execute() { EnsurePathExists(PendingPath); try { if (_Mode == ModeEnum.CompressEncrypt) { Program.log.Debug("starting backup of " + FromPath); Program.log.Debug("=> compress and encrypt to " + PendingPath); using (Stream encryptedStream = GetEncryptedStream()) using (StreamCryptoWrapper encryptedTarget = StreamCryptoWrapper.Encrypt(encryptedStream, _Password, _Salt)) { using (Stream compressedStream = GetCompressedStream()) { using (FileStream source = new FileStream(FromPath, FileMode.Open, FileAccess.Read, FileShare.Read)) KeepLzmaCompressor.Compress(source, compressedStream); compressedStream.Position = 0; Program.log.Debug("compression done, start encrypting"); compressedStream.CopyTo(encryptedTarget.CryptoStream); encryptedTarget.CryptoStream.FlushFinalBlock(); Program.log.Debug("encryption done, hashing result"); } encryptedStream.Position = 0; ChecksumTarget = KeepHasher.GetHash(encryptedStream); encryptedStream.Position = 0; SizeTarget = encryptedStream.Length; Program.log.Debug("done"); if (encryptedStream is MemoryStream) { using (Stream target = new FileStream(PendingPath, FileMode.Create, FileAccess.Write, FileShare.Read)) encryptedStream.CopyTo(target); } } } else if (_Mode == ModeEnum.DecryptDecompress) { Program.log.Debug("<= decrypt and decompress " + FromPath); using (FileStream source = new FileStream(FromPath, FileMode.Open, FileAccess.Read, FileShare.Read)) using (StreamCryptoWrapper decryptedSource = StreamCryptoWrapper.Decrypt(source, _Password, _Salt)) using (Stream decryptedMemory = GetDecryptedStream()) using (FileStream target = new FileStream(PendingPath, FileMode.Create, FileAccess.Write, FileShare.Read)) { decryptedSource.CryptoStream.CopyTo(decryptedMemory); decryptedMemory.Position = 0; Program.log.Debug("decryption to memory done, decompressing to disc"); KeepLzmaCompressor.Decompress(decryptedMemory, target); Program.log.Debug("decompression done"); } } } finally { foreach (string file in _TempFilesToDelete) { Program.log.Debug("deleting temp file " + file); File.Delete(file); } } EnsurePathExists(ToPath); Program.log.Debug("renaming " + PendingPath + " -> " + ToPath); File.Move(PendingPath, ToPath); if (_Mode == ModeEnum.DecryptDecompress) { Program.log.Debug("restoring metadata " + ToPath); FileInfo fi = new FileInfo(ToPath); fi.LastWriteTimeUtc = SourceFile.LastWriteTimeUtc; fi.CreationTimeUtc = SourceFile.CreationTimeUtc; if (fi.Length != SourceFile.SizeBytes) { Program.log.Error("size mismatch " + ToPath + " " + SourceFile.Sha256); } //string sha = Hashing.GetHash(fi); //if (sha != SourceFile.Sha256) // Program.log.Error("hash mismatch " + ToPath + " " + sha + " but should be "+ SourceFile.Sha256); } }
public void HashFile() { _Sha256 = KeepHasher.GetHash(_FileInfo); }
public void Analyze() { int errorCount = 0; ILog log = Program.log; log.Info("getting all files ... might take some time"); FileInfo[] allFiles = (from x in _Storage.PartitionManager.AllDirectories from f in x.EnumerateFiles("*", SearchOption.AllDirectories) select f).ToArray(); log.Info("done"); log.Info("parsing the sha256 from filename"); IEnumerable <FileInfo> files = allFiles.Where(x => x.Name.EndsWith(ObjectStorage.OBJ_EXTENSION, StringComparison.OrdinalIgnoreCase)); Dictionary <string, FileInfo> filesBySha256 = new Dictionary <string, FileInfo>(); foreach (FileInfo f in files) { string sha256 = f.Name.Substring(0, f.Name.Length - ObjectStorage.OBJ_EXTENSION.Length); filesBySha256.Add(sha256, f); } log.Info("collecting all stored objects from manifest"); Dictionary <string, ManifestItem> manifestBySha256 = _Storage.Manifest.ManifestObjects.ToDictionary(x => x.Sha256source); /// Phase 1 log.Info("check if all manifest files are present as files"); foreach (var x in manifestBySha256.Keys) { if (!filesBySha256.ContainsKey(x)) { log.Error("missing file in storage, which is present in the manifest! " + x); errorCount++; } } /// Phase 2 log.Info("check if all files are present in the manifest"); foreach (var x in filesBySha256.Keys) { if (!manifestBySha256.ContainsKey(x)) { log.Error("missing file in manifest, that was found in storage! " + x); errorCount++; } } /// Phase 3 log.Info("check size of all files with the manifest"); foreach (var hash in filesBySha256.Keys) { ManifestItem mf; if (!manifestBySha256.TryGetValue(hash, out mf)) { continue; } long size = mf.SizeTarget; long size2 = filesBySha256[hash].Length; if (size != size2) { log.Error("storage size mismatch! for " + hash); errorCount++; } } /// Phase 4 log.Info("look for pending files"); IEnumerable <FileInfo> pendingFiles = allFiles.Where(x => x.Name.EndsWith(ObjectStorage.PENDING_EXTENSION, StringComparison.OrdinalIgnoreCase)); foreach (var x in pendingFiles) { log.Error("found pending file " + x.FullName); errorCount++; } /// Phase 5 log.Info("look for unknown files"); foreach (var x in allFiles) { if (!x.Name.EndsWith(ObjectStorage.OBJ_EXTENSION, StringComparison.OrdinalIgnoreCase) && !x.Name.EndsWith(ObjectStorage.PENDING_EXTENSION, StringComparison.OrdinalIgnoreCase)) { log.Error("unknown file " + x.FullName); errorCount++; } } /// Phase 6 log.Info("look, if files from all inventories are present in manifest and nothing more"); FileInfo[] invFiles = _Storage.GetInventoryFiles(); HashSet <string> hashesFromAllInventories = new HashSet <string>(); foreach (FileInfo fi in invFiles) { log.Info(fi.FullName); Inventory i = Inventory.FromXml(fi, _Configuration); IList <string> shas = i.Folder.AllFiles.Select(x => x.Sha256).ToList(); foreach (string sha in shas) { hashesFromAllInventories.Add(sha); if (!manifestBySha256.ContainsKey(sha)) { log.Error("missing file in manifest from inventory " + fi.FullName + " " + sha); errorCount++; } } } foreach (string sha in manifestBySha256.Keys) { if (!hashesFromAllInventories.Contains(sha)) { log.Error("found file in manifest, which is in no inventory " + sha); errorCount++; } } ManifestItem[] ml = manifestBySha256.Values.ToArray(); int count = ml.Count(); if (!_Fast) { /// Phase 7 { log.Info("test cecksums of all files in storage"); Action <ManifestItem> check = delegate(ManifestItem x) { string shaTarget = x.Sha256target; FileInfo fi = new FileInfo(ObjectStorage.GetStorageTarget(x.Sha256source, _Storage.PartitionManager.GetDirByNumber(x.Partition))); string shaFile = KeepHasher.GetHash(fi); if (!string.Equals(shaTarget, shaFile, StringComparison.Ordinal)) { log.Error("storage hash mismatch! for " + x.Sha256source); Interlocked.Increment(ref errorCount); } }; if (ml.Length > 0) { ml.ParallelExecute( x => x.SizeSource, x => x.Sha256source, x => x.Sha256target, x => check(x), 6); } } } if (_Decompress) { // Phase 8 - SLOOWWW :-) log.Info("actually decode file and check size and checksum"); for (int i = 0; i < count; i++) { double percentFiles = ((double)i / (double)count) * 100.0; Program.log.Info(string.Format("file {0} / {1} ({2:0.00}%)", i + 1, count, percentFiles)); var x = ml[i]; log.Info("checking decompression and decryption for " + x.Sha256source); FileInfo fi = new FileInfo(ObjectStorage.GetStorageTarget(x.Sha256source, _Storage.PartitionManager.GetDirByNumber(x.Partition))); long sizeFile; string shaFile = null; using (Stream s = FileJob.JustDecryptAndDecompress(fi, _Configuration.Password, _Configuration.Salt)) { sizeFile = s.Length; shaFile = KeepHasher.GetHash(s); } if (!string.Equals(x.Sha256source, shaFile, StringComparison.Ordinal)) { log.Error("decompressed and decrypted hash mismatch! for " + x.Sha256source); errorCount++; } if (x.SizeSource != sizeFile) { log.Error("decompressed and decrypted size mismatch! for " + x.Sha256source); errorCount++; } } } log.Info(errorCount + " errors !!"); }