public static void DoTest(long size, string password, string salt)
{
FileInfo source = null;
FileInfo pending = null;
FileInfo target = null;
FileInfo restored = null;
try
{
source = CreateRandomFile(size);
string sourceHash = KeepHasher.GetHash(source);
pending = new FileInfo(source.FullName + ".pending");
target = new FileInfo(source.FullName + ".target");
restored = new FileInfo(source.FullName + ".restored");
IFile testFile = new TestFile()
{
Name = source.Name,
Sha256 = sourceHash,
SizeBytes = size
};
FileJob job = new FileJob(testFile, source.FullName, pending.FullName, target.FullName, FileJob.ModeEnum.CompressEncrypt, password, salt);
job.Execute();
FileJob job2 = new FileJob(testFile, target.FullName, pending.FullName, restored.FullName, FileJob.ModeEnum.DecryptDecompress, password, salt);
job2.Execute();
string restoredHash = KeepHasher.GetHash(restored);
if (sourceHash != restoredHash)
{
throw new Exception();
}
}
finally
{
if (source != null && source.Exists)
{
source.Delete();
}
if (pending != null && pending.Exists)
{
pending.Delete();
}
if (target != null && target.Exists)
{
target.Delete();
}
if (restored != null && restored.Exists)
{
restored.Delete();
}
}
}
public override void Execute()
{
EnsurePathExists(PendingPath);
try
{
if (_Mode == ModeEnum.CompressEncrypt)
{
Program.log.Debug("starting backup of " + FromPath);
Program.log.Debug("=> compress and encrypt to " + PendingPath);
using (Stream encryptedStream = GetEncryptedStream())
using (StreamCryptoWrapper encryptedTarget = StreamCryptoWrapper.Encrypt(encryptedStream, _Password, _Salt))
{
using (Stream compressedStream = GetCompressedStream())
{
using (FileStream source = new FileStream(FromPath, FileMode.Open, FileAccess.Read, FileShare.Read))
KeepLzmaCompressor.Compress(source, compressedStream);
compressedStream.Position = 0;
Program.log.Debug("compression done, start encrypting");
compressedStream.CopyTo(encryptedTarget.CryptoStream);
encryptedTarget.CryptoStream.FlushFinalBlock();
Program.log.Debug("encryption done, hashing result");
}
encryptedStream.Position = 0;
ChecksumTarget = KeepHasher.GetHash(encryptedStream);
encryptedStream.Position = 0;
SizeTarget = encryptedStream.Length;
Program.log.Debug("done");
if (encryptedStream is MemoryStream)
{
using (Stream target = new FileStream(PendingPath, FileMode.Create, FileAccess.Write, FileShare.Read))
encryptedStream.CopyTo(target);
}
}
}
else if (_Mode == ModeEnum.DecryptDecompress)
{
Program.log.Debug("<= decrypt and decompress " + FromPath);
using (FileStream source = new FileStream(FromPath, FileMode.Open, FileAccess.Read, FileShare.Read))
using (StreamCryptoWrapper decryptedSource = StreamCryptoWrapper.Decrypt(source, _Password, _Salt))
using (Stream decryptedMemory = GetDecryptedStream())
using (FileStream target = new FileStream(PendingPath, FileMode.Create, FileAccess.Write, FileShare.Read))
{
decryptedSource.CryptoStream.CopyTo(decryptedMemory);
decryptedMemory.Position = 0;
Program.log.Debug("decryption to memory done, decompressing to disc");
KeepLzmaCompressor.Decompress(decryptedMemory, target);
Program.log.Debug("decompression done");
}
}
}
finally
{
foreach (string file in _TempFilesToDelete)
{
Program.log.Debug("deleting temp file " + file);
File.Delete(file);
}
}
EnsurePathExists(ToPath);
Program.log.Debug("renaming " + PendingPath + " -> " + ToPath);
File.Move(PendingPath, ToPath);
if (_Mode == ModeEnum.DecryptDecompress)
{
Program.log.Debug("restoring metadata " + ToPath);
FileInfo fi = new FileInfo(ToPath);
fi.LastWriteTimeUtc = SourceFile.LastWriteTimeUtc;
fi.CreationTimeUtc = SourceFile.CreationTimeUtc;
if (fi.Length != SourceFile.SizeBytes)
{
Program.log.Error("size mismatch " + ToPath + " " + SourceFile.Sha256);
}
//string sha = Hashing.GetHash(fi);
//if (sha != SourceFile.Sha256)
// Program.log.Error("hash mismatch " + ToPath + " " + sha + " but should be "+ SourceFile.Sha256);
}
}
public void HashFile()
{
_Sha256 = KeepHasher.GetHash(_FileInfo);
}
public void Analyze()
{
int errorCount = 0;
ILog log = Program.log;
log.Info("getting all files ... might take some time");
FileInfo[] allFiles = (from x in _Storage.PartitionManager.AllDirectories
from f in x.EnumerateFiles("*", SearchOption.AllDirectories)
select f).ToArray();
log.Info("done");
log.Info("parsing the sha256 from filename");
IEnumerable <FileInfo> files = allFiles.Where(x => x.Name.EndsWith(ObjectStorage.OBJ_EXTENSION, StringComparison.OrdinalIgnoreCase));
Dictionary <string, FileInfo> filesBySha256 = new Dictionary <string, FileInfo>();
foreach (FileInfo f in files)
{
string sha256 = f.Name.Substring(0, f.Name.Length - ObjectStorage.OBJ_EXTENSION.Length);
filesBySha256.Add(sha256, f);
}
log.Info("collecting all stored objects from manifest");
Dictionary <string, ManifestItem> manifestBySha256 = _Storage.Manifest.ManifestObjects.ToDictionary(x => x.Sha256source);
/// Phase 1
log.Info("check if all manifest files are present as files");
foreach (var x in manifestBySha256.Keys)
{
if (!filesBySha256.ContainsKey(x))
{
log.Error("missing file in storage, which is present in the manifest! " + x);
errorCount++;
}
}
/// Phase 2
log.Info("check if all files are present in the manifest");
foreach (var x in filesBySha256.Keys)
{
if (!manifestBySha256.ContainsKey(x))
{
log.Error("missing file in manifest, that was found in storage! " + x);
errorCount++;
}
}
/// Phase 3
log.Info("check size of all files with the manifest");
foreach (var hash in filesBySha256.Keys)
{
ManifestItem mf;
if (!manifestBySha256.TryGetValue(hash, out mf))
{
continue;
}
long size = mf.SizeTarget;
long size2 = filesBySha256[hash].Length;
if (size != size2)
{
log.Error("storage size mismatch! for " + hash);
errorCount++;
}
}
/// Phase 4
log.Info("look for pending files");
IEnumerable <FileInfo> pendingFiles = allFiles.Where(x => x.Name.EndsWith(ObjectStorage.PENDING_EXTENSION, StringComparison.OrdinalIgnoreCase));
foreach (var x in pendingFiles)
{
log.Error("found pending file " + x.FullName);
errorCount++;
}
/// Phase 5
log.Info("look for unknown files");
foreach (var x in allFiles)
{
if (!x.Name.EndsWith(ObjectStorage.OBJ_EXTENSION, StringComparison.OrdinalIgnoreCase) &&
!x.Name.EndsWith(ObjectStorage.PENDING_EXTENSION, StringComparison.OrdinalIgnoreCase))
{
log.Error("unknown file " + x.FullName);
errorCount++;
}
}
/// Phase 6
log.Info("look, if files from all inventories are present in manifest and nothing more");
FileInfo[] invFiles = _Storage.GetInventoryFiles();
HashSet <string> hashesFromAllInventories = new HashSet <string>();
foreach (FileInfo fi in invFiles)
{
log.Info(fi.FullName);
Inventory i = Inventory.FromXml(fi, _Configuration);
IList <string> shas = i.Folder.AllFiles.Select(x => x.Sha256).ToList();
foreach (string sha in shas)
{
hashesFromAllInventories.Add(sha);
if (!manifestBySha256.ContainsKey(sha))
{
log.Error("missing file in manifest from inventory " + fi.FullName + " " + sha);
errorCount++;
}
}
}
foreach (string sha in manifestBySha256.Keys)
{
if (!hashesFromAllInventories.Contains(sha))
{
log.Error("found file in manifest, which is in no inventory " + sha);
errorCount++;
}
}
ManifestItem[] ml = manifestBySha256.Values.ToArray();
int count = ml.Count();
if (!_Fast)
{
/// Phase 7
{
log.Info("test cecksums of all files in storage");
Action <ManifestItem> check = delegate(ManifestItem x)
{
string shaTarget = x.Sha256target;
FileInfo fi = new FileInfo(ObjectStorage.GetStorageTarget(x.Sha256source, _Storage.PartitionManager.GetDirByNumber(x.Partition)));
string shaFile = KeepHasher.GetHash(fi);
if (!string.Equals(shaTarget, shaFile, StringComparison.Ordinal))
{
log.Error("storage hash mismatch! for " + x.Sha256source);
Interlocked.Increment(ref errorCount);
}
};
if (ml.Length > 0)
{
ml.ParallelExecute(
x => x.SizeSource,
x => x.Sha256source,
x => x.Sha256target,
x => check(x),
6);
}
}
}
if (_Decompress)
{
// Phase 8 - SLOOWWW :-)
log.Info("actually decode file and check size and checksum");
for (int i = 0; i < count; i++)
{
double percentFiles = ((double)i / (double)count) * 100.0;
Program.log.Info(string.Format("file {0} / {1} ({2:0.00}%)", i + 1, count, percentFiles));
var x = ml[i];
log.Info("checking decompression and decryption for " + x.Sha256source);
FileInfo fi = new FileInfo(ObjectStorage.GetStorageTarget(x.Sha256source, _Storage.PartitionManager.GetDirByNumber(x.Partition)));
long sizeFile;
string shaFile = null;
using (Stream s = FileJob.JustDecryptAndDecompress(fi, _Configuration.Password, _Configuration.Salt))
{
sizeFile = s.Length;
shaFile = KeepHasher.GetHash(s);
}
if (!string.Equals(x.Sha256source, shaFile, StringComparison.Ordinal))
{
log.Error("decompressed and decrypted hash mismatch! for " + x.Sha256source);
errorCount++;
}
if (x.SizeSource != sizeFile)
{
log.Error("decompressed and decrypted size mismatch! for " + x.Sha256source);
errorCount++;
}
}
}
log.Info(errorCount + " errors !!");
}
