static byte[] HashOf(FileDetail file)
{
using (var md5 = MD5.Create())
using (var stream = NativeIO.OpenFileStream(file.PathInfo, FileAccess.Read))
{
return (md5.ComputeHash(stream));
}
}
static void CopyLength(Stream fs, string dstFilePath, long fileLength, IEnumerable <byte> expectedHash)
{
const int bufSz = 65536;
var remain = fileLength;
var buffer = new byte[bufSz];
using (var md5 = MD5.Create())
using (var fout = NativeIO.OpenFileStream(new PathInfo(dstFilePath), FileAccess.Write, FileMode.CreateNew))
{
int len;
while (remain > bufSz)
{
len = fs.Read(buffer, 0, bufSz);
if (len != bufSz)
{
throw new Exception("Malformed file: data truncated");
}
md5.TransformBlock(buffer, 0, len, null, 0);
fout.Write(buffer, 0, bufSz);
remain -= bufSz;
}
if (remain != 0)
{
len = fs.Read(buffer, 0, (int)remain);
if (len != remain)
{
throw new Exception("Malformed file: data truncated at end");
}
md5.TransformBlock(buffer, 0, (int)remain, null, 0);
fout.Write(buffer, 0, (int)remain);
}
md5.TransformFinalBlock(new byte[0], 0, 0);
if (!HashesEqual(expectedHash, md5.Hash))
{
throw new Exception("Damaged archive: File at " + dstFilePath + " failed a checksum");
}
}
}
public static void files_can_be_created_and_written_and_read_and_copied_in_very_long_paths()
{
// Create a >255 length path
const string path = TempRoot + "\\QIO\\Pseudopseudohypoparathyroidism\\Pneumonoultramicroscopicsilicovolcanoconiosis\\Floccinaucinihilipilification\\Antidisestablishmentarianism\\Honorificabilitudinitatibus\\Donaudampfschiffahrtselektrizitätenhauptbetriebswerkbauunterbeamtengesellschaft";
NativeIO.CreateDirectory(new PathInfo(path), recursive: true);
var sampleData = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8 };
var srcFile = new PathInfo(path + "\\example.file.txt");
var dstFile = new PathInfo(path + "\\example.copy.txt");
// write a file
using (var fs = NativeIO.OpenFileStream(srcFile, FileAccess.Write, FileMode.Create, FileShare.None)) {
fs.Write(sampleData, 4, 4);
fs.Write(sampleData, 0, 4);
fs.Flush();
}
// copy the file elsewhere
Assert.True(NativeIO.Exists(srcFile), "Source file can't be found (didn't write correctly?)");
Assert.False(NativeIO.SymbolicLink.IsSymLink(srcFile), "File was a sym-link");
Assert.True(NativeIO.CopyFile(srcFile, dstFile), "Failed to copy file");
Assert.True(NativeIO.Exists(dstFile), "Target file can't be found");
// Check the contents
using (var fs = NativeIO.OpenFileStream(srcFile, FileAccess.Read))
{
var buf = new byte[8];
var length = fs.Read(buf, 0, 8);
Assert.That(length, Is.EqualTo(8));
Assert.That(buf, Is.EquivalentTo(new byte[] { 5, 6, 7, 8, 1, 2, 3, 4 }));
}
// cleanup
NativeIO.DeleteDirectory(new DirectoryDetail(TempRoot + "\\QIO"), recursive: true);
}
/// <summary>
/// Read all files under the source path into a single destination file.
/// </summary>
/// <param name="srcPath">Full path of source directory</param>
/// <param name="dstFilePath">Full path of destination file</param>
/// <param name="signingCertPath">PFX file containing </param>
/// <param name="certPassword">password securing the pfx file</param>
public static void FolderToFile(string srcPath, string dstFilePath, string signingCertPath = null, string certPassword = null) {
// list out name+hash -> [path]
// write this to a single file
// gzip that file
var tmpPath = dstFilePath+".tmp";
var filePaths = new Dictionary<string, PathList>();
var symLinks = new Dictionary<string, string>(); // link path -> target path
// find distinct files
var files = NativeIO.EnumerateFiles(new PathInfo(srcPath).FullNameUnc, (symPath, targetPath)=>{
if (IsSubpath(srcPath, targetPath))
{
symLinks.Add(symPath, targetPath);
return false;
}
return true;
}, searchOption: SearchOption.AllDirectories);
foreach (var file in files)
{
var hash = HashOf(file);
Add(hash, file, filePaths);
}
// pack everything into a temp file
if (File.Exists(tmpPath)) File.Delete(tmpPath);
using (var fs = File.OpenWrite(tmpPath))
{
// Write data files
foreach (var fileKey in filePaths.Keys)
{
var pathList = filePaths[fileKey];
var catPaths = Encoding.UTF8.GetBytes(string.Join("|", Filter(pathList.Paths, srcPath)));
// Write <MD5:16 bytes>
fs.Write(pathList.HashData, 0, 16);
// Write <length:8 bytes><paths:utf8 str>
WriteLength(catPaths.Length, fs);
fs.Write(catPaths, 0, catPaths.Length);
var info = NativeIO.ReadFileDetails(new PathInfo(pathList.Paths[0]));
// Write <length:8 bytes><data:byte array>
WriteLength((long)info.Length, fs);
using (var inf = NativeIO.OpenFileStream(info.PathInfo, FileAccess.Read)) inf.CopyTo(fs);
fs.Flush();
}
// Write symbolic links
foreach (var linkSrc in symLinks.Keys)
{
var linkTarget = symLinks[linkSrc];
var linkData = Encoding.UTF8.GetBytes(string.Join("|", Filter(new[] { linkSrc, linkTarget }, srcPath)));
// Write <zeros:16 bytes>
WriteLength(0, fs);
WriteLength(0, fs);
// Write <length:8 bytes><path pair:utf8 str>, path pair is 'src|target'
WriteLength(linkData.Length, fs);
fs.Write(linkData, 0, linkData.Length);
// Write <length:8 bytes>, always zero (there is not file content in a link)
WriteLength(0, fs);
}
fs.Flush();
}
// If cert, write to *another* temp file with the signing header in place
if ( ! string.IsNullOrWhiteSpace(signingCertPath)) {
var tmpSignPath = tmpPath + ".signed";
try
{
using (var cat = File.OpenRead(tmpPath))
{
var signingBytes = Crypto.BuildSigningHeader(cat,signingCertPath, certPassword);
cat.Seek(0, SeekOrigin.Begin);
using (var final = File.Open(tmpSignPath, FileMode.Create, FileAccess.Write)) {
final.Write(signingBytes, 0, signingBytes.Length);
cat.CopyTo(final);
final.Flush();
}
}
File.Delete(tmpPath); // wipe the old one
File.Move(tmpSignPath, tmpPath); // use the new one for compression
} catch (Exception ex) {
Console.WriteLine("Signing failed: "+ex);
throw;
}
}
// Compress the file
if (File.Exists(dstFilePath)) File.Delete(dstFilePath);
using (var compressing = new GZipStream(File.OpenWrite(dstFilePath), CompressionLevel.Optimal))
using (var cat = File.OpenRead(tmpPath))
{
cat.CopyTo(compressing, 65536);
compressing.Flush();
}
// Kill the temp file
File.Delete(tmpPath);
}
