public static void CreateUpdatePackage(System.Security.Cryptography.RSACryptoServiceProvider key, string inputfolder, string outputfolder, string manifest = null)
{
// Read the existing manifest
UpdateInfo remoteManifest;
var manifestpath = manifest ?? System.IO.Path.Combine(inputfolder, UPDATE_MANIFEST_FILENAME);
using (var s = System.IO.File.OpenRead(manifestpath))
using (var sr = new System.IO.StreamReader(s))
using (var jr = new Newtonsoft.Json.JsonTextReader(sr))
remoteManifest = new Newtonsoft.Json.JsonSerializer().Deserialize <UpdateInfo>(jr);
if (remoteManifest.Files == null)
{
remoteManifest.Files = new FileEntry[0];
}
if (remoteManifest.ReleaseTime.Ticks == 0)
{
remoteManifest.ReleaseTime = DateTime.UtcNow;
}
var ignoreFiles = (from n in remoteManifest.Files
where n.Ignore
select n).ToArray();
var ignoreMap = ignoreFiles.ToDictionary(k => k.Path, k => "", Duplicati.Library.Utility.Utility.ClientFilenameStringComparer);
remoteManifest.MD5 = null;
remoteManifest.SHA256 = null;
remoteManifest.Files = null;
remoteManifest.UncompressedSize = 0;
var localManifest = remoteManifest.Clone();
localManifest.RemoteURLS = null;
inputfolder = Duplicati.Library.Utility.Utility.AppendDirSeparator(inputfolder);
var baselen = inputfolder.Length;
var dirsep = System.IO.Path.DirectorySeparatorChar.ToString();
ignoreMap.Add(UPDATE_MANIFEST_FILENAME, "");
var md5 = System.Security.Cryptography.MD5.Create();
var sha256 = System.Security.Cryptography.SHA256.Create();
Func <string, string> computeMD5 = (path) =>
{
md5.Initialize();
using (var fs = System.IO.File.OpenRead(path))
return(Convert.ToBase64String(md5.ComputeHash(fs)));
};
Func <string, string> computeSHA256 = (path) =>
{
sha256.Initialize();
using (var fs = System.IO.File.OpenRead(path))
return(Convert.ToBase64String(sha256.ComputeHash(fs)));
};
// Build a zip
using (var archive_temp = new Duplicati.Library.Utility.TempFile())
{
using (var zipfile = new Duplicati.Library.Compression.FileArchiveZip(archive_temp, new Dictionary <string, string>()))
{
Func <string, string, bool> addToArchive = (path, relpath) =>
{
if (ignoreMap.ContainsKey(relpath))
{
return(false);
}
if (path.EndsWith(dirsep))
{
return(true);
}
using (var source = System.IO.File.OpenRead(path))
using (var target = zipfile.CreateFile(relpath,
Duplicati.Library.Interface.CompressionHint.Compressible,
System.IO.File.GetLastAccessTimeUtc(path)))
{
source.CopyTo(target);
remoteManifest.UncompressedSize += source.Length;
}
return(true);
};
// Build the update manifest
localManifest.Files =
(from fse in Duplicati.Library.Utility.Utility.EnumerateFileSystemEntries(inputfolder)
let relpath = fse.Substring(baselen)
where addToArchive(fse, relpath)
select new FileEntry()
{
Path = relpath,
LastWriteTime = System.IO.File.GetLastAccessTimeUtc(fse),
MD5 = fse.EndsWith(dirsep) ? null : computeMD5(fse),
SHA256 = fse.EndsWith(dirsep) ? null : computeSHA256(fse)
})
.Union(ignoreFiles).ToArray();
// Write a signed manifest with the files
using (var ms = new System.IO.MemoryStream())
using (var sw = new System.IO.StreamWriter(ms))
{
new Newtonsoft.Json.JsonSerializer().Serialize(sw, localManifest);
sw.Flush();
using (var ms2 = new System.IO.MemoryStream())
{
SignatureReadingStream.CreateSignedStream(ms, ms2, key);
ms2.Position = 0;
using (var sigfile = zipfile.CreateFile(UPDATE_MANIFEST_FILENAME,
Duplicati.Library.Interface.CompressionHint.Compressible,
DateTime.UtcNow))
ms2.CopyTo(sigfile);
}
}
}
remoteManifest.CompressedSize = new System.IO.FileInfo(archive_temp).Length;
remoteManifest.MD5 = computeMD5(archive_temp);
remoteManifest.SHA256 = computeSHA256(archive_temp);
System.IO.File.Move(archive_temp, System.IO.Path.Combine(outputfolder, "package.zip"));
}
// Write a signed manifest for upload
using (var tf = new Duplicati.Library.Utility.TempFile())
{
using (var ms = new System.IO.MemoryStream())
using (var sw = new System.IO.StreamWriter(ms))
{
new Newtonsoft.Json.JsonSerializer().Serialize(sw, remoteManifest);
sw.Flush();
using (var fs = System.IO.File.Create(tf))
SignatureReadingStream.CreateSignedStream(ms, fs, key);
}
System.IO.File.Move(tf, System.IO.Path.Combine(outputfolder, UPDATE_MANIFEST_FILENAME));
}
}
public static bool DownloadAndUnpackUpdate(UpdateInfo version, Action <double> progress = null)
{
if (INSTALLDIR == null)
{
return(false);
}
using (var tempfile = new Library.Utility.TempFile())
{
foreach (var url in version.RemoteURLS)
{
try
{
Action <long> cb = null;
if (progress != null)
{
cb = (s) => { progress(Math.Min(1.0, Math.Max(0.0, (double)s / version.CompressedSize))); }
}
;
var wreq = (System.Net.HttpWebRequest)System.Net.WebRequest.Create(url);
wreq.UserAgent = string.Format("{0} v{1}", APPNAME, SelfVersion.Version);
wreq.Headers.Add("X-Install-ID", InstallID);
using (var resp = wreq.GetResponse())
using (var rss = resp.GetResponseStream())
using (var pgs = new Duplicati.Library.Utility.ProgressReportingStream(rss, version.CompressedSize, cb))
using (var fs = System.IO.File.Open(tempfile, System.IO.FileMode.Create))
Duplicati.Library.Utility.Utility.CopyStream(pgs, fs);
var sha256 = System.Security.Cryptography.SHA256.Create();
var md5 = System.Security.Cryptography.MD5.Create();
using (var s = System.IO.File.OpenRead(tempfile))
{
if (s.Length != version.CompressedSize)
{
throw new Exception(string.Format("Invalid file size {0}, expected {1} for {2}", s.Length, version.CompressedSize, url));
}
var sha256hash = Convert.ToBase64String(sha256.ComputeHash(s));
if (sha256hash != version.SHA256)
{
throw new Exception(string.Format("Damaged or corrupted file, sha256 mismatch for {0}", url));
}
}
using (var s = System.IO.File.OpenRead(tempfile))
{
var md5hash = Convert.ToBase64String(md5.ComputeHash(s));
if (md5hash != version.MD5)
{
throw new Exception(string.Format("Damaged or corrupted file, md5 mismatch for {0}", url));
}
}
using (var tempfolder = new Duplicati.Library.Utility.TempFolder())
using (var zip = new Duplicati.Library.Compression.FileArchiveZip(tempfile, new Dictionary <string, string>()))
{
foreach (var file in zip.ListFilesWithSize(""))
{
if (System.IO.Path.IsPathRooted(file.Key) || file.Key.Trim().StartsWith("..", StringComparison.InvariantCultureIgnoreCase))
{
throw new Exception(string.Format("Out-of-place file path detected: {0}", file.Key));
}
var targetpath = System.IO.Path.Combine(tempfolder, file.Key);
var targetfolder = System.IO.Path.GetDirectoryName(targetpath);
if (!System.IO.Directory.Exists(targetfolder))
{
System.IO.Directory.CreateDirectory(targetfolder);
}
using (var zs = zip.OpenRead(file.Key))
using (var fs = System.IO.File.Create(targetpath))
zs.CopyTo(fs);
}
if (VerifyUnpackedFolder(tempfolder, version))
{
var versionstring = TryParseVersion(version.Version).ToString();
var targetfolder = System.IO.Path.Combine(INSTALLDIR, versionstring);
if (System.IO.Directory.Exists(targetfolder))
{
System.IO.Directory.Delete(targetfolder, true);
}
System.IO.Directory.CreateDirectory(targetfolder);
var tempfolderpath = Duplicati.Library.Utility.Utility.AppendDirSeparator(tempfolder);
var tempfolderlength = tempfolderpath.Length;
// Would be nice, but does not work :(
//System.IO.Directory.Move(tempfolder, targetfolder);
foreach (var e in Duplicati.Library.Utility.Utility.EnumerateFileSystemEntries(tempfolder))
{
var relpath = e.Substring(tempfolderlength);
if (string.IsNullOrWhiteSpace(relpath))
{
continue;
}
var fullpath = System.IO.Path.Combine(targetfolder, relpath);
if (relpath.EndsWith(System.IO.Path.DirectorySeparatorChar.ToString()))
{
System.IO.Directory.CreateDirectory(fullpath);
}
else
{
System.IO.File.Copy(e, fullpath);
}
}
// Verification will kick in when we list the installed updates
//VerifyUnpackedFolder(targetfolder, version);
System.IO.File.WriteAllText(System.IO.Path.Combine(INSTALLDIR, CURRENT_FILE), versionstring);
m_hasUpdateInstalled = null;
var obsolete = (from n in FindInstalledVersions()
where n.Value.Version != version.Version && n.Value.Version != SelfVersion.Version
let x = TryParseVersion(n.Value.Version)
orderby x descending
select n).Skip(1).ToArray();
foreach (var f in obsolete)
{
try { System.IO.Directory.Delete(f.Key, true); }
catch { }
}
return(true);
}
else
{
throw new Exception(string.Format("Unable to verify unpacked folder for url: {0}", url));
}
}
}
catch (Exception ex)
{
if (OnError != null)
{
OnError(ex);
}
}
}
}
return(false);
}
public static void CreateUpdatePackage(System.Security.Cryptography.RSACryptoServiceProvider key, string inputfolder, string outputfolder, string manifest = null)
{
// Read the existing manifest
UpdateInfo remoteManifest;
var manifestpath = manifest ?? System.IO.Path.Combine(inputfolder, UPDATE_MANIFEST_FILENAME);
using(var s = System.IO.File.OpenRead(manifestpath))
using(var sr = new System.IO.StreamReader(s))
using(var jr = new Newtonsoft.Json.JsonTextReader(sr))
remoteManifest = new Newtonsoft.Json.JsonSerializer().Deserialize<UpdateInfo>(jr);
if (remoteManifest.Files == null)
remoteManifest.Files = new FileEntry[0];
if (remoteManifest.ReleaseTime.Ticks == 0)
remoteManifest.ReleaseTime = DateTime.UtcNow;
var ignoreFiles = (from n in remoteManifest.Files
where n.Ignore
select n).ToArray();
var ignoreMap = ignoreFiles.ToDictionary(k => k.Path, k => "", Duplicati.Library.Utility.Utility.ClientFilenameStringComparer);
remoteManifest.MD5 = null;
remoteManifest.SHA256 = null;
remoteManifest.Files = null;
remoteManifest.UncompressedSize = 0;
var localManifest = remoteManifest.Clone();
localManifest.RemoteURLS = null;
inputfolder = Duplicati.Library.Utility.Utility.AppendDirSeparator(inputfolder);
var baselen = inputfolder.Length;
var dirsep = System.IO.Path.DirectorySeparatorChar.ToString();
ignoreMap.Add(UPDATE_MANIFEST_FILENAME, "");
var md5 = System.Security.Cryptography.MD5.Create();
var sha256 = System.Security.Cryptography.SHA256.Create();
Func<string, string> computeMD5 = (path) =>
{
md5.Initialize();
using(var fs = System.IO.File.OpenRead(path))
return Convert.ToBase64String(md5.ComputeHash(fs));
};
Func<string, string> computeSHA256 = (path) =>
{
sha256.Initialize();
using(var fs = System.IO.File.OpenRead(path))
return Convert.ToBase64String(sha256.ComputeHash(fs));
};
// Build a zip
using (var archive_temp = new Duplicati.Library.Utility.TempFile())
{
using (var zipfile = new Duplicati.Library.Compression.FileArchiveZip(archive_temp, new Dictionary<string, string>()))
{
Func<string, string, bool> addToArchive = (path, relpath) =>
{
if (ignoreMap.ContainsKey(relpath))
return false;
if (path.EndsWith(dirsep))
return true;
using (var source = System.IO.File.OpenRead(path))
using (var target = zipfile.CreateFile(relpath,
Duplicati.Library.Interface.CompressionHint.Compressible,
System.IO.File.GetLastAccessTimeUtc(path)))
{
source.CopyTo(target);
remoteManifest.UncompressedSize += source.Length;
}
return true;
};
// Build the update manifest
localManifest.Files =
(from fse in Duplicati.Library.Utility.Utility.EnumerateFileSystemEntries(inputfolder)
let relpath = fse.Substring(baselen)
where addToArchive(fse, relpath)
select new FileEntry() {
Path = relpath,
LastWriteTime = System.IO.File.GetLastAccessTimeUtc(fse),
MD5 = fse.EndsWith(dirsep) ? null : computeMD5(fse),
SHA256 = fse.EndsWith(dirsep) ? null : computeSHA256(fse)
})
.Union(ignoreFiles).ToArray();
// Write a signed manifest with the files
using (var ms = new System.IO.MemoryStream())
using (var sw = new System.IO.StreamWriter(ms))
{
new Newtonsoft.Json.JsonSerializer().Serialize(sw, localManifest);
sw.Flush();
using (var ms2 = new System.IO.MemoryStream())
{
SignatureReadingStream.CreateSignedStream(ms, ms2, key);
ms2.Position = 0;
using (var sigfile = zipfile.CreateFile(UPDATE_MANIFEST_FILENAME,
Duplicati.Library.Interface.CompressionHint.Compressible,
DateTime.UtcNow))
ms2.CopyTo(sigfile);
}
}
}
remoteManifest.CompressedSize = new System.IO.FileInfo(archive_temp).Length;
remoteManifest.MD5 = computeMD5(archive_temp);
remoteManifest.SHA256 = computeSHA256(archive_temp);
System.IO.File.Move(archive_temp, System.IO.Path.Combine(outputfolder, "package.zip"));
}
// Write a signed manifest for upload
using(var tf = new Duplicati.Library.Utility.TempFile())
{
using (var ms = new System.IO.MemoryStream())
using (var sw = new System.IO.StreamWriter(ms))
{
new Newtonsoft.Json.JsonSerializer().Serialize(sw, remoteManifest);
sw.Flush();
using (var fs = System.IO.File.Create(tf))
SignatureReadingStream.CreateSignedStream(ms, fs, key);
}
System.IO.File.Move(tf, System.IO.Path.Combine(outputfolder, UPDATE_MANIFEST_FILENAME));
}
}
public static bool DownloadAndUnpackUpdate(UpdateInfo version, Action<double> progress = null)
{
if (INSTALLDIR == null)
return false;
var updates = version.RemoteURLS.ToList();
// If alternate update URLs are specified,
// we look for packages there as well
if (AutoUpdateSettings.UsesAlternateURLs)
{
var packagepath = new Library.Utility.Uri(updates[0]).Path;
var packagename = packagepath.Split('/').Last();
foreach(var alt_url in AutoUpdateSettings.URLs.Reverse())
{
var alt_uri = new Library.Utility.Uri(alt_url);
var path_components = alt_uri.Path.Split('/');
var path = string.Join("/", path_components.Take(path_components.Count() - 1).Union(new string[] { packagename}));
var new_path = alt_uri.SetPath(path);
updates.Insert(0, new_path.ToString());
}
}
using(var tempfile = new Library.Utility.TempFile())
{
foreach(var url in updates)
{
try
{
Action<long> cb = null;
if (progress != null)
cb = (s) => { progress(Math.Min(1.0, Math.Max(0.0, (double)s / version.CompressedSize))); };
var wreq = (System.Net.HttpWebRequest)System.Net.WebRequest.Create(url);
wreq.UserAgent = string.Format("{0} v{1}", APPNAME, SelfVersion.Version);
wreq.Headers.Add("X-Install-ID", InstallID);
using(var resp = wreq.GetResponse())
using(var rss = resp.GetResponseStream())
using(var pgs = new Duplicati.Library.Utility.ProgressReportingStream(rss, version.CompressedSize, cb))
using(var fs = System.IO.File.Open(tempfile, System.IO.FileMode.Create))
Duplicati.Library.Utility.Utility.CopyStream(pgs, fs);
var sha256 = System.Security.Cryptography.SHA256.Create();
var md5 = System.Security.Cryptography.MD5.Create();
using(var s = System.IO.File.OpenRead(tempfile))
{
if (s.Length != version.CompressedSize)
throw new Exception(string.Format("Invalid file size {0}, expected {1} for {2}", s.Length, version.CompressedSize, url));
var sha256hash = Convert.ToBase64String(sha256.ComputeHash(s));
if (sha256hash != version.SHA256)
throw new Exception(string.Format("Damaged or corrupted file, sha256 mismatch for {0}", url));
}
using(var s = System.IO.File.OpenRead(tempfile))
{
var md5hash = Convert.ToBase64String(md5.ComputeHash(s));
if (md5hash != version.MD5)
throw new Exception(string.Format("Damaged or corrupted file, md5 mismatch for {0}", url));
}
using(var tempfolder = new Duplicati.Library.Utility.TempFolder())
using(var zip = new Duplicati.Library.Compression.FileArchiveZip(tempfile, new Dictionary<string, string>()))
{
foreach(var file in zip.ListFilesWithSize(""))
{
if (System.IO.Path.IsPathRooted(file.Key) || file.Key.Trim().StartsWith("..", StringComparison.InvariantCultureIgnoreCase))
throw new Exception(string.Format("Out-of-place file path detected: {0}", file.Key));
var targetpath = System.IO.Path.Combine(tempfolder, file.Key);
var targetfolder = System.IO.Path.GetDirectoryName(targetpath);
if (!System.IO.Directory.Exists(targetfolder))
System.IO.Directory.CreateDirectory(targetfolder);
using(var zs = zip.OpenRead(file.Key))
using(var fs = System.IO.File.Create(targetpath))
zs.CopyTo(fs);
}
if (VerifyUnpackedFolder(tempfolder, version))
{
var versionstring = TryParseVersion(version.Version).ToString();
var targetfolder = System.IO.Path.Combine(INSTALLDIR, versionstring);
if (System.IO.Directory.Exists(targetfolder))
System.IO.Directory.Delete(targetfolder, true);
System.IO.Directory.CreateDirectory(targetfolder);
var tempfolderpath = Duplicati.Library.Utility.Utility.AppendDirSeparator(tempfolder);
var tempfolderlength = tempfolderpath.Length;
// Would be nice, but does not work :(
//System.IO.Directory.Move(tempfolder, targetfolder);
foreach(var e in Duplicati.Library.Utility.Utility.EnumerateFileSystemEntries(tempfolder))
{
var relpath = e.Substring(tempfolderlength);
if (string.IsNullOrWhiteSpace(relpath))
continue;
var fullpath = System.IO.Path.Combine(targetfolder, relpath);
if (relpath.EndsWith(System.IO.Path.DirectorySeparatorChar.ToString()))
System.IO.Directory.CreateDirectory(fullpath);
else
System.IO.File.Copy(e, fullpath);
}
// Verification will kick in when we list the installed updates
//VerifyUnpackedFolder(targetfolder, version);
System.IO.File.WriteAllText(System.IO.Path.Combine(INSTALLDIR, CURRENT_FILE), versionstring);
m_hasUpdateInstalled = null;
var obsolete = (from n in FindInstalledVersions()
where n.Value.Version != version.Version && n.Value.Version != SelfVersion.Version
let x = TryParseVersion(n.Value.Version)
orderby x descending
select n).Skip(1).ToArray();
foreach(var f in obsolete)
try { System.IO.Directory.Delete(f.Key, true); }
catch { }
return true;
}
else
{
throw new Exception(string.Format("Unable to verify unpacked folder for url: {0}", url));
}
}
}
catch(Exception ex)
{
if (OnError != null)
OnError(ex);
}
}
}
return false;
}
public static void Main(string[] _args)
{
var args = new List <string>(_args);
var opts = Duplicati.Library.Utility.CommandLineParser.ExtractOptions(args, null);
if (args == null || args.Count == 0)
{
Console.WriteLine("Usage:");
Console.WriteLine("{0} <filename.zip>", System.Reflection.Assembly.GetEntryAssembly().Location);
return;
}
Duplicati.Library.Logging.Log.LogLevel = Duplicati.Library.Logging.LogMessageType.Profiling;
Duplicati.Library.Logging.Log.CurrentLog = new Duplicati.Library.Logging.StreamLog(Console.OpenStandardOutput());
var filecount = 0;
var errorcount = 0;
foreach (var file in args)
{
if (!File.Exists(file))
{
Console.WriteLine("File not found: {0}", file);
continue;
}
if (new FileInfo(file).Length == 0)
{
Console.WriteLine("File is emtpy: {0}", file);
continue;
}
Console.WriteLine("Opening zip file {0}", file);
var errors = false;
try
{
filecount++;
using (var zr = new Duplicati.Library.Compression.FileArchiveZip(file, opts))
{
var files = zr.ListFilesWithSize(null).ToArray();
Console.WriteLine("Found {0} files in archive, testing read-ability for each", files.Length);
for (var i = 0; i < files.Length; i++)
{
Console.Write("Opening file #{0} - {1}", i + 1, files[i].Key);
try
{
using (var ms = new MemoryStream())
using (var sr = zr.OpenRead(files[i].Key))
{
sr.CopyTo(ms);
if (ms.Length == files[i].Value)
{
Console.WriteLine(" -- success");
}
else
{
Console.WriteLine(" -- bad length: {0} vs {1}", ms.Length, files[i].Value);
}
}
}
catch (Exception ex)
{
errors = true;
Console.WriteLine(" -- failed: {0}", ex);
}
}
}
if (errors)
{
Console.WriteLine("Found errors while processing file {0}", file);
errorcount++;
}
else
{
Console.WriteLine("Processed file {0} with no errors", file);
}
}
catch (Exception ex)
{
errorcount++;
Console.WriteLine("Open failed: {0}", ex);
}
}
Console.Write("Processed {0} zip files", filecount);
if (errorcount == 0)
{
Console.WriteLine(" without errors");
}
else
{
Console.WriteLine(" and found {0} errors", errorcount);
}
}
