public static void CreateUpdatePackage(System.Security.Cryptography.RSACryptoServiceProvider key, string inputfolder, string outputfolder, string manifest = null) { // Read the existing manifest UpdateInfo remoteManifest; var manifestpath = manifest ?? System.IO.Path.Combine(inputfolder, UPDATE_MANIFEST_FILENAME); using (var s = System.IO.File.OpenRead(manifestpath)) using (var sr = new System.IO.StreamReader(s)) using (var jr = new Newtonsoft.Json.JsonTextReader(sr)) remoteManifest = new Newtonsoft.Json.JsonSerializer().Deserialize <UpdateInfo>(jr); if (remoteManifest.Files == null) { remoteManifest.Files = new FileEntry[0]; } if (remoteManifest.ReleaseTime.Ticks == 0) { remoteManifest.ReleaseTime = DateTime.UtcNow; } var ignoreFiles = (from n in remoteManifest.Files where n.Ignore select n).ToArray(); var ignoreMap = ignoreFiles.ToDictionary(k => k.Path, k => "", Duplicati.Library.Utility.Utility.ClientFilenameStringComparer); remoteManifest.MD5 = null; remoteManifest.SHA256 = null; remoteManifest.Files = null; remoteManifest.UncompressedSize = 0; var localManifest = remoteManifest.Clone(); localManifest.RemoteURLS = null; inputfolder = Duplicati.Library.Utility.Utility.AppendDirSeparator(inputfolder); var baselen = inputfolder.Length; var dirsep = System.IO.Path.DirectorySeparatorChar.ToString(); ignoreMap.Add(UPDATE_MANIFEST_FILENAME, ""); var md5 = System.Security.Cryptography.MD5.Create(); var sha256 = System.Security.Cryptography.SHA256.Create(); Func <string, string> computeMD5 = (path) => { md5.Initialize(); using (var fs = System.IO.File.OpenRead(path)) return(Convert.ToBase64String(md5.ComputeHash(fs))); }; Func <string, string> computeSHA256 = (path) => { sha256.Initialize(); using (var fs = System.IO.File.OpenRead(path)) return(Convert.ToBase64String(sha256.ComputeHash(fs))); }; // Build a zip using (var archive_temp = new Duplicati.Library.Utility.TempFile()) { using (var zipfile = new Duplicati.Library.Compression.FileArchiveZip(archive_temp, new Dictionary <string, string>())) { Func <string, string, bool> addToArchive = (path, relpath) => { if (ignoreMap.ContainsKey(relpath)) { return(false); } if (path.EndsWith(dirsep)) { return(true); } using (var source = System.IO.File.OpenRead(path)) using (var target = zipfile.CreateFile(relpath, Duplicati.Library.Interface.CompressionHint.Compressible, System.IO.File.GetLastAccessTimeUtc(path))) { source.CopyTo(target); remoteManifest.UncompressedSize += source.Length; } return(true); }; // Build the update manifest localManifest.Files = (from fse in Duplicati.Library.Utility.Utility.EnumerateFileSystemEntries(inputfolder) let relpath = fse.Substring(baselen) where addToArchive(fse, relpath) select new FileEntry() { Path = relpath, LastWriteTime = System.IO.File.GetLastAccessTimeUtc(fse), MD5 = fse.EndsWith(dirsep) ? null : computeMD5(fse), SHA256 = fse.EndsWith(dirsep) ? null : computeSHA256(fse) }) .Union(ignoreFiles).ToArray(); // Write a signed manifest with the files using (var ms = new System.IO.MemoryStream()) using (var sw = new System.IO.StreamWriter(ms)) { new Newtonsoft.Json.JsonSerializer().Serialize(sw, localManifest); sw.Flush(); using (var ms2 = new System.IO.MemoryStream()) { SignatureReadingStream.CreateSignedStream(ms, ms2, key); ms2.Position = 0; using (var sigfile = zipfile.CreateFile(UPDATE_MANIFEST_FILENAME, Duplicati.Library.Interface.CompressionHint.Compressible, DateTime.UtcNow)) ms2.CopyTo(sigfile); } } } remoteManifest.CompressedSize = new System.IO.FileInfo(archive_temp).Length; remoteManifest.MD5 = computeMD5(archive_temp); remoteManifest.SHA256 = computeSHA256(archive_temp); System.IO.File.Move(archive_temp, System.IO.Path.Combine(outputfolder, "package.zip")); } // Write a signed manifest for upload using (var tf = new Duplicati.Library.Utility.TempFile()) { using (var ms = new System.IO.MemoryStream()) using (var sw = new System.IO.StreamWriter(ms)) { new Newtonsoft.Json.JsonSerializer().Serialize(sw, remoteManifest); sw.Flush(); using (var fs = System.IO.File.Create(tf)) SignatureReadingStream.CreateSignedStream(ms, fs, key); } System.IO.File.Move(tf, System.IO.Path.Combine(outputfolder, UPDATE_MANIFEST_FILENAME)); } }
public static bool DownloadAndUnpackUpdate(UpdateInfo version, Action <double> progress = null) { if (INSTALLDIR == null) { return(false); } using (var tempfile = new Library.Utility.TempFile()) { foreach (var url in version.RemoteURLS) { try { Action <long> cb = null; if (progress != null) { cb = (s) => { progress(Math.Min(1.0, Math.Max(0.0, (double)s / version.CompressedSize))); } } ; var wreq = (System.Net.HttpWebRequest)System.Net.WebRequest.Create(url); wreq.UserAgent = string.Format("{0} v{1}", APPNAME, SelfVersion.Version); wreq.Headers.Add("X-Install-ID", InstallID); using (var resp = wreq.GetResponse()) using (var rss = resp.GetResponseStream()) using (var pgs = new Duplicati.Library.Utility.ProgressReportingStream(rss, version.CompressedSize, cb)) using (var fs = System.IO.File.Open(tempfile, System.IO.FileMode.Create)) Duplicati.Library.Utility.Utility.CopyStream(pgs, fs); var sha256 = System.Security.Cryptography.SHA256.Create(); var md5 = System.Security.Cryptography.MD5.Create(); using (var s = System.IO.File.OpenRead(tempfile)) { if (s.Length != version.CompressedSize) { throw new Exception(string.Format("Invalid file size {0}, expected {1} for {2}", s.Length, version.CompressedSize, url)); } var sha256hash = Convert.ToBase64String(sha256.ComputeHash(s)); if (sha256hash != version.SHA256) { throw new Exception(string.Format("Damaged or corrupted file, sha256 mismatch for {0}", url)); } } using (var s = System.IO.File.OpenRead(tempfile)) { var md5hash = Convert.ToBase64String(md5.ComputeHash(s)); if (md5hash != version.MD5) { throw new Exception(string.Format("Damaged or corrupted file, md5 mismatch for {0}", url)); } } using (var tempfolder = new Duplicati.Library.Utility.TempFolder()) using (var zip = new Duplicati.Library.Compression.FileArchiveZip(tempfile, new Dictionary <string, string>())) { foreach (var file in zip.ListFilesWithSize("")) { if (System.IO.Path.IsPathRooted(file.Key) || file.Key.Trim().StartsWith("..", StringComparison.InvariantCultureIgnoreCase)) { throw new Exception(string.Format("Out-of-place file path detected: {0}", file.Key)); } var targetpath = System.IO.Path.Combine(tempfolder, file.Key); var targetfolder = System.IO.Path.GetDirectoryName(targetpath); if (!System.IO.Directory.Exists(targetfolder)) { System.IO.Directory.CreateDirectory(targetfolder); } using (var zs = zip.OpenRead(file.Key)) using (var fs = System.IO.File.Create(targetpath)) zs.CopyTo(fs); } if (VerifyUnpackedFolder(tempfolder, version)) { var versionstring = TryParseVersion(version.Version).ToString(); var targetfolder = System.IO.Path.Combine(INSTALLDIR, versionstring); if (System.IO.Directory.Exists(targetfolder)) { System.IO.Directory.Delete(targetfolder, true); } System.IO.Directory.CreateDirectory(targetfolder); var tempfolderpath = Duplicati.Library.Utility.Utility.AppendDirSeparator(tempfolder); var tempfolderlength = tempfolderpath.Length; // Would be nice, but does not work :( //System.IO.Directory.Move(tempfolder, targetfolder); foreach (var e in Duplicati.Library.Utility.Utility.EnumerateFileSystemEntries(tempfolder)) { var relpath = e.Substring(tempfolderlength); if (string.IsNullOrWhiteSpace(relpath)) { continue; } var fullpath = System.IO.Path.Combine(targetfolder, relpath); if (relpath.EndsWith(System.IO.Path.DirectorySeparatorChar.ToString())) { System.IO.Directory.CreateDirectory(fullpath); } else { System.IO.File.Copy(e, fullpath); } } // Verification will kick in when we list the installed updates //VerifyUnpackedFolder(targetfolder, version); System.IO.File.WriteAllText(System.IO.Path.Combine(INSTALLDIR, CURRENT_FILE), versionstring); m_hasUpdateInstalled = null; var obsolete = (from n in FindInstalledVersions() where n.Value.Version != version.Version && n.Value.Version != SelfVersion.Version let x = TryParseVersion(n.Value.Version) orderby x descending select n).Skip(1).ToArray(); foreach (var f in obsolete) { try { System.IO.Directory.Delete(f.Key, true); } catch { } } return(true); } else { throw new Exception(string.Format("Unable to verify unpacked folder for url: {0}", url)); } } } catch (Exception ex) { if (OnError != null) { OnError(ex); } } } } return(false); }
public static void CreateUpdatePackage(System.Security.Cryptography.RSACryptoServiceProvider key, string inputfolder, string outputfolder, string manifest = null) { // Read the existing manifest UpdateInfo remoteManifest; var manifestpath = manifest ?? System.IO.Path.Combine(inputfolder, UPDATE_MANIFEST_FILENAME); using(var s = System.IO.File.OpenRead(manifestpath)) using(var sr = new System.IO.StreamReader(s)) using(var jr = new Newtonsoft.Json.JsonTextReader(sr)) remoteManifest = new Newtonsoft.Json.JsonSerializer().Deserialize<UpdateInfo>(jr); if (remoteManifest.Files == null) remoteManifest.Files = new FileEntry[0]; if (remoteManifest.ReleaseTime.Ticks == 0) remoteManifest.ReleaseTime = DateTime.UtcNow; var ignoreFiles = (from n in remoteManifest.Files where n.Ignore select n).ToArray(); var ignoreMap = ignoreFiles.ToDictionary(k => k.Path, k => "", Duplicati.Library.Utility.Utility.ClientFilenameStringComparer); remoteManifest.MD5 = null; remoteManifest.SHA256 = null; remoteManifest.Files = null; remoteManifest.UncompressedSize = 0; var localManifest = remoteManifest.Clone(); localManifest.RemoteURLS = null; inputfolder = Duplicati.Library.Utility.Utility.AppendDirSeparator(inputfolder); var baselen = inputfolder.Length; var dirsep = System.IO.Path.DirectorySeparatorChar.ToString(); ignoreMap.Add(UPDATE_MANIFEST_FILENAME, ""); var md5 = System.Security.Cryptography.MD5.Create(); var sha256 = System.Security.Cryptography.SHA256.Create(); Func<string, string> computeMD5 = (path) => { md5.Initialize(); using(var fs = System.IO.File.OpenRead(path)) return Convert.ToBase64String(md5.ComputeHash(fs)); }; Func<string, string> computeSHA256 = (path) => { sha256.Initialize(); using(var fs = System.IO.File.OpenRead(path)) return Convert.ToBase64String(sha256.ComputeHash(fs)); }; // Build a zip using (var archive_temp = new Duplicati.Library.Utility.TempFile()) { using (var zipfile = new Duplicati.Library.Compression.FileArchiveZip(archive_temp, new Dictionary<string, string>())) { Func<string, string, bool> addToArchive = (path, relpath) => { if (ignoreMap.ContainsKey(relpath)) return false; if (path.EndsWith(dirsep)) return true; using (var source = System.IO.File.OpenRead(path)) using (var target = zipfile.CreateFile(relpath, Duplicati.Library.Interface.CompressionHint.Compressible, System.IO.File.GetLastAccessTimeUtc(path))) { source.CopyTo(target); remoteManifest.UncompressedSize += source.Length; } return true; }; // Build the update manifest localManifest.Files = (from fse in Duplicati.Library.Utility.Utility.EnumerateFileSystemEntries(inputfolder) let relpath = fse.Substring(baselen) where addToArchive(fse, relpath) select new FileEntry() { Path = relpath, LastWriteTime = System.IO.File.GetLastAccessTimeUtc(fse), MD5 = fse.EndsWith(dirsep) ? null : computeMD5(fse), SHA256 = fse.EndsWith(dirsep) ? null : computeSHA256(fse) }) .Union(ignoreFiles).ToArray(); // Write a signed manifest with the files using (var ms = new System.IO.MemoryStream()) using (var sw = new System.IO.StreamWriter(ms)) { new Newtonsoft.Json.JsonSerializer().Serialize(sw, localManifest); sw.Flush(); using (var ms2 = new System.IO.MemoryStream()) { SignatureReadingStream.CreateSignedStream(ms, ms2, key); ms2.Position = 0; using (var sigfile = zipfile.CreateFile(UPDATE_MANIFEST_FILENAME, Duplicati.Library.Interface.CompressionHint.Compressible, DateTime.UtcNow)) ms2.CopyTo(sigfile); } } } remoteManifest.CompressedSize = new System.IO.FileInfo(archive_temp).Length; remoteManifest.MD5 = computeMD5(archive_temp); remoteManifest.SHA256 = computeSHA256(archive_temp); System.IO.File.Move(archive_temp, System.IO.Path.Combine(outputfolder, "package.zip")); } // Write a signed manifest for upload using(var tf = new Duplicati.Library.Utility.TempFile()) { using (var ms = new System.IO.MemoryStream()) using (var sw = new System.IO.StreamWriter(ms)) { new Newtonsoft.Json.JsonSerializer().Serialize(sw, remoteManifest); sw.Flush(); using (var fs = System.IO.File.Create(tf)) SignatureReadingStream.CreateSignedStream(ms, fs, key); } System.IO.File.Move(tf, System.IO.Path.Combine(outputfolder, UPDATE_MANIFEST_FILENAME)); } }
public static bool DownloadAndUnpackUpdate(UpdateInfo version, Action<double> progress = null) { if (INSTALLDIR == null) return false; var updates = version.RemoteURLS.ToList(); // If alternate update URLs are specified, // we look for packages there as well if (AutoUpdateSettings.UsesAlternateURLs) { var packagepath = new Library.Utility.Uri(updates[0]).Path; var packagename = packagepath.Split('/').Last(); foreach(var alt_url in AutoUpdateSettings.URLs.Reverse()) { var alt_uri = new Library.Utility.Uri(alt_url); var path_components = alt_uri.Path.Split('/'); var path = string.Join("/", path_components.Take(path_components.Count() - 1).Union(new string[] { packagename})); var new_path = alt_uri.SetPath(path); updates.Insert(0, new_path.ToString()); } } using(var tempfile = new Library.Utility.TempFile()) { foreach(var url in updates) { try { Action<long> cb = null; if (progress != null) cb = (s) => { progress(Math.Min(1.0, Math.Max(0.0, (double)s / version.CompressedSize))); }; var wreq = (System.Net.HttpWebRequest)System.Net.WebRequest.Create(url); wreq.UserAgent = string.Format("{0} v{1}", APPNAME, SelfVersion.Version); wreq.Headers.Add("X-Install-ID", InstallID); using(var resp = wreq.GetResponse()) using(var rss = resp.GetResponseStream()) using(var pgs = new Duplicati.Library.Utility.ProgressReportingStream(rss, version.CompressedSize, cb)) using(var fs = System.IO.File.Open(tempfile, System.IO.FileMode.Create)) Duplicati.Library.Utility.Utility.CopyStream(pgs, fs); var sha256 = System.Security.Cryptography.SHA256.Create(); var md5 = System.Security.Cryptography.MD5.Create(); using(var s = System.IO.File.OpenRead(tempfile)) { if (s.Length != version.CompressedSize) throw new Exception(string.Format("Invalid file size {0}, expected {1} for {2}", s.Length, version.CompressedSize, url)); var sha256hash = Convert.ToBase64String(sha256.ComputeHash(s)); if (sha256hash != version.SHA256) throw new Exception(string.Format("Damaged or corrupted file, sha256 mismatch for {0}", url)); } using(var s = System.IO.File.OpenRead(tempfile)) { var md5hash = Convert.ToBase64String(md5.ComputeHash(s)); if (md5hash != version.MD5) throw new Exception(string.Format("Damaged or corrupted file, md5 mismatch for {0}", url)); } using(var tempfolder = new Duplicati.Library.Utility.TempFolder()) using(var zip = new Duplicati.Library.Compression.FileArchiveZip(tempfile, new Dictionary<string, string>())) { foreach(var file in zip.ListFilesWithSize("")) { if (System.IO.Path.IsPathRooted(file.Key) || file.Key.Trim().StartsWith("..", StringComparison.InvariantCultureIgnoreCase)) throw new Exception(string.Format("Out-of-place file path detected: {0}", file.Key)); var targetpath = System.IO.Path.Combine(tempfolder, file.Key); var targetfolder = System.IO.Path.GetDirectoryName(targetpath); if (!System.IO.Directory.Exists(targetfolder)) System.IO.Directory.CreateDirectory(targetfolder); using(var zs = zip.OpenRead(file.Key)) using(var fs = System.IO.File.Create(targetpath)) zs.CopyTo(fs); } if (VerifyUnpackedFolder(tempfolder, version)) { var versionstring = TryParseVersion(version.Version).ToString(); var targetfolder = System.IO.Path.Combine(INSTALLDIR, versionstring); if (System.IO.Directory.Exists(targetfolder)) System.IO.Directory.Delete(targetfolder, true); System.IO.Directory.CreateDirectory(targetfolder); var tempfolderpath = Duplicati.Library.Utility.Utility.AppendDirSeparator(tempfolder); var tempfolderlength = tempfolderpath.Length; // Would be nice, but does not work :( //System.IO.Directory.Move(tempfolder, targetfolder); foreach(var e in Duplicati.Library.Utility.Utility.EnumerateFileSystemEntries(tempfolder)) { var relpath = e.Substring(tempfolderlength); if (string.IsNullOrWhiteSpace(relpath)) continue; var fullpath = System.IO.Path.Combine(targetfolder, relpath); if (relpath.EndsWith(System.IO.Path.DirectorySeparatorChar.ToString())) System.IO.Directory.CreateDirectory(fullpath); else System.IO.File.Copy(e, fullpath); } // Verification will kick in when we list the installed updates //VerifyUnpackedFolder(targetfolder, version); System.IO.File.WriteAllText(System.IO.Path.Combine(INSTALLDIR, CURRENT_FILE), versionstring); m_hasUpdateInstalled = null; var obsolete = (from n in FindInstalledVersions() where n.Value.Version != version.Version && n.Value.Version != SelfVersion.Version let x = TryParseVersion(n.Value.Version) orderby x descending select n).Skip(1).ToArray(); foreach(var f in obsolete) try { System.IO.Directory.Delete(f.Key, true); } catch { } return true; } else { throw new Exception(string.Format("Unable to verify unpacked folder for url: {0}", url)); } } } catch(Exception ex) { if (OnError != null) OnError(ex); } } } return false; }
public static void Main(string[] _args) { var args = new List <string>(_args); var opts = Duplicati.Library.Utility.CommandLineParser.ExtractOptions(args, null); if (args == null || args.Count == 0) { Console.WriteLine("Usage:"); Console.WriteLine("{0} <filename.zip>", System.Reflection.Assembly.GetEntryAssembly().Location); return; } Duplicati.Library.Logging.Log.LogLevel = Duplicati.Library.Logging.LogMessageType.Profiling; Duplicati.Library.Logging.Log.CurrentLog = new Duplicati.Library.Logging.StreamLog(Console.OpenStandardOutput()); var filecount = 0; var errorcount = 0; foreach (var file in args) { if (!File.Exists(file)) { Console.WriteLine("File not found: {0}", file); continue; } if (new FileInfo(file).Length == 0) { Console.WriteLine("File is emtpy: {0}", file); continue; } Console.WriteLine("Opening zip file {0}", file); var errors = false; try { filecount++; using (var zr = new Duplicati.Library.Compression.FileArchiveZip(file, opts)) { var files = zr.ListFilesWithSize(null).ToArray(); Console.WriteLine("Found {0} files in archive, testing read-ability for each", files.Length); for (var i = 0; i < files.Length; i++) { Console.Write("Opening file #{0} - {1}", i + 1, files[i].Key); try { using (var ms = new MemoryStream()) using (var sr = zr.OpenRead(files[i].Key)) { sr.CopyTo(ms); if (ms.Length == files[i].Value) { Console.WriteLine(" -- success"); } else { Console.WriteLine(" -- bad length: {0} vs {1}", ms.Length, files[i].Value); } } } catch (Exception ex) { errors = true; Console.WriteLine(" -- failed: {0}", ex); } } } if (errors) { Console.WriteLine("Found errors while processing file {0}", file); errorcount++; } else { Console.WriteLine("Processed file {0} with no errors", file); } } catch (Exception ex) { errorcount++; Console.WriteLine("Open failed: {0}", ex); } } Console.Write("Processed {0} zip files", filecount); if (errorcount == 0) { Console.WriteLine(" without errors"); } else { Console.WriteLine(" and found {0} errors", errorcount); } }