public static bool DownloadAndUnpackUpdate(UpdateInfo version, Action <double> progress = null)
{
if (INSTALLDIR == null)
{
return(false);
}
using (var tempfile = new Library.Utility.TempFile())
{
foreach (var url in version.RemoteURLS)
{
try
{
Action <long> cb = null;
if (progress != null)
{
cb = (s) => { progress(Math.Min(1.0, Math.Max(0.0, (double)s / version.CompressedSize))); }
}
;
var wreq = (System.Net.HttpWebRequest)System.Net.WebRequest.Create(url);
wreq.UserAgent = string.Format("{0} v{1}", APPNAME, SelfVersion.Version);
wreq.Headers.Add("X-Install-ID", InstallID);
using (var resp = wreq.GetResponse())
using (var rss = resp.GetResponseStream())
using (var pgs = new Duplicati.Library.Utility.ProgressReportingStream(rss, version.CompressedSize, cb))
using (var fs = System.IO.File.Open(tempfile, System.IO.FileMode.Create))
Duplicati.Library.Utility.Utility.CopyStream(pgs, fs);
var sha256 = System.Security.Cryptography.SHA256.Create();
var md5 = System.Security.Cryptography.MD5.Create();
using (var s = System.IO.File.OpenRead(tempfile))
{
if (s.Length != version.CompressedSize)
{
throw new Exception(string.Format("Invalid file size {0}, expected {1} for {2}", s.Length, version.CompressedSize, url));
}
var sha256hash = Convert.ToBase64String(sha256.ComputeHash(s));
if (sha256hash != version.SHA256)
{
throw new Exception(string.Format("Damaged or corrupted file, sha256 mismatch for {0}", url));
}
}
using (var s = System.IO.File.OpenRead(tempfile))
{
var md5hash = Convert.ToBase64String(md5.ComputeHash(s));
if (md5hash != version.MD5)
{
throw new Exception(string.Format("Damaged or corrupted file, md5 mismatch for {0}", url));
}
}
using (var tempfolder = new Duplicati.Library.Utility.TempFolder())
using (var zip = new Duplicati.Library.Compression.FileArchiveZip(tempfile, new Dictionary <string, string>()))
{
foreach (var file in zip.ListFilesWithSize(""))
{
if (System.IO.Path.IsPathRooted(file.Key) || file.Key.Trim().StartsWith("..", StringComparison.InvariantCultureIgnoreCase))
{
throw new Exception(string.Format("Out-of-place file path detected: {0}", file.Key));
}
var targetpath = System.IO.Path.Combine(tempfolder, file.Key);
var targetfolder = System.IO.Path.GetDirectoryName(targetpath);
if (!System.IO.Directory.Exists(targetfolder))
{
System.IO.Directory.CreateDirectory(targetfolder);
}
using (var zs = zip.OpenRead(file.Key))
using (var fs = System.IO.File.Create(targetpath))
zs.CopyTo(fs);
}
if (VerifyUnpackedFolder(tempfolder, version))
{
var versionstring = TryParseVersion(version.Version).ToString();
var targetfolder = System.IO.Path.Combine(INSTALLDIR, versionstring);
if (System.IO.Directory.Exists(targetfolder))
{
System.IO.Directory.Delete(targetfolder, true);
}
System.IO.Directory.CreateDirectory(targetfolder);
var tempfolderpath = Duplicati.Library.Utility.Utility.AppendDirSeparator(tempfolder);
var tempfolderlength = tempfolderpath.Length;
// Would be nice, but does not work :(
//System.IO.Directory.Move(tempfolder, targetfolder);
foreach (var e in Duplicati.Library.Utility.Utility.EnumerateFileSystemEntries(tempfolder))
{
var relpath = e.Substring(tempfolderlength);
if (string.IsNullOrWhiteSpace(relpath))
{
continue;
}
var fullpath = System.IO.Path.Combine(targetfolder, relpath);
if (relpath.EndsWith(System.IO.Path.DirectorySeparatorChar.ToString()))
{
System.IO.Directory.CreateDirectory(fullpath);
}
else
{
System.IO.File.Copy(e, fullpath);
}
}
// Verification will kick in when we list the installed updates
//VerifyUnpackedFolder(targetfolder, version);
System.IO.File.WriteAllText(System.IO.Path.Combine(INSTALLDIR, CURRENT_FILE), versionstring);
m_hasUpdateInstalled = null;
var obsolete = (from n in FindInstalledVersions()
where n.Value.Version != version.Version && n.Value.Version != SelfVersion.Version
let x = TryParseVersion(n.Value.Version)
orderby x descending
select n).Skip(1).ToArray();
foreach (var f in obsolete)
{
try { System.IO.Directory.Delete(f.Key, true); }
catch { }
}
return(true);
}
else
{
throw new Exception(string.Format("Unable to verify unpacked folder for url: {0}", url));
}
}
}
catch (Exception ex)
{
if (OnError != null)
{
OnError(ex);
}
}
}
}
return(false);
}
private void GetInternal(BackupEntryBase remote, Manifestfile manifest, string filename, Manifestfile.HashEntry hash)
{
int retries = m_options.NumberOfRetries;
Exception lastEx = null;
m_statusmessage = string.Format(Strings.BackendWrapper.StatusMessageDownloading, remote.Filename);
do
{
try
{
if (manifest != null && !string.IsNullOrEmpty(m_options.SignatureCachePath) && hash != null && remote is SignatureEntry)
{
string cachefilename = FindCacheEntry(remote as SignatureEntry);
if (cachefilename != null && System.IO.File.Exists(cachefilename))
{
if ((hash.Size < 0 || new System.IO.FileInfo(cachefilename).Length == hash.Size) && Utility.Utility.CalculateHash(cachefilename) == hash.Hash)
{
if (manifest.Version > 2 && !string.IsNullOrEmpty(remote.EncryptionMode))
{
try
{
using (Library.Interface.IEncryption enc = DynamicLoader.EncryptionLoader.GetModule(remote.EncryptionMode, m_options.Passphrase, m_options.RawOptions))
enc.Decrypt(cachefilename, filename);
return;
}
catch (Exception ex)
{
m_statistics.LogWarning(string.Format(Strings.BackendWrapper.CachedSignatureDecryptWarning, cachefilename, ex.Message), null);
try { System.IO.File.Delete(cachefilename); }
catch { }
}
}
else
{
//TODO: Don't copy, but just return it as write protected
System.IO.File.Copy(cachefilename, filename, true);
return;
}
}
else
{
m_statistics.LogWarning(string.Format(Strings.BackendWrapper.CachedSignatureHashMismatchWarning, cachefilename), null);
try { System.IO.File.Delete(cachefilename); }
catch { }
}
}
}
Utility.TempFile tempfile = null;
try
{
if (!string.IsNullOrEmpty(remote.EncryptionMode))
tempfile = new Duplicati.Library.Utility.TempFile();
else
tempfile = new Duplicati.Library.Utility.TempFile(filename);
ResetBackend();
m_statistics.AddNumberOfRemoteCalls(1);
if (m_backend is Duplicati.Library.Interface.IStreamingBackend && !m_options.DisableStreamingTransfers)
{
using (System.IO.FileStream fs = System.IO.File.Open(tempfile, System.IO.FileMode.Create, System.IO.FileAccess.Write, System.IO.FileShare.None))
using (Utility.ProgressReportingStream pgs = new Duplicati.Library.Utility.ProgressReportingStream(fs, remote.Fileentry.Size))
using (Utility.ThrottledStream ts = new Duplicati.Library.Utility.ThrottledStream(pgs, m_options.MaxUploadPrSecond, m_options.MaxDownloadPrSecond))
{
pgs.Progress += new Duplicati.Library.Utility.ProgressReportingStream.ProgressDelegate(pgs_Progress);
ts.Callback += new Duplicati.Library.Utility.ThrottledStream.ThrottledStreamCallback(ThrottledStream_Callback);
((Duplicati.Library.Interface.IStreamingBackend)m_backend).Get(remote.Filename, ts);
}
}
else
{
if (!m_async && ProgressEvent != null)
ProgressEvent(50, m_statusmessage);
m_backend.Get(remote.Filename, tempfile);
if (!m_async && ProgressEvent != null)
ProgressEvent(100, m_statusmessage);
}
//This is required so we are sure that the file was downloaded completely and not partially,
// as any exception here will cause a retry, but using a partial file may cause random errors
if (remote.Fileentry.Size > 0 && remote.Fileentry.Size != new System.IO.FileInfo(tempfile).Length)
throw new Exception(string.Format(Strings.BackendWrapper.DownloadedFileSizeError, remote.Filename, remote.Fileentry.Size, new System.IO.FileInfo(tempfile).Length));
remote.RemoteHash = Utility.Utility.CalculateHash(tempfile);
//Manifest version 3 has hashes WITH encryption
if (manifest != null && manifest.Version > 2)
{
if (hash != null && remote.RemoteHash != hash.Hash)
throw new HashMismathcException(string.Format(Strings.BackendWrapper.HashMismatchError, remote.Filename, hash.Hash, Utility.Utility.CalculateHash(tempfile)));
if (!string.IsNullOrEmpty(m_options.SignatureCachePath) && remote is SignatureEntry)
{
string cachefilename = System.IO.Path.Combine(m_options.SignatureCachePath, m_cachefilenamestrategy.GenerateFilename(remote));
try { System.IO.File.Copy(tempfile, cachefilename, true); }
catch (Exception ex) { m_statistics.LogWarning(string.Format(Strings.BackendWrapper.SaveCacheFileError, cachefilename), ex); }
}
}
if (!string.IsNullOrEmpty(remote.EncryptionMode))
{
try
{
using (Library.Interface.IEncryption enc = DynamicLoader.EncryptionLoader.GetModule(remote.EncryptionMode, m_options.Passphrase, m_options.RawOptions))
enc.Decrypt(tempfile, filename);
}
catch (Exception ex)
{
//If we fail here, make sure that we throw a crypto exception
if (ex is System.Security.Cryptography.CryptographicException)
throw;
else
throw new System.Security.Cryptography.CryptographicException(ex.Message, ex);
}
tempfile.Dispose(); //Remove the encrypted file
//Wrap the new file as a temp file
tempfile = new Duplicati.Library.Utility.TempFile(filename);
}
//Manifest version 1+2 has hashes WITHOUT encryption
if (manifest != null && manifest.Version <= 2)
{
if (hash != null && Utility.Utility.CalculateHash(tempfile) != hash.Hash)
throw new HashMismathcException(string.Format(Strings.BackendWrapper.HashMismatchError, remote.Filename, hash.Hash, Utility.Utility.CalculateHash(tempfile)));
if (!string.IsNullOrEmpty(m_options.SignatureCachePath) && remote is SignatureEntry)
{
string cachefilename = System.IO.Path.Combine(m_options.SignatureCachePath, m_cachefilenamestrategy.GenerateFilename(remote));
try { System.IO.File.Copy(tempfile, cachefilename, true); }
catch (Exception ex) { m_statistics.LogWarning(string.Format(Strings.BackendWrapper.SaveCacheFileError, cachefilename), ex); }
}
}
lastEx = null;
tempfile.Protected = true; //Don't delete it
if (m_backendInterfaceLogger != null)
{
if (remote is ManifestEntry)
m_backendInterfaceLogger.RegisterGet(remote.Fileentry, true, System.IO.File.ReadAllText(tempfile));
else
m_backendInterfaceLogger.RegisterGet(remote.Fileentry, true, null);
}
}
finally
{
try
{
if (tempfile != null)
tempfile.Dispose();
}
catch
{
}
}
}
catch (System.Threading.ThreadAbortException tex)
{
if (m_backendInterfaceLogger != null)
m_backendInterfaceLogger.RegisterGet(remote.Fileentry, false, tex.ToString());
throw;
}
catch (Exception ex)
{
lastEx = ex;
m_statistics.LogRetryAttempt(ex.Message, ex);
if (m_backendInterfaceLogger != null)
m_backendInterfaceLogger.RegisterGet(remote.Fileentry, false, ex.ToString());
DisposeBackend();
retries--;
if (retries > 0 && m_options.RetryDelay.Ticks > 0)
System.Threading.Thread.Sleep(m_options.RetryDelay);
}
} while (lastEx != null && retries > 0);
if (lastEx != null)
if (lastEx is HashMismathcException)
throw lastEx;
else if (lastEx is System.Security.Cryptography.CryptographicException)
throw lastEx;
else
throw new Exception(string.Format(Strings.BackendWrapper.FileDownloadError2, filename, lastEx.Message), lastEx);
m_statistics.AddBytesDownloaded(new System.IO.FileInfo(filename).Length);
}
private void PutInternal(BackupEntryBase remote, string filename)
{
string remotename = remote.Filename;
m_statusmessage = string.Format(Strings.BackendWrapper.StatusMessageUploading, remotename, Utility.Utility.FormatSizeString(new System.IO.FileInfo(filename).Length));
Duplicati.Library.Interface.IFileEntry log_fe = new Duplicati.Library.Interface.FileEntry(remote.Filename, remote.Filesize, DateTime.Now, DateTime.Now);
long sourceFileSize = new System.IO.FileInfo(filename).Length;
try
{
int retries = m_options.NumberOfRetries;
bool success = false;
Exception lastEx = null;
do
{
try
{
ResetBackend();
m_statistics.AddNumberOfRemoteCalls(1);
if (m_backend is Library.Interface.IStreamingBackend && !m_options.DisableStreamingTransfers)
{
#if DEBUG_THROTTLE
DateTime begin = DateTime.Now;
#endif
using (System.IO.FileStream fs = System.IO.File.Open(filename, System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.Read))
using (Utility.ProgressReportingStream pgs = new Duplicati.Library.Utility.ProgressReportingStream(fs, fs.Length))
{
pgs.Progress += new Duplicati.Library.Utility.ProgressReportingStream.ProgressDelegate(pgs_Progress);
using (Utility.ThrottledStream ts = new Utility.ThrottledStream(pgs, m_options.MaxUploadPrSecond, m_options.MaxDownloadPrSecond))
{
ts.Callback += new Duplicati.Library.Utility.ThrottledStream.ThrottledStreamCallback(ThrottledStream_Callback);
((Library.Interface.IStreamingBackend)m_backend).Put(remotename, ts);
}
}
#if DEBUG_THROTTLE
TimeSpan duration = DateTime.Now - begin;
long size = new System.IO.FileInfo(encryptedFile).Length;
Console.WriteLine("Transferred " + Core.Utility.FormatSizeString(size) + " in " + duration.TotalSeconds.ToString() + ", yielding : " + ((size / (double)1024.0) / duration.TotalSeconds) + " kb/s");
#endif
}
else
{
if (ProgressEvent != null)
ProgressEvent(50, m_statusmessage);
m_backend.Put(remotename, filename);
if (ProgressEvent != null)
ProgressEvent(50, m_statusmessage);
}
if (m_options.ListVerifyUploads)
{
Library.Interface.FileEntry m = null;
foreach (Library.Interface.FileEntry fe in ListInternal())
if (fe.Name == remotename)
{
m = fe;
break;
}
if (m == null)
throw new Exception(string.Format(Strings.BackendWrapper.UploadVerificationFailure, remotename));
long size = new System.IO.FileInfo(filename).Length;
if (m.Size >= 0 && m.Size != size)
throw new Exception(string.Format(Strings.BackendWrapper.UploadSizeVerificationFailure, remotename, m.Size, size));
}
if (remote is SignatureEntry && !string.IsNullOrEmpty(m_options.SignatureCachePath) && System.IO.File.Exists(filename))
System.IO.File.Copy(filename, System.IO.Path.Combine(m_options.SignatureCachePath, m_cachefilenamestrategy.GenerateFilename(remote)), true);
if (remote is ManifestEntry)
{
if (m_queuelock != null)
{
lock (m_queuelock)
m_manifestUploads++;
}
else
m_manifestUploads++;
}
success = true;
lastEx = null;
if (m_backendInterfaceLogger != null)
m_backendInterfaceLogger.RegisterPut(log_fe, true, null);
}
catch (System.Threading.ThreadAbortException tex)
{
if (m_backendInterfaceLogger != null)
m_backendInterfaceLogger.RegisterPut(log_fe, false, tex.ToString());
throw;
}
catch (Exception ex)
{
if (m_backendInterfaceLogger != null)
m_backendInterfaceLogger.RegisterPut(log_fe, false, ex.ToString());
DisposeBackend();
lastEx = ex;
m_statistics.LogRetryAttempt(ex.Message, ex);
retries--;
if (retries > 0 && m_options.RetryDelay.Ticks > 0)
System.Threading.Thread.Sleep(m_options.RetryDelay);
}
} while (!success && retries > 0);
if (!success)
throw new Exception(string.Format(Strings.BackendWrapper.FileUploadError2, remotename, lastEx == null ? "<null>" : lastEx.Message), lastEx);
m_statistics.AddBytesUploaded(sourceFileSize);
}
finally
{
//We delete the file here, because the backend leaves the file,
//in the case where we use async methods
try
{
if (System.IO.File.Exists(filename))
System.IO.File.Delete(filename);
}
catch { }
try
{
if (ProgressEvent != null)
ProgressEvent(-1, "");
}
catch
{
}
}
}
public static bool DownloadAndUnpackUpdate(UpdateInfo version, Action<double> progress = null)
{
if (INSTALLDIR == null)
return false;
var updates = version.RemoteURLS.ToList();
// If alternate update URLs are specified,
// we look for packages there as well
if (AutoUpdateSettings.UsesAlternateURLs)
{
var packagepath = new Library.Utility.Uri(updates[0]).Path;
var packagename = packagepath.Split('/').Last();
foreach(var alt_url in AutoUpdateSettings.URLs.Reverse())
{
var alt_uri = new Library.Utility.Uri(alt_url);
var path_components = alt_uri.Path.Split('/');
var path = string.Join("/", path_components.Take(path_components.Count() - 1).Union(new string[] { packagename}));
var new_path = alt_uri.SetPath(path);
updates.Insert(0, new_path.ToString());
}
}
using(var tempfile = new Library.Utility.TempFile())
{
foreach(var url in updates)
{
try
{
Action<long> cb = null;
if (progress != null)
cb = (s) => { progress(Math.Min(1.0, Math.Max(0.0, (double)s / version.CompressedSize))); };
var wreq = (System.Net.HttpWebRequest)System.Net.WebRequest.Create(url);
wreq.UserAgent = string.Format("{0} v{1}", APPNAME, SelfVersion.Version);
wreq.Headers.Add("X-Install-ID", InstallID);
using(var resp = wreq.GetResponse())
using(var rss = resp.GetResponseStream())
using(var pgs = new Duplicati.Library.Utility.ProgressReportingStream(rss, version.CompressedSize, cb))
using(var fs = System.IO.File.Open(tempfile, System.IO.FileMode.Create))
Duplicati.Library.Utility.Utility.CopyStream(pgs, fs);
var sha256 = System.Security.Cryptography.SHA256.Create();
var md5 = System.Security.Cryptography.MD5.Create();
using(var s = System.IO.File.OpenRead(tempfile))
{
if (s.Length != version.CompressedSize)
throw new Exception(string.Format("Invalid file size {0}, expected {1} for {2}", s.Length, version.CompressedSize, url));
var sha256hash = Convert.ToBase64String(sha256.ComputeHash(s));
if (sha256hash != version.SHA256)
throw new Exception(string.Format("Damaged or corrupted file, sha256 mismatch for {0}", url));
}
using(var s = System.IO.File.OpenRead(tempfile))
{
var md5hash = Convert.ToBase64String(md5.ComputeHash(s));
if (md5hash != version.MD5)
throw new Exception(string.Format("Damaged or corrupted file, md5 mismatch for {0}", url));
}
using(var tempfolder = new Duplicati.Library.Utility.TempFolder())
using(var zip = new Duplicati.Library.Compression.FileArchiveZip(tempfile, new Dictionary<string, string>()))
{
foreach(var file in zip.ListFilesWithSize(""))
{
if (System.IO.Path.IsPathRooted(file.Key) || file.Key.Trim().StartsWith("..", StringComparison.InvariantCultureIgnoreCase))
throw new Exception(string.Format("Out-of-place file path detected: {0}", file.Key));
var targetpath = System.IO.Path.Combine(tempfolder, file.Key);
var targetfolder = System.IO.Path.GetDirectoryName(targetpath);
if (!System.IO.Directory.Exists(targetfolder))
System.IO.Directory.CreateDirectory(targetfolder);
using(var zs = zip.OpenRead(file.Key))
using(var fs = System.IO.File.Create(targetpath))
zs.CopyTo(fs);
}
if (VerifyUnpackedFolder(tempfolder, version))
{
var versionstring = TryParseVersion(version.Version).ToString();
var targetfolder = System.IO.Path.Combine(INSTALLDIR, versionstring);
if (System.IO.Directory.Exists(targetfolder))
System.IO.Directory.Delete(targetfolder, true);
System.IO.Directory.CreateDirectory(targetfolder);
var tempfolderpath = Duplicati.Library.Utility.Utility.AppendDirSeparator(tempfolder);
var tempfolderlength = tempfolderpath.Length;
// Would be nice, but does not work :(
//System.IO.Directory.Move(tempfolder, targetfolder);
foreach(var e in Duplicati.Library.Utility.Utility.EnumerateFileSystemEntries(tempfolder))
{
var relpath = e.Substring(tempfolderlength);
if (string.IsNullOrWhiteSpace(relpath))
continue;
var fullpath = System.IO.Path.Combine(targetfolder, relpath);
if (relpath.EndsWith(System.IO.Path.DirectorySeparatorChar.ToString()))
System.IO.Directory.CreateDirectory(fullpath);
else
System.IO.File.Copy(e, fullpath);
}
// Verification will kick in when we list the installed updates
//VerifyUnpackedFolder(targetfolder, version);
System.IO.File.WriteAllText(System.IO.Path.Combine(INSTALLDIR, CURRENT_FILE), versionstring);
m_hasUpdateInstalled = null;
var obsolete = (from n in FindInstalledVersions()
where n.Value.Version != version.Version && n.Value.Version != SelfVersion.Version
let x = TryParseVersion(n.Value.Version)
orderby x descending
select n).Skip(1).ToArray();
foreach(var f in obsolete)
try { System.IO.Directory.Delete(f.Key, true); }
catch { }
return true;
}
else
{
throw new Exception(string.Format("Unable to verify unpacked folder for url: {0}", url));
}
}
}
catch(Exception ex)
{
if (OnError != null)
OnError(ex);
}
}
}
return false;
}
