/// <summary>
/// Constructs a HashEntry from a remote file descriptor
/// </summary>
/// <param name="item">The remote file to mimic</param>
public HashEntry(BackupEntryBase item)
{
m_hash = item.RemoteHash;
m_name = item.Filename;
m_size = item.Filesize;
if (string.IsNullOrEmpty(m_hash) || m_size < 0 || string.IsNullOrEmpty(m_name))
throw new Exception(Strings.Manifestfile.InternalError);
}
public string GenerateFilename(BackupEntryBase type)
{
string t;
if (type is ManifestEntry && ((ManifestEntry)type).IsPrimary)
t = m_useShortFilenames ? MANIFEST_A_SHORT : MANIFEST_A;
else if (type is ManifestEntry && !((ManifestEntry)type).IsPrimary)
t = m_useShortFilenames ? MANIFEST_B_SHORT : MANIFEST_B;
else if (type is ContentEntry)
t = m_useShortFilenames ? CONTENT_SHORT : CONTENT;
else if (type is SignatureEntry)
t = m_useShortFilenames ? SIGNATURE_SHORT : SIGNATURE;
else if (type is VerificationEntry)
return m_prefix + "." + type.Time.ToUniversalTime().ToString(TIMESTAMP_FORMAT) + ".verification";
else if (type is DeleteTransactionEntry)
return m_prefix + "-" + DELETE_TRANSACTION_FILENAME;
else
throw new Exception(string.Format(Strings.FilenameStrategy.InvalidEntryTypeError, type));
string filename;
if (m_useShortFilenames)
{
filename = m_prefix + "-" + t + (type.IsFull ? FULL_SHORT : INCREMENTAL_SHORT) + (type.Time.ToUniversalTime().Ticks / TimeSpan.TicksPerSecond).ToString("X");
}
else
{
string datetime;
if (m_useOldFilenames)
{
//Make sure the same DateTime is always the same string
if (!m_timeStringCache.ContainsKey(type.Time))
m_timeStringCache[type.Time] = type.Time.ToString("yyyy-MM-ddTHH:mm:ssK");
datetime = m_timeStringCache[type.Time].Replace(":", m_timeSeparator ?? (Utility.Utility.IsClientLinux ? ":" : "'"));
}
else
{
datetime = type.Time.ToUniversalTime().ToString(TIMESTAMP_FORMAT);
}
filename = m_prefix + "-" + (type.IsFull ? FULL : INCREMENTAL) + "-" + t + "." + datetime;
}
if (type is ManifestEntry)
return filename;
else
return filename + "." + VOLUME + ((PayloadEntryBase)type).Volumenumber.ToString();
}
public void AddFile(BackupEntryBase file)
{
System.Xml.XmlNode f = m_node.AppendChild(m_doc.CreateElement("File"));
if (file is ManifestEntry)
f.Attributes.Append(m_doc.CreateAttribute("type")).Value = "manifest";
else if (file is SignatureEntry)
f.Attributes.Append(m_doc.CreateAttribute("type")).Value = "signature";
else if (file is ContentEntry)
f.Attributes.Append(m_doc.CreateAttribute("type")).Value = "content";
f.Attributes.Append(m_doc.CreateAttribute("name")).Value = file.Filename;
f.Attributes.Append(m_doc.CreateAttribute("size")).Value = file.Filesize.ToString();
f.InnerText = Utility.Utility.ByteArrayAsHexString(Convert.FromBase64String(file.RemoteHash));
}
public void Put(BackupEntryBase remote, string filename)
{
Put(remote, filename, false);
}
private void DeleteInternal(BackupEntryBase remote)
{
int retries = m_options.NumberOfRetries;
Exception lastEx = null;
do
{
try
{
ResetBackend();
m_statistics.AddNumberOfRemoteCalls(1);
m_backend.Delete(remote.Filename);
lastEx = null;
if (m_backendInterfaceLogger != null)
m_backendInterfaceLogger.RegisterDelete(remote.Fileentry, true, null);
}
catch (System.Threading.ThreadAbortException tex)
{
if (m_backendInterfaceLogger != null)
m_backendInterfaceLogger.RegisterDelete(remote.Fileentry, false, tex.ToString());
throw;
}
catch (Exception ex)
{
lastEx = ex;
m_statistics.LogRetryAttempt(ex.Message, ex);
if (m_backendInterfaceLogger != null)
m_backendInterfaceLogger.RegisterDelete(remote.Fileentry, false, ex.ToString());
DisposeBackend();
retries--;
if (retries > 0 && m_options.RetryDelay.Ticks > 0)
System.Threading.Thread.Sleep(m_options.RetryDelay);
}
} while (lastEx != null && retries > 0);
if (lastEx != null)
throw new Exception(string.Format(Strings.BackendWrapper.FileDeleteError2, remote.Filename, lastEx.Message), lastEx);
if (remote is SignatureEntry && !string.IsNullOrEmpty(m_options.SignatureCachePath))
{
try
{
string file = FindCacheEntry(remote as SignatureEntry);
try
{
while (file != null)
{
System.IO.File.Delete(file);
file = FindCacheEntry(remote as SignatureEntry);
}
}
catch (Exception ex)
{
m_statistics.LogWarning(string.Format(Strings.BackendWrapper.DeleteCacheFileError, file), ex);
}
}
catch {}
}
}
/// <summary>
/// Internal helper to consistenly name remote files beyond what the filenamestrategy supports
/// </summary>
/// <param name="remote">The entry to create a filename for</param>
/// <returns>A filename with extensions</returns>
public string GenerateFilename(BackupEntryBase remote)
{
string remotename = m_filenamestrategy.GenerateFilename(remote);
if (remote is ManifestEntry)
remotename += ".manifest";
else if (remote is VerificationEntry)
return remotename;
else if (!(remote is DeleteTransactionEntry))
remotename += "." + m_options.CompressionModule;
if (!m_options.NoEncryption)
{
if (m_encryption == null)
m_encryption = DynamicLoader.EncryptionLoader.GetModule(m_options.EncryptionModule, m_options.Passphrase, m_options.RawOptions);
remotename += "." + m_encryption.FilenameExtension;
}
return remotename;
}
/// <summary>
/// Gets a file from the remote store, verifies the hash and decrypts the content
/// </summary>
/// <param name="remote">The entry to get</param>
/// <param name="manifest">The manifest that protectes the file</param>
/// <param name="filename">The remote filename</param>
/// <param name="filehash">The hash of the remote file</param>
public void Get(BackupEntryBase remote, Manifestfile manifest, string filename, Manifestfile.HashEntry hash)
{
ProtectedInvoke("GetInternal", remote, manifest, filename, hash);
}
public void Delete(BackupEntryBase remote)
{
ProtectedInvoke("DeleteInternal", remote);
}
public void AddOrphan(BackupEntryBase entry)
{
if (m_orphans != null)
m_orphans.Add(entry);
else
Logging.Log.WriteMessage(string.Format(Strings.BackendWrapper.PartialFileFoundMessage, entry.Filename), Duplicati.Library.Logging.LogMessageType.Warning);
}
private void PutInternal(BackupEntryBase remote, string filename)
{
string remotename = remote.Filename;
m_statusmessage = string.Format(Strings.BackendWrapper.StatusMessageUploading, remotename, Utility.Utility.FormatSizeString(new System.IO.FileInfo(filename).Length));
Duplicati.Library.Interface.IFileEntry log_fe = new Duplicati.Library.Interface.FileEntry(remote.Filename, remote.Filesize, DateTime.Now, DateTime.Now);
long sourceFileSize = new System.IO.FileInfo(filename).Length;
try
{
int retries = m_options.NumberOfRetries;
bool success = false;
Exception lastEx = null;
do
{
try
{
ResetBackend();
m_statistics.AddNumberOfRemoteCalls(1);
if (m_backend is Library.Interface.IStreamingBackend && !m_options.DisableStreamingTransfers)
{
#if DEBUG_THROTTLE
DateTime begin = DateTime.Now;
#endif
using (System.IO.FileStream fs = System.IO.File.Open(filename, System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.Read))
using (Utility.ProgressReportingStream pgs = new Duplicati.Library.Utility.ProgressReportingStream(fs, fs.Length))
{
pgs.Progress += new Duplicati.Library.Utility.ProgressReportingStream.ProgressDelegate(pgs_Progress);
using (Utility.ThrottledStream ts = new Utility.ThrottledStream(pgs, m_options.MaxUploadPrSecond, m_options.MaxDownloadPrSecond))
{
ts.Callback += new Duplicati.Library.Utility.ThrottledStream.ThrottledStreamCallback(ThrottledStream_Callback);
((Library.Interface.IStreamingBackend)m_backend).Put(remotename, ts);
}
}
#if DEBUG_THROTTLE
TimeSpan duration = DateTime.Now - begin;
long size = new System.IO.FileInfo(encryptedFile).Length;
Console.WriteLine("Transferred " + Core.Utility.FormatSizeString(size) + " in " + duration.TotalSeconds.ToString() + ", yielding : " + ((size / (double)1024.0) / duration.TotalSeconds) + " kb/s");
#endif
}
else
{
if (ProgressEvent != null)
ProgressEvent(50, m_statusmessage);
m_backend.Put(remotename, filename);
if (ProgressEvent != null)
ProgressEvent(50, m_statusmessage);
}
if (m_options.ListVerifyUploads)
{
Library.Interface.FileEntry m = null;
foreach (Library.Interface.FileEntry fe in ListInternal())
if (fe.Name == remotename)
{
m = fe;
break;
}
if (m == null)
throw new Exception(string.Format(Strings.BackendWrapper.UploadVerificationFailure, remotename));
long size = new System.IO.FileInfo(filename).Length;
if (m.Size >= 0 && m.Size != size)
throw new Exception(string.Format(Strings.BackendWrapper.UploadSizeVerificationFailure, remotename, m.Size, size));
}
if (remote is SignatureEntry && !string.IsNullOrEmpty(m_options.SignatureCachePath) && System.IO.File.Exists(filename))
System.IO.File.Copy(filename, System.IO.Path.Combine(m_options.SignatureCachePath, m_cachefilenamestrategy.GenerateFilename(remote)), true);
if (remote is ManifestEntry)
{
if (m_queuelock != null)
{
lock (m_queuelock)
m_manifestUploads++;
}
else
m_manifestUploads++;
}
success = true;
lastEx = null;
if (m_backendInterfaceLogger != null)
m_backendInterfaceLogger.RegisterPut(log_fe, true, null);
}
catch (System.Threading.ThreadAbortException tex)
{
if (m_backendInterfaceLogger != null)
m_backendInterfaceLogger.RegisterPut(log_fe, false, tex.ToString());
throw;
}
catch (Exception ex)
{
if (m_backendInterfaceLogger != null)
m_backendInterfaceLogger.RegisterPut(log_fe, false, ex.ToString());
DisposeBackend();
lastEx = ex;
m_statistics.LogRetryAttempt(ex.Message, ex);
retries--;
if (retries > 0 && m_options.RetryDelay.Ticks > 0)
System.Threading.Thread.Sleep(m_options.RetryDelay);
}
} while (!success && retries > 0);
if (!success)
throw new Exception(string.Format(Strings.BackendWrapper.FileUploadError2, remotename, lastEx == null ? "<null>" : lastEx.Message), lastEx);
m_statistics.AddBytesUploaded(sourceFileSize);
}
finally
{
//We delete the file here, because the backend leaves the file,
//in the case where we use async methods
try
{
if (System.IO.File.Exists(filename))
System.IO.File.Delete(filename);
}
catch { }
try
{
if (ProgressEvent != null)
ProgressEvent(-1, "");
}
catch
{
}
}
}
private void Put(BackupEntryBase remote, string filename, bool forcesync)
{
if (!remote.IsEncrypted && !m_options.NoEncryption && remote as VerificationEntry == null)
{
if (m_encryption == null)
m_encryption = DynamicLoader.EncryptionLoader.GetModule(m_options.EncryptionModule, m_options.Passphrase, m_options.RawOptions);
using (Utility.TempFile raw = new Duplicati.Library.Utility.TempFile(filename))
using (Utility.TempFile enc = new Duplicati.Library.Utility.TempFile())
{
m_encryption.Encrypt(raw, enc);
filename = enc;
enc.Protected = true;
raw.Protected = false;
}
remote.IsEncrypted = true;
}
remote.RemoteHash = Utility.Utility.CalculateHash(filename);
remote.Filename = GenerateFilename(remote);
remote.Filesize = new System.IO.FileInfo(filename).Length;
if (!m_async)
PutInternal(remote, filename);
else
{
if (forcesync)
{
int count;
lock (m_queuelock)
count = m_pendingOperations.Count;
while (count > 0)
{
m_asyncItemProcessed.WaitOne(1000 * 5, false);
lock (m_queuelock)
count = m_pendingOperations.Count;
}
PutInternal(remote, filename);
}
else
{
bool waitForCompletion;
//There are 3 files in a volume (signature, content and manifest) + a verification file
int uploads_in_set = m_options.CreateVerificationFile ? 4 : 3;
lock (m_queuelock)
{
if (m_workerException != null)
throw m_workerException;
m_pendingOperations.Enqueue(new KeyValuePair<BackupEntryBase, string>(remote, filename));
m_asyncItemReady.Set();
waitForCompletion = m_options.AsynchronousUploadLimit > 0 && m_pendingOperations.Count > (m_options.AsynchronousUploadLimit * uploads_in_set);
}
while (waitForCompletion)
{
m_asyncItemProcessed.WaitOne(1000 * 5, false);
lock (m_queuelock)
{
if (m_workerException != null)
throw m_workerException;
waitForCompletion = m_options.AsynchronousUploadLimit > 0 && m_pendingOperations.Count > (m_options.AsynchronousUploadLimit * uploads_in_set);
}
}
}
}
}
private void GetInternal(BackupEntryBase remote, Manifestfile manifest, string filename, Manifestfile.HashEntry hash)
{
int retries = m_options.NumberOfRetries;
Exception lastEx = null;
m_statusmessage = string.Format(Strings.BackendWrapper.StatusMessageDownloading, remote.Filename);
do
{
try
{
if (manifest != null && !string.IsNullOrEmpty(m_options.SignatureCachePath) && hash != null && remote is SignatureEntry)
{
string cachefilename = FindCacheEntry(remote as SignatureEntry);
if (cachefilename != null && System.IO.File.Exists(cachefilename))
{
if ((hash.Size < 0 || new System.IO.FileInfo(cachefilename).Length == hash.Size) && Utility.Utility.CalculateHash(cachefilename) == hash.Hash)
{
if (manifest.Version > 2 && !string.IsNullOrEmpty(remote.EncryptionMode))
{
try
{
using (Library.Interface.IEncryption enc = DynamicLoader.EncryptionLoader.GetModule(remote.EncryptionMode, m_options.Passphrase, m_options.RawOptions))
enc.Decrypt(cachefilename, filename);
return;
}
catch (Exception ex)
{
m_statistics.LogWarning(string.Format(Strings.BackendWrapper.CachedSignatureDecryptWarning, cachefilename, ex.Message), null);
try { System.IO.File.Delete(cachefilename); }
catch { }
}
}
else
{
//TODO: Don't copy, but just return it as write protected
System.IO.File.Copy(cachefilename, filename, true);
return;
}
}
else
{
m_statistics.LogWarning(string.Format(Strings.BackendWrapper.CachedSignatureHashMismatchWarning, cachefilename), null);
try { System.IO.File.Delete(cachefilename); }
catch { }
}
}
}
Utility.TempFile tempfile = null;
try
{
if (!string.IsNullOrEmpty(remote.EncryptionMode))
tempfile = new Duplicati.Library.Utility.TempFile();
else
tempfile = new Duplicati.Library.Utility.TempFile(filename);
ResetBackend();
m_statistics.AddNumberOfRemoteCalls(1);
if (m_backend is Duplicati.Library.Interface.IStreamingBackend && !m_options.DisableStreamingTransfers)
{
using (System.IO.FileStream fs = System.IO.File.Open(tempfile, System.IO.FileMode.Create, System.IO.FileAccess.Write, System.IO.FileShare.None))
using (Utility.ProgressReportingStream pgs = new Duplicati.Library.Utility.ProgressReportingStream(fs, remote.Fileentry.Size))
using (Utility.ThrottledStream ts = new Duplicati.Library.Utility.ThrottledStream(pgs, m_options.MaxUploadPrSecond, m_options.MaxDownloadPrSecond))
{
pgs.Progress += new Duplicati.Library.Utility.ProgressReportingStream.ProgressDelegate(pgs_Progress);
ts.Callback += new Duplicati.Library.Utility.ThrottledStream.ThrottledStreamCallback(ThrottledStream_Callback);
((Duplicati.Library.Interface.IStreamingBackend)m_backend).Get(remote.Filename, ts);
}
}
else
{
if (!m_async && ProgressEvent != null)
ProgressEvent(50, m_statusmessage);
m_backend.Get(remote.Filename, tempfile);
if (!m_async && ProgressEvent != null)
ProgressEvent(100, m_statusmessage);
}
//This is required so we are sure that the file was downloaded completely and not partially,
// as any exception here will cause a retry, but using a partial file may cause random errors
if (remote.Fileentry.Size > 0 && remote.Fileentry.Size != new System.IO.FileInfo(tempfile).Length)
throw new Exception(string.Format(Strings.BackendWrapper.DownloadedFileSizeError, remote.Filename, remote.Fileentry.Size, new System.IO.FileInfo(tempfile).Length));
remote.RemoteHash = Utility.Utility.CalculateHash(tempfile);
//Manifest version 3 has hashes WITH encryption
if (manifest != null && manifest.Version > 2)
{
if (hash != null && remote.RemoteHash != hash.Hash)
throw new HashMismathcException(string.Format(Strings.BackendWrapper.HashMismatchError, remote.Filename, hash.Hash, Utility.Utility.CalculateHash(tempfile)));
if (!string.IsNullOrEmpty(m_options.SignatureCachePath) && remote is SignatureEntry)
{
string cachefilename = System.IO.Path.Combine(m_options.SignatureCachePath, m_cachefilenamestrategy.GenerateFilename(remote));
try { System.IO.File.Copy(tempfile, cachefilename, true); }
catch (Exception ex) { m_statistics.LogWarning(string.Format(Strings.BackendWrapper.SaveCacheFileError, cachefilename), ex); }
}
}
if (!string.IsNullOrEmpty(remote.EncryptionMode))
{
try
{
using (Library.Interface.IEncryption enc = DynamicLoader.EncryptionLoader.GetModule(remote.EncryptionMode, m_options.Passphrase, m_options.RawOptions))
enc.Decrypt(tempfile, filename);
}
catch (Exception ex)
{
//If we fail here, make sure that we throw a crypto exception
if (ex is System.Security.Cryptography.CryptographicException)
throw;
else
throw new System.Security.Cryptography.CryptographicException(ex.Message, ex);
}
tempfile.Dispose(); //Remove the encrypted file
//Wrap the new file as a temp file
tempfile = new Duplicati.Library.Utility.TempFile(filename);
}
//Manifest version 1+2 has hashes WITHOUT encryption
if (manifest != null && manifest.Version <= 2)
{
if (hash != null && Utility.Utility.CalculateHash(tempfile) != hash.Hash)
throw new HashMismathcException(string.Format(Strings.BackendWrapper.HashMismatchError, remote.Filename, hash.Hash, Utility.Utility.CalculateHash(tempfile)));
if (!string.IsNullOrEmpty(m_options.SignatureCachePath) && remote is SignatureEntry)
{
string cachefilename = System.IO.Path.Combine(m_options.SignatureCachePath, m_cachefilenamestrategy.GenerateFilename(remote));
try { System.IO.File.Copy(tempfile, cachefilename, true); }
catch (Exception ex) { m_statistics.LogWarning(string.Format(Strings.BackendWrapper.SaveCacheFileError, cachefilename), ex); }
}
}
lastEx = null;
tempfile.Protected = true; //Don't delete it
if (m_backendInterfaceLogger != null)
{
if (remote is ManifestEntry)
m_backendInterfaceLogger.RegisterGet(remote.Fileentry, true, System.IO.File.ReadAllText(tempfile));
else
m_backendInterfaceLogger.RegisterGet(remote.Fileentry, true, null);
}
}
finally
{
try
{
if (tempfile != null)
tempfile.Dispose();
}
catch
{
}
}
}
catch (System.Threading.ThreadAbortException tex)
{
if (m_backendInterfaceLogger != null)
m_backendInterfaceLogger.RegisterGet(remote.Fileentry, false, tex.ToString());
throw;
}
catch (Exception ex)
{
lastEx = ex;
m_statistics.LogRetryAttempt(ex.Message, ex);
if (m_backendInterfaceLogger != null)
m_backendInterfaceLogger.RegisterGet(remote.Fileentry, false, ex.ToString());
DisposeBackend();
retries--;
if (retries > 0 && m_options.RetryDelay.Ticks > 0)
System.Threading.Thread.Sleep(m_options.RetryDelay);
}
} while (lastEx != null && retries > 0);
if (lastEx != null)
if (lastEx is HashMismathcException)
throw lastEx;
else if (lastEx is System.Security.Cryptography.CryptographicException)
throw lastEx;
else
throw new Exception(string.Format(Strings.BackendWrapper.FileDownloadError2, filename, lastEx.Message), lastEx);
m_statistics.AddBytesDownloaded(new System.IO.FileInfo(filename).Length);
}