/// <summary> /// Constructs a HashEntry from a remote file descriptor /// </summary> /// <param name="item">The remote file to mimic</param> public HashEntry(BackupEntryBase item) { m_hash = item.RemoteHash; m_name = item.Filename; m_size = item.Filesize; if (string.IsNullOrEmpty(m_hash) || m_size < 0 || string.IsNullOrEmpty(m_name)) throw new Exception(Strings.Manifestfile.InternalError); }
public string GenerateFilename(BackupEntryBase type) { string t; if (type is ManifestEntry && ((ManifestEntry)type).IsPrimary) t = m_useShortFilenames ? MANIFEST_A_SHORT : MANIFEST_A; else if (type is ManifestEntry && !((ManifestEntry)type).IsPrimary) t = m_useShortFilenames ? MANIFEST_B_SHORT : MANIFEST_B; else if (type is ContentEntry) t = m_useShortFilenames ? CONTENT_SHORT : CONTENT; else if (type is SignatureEntry) t = m_useShortFilenames ? SIGNATURE_SHORT : SIGNATURE; else if (type is VerificationEntry) return m_prefix + "." + type.Time.ToUniversalTime().ToString(TIMESTAMP_FORMAT) + ".verification"; else if (type is DeleteTransactionEntry) return m_prefix + "-" + DELETE_TRANSACTION_FILENAME; else throw new Exception(string.Format(Strings.FilenameStrategy.InvalidEntryTypeError, type)); string filename; if (m_useShortFilenames) { filename = m_prefix + "-" + t + (type.IsFull ? FULL_SHORT : INCREMENTAL_SHORT) + (type.Time.ToUniversalTime().Ticks / TimeSpan.TicksPerSecond).ToString("X"); } else { string datetime; if (m_useOldFilenames) { //Make sure the same DateTime is always the same string if (!m_timeStringCache.ContainsKey(type.Time)) m_timeStringCache[type.Time] = type.Time.ToString("yyyy-MM-ddTHH:mm:ssK"); datetime = m_timeStringCache[type.Time].Replace(":", m_timeSeparator ?? (Utility.Utility.IsClientLinux ? ":" : "'")); } else { datetime = type.Time.ToUniversalTime().ToString(TIMESTAMP_FORMAT); } filename = m_prefix + "-" + (type.IsFull ? FULL : INCREMENTAL) + "-" + t + "." + datetime; } if (type is ManifestEntry) return filename; else return filename + "." + VOLUME + ((PayloadEntryBase)type).Volumenumber.ToString(); }
public void AddFile(BackupEntryBase file) { System.Xml.XmlNode f = m_node.AppendChild(m_doc.CreateElement("File")); if (file is ManifestEntry) f.Attributes.Append(m_doc.CreateAttribute("type")).Value = "manifest"; else if (file is SignatureEntry) f.Attributes.Append(m_doc.CreateAttribute("type")).Value = "signature"; else if (file is ContentEntry) f.Attributes.Append(m_doc.CreateAttribute("type")).Value = "content"; f.Attributes.Append(m_doc.CreateAttribute("name")).Value = file.Filename; f.Attributes.Append(m_doc.CreateAttribute("size")).Value = file.Filesize.ToString(); f.InnerText = Utility.Utility.ByteArrayAsHexString(Convert.FromBase64String(file.RemoteHash)); }
public void Put(BackupEntryBase remote, string filename) { Put(remote, filename, false); }
private void DeleteInternal(BackupEntryBase remote) { int retries = m_options.NumberOfRetries; Exception lastEx = null; do { try { ResetBackend(); m_statistics.AddNumberOfRemoteCalls(1); m_backend.Delete(remote.Filename); lastEx = null; if (m_backendInterfaceLogger != null) m_backendInterfaceLogger.RegisterDelete(remote.Fileentry, true, null); } catch (System.Threading.ThreadAbortException tex) { if (m_backendInterfaceLogger != null) m_backendInterfaceLogger.RegisterDelete(remote.Fileentry, false, tex.ToString()); throw; } catch (Exception ex) { lastEx = ex; m_statistics.LogRetryAttempt(ex.Message, ex); if (m_backendInterfaceLogger != null) m_backendInterfaceLogger.RegisterDelete(remote.Fileentry, false, ex.ToString()); DisposeBackend(); retries--; if (retries > 0 && m_options.RetryDelay.Ticks > 0) System.Threading.Thread.Sleep(m_options.RetryDelay); } } while (lastEx != null && retries > 0); if (lastEx != null) throw new Exception(string.Format(Strings.BackendWrapper.FileDeleteError2, remote.Filename, lastEx.Message), lastEx); if (remote is SignatureEntry && !string.IsNullOrEmpty(m_options.SignatureCachePath)) { try { string file = FindCacheEntry(remote as SignatureEntry); try { while (file != null) { System.IO.File.Delete(file); file = FindCacheEntry(remote as SignatureEntry); } } catch (Exception ex) { m_statistics.LogWarning(string.Format(Strings.BackendWrapper.DeleteCacheFileError, file), ex); } } catch {} } }
/// <summary> /// Internal helper to consistenly name remote files beyond what the filenamestrategy supports /// </summary> /// <param name="remote">The entry to create a filename for</param> /// <returns>A filename with extensions</returns> public string GenerateFilename(BackupEntryBase remote) { string remotename = m_filenamestrategy.GenerateFilename(remote); if (remote is ManifestEntry) remotename += ".manifest"; else if (remote is VerificationEntry) return remotename; else if (!(remote is DeleteTransactionEntry)) remotename += "." + m_options.CompressionModule; if (!m_options.NoEncryption) { if (m_encryption == null) m_encryption = DynamicLoader.EncryptionLoader.GetModule(m_options.EncryptionModule, m_options.Passphrase, m_options.RawOptions); remotename += "." + m_encryption.FilenameExtension; } return remotename; }
/// <summary> /// Gets a file from the remote store, verifies the hash and decrypts the content /// </summary> /// <param name="remote">The entry to get</param> /// <param name="manifest">The manifest that protectes the file</param> /// <param name="filename">The remote filename</param> /// <param name="filehash">The hash of the remote file</param> public void Get(BackupEntryBase remote, Manifestfile manifest, string filename, Manifestfile.HashEntry hash) { ProtectedInvoke("GetInternal", remote, manifest, filename, hash); }
public void Delete(BackupEntryBase remote) { ProtectedInvoke("DeleteInternal", remote); }
public void AddOrphan(BackupEntryBase entry) { if (m_orphans != null) m_orphans.Add(entry); else Logging.Log.WriteMessage(string.Format(Strings.BackendWrapper.PartialFileFoundMessage, entry.Filename), Duplicati.Library.Logging.LogMessageType.Warning); }
private void PutInternal(BackupEntryBase remote, string filename) { string remotename = remote.Filename; m_statusmessage = string.Format(Strings.BackendWrapper.StatusMessageUploading, remotename, Utility.Utility.FormatSizeString(new System.IO.FileInfo(filename).Length)); Duplicati.Library.Interface.IFileEntry log_fe = new Duplicati.Library.Interface.FileEntry(remote.Filename, remote.Filesize, DateTime.Now, DateTime.Now); long sourceFileSize = new System.IO.FileInfo(filename).Length; try { int retries = m_options.NumberOfRetries; bool success = false; Exception lastEx = null; do { try { ResetBackend(); m_statistics.AddNumberOfRemoteCalls(1); if (m_backend is Library.Interface.IStreamingBackend && !m_options.DisableStreamingTransfers) { #if DEBUG_THROTTLE DateTime begin = DateTime.Now; #endif using (System.IO.FileStream fs = System.IO.File.Open(filename, System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.Read)) using (Utility.ProgressReportingStream pgs = new Duplicati.Library.Utility.ProgressReportingStream(fs, fs.Length)) { pgs.Progress += new Duplicati.Library.Utility.ProgressReportingStream.ProgressDelegate(pgs_Progress); using (Utility.ThrottledStream ts = new Utility.ThrottledStream(pgs, m_options.MaxUploadPrSecond, m_options.MaxDownloadPrSecond)) { ts.Callback += new Duplicati.Library.Utility.ThrottledStream.ThrottledStreamCallback(ThrottledStream_Callback); ((Library.Interface.IStreamingBackend)m_backend).Put(remotename, ts); } } #if DEBUG_THROTTLE TimeSpan duration = DateTime.Now - begin; long size = new System.IO.FileInfo(encryptedFile).Length; Console.WriteLine("Transferred " + Core.Utility.FormatSizeString(size) + " in " + duration.TotalSeconds.ToString() + ", yielding : " + ((size / (double)1024.0) / duration.TotalSeconds) + " kb/s"); #endif } else { if (ProgressEvent != null) ProgressEvent(50, m_statusmessage); m_backend.Put(remotename, filename); if (ProgressEvent != null) ProgressEvent(50, m_statusmessage); } if (m_options.ListVerifyUploads) { Library.Interface.FileEntry m = null; foreach (Library.Interface.FileEntry fe in ListInternal()) if (fe.Name == remotename) { m = fe; break; } if (m == null) throw new Exception(string.Format(Strings.BackendWrapper.UploadVerificationFailure, remotename)); long size = new System.IO.FileInfo(filename).Length; if (m.Size >= 0 && m.Size != size) throw new Exception(string.Format(Strings.BackendWrapper.UploadSizeVerificationFailure, remotename, m.Size, size)); } if (remote is SignatureEntry && !string.IsNullOrEmpty(m_options.SignatureCachePath) && System.IO.File.Exists(filename)) System.IO.File.Copy(filename, System.IO.Path.Combine(m_options.SignatureCachePath, m_cachefilenamestrategy.GenerateFilename(remote)), true); if (remote is ManifestEntry) { if (m_queuelock != null) { lock (m_queuelock) m_manifestUploads++; } else m_manifestUploads++; } success = true; lastEx = null; if (m_backendInterfaceLogger != null) m_backendInterfaceLogger.RegisterPut(log_fe, true, null); } catch (System.Threading.ThreadAbortException tex) { if (m_backendInterfaceLogger != null) m_backendInterfaceLogger.RegisterPut(log_fe, false, tex.ToString()); throw; } catch (Exception ex) { if (m_backendInterfaceLogger != null) m_backendInterfaceLogger.RegisterPut(log_fe, false, ex.ToString()); DisposeBackend(); lastEx = ex; m_statistics.LogRetryAttempt(ex.Message, ex); retries--; if (retries > 0 && m_options.RetryDelay.Ticks > 0) System.Threading.Thread.Sleep(m_options.RetryDelay); } } while (!success && retries > 0); if (!success) throw new Exception(string.Format(Strings.BackendWrapper.FileUploadError2, remotename, lastEx == null ? "<null>" : lastEx.Message), lastEx); m_statistics.AddBytesUploaded(sourceFileSize); } finally { //We delete the file here, because the backend leaves the file, //in the case where we use async methods try { if (System.IO.File.Exists(filename)) System.IO.File.Delete(filename); } catch { } try { if (ProgressEvent != null) ProgressEvent(-1, ""); } catch { } } }
private void Put(BackupEntryBase remote, string filename, bool forcesync) { if (!remote.IsEncrypted && !m_options.NoEncryption && remote as VerificationEntry == null) { if (m_encryption == null) m_encryption = DynamicLoader.EncryptionLoader.GetModule(m_options.EncryptionModule, m_options.Passphrase, m_options.RawOptions); using (Utility.TempFile raw = new Duplicati.Library.Utility.TempFile(filename)) using (Utility.TempFile enc = new Duplicati.Library.Utility.TempFile()) { m_encryption.Encrypt(raw, enc); filename = enc; enc.Protected = true; raw.Protected = false; } remote.IsEncrypted = true; } remote.RemoteHash = Utility.Utility.CalculateHash(filename); remote.Filename = GenerateFilename(remote); remote.Filesize = new System.IO.FileInfo(filename).Length; if (!m_async) PutInternal(remote, filename); else { if (forcesync) { int count; lock (m_queuelock) count = m_pendingOperations.Count; while (count > 0) { m_asyncItemProcessed.WaitOne(1000 * 5, false); lock (m_queuelock) count = m_pendingOperations.Count; } PutInternal(remote, filename); } else { bool waitForCompletion; //There are 3 files in a volume (signature, content and manifest) + a verification file int uploads_in_set = m_options.CreateVerificationFile ? 4 : 3; lock (m_queuelock) { if (m_workerException != null) throw m_workerException; m_pendingOperations.Enqueue(new KeyValuePair<BackupEntryBase, string>(remote, filename)); m_asyncItemReady.Set(); waitForCompletion = m_options.AsynchronousUploadLimit > 0 && m_pendingOperations.Count > (m_options.AsynchronousUploadLimit * uploads_in_set); } while (waitForCompletion) { m_asyncItemProcessed.WaitOne(1000 * 5, false); lock (m_queuelock) { if (m_workerException != null) throw m_workerException; waitForCompletion = m_options.AsynchronousUploadLimit > 0 && m_pendingOperations.Count > (m_options.AsynchronousUploadLimit * uploads_in_set); } } } } }
private void GetInternal(BackupEntryBase remote, Manifestfile manifest, string filename, Manifestfile.HashEntry hash) { int retries = m_options.NumberOfRetries; Exception lastEx = null; m_statusmessage = string.Format(Strings.BackendWrapper.StatusMessageDownloading, remote.Filename); do { try { if (manifest != null && !string.IsNullOrEmpty(m_options.SignatureCachePath) && hash != null && remote is SignatureEntry) { string cachefilename = FindCacheEntry(remote as SignatureEntry); if (cachefilename != null && System.IO.File.Exists(cachefilename)) { if ((hash.Size < 0 || new System.IO.FileInfo(cachefilename).Length == hash.Size) && Utility.Utility.CalculateHash(cachefilename) == hash.Hash) { if (manifest.Version > 2 && !string.IsNullOrEmpty(remote.EncryptionMode)) { try { using (Library.Interface.IEncryption enc = DynamicLoader.EncryptionLoader.GetModule(remote.EncryptionMode, m_options.Passphrase, m_options.RawOptions)) enc.Decrypt(cachefilename, filename); return; } catch (Exception ex) { m_statistics.LogWarning(string.Format(Strings.BackendWrapper.CachedSignatureDecryptWarning, cachefilename, ex.Message), null); try { System.IO.File.Delete(cachefilename); } catch { } } } else { //TODO: Don't copy, but just return it as write protected System.IO.File.Copy(cachefilename, filename, true); return; } } else { m_statistics.LogWarning(string.Format(Strings.BackendWrapper.CachedSignatureHashMismatchWarning, cachefilename), null); try { System.IO.File.Delete(cachefilename); } catch { } } } } Utility.TempFile tempfile = null; try { if (!string.IsNullOrEmpty(remote.EncryptionMode)) tempfile = new Duplicati.Library.Utility.TempFile(); else tempfile = new Duplicati.Library.Utility.TempFile(filename); ResetBackend(); m_statistics.AddNumberOfRemoteCalls(1); if (m_backend is Duplicati.Library.Interface.IStreamingBackend && !m_options.DisableStreamingTransfers) { using (System.IO.FileStream fs = System.IO.File.Open(tempfile, System.IO.FileMode.Create, System.IO.FileAccess.Write, System.IO.FileShare.None)) using (Utility.ProgressReportingStream pgs = new Duplicati.Library.Utility.ProgressReportingStream(fs, remote.Fileentry.Size)) using (Utility.ThrottledStream ts = new Duplicati.Library.Utility.ThrottledStream(pgs, m_options.MaxUploadPrSecond, m_options.MaxDownloadPrSecond)) { pgs.Progress += new Duplicati.Library.Utility.ProgressReportingStream.ProgressDelegate(pgs_Progress); ts.Callback += new Duplicati.Library.Utility.ThrottledStream.ThrottledStreamCallback(ThrottledStream_Callback); ((Duplicati.Library.Interface.IStreamingBackend)m_backend).Get(remote.Filename, ts); } } else { if (!m_async && ProgressEvent != null) ProgressEvent(50, m_statusmessage); m_backend.Get(remote.Filename, tempfile); if (!m_async && ProgressEvent != null) ProgressEvent(100, m_statusmessage); } //This is required so we are sure that the file was downloaded completely and not partially, // as any exception here will cause a retry, but using a partial file may cause random errors if (remote.Fileentry.Size > 0 && remote.Fileentry.Size != new System.IO.FileInfo(tempfile).Length) throw new Exception(string.Format(Strings.BackendWrapper.DownloadedFileSizeError, remote.Filename, remote.Fileentry.Size, new System.IO.FileInfo(tempfile).Length)); remote.RemoteHash = Utility.Utility.CalculateHash(tempfile); //Manifest version 3 has hashes WITH encryption if (manifest != null && manifest.Version > 2) { if (hash != null && remote.RemoteHash != hash.Hash) throw new HashMismathcException(string.Format(Strings.BackendWrapper.HashMismatchError, remote.Filename, hash.Hash, Utility.Utility.CalculateHash(tempfile))); if (!string.IsNullOrEmpty(m_options.SignatureCachePath) && remote is SignatureEntry) { string cachefilename = System.IO.Path.Combine(m_options.SignatureCachePath, m_cachefilenamestrategy.GenerateFilename(remote)); try { System.IO.File.Copy(tempfile, cachefilename, true); } catch (Exception ex) { m_statistics.LogWarning(string.Format(Strings.BackendWrapper.SaveCacheFileError, cachefilename), ex); } } } if (!string.IsNullOrEmpty(remote.EncryptionMode)) { try { using (Library.Interface.IEncryption enc = DynamicLoader.EncryptionLoader.GetModule(remote.EncryptionMode, m_options.Passphrase, m_options.RawOptions)) enc.Decrypt(tempfile, filename); } catch (Exception ex) { //If we fail here, make sure that we throw a crypto exception if (ex is System.Security.Cryptography.CryptographicException) throw; else throw new System.Security.Cryptography.CryptographicException(ex.Message, ex); } tempfile.Dispose(); //Remove the encrypted file //Wrap the new file as a temp file tempfile = new Duplicati.Library.Utility.TempFile(filename); } //Manifest version 1+2 has hashes WITHOUT encryption if (manifest != null && manifest.Version <= 2) { if (hash != null && Utility.Utility.CalculateHash(tempfile) != hash.Hash) throw new HashMismathcException(string.Format(Strings.BackendWrapper.HashMismatchError, remote.Filename, hash.Hash, Utility.Utility.CalculateHash(tempfile))); if (!string.IsNullOrEmpty(m_options.SignatureCachePath) && remote is SignatureEntry) { string cachefilename = System.IO.Path.Combine(m_options.SignatureCachePath, m_cachefilenamestrategy.GenerateFilename(remote)); try { System.IO.File.Copy(tempfile, cachefilename, true); } catch (Exception ex) { m_statistics.LogWarning(string.Format(Strings.BackendWrapper.SaveCacheFileError, cachefilename), ex); } } } lastEx = null; tempfile.Protected = true; //Don't delete it if (m_backendInterfaceLogger != null) { if (remote is ManifestEntry) m_backendInterfaceLogger.RegisterGet(remote.Fileentry, true, System.IO.File.ReadAllText(tempfile)); else m_backendInterfaceLogger.RegisterGet(remote.Fileentry, true, null); } } finally { try { if (tempfile != null) tempfile.Dispose(); } catch { } } } catch (System.Threading.ThreadAbortException tex) { if (m_backendInterfaceLogger != null) m_backendInterfaceLogger.RegisterGet(remote.Fileentry, false, tex.ToString()); throw; } catch (Exception ex) { lastEx = ex; m_statistics.LogRetryAttempt(ex.Message, ex); if (m_backendInterfaceLogger != null) m_backendInterfaceLogger.RegisterGet(remote.Fileentry, false, ex.ToString()); DisposeBackend(); retries--; if (retries > 0 && m_options.RetryDelay.Ticks > 0) System.Threading.Thread.Sleep(m_options.RetryDelay); } } while (lastEx != null && retries > 0); if (lastEx != null) if (lastEx is HashMismathcException) throw lastEx; else if (lastEx is System.Security.Cryptography.CryptographicException) throw lastEx; else throw new Exception(string.Format(Strings.BackendWrapper.FileDownloadError2, filename, lastEx.Message), lastEx); m_statistics.AddBytesDownloaded(new System.IO.FileInfo(filename).Length); }