private void DoPut(FileEntryItem item) { if (m_encryption != null) lock(m_encryptionLock) item.Encrypt(m_encryption, m_statwriter); if (item.UpdateHashAndSize(m_options) && !item.NotTrackedInDb) m_db.LogDbUpdate(item.RemoteFilename, RemoteVolumeState.Uploading, item.Size, item.Hash); if (item.Indexfile != null && !item.IndexfileUpdated) { item.Indexfile.Item1.FinishVolume(item.Hash, item.Size); item.Indexfile.Item1.Close(); item.IndexfileUpdated = true; } m_db.LogDbOperation("put", item.RemoteFilename, JsonConvert.SerializeObject(new { Size = item.Size, Hash = item.Hash })); m_statwriter.SendEvent(BackendActionType.Put, BackendEventType.Started, item.RemoteFilename, item.Size); if (m_backend is Library.Interface.IStreamingBackend && !m_options.DisableStreamingTransfers) { using (var fs = System.IO.File.OpenRead(item.LocalFilename)) using (var ts = new ThrottledStream(fs, m_options.MaxDownloadPrSecond, m_options.MaxUploadPrSecond)) using (var pgs = new Library.Utility.ProgressReportingStream(ts, item.Size, HandleProgress)) ((Library.Interface.IStreamingBackend)m_backend).Put(item.RemoteFilename, pgs); } else m_backend.Put(item.RemoteFilename, item.LocalFilename); if (!item.NotTrackedInDb) m_db.LogDbUpdate(item.RemoteFilename, RemoteVolumeState.Uploaded, item.Size, item.Hash); m_statwriter.SendEvent(BackendActionType.Put, BackendEventType.Completed, item.RemoteFilename, item.Size); if (m_options.ListVerifyUploads) { var f = m_backend.List().Where(n => n.Name.Equals(item.RemoteFilename, StringComparison.InvariantCultureIgnoreCase)).FirstOrDefault(); if (f == null) throw new Exception(string.Format("List verify failed, file was not found after upload: {0}", f.Name)); else if (f.Size != item.Size && f.Size >= 0) throw new Exception(string.Format("List verify failed for file: {0}, size was {1} but expected to be {2}", f.Name, f.Size, item.Size)); } item.DeleteLocalFile(m_statwriter); }
private void DoGet(FileEntryItem item) { Library.Utility.TempFile tmpfile = null; m_statwriter.SendEvent(BackendActionType.Get, BackendEventType.Started, item.RemoteFilename, item.Size); try { tmpfile = new Library.Utility.TempFile(); if (m_backend is Library.Interface.IStreamingBackend && !m_options.DisableStreamingTransfers) { using (var fs = System.IO.File.OpenWrite(tmpfile)) using (var ts = new ThrottledStream(fs, m_options.MaxDownloadPrSecond, m_options.MaxUploadPrSecond)) using (var pgs = new Library.Utility.ProgressReportingStream(ts, item.Size, HandleProgress)) ((Library.Interface.IStreamingBackend)m_backend).Get(item.RemoteFilename, pgs); } else m_backend.Get(item.RemoteFilename, tmpfile); m_db.LogDbOperation("get", item.RemoteFilename, JsonConvert.SerializeObject(new { Size = new System.IO.FileInfo(tmpfile).Length, Hash = FileEntryItem.CalculateFileHash(tmpfile) })); m_statwriter.SendEvent(BackendActionType.Get, BackendEventType.Completed, item.RemoteFilename, new System.IO.FileInfo(tmpfile).Length); if (!m_options.SkipFileHashChecks) { var nl = new System.IO.FileInfo(tmpfile).Length; if (item.Size >= 0) { if (nl != item.Size) throw new Exception(string.Format(Strings.Controller.DownloadedFileSizeError, item.RemoteFilename, nl, item.Size)); } else item.Size = nl; var nh = FileEntryItem.CalculateFileHash(tmpfile); if (!string.IsNullOrEmpty(item.Hash)) { if (nh != item.Hash) throw new HashMismathcException(string.Format(Strings.Controller.HashMismatchError, tmpfile, item.Hash, nh)); } else item.Hash = nh; } if (!item.VerifyHashOnly) { // Decrypt before returning if (!m_options.NoEncryption) { try { using(var tmpfile2 = tmpfile) { tmpfile = new Library.Utility.TempFile(); lock(m_encryptionLock) m_encryption.Decrypt(tmpfile2, tmpfile); } } catch (Exception ex) { //If we fail here, make sure that we throw a crypto exception if (ex is System.Security.Cryptography.CryptographicException) throw; else throw new System.Security.Cryptography.CryptographicException(ex.Message, ex); } } item.Result = tmpfile; tmpfile = null; } } catch { if (tmpfile != null) tmpfile.Dispose(); throw; } }
private TempFile coreDoGetPiping(FileEntryItem item, Interface.IEncryption useDecrypter, out long retDownloadSize, out string retHashcode) { // With piping allowed, we will parallelize the operation with buffered pipes to maximize throughput: // Separated: Download (only for streaming) - Hashing - Decryption // The idea is to use DirectStreamLink's that are inserted in the stream stack, creating a fork to run // the crypto operations on. retDownloadSize = -1; retHashcode = null; bool enableStreaming = (m_backend is Library.Interface.IStreamingBackend && !m_options.DisableStreamingTransfers); System.Threading.Tasks.Task<string> taskHasher = null; DirectStreamLink linkForkHasher = null; System.Threading.Tasks.Task taskDecrypter = null; DirectStreamLink linkForkDecryptor = null; // keep potential temp files and their streams for cleanup (cannot use using here). TempFile retTarget = null, dlTarget = null, decryptTarget = null; System.IO.Stream dlToStream = null, decryptToStream = null; try { System.IO.Stream nextTierWriter = null; // target of our stacked streams if (!enableStreaming) // we will always need dlTarget if not streaming... dlTarget = new TempFile(); else if (enableStreaming && useDecrypter == null) { dlTarget = new TempFile(); dlToStream = System.IO.File.OpenWrite(dlTarget); nextTierWriter = dlToStream; // actually write through to file. } // setup decryption: fork off a StreamLink from stack, and setup decryptor task if (useDecrypter != null) { linkForkDecryptor = new DirectStreamLink(1 << 16, false, false, nextTierWriter); nextTierWriter = linkForkDecryptor.WriterStream; linkForkDecryptor.SetKnownLength(item.Size, false); // Set length to allow AES-decryption (not streamable yet) decryptTarget = new TempFile(); decryptToStream = System.IO.File.OpenWrite(decryptTarget); taskDecrypter = new System.Threading.Tasks.Task(() => { using (var input = linkForkDecryptor.ReaderStream) using (var output = decryptToStream) lock (m_encryptionLock) { useDecrypter.Decrypt(input, output); } } ); } // setup hashing: fork off a StreamLink from stack, then task computes hash linkForkHasher = new DirectStreamLink(1 << 16, false, false, nextTierWriter); nextTierWriter = linkForkHasher.WriterStream; taskHasher = new System.Threading.Tasks.Task<string>(() => { using (var input = linkForkHasher.ReaderStream) return CalculateFileHash(input); } ); // OK, forks with tasks are set up, so let's do the download which is performed in main thread. bool hadException = false; try { if (enableStreaming) { using (var ss = new ShaderStream(nextTierWriter, false)) { using (var ts = new ThrottledStream(ss, m_options.MaxUploadPrSecond, m_options.MaxDownloadPrSecond)) using (var pgs = new Library.Utility.ProgressReportingStream(ts, item.Size, HandleProgress)) { taskHasher.Start(); // We do not start tasks earlier to be sure the input always gets closed. if (taskDecrypter != null) taskDecrypter.Start(); ((Library.Interface.IStreamingBackend)m_backend).Get(item.RemoteFilename, pgs); } retDownloadSize = ss.TotalBytesWritten; } } else { m_backend.Get(item.RemoteFilename, dlTarget); retDownloadSize = new System.IO.FileInfo(dlTarget).Length; using (dlToStream = System.IO.File.OpenRead(dlTarget)) { taskHasher.Start(); // We do not start tasks earlier to be sure the input always gets closed. if (taskDecrypter != null) taskDecrypter.Start(); new DirectStreamLink.DataPump(dlToStream, nextTierWriter).Run(); } } } catch (Exception) { hadException = true; throw; } finally { // This nested try-catch-finally blocks will make sure we do not miss any exceptions ans all started tasks // are properly ended and tidied up. For what is thrown: If exceptions in main thread occured (download) it is thrown, // then hasher task is checked and last decryption. This resembles old logic. try { retHashcode = taskHasher.Result; } catch (AggregateException ex) { if (!hadException) { hadException = true; throw ex.InnerExceptions[0]; } } finally { if (taskDecrypter != null) { try { taskDecrypter.Wait(); } catch (AggregateException ex) { if (!hadException) { hadException = true; if (ex.InnerExceptions[0] is System.Security.Cryptography.CryptographicException) throw ex.InnerExceptions[0]; else throw new System.Security.Cryptography.CryptographicException(ex.InnerExceptions[0].Message, ex.InnerExceptions[0]); } } } } } if (useDecrypter != null) // return decrypted temp file { retTarget = decryptTarget; decryptTarget = null; } else // return downloaded file { retTarget = dlTarget; dlTarget = null; } } finally { // Be tidy: manually do some cleanup to temp files, as we could not use using's. // Unclosed streams should only occur if we failed even before tasks were started. if (dlToStream != null) dlToStream.Dispose(); if (dlTarget != null) dlTarget.Dispose(); if (decryptToStream != null) decryptToStream.Dispose(); if (decryptTarget != null) decryptTarget.Dispose(); } return retTarget; }
private TempFile coreDoGetSequential(FileEntryItem item, Interface.IEncryption useDecrypter, out long retDownloadSize, out string retHashcode) { retHashcode = null; retDownloadSize = -1; TempFile retTarget, dlTarget = null, decryptTarget = null; try { dlTarget = new Library.Utility.TempFile(); if (m_backend is Library.Interface.IStreamingBackend && !m_options.DisableStreamingTransfers) { Func<string> getFileHash; // extended to use stacked streams using (var fs = System.IO.File.OpenWrite(dlTarget)) using (var hs = GetFileHasherStream(fs, System.Security.Cryptography.CryptoStreamMode.Write, out getFileHash)) using (var ss = new ShaderStream(hs, true)) { using (var ts = new ThrottledStream(ss, m_options.MaxUploadPrSecond, m_options.MaxDownloadPrSecond)) using (var pgs = new Library.Utility.ProgressReportingStream(ts, item.Size, HandleProgress)) { ((Library.Interface.IStreamingBackend)m_backend).Get(item.RemoteFilename, pgs); } ss.Flush(); retDownloadSize = ss.TotalBytesWritten; retHashcode = getFileHash(); } } else { m_backend.Get(item.RemoteFilename, dlTarget); retDownloadSize = new System.IO.FileInfo(dlTarget).Length; retHashcode = CalculateFileHash(dlTarget); } // Decryption is not placed in the stream stack because there seemed to be an effort // to throw a CryptographicException on fail. If in main stack, we cannot differentiate // in which part of the stack the source of an exception resides. if (useDecrypter != null) { decryptTarget = new Library.Utility.TempFile(); lock (m_encryptionLock) { try { useDecrypter.Decrypt(dlTarget, decryptTarget); } // If we fail here, make sure that we throw a crypto exception catch (System.Security.Cryptography.CryptographicException) { throw; } catch (Exception ex) { throw new System.Security.Cryptography.CryptographicException(ex.Message, ex); } } retTarget = decryptTarget; decryptTarget = null; } else { retTarget = dlTarget; dlTarget = null; } } finally { if (dlTarget != null) dlTarget.Dispose(); if (decryptTarget != null) decryptTarget.Dispose(); } return retTarget; }
private void DoGet(FileEntryItem item) { Library.Utility.TempFile tmpfile = null; m_statwriter.SendEvent(BackendActionType.Get, BackendEventType.Started, item.RemoteFilename, item.Size); try { var begin = DateTime.Now; tmpfile = new Library.Utility.TempFile(); if (m_backend is Library.Interface.IStreamingBackend && !m_options.DisableStreamingTransfers) { using (var fs = System.IO.File.OpenWrite(tmpfile)) using (var ts = new ThrottledStream(fs, m_options.MaxUploadPrSecond, m_options.MaxDownloadPrSecond)) using (var pgs = new Library.Utility.ProgressReportingStream(ts, item.Size, HandleProgress)) ((Library.Interface.IStreamingBackend)m_backend).Get(item.RemoteFilename, pgs); } else m_backend.Get(item.RemoteFilename, tmpfile); var duration = DateTime.Now - begin; Logging.Log.WriteMessage(string.Format("Downloaded {0} in {1}, {2}/s", Library.Utility.Utility.FormatSizeString(item.Size), duration, Library.Utility.Utility.FormatSizeString((long)(item.Size / duration.TotalSeconds))), Duplicati.Library.Logging.LogMessageType.Profiling); m_db.LogDbOperation("get", item.RemoteFilename, JsonConvert.SerializeObject(new { Size = new System.IO.FileInfo(tmpfile).Length, Hash = CalculateFileHash(tmpfile) })); m_statwriter.SendEvent(BackendActionType.Get, BackendEventType.Completed, item.RemoteFilename, new System.IO.FileInfo(tmpfile).Length); if (!m_options.SkipFileHashChecks) { var nl = new System.IO.FileInfo(tmpfile).Length; if (item.Size >= 0) { if (nl != item.Size) throw new Exception(Strings.Controller.DownloadedFileSizeError(item.RemoteFilename, nl, item.Size)); } else item.Size = nl; var nh = CalculateFileHash(tmpfile); if (!string.IsNullOrEmpty(item.Hash)) { if (nh != item.Hash) throw new HashMismathcException(Strings.Controller.HashMismatchError(tmpfile, item.Hash, nh)); } else item.Hash = nh; } if (!item.VerifyHashOnly) { // Decrypt before returning if (!m_options.NoEncryption) { try { using(var tmpfile2 = tmpfile) { tmpfile = new Library.Utility.TempFile(); lock(m_encryptionLock) { // Auto-guess the encryption module var ext = (System.IO.Path.GetExtension(item.RemoteFilename) ?? "").TrimStart('.'); if (!m_encryption.FilenameExtension.Equals(ext, StringComparison.InvariantCultureIgnoreCase)) { // Check if the file is encrypted with something else if (DynamicLoader.EncryptionLoader.Keys.Contains(ext, StringComparer.InvariantCultureIgnoreCase)) { m_statwriter.AddVerboseMessage("Filename extension \"{0}\" does not match encryption module \"{1}\", using matching encryption module", ext, m_options.EncryptionModule); using(var encmodule = DynamicLoader.EncryptionLoader.GetModule(ext, m_options.Passphrase, m_options.RawOptions)) (encmodule ?? m_encryption).Decrypt(tmpfile2, tmpfile); } // Check if the file is not encrypted else if (DynamicLoader.CompressionLoader.Keys.Contains(ext, StringComparer.InvariantCultureIgnoreCase)) { m_statwriter.AddVerboseMessage("Filename extension \"{0}\" does not match encryption module \"{1}\", guessing that it is not encrypted", ext, m_options.EncryptionModule); } // Fallback, lets see what happens... else { m_statwriter.AddVerboseMessage("Filename extension \"{0}\" does not match encryption module \"{1}\", attempting to use specified encryption module as no others match", ext, m_options.EncryptionModule); m_encryption.Decrypt(tmpfile2, tmpfile); } } else { m_encryption.Decrypt(tmpfile2, tmpfile); } } } } catch (Exception ex) { //If we fail here, make sure that we throw a crypto exception if (ex is System.Security.Cryptography.CryptographicException) throw; else throw new System.Security.Cryptography.CryptographicException(ex.Message, ex); } } item.Result = tmpfile; tmpfile = null; } } catch { if (tmpfile != null) tmpfile.Dispose(); throw; } }