private void DoPut(FileEntryItem item)
{
if (m_encryption != null)
lock(m_encryptionLock)
item.Encrypt(m_encryption, m_statwriter);
if (item.UpdateHashAndSize(m_options) && !item.NotTrackedInDb)
m_db.LogDbUpdate(item.RemoteFilename, RemoteVolumeState.Uploading, item.Size, item.Hash);
if (item.Indexfile != null && !item.IndexfileUpdated)
{
item.Indexfile.Item1.FinishVolume(item.Hash, item.Size);
item.Indexfile.Item1.Close();
item.IndexfileUpdated = true;
}
m_db.LogDbOperation("put", item.RemoteFilename, JsonConvert.SerializeObject(new { Size = item.Size, Hash = item.Hash }));
m_statwriter.SendEvent(BackendActionType.Put, BackendEventType.Started, item.RemoteFilename, item.Size);
if (m_backend is Library.Interface.IStreamingBackend && !m_options.DisableStreamingTransfers)
{
using (var fs = System.IO.File.OpenRead(item.LocalFilename))
using (var ts = new ThrottledStream(fs, m_options.MaxDownloadPrSecond, m_options.MaxUploadPrSecond))
using (var pgs = new Library.Utility.ProgressReportingStream(ts, item.Size, HandleProgress))
((Library.Interface.IStreamingBackend)m_backend).Put(item.RemoteFilename, pgs);
}
else
m_backend.Put(item.RemoteFilename, item.LocalFilename);
if (!item.NotTrackedInDb)
m_db.LogDbUpdate(item.RemoteFilename, RemoteVolumeState.Uploaded, item.Size, item.Hash);
m_statwriter.SendEvent(BackendActionType.Put, BackendEventType.Completed, item.RemoteFilename, item.Size);
if (m_options.ListVerifyUploads)
{
var f = m_backend.List().Where(n => n.Name.Equals(item.RemoteFilename, StringComparison.InvariantCultureIgnoreCase)).FirstOrDefault();
if (f == null)
throw new Exception(string.Format("List verify failed, file was not found after upload: {0}", f.Name));
else if (f.Size != item.Size && f.Size >= 0)
throw new Exception(string.Format("List verify failed for file: {0}, size was {1} but expected to be {2}", f.Name, f.Size, item.Size));
}
item.DeleteLocalFile(m_statwriter);
}
private void DoGet(FileEntryItem item)
{
Library.Utility.TempFile tmpfile = null;
m_statwriter.SendEvent(BackendActionType.Get, BackendEventType.Started, item.RemoteFilename, item.Size);
try
{
tmpfile = new Library.Utility.TempFile();
if (m_backend is Library.Interface.IStreamingBackend && !m_options.DisableStreamingTransfers)
{
using (var fs = System.IO.File.OpenWrite(tmpfile))
using (var ts = new ThrottledStream(fs, m_options.MaxDownloadPrSecond, m_options.MaxUploadPrSecond))
using (var pgs = new Library.Utility.ProgressReportingStream(ts, item.Size, HandleProgress))
((Library.Interface.IStreamingBackend)m_backend).Get(item.RemoteFilename, pgs);
}
else
m_backend.Get(item.RemoteFilename, tmpfile);
m_db.LogDbOperation("get", item.RemoteFilename, JsonConvert.SerializeObject(new { Size = new System.IO.FileInfo(tmpfile).Length, Hash = FileEntryItem.CalculateFileHash(tmpfile) }));
m_statwriter.SendEvent(BackendActionType.Get, BackendEventType.Completed, item.RemoteFilename, new System.IO.FileInfo(tmpfile).Length);
if (!m_options.SkipFileHashChecks)
{
var nl = new System.IO.FileInfo(tmpfile).Length;
if (item.Size >= 0)
{
if (nl != item.Size)
throw new Exception(string.Format(Strings.Controller.DownloadedFileSizeError, item.RemoteFilename, nl, item.Size));
}
else
item.Size = nl;
var nh = FileEntryItem.CalculateFileHash(tmpfile);
if (!string.IsNullOrEmpty(item.Hash))
{
if (nh != item.Hash)
throw new HashMismathcException(string.Format(Strings.Controller.HashMismatchError, tmpfile, item.Hash, nh));
}
else
item.Hash = nh;
}
if (!item.VerifyHashOnly)
{
// Decrypt before returning
if (!m_options.NoEncryption)
{
try
{
using(var tmpfile2 = tmpfile)
{
tmpfile = new Library.Utility.TempFile();
lock(m_encryptionLock)
m_encryption.Decrypt(tmpfile2, tmpfile);
}
}
catch (Exception ex)
{
//If we fail here, make sure that we throw a crypto exception
if (ex is System.Security.Cryptography.CryptographicException)
throw;
else
throw new System.Security.Cryptography.CryptographicException(ex.Message, ex);
}
}
item.Result = tmpfile;
tmpfile = null;
}
}
catch
{
if (tmpfile != null)
tmpfile.Dispose();
throw;
}
}
private TempFile coreDoGetPiping(FileEntryItem item, Interface.IEncryption useDecrypter, out long retDownloadSize, out string retHashcode)
{
// With piping allowed, we will parallelize the operation with buffered pipes to maximize throughput:
// Separated: Download (only for streaming) - Hashing - Decryption
// The idea is to use DirectStreamLink's that are inserted in the stream stack, creating a fork to run
// the crypto operations on.
retDownloadSize = -1;
retHashcode = null;
bool enableStreaming = (m_backend is Library.Interface.IStreamingBackend && !m_options.DisableStreamingTransfers);
System.Threading.Tasks.Task<string> taskHasher = null;
DirectStreamLink linkForkHasher = null;
System.Threading.Tasks.Task taskDecrypter = null;
DirectStreamLink linkForkDecryptor = null;
// keep potential temp files and their streams for cleanup (cannot use using here).
TempFile retTarget = null, dlTarget = null, decryptTarget = null;
System.IO.Stream dlToStream = null, decryptToStream = null;
try
{
System.IO.Stream nextTierWriter = null; // target of our stacked streams
if (!enableStreaming) // we will always need dlTarget if not streaming...
dlTarget = new TempFile();
else if (enableStreaming && useDecrypter == null)
{
dlTarget = new TempFile();
dlToStream = System.IO.File.OpenWrite(dlTarget);
nextTierWriter = dlToStream; // actually write through to file.
}
// setup decryption: fork off a StreamLink from stack, and setup decryptor task
if (useDecrypter != null)
{
linkForkDecryptor = new DirectStreamLink(1 << 16, false, false, nextTierWriter);
nextTierWriter = linkForkDecryptor.WriterStream;
linkForkDecryptor.SetKnownLength(item.Size, false); // Set length to allow AES-decryption (not streamable yet)
decryptTarget = new TempFile();
decryptToStream = System.IO.File.OpenWrite(decryptTarget);
taskDecrypter = new System.Threading.Tasks.Task(() =>
{
using (var input = linkForkDecryptor.ReaderStream)
using (var output = decryptToStream)
lock (m_encryptionLock) { useDecrypter.Decrypt(input, output); }
}
);
}
// setup hashing: fork off a StreamLink from stack, then task computes hash
linkForkHasher = new DirectStreamLink(1 << 16, false, false, nextTierWriter);
nextTierWriter = linkForkHasher.WriterStream;
taskHasher = new System.Threading.Tasks.Task<string>(() =>
{
using (var input = linkForkHasher.ReaderStream)
return CalculateFileHash(input);
}
);
// OK, forks with tasks are set up, so let's do the download which is performed in main thread.
bool hadException = false;
try
{
if (enableStreaming)
{
using (var ss = new ShaderStream(nextTierWriter, false))
{
using (var ts = new ThrottledStream(ss, m_options.MaxUploadPrSecond, m_options.MaxDownloadPrSecond))
using (var pgs = new Library.Utility.ProgressReportingStream(ts, item.Size, HandleProgress))
{
taskHasher.Start(); // We do not start tasks earlier to be sure the input always gets closed.
if (taskDecrypter != null) taskDecrypter.Start();
((Library.Interface.IStreamingBackend)m_backend).Get(item.RemoteFilename, pgs);
}
retDownloadSize = ss.TotalBytesWritten;
}
}
else
{
m_backend.Get(item.RemoteFilename, dlTarget);
retDownloadSize = new System.IO.FileInfo(dlTarget).Length;
using (dlToStream = System.IO.File.OpenRead(dlTarget))
{
taskHasher.Start(); // We do not start tasks earlier to be sure the input always gets closed.
if (taskDecrypter != null) taskDecrypter.Start();
new DirectStreamLink.DataPump(dlToStream, nextTierWriter).Run();
}
}
}
catch (Exception)
{ hadException = true; throw; }
finally
{
// This nested try-catch-finally blocks will make sure we do not miss any exceptions ans all started tasks
// are properly ended and tidied up. For what is thrown: If exceptions in main thread occured (download) it is thrown,
// then hasher task is checked and last decryption. This resembles old logic.
try { retHashcode = taskHasher.Result; }
catch (AggregateException ex) { if (!hadException) { hadException = true; throw ex.InnerExceptions[0]; } }
finally
{
if (taskDecrypter != null)
{
try { taskDecrypter.Wait(); }
catch (AggregateException ex)
{
if (!hadException)
{
hadException = true;
if (ex.InnerExceptions[0] is System.Security.Cryptography.CryptographicException)
throw ex.InnerExceptions[0];
else
throw new System.Security.Cryptography.CryptographicException(ex.InnerExceptions[0].Message, ex.InnerExceptions[0]);
}
}
}
}
}
if (useDecrypter != null) // return decrypted temp file
{ retTarget = decryptTarget; decryptTarget = null; }
else // return downloaded file
{ retTarget = dlTarget; dlTarget = null; }
}
finally
{
// Be tidy: manually do some cleanup to temp files, as we could not use using's.
// Unclosed streams should only occur if we failed even before tasks were started.
if (dlToStream != null) dlToStream.Dispose();
if (dlTarget != null) dlTarget.Dispose();
if (decryptToStream != null) decryptToStream.Dispose();
if (decryptTarget != null) decryptTarget.Dispose();
}
return retTarget;
}
private TempFile coreDoGetSequential(FileEntryItem item, Interface.IEncryption useDecrypter, out long retDownloadSize, out string retHashcode)
{
retHashcode = null;
retDownloadSize = -1;
TempFile retTarget, dlTarget = null, decryptTarget = null;
try
{
dlTarget = new Library.Utility.TempFile();
if (m_backend is Library.Interface.IStreamingBackend && !m_options.DisableStreamingTransfers)
{
Func<string> getFileHash;
// extended to use stacked streams
using (var fs = System.IO.File.OpenWrite(dlTarget))
using (var hs = GetFileHasherStream(fs, System.Security.Cryptography.CryptoStreamMode.Write, out getFileHash))
using (var ss = new ShaderStream(hs, true))
{
using (var ts = new ThrottledStream(ss, m_options.MaxUploadPrSecond, m_options.MaxDownloadPrSecond))
using (var pgs = new Library.Utility.ProgressReportingStream(ts, item.Size, HandleProgress))
{ ((Library.Interface.IStreamingBackend)m_backend).Get(item.RemoteFilename, pgs); }
ss.Flush();
retDownloadSize = ss.TotalBytesWritten;
retHashcode = getFileHash();
}
}
else
{
m_backend.Get(item.RemoteFilename, dlTarget);
retDownloadSize = new System.IO.FileInfo(dlTarget).Length;
retHashcode = CalculateFileHash(dlTarget);
}
// Decryption is not placed in the stream stack because there seemed to be an effort
// to throw a CryptographicException on fail. If in main stack, we cannot differentiate
// in which part of the stack the source of an exception resides.
if (useDecrypter != null)
{
decryptTarget = new Library.Utility.TempFile();
lock (m_encryptionLock)
{
try { useDecrypter.Decrypt(dlTarget, decryptTarget); }
// If we fail here, make sure that we throw a crypto exception
catch (System.Security.Cryptography.CryptographicException) { throw; }
catch (Exception ex) { throw new System.Security.Cryptography.CryptographicException(ex.Message, ex); }
}
retTarget = decryptTarget;
decryptTarget = null;
}
else
{
retTarget = dlTarget;
dlTarget = null;
}
}
finally
{
if (dlTarget != null) dlTarget.Dispose();
if (decryptTarget != null) decryptTarget.Dispose();
}
return retTarget;
}
private void DoGet(FileEntryItem item)
{
Library.Utility.TempFile tmpfile = null;
m_statwriter.SendEvent(BackendActionType.Get, BackendEventType.Started, item.RemoteFilename, item.Size);
try
{
var begin = DateTime.Now;
tmpfile = new Library.Utility.TempFile();
if (m_backend is Library.Interface.IStreamingBackend && !m_options.DisableStreamingTransfers)
{
using (var fs = System.IO.File.OpenWrite(tmpfile))
using (var ts = new ThrottledStream(fs, m_options.MaxUploadPrSecond, m_options.MaxDownloadPrSecond))
using (var pgs = new Library.Utility.ProgressReportingStream(ts, item.Size, HandleProgress))
((Library.Interface.IStreamingBackend)m_backend).Get(item.RemoteFilename, pgs);
}
else
m_backend.Get(item.RemoteFilename, tmpfile);
var duration = DateTime.Now - begin;
Logging.Log.WriteMessage(string.Format("Downloaded {0} in {1}, {2}/s", Library.Utility.Utility.FormatSizeString(item.Size), duration, Library.Utility.Utility.FormatSizeString((long)(item.Size / duration.TotalSeconds))), Duplicati.Library.Logging.LogMessageType.Profiling);
m_db.LogDbOperation("get", item.RemoteFilename, JsonConvert.SerializeObject(new { Size = new System.IO.FileInfo(tmpfile).Length, Hash = CalculateFileHash(tmpfile) }));
m_statwriter.SendEvent(BackendActionType.Get, BackendEventType.Completed, item.RemoteFilename, new System.IO.FileInfo(tmpfile).Length);
if (!m_options.SkipFileHashChecks)
{
var nl = new System.IO.FileInfo(tmpfile).Length;
if (item.Size >= 0)
{
if (nl != item.Size)
throw new Exception(Strings.Controller.DownloadedFileSizeError(item.RemoteFilename, nl, item.Size));
}
else
item.Size = nl;
var nh = CalculateFileHash(tmpfile);
if (!string.IsNullOrEmpty(item.Hash))
{
if (nh != item.Hash)
throw new HashMismathcException(Strings.Controller.HashMismatchError(tmpfile, item.Hash, nh));
}
else
item.Hash = nh;
}
if (!item.VerifyHashOnly)
{
// Decrypt before returning
if (!m_options.NoEncryption)
{
try
{
using(var tmpfile2 = tmpfile)
{
tmpfile = new Library.Utility.TempFile();
lock(m_encryptionLock)
{
// Auto-guess the encryption module
var ext = (System.IO.Path.GetExtension(item.RemoteFilename) ?? "").TrimStart('.');
if (!m_encryption.FilenameExtension.Equals(ext, StringComparison.InvariantCultureIgnoreCase))
{
// Check if the file is encrypted with something else
if (DynamicLoader.EncryptionLoader.Keys.Contains(ext, StringComparer.InvariantCultureIgnoreCase))
{
m_statwriter.AddVerboseMessage("Filename extension \"{0}\" does not match encryption module \"{1}\", using matching encryption module", ext, m_options.EncryptionModule);
using(var encmodule = DynamicLoader.EncryptionLoader.GetModule(ext, m_options.Passphrase, m_options.RawOptions))
(encmodule ?? m_encryption).Decrypt(tmpfile2, tmpfile);
}
// Check if the file is not encrypted
else if (DynamicLoader.CompressionLoader.Keys.Contains(ext, StringComparer.InvariantCultureIgnoreCase))
{
m_statwriter.AddVerboseMessage("Filename extension \"{0}\" does not match encryption module \"{1}\", guessing that it is not encrypted", ext, m_options.EncryptionModule);
}
// Fallback, lets see what happens...
else
{
m_statwriter.AddVerboseMessage("Filename extension \"{0}\" does not match encryption module \"{1}\", attempting to use specified encryption module as no others match", ext, m_options.EncryptionModule);
m_encryption.Decrypt(tmpfile2, tmpfile);
}
}
else
{
m_encryption.Decrypt(tmpfile2, tmpfile);
}
}
}
}
catch (Exception ex)
{
//If we fail here, make sure that we throw a crypto exception
if (ex is System.Security.Cryptography.CryptographicException)
throw;
else
throw new System.Security.Cryptography.CryptographicException(ex.Message, ex);
}
}
item.Result = tmpfile;
tmpfile = null;
}
}
catch
{
if (tmpfile != null)
tmpfile.Dispose();
throw;
}
}