Example #1
0
        private void DoPut(FileEntryItem item)
        {
            if (m_encryption != null)
                lock(m_encryptionLock)
                    item.Encrypt(m_encryption, m_statwriter);

            if (item.UpdateHashAndSize(m_options) && !item.NotTrackedInDb)
                m_db.LogDbUpdate(item.RemoteFilename, RemoteVolumeState.Uploading, item.Size, item.Hash);

            if (item.Indexfile != null && !item.IndexfileUpdated)
            {
                item.Indexfile.Item1.FinishVolume(item.Hash, item.Size);
                item.Indexfile.Item1.Close();
                item.IndexfileUpdated = true;
            }

            m_db.LogDbOperation("put", item.RemoteFilename, JsonConvert.SerializeObject(new { Size = item.Size, Hash = item.Hash }));
            m_statwriter.SendEvent(BackendActionType.Put, BackendEventType.Started, item.RemoteFilename, item.Size);

            if (m_backend is Library.Interface.IStreamingBackend && !m_options.DisableStreamingTransfers)
            {
                using (var fs = System.IO.File.OpenRead(item.LocalFilename))
                using (var ts = new ThrottledStream(fs, m_options.MaxDownloadPrSecond, m_options.MaxUploadPrSecond))
                using (var pgs = new Library.Utility.ProgressReportingStream(ts, item.Size, HandleProgress))
                    ((Library.Interface.IStreamingBackend)m_backend).Put(item.RemoteFilename, pgs);
            }
            else
                m_backend.Put(item.RemoteFilename, item.LocalFilename);

            if (!item.NotTrackedInDb)
                m_db.LogDbUpdate(item.RemoteFilename, RemoteVolumeState.Uploaded, item.Size, item.Hash);

            m_statwriter.SendEvent(BackendActionType.Put, BackendEventType.Completed, item.RemoteFilename, item.Size);

            if (m_options.ListVerifyUploads)
            {
                var f = m_backend.List().Where(n => n.Name.Equals(item.RemoteFilename, StringComparison.InvariantCultureIgnoreCase)).FirstOrDefault();
                if (f == null)
                    throw new Exception(string.Format("List verify failed, file was not found after upload: {0}", f.Name));
                else if (f.Size != item.Size && f.Size >= 0)
                    throw new Exception(string.Format("List verify failed for file: {0}, size was {1} but expected to be {2}", f.Name, f.Size, item.Size));
            }

            item.DeleteLocalFile(m_statwriter);
        }
Example #2
0
        private void DoGet(FileEntryItem item)
        {
            Library.Utility.TempFile tmpfile = null;
            m_statwriter.SendEvent(BackendActionType.Get, BackendEventType.Started, item.RemoteFilename, item.Size);

            try
            {
                tmpfile = new Library.Utility.TempFile();
                if (m_backend is Library.Interface.IStreamingBackend && !m_options.DisableStreamingTransfers)
                {
                    using (var fs = System.IO.File.OpenWrite(tmpfile))
                    using (var ts = new ThrottledStream(fs, m_options.MaxDownloadPrSecond, m_options.MaxUploadPrSecond))
                    using (var pgs = new Library.Utility.ProgressReportingStream(ts, item.Size, HandleProgress))
                        ((Library.Interface.IStreamingBackend)m_backend).Get(item.RemoteFilename, pgs);
                }
                else
                    m_backend.Get(item.RemoteFilename, tmpfile);

                m_db.LogDbOperation("get", item.RemoteFilename, JsonConvert.SerializeObject(new { Size = new System.IO.FileInfo(tmpfile).Length, Hash = FileEntryItem.CalculateFileHash(tmpfile) }));
                m_statwriter.SendEvent(BackendActionType.Get, BackendEventType.Completed, item.RemoteFilename, new System.IO.FileInfo(tmpfile).Length);

                if (!m_options.SkipFileHashChecks)
                {
                    var nl = new System.IO.FileInfo(tmpfile).Length;
                    if (item.Size >= 0)
                    {
                        if (nl != item.Size)
                            throw new Exception(string.Format(Strings.Controller.DownloadedFileSizeError, item.RemoteFilename, nl, item.Size));
                    }
                    else
                        item.Size = nl;

                    var nh = FileEntryItem.CalculateFileHash(tmpfile);
                    if (!string.IsNullOrEmpty(item.Hash))
                    {
                        if (nh != item.Hash)
                            throw new HashMismathcException(string.Format(Strings.Controller.HashMismatchError, tmpfile, item.Hash, nh));
                    }
                    else
                        item.Hash = nh;
                }

                if (!item.VerifyHashOnly)
                {
                    // Decrypt before returning
                    if (!m_options.NoEncryption)
                    {
                        try
                        {
                            using(var tmpfile2 = tmpfile)
                            {
                                tmpfile = new Library.Utility.TempFile();
                                lock(m_encryptionLock)
                                    m_encryption.Decrypt(tmpfile2, tmpfile);
                            }
                        }
                        catch (Exception ex)
                        {
                            //If we fail here, make sure that we throw a crypto exception
                            if (ex is System.Security.Cryptography.CryptographicException)
                                throw;
                            else
                                throw new System.Security.Cryptography.CryptographicException(ex.Message, ex);
                        }
                    }

                    item.Result = tmpfile;
                    tmpfile = null;
                }
            }
            catch
            {
                if (tmpfile != null)
                    tmpfile.Dispose();

                throw;
            }
        }
Example #3
0
        private TempFile coreDoGetPiping(FileEntryItem item, Interface.IEncryption useDecrypter, out long retDownloadSize, out string retHashcode)
        {
            // With piping allowed, we will parallelize the operation with buffered pipes to maximize throughput:
            // Separated: Download (only for streaming) - Hashing - Decryption
            // The idea is to use DirectStreamLink's that are inserted in the stream stack, creating a fork to run
            // the crypto operations on.

            retDownloadSize = -1;
            retHashcode = null;

            bool enableStreaming = (m_backend is Library.Interface.IStreamingBackend && !m_options.DisableStreamingTransfers);

            System.Threading.Tasks.Task<string> taskHasher = null;
            DirectStreamLink linkForkHasher = null;
            System.Threading.Tasks.Task taskDecrypter = null;
            DirectStreamLink linkForkDecryptor = null;

            // keep potential temp files and their streams for cleanup (cannot use using here).
            TempFile retTarget = null, dlTarget = null, decryptTarget = null;
            System.IO.Stream dlToStream = null, decryptToStream = null;
            try
            {
                System.IO.Stream nextTierWriter = null; // target of our stacked streams
                if (!enableStreaming) // we will always need dlTarget if not streaming...
                    dlTarget = new TempFile();
                else if (enableStreaming && useDecrypter == null)
                {
                    dlTarget = new TempFile();
                    dlToStream = System.IO.File.OpenWrite(dlTarget);
                    nextTierWriter = dlToStream; // actually write through to file.
                }

                // setup decryption: fork off a StreamLink from stack, and setup decryptor task
                if (useDecrypter != null)
                {
                    linkForkDecryptor = new DirectStreamLink(1 << 16, false, false, nextTierWriter);
                    nextTierWriter = linkForkDecryptor.WriterStream;
                    linkForkDecryptor.SetKnownLength(item.Size, false); // Set length to allow AES-decryption (not streamable yet)
                    decryptTarget = new TempFile();
                    decryptToStream = System.IO.File.OpenWrite(decryptTarget);
                    taskDecrypter = new System.Threading.Tasks.Task(() =>
                            {
                                using (var input = linkForkDecryptor.ReaderStream)
                                using (var output = decryptToStream)
                                    lock (m_encryptionLock) { useDecrypter.Decrypt(input, output); }
                            }
                        );
                }

                // setup hashing: fork off a StreamLink from stack, then task computes hash
                linkForkHasher = new DirectStreamLink(1 << 16, false, false, nextTierWriter);
                nextTierWriter = linkForkHasher.WriterStream;
                taskHasher = new System.Threading.Tasks.Task<string>(() =>
                        {
                            using (var input = linkForkHasher.ReaderStream)
                                return CalculateFileHash(input);
                        }
                    );

                // OK, forks with tasks are set up, so let's do the download which is performed in main thread.
                bool hadException = false;
                try
                {
                    if (enableStreaming)
                    {
                        using (var ss = new ShaderStream(nextTierWriter, false))
                        {
                            using (var ts = new ThrottledStream(ss, m_options.MaxUploadPrSecond, m_options.MaxDownloadPrSecond))
                            using (var pgs = new Library.Utility.ProgressReportingStream(ts, item.Size, HandleProgress))
                            {
                                taskHasher.Start(); // We do not start tasks earlier to be sure the input always gets closed.
                                if (taskDecrypter != null) taskDecrypter.Start();
                                ((Library.Interface.IStreamingBackend)m_backend).Get(item.RemoteFilename, pgs);
                            }
                            retDownloadSize = ss.TotalBytesWritten;
                        }
                    }
                    else
                    {
                        m_backend.Get(item.RemoteFilename, dlTarget);
                        retDownloadSize = new System.IO.FileInfo(dlTarget).Length;
                        using (dlToStream = System.IO.File.OpenRead(dlTarget))
                        {
                            taskHasher.Start(); // We do not start tasks earlier to be sure the input always gets closed.
                            if (taskDecrypter != null) taskDecrypter.Start();
                            new DirectStreamLink.DataPump(dlToStream, nextTierWriter).Run();
                        }
                    }
                }
                catch (Exception)
                { hadException = true; throw; }
                finally
                {
                    // This nested try-catch-finally blocks will make sure we do not miss any exceptions ans all started tasks
                    // are properly ended and tidied up. For what is thrown: If exceptions in main thread occured (download) it is thrown,
                    // then hasher task is checked and last decryption. This resembles old logic.
                    try { retHashcode = taskHasher.Result; }
                    catch (AggregateException ex) { if (!hadException) { hadException = true; throw ex.InnerExceptions[0]; } }
                    finally
                    {
                        if (taskDecrypter != null)
                        {
                            try { taskDecrypter.Wait(); }
                            catch (AggregateException ex)
                            {
                                if (!hadException)
                                {
                                    hadException = true;
                                    if (ex.InnerExceptions[0] is System.Security.Cryptography.CryptographicException)
                                        throw ex.InnerExceptions[0];
                                    else
                                        throw new System.Security.Cryptography.CryptographicException(ex.InnerExceptions[0].Message, ex.InnerExceptions[0]);
                                }
                            }
                        }
                    }
                }

                if (useDecrypter != null) // return decrypted temp file
                { retTarget = decryptTarget; decryptTarget = null; }
                else // return downloaded file
                { retTarget = dlTarget; dlTarget = null; }
            }
            finally
            {
                // Be tidy: manually do some cleanup to temp files, as we could not use using's.
                // Unclosed streams should only occur if we failed even before tasks were started.
                if (dlToStream != null) dlToStream.Dispose();
                if (dlTarget != null) dlTarget.Dispose();
                if (decryptToStream != null) decryptToStream.Dispose();
                if (decryptTarget != null) decryptTarget.Dispose();
            }

            return retTarget;
        }
Example #4
0
        private TempFile coreDoGetSequential(FileEntryItem item, Interface.IEncryption useDecrypter, out long retDownloadSize, out string retHashcode)
        {
            retHashcode = null;
            retDownloadSize = -1;
            TempFile retTarget, dlTarget = null, decryptTarget = null;
            try
            {
                dlTarget = new Library.Utility.TempFile();
                if (m_backend is Library.Interface.IStreamingBackend && !m_options.DisableStreamingTransfers)
                {
                    Func<string> getFileHash;
                    // extended to use stacked streams
                    using (var fs = System.IO.File.OpenWrite(dlTarget))
                    using (var hs = GetFileHasherStream(fs, System.Security.Cryptography.CryptoStreamMode.Write, out getFileHash))
                    using (var ss = new ShaderStream(hs, true))
                    {
                        using (var ts = new ThrottledStream(ss, m_options.MaxUploadPrSecond, m_options.MaxDownloadPrSecond))
                        using (var pgs = new Library.Utility.ProgressReportingStream(ts, item.Size, HandleProgress))
                        { ((Library.Interface.IStreamingBackend)m_backend).Get(item.RemoteFilename, pgs); }
                        ss.Flush();
                        retDownloadSize = ss.TotalBytesWritten;
                        retHashcode = getFileHash();
                    }
                }
                else
                {
                    m_backend.Get(item.RemoteFilename, dlTarget);
                    retDownloadSize = new System.IO.FileInfo(dlTarget).Length;
                    retHashcode = CalculateFileHash(dlTarget);
                }

                // Decryption is not placed in the stream stack because there seemed to be an effort
                // to throw a CryptographicException on fail. If in main stack, we cannot differentiate
                // in which part of the stack the source of an exception resides.
                if (useDecrypter != null)
                {
                    decryptTarget = new Library.Utility.TempFile();
                    lock (m_encryptionLock)
                    {
                        try { useDecrypter.Decrypt(dlTarget, decryptTarget); }
                        // If we fail here, make sure that we throw a crypto exception
                        catch (System.Security.Cryptography.CryptographicException) { throw; }
                        catch (Exception ex) { throw new System.Security.Cryptography.CryptographicException(ex.Message, ex); }
                    }
                    retTarget = decryptTarget;
                    decryptTarget = null;
                }
                else
                {
                    retTarget = dlTarget;
                    dlTarget = null;
                }
            }
            finally
            {
                if (dlTarget != null) dlTarget.Dispose();
                if (decryptTarget != null) decryptTarget.Dispose();
            }

            return retTarget;
        }
Example #5
0
        private void DoGet(FileEntryItem item)
        {
            Library.Utility.TempFile tmpfile = null;
            m_statwriter.SendEvent(BackendActionType.Get, BackendEventType.Started, item.RemoteFilename, item.Size);

            try
            {
                var begin = DateTime.Now;

                tmpfile = new Library.Utility.TempFile();
                if (m_backend is Library.Interface.IStreamingBackend && !m_options.DisableStreamingTransfers)
                {
                    using (var fs = System.IO.File.OpenWrite(tmpfile))
                    using (var ts = new ThrottledStream(fs, m_options.MaxUploadPrSecond, m_options.MaxDownloadPrSecond))
                    using (var pgs = new Library.Utility.ProgressReportingStream(ts, item.Size, HandleProgress))
                        ((Library.Interface.IStreamingBackend)m_backend).Get(item.RemoteFilename, pgs);
                }
                else
                    m_backend.Get(item.RemoteFilename, tmpfile);

                var duration = DateTime.Now - begin;
                Logging.Log.WriteMessage(string.Format("Downloaded {0} in {1}, {2}/s", Library.Utility.Utility.FormatSizeString(item.Size), duration, Library.Utility.Utility.FormatSizeString((long)(item.Size / duration.TotalSeconds))), Duplicati.Library.Logging.LogMessageType.Profiling);

                m_db.LogDbOperation("get", item.RemoteFilename, JsonConvert.SerializeObject(new { Size = new System.IO.FileInfo(tmpfile).Length, Hash = CalculateFileHash(tmpfile) }));
                m_statwriter.SendEvent(BackendActionType.Get, BackendEventType.Completed, item.RemoteFilename, new System.IO.FileInfo(tmpfile).Length);

                if (!m_options.SkipFileHashChecks)
                {
                    var nl = new System.IO.FileInfo(tmpfile).Length;
                    if (item.Size >= 0)
                    {
                        if (nl != item.Size)
                            throw new Exception(Strings.Controller.DownloadedFileSizeError(item.RemoteFilename, nl, item.Size));
                    }
                    else
                    	item.Size = nl;

                    var nh = CalculateFileHash(tmpfile);
                    if (!string.IsNullOrEmpty(item.Hash))
                    {
                        if (nh != item.Hash)
                            throw new HashMismathcException(Strings.Controller.HashMismatchError(tmpfile, item.Hash, nh));
                    }
                    else
                    	item.Hash = nh;
                }
                
                if (!item.VerifyHashOnly)
                {
                    // Decrypt before returning
                    if (!m_options.NoEncryption)
                    {
                        try
                        {
                            using(var tmpfile2 = tmpfile)
                            { 
                            	tmpfile = new Library.Utility.TempFile();
                                lock(m_encryptionLock)
                                {
                                    // Auto-guess the encryption module
                                    var ext = (System.IO.Path.GetExtension(item.RemoteFilename) ?? "").TrimStart('.');
                                    if (!m_encryption.FilenameExtension.Equals(ext, StringComparison.InvariantCultureIgnoreCase))
                                    {
                                        // Check if the file is encrypted with something else
                                        if (DynamicLoader.EncryptionLoader.Keys.Contains(ext, StringComparer.InvariantCultureIgnoreCase))
                                        {
                                            m_statwriter.AddVerboseMessage("Filename extension \"{0}\" does not match encryption module \"{1}\", using matching encryption module", ext, m_options.EncryptionModule);
                                            using(var encmodule = DynamicLoader.EncryptionLoader.GetModule(ext, m_options.Passphrase, m_options.RawOptions))
                                                (encmodule ?? m_encryption).Decrypt(tmpfile2, tmpfile);
                                        }
                                        // Check if the file is not encrypted
                                        else if (DynamicLoader.CompressionLoader.Keys.Contains(ext, StringComparer.InvariantCultureIgnoreCase))
                                        {
                                            m_statwriter.AddVerboseMessage("Filename extension \"{0}\" does not match encryption module \"{1}\", guessing that it is not encrypted", ext, m_options.EncryptionModule);
                                        }
                                        // Fallback, lets see what happens...
                                        else
                                        {
                                            m_statwriter.AddVerboseMessage("Filename extension \"{0}\" does not match encryption module \"{1}\", attempting to use specified encryption module as no others match", ext, m_options.EncryptionModule);
                                            m_encryption.Decrypt(tmpfile2, tmpfile);
                                        }
                                    }
                                    else
                                    {
                                	    m_encryption.Decrypt(tmpfile2, tmpfile);
                                    }
                                }
                            }
                        }
                        catch (Exception ex)
                        {
                            //If we fail here, make sure that we throw a crypto exception
                            if (ex is System.Security.Cryptography.CryptographicException)
                                throw;
                            else
                                throw new System.Security.Cryptography.CryptographicException(ex.Message, ex);
                        }
                    }
    
                    item.Result = tmpfile;
                    tmpfile = null;
                }
            }
            catch
            {
                if (tmpfile != null)
                    tmpfile.Dispose();

                throw;
            }
        }