Ejemplo n.º 1
0
        private async Task <IList <Library.Interface.IFileEntry> > DoList()
        {
            await m_stats.SendEventAsync(BackendActionType.List, BackendEventType.Started, null, -1);

            var r = m_backend.List().ToList();

            var sb = new StringBuilder();

            sb.AppendLine("[");
            long count = 0;

            foreach (var e in r)
            {
                if (count != 0)
                {
                    sb.AppendLine(",");
                }
                count++;
                sb.Append(JsonConvert.SerializeObject(e));
            }

            sb.AppendLine();
            sb.Append("]");
            await m_database.LogRemoteOperationAsync("list", "", sb.ToString());

            await m_stats.SendEventAsync(BackendActionType.List, BackendEventType.Completed, null, r.Count);

            return(r);
        }
 /// <summary>
 /// Tests a backend by invoking the List() method.
 /// As long as the iteration can either complete or find at least one file without throwing, the test is successful
 /// </summary>
 /// <param name="backend">Backend to test</param>
 public static void TestList(this IBackend backend)
 {
     // If we can iterate successfully, even if it's empty, then the backend test is successful
     foreach (IFileEntry file in backend.List())
     {
         break;
     }
 }
Ejemplo n.º 3
0
        private async Task DoPut(FileEntryItem item, IBackend backend, CancellationToken cancelToken)
        {
            if (cancelToken.IsCancellationRequested)
            {
                return;
            }

            if (item.TrackedInDb)
            {
                await m_database.UpdateRemoteVolumeAsync(item.RemoteFilename, RemoteVolumeState.Uploading, item.Size, item.Hash);
            }

            if (m_options.Dryrun)
            {
                Logging.Log.WriteDryrunMessage(LOGTAG, "WouldUploadVolume", "Would upload volume: {0}, size: {1}", item.RemoteFilename, Library.Utility.Utility.FormatSizeString(new FileInfo(item.LocalFilename).Length));
                item.DeleteLocalFile();
                return;
            }

            await m_database.LogRemoteOperationAsync("put", item.RemoteFilename, JsonConvert.SerializeObject(new { Size = item.Size, Hash = item.Hash }));

            await m_stats.SendEventAsync(BackendActionType.Put, BackendEventType.Started, item.RemoteFilename, item.Size, updateProgress : false);

            m_progressUpdater.StartFileProgress(item.RemoteFilename, item.Size);

            var begin = DateTime.Now;

            if (!m_options.DisableStreamingTransfers && backend is IStreamingBackend streamingBackend)
            {
                // A download throttle speed is not given to the ThrottledStream as we are only uploading data here
                using (var fs = File.OpenRead(item.LocalFilename))
                    using (var ts = new ThrottledStream(fs, m_initialUploadThrottleSpeed, 0))
                        using (var pgs = new ProgressReportingStream(ts, pg => HandleProgress(ts, pg, item.RemoteFilename)))
                            await streamingBackend.PutAsync(item.RemoteFilename, pgs, cancelToken).ConfigureAwait(false);
            }
            else
            {
                await backend.PutAsync(item.RemoteFilename, item.LocalFilename, cancelToken).ConfigureAwait(false);
            }

            var duration = DateTime.Now - begin;

            m_progressUpdater.EndFileProgress(item.RemoteFilename);
            Logging.Log.WriteProfilingMessage(LOGTAG, "UploadSpeed", "Uploaded {0} in {1}, {2}/s", Library.Utility.Utility.FormatSizeString(item.Size), duration, Library.Utility.Utility.FormatSizeString((long)(item.Size / duration.TotalSeconds)));

            if (item.TrackedInDb)
            {
                await m_database.UpdateRemoteVolumeAsync(item.RemoteFilename, RemoteVolumeState.Uploaded, item.Size, item.Hash);
            }

            await m_stats.SendEventAsync(BackendActionType.Put, BackendEventType.Completed, item.RemoteFilename, item.Size);

            if (m_options.ListVerifyUploads)
            {
                var f = backend.List().FirstOrDefault(n => n.Name.Equals(item.RemoteFilename, StringComparison.OrdinalIgnoreCase));
                if (f == null)
                {
                    throw new Exception(string.Format("List verify failed, file was not found after upload: {0}", item.RemoteFilename));
                }
                else if (f.Size != item.Size && f.Size >= 0)
                {
                    throw new Exception(string.Format("List verify failed for file: {0}, size was {1} but expected to be {2}", f.Name, f.Size, item.Size));
                }
            }

            item.DeleteLocalFile();
            await m_database.CommitTransactionAsync("CommitAfterUpload");
        }
Ejemplo n.º 4
0
        private async Task <bool> DoPut(FileEntryItem item, bool updatedHash = false)
        {
            // If this is not already encrypted, do it now
            item.Encrypt(m_options);

            updatedHash |= item.UpdateHashAndSize(m_options);

            if (updatedHash && item.TrackedInDb)
            {
                await m_database.UpdateRemoteVolumeAsync(item.RemoteFilename, RemoteVolumeState.Uploading, item.Size, item.Hash);
            }

            if (m_options.Dryrun)
            {
                Logging.Log.WriteDryrunMessage(LOGTAG, "WouldUploadVolume", "Would upload volume: {0}, size: {1}", item.RemoteFilename, Library.Utility.Utility.FormatSizeString(new FileInfo(item.LocalFilename).Length));
                item.DeleteLocalFile();
                return(true);
            }

            await m_database.LogRemoteOperationAsync("put", item.RemoteFilename, JsonConvert.SerializeObject(new { Size = item.Size, Hash = item.Hash }));

            await m_stats.SendEventAsync(BackendActionType.Put, BackendEventType.Started, item.RemoteFilename, item.Size);

            var begin = DateTime.Now;

            if (m_backend is Library.Interface.IStreamingBackend && !m_options.DisableStreamingTransfers)
            {
                using (var fs = System.IO.File.OpenRead(item.LocalFilename))
                    using (var ts = new ThrottledStream(fs, m_options.MaxUploadPrSecond, m_options.MaxDownloadPrSecond))
                        using (var pgs = new Library.Utility.ProgressReportingStream(ts, item.Size, pg => HandleProgress(ts, pg)))
                            ((Library.Interface.IStreamingBackend)m_backend).Put(item.RemoteFilename, pgs);
            }
            else
            {
                m_backend.Put(item.RemoteFilename, item.LocalFilename);
            }

            var duration = DateTime.Now - begin;

            Logging.Log.WriteProfilingMessage(LOGTAG, "UploadSpeed", "Uploaded {0} in {1}, {2}/s", Library.Utility.Utility.FormatSizeString(item.Size), duration, Library.Utility.Utility.FormatSizeString((long)(item.Size / duration.TotalSeconds)));

            if (item.TrackedInDb)
            {
                await m_database.UpdateRemoteVolumeAsync(item.RemoteFilename, RemoteVolumeState.Uploaded, item.Size, item.Hash);
            }

            await m_stats.SendEventAsync(BackendActionType.Put, BackendEventType.Completed, item.RemoteFilename, item.Size);

            if (m_options.ListVerifyUploads)
            {
                var f = m_backend.List().Where(n => n.Name.Equals(item.RemoteFilename, StringComparison.OrdinalIgnoreCase)).FirstOrDefault();
                if (f == null)
                {
                    throw new Exception(string.Format("List verify failed, file was not found after upload: {0}", item.RemoteFilename));
                }
                else if (f.Size != item.Size && f.Size >= 0)
                {
                    throw new Exception(string.Format("List verify failed for file: {0}, size was {1} but expected to be {2}", f.Name, f.Size, item.Size));
                }
            }

            item.DeleteLocalFile();
            await m_database.CommitTransactionAsync("CommitAfterUpload");

            return(true);
        }
Ejemplo n.º 5
0
        public static void TestBackend(IBackend b)
        {
            // Should be empty
            //~ keys, err := b.List("")
            //~ if err != nil {
            //~     t.Fatalf("err: %v", err)
            //~ }
            //~ if len(keys) != 0 {
            //~     t.Fatalf("bad: %v", keys)
            //~ }
            var keys = b.List("");

            Assert.AreEqual(0, keys.Count());

            // Delete should work if it does not exist
            //~ err = b.Delete("foo")
            //~ if err != nil {
            //~     t.Fatalf("err: %v", err)
            //~ }
            b.Delete("foo");

            // Get should fail
            //~ out, err := b.Get("foo")
            //~ if err != nil {
            //~     t.Fatalf("err: %v", err)
            //~ }
            //~ if out != nil {
            //~     t.Fatalf("bad: %v", out)
            //~ }
            var @out = b.Get("foo");

            Assert.IsNull(@out);

            // Make an entry
            //~ e := &Entry{Key: "foo", Value: []byte("test")}
            //~ err = b.Put(e)
            //~ if err != nil {
            //~     t.Fatalf("err: %v", err)
            //~ }
            var e = new Entry {
                Key = "foo", Value = "test".ToUtf8Bytes()
            };

            b.Put(e);

            // Get should work
            //~ out, err = b.Get("foo")
            //~ if err != nil {
            //~     t.Fatalf("err: %v", err)
            //~ }
            //~ if !reflect.DeepEqual(out, e) {
            //~     t.Fatalf("bad: %v expected: %v", out, e)
            //~ }
            @out = b.Get("foo");
            Assert.IsNotNull(@out);

            // List should not be empty
            //~ keys, err = b.List("")
            //~ if err != nil {
            //~     t.Fatalf("err: %v", err)
            //~ }
            //~ if len(keys) != 1 {
            //~     t.Fatalf("bad: %v", keys)
            //~ }
            //~ if keys[0] != "foo" {
            //~     t.Fatalf("bad: %v", keys)
            //~ }
            keys = b.List("");
            Assert.AreEqual(1, keys.Count());
            Assert.AreEqual("foo", keys.ElementAt(0));

            // Delete should work
            //~ err = b.Delete("foo")
            //~ if err != nil {
            //~     t.Fatalf("err: %v", err)
            //~ }
            b.Delete("foo");

            // Should be empty
            //~ keys, err = b.List("")
            //~ if err != nil {
            //~     t.Fatalf("err: %v", err)
            //~ }
            //~ if len(keys) != 0 {
            //~     t.Fatalf("bad: %v", keys)
            //~ }
            keys = b.List("");
            Assert.AreEqual(0, keys.Count());

            // Get should fail
            //~ out, err = b.Get("foo")
            //~ if err != nil {
            //~     t.Fatalf("err: %v", err)
            //~ }
            //~ if out != nil {
            //~     t.Fatalf("bad: %v", out)
            //~ }
            @out = b.Get("foo");
            Assert.IsNull(@out);

            // Multiple Puts should work; GH-189
            //~ e = &Entry{Key: "foo", Value: []byte("test")}
            //~ err = b.Put(e)
            //~ if err != nil {
            //~     t.Fatalf("err: %v", err)
            //~ }
            //~ e = &Entry{Key: "foo", Value: []byte("test")}
            //~ err = b.Put(e)
            //~ if err != nil {
            //~     t.Fatalf("err: %v", err)
            //~ }
            e = new Entry {
                Key = "foo", Value = "test".ToUtf8Bytes()
            };
            b.Put(e);
            e = new Entry {
                Key = "foo", Value = "test".ToUtf8Bytes()
            };
            b.Put(e);

            // Make a nested entry
            //~ e = &Entry{Key: "foo/bar", Value: []byte("baz")}
            //~ err = b.Put(e)
            //~ if err != nil {
            //~     t.Fatalf("err: %v", err)
            //~ }
            e = new Entry {
                Key = "foo/bar", Value = "baz".ToUtf8Bytes()
            };
            b.Put(e);

            //~ keys, err = b.List("")
            //~ if err != nil {
            //~     t.Fatalf("err: %v", err)
            //~ }
            //~ if len(keys) != 2 {
            //~     t.Fatalf("bad: %v", keys)
            //~ }
            //~ sort.Strings(keys)
            //~ if keys[0] != "foo" || keys[1] != "foo/" {
            //~     t.Fatalf("bad: %v", keys)
            //~ }
            keys = b.List("");
            Assert.AreEqual(2, keys.Count());
            keys = keys.OrderBy(x => x);
            Assert.AreEqual("foo", keys.ElementAt(0));
            Assert.AreEqual("foo/", keys.ElementAt(1));

            // Delete with children should work
            //~ err = b.Delete("foo")
            //~ if err != nil {
            //~     t.Fatalf("err: %v", err)
            //~ }
            b.Delete("foo");

            // Get should return the child
            //~ out, err = b.Get("foo/bar")
            //~ if err != nil {
            //~     t.Fatalf("err: %v", err)
            //~ }
            //~ if out == nil {
            //~     t.Fatalf("missing child")
            //~ }
            @out = b.Get("foo/bar");
            Assert.IsNotNull(@out);

            // Removal of nested secret should not leave artifacts
            //~ e = &Entry{Key: "foo/nested1/nested2/nested3", Value: []byte("baz")}
            //~ err = b.Put(e)
            //~ if err != nil {
            //~     t.Fatalf("err: %v", err)
            //~ }
            e = new Entry {
                Key = "foo/nested1/nested2/nested3", Value = "baz".ToUtf8Bytes()
            };
            b.Put(e);

            //~ err = b.Delete("foo/nested1/nested2/nested3")
            //~ if err != nil {
            //~     t.Fatalf("failed to remove nested secret: %v", err)
            //~ }
            b.Delete("foo/nested1/nested2/nested3");

            //~ keys, err = b.List("foo/")
            //~ if err != nil {
            //~     t.Fatalf("err: %v", err)
            //~ }
            keys = b.List("foo/");

            //~ if len(keys) != 1 {
            //~     t.Fatalf("there should be only one key left after deleting nested "+
            //~         "secret: %v", keys)
            //~ }
            Assert.AreEqual(1, keys.Count());

            //~ if keys[0] != "bar" {
            //~     t.Fatalf("bad keys after deleting nested: %v", keys)
            //~ }
            Assert.AreEqual("bar", keys.ElementAt(0));

            // Make a second nested entry to test prefix removal
            //~ e = &Entry{Key: "foo/zip", Value: []byte("zap")}
            //~ err = b.Put(e)
            //~ if err != nil {
            //~     t.Fatalf("err: %v", err)
            //~ }
            e = new Entry {
                Key = "foo/zip", Value = "zap".ToUtf8Bytes()
            };
            b.Put(e);

            // Delete should not remove the prefix
            //~ err = b.Delete("foo/bar")
            //~ if err != nil {
            //~     t.Fatalf("err: %v", err)
            //~ }
            b.Delete("foo/bar");

            //~ keys, err = b.List("")
            //~ if err != nil {
            //~     t.Fatalf("err: %v", err)
            //~ }
            //~ if len(keys) != 1 {
            //~     t.Fatalf("bad: %v", keys)
            //~ }
            //~ if keys[0] != "foo/" {
            //~     t.Fatalf("bad: %v", keys)
            //~ }
            keys = b.List("");
            Assert.AreEqual(1, keys.Count());
            Assert.AreEqual("foo/", keys.ElementAt(0));

            // Delete should remove the prefix
            //~ err = b.Delete("foo/zip")
            //~ if err != nil {
            //~     t.Fatalf("err: %v", err)
            //~ }
            b.Delete("foo/zip");

            //~ keys, err = b.List("")
            //~ if err != nil {
            //~     t.Fatalf("err: %v", err)
            //~ }
            //~ if len(keys) != 0 {
            //~     t.Fatalf("bad: %v", keys)
            //~ }
            keys = b.List("");
            Assert.AreEqual(0, keys.Count());
        }
Ejemplo n.º 6
0
        public static void TestBackend_ListPrefix(IBackend b)
        {
            //~ e1 := &Entry{Key: "foo", Value: []byte("test")}
            //~ e2 := &Entry{Key: "foo/bar", Value: []byte("test")}
            //~ e3 := &Entry{Key: "foo/bar/baz", Value: []byte("test")}
            var e1 = new Entry {
                Key = "foo", Value = "test".ToUtf8Bytes()
            };
            var e2 = new Entry {
                Key = "foo/bar", Value = "test".ToUtf8Bytes()
            };
            var e3 = new Entry {
                Key = "foo/bar/baz", Value = "test".ToUtf8Bytes()
            };

            //~ defer func() {
            //~     b.Delete("foo")
            //~     b.Delete("foo/bar")
            //~     b.Delete("foo/bar/baz")
            //~ }()

            using (var defer = new Util.Defer())
            {
                defer.Add(() =>
                {
                    b.Delete("foo");
                    b.Delete("foo/bar");
                    b.Delete("foo/bar/baz");
                });

                //~ err := b.Put(e1)
                //~ if err != nil {
                //~     t.Fatalf("err: %v", err)
                //~ }
                //~ err = b.Put(e2)
                //~ if err != nil {
                //~     t.Fatalf("err: %v", err)
                //~ }
                //~ err = b.Put(e3)
                //~ if err != nil {
                //~     t.Fatalf("err: %v", err)
                //~ }
                b.Put(e1);
                b.Put(e2);
                b.Put(e3);

                // Scan the root
                //~ keys, err := b.List("")
                //~ if err != nil {
                //~     t.Fatalf("err: %v", err)
                //~ }
                //~ if len(keys) != 2 {
                //~     t.Fatalf("bad: %v", keys)
                //~ }
                //~ sort.Strings(keys)
                //~ if keys[0] != "foo" {
                //~     t.Fatalf("bad: %v", keys)
                //~ }
                //~ if keys[1] != "foo/" {
                //~     t.Fatalf("bad: %v", keys)
                //~ }
                var keys = b.List("");
                Assert.AreEqual(2, keys.Count());
                keys = keys.OrderBy(x => x);
                Assert.AreEqual("foo", keys.ElementAt(0));
                Assert.AreEqual("foo/", keys.ElementAt(1));

                // Scan foo/
                //~ keys, err = b.List("foo/")
                //~ if err != nil {
                //~     t.Fatalf("err: %v", err)
                //~ }
                //~ if len(keys) != 2 {
                //~     t.Fatalf("bad: %v", keys)
                //~ }
                //~ sort.Strings(keys)
                //~ if keys[0] != "bar" {
                //~     t.Fatalf("bad: %v", keys)
                //~ }
                //~ if keys[1] != "bar/" {
                //~     t.Fatalf("bad: %v", keys)
                //~ }
                keys = b.List("foo/");
                Assert.AreEqual(2, keys.Count());
                keys = keys.OrderBy(x => x);
                Assert.AreEqual("bar", keys.ElementAt(0));
                Assert.AreEqual("bar/", keys.ElementAt(1));

                // Scan foo/bar/
                //~ keys, err = b.List("foo/bar/")
                //~ if err != nil {
                //~     t.Fatalf("err: %v", err)
                //~ }
                //~ sort.Strings(keys)
                //~ if len(keys) != 1 {
                //~     t.Fatalf("bad: %v", keys)
                //~ }
                //~ if keys[0] != "baz" {
                //~     t.Fatalf("bad: %v", keys)
                //~ }
                keys = b.List("foo/bar/");
                keys = keys.OrderBy(x => x);
                Assert.AreEqual(1, keys.Count());
                Assert.AreEqual("baz", keys.ElementAt(0));
            }
        }