private async Task <IList <Library.Interface.IFileEntry> > DoList()
{
await m_stats.SendEventAsync(BackendActionType.List, BackendEventType.Started, null, -1);
var r = m_backend.List().ToList();
var sb = new StringBuilder();
sb.AppendLine("[");
long count = 0;
foreach (var e in r)
{
if (count != 0)
{
sb.AppendLine(",");
}
count++;
sb.Append(JsonConvert.SerializeObject(e));
}
sb.AppendLine();
sb.Append("]");
await m_database.LogRemoteOperationAsync("list", "", sb.ToString());
await m_stats.SendEventAsync(BackendActionType.List, BackendEventType.Completed, null, r.Count);
return(r);
}
/// <summary>
/// Tests a backend by invoking the List() method.
/// As long as the iteration can either complete or find at least one file without throwing, the test is successful
/// </summary>
/// <param name="backend">Backend to test</param>
public static void TestList(this IBackend backend)
{
// If we can iterate successfully, even if it's empty, then the backend test is successful
foreach (IFileEntry file in backend.List())
{
break;
}
}
private async Task DoPut(FileEntryItem item, IBackend backend, CancellationToken cancelToken)
{
if (cancelToken.IsCancellationRequested)
{
return;
}
if (item.TrackedInDb)
{
await m_database.UpdateRemoteVolumeAsync(item.RemoteFilename, RemoteVolumeState.Uploading, item.Size, item.Hash);
}
if (m_options.Dryrun)
{
Logging.Log.WriteDryrunMessage(LOGTAG, "WouldUploadVolume", "Would upload volume: {0}, size: {1}", item.RemoteFilename, Library.Utility.Utility.FormatSizeString(new FileInfo(item.LocalFilename).Length));
item.DeleteLocalFile();
return;
}
await m_database.LogRemoteOperationAsync("put", item.RemoteFilename, JsonConvert.SerializeObject(new { Size = item.Size, Hash = item.Hash }));
await m_stats.SendEventAsync(BackendActionType.Put, BackendEventType.Started, item.RemoteFilename, item.Size, updateProgress : false);
m_progressUpdater.StartFileProgress(item.RemoteFilename, item.Size);
var begin = DateTime.Now;
if (!m_options.DisableStreamingTransfers && backend is IStreamingBackend streamingBackend)
{
// A download throttle speed is not given to the ThrottledStream as we are only uploading data here
using (var fs = File.OpenRead(item.LocalFilename))
using (var ts = new ThrottledStream(fs, m_initialUploadThrottleSpeed, 0))
using (var pgs = new ProgressReportingStream(ts, pg => HandleProgress(ts, pg, item.RemoteFilename)))
await streamingBackend.PutAsync(item.RemoteFilename, pgs, cancelToken).ConfigureAwait(false);
}
else
{
await backend.PutAsync(item.RemoteFilename, item.LocalFilename, cancelToken).ConfigureAwait(false);
}
var duration = DateTime.Now - begin;
m_progressUpdater.EndFileProgress(item.RemoteFilename);
Logging.Log.WriteProfilingMessage(LOGTAG, "UploadSpeed", "Uploaded {0} in {1}, {2}/s", Library.Utility.Utility.FormatSizeString(item.Size), duration, Library.Utility.Utility.FormatSizeString((long)(item.Size / duration.TotalSeconds)));
if (item.TrackedInDb)
{
await m_database.UpdateRemoteVolumeAsync(item.RemoteFilename, RemoteVolumeState.Uploaded, item.Size, item.Hash);
}
await m_stats.SendEventAsync(BackendActionType.Put, BackendEventType.Completed, item.RemoteFilename, item.Size);
if (m_options.ListVerifyUploads)
{
var f = backend.List().FirstOrDefault(n => n.Name.Equals(item.RemoteFilename, StringComparison.OrdinalIgnoreCase));
if (f == null)
{
throw new Exception(string.Format("List verify failed, file was not found after upload: {0}", item.RemoteFilename));
}
else if (f.Size != item.Size && f.Size >= 0)
{
throw new Exception(string.Format("List verify failed for file: {0}, size was {1} but expected to be {2}", f.Name, f.Size, item.Size));
}
}
item.DeleteLocalFile();
await m_database.CommitTransactionAsync("CommitAfterUpload");
}
private async Task <bool> DoPut(FileEntryItem item, bool updatedHash = false)
{
// If this is not already encrypted, do it now
item.Encrypt(m_options);
updatedHash |= item.UpdateHashAndSize(m_options);
if (updatedHash && item.TrackedInDb)
{
await m_database.UpdateRemoteVolumeAsync(item.RemoteFilename, RemoteVolumeState.Uploading, item.Size, item.Hash);
}
if (m_options.Dryrun)
{
Logging.Log.WriteDryrunMessage(LOGTAG, "WouldUploadVolume", "Would upload volume: {0}, size: {1}", item.RemoteFilename, Library.Utility.Utility.FormatSizeString(new FileInfo(item.LocalFilename).Length));
item.DeleteLocalFile();
return(true);
}
await m_database.LogRemoteOperationAsync("put", item.RemoteFilename, JsonConvert.SerializeObject(new { Size = item.Size, Hash = item.Hash }));
await m_stats.SendEventAsync(BackendActionType.Put, BackendEventType.Started, item.RemoteFilename, item.Size);
var begin = DateTime.Now;
if (m_backend is Library.Interface.IStreamingBackend && !m_options.DisableStreamingTransfers)
{
using (var fs = System.IO.File.OpenRead(item.LocalFilename))
using (var ts = new ThrottledStream(fs, m_options.MaxUploadPrSecond, m_options.MaxDownloadPrSecond))
using (var pgs = new Library.Utility.ProgressReportingStream(ts, item.Size, pg => HandleProgress(ts, pg)))
((Library.Interface.IStreamingBackend)m_backend).Put(item.RemoteFilename, pgs);
}
else
{
m_backend.Put(item.RemoteFilename, item.LocalFilename);
}
var duration = DateTime.Now - begin;
Logging.Log.WriteProfilingMessage(LOGTAG, "UploadSpeed", "Uploaded {0} in {1}, {2}/s", Library.Utility.Utility.FormatSizeString(item.Size), duration, Library.Utility.Utility.FormatSizeString((long)(item.Size / duration.TotalSeconds)));
if (item.TrackedInDb)
{
await m_database.UpdateRemoteVolumeAsync(item.RemoteFilename, RemoteVolumeState.Uploaded, item.Size, item.Hash);
}
await m_stats.SendEventAsync(BackendActionType.Put, BackendEventType.Completed, item.RemoteFilename, item.Size);
if (m_options.ListVerifyUploads)
{
var f = m_backend.List().Where(n => n.Name.Equals(item.RemoteFilename, StringComparison.OrdinalIgnoreCase)).FirstOrDefault();
if (f == null)
{
throw new Exception(string.Format("List verify failed, file was not found after upload: {0}", item.RemoteFilename));
}
else if (f.Size != item.Size && f.Size >= 0)
{
throw new Exception(string.Format("List verify failed for file: {0}, size was {1} but expected to be {2}", f.Name, f.Size, item.Size));
}
}
item.DeleteLocalFile();
await m_database.CommitTransactionAsync("CommitAfterUpload");
return(true);
}
public static void TestBackend(IBackend b)
{
// Should be empty
//~ keys, err := b.List("")
//~ if err != nil {
//~ t.Fatalf("err: %v", err)
//~ }
//~ if len(keys) != 0 {
//~ t.Fatalf("bad: %v", keys)
//~ }
var keys = b.List("");
Assert.AreEqual(0, keys.Count());
// Delete should work if it does not exist
//~ err = b.Delete("foo")
//~ if err != nil {
//~ t.Fatalf("err: %v", err)
//~ }
b.Delete("foo");
// Get should fail
//~ out, err := b.Get("foo")
//~ if err != nil {
//~ t.Fatalf("err: %v", err)
//~ }
//~ if out != nil {
//~ t.Fatalf("bad: %v", out)
//~ }
var @out = b.Get("foo");
Assert.IsNull(@out);
// Make an entry
//~ e := &Entry{Key: "foo", Value: []byte("test")}
//~ err = b.Put(e)
//~ if err != nil {
//~ t.Fatalf("err: %v", err)
//~ }
var e = new Entry {
Key = "foo", Value = "test".ToUtf8Bytes()
};
b.Put(e);
// Get should work
//~ out, err = b.Get("foo")
//~ if err != nil {
//~ t.Fatalf("err: %v", err)
//~ }
//~ if !reflect.DeepEqual(out, e) {
//~ t.Fatalf("bad: %v expected: %v", out, e)
//~ }
@out = b.Get("foo");
Assert.IsNotNull(@out);
// List should not be empty
//~ keys, err = b.List("")
//~ if err != nil {
//~ t.Fatalf("err: %v", err)
//~ }
//~ if len(keys) != 1 {
//~ t.Fatalf("bad: %v", keys)
//~ }
//~ if keys[0] != "foo" {
//~ t.Fatalf("bad: %v", keys)
//~ }
keys = b.List("");
Assert.AreEqual(1, keys.Count());
Assert.AreEqual("foo", keys.ElementAt(0));
// Delete should work
//~ err = b.Delete("foo")
//~ if err != nil {
//~ t.Fatalf("err: %v", err)
//~ }
b.Delete("foo");
// Should be empty
//~ keys, err = b.List("")
//~ if err != nil {
//~ t.Fatalf("err: %v", err)
//~ }
//~ if len(keys) != 0 {
//~ t.Fatalf("bad: %v", keys)
//~ }
keys = b.List("");
Assert.AreEqual(0, keys.Count());
// Get should fail
//~ out, err = b.Get("foo")
//~ if err != nil {
//~ t.Fatalf("err: %v", err)
//~ }
//~ if out != nil {
//~ t.Fatalf("bad: %v", out)
//~ }
@out = b.Get("foo");
Assert.IsNull(@out);
// Multiple Puts should work; GH-189
//~ e = &Entry{Key: "foo", Value: []byte("test")}
//~ err = b.Put(e)
//~ if err != nil {
//~ t.Fatalf("err: %v", err)
//~ }
//~ e = &Entry{Key: "foo", Value: []byte("test")}
//~ err = b.Put(e)
//~ if err != nil {
//~ t.Fatalf("err: %v", err)
//~ }
e = new Entry {
Key = "foo", Value = "test".ToUtf8Bytes()
};
b.Put(e);
e = new Entry {
Key = "foo", Value = "test".ToUtf8Bytes()
};
b.Put(e);
// Make a nested entry
//~ e = &Entry{Key: "foo/bar", Value: []byte("baz")}
//~ err = b.Put(e)
//~ if err != nil {
//~ t.Fatalf("err: %v", err)
//~ }
e = new Entry {
Key = "foo/bar", Value = "baz".ToUtf8Bytes()
};
b.Put(e);
//~ keys, err = b.List("")
//~ if err != nil {
//~ t.Fatalf("err: %v", err)
//~ }
//~ if len(keys) != 2 {
//~ t.Fatalf("bad: %v", keys)
//~ }
//~ sort.Strings(keys)
//~ if keys[0] != "foo" || keys[1] != "foo/" {
//~ t.Fatalf("bad: %v", keys)
//~ }
keys = b.List("");
Assert.AreEqual(2, keys.Count());
keys = keys.OrderBy(x => x);
Assert.AreEqual("foo", keys.ElementAt(0));
Assert.AreEqual("foo/", keys.ElementAt(1));
// Delete with children should work
//~ err = b.Delete("foo")
//~ if err != nil {
//~ t.Fatalf("err: %v", err)
//~ }
b.Delete("foo");
// Get should return the child
//~ out, err = b.Get("foo/bar")
//~ if err != nil {
//~ t.Fatalf("err: %v", err)
//~ }
//~ if out == nil {
//~ t.Fatalf("missing child")
//~ }
@out = b.Get("foo/bar");
Assert.IsNotNull(@out);
// Removal of nested secret should not leave artifacts
//~ e = &Entry{Key: "foo/nested1/nested2/nested3", Value: []byte("baz")}
//~ err = b.Put(e)
//~ if err != nil {
//~ t.Fatalf("err: %v", err)
//~ }
e = new Entry {
Key = "foo/nested1/nested2/nested3", Value = "baz".ToUtf8Bytes()
};
b.Put(e);
//~ err = b.Delete("foo/nested1/nested2/nested3")
//~ if err != nil {
//~ t.Fatalf("failed to remove nested secret: %v", err)
//~ }
b.Delete("foo/nested1/nested2/nested3");
//~ keys, err = b.List("foo/")
//~ if err != nil {
//~ t.Fatalf("err: %v", err)
//~ }
keys = b.List("foo/");
//~ if len(keys) != 1 {
//~ t.Fatalf("there should be only one key left after deleting nested "+
//~ "secret: %v", keys)
//~ }
Assert.AreEqual(1, keys.Count());
//~ if keys[0] != "bar" {
//~ t.Fatalf("bad keys after deleting nested: %v", keys)
//~ }
Assert.AreEqual("bar", keys.ElementAt(0));
// Make a second nested entry to test prefix removal
//~ e = &Entry{Key: "foo/zip", Value: []byte("zap")}
//~ err = b.Put(e)
//~ if err != nil {
//~ t.Fatalf("err: %v", err)
//~ }
e = new Entry {
Key = "foo/zip", Value = "zap".ToUtf8Bytes()
};
b.Put(e);
// Delete should not remove the prefix
//~ err = b.Delete("foo/bar")
//~ if err != nil {
//~ t.Fatalf("err: %v", err)
//~ }
b.Delete("foo/bar");
//~ keys, err = b.List("")
//~ if err != nil {
//~ t.Fatalf("err: %v", err)
//~ }
//~ if len(keys) != 1 {
//~ t.Fatalf("bad: %v", keys)
//~ }
//~ if keys[0] != "foo/" {
//~ t.Fatalf("bad: %v", keys)
//~ }
keys = b.List("");
Assert.AreEqual(1, keys.Count());
Assert.AreEqual("foo/", keys.ElementAt(0));
// Delete should remove the prefix
//~ err = b.Delete("foo/zip")
//~ if err != nil {
//~ t.Fatalf("err: %v", err)
//~ }
b.Delete("foo/zip");
//~ keys, err = b.List("")
//~ if err != nil {
//~ t.Fatalf("err: %v", err)
//~ }
//~ if len(keys) != 0 {
//~ t.Fatalf("bad: %v", keys)
//~ }
keys = b.List("");
Assert.AreEqual(0, keys.Count());
}
public static void TestBackend_ListPrefix(IBackend b)
{
//~ e1 := &Entry{Key: "foo", Value: []byte("test")}
//~ e2 := &Entry{Key: "foo/bar", Value: []byte("test")}
//~ e3 := &Entry{Key: "foo/bar/baz", Value: []byte("test")}
var e1 = new Entry {
Key = "foo", Value = "test".ToUtf8Bytes()
};
var e2 = new Entry {
Key = "foo/bar", Value = "test".ToUtf8Bytes()
};
var e3 = new Entry {
Key = "foo/bar/baz", Value = "test".ToUtf8Bytes()
};
//~ defer func() {
//~ b.Delete("foo")
//~ b.Delete("foo/bar")
//~ b.Delete("foo/bar/baz")
//~ }()
using (var defer = new Util.Defer())
{
defer.Add(() =>
{
b.Delete("foo");
b.Delete("foo/bar");
b.Delete("foo/bar/baz");
});
//~ err := b.Put(e1)
//~ if err != nil {
//~ t.Fatalf("err: %v", err)
//~ }
//~ err = b.Put(e2)
//~ if err != nil {
//~ t.Fatalf("err: %v", err)
//~ }
//~ err = b.Put(e3)
//~ if err != nil {
//~ t.Fatalf("err: %v", err)
//~ }
b.Put(e1);
b.Put(e2);
b.Put(e3);
// Scan the root
//~ keys, err := b.List("")
//~ if err != nil {
//~ t.Fatalf("err: %v", err)
//~ }
//~ if len(keys) != 2 {
//~ t.Fatalf("bad: %v", keys)
//~ }
//~ sort.Strings(keys)
//~ if keys[0] != "foo" {
//~ t.Fatalf("bad: %v", keys)
//~ }
//~ if keys[1] != "foo/" {
//~ t.Fatalf("bad: %v", keys)
//~ }
var keys = b.List("");
Assert.AreEqual(2, keys.Count());
keys = keys.OrderBy(x => x);
Assert.AreEqual("foo", keys.ElementAt(0));
Assert.AreEqual("foo/", keys.ElementAt(1));
// Scan foo/
//~ keys, err = b.List("foo/")
//~ if err != nil {
//~ t.Fatalf("err: %v", err)
//~ }
//~ if len(keys) != 2 {
//~ t.Fatalf("bad: %v", keys)
//~ }
//~ sort.Strings(keys)
//~ if keys[0] != "bar" {
//~ t.Fatalf("bad: %v", keys)
//~ }
//~ if keys[1] != "bar/" {
//~ t.Fatalf("bad: %v", keys)
//~ }
keys = b.List("foo/");
Assert.AreEqual(2, keys.Count());
keys = keys.OrderBy(x => x);
Assert.AreEqual("bar", keys.ElementAt(0));
Assert.AreEqual("bar/", keys.ElementAt(1));
// Scan foo/bar/
//~ keys, err = b.List("foo/bar/")
//~ if err != nil {
//~ t.Fatalf("err: %v", err)
//~ }
//~ sort.Strings(keys)
//~ if len(keys) != 1 {
//~ t.Fatalf("bad: %v", keys)
//~ }
//~ if keys[0] != "baz" {
//~ t.Fatalf("bad: %v", keys)
//~ }
keys = b.List("foo/bar/");
keys = keys.OrderBy(x => x);
Assert.AreEqual(1, keys.Count());
Assert.AreEqual("baz", keys.ElementAt(0));
}
}