/// <summary>
/// Runs the upload process
/// </summary>
/// <returns>A tuple with the completion task and the channel to use</returns>
public static Tuple <Task, IWriteChannel <string> > Run()
{
var channel = ChannelManager.CreateChannel <string>(
buffersize: MAX_PENDING_UPLOADS,
pendingWritersOverflowStrategy: QueueOverflowStrategy.LIFO
);
var task = AutomationExtensions.RunTask(
channel.AsRead(),
async(chan) =>
{
while (true)
{
var f = await chan.ReadAsync();
try
{
if (File.Exists(f))
{
var req = (HttpWebRequest)WebRequest.Create(UPLOAD_URL);
req.Method = "POST";
req.ContentType = "application/json; charset=utf-8";
int rc;
using (var fs = File.OpenRead(f))
{
if (fs.Length > 0)
{
req.ContentLength = fs.Length;
var areq = new Library.Utility.AsyncHttpRequest(req);
using (var rs = areq.GetRequestStream())
Library.Utility.Utility.CopyStream(fs, rs);
using (var resp = (HttpWebResponse)areq.GetResponse())
rc = (int)resp.StatusCode;
}
else
{
rc = 200;
}
}
if (rc >= 200 && rc <= 299)
{
File.Delete(f);
}
}
}
catch (Exception ex)
{
Logging.Log.WriteMessage("UsageReporter failed", Duplicati.Library.Logging.LogMessageType.Error, ex);
}
}
}
);
return(new Tuple <Task, IWriteChannel <string> >(task, channel));
}
public static Task RunGranterAsync(IWriteChannel <bool> channel, long count, CancellationToken token)
{
var canceltask = new TaskCompletionSource <bool>();
token.Register(() => canceltask.TrySetCanceled());
var total = count;
return(AutomationExtensions.RunTask(
new { channel },
async self => {
while (count > 0)
{
DebugWriteLine($"Emitting task {total - count} of {total}");
if (await Task.WhenAny(new [] { canceltask.Task, channel.WriteAsync(true) }) == canceltask.Task)
{
throw new TaskCanceledException();
}
count--;
DebugWriteLine($"Emitted task {total - count} of {total}");
}
DebugWriteLine("Stopping task granter");
}
));
}
/// <summary>
/// Reads input and applies the method to each input, and emits the output
/// </summary>
/// <param name="method">The worker method to apply to each element.</param>
/// <typeparam name="TInput">The input type parameter.</typeparam>
/// <typeparam name="TOutput">The output type parameter.</typeparam>
private static Task Worker <TInput, TOutput>(Func <TInput, TOutput> method)
{
return(AutomationExtensions.RunTask(
new {
input = ChannelMarker.ForRead <TInput>(WORKERINPUT),
output = ChannelMarker.ForWrite <TOutput>(WORKEROUTPUT)
},
async self => {
try
{
while (true)
{
await self.output.WriteAsync(method(await self.input.ReadAsync().ConfigureAwait(false))).ConfigureAwait(false);
}
}
catch (Exception ex)
{
if (!(ex is RetiredException))
{
Console.WriteLine("ex: {0}", ex);
}
throw;
}
}
));
}
public static Task Run(BackupDatabase database, Options options, ITaskReader taskreader)
{
return(AutomationExtensions.RunTask(new
{
UploadChannel = Channels.BackendRequest.ForWrite
},
async self =>
{
if (options.IndexfilePolicy != Options.IndexFileStrategy.None)
{
foreach (var blockfile in await database.GetMissingIndexFilesAsync())
{
if (!await taskreader.ProgressAsync)
{
return;
}
Logging.Log.WriteInformationMessage(LOGTAG, "RecreateMissingIndexFile", "Re-creating missing index file for {0}", blockfile);
var w = await Common.IndexVolumeCreator.CreateIndexVolume(blockfile, options, database);
if (!await taskreader.ProgressAsync)
{
return;
}
await database.UpdateRemoteVolumeAsync(w.RemoteFilename, RemoteVolumeState.Uploading, -1, null);
await self.UploadChannel.WriteAsync(new IndexVolumeUploadRequest(w));
}
}
}));
}
public SingleRunner()
{
AutomationExtensions.AutoWireChannels(this, null);
m_channel = ChannelManager.CreateChannel <Func <Task> >();
m_workerSource = new System.Threading.CancellationTokenSource();
m_worker = AutomationExtensions.RunProtected(this, Start);
}
/// <summary>
/// Runs the MRU cache
/// </summary>
/// <returns>An awaitable task.</returns>
/// <param name="selfinfo">This peer's information</param>
/// <param name="storesize">The size of the MRU store</param>
/// <param name="maxage">The maximum amount of time items are stored</param>
/// <param name="buffersize">The size of the forwarding buffer.</param>
public static Task RunAsync(PeerInfo selfinfo, int storesize, TimeSpan maxage, int buffersize = 10)
{
var parent = RunMRUAsync(selfinfo, storesize, maxage, buffersize);
return(Task.WhenAll(
parent,
AutomationExtensions.RunTask(
new { Request = Channels.MRURequests.ForWrite },
async self =>
{
while (true)
{
// Sleep, but quit if the MRU stops
if (await Task.WhenAny(parent, Task.Delay(new TimeSpan(maxage.Ticks / 3))) == parent)
{
return;
}
log.Debug("Invoking store expiration");
await self.Request.SendExpireAsync();
log.Debug("Store expiration completed, waiting ...");
}
}
)
));
}
private static void EnsureNotepadHasExpectedMenus(AutomationElement windowElement)
{
var menuBar = windowElement.FindFirstDescendantById("MenuBar");
menuBar.ExecuteWithUpdatedCache(
AutomationExtensions.BuildCacheRequest(TreeScope.Element | TreeScope.Children, AutomationElement.NameProperty),
m => CollectionAssert.AreEqual(NotepadMenuNames, GetChildNames(m)));
}
public static Task Run(Snapshots.ISnapshotService snapshot, Options options, BackupStatsCollector stats, BackupDatabase database)
{
return(AutomationExtensions.RunTask(
new
{
Input = Channels.ProcessedFiles.ForRead,
Output = Channels.AcceptedChangedFile.ForWrite
},
async self =>
{
var EMPTY_METADATA = Utility.WrapMetadata(new Dictionary <string, string>(), options);
var blocksize = options.Blocksize;
while (true)
{
var e = await self.Input.ReadAsync();
long filestatsize = -1;
try
{
filestatsize = snapshot.GetFileSize(e.Path);
}
catch (Exception ex)
{
Logging.Log.WriteExplicitMessage(FILELOGTAG, "FailedToReadSize", ex, "Failed tp read size of file: {0}", e.Path);
}
await stats.AddExaminedFile(filestatsize);
e.MetaHashAndSize = options.StoreMetadata ? Utility.WrapMetadata(await MetadataGenerator.GenerateMetadataAsync(e.Path, e.Attributes, options, snapshot), options) : EMPTY_METADATA;
var timestampChanged = e.LastWrite != e.OldModified || e.LastWrite.Ticks == 0 || e.OldModified.Ticks == 0;
var filesizeChanged = filestatsize < 0 || e.LastFileSize < 0 || filestatsize != e.LastFileSize;
var tooLargeFile = options.SkipFilesLargerThan != long.MaxValue && options.SkipFilesLargerThan != 0 && filestatsize >= 0 && filestatsize > options.SkipFilesLargerThan;
e.MetadataChanged = !options.CheckFiletimeOnly && !options.SkipMetadata && (e.MetaHashAndSize.Blob.Length != e.OldMetaSize || e.MetaHashAndSize.FileHash != e.OldMetaHash);
if ((e.OldId < 0 || options.DisableFiletimeCheck || timestampChanged || filesizeChanged || e.MetadataChanged) && !tooLargeFile)
{
Logging.Log.WriteVerboseMessage(FILELOGTAG, "CheckFileForChanges", "Checking file for changes {0}, new: {1}, timestamp changed: {2}, size changed: {3}, metadatachanged: {4}, {5} vs {6}", e.Path, e.OldId <= 0, timestampChanged, filesizeChanged, e.MetadataChanged, e.LastWrite, e.OldModified);
await self.Output.WriteAsync(e);
}
else
{
if (tooLargeFile)
{
Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckTooLarge", "Skipped checking file, because the size exceeds limit {0}", e.Path);
}
else
{
Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckNoTimestampChange", "Skipped checking file, because timestamp was not updated {0}", e.Path);
}
await database.AddUnmodifiedAsync(e.OldId, e.LastWrite);
}
}
}));
}
public static void Run(Arguments arguments)
{
if (arguments.ShowHelp)
{
Usage();
return;
}
AutomationElement.FromHandle(arguments.WindowHandle).ExecuteWithUpdatedCache(AutomationExtensions.BuildCacheRequest(TreeScope.Element | TreeScope.Descendants, AutomationElement.NameProperty),
element => DumpValuesRecursive(element, 0));
}
public void TestRetireWithoutLoss()
{
Task[] tasks;
int count = 0;
using (new ChannelScope())
{
tasks = new Task[] {
AutomationExtensions.RunTask(
new { channel = ChannelMarker.ForWrite <int>(CHANNEL_NAME) },
async self =>
{
await Task.Delay(500);
await self.channel.WriteAsync(1);
}
),
AutomationExtensions.RunTask(
new { channel = ChannelMarker.ForWrite <int>(CHANNEL_NAME) },
async self =>
{
await Task.Delay(1000);
await self.channel.WriteAsync(1);
}
),
AutomationExtensions.RunTask(
new { channel = ChannelMarker.ForRead <int>(CHANNEL_NAME) },
async self =>
{
while (true)
{
await self.channel.ReadAsync();
count++;
}
}
)
};
}
var all = Task.WhenAll(tasks).WaitForTask();
if (count != 2)
{
throw new Exception(string.Format("Unexpected count, expected {0} but got {1}", 2, count));
}
if (all.IsFaulted || !all.IsCompleted)
{
throw new Exception("Unexpected task state");
}
}
/// <summary>
/// Emits all values from the enumerable into the network
/// </summary>
/// <param name="values">Values.</param>
/// <typeparam name="TInput">The 1st type parameter.</typeparam>
private static Task Generator <TInput>(IEnumerable <TInput> values)
{
return(AutomationExtensions.RunTask(
new { channel = ChannelMarker.ForWrite <TInput>(WORKERINPUT) },
async self => {
foreach (var value in values)
{
await self.channel.WriteAsync(value).ConfigureAwait(false);
}
}
));
}
protected virtual void Dispose(bool isDisposing)
{
m_workerSource.Cancel();
if (m_channel != null)
{
try { m_channel.Retire(); }
catch { }
finally { }
}
AutomationExtensions.RetireAllChannels(this);
}
public static Task RunWriter <T>(IChannel <T> channel, IEnumerable <T> values)
{
return(AutomationExtensions.RunTask(
new { chan = channel.AsWriteOnly() },
async self =>
{
foreach (var v in values)
{
await self.chan.WriteAsync(v);
}
}
));
}
private static void EnsureNotepadHasExpectedMenusWithHierarchicalValueProvider(AutomationElement windowElement)
{
var hvp = new HierarchicalValueProvider <AutomationElement, string, string>(windowElement);
var menuBar = windowElement.FindFirstDescendantById("MenuBar");
menuBar.ExecuteWithUpdatedCache(
AutomationExtensions.BuildCacheRequest(TreeScope.Element | TreeScope.Children, AutomationElement.NameProperty),
m =>
{
var values = hvp.GetValues(BuildNotepadMenuTree());
CollectionAssert.AreEqual(NotepadMenuNames, values["menu"]);
});
}
public void TestChannelWire()
{
Reader x1, x2;
Writer y;
IRetireAbleChannel c;
using (new ChannelScope())
{
AutomationExtensions.AutoWireChannels(new object[] {
x1 = new Reader(),
x2 = new Reader(),
y = new Writer()
});
c = ChannelManager.GetChannel <int>("input");
}
if (x1 == null || !x1.HasChannel || x2 == null || !x2.HasChannel)
{
throw new Exception("Autoloader failed to load channel");
}
if (ChannelScope.Current != ChannelScope.Root)
{
throw new Exception("Unexpected current scope");
}
AutomationExtensions.RetireAllChannels(x1);
if (c.IsRetiredAsync.WaitForTask().Result)
{
throw new Exception("Unexpected early retire");
}
AutomationExtensions.RetireAllChannels(x2);
if (!c.IsRetiredAsync.WaitForTask().Result)
{
throw new Exception("Unexpected non-retire");
}
using (new ChannelScope())
AutomationExtensions.AutoWireChannels(y = new Writer());
if (y == null || !y.HasChannel || y.IsChannelRetired)
{
throw new Exception("Scope does not appear isolated");
}
}
private void TestReaderOverflow(QueueOverflowStrategy strategy)
{
using (new IsolatedChannelScope())
{
var readertasks = Enumerable.Range(0, 4).Select(count =>
AutomationExtensions.RunTask(new {
Input = ChannelMarker.ForRead <int>("channel", maxPendingReaders: 3, pendingReadersOverflowStrategy: strategy)
},
async x =>
{
//Console.WriteLine("Started {0}", count);
while (true)
{
await x.Input.ReadAsync();
}
})
).ToList();
using (ChannelManager.GetChannel <int>("channel").AsWriteOnly())
Task.Delay(500).WaitForTaskOrThrow();
Task.WhenAny(readertasks.Union(new [] { Task.Delay(1000) })).WaitForTaskOrThrow();
Task.Delay(500).WaitForTaskOrThrow();
int discard;
switch (strategy)
{
case QueueOverflowStrategy.FIFO:
discard = 0;
break;
case QueueOverflowStrategy.LIFO:
discard = readertasks.Count - 2;
break;
case QueueOverflowStrategy.Reject:
default:
discard = readertasks.Count - 1;
break;
}
Assert.IsTrue(readertasks[discard].IsFaulted);
TestAssert.IsInstanceOf <ChannelOverflowException>(readertasks[discard].Exception.Flatten().InnerExceptions.First());
readertasks.RemoveAt(discard);
Assert.IsTrue(readertasks.All(x => x.IsCompleted && !x.IsFaulted && !x.IsCanceled));
}
}
/// <summary>
/// Collects input and combines it with the join method
/// </summary>
/// <param name="joinmethod">The method used to join results.</param>
/// <param name="initial">The initial input to the join method, aka. the neutral element.</param>
/// <typeparam name="TOutput">The type parameter for the data to join.</typeparam>
/// <typeparam name="TResult">The type parameter for the aggregated data.</typeparam>
private static async Task <TResult> Collector <TOutput, TResult>(Func <TResult, TOutput, TResult> joinmethod, TResult initial)
{
var current = initial;
await AutomationExtensions.RunTask(
new { channel = ChannelMarker.ForRead <TOutput>(WORKEROUTPUT) },
async self => {
while (true)
{
current = joinmethod(current, await self.channel.ReadAsync().ConfigureAwait(false));
}
}
).ConfigureAwait(false);
return(current);
}
public static Task Run(Snapshots.ISnapshotService snapshot, BackupResults result, Options options, IFilter sourcefilter, IFilter filter, Common.ITaskReader taskreader, System.Threading.CancellationToken token)
{
// Make sure we create the enumeration process in a seperate scope,
// but keep the log channel from the parent scope
using (Logging.Log.StartIsolatingScope())
using (new IsolatedChannelScope())
{
var enumeratorTask = Backup.FileEnumerationProcess.Run(snapshot, options.FileAttributeFilter, sourcefilter, filter, options.SymlinkPolicy, options.HardlinkPolicy, options.ExcludeEmptyFolders, options.IgnoreFilenames, options.ChangedFilelist, taskreader);
var counterTask = AutomationExtensions.RunTask(new
{
Input = Backup.Channels.SourcePaths.ForRead
},
async self =>
{
var count = 0L;
var size = 0L;
try
{
while (await taskreader.ProgressAsync && !token.IsCancellationRequested)
{
var path = await self.Input.ReadAsync();
count++;
try
{
size += snapshot.GetFileSize(path);
}
catch
{
}
result.OperationProgressUpdater.UpdatefileCount(count, size, false);
}
}
finally
{
result.OperationProgressUpdater.UpdatefileCount(count, size, true);
}
});
return(Task.WhenAll(enumeratorTask, counterTask));
}
}
public static Task RunReader <T>(IChannel <T> channel, IEnumerable <T> values, CounterShim counter)
{
return(AutomationExtensions.RunTask(
new { chan = channel.AsReadOnly() },
async self =>
{
foreach (var v in values)
{
var r = await self.chan.ReadAsync();
counter.Increment();
if (Comparer <T> .Default.Compare(v, r) != 0)
{
throw new Exception(string.Format("Got {0} but expected {1}", r, v));
}
}
}
));
}
public void TestAttributes()
{
var values = new[] { 0, 1, 2, 3, 4 };
var counter = new CounterShim();
var readercount = 10;
var name = "bcast";
using (new IsolatedChannelScope())
{
var writer = AutomationExtensions.RunTask(
new Writer(),
async self =>
{
foreach (var v in values)
{
await self.chan.WriteAsync(v);
}
}
);
var readers = Enumerable.Range(0, readercount).Select(x => AutomationExtensions.RunTask(
new { chan = ChannelMarker.ForRead <int>(name) },
async self =>
{
foreach (var v in values)
{
var r = await self.chan.ReadAsync();
counter.Increment();
if (Comparer <int> .Default.Compare(v, r) != 0)
{
throw new Exception(string.Format("Got {0} but expected {1}", r, v));
}
}
}
)).ToArray();
Task.WhenAll(readers.Union(new[] { writer })).WaitForTaskOrThrow();
if (counter.Count != readercount * values.Length)
{
throw new Exception(string.Format("The counter said {0} values were read, but {1} was expected", counter.Count, readercount * values.Length));
}
}
}
public static Task RunStatPrinterAsync(IWriteChannel <StatRequest> channel, TimeSpan period, long total, CancellationToken token)
{
return(AutomationExtensions.RunTask(
new { channel },
async _ => {
while (true)
{
await Task.Delay(period, token);
var tcs = new TaskCompletionSource <ResultStats>();
await channel.WriteAsync(new StatRequest()
{
Result = tcs
});
var res = await tcs.Task;
var pg = (res.Requests / (double)total) * 100;
Console.WriteLine($" {pg:0.00}% ({res.Requests} of {total}) {(res.Failures == 0 ? "" : $"{res.Failures} {(res.Failures == 1 ? "failure" : "failures")}")}");
}
}
private static Task WriteChannel(StreamWriter stream, IReadChannelEnd <string> ch)
{
stream.AutoFlush = true;
return(AutomationExtensions.RunTask(
new {
Input = ch
},
async self =>
{
using (stream)
{
while (true)
{
var line = await self.Input.ReadAsync();
await stream.WriteLineAsync(line).ConfigureAwait(false);
//Console.WriteLine("Wrote {0}", line);
}
}
}
));
}
public static List <AutomationElement> FindAll(AutomationElement root, TreeScope scope, Condition condition, int processId)
{
return((List <AutomationElement>) STAHelper.Invoke(
delegate() {
int elementProcessId = (int)root.GetCurrentPropertyValue(AutomationElement.ProcessIdProperty);
if (elementProcessId != processId)
{
// This happens when the element represents the desktop.
// We could just filter using the ProcessIdProperty but this searches all nodes and *then* filters,
// which is incredibly slow if we're searching a lot of nodes (i.e. TreeScope is Descendant).
// Instead we find all direct children with the right process id and then search them inclusively.
// Helpfully, there's a Subtree TreeScope which does what we want
Condition processCondition = new PropertyCondition2(AutomationElement.ProcessIdProperty, processId);
if (scope == TreeScope.Descendants)
{
List <AutomationElement> roots = AutomationExtensions.FindAllRaw(root, TreeScope.Children, processCondition);
List <AutomationElement> mergedResults = new List <AutomationElement>();
foreach (AutomationElement currentRoot in roots)
{
mergedResults.AddRange(FindAll(currentRoot, TreeScope.Subtree, condition, processId));
}
return mergedResults;
}
else
{
condition = (condition == null) ? processCondition : new AndCondition(condition, processCondition);
}
}
if (condition == null)
{
condition = Condition.TrueCondition;
}
return AutomationExtensions.FindAllRaw(root, scope, condition);
}
));
}
/// <summary>
/// The runner helper method that calls the abstract request method
/// </summary>
/// <returns>An awaitable task</returns>
protected Task RunAsync(IReadChannel <bool> reqchan, IWriteChannel <RequestResult> respchan)
{
return(AutomationExtensions.RunTask(
new { reqchan, respchan },
async _ =>
{
while (await reqchan.ReadAsync())
{
var start = DateTime.Now;
try
{
var resp = await PeformRequestAsync();
await respchan.WriteAsync(new RequestResult()
{
Started = start,
Finished = DateTime.Now,
Failed = m_expectedresponse != null && m_expectedresponse != resp
});
}
catch (System.Exception ex)
{
await respchan.WriteAsync(new RequestResult()
{
Started = start,
Finished = DateTime.Now,
Failed = true,
Exception = ex
});
if (m_options.Verbose)
{
Console.WriteLine(ex.Message);
}
}
}
}));
}
public static Task Run(BackupDatabase database, Options options, ITaskReader taskreader)
{
return(AutomationExtensions.RunTask(
new
{
Input = Channels.OutputBlocks.ForRead,
Output = Channels.BackendRequest.ForWrite,
SpillPickup = Channels.SpillPickup.ForWrite,
},
async self =>
{
var noIndexFiles = options.IndexfilePolicy == Options.IndexFileStrategy.None;
var fullIndexFiles = options.IndexfilePolicy == Options.IndexFileStrategy.Full;
BlockVolumeWriter blockvolume = null;
TemporaryIndexVolume indexvolume = null;
try
{
while (true)
{
var b = await self.Input.ReadAsync();
// Lazy-start a new block volume
if (blockvolume == null)
{
// Before we start a new volume, probe to see if it exists
// This will delay creation of volumes for differential backups
// There can be a race, such that two workers determine that
// the block is missing, but this will be solved by the AddBlock call
// which runs atomically
if (await database.FindBlockIDAsync(b.HashKey, b.Size) >= 0)
{
b.TaskCompletion.TrySetResult(false);
continue;
}
blockvolume = new BlockVolumeWriter(options);
blockvolume.VolumeID = await database.RegisterRemoteVolumeAsync(blockvolume.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary);
indexvolume = noIndexFiles ? null : new TemporaryIndexVolume(options);
}
var newBlock = await database.AddBlockAsync(b.HashKey, b.Size, blockvolume.VolumeID);
b.TaskCompletion.TrySetResult(newBlock);
if (newBlock)
{
blockvolume.AddBlock(b.HashKey, b.Data, b.Offset, (int)b.Size, b.Hint);
if (indexvolume != null)
{
indexvolume.AddBlock(b.HashKey, b.Size);
if (b.IsBlocklistHashes && fullIndexFiles)
{
indexvolume.AddBlockListHash(b.HashKey, b.Size, b.Data);
}
}
// If the volume is full, send to upload
if (blockvolume.Filesize > options.VolumeSize - options.Blocksize)
{
//When uploading a new volume, we register the volumes and then flush the transaction
// this ensures that the local database and remote storage are as closely related as possible
await database.UpdateRemoteVolumeAsync(blockvolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null);
blockvolume.Close();
await database.CommitTransactionAsync("CommitAddBlockToOutputFlush");
FileEntryItem blockEntry = blockvolume.CreateFileEntryForUpload(options);
TemporaryIndexVolume indexVolumeCopy = null;
if (indexvolume != null)
{
indexVolumeCopy = new TemporaryIndexVolume(options);
indexvolume.CopyTo(indexVolumeCopy, false);
}
var uploadRequest = new VolumeUploadRequest(blockvolume, blockEntry, indexVolumeCopy, options, database);
blockvolume = null;
indexvolume = null;
// Write to output at the end here to prevent sending a full volume to the SpillCollector
await self.Output.WriteAsync(uploadRequest);
}
}
// We ignore the stop signal, but not the pause and terminate
await taskreader.ProgressAsync;
}
}
catch (Exception ex)
{
if (ex.IsRetiredException())
{
// If we have collected data, merge all pending volumes into a single volume
if (blockvolume != null && blockvolume.SourceSize > 0)
{
await self.SpillPickup.WriteAsync(new SpillVolumeRequest(blockvolume, indexvolume));
}
}
throw;
}
}));
}
public static Task Run(Snapshots.ISnapshotService snapshot, Options options, BackupDatabase database, long lastfilesetid, CancellationToken token)
{
return(AutomationExtensions.RunTask(new
{
Input = Backup.Channels.SourcePaths.ForRead,
StreamBlockChannel = Channels.StreamBlock.ForWrite,
Output = Backup.Channels.ProcessedFiles.ForWrite,
},
async self =>
{
var emptymetadata = Utility.WrapMetadata(new Dictionary <string, string>(), options);
var prevprefix = new KeyValuePair <string, long>(null, -1);
var CHECKFILETIMEONLY = options.CheckFiletimeOnly;
var DISABLEFILETIMECHECK = options.DisableFiletimeCheck;
while (true)
{
var path = await self.Input.ReadAsync();
var lastwrite = new DateTime(0, DateTimeKind.Utc);
var attributes = default(FileAttributes);
try
{
lastwrite = snapshot.GetLastWriteTimeUtc(path);
}
catch (Exception ex)
{
Logging.Log.WriteWarningMessage(FILELOGTAG, "TimestampReadFailed", ex, "Failed to read timestamp on \"{0}\"", path);
}
try
{
attributes = snapshot.GetAttributes(path);
}
catch (Exception ex)
{
Logging.Log.WriteVerboseMessage(FILELOGTAG, "FailedAttributeRead", "Failed to read attributes from {0}: {1}", path, ex.Message);
}
// If we only have metadata, stop here
if (await ProcessMetadata(path, attributes, lastwrite, options, snapshot, emptymetadata, database, self.StreamBlockChannel).ConfigureAwait(false))
{
try
{
var split = Database.LocalDatabase.SplitIntoPrefixAndName(path);
long prefixid;
if (string.Equals(prevprefix.Key, split.Key, StringComparison.Ordinal))
{
prefixid = prevprefix.Value;
}
else
{
prefixid = await database.GetOrCreatePathPrefix(split.Key);
prevprefix = new KeyValuePair <string, long>(split.Key, prefixid);
}
if (CHECKFILETIMEONLY || DISABLEFILETIMECHECK)
{
var tmp = await database.GetFileLastModifiedAsync(prefixid, split.Value, lastfilesetid, false);
await self.Output.WriteAsync(new FileEntry
{
OldId = tmp.Item1,
Path = path,
PathPrefixID = prefixid,
Filename = split.Value,
Attributes = attributes,
LastWrite = lastwrite,
OldModified = tmp.Item2,
LastFileSize = tmp.Item3,
OldMetaHash = null,
OldMetaSize = -1
});
}
else
{
var res = await database.GetFileEntryAsync(prefixid, split.Value, lastfilesetid);
await self.Output.WriteAsync(new FileEntry
{
OldId = res == null ? -1 : res.id,
Path = path,
PathPrefixID = prefixid,
Filename = split.Value,
Attributes = attributes,
LastWrite = lastwrite,
OldModified = res == null ? new DateTime(0) : res.modified,
LastFileSize = res == null ? -1 : res.filesize,
OldMetaHash = res == null ? null : res.metahash,
OldMetaSize = res == null ? -1 : res.metasize
});
}
}
catch (Exception ex)
{
if (ex.IsRetiredException() || token.IsCancellationRequested)
{
continue;
}
Logging.Log.WriteWarningMessage(FILELOGTAG, "ProcessingMetadataFailed", ex,
"Failed to process entry, path: {0}", path);
}
}
}
}));
}
/// <summary>
/// Runs the report processor
/// </summary>
/// <param name="forward">The channel accepting filenames with usage reports.</param>
internal static Tuple <Task, IWriteChannel <ReportItem> > Run(IWriteChannel <string> forward)
{
var instanceid = System.Diagnostics.Process.GetCurrentProcess().Id.ToString();
var channel = ChannelManager.CreateChannel <ReportItem>(
maxPendingWriters: MAX_QUEUE_SIZE,
pendingWritersOverflowStrategy: QueueOverflowStrategy.LIFO
);
var task = AutomationExtensions.RunTask(
new
{
Input = channel.AsRead(),
Output = forward
},
async(self) =>
{
// Wait 20 seconds before we start transmitting
for (var i = 0; i < 20; i++)
{
await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
if (await self.Input.IsRetiredAsync)
{
return;
}
}
await ProcessAbandonedFiles(self.Output, self.Input, null).ConfigureAwait(false);
var rs = new ReportSet();
var tf = GetTempFilename(instanceid);
var nextTransmitTarget = new DateTime(0);
while (true)
{
var forceSend = false;
try
{
// We wait until we get an item, or WAIT_TIME from the last event
var waittime =
rs.Items.Count == 0
? Timeout.Infinite
: new TimeSpan(Math.Max(0, (nextTransmitTarget - DateTime.UtcNow).Ticks));
var item = await self.Input.ReadAsync(waittime);
if (item != null)
{
if (rs.Items.Count == 0)
{
nextTransmitTarget = DateTime.UtcNow + WAIT_TIME;
}
forceSend = item.Type == ReportType.Crash;
rs.Items.Add(item);
File.WriteAllText(tf, JsonConvert.SerializeObject(rs));
}
}
catch (TimeoutException)
{
forceSend = true;
}
if ((forceSend && rs.Items.Count > 0) || (rs.Items.Count > MAX_ITEMS_IN_SET))
{
var nextFilename = GetTempFilename(instanceid);
self.Output.WriteNoWait(tf);
rs = new ReportSet();
await ProcessAbandonedFiles(self.Output, self.Input, null);
tf = nextFilename;
}
}
}
);
return(new Tuple <Task, IWriteChannel <ReportItem> >(task, channel));
}
public Task Run()
{
return(AutomationExtensions.RunTask(new
{
Input = Channels.BackendRequest.ForRead,
},
async self =>
{
var workers = new List <Worker>();
m_maxConcurrentUploads = m_options.AsynchronousConcurrentUploadLimit <= 0 ? int.MaxValue : m_options.AsynchronousConcurrentUploadLimit;
m_initialUploadThrottleSpeed = m_options.AsynchronousConcurrentUploadLimit <= 0 ? int.MaxValue : m_options.MaxUploadPrSecond / m_maxConcurrentUploads;
var lastSize = -1L;
var uploadsInProgress = 0;
m_cancelTokenSource = new CancellationTokenSource();
m_progressUpdater.Run(m_cancelTokenSource.Token);
try
{
while (!await self.Input.IsRetiredAsync && await m_taskReader.ProgressAsync)
{
var req = await self.Input.ReadAsync();
if (!await m_taskReader.ProgressAsync)
{
break;
}
var worker = workers.FirstOrDefault(w => w.Task.IsCompleted && !w.Task.IsFaulted);
if (worker == null)
{
worker = new Worker(m_backendFactory());
workers.Add(worker);
}
if (req is VolumeUploadRequest volumeUpload)
{
if (volumeUpload.IndexVolume == null)
{
worker.Task = Task.Run(() => UploadFileAsync(volumeUpload.BlockEntry, worker, m_cancelTokenSource.Token));
}
else
{
worker.Task = Task.Run(() => UploadBlockAndIndexAsync(volumeUpload, worker, m_cancelTokenSource.Token));
}
lastSize = volumeUpload.BlockVolume.SourceSize;
uploadsInProgress++;
}
else if (req is FilesetUploadRequest filesetUpload)
{
worker.Task = Task.Run(() => UploadVolumeWriter(filesetUpload.Fileset, worker, m_cancelTokenSource.Token));
uploadsInProgress++;
}
else if (req is IndexVolumeUploadRequest indexUpload)
{
worker.Task = Task.Run(() => UploadVolumeWriter(indexUpload.IndexVolume, worker, m_cancelTokenSource.Token));
uploadsInProgress++;
}
else if (req is FlushRequest flush)
{
try
{
while (workers.Any())
{
var finishedTask = await Task.WhenAny(workers.Select(w => w.Task)).ConfigureAwait(false);
if (finishedTask.IsFaulted)
{
ExceptionDispatchInfo.Capture(finishedTask.Exception).Throw();
}
workers.RemoveAll(w => w.Task == finishedTask);
}
uploadsInProgress = 0;
}
finally
{
flush.SetFlushed(lastSize);
}
break;
}
if (uploadsInProgress >= m_maxConcurrentUploads)
{
await Task.WhenAny(workers.Select(w => w.Task)).ConfigureAwait(false);
uploadsInProgress--;
var failedUploads = workers.Where(w => w.Task.IsFaulted).Select(w => GetInnerMostException(w.Task.Exception)).ToList();
if (failedUploads.Any())
{
if (failedUploads.Count == 1)
{
ExceptionDispatchInfo.Capture(failedUploads.First()).Throw();
}
else
{
throw new AggregateException(failedUploads);
}
}
}
}
}
catch (Exception ex) when(!ex.IsRetiredException())
{
m_cancelTokenSource.Cancel();
try
{
await Task.WhenAll(workers.Select(w => w.Task));
}
catch { /* As we are cancelling all threads we do not need to alert the user to any of these exceptions */ }
throw;
}
try
{
m_stats.SetBlocking(true);
await Task.WhenAll(workers.Select(w => w.Task));
}
finally
{
m_stats.SetBlocking(false);
}
}));
}
public static Task Run(IEnumerable <string> sources, Snapshots.ISnapshotService snapshot, UsnJournalService journalService, FileAttributes fileAttributes, Duplicati.Library.Utility.IFilter sourcefilter, Duplicati.Library.Utility.IFilter emitfilter, Options.SymlinkStrategy symlinkPolicy, Options.HardlinkStrategy hardlinkPolicy, bool excludeemptyfolders, string[] ignorenames, string[] changedfilelist, ITaskReader taskreader)
{
return(AutomationExtensions.RunTask(
new
{
Output = Backup.Channels.SourcePaths.ForWrite
},
async self =>
{
var hardlinkmap = new Dictionary <string, string>();
var mixinqueue = new Queue <string>();
Duplicati.Library.Utility.IFilter enumeratefilter = emitfilter;
bool includes;
bool excludes;
Library.Utility.FilterExpression.AnalyzeFilters(emitfilter, out includes, out excludes);
if (includes && !excludes)
{
enumeratefilter = Library.Utility.FilterExpression.Combine(emitfilter, new Duplicati.Library.Utility.FilterExpression("*" + System.IO.Path.DirectorySeparatorChar, true));
}
// Simplify checking for an empty list
if (ignorenames != null && ignorenames.Length == 0)
{
ignorenames = null;
}
// If we have a specific list, use that instead of enumerating the filesystem
IEnumerable <string> worklist;
if (changedfilelist != null && changedfilelist.Length > 0)
{
worklist = changedfilelist.Where(x =>
{
var fa = FileAttributes.Normal;
try
{
fa = snapshot.GetAttributes(x);
}
catch
{
}
return AttributeFilter(x, fa, snapshot, sourcefilter, hardlinkPolicy, symlinkPolicy, hardlinkmap, fileAttributes, enumeratefilter, ignorenames, mixinqueue);
});
}
else
{
Library.Utility.Utility.EnumerationFilterDelegate attributeFilter = (root, path, attr) =>
AttributeFilter(path, attr, snapshot, sourcefilter, hardlinkPolicy, symlinkPolicy, hardlinkmap, fileAttributes, enumeratefilter, ignorenames, mixinqueue);
if (journalService != null)
{
// filter sources using USN journal, to obtain a sub-set of files / folders that may have been modified
sources = journalService.GetModifiedSources(attributeFilter);
}
worklist = snapshot.EnumerateFilesAndFolders(sources, attributeFilter, (rootpath, errorpath, ex) =>
{
Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "FileAccessError", ex, "Error reported while accessing file: {0}", errorpath);
});
}
var source = ExpandWorkList(worklist, mixinqueue, emitfilter, enumeratefilter);
if (excludeemptyfolders)
{
source = ExcludeEmptyFolders(source);
}
// Process each path, and dequeue the mixins with symlinks as we go
foreach (var s in source)
{
if (!await taskreader.ProgressAsync)
{
return;
}
await self.Output.WriteAsync(s);
}
}));
}
public static Task Run(Options options, BackupDatabase database, ITaskReader taskreader)
{
return(AutomationExtensions.RunTask(
new
{
Input = Channels.SpillPickup.ForRead,
Output = Channels.BackendRequest.ForWrite,
},
async self =>
{
var lst = new List <VolumeUploadRequest>();
while (!await self.Input.IsRetiredAsync)
{
try
{
lst.Add((VolumeUploadRequest)await self.Input.ReadAsync());
}
catch (Exception ex)
{
if (ex.IsRetiredException())
{
break;
}
throw;
}
}
while (lst.Count > 1)
{
// We ignore the stop signal, but not the pause and terminate
await taskreader.ProgressAsync;
VolumeUploadRequest target = null;
var source = lst[0];
// Finalize the current work
source.BlockVolume.Close();
// Remove it from the list of active operations
lst.RemoveAt(0);
var buffer = new byte[options.Blocksize];
using (var rd = new BlockVolumeReader(options.CompressionModule, source.BlockVolume.LocalFilename, options))
{
foreach (var file in rd.Blocks)
{
// Grab a target
if (target == null)
{
if (lst.Count == 0)
{
// No more targets, make one
target = new VolumeUploadRequest(new BlockVolumeWriter(options), source.IndexVolume == null ? null : new TemporaryIndexVolume(options));
target.BlockVolume.VolumeID = await database.RegisterRemoteVolumeAsync(target.BlockVolume.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary);
}
else
{
// Grab the next target
target = lst[0];
lst.RemoveAt(0);
}
// We copy all the blocklisthashes, which may create duplicates
// but otherwise we need to query all hashes to see if they are blocklisthashes
if (source.IndexVolume != null)
{
source.IndexVolume.CopyTo(target.IndexVolume, true);
}
}
var len = rd.ReadBlock(file.Key, buffer);
target.BlockVolume.AddBlock(file.Key, buffer, 0, len, Duplicati.Library.Interface.CompressionHint.Default);
await database.MoveBlockToVolumeAsync(file.Key, len, source.BlockVolume.VolumeID, target.BlockVolume.VolumeID);
if (target.IndexVolume != null)
{
target.IndexVolume.AddBlock(file.Key, len);
}
if (target.BlockVolume.Filesize > options.VolumeSize - options.Blocksize)
{
target.BlockVolume.Close();
await self.Output.WriteAsync(target);
target = null;
}
}
}
// Make sure they are out of the database
System.IO.File.Delete(source.BlockVolume.LocalFilename);
await database.SafeDeleteRemoteVolumeAsync(source.BlockVolume.RemoteFilename);
// Re-inject the target if it has content
if (target != null)
{
lst.Insert(lst.Count == 0 ? 0 : 1, target);
}
}
foreach (var n in lst)
{
// We ignore the stop signal, but not the pause and terminate
await taskreader.ProgressAsync;
n.BlockVolume.Close();
await self.Output.WriteAsync(n);
}
}));
}
