public static Task RunGranterAsync(IWriteChannel <bool> channel, long count, CancellationToken token)
{
var canceltask = new TaskCompletionSource <bool>();
token.Register(() => canceltask.TrySetCanceled());
var total = count;
return(AutomationExtensions.RunTask(
new { channel },
async self => {
while (count > 0)
{
DebugWriteLine($"Emitting task {total - count} of {total}");
if (await Task.WhenAny(new [] { canceltask.Task, channel.WriteAsync(true) }) == canceltask.Task)
{
throw new TaskCanceledException();
}
count--;
DebugWriteLine($"Emitted task {total - count} of {total}");
}
DebugWriteLine("Stopping task granter");
}
));
}
/// <summary>
/// Initializes a new instance of the <see cref="T:CoCoL.Blocks.Prefix`1"/> class.
/// </summary>
/// <param name="input">The input channel.</param>
/// <param name="output">The output channel.</param>
/// <param name="value">The initial value to emit.</param>
/// <param name="repeat">The number of copies to initially emit.</param>
public Prefix(IReadChannel <T> input, IWriteChannel <T> output, T value, long repeat = 1)
{
m_input = input ?? throw new ArgumentNullException(nameof(input));
m_output = output ?? throw new ArgumentNullException(nameof(output));
m_value = value;
m_repeat = repeat;
}
/// <summary>
/// Stores the value in long-term storage if required, and broadcasts the value,
/// if we are the closest peer
/// </summary>
/// <returns>An awaitable task.</returns>
/// <param name="self">The node's own information</param>
/// <param name="router">The router request channel</param>
/// <param name="key">The key for the data.</param>
/// <param name="data">The data to store.</param>
/// <param name="storechan">The channel pointing back to the store process</param>
private static async Task StoreLongTermAsync(PeerInfo self, IWriteChannel <RoutingRequest> router, IWriteChannel <MRUInternalStore> storechan, Key key, byte[] data)
{
// We could use a small cache of peers here
// to avoid hammering the routing process
// for each insert
log.Debug($"Checking if {key} should be inserted into long-term storage for {self.Key}");
var list = await router.LookupPeerAsync(key, true);
// If we are in the k-bucket closest peers, store it
if (list.Any(x => self.Key.Equals(x.Key)))
{
// Check if we are the closest node
var closest = self.Key.Equals(list.OrderBy(x => new KeyDistance(key, x.Key)).First().Key);
log.Debug($"We are {(closest ? "*the* peer": "one of the peers")} for {key}");
await storechan.WriteAsync(new MRUInternalStore()
{
Key = key,
Data = data,
Peers = closest ? list : null
});
}
else
{
log.Debug($"We are not one of the closest peers for {key}");
}
}
/// <summary>
/// Sends an expire request
/// </summary>
/// <returns>The result of the expire operation.</returns>
/// <param name="channel">The request channel.</param>
public static async Task <bool> SendExpireAsync(this IWriteChannel <MRURequest> channel)
{
return((await SendMessageAsync <MRURequest, MRUResponse>(channel, new MRURequest()
{
Operation = MRUOperation.Expire
})).Success);
}
/// <summary>
/// Wraps the channel with a latency hiding instance, if required by config
/// </summary>
/// <returns>The buffered write channel.</returns>
/// <param name="input">The input channel.</param>
private static IWriteChannel <T> AsBufferedWrite(IWriteChannel <T> input)
{
if (Config.NetworkChannelLatencyBufferSize != 0 && input is NetworkChannel <T> && (Config.AllChannelsNetworked || ((INamedItem)input).Name == "tick"))
{
return(new LatencyHidingWriter <T>(input, Config.NetworkChannelLatencyBufferSize));
}
return(input);
}
/// <summary>
/// Initializes a new instance of the <see cref="Duplicati.Library.UsageReporter.EventProcessor"/> class.
/// </summary>
/// <param name="forwardChannel">The forwarding destination.</param>
public EventProcessor(IWriteChannel <string> forwardChannel)
{
INSTANCE_ID = System.Diagnostics.Process.GetCurrentProcess().Id.ToString();
Channel = m_channel = ChannelManager.CreateChannel <ReportItem>(null, MAX_QUEUE_SIZE);
m_forward = forwardChannel;
Terminated = RunProtected(Run);
}
/// <summary>
/// Wraps the channel with a latency hiding instance, if required by config
/// </summary>
/// <returns>The buffered write channel.</returns>
/// <param name="input">The input channel.</param>
private static IWriteChannel <T> AsBufferedWrite <T>(IWriteChannel <T> input)
{
if (Config.NetworkChannelLatencyBufferSize != 0 && input is NetworkChannel <T> && Config.NetworkedChannels)
{
return(new LatencyHidingWriter <T>(input, Config.NetworkChannelLatencyBufferSize));
}
return(input);
}
/// <summary>
/// Removes the peer with the given key from the routing table.
/// </summary>
/// <returns>The peer to remove.</returns>
/// <param name="channel">The channel to send the request on.</param>
/// <param name="key">The key for the peer to remove.</param>
public static Task RemovePeerAsync(this IWriteChannel <RoutingRequest> channel, Key key)
{
return(SendMessageAsync <RoutingRequest, RoutingResponse>(channel, new RoutingRequest()
{
Operation = RoutingOperation.Remove,
Key = key
}));
}
/// <summary>
/// Adds a peer to the routing table
/// </summary>
/// <returns><c>true</c> if the peer is new, <c>false</c> otherwise.</returns>
/// <param name="channel">The channel to send the request on.</param>
/// <param name="key">The key for the peer.</param>
/// <param name="peer">The peer.</param>
public static async Task <bool> AddPeerAsync(this IWriteChannel <RoutingRequest> channel, Key key, PeerInfo peer)
{
return((await SendMessageAsync <RoutingRequest, RoutingResponse>(channel, new RoutingRequest()
{
Operation = RoutingOperation.Add,
Key = key,
Data = peer
})).IsNew);
}
/// <summary>
/// Looks up peers in the routing table
/// </summary>
/// <returns>The peers.</returns>
/// <param name="channel">The channel to send the request on</param>
/// <param name="onlyKbucket"><c>true</c> if the response must originate from the closest bucket, <c>false</c> otherwise.</param>
/// <param name="key">The key to return the results for.</param>
public static async Task <List <PeerInfo> > LookupPeerAsync(this IWriteChannel <RoutingRequest> channel, Key key, bool onlyKbucket = false)
{
return((await SendMessageAsync <RoutingRequest, RoutingResponse>(channel, new RoutingRequest()
{
Operation = RoutingOperation.Lookup,
Key = key,
OnlyKBucket = onlyKbucket
})).Peers);
}
/// <summary>
/// Adds a value to the MRU cache and long-term storage
/// </summary>
/// <returns>The result of the add operation.</returns>
/// <param name="channel">The channel to send the request over.</param>
/// <param name="key">The key of the data to add.</param>
/// <param name="data">The data to add.</param>
public static async Task <bool> SendAddAsync(this IWriteChannel <MRURequest> channel, Key key, byte[] data)
{
return((await SendMessageAsync <MRURequest, MRUResponse>(channel, new MRURequest()
{
Operation = MRUOperation.Add,
Key = key,
Data = data
})).Success);
}
/// <summary>
/// Requests a value from the MRU cache or value store
/// </summary>
/// <returns>The result or null.</returns>
/// <param name="channel">The channel to place the request on.</param>
/// <param name="key">The key to get the data for.</param>
public static async Task <byte[]> SendGetAsync(this IWriteChannel <MRURequest> channel, Key key)
{
var res = await SendMessageAsync <MRURequest, MRUResponse>(channel, new MRURequest()
{
Operation = MRUOperation.Get,
Key = key
});
return(res.Success ? res.Data : null);
}
/// <summary>
/// Sends a request and waits for the response
/// </summary>
/// <returns>The connection request async.</returns>
/// <param name="channel">The channel to send the request over</param>
/// <param name="endPoint">End point.</param>
/// <param name="key">Key.</param>
/// <param name="requestId">The request ID</param>
/// <param name="request">Request.</param>
public static Task <ConnectionResponse> SendConnectionRequestAsync(this IWriteChannel <ConnectionRequest> channel, EndPoint endPoint, Key key, long requestId, Protocol.Request request)
{
return(SendMessageAsync <ConnectionRequest, ConnectionResponse>(channel, new ConnectionRequest()
{
EndPoint = endPoint,
Key = key,
RequestID = requestId,
Request = request
}));
}
/// <summary>
/// Initializes a new instance of the <see cref="CoCoL.ChannelMarkerWrapper<T>"/> class.
/// </summary>
/// <param name="attribute">The attribute describing the channel.</param>
public ChannelMarkerWrapper(ChannelNameAttribute attribute)
{
if (attribute == null)
{
throw new ArgumentNullException("attribute");
}
Attribute = attribute;
ForWrite = ChannelMarker.ForWrite <T>(attribute);
ForRead = ChannelMarker.ForRead <T>(attribute);
}
public Prefix(IReadChannel<T> input, IWriteChannel<T> output, T value, long repeat = 1)
{
if (input == null)
throw new ArgumentNullException("input");
if (output == null)
throw new ArgumentNullException("output");
m_input = input;
m_output = output;
m_value = value;
m_repeat = repeat;
}
private static async Task <long> FlushBackend(BackupResults result, IWriteChannel <Backup.IUploadRequest> uploadtarget, Task uploader)
{
var flushReq = new Backup.FlushRequest();
// Wait for upload completion
result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_WaitForUpload);
await uploadtarget.WriteAsync(flushReq).ConfigureAwait(false);
await uploader.ConfigureAwait(false);
// Grab the size of the last uploaded volume
return(await flushReq.LastWriteSizeAsync);
}
/// <summary>
/// Sets up a simple channel transformation process
/// </summary>
/// <returns>An awaitable task.</returns>
/// <param name="input">The input channel.</param>
/// <param name="output">The output channel.</param>
/// <param name="transform">The method transforming the input</param>
/// <param name="maxparallel">The maximum number of parallel handlers</param>
/// <typeparam name="TIn">The input type parameter.</typeparam>
/// <typeparam name="TOut">The output type parameter.</typeparam>
public static Task TransformAsync <TIn, TOut>(IReadChannel <TIn> input, IWriteChannel <TOut> output, Func <TIn, Task <TOut> > transform, int maxparallel = 1)
{
if (output == null)
{
throw new ArgumentNullException(nameof(output));
}
if (transform == null)
{
throw new ArgumentNullException(nameof(transform));
}
return(CollectAsync(input, async data => await output.WriteAsync(await transform(data)), maxparallel));
}
public Identity(IReadChannel <T> input, IWriteChannel <T> output)
{
if (input == null)
{
throw new ArgumentNullException("input");
}
if (output == null)
{
throw new ArgumentNullException("output");
}
m_input = input;
m_output = output;
}
public static async Task <StreamProcessResult> ProcessStream(IWriteChannel <StreamBlock> channel, string path, Stream stream, bool isMetadata, CompressionHint hint)
{
var tcs = new TaskCompletionSource <StreamProcessResult>();
await channel.WriteAsync(new StreamBlock()
{
Path = path,
Stream = stream,
IsMetadata = isMetadata,
Hint = hint,
Result = tcs
});
return(await tcs.Task);
}
public Successor(IReadChannel <long> input, IWriteChannel <long> output)
{
if (input == null)
{
throw new ArgumentNullException("input");
}
if (output == null)
{
throw new ArgumentNullException("output");
}
m_input = input;
m_output = output;
}
private static async Task UploadVolumeAndIndex(SpillVolumeRequest target, IWriteChannel <IUploadRequest> outputChannel, Options options, BackupDatabase database)
{
var blockEntry = target.BlockVolume.CreateFileEntryForUpload(options);
TemporaryIndexVolume indexVolumeCopy = null;
if (target.IndexVolume != null)
{
indexVolumeCopy = new TemporaryIndexVolume(options);
target.IndexVolume.CopyTo(indexVolumeCopy, false);
}
var uploadRequest = new VolumeUploadRequest(target.BlockVolume, blockEntry, indexVolumeCopy, options, database);
await outputChannel.WriteAsync(uploadRequest).ConfigureAwait(false);
}
/// <summary>
/// Runs the identity process, which simply forwards a value.
/// </summary>
/// <param name="chan_read">The channel to read from</param>
/// <param name="chan_write">The channel to write to</param>
private static async void RunIdentity(IReadChannel <T> chan_read, IWriteChannel <T> chan_write)
{
try
{
while (true)
{
await chan_write.WriteAsync(await chan_read.ReadAsync());
}
}
catch (RetiredException)
{
chan_read.Retire();
chan_write.Retire();
}
}
public ReportSetUploader()
{
Channel = m_channel = ChannelManager.CreateChannel<string>(buffersize: MAX_PENDING_UPLOADS);
FilterChannel = ChannelManager.CreateChannel<string>();
Terminated = Task.WhenAll(
RunProtected(async () => {
// If we have more than MAX_PENDING_UPLOADS
// the newest ones are just discarded
// they will be picked up on the later runs
while (true)
FilterChannel.TryWrite(await m_channel.ReadAsync());
}),
RunProtected(Run)
);
}
private static async Task UploadVolumeAndIndex(SpillVolumeRequest target, IWriteChannel <IUploadRequest> outputChannel, Options options, BackupDatabase database)
{
var blockEntry = CreateFileEntryForUpload(target.BlockVolume, options);
IndexVolumeWriter indexVolume = null;
FileEntryItem indexEntry = null;
if (target.IndexVolume != null)
{
indexVolume = await target.IndexVolume.CreateVolume(target.BlockVolume.RemoteFilename, blockEntry.Hash, blockEntry.Size, options, database).ConfigureAwait(false);
indexEntry = CreateFileEntryForUpload(indexVolume, options);
}
var uploadRequest = new VolumeUploadRequest(target.BlockVolume, blockEntry, indexVolume, indexEntry);
await outputChannel.WriteAsync(uploadRequest).ConfigureAwait(false);
}
/// <summary>
/// A process that produces an infinite amount of numbers,
/// given an offset and an increment
/// </summary>
/// <returns>The awaitable task that represents the process</returns>
/// <param name="from">The number offset.</param>
/// <param name="increment">The increment.</param>
/// <param name="target">The channel to which the numbers are written.</param>
private static async Task RunNumbersFromAsync(long from, long increment, IWriteChannel <long> target)
{
var n = from;
try
{
while (true)
{
await target.WriteAsync(n);
n += increment;
}
}
catch (RetiredException)
{
}
}
/// <summary>
/// Write to the channel in a blocking manner
/// </summary>
/// <param name="value">The value to write into the channel</param>
/// <param name="self">The channel to read from</param>
/// <typeparam name="T">The channel data type parameter.</typeparam>
public static void Write <T>(this IWriteChannel <T> self, T value)
{
var res = self.WriteAsync(value, Timeout.Infinite).WaitForTask();
if (res.Exception != null)
{
if (res.Exception is AggregateException && ((AggregateException)res.Exception).Flatten().InnerExceptions.Count == 1)
{
throw ((AggregateException)res.Exception).InnerException;
}
throw res.Exception;
}
else if (res.IsCanceled)
{
throw new OperationCanceledException();
}
}
/// <summary>
/// Constructs a new runner
/// </summary>
/// <param name="options">The options to use</param>
/// <param name="reqchan">The request channel</param>
/// <param name="respchan">The response channel</param>
/// <param name="response">The expected response string</param>
public WebRequestWorker(IWorkerOptions options, IReadChannel <bool> reqchan, IWriteChannel <RequestResult> respchan, string response)
//: base(options, maximumrequests, response)
: base(options, reqchan, respchan, response)
{
foreach (var h in options.Headers)
{
var parts = h?.Split(':', 2);
if (parts == null || parts.Length < 2)
{
throw new ArgumentException($"Bad header format: {h}");
}
m_headers.Add(parts[0], parts[1]);
}
m_data = string.IsNullOrEmpty(options.Body) ? null : System.Text.Encoding.UTF8.GetBytes(options.Body);
m_initialized.TrySetResult(true);
}
/// <summary>
/// Write to the channel in a blocking manner
/// </summary>
/// <param name="value">The value to write into the channel</param>
/// <param name="self">The channel to read from</param>
/// <param name="timeout">The maximum time to wait for an available slot</param>
/// <typeparam name="T">The channel data type parameter.</typeparam>
public static void Write <T>(this IWriteChannel <T> self, T value, TimeSpan timeout)
{
var res = self.WriteAsync(value, new TimeoutOffer(timeout)).WaitForTask();
if (res.Exception != null)
{
if (res.Exception.Flatten().InnerExceptions.Count == 1)
{
throw res.Exception.InnerException;
}
throw res.Exception;
}
else if (res.IsCanceled)
{
throw new OperationCanceledException();
}
}
public Delta(IReadChannel <T> input, IWriteChannel <T> outputA, IWriteChannel <T> outputB)
{
if (input == null)
{
throw new ArgumentNullException("input");
}
if (outputA == null)
{
throw new ArgumentNullException("outputA");
}
if (outputB == null)
{
throw new ArgumentNullException("outputB");
}
m_input = input;
m_outputA = outputA;
m_outputB = outputB;
}
// The Primes process
/// <summary>
/// A process that outputs the number 2, then
/// spawns the sieve and a number generating process
/// that produces numbers from 3 and forward
/// </summary>
/// <returns>The awaitable task that represents the process</returns>
/// <param name="target">The channel the numbers are written into.</param>
private static async Task RunPrimesAsync(IWriteChannel <long> target)
{
var chan = ChannelManager.CreateChannel <long>();
try
{
await target.WriteAsync(2);
await Task.WhenAll(
RunNumbersFromAsync(3, 2, chan),
RunSieveAsync(chan, target)
);
}
catch (RetiredException)
{
chan.Retire();
target.Retire();
}
}
public ReportSetUploader()
{
Channel = m_channel = ChannelManager.CreateChannel <string>(buffersize: MAX_PENDING_UPLOADS);
FilterChannel = ChannelManager.CreateChannel <string>();
Terminated = Task.WhenAll(
RunProtected(async() => {
// If we have more than MAX_PENDING_UPLOADS
// the newest ones are just discarded
// they will be picked up on the later runs
while (true)
{
FilterChannel.TryWrite(await m_channel.ReadAsync());
}
}),
RunProtected(Run)
);
}
/// <summary>
/// Initializes a new instance of the <see cref="Duplicati.Library.UsageReporter.EventProcessor"/> class.
/// </summary>
/// <param name="forwardChannel">The forwarding destination.</param>
public EventProcessor(IWriteChannel<string> forwardChannel)
{
INSTANCE_ID = System.Diagnostics.Process.GetCurrentProcess().Id.ToString();
Channel = m_channel = ChannelManager.CreateChannel<ReportItem>(null, MAX_QUEUE_SIZE);
m_forward = forwardChannel;
Terminated = RunProtected(Run);
}
/// <summary>
/// Initializes the usage reporter library
/// </summary>
public static void Initialize()
{
if (_eventChannel == null || _eventChannel.IsRetired)
{
if (IsDisabled)
return;
var rsu = new ReportSetUploader();
var ep = new EventProcessor(rsu.Channel);
_eventChannel = ep.Channel;
ShutdownTask = Task.WhenAll(ep.Terminated, rsu.Terminated);
// TODO: Disable on debug builds
AppDomain.CurrentDomain.UnhandledException += HandleUncaughtException;
//AppDomain.CurrentDomain.UnhandledException += HandleUncaughtException;
//AppDomain.CurrentDomain.ProcessExit
Report("Started");
}
}
