public async Task <List <IDownload> > Process(CancellationToken token, ILog log)
{
log.Debug($"{nameof(BackgroundDownloader)} started");
var errors = new AsyncProducerConsumerQueue <IDownload>();
IDownload item;
int success;
for (success = 0; (item = await GetItem(token)) != null;)
{
// TODO: add second queue, semaphore and support for retries
try
{
await filesystemTools.WriteFileWithCompletionMark(item.Location, item.DesiredName, async() => await item.GetContent(token), token, log);
success++;
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception e)
{
item.AddError(e);
await errors.EnqueueAsync(item, token);
log.Warn($"Error downloading {item}", e);
}
}
errors.CompleteAdding();
var errorList = errors.GetConsumingEnumerable(token).ToList();
log.Info($"All downloads completed. {success} successful, {errorList.Count} failed");
return(errorList);
}
//如果平台不支持TPL数据流库，可以使用Nito.AsyncEx中的AsyncProducerConsumerQueue<T>类，它同时也支持同步和异步方法。
async void Example17()
{
AsyncProducerConsumerQueue <int> queue = new AsyncProducerConsumerQueue <int>();
//异步的生产者代码
await queue.EnqueueAsync(7);
await queue.EnqueueAsync(13);
//同步的生产者代码
queue.Enqueue(7);
queue.Enqueue(13);
queue.CompleteAdding();
//单个消费者时的异步代码
while (await queue.OutputAvailableAsync())
{
Trace.WriteLine(await queue.DequeueAsync());
}
//多个消费者时的异步代码
while (true)
{
//var result = await queue.TryDequeueAsync();
//if (result.Success) break;
//Trace.WriteLine(result.Item);
}
//同步的消费者代码
foreach (var item in queue.GetConsumingEnumerable())
{
Trace.WriteLine(item);
}
}
async Task Test()
{
var queue = new AsyncProducerConsumerQueue <int>();
// Asynchronous producer code
await queue.EnqueueAsync(7);
await queue.EnqueueAsync(13);
// Synchronous producer code
queue.Enqueue(7);
queue.Enqueue(13);
queue.CompleteAdding();
// Asynchronous single consumer code
while (await queue.OutputAvailableAsync())
{
Trace.WriteLine(await queue.DequeueAsync());
}
// Asynchronous multi-consumer code
while (true)
{
int item;
try
{
item = await queue.DequeueAsync();
}
catch (InvalidOperationException)
{
break;
}
Trace.WriteLine(item);
}
// Synchronous consumer code
foreach (int item in queue.GetConsumingEnumerable())
{
Trace.WriteLine(item);
}
}
public async Task <IEnumerable <Content> > GetContentChanges(IEnumerable <Content> existingContent, bool fastMode)
{
var messages = DiscordChannel.GetMessagesAsync(int.MaxValue);
ConcurrentStack <Content> messagesToGet = new ConcurrentStack <Content>();
int totalEnumerated = 0;
AsyncProducerConsumerQueue <IMessage> messageCollection = new AsyncProducerConsumerQueue <IMessage>(8000);
bool readingComplete = false;
var enqueueTask = Task.Run(async() =>
{
try
{
using (var enumerator = messages.GetEnumerator())
{
while (!readingComplete && await enumerator.MoveNext())
{
Console.WriteLine($"Enumerated {totalEnumerated} / 0");
foreach (var message in enumerator.Current)
{
totalEnumerated++;
messageCollection.Enqueue(message);
}
}
}
Console.WriteLine($"Enumerated {totalEnumerated} / {totalEnumerated}");
}
finally
{
messageCollection.CompleteAdding();
}
});
List <Task> embedTasks = new List <Task>();
foreach (var message in messageCollection.GetConsumingEnumerable())
{
if (existingContent.Any(y => (ulong)y.UserHash == message.Id))
{
if (fastMode)
{
break;
}
continue;
}
foreach (var attachment in message.Attachments)
{
var content = new Content
{
LastModifiedTime = message.Timestamp.DateTime,
Text = attachment.Filename,
UserHash = (long)message.Id,
Metadata = new Dictionary <string, string>(5)
{
["filename"] = attachment.Filename,
["ID"] = attachment.Id.ToString(),
["messageText"] = message.Content,
["url"] = attachment.Url,
["proxyUrl"] = attachment.ProxyUrl
}
};
if (message.Channel.Id == 464602747786887178)
{
MessagesToTick[message.Id] = (RestUserMessage)message;
}
messagesToGet.Push(content);
}
foreach (var embed in message.Embeds)
{
embedTasks.Add(ProcessEmbed(messagesToGet, message, embed));
}
}
readingComplete = true;
await Task.WhenAll(embedTasks.ToArray());
GCSettings.LargeObjectHeapCompactionMode = GCLargeObjectHeapCompactionMode.CompactOnce;
GC.Collect();
return(messagesToGet);
}