private async Task CompleteTaskAsync(CollectionTask task) { if (AppSettings.Default.UseProxy && AppSettings.Default.Proxies.Count > 0) { var newTask = Task.Run (() => { _proxiedClients.AsParallel().WithExecutionMode(ParallelExecutionMode.ForceParallelism).ForAll( client => { for (var i = 0; i < 5; i++) { try { task.Func.Invoke(client, _cancellationTokenSource.Token); break; } catch { // ignored } } }); } ); await newTask; } else { await Task.Run(() => task.Func.Invoke(_proxylessClient, CancellationToken.None)); } }
public IContext StartBaseCollectionTask(IContext context) { context.Logger.LogTrace("Entering StartBaseCollectionTask"); context.Logger.LogInformation("Flags: {flags}", context.ResolvedCollectionMethods.GetIndividualFlags()); //5. Start the collection var task = new CollectionTask(context); context.CollectionTask = task.StartCollection(); context.Logger.LogTrace("Exiting StartBaseCollectionTask"); return(context); }
protected TaskCollection(int initialSize) { _currentTask = new CollectionTask(this); _listOfStacks = new FasterList <StructFriendlyStack>(initialSize); var buffer = _listOfStacks.ToArrayFast(); for (int i = 0; i < initialSize; i++) { buffer[i] = new StructFriendlyStack(1); } }
public async Task RegisterActionAsync(string uniqueName, Action <HttpClient, CancellationToken> collectFunc, TimeSpan repeatInterval) { var task = new CollectionTask() { Name = uniqueName, Func = collectFunc, Interval = repeatInterval, NextRun = DateTime.Now + repeatInterval }; _registeredTasks.Add(task); await CompleteTaskAsync(task); }
public void ProccesCollectionTasks() { bool loop = true; while (loop) { IEnumerable <CloudQueueMessage> queueMessages = StorageClientHelper.GetQueueMessages(QueueType.Processing, SourceType.Twitter, 32); int messageCount = queueMessages.Count <CloudQueueMessage>(); if ((messageCount < 1) || (messageCount == null)) { Console.WriteLine("No Messages"); loop = false; } else { Console.WriteLine("GetQueueMessage = " + queueMessages.Count <CloudQueueMessage>()); foreach (CloudQueueMessage queueMessage in queueMessages) { //1. Get Queue HaystackLibrary.CollectionTask newTask = CollectionsTaskHelper.MessageToDatabaseTask(queueMessage.AsString); string taskString = CollectionsTaskHelper.DatabaseTaskString(newTask); Console.WriteLine(taskString); //2. Get Blob CloudBlob blob = StorageClientHelper.GetBlobReference(newTask.Project, newTask.BlobName); string blobText = blob.DownloadText(); //TODO: Blob Validate Hash Console.WriteLine(blobText); //3. Update Database: Results //TODO: Update Database: Results //4. Update Database: Tasks LinqToSqlDataContext context = new LinqToSqlDataContext(); CollectionTask oldTask = (from t in context.CollectionTasks where t.Id == newTask.Id orderby t.Id descending select t).First <CollectionTask>(); oldTask.State = CollectionsTaskState.Processed.ToString(); oldTask.Collected = newTask.Collected; oldTask.Station = newTask.Station; oldTask.Collector = newTask.Collector; oldTask.BlobName = newTask.BlobName; oldTask.BlobHash = blob.Attributes.Properties.ContentMD5; oldTask.Processed = DateTime.UtcNow; context.SubmitChanges(); //5. Update Blob Metadata blob.Attributes.Metadata["State"] = CollectionsTaskState.Processed.ToString(); //6. Delete Queue StorageClientHelper.DeleteQueueMessage(QueueType.Processing, SourceType.Twitter, queueMessage); //7. Print Success string output = CollectionsTaskHelper.DatabaseTaskString(newTask); Console.WriteLine(output); } } } }