private async Task CompleteTaskAsync(CollectionTask task)
{
if (AppSettings.Default.UseProxy && AppSettings.Default.Proxies.Count > 0)
{
var newTask = Task.Run
(() =>
{
_proxiedClients.AsParallel().WithExecutionMode(ParallelExecutionMode.ForceParallelism).ForAll(
client =>
{
for (var i = 0; i < 5; i++)
{
try
{
task.Func.Invoke(client, _cancellationTokenSource.Token);
break;
}
catch
{
// ignored
}
}
});
}
);
await newTask;
}
else
{
await Task.Run(() => task.Func.Invoke(_proxylessClient, CancellationToken.None));
}
}
public IContext StartBaseCollectionTask(IContext context)
{
context.Logger.LogTrace("Entering StartBaseCollectionTask");
context.Logger.LogInformation("Flags: {flags}", context.ResolvedCollectionMethods.GetIndividualFlags());
//5. Start the collection
var task = new CollectionTask(context);
context.CollectionTask = task.StartCollection();
context.Logger.LogTrace("Exiting StartBaseCollectionTask");
return(context);
}
protected TaskCollection(int initialSize)
{
_currentTask = new CollectionTask(this);
_listOfStacks = new FasterList <StructFriendlyStack>(initialSize);
var buffer = _listOfStacks.ToArrayFast();
for (int i = 0; i < initialSize; i++)
{
buffer[i] = new StructFriendlyStack(1);
}
}
public async Task RegisterActionAsync(string uniqueName, Action <HttpClient, CancellationToken> collectFunc, TimeSpan repeatInterval)
{
var task = new CollectionTask()
{
Name = uniqueName,
Func = collectFunc,
Interval = repeatInterval,
NextRun = DateTime.Now + repeatInterval
};
_registeredTasks.Add(task);
await CompleteTaskAsync(task);
}
public void ProccesCollectionTasks()
{
bool loop = true;
while (loop)
{
IEnumerable <CloudQueueMessage> queueMessages = StorageClientHelper.GetQueueMessages(QueueType.Processing, SourceType.Twitter, 32);
int messageCount = queueMessages.Count <CloudQueueMessage>();
if ((messageCount < 1) || (messageCount == null))
{
Console.WriteLine("No Messages");
loop = false;
}
else
{
Console.WriteLine("GetQueueMessage = " + queueMessages.Count <CloudQueueMessage>());
foreach (CloudQueueMessage queueMessage in queueMessages)
{
//1. Get Queue
HaystackLibrary.CollectionTask newTask = CollectionsTaskHelper.MessageToDatabaseTask(queueMessage.AsString);
string taskString = CollectionsTaskHelper.DatabaseTaskString(newTask);
Console.WriteLine(taskString);
//2. Get Blob
CloudBlob blob = StorageClientHelper.GetBlobReference(newTask.Project, newTask.BlobName);
string blobText = blob.DownloadText();
//TODO: Blob Validate Hash
Console.WriteLine(blobText);
//3. Update Database: Results
//TODO: Update Database: Results
//4. Update Database: Tasks
LinqToSqlDataContext context = new LinqToSqlDataContext();
CollectionTask oldTask = (from t in context.CollectionTasks
where t.Id == newTask.Id
orderby t.Id descending
select t).First <CollectionTask>();
oldTask.State = CollectionsTaskState.Processed.ToString();
oldTask.Collected = newTask.Collected;
oldTask.Station = newTask.Station;
oldTask.Collector = newTask.Collector;
oldTask.BlobName = newTask.BlobName;
oldTask.BlobHash = blob.Attributes.Properties.ContentMD5;
oldTask.Processed = DateTime.UtcNow;
context.SubmitChanges();
//5. Update Blob Metadata
blob.Attributes.Metadata["State"] = CollectionsTaskState.Processed.ToString();
//6. Delete Queue
StorageClientHelper.DeleteQueueMessage(QueueType.Processing, SourceType.Twitter, queueMessage);
//7. Print Success
string output = CollectionsTaskHelper.DatabaseTaskString(newTask);
Console.WriteLine(output);
}
}
}
}
