public int IssueCollectionTasks()
{
int tasksIssued = 0;
//Query Collection Tasks
LinqToSqlAzureHaystackDataContext context = new LinqToSqlAzureHaystackDataContext();
IQueryable <T_CollectionTask> results =
from r in context.T_CollectionTasks
where r.State == StateTypeEnum.Approved.ToString()
select r;
//Write Queue Message, Update State to "Issued"
foreach (T_CollectionTask r in results)
{
r.State = StateTypeEnum.Issued.ToString();
r.Issued = DateTime.UtcNow;
string message = CollectionsTaskHelper.T_CollectionsTaskToMessage(r);
QueueTypeEnum queueName;
if (r.Command == TwitterCommandEnum.Followers.ToString())
{
queueName = QueueTypeEnum.CollectionsTwitterFollowers;
}
else if (r.Command == TwitterCommandEnum.Friends.ToString())
{
queueName = QueueTypeEnum.CollectionsTwitterFriends;
}
else if (r.Command == TwitterCommandEnum.Profile.ToString())
{
queueName = QueueTypeEnum.CollectionsTwitterProfile;
}
else if (r.Command == TwitterCommandEnum.Search.ToString())
{
queueName = QueueTypeEnum.CollectionsTwitterSearch;
}
else if (r.Command == TwitterCommandEnum.Tweets.ToString())
{
queueName = QueueTypeEnum.CollectionsTwitterTweets;
}
else
{
throw new Exception("Command Not Found");
}
AzureClientService.AddQueueMessage(queueName, message);
tasksIssued++;
string task = string.Format("{0} {1} {2} {3} {4} {5} {6} ",
r.Id.ToString(), r.State, r.Created.ToString(), r.Project, r.Source, r.Command, r.Target);
Console.WriteLine(task);
}
//Submit Changes
context.SubmitChanges();
return(tasksIssued);
}
public static void Display()
{
CloudQueue queue = AzureClientService.GetCloudQueue(QueueTypeEnum.ProcessingTwitter);
int messageCount = queue.RetrieveApproximateMessageCount();
for (int i = 0; i < messageCount; i++)
{
CloudQueueMessage message = queue.GetMessage();
CollectionsTask task = CollectionsTaskHelper.MessageToCollectionsTask(message.AsString);
Console.WriteLine(string.Format("ID={0}, Command{1}, Target={2}",
task.Id, task.Command, task.Target));
//GetBlob
CloudBlob blob = AzureClientService.GetBlobReference(task.Project, task.BlobName);
string resultsXml = blob.DownloadText();
Console.WriteLine(string.Format("BLOB: {0}, {1}, {2}, {3}, {4},{5}",
blob.Attributes.Metadata["Id"].ToString(),
blob.Attributes.Metadata["Project"].ToString(),
blob.Attributes.Metadata["State"].ToString(),
blob.Attributes.Metadata["Collected"].ToString(),
blob.Name,
blob.Attributes.Properties.ContentMD5.ToString()));
Console.WriteLine(string.Format("BLOB XML: {0}", resultsXml));
}
}
//5-ReportCollectionTask
protected void ReportCollectionTask()
{
Logger.Log(GetContext() + "ReportCollectionTask(): #" + _currentTask.Id);
//MetaData
_currentTask.Station = _station;
_currentTask.Collector = _collector;
_currentTask.Collected = CollectionsTaskHelper.Now();
//SaveBlob
CloudBlob blob = AzureClientService.GetBlobReference(_currentTask.Project, _currentTask.BlobName);
blob.Attributes.Properties.ContentMD5 = _currentTask.ResultHash;
blob.Attributes.Metadata["Id"] = _currentTask.Id.ToString();
blob.Attributes.Metadata["State"] = StateTypeEnum.Collected.ToString();
blob.Attributes.Metadata["Project"] = _currentTask.Project;
blob.Attributes.Metadata["Collected"] = _currentTask.Collected;
blob.Attributes.Metadata["Command"] = _currentTask.Command;
//blob.Attributes.Metadata["TargetId"] = _currentTask.
blob.UploadText(_currentTask.ResultJson);
//AddQueue
string message = CollectionsTaskHelper.CollectionsTaskToMessage(_currentTask);
AzureClientService.AddQueueMessage(QueueTypeEnum.ProcessingTwitter, message);
AzureClientService.DeleteQueueMessage(_collectionsQueue, _currentQueueMessage);
}
//Step 5, Inner Loop
private void ReportCollectionTask()
{
Logger.Log("ReportCollectionTask: #" + _currentTask.Command);
Logger.Log("");
//MetaData
_currentTask.CollectionsStation = _collectionsStation;
_currentTask.CollectorInstance = _collectorInstance;
_currentTask.Collected = CollectionsTaskHelper.Now();
//SaveBlob
CloudBlob blob = StorageClientHelper.GetBlobReference(_currentTask.Project, _currentTask.BlobName);
blob.Attributes.Properties.ContentMD5 = _currentTask.ResultsHash;
blob.Attributes.Metadata["Id"] = _currentTask.Id.ToString();
blob.Attributes.Metadata["Project"] = _currentTask.Project;
blob.Attributes.Metadata["State"] = CollectionsTaskState.Collected.ToString();
blob.Attributes.Metadata["Collected"] = _currentTask.Collected;
blob.UploadText(_currentTask.ResultsXml);
//AddQueue
string message = CollectionsTaskHelper.LocalTaskToMessage(_currentTask);
StorageClientHelper.AddQueueMessage(QueueType.Processing, SourceType.Twitter, message);
}
//Step 3, Outer Loop
private List <CollectionsTaskLocal> RequestCollectionTasks(int collectionCapacity)
{
Logger.Log("RequestCollectionTasks");
//Hard Limit by Azure
int messageCount = Math.Min(collectionCapacity, 32);
//Get Q Messages
IEnumerable <CloudQueueMessage> queueMessages = StorageClientHelper.GetQueueMessages(QueueType.Collections, _sourceType, messageCount);
//Convert to Tasks
List <CollectionsTaskLocal> collectionsTaskList = new List <CollectionsTaskLocal>();
if (queueMessages == null)
{
return(collectionsTaskList);
}
else
{
Logger.Log("Received #" + queueMessages.Count <CloudQueueMessage>().ToString() + " messages");
}
foreach (CloudQueueMessage qm in queueMessages)
{
string messag = qm.AsString;
CollectionsTaskLocal task = CollectionsTaskHelper.MessageToLocalTask(messag);
task.QueueMessage = qm;
collectionsTaskList.Add(task);
}
return(collectionsTaskList);
}
public static void Delete()
{
CloudQueue queue = AzureClientService.GetCloudQueue(QueueTypeEnum.ProcessingTwitter);
int messageCount = queue.RetrieveApproximateMessageCount();
for (int i = 0; i < messageCount; i++)
{
CloudQueueMessage message = queue.GetMessage();
CollectionsTask task = CollectionsTaskHelper.MessageToCollectionsTask(message.AsString);
//GetBlob
CloudBlob blob = AzureClientService.GetBlobReference(task.Project, task.BlobName);
#if DEBUG
blob.DeleteIfExists();
#endif
queue.DeleteMessage(message);
}
}
//2-GetCollectionTask
protected bool GetCollectionTask()
{
Logger.Log(GetContext() + "GetCollectionTask()");
CloudQueueMessage queueMessage = AzureClientService.GetQueueMessage(_collectionsQueue);
if (queueMessage != null)
{
_currentQueueMessage = queueMessage;
_currentTask = CollectionsTaskHelper.MessageToCollectionsTask(_currentQueueMessage.AsString);
_currentTask.QueueMessage = _currentQueueMessage;
return(true);
}
else
{
return(false);
}
}
public void ProccesCollectionTasks()
{
bool loop = true;
while (loop)
{
IEnumerable <CloudQueueMessage> queueMessages = AzureClientService.GetQueueMessages(QueueTypeEnum.ProcessingTwitter, 32);
int messageCount = queueMessages.Count <CloudQueueMessage>();
if (messageCount < 1)
{
Console.WriteLine("No Messages");
loop = false;
}
else
{
Console.WriteLine("GetQueueMessage = " + queueMessages.Count <CloudQueueMessage>());
foreach (CloudQueueMessage queueMessage in queueMessages)
{
//1. Get Queue
T_CollectionTask t_newTask = CollectionsTaskHelper.MessageToT_CollectionTask(queueMessage.AsString);
//string taskString = CollectionsTaskHelper.T_CollectionsTaskToMessage(newTask);
//Console.WriteLine(taskString);
//2. Get Blob
CloudBlob blob = AzureClientService.GetBlobReference(t_newTask.Project, t_newTask.BlobName);
string blobText = blob.DownloadText();
//TODO: Blob Validate Hash
//Console.WriteLine(blobText);
//3. Update Database: Results
if (t_newTask.Command == TwitterCommandEnum.Followers.ToString())
{
ProcessTwitterFollowers(blobText, t_newTask);
}
else if (t_newTask.Command == TwitterCommandEnum.Friends.ToString())
{
ProcessTwitterFriends(blobText, t_newTask);
}
else if (t_newTask.Command == TwitterCommandEnum.Profile.ToString())
{
ProcessTwitterProfile(blobText, t_newTask);
}
else if (t_newTask.Command == TwitterCommandEnum.Tweets.ToString())
{
ProcessTwitterTweets(blobText, t_newTask);
}
else if (t_newTask.Command == TwitterCommandEnum.Search.ToString())
{
ProcessTwitterSearch(blobText, t_newTask);
}
else
{
throw new NotImplementedException();
}
//4. Update Database: Tasks
LinqToSqlAzureHaystackDataContext context = new LinqToSqlAzureHaystackDataContext();
T_CollectionTask t_oldTask = (from t in context.T_CollectionTasks
where t.Id == t_newTask.Id
orderby t.Id descending
select t).First <T_CollectionTask>();
t_oldTask.State = StateTypeEnum.Processed.ToString();
t_oldTask.Collected = t_newTask.Collected;
t_oldTask.Station = t_newTask.Station;
t_oldTask.Collector = t_newTask.Collector;
t_oldTask.BlobName = t_newTask.BlobName;
t_oldTask.ResultHash = blob.Attributes.Properties.ContentMD5;
t_oldTask.Processed = DateTime.UtcNow;
context.SubmitChanges();
//5. Update Blob Metadata
blob.Attributes.Metadata["State"] = StateTypeEnum.Processed.ToString();
//6. Delete Queue
AzureClientService.DeleteQueueMessage(QueueTypeEnum.ProcessingTwitter, queueMessage);
//7. Print Success
string output = CollectionsTaskHelper.T_CollectionsTaskToMessage(t_newTask);
Console.WriteLine(output);
}
}
}
}