public int IssueCollectionTasks() { int tasksIssued = 0; //Query Collection Tasks LinqToSqlAzureHaystackDataContext context = new LinqToSqlAzureHaystackDataContext(); IQueryable <T_CollectionTask> results = from r in context.T_CollectionTasks where r.State == StateTypeEnum.Approved.ToString() select r; //Write Queue Message, Update State to "Issued" foreach (T_CollectionTask r in results) { r.State = StateTypeEnum.Issued.ToString(); r.Issued = DateTime.UtcNow; string message = CollectionsTaskHelper.T_CollectionsTaskToMessage(r); QueueTypeEnum queueName; if (r.Command == TwitterCommandEnum.Followers.ToString()) { queueName = QueueTypeEnum.CollectionsTwitterFollowers; } else if (r.Command == TwitterCommandEnum.Friends.ToString()) { queueName = QueueTypeEnum.CollectionsTwitterFriends; } else if (r.Command == TwitterCommandEnum.Profile.ToString()) { queueName = QueueTypeEnum.CollectionsTwitterProfile; } else if (r.Command == TwitterCommandEnum.Search.ToString()) { queueName = QueueTypeEnum.CollectionsTwitterSearch; } else if (r.Command == TwitterCommandEnum.Tweets.ToString()) { queueName = QueueTypeEnum.CollectionsTwitterTweets; } else { throw new Exception("Command Not Found"); } AzureClientService.AddQueueMessage(queueName, message); tasksIssued++; string task = string.Format("{0} {1} {2} {3} {4} {5} {6} ", r.Id.ToString(), r.State, r.Created.ToString(), r.Project, r.Source, r.Command, r.Target); Console.WriteLine(task); } //Submit Changes context.SubmitChanges(); return(tasksIssued); }
public static void Display() { CloudQueue queue = AzureClientService.GetCloudQueue(QueueTypeEnum.ProcessingTwitter); int messageCount = queue.RetrieveApproximateMessageCount(); for (int i = 0; i < messageCount; i++) { CloudQueueMessage message = queue.GetMessage(); CollectionsTask task = CollectionsTaskHelper.MessageToCollectionsTask(message.AsString); Console.WriteLine(string.Format("ID={0}, Command{1}, Target={2}", task.Id, task.Command, task.Target)); //GetBlob CloudBlob blob = AzureClientService.GetBlobReference(task.Project, task.BlobName); string resultsXml = blob.DownloadText(); Console.WriteLine(string.Format("BLOB: {0}, {1}, {2}, {3}, {4},{5}", blob.Attributes.Metadata["Id"].ToString(), blob.Attributes.Metadata["Project"].ToString(), blob.Attributes.Metadata["State"].ToString(), blob.Attributes.Metadata["Collected"].ToString(), blob.Name, blob.Attributes.Properties.ContentMD5.ToString())); Console.WriteLine(string.Format("BLOB XML: {0}", resultsXml)); } }
//5-ReportCollectionTask protected void ReportCollectionTask() { Logger.Log(GetContext() + "ReportCollectionTask(): #" + _currentTask.Id); //MetaData _currentTask.Station = _station; _currentTask.Collector = _collector; _currentTask.Collected = CollectionsTaskHelper.Now(); //SaveBlob CloudBlob blob = AzureClientService.GetBlobReference(_currentTask.Project, _currentTask.BlobName); blob.Attributes.Properties.ContentMD5 = _currentTask.ResultHash; blob.Attributes.Metadata["Id"] = _currentTask.Id.ToString(); blob.Attributes.Metadata["State"] = StateTypeEnum.Collected.ToString(); blob.Attributes.Metadata["Project"] = _currentTask.Project; blob.Attributes.Metadata["Collected"] = _currentTask.Collected; blob.Attributes.Metadata["Command"] = _currentTask.Command; //blob.Attributes.Metadata["TargetId"] = _currentTask. blob.UploadText(_currentTask.ResultJson); //AddQueue string message = CollectionsTaskHelper.CollectionsTaskToMessage(_currentTask); AzureClientService.AddQueueMessage(QueueTypeEnum.ProcessingTwitter, message); AzureClientService.DeleteQueueMessage(_collectionsQueue, _currentQueueMessage); }
//Step 5, Inner Loop private void ReportCollectionTask() { Logger.Log("ReportCollectionTask: #" + _currentTask.Command); Logger.Log(""); //MetaData _currentTask.CollectionsStation = _collectionsStation; _currentTask.CollectorInstance = _collectorInstance; _currentTask.Collected = CollectionsTaskHelper.Now(); //SaveBlob CloudBlob blob = StorageClientHelper.GetBlobReference(_currentTask.Project, _currentTask.BlobName); blob.Attributes.Properties.ContentMD5 = _currentTask.ResultsHash; blob.Attributes.Metadata["Id"] = _currentTask.Id.ToString(); blob.Attributes.Metadata["Project"] = _currentTask.Project; blob.Attributes.Metadata["State"] = CollectionsTaskState.Collected.ToString(); blob.Attributes.Metadata["Collected"] = _currentTask.Collected; blob.UploadText(_currentTask.ResultsXml); //AddQueue string message = CollectionsTaskHelper.LocalTaskToMessage(_currentTask); StorageClientHelper.AddQueueMessage(QueueType.Processing, SourceType.Twitter, message); }
//Step 3, Outer Loop private List <CollectionsTaskLocal> RequestCollectionTasks(int collectionCapacity) { Logger.Log("RequestCollectionTasks"); //Hard Limit by Azure int messageCount = Math.Min(collectionCapacity, 32); //Get Q Messages IEnumerable <CloudQueueMessage> queueMessages = StorageClientHelper.GetQueueMessages(QueueType.Collections, _sourceType, messageCount); //Convert to Tasks List <CollectionsTaskLocal> collectionsTaskList = new List <CollectionsTaskLocal>(); if (queueMessages == null) { return(collectionsTaskList); } else { Logger.Log("Received #" + queueMessages.Count <CloudQueueMessage>().ToString() + " messages"); } foreach (CloudQueueMessage qm in queueMessages) { string messag = qm.AsString; CollectionsTaskLocal task = CollectionsTaskHelper.MessageToLocalTask(messag); task.QueueMessage = qm; collectionsTaskList.Add(task); } return(collectionsTaskList); }
public static void Delete() { CloudQueue queue = AzureClientService.GetCloudQueue(QueueTypeEnum.ProcessingTwitter); int messageCount = queue.RetrieveApproximateMessageCount(); for (int i = 0; i < messageCount; i++) { CloudQueueMessage message = queue.GetMessage(); CollectionsTask task = CollectionsTaskHelper.MessageToCollectionsTask(message.AsString); //GetBlob CloudBlob blob = AzureClientService.GetBlobReference(task.Project, task.BlobName); #if DEBUG blob.DeleteIfExists(); #endif queue.DeleteMessage(message); } }
//2-GetCollectionTask protected bool GetCollectionTask() { Logger.Log(GetContext() + "GetCollectionTask()"); CloudQueueMessage queueMessage = AzureClientService.GetQueueMessage(_collectionsQueue); if (queueMessage != null) { _currentQueueMessage = queueMessage; _currentTask = CollectionsTaskHelper.MessageToCollectionsTask(_currentQueueMessage.AsString); _currentTask.QueueMessage = _currentQueueMessage; return(true); } else { return(false); } }
public void ProccesCollectionTasks() { bool loop = true; while (loop) { IEnumerable <CloudQueueMessage> queueMessages = AzureClientService.GetQueueMessages(QueueTypeEnum.ProcessingTwitter, 32); int messageCount = queueMessages.Count <CloudQueueMessage>(); if (messageCount < 1) { Console.WriteLine("No Messages"); loop = false; } else { Console.WriteLine("GetQueueMessage = " + queueMessages.Count <CloudQueueMessage>()); foreach (CloudQueueMessage queueMessage in queueMessages) { //1. Get Queue T_CollectionTask t_newTask = CollectionsTaskHelper.MessageToT_CollectionTask(queueMessage.AsString); //string taskString = CollectionsTaskHelper.T_CollectionsTaskToMessage(newTask); //Console.WriteLine(taskString); //2. Get Blob CloudBlob blob = AzureClientService.GetBlobReference(t_newTask.Project, t_newTask.BlobName); string blobText = blob.DownloadText(); //TODO: Blob Validate Hash //Console.WriteLine(blobText); //3. Update Database: Results if (t_newTask.Command == TwitterCommandEnum.Followers.ToString()) { ProcessTwitterFollowers(blobText, t_newTask); } else if (t_newTask.Command == TwitterCommandEnum.Friends.ToString()) { ProcessTwitterFriends(blobText, t_newTask); } else if (t_newTask.Command == TwitterCommandEnum.Profile.ToString()) { ProcessTwitterProfile(blobText, t_newTask); } else if (t_newTask.Command == TwitterCommandEnum.Tweets.ToString()) { ProcessTwitterTweets(blobText, t_newTask); } else if (t_newTask.Command == TwitterCommandEnum.Search.ToString()) { ProcessTwitterSearch(blobText, t_newTask); } else { throw new NotImplementedException(); } //4. Update Database: Tasks LinqToSqlAzureHaystackDataContext context = new LinqToSqlAzureHaystackDataContext(); T_CollectionTask t_oldTask = (from t in context.T_CollectionTasks where t.Id == t_newTask.Id orderby t.Id descending select t).First <T_CollectionTask>(); t_oldTask.State = StateTypeEnum.Processed.ToString(); t_oldTask.Collected = t_newTask.Collected; t_oldTask.Station = t_newTask.Station; t_oldTask.Collector = t_newTask.Collector; t_oldTask.BlobName = t_newTask.BlobName; t_oldTask.ResultHash = blob.Attributes.Properties.ContentMD5; t_oldTask.Processed = DateTime.UtcNow; context.SubmitChanges(); //5. Update Blob Metadata blob.Attributes.Metadata["State"] = StateTypeEnum.Processed.ToString(); //6. Delete Queue AzureClientService.DeleteQueueMessage(QueueTypeEnum.ProcessingTwitter, queueMessage); //7. Print Success string output = CollectionsTaskHelper.T_CollectionsTaskToMessage(t_newTask); Console.WriteLine(output); } } } }