Esempio n. 1
0
        public int IssueCollectionTasks()
        {
            int tasksIssued = 0;

            //Query Collection Tasks
            LinqToSqlAzureHaystackDataContext context = new LinqToSqlAzureHaystackDataContext();
            IQueryable <T_CollectionTask>     results =
                from r in context.T_CollectionTasks
                where r.State == StateTypeEnum.Approved.ToString()
                select r;

            //Write Queue Message, Update State to "Issued"
            foreach (T_CollectionTask r in results)
            {
                r.State  = StateTypeEnum.Issued.ToString();
                r.Issued = DateTime.UtcNow;

                string message = CollectionsTaskHelper.T_CollectionsTaskToMessage(r);

                QueueTypeEnum queueName;

                if (r.Command == TwitterCommandEnum.Followers.ToString())
                {
                    queueName = QueueTypeEnum.CollectionsTwitterFollowers;
                }
                else if (r.Command == TwitterCommandEnum.Friends.ToString())
                {
                    queueName = QueueTypeEnum.CollectionsTwitterFriends;
                }
                else if (r.Command == TwitterCommandEnum.Profile.ToString())
                {
                    queueName = QueueTypeEnum.CollectionsTwitterProfile;
                }
                else if (r.Command == TwitterCommandEnum.Search.ToString())
                {
                    queueName = QueueTypeEnum.CollectionsTwitterSearch;
                }
                else if (r.Command == TwitterCommandEnum.Tweets.ToString())
                {
                    queueName = QueueTypeEnum.CollectionsTwitterTweets;
                }
                else
                {
                    throw new Exception("Command Not Found");
                }

                AzureClientService.AddQueueMessage(queueName, message);
                tasksIssued++;

                string task = string.Format("{0} {1} {2} {3} {4} {5} {6} ",
                                            r.Id.ToString(), r.State, r.Created.ToString(), r.Project, r.Source, r.Command, r.Target);

                Console.WriteLine(task);
            }

            //Submit Changes
            context.SubmitChanges();

            return(tasksIssued);
        }
        public static void Display()
        {
            CloudQueue queue        = AzureClientService.GetCloudQueue(QueueTypeEnum.ProcessingTwitter);
            int        messageCount = queue.RetrieveApproximateMessageCount();

            for (int i = 0; i < messageCount; i++)
            {
                CloudQueueMessage message = queue.GetMessage();
                CollectionsTask   task    = CollectionsTaskHelper.MessageToCollectionsTask(message.AsString);
                Console.WriteLine(string.Format("ID={0}, Command{1}, Target={2}",
                                                task.Id, task.Command, task.Target));

                //GetBlob
                CloudBlob blob       = AzureClientService.GetBlobReference(task.Project, task.BlobName);
                string    resultsXml = blob.DownloadText();

                Console.WriteLine(string.Format("BLOB: {0}, {1}, {2}, {3}, {4},{5}",
                                                blob.Attributes.Metadata["Id"].ToString(),
                                                blob.Attributes.Metadata["Project"].ToString(),
                                                blob.Attributes.Metadata["State"].ToString(),
                                                blob.Attributes.Metadata["Collected"].ToString(),
                                                blob.Name,
                                                blob.Attributes.Properties.ContentMD5.ToString()));
                Console.WriteLine(string.Format("BLOB XML: {0}", resultsXml));
            }
        }
Esempio n. 3
0
        //5-ReportCollectionTask
        protected void ReportCollectionTask()
        {
            Logger.Log(GetContext() + "ReportCollectionTask(): #" + _currentTask.Id);

            //MetaData
            _currentTask.Station   = _station;
            _currentTask.Collector = _collector;
            _currentTask.Collected = CollectionsTaskHelper.Now();

            //SaveBlob
            CloudBlob blob = AzureClientService.GetBlobReference(_currentTask.Project, _currentTask.BlobName);

            blob.Attributes.Properties.ContentMD5 = _currentTask.ResultHash;
            blob.Attributes.Metadata["Id"]        = _currentTask.Id.ToString();
            blob.Attributes.Metadata["State"]     = StateTypeEnum.Collected.ToString();
            blob.Attributes.Metadata["Project"]   = _currentTask.Project;
            blob.Attributes.Metadata["Collected"] = _currentTask.Collected;
            blob.Attributes.Metadata["Command"]   = _currentTask.Command;
            //blob.Attributes.Metadata["TargetId"] = _currentTask.
            blob.UploadText(_currentTask.ResultJson);

            //AddQueue
            string message = CollectionsTaskHelper.CollectionsTaskToMessage(_currentTask);

            AzureClientService.AddQueueMessage(QueueTypeEnum.ProcessingTwitter, message);
            AzureClientService.DeleteQueueMessage(_collectionsQueue, _currentQueueMessage);
        }
        //Step 5, Inner Loop
        private void ReportCollectionTask()
        {
            Logger.Log("ReportCollectionTask: #" + _currentTask.Command);
            Logger.Log("");

            //MetaData
            _currentTask.CollectionsStation = _collectionsStation;
            _currentTask.CollectorInstance  = _collectorInstance;
            _currentTask.Collected          = CollectionsTaskHelper.Now();

            //SaveBlob
            CloudBlob blob = StorageClientHelper.GetBlobReference(_currentTask.Project, _currentTask.BlobName);

            blob.Attributes.Properties.ContentMD5 = _currentTask.ResultsHash;
            blob.Attributes.Metadata["Id"]        = _currentTask.Id.ToString();
            blob.Attributes.Metadata["Project"]   = _currentTask.Project;
            blob.Attributes.Metadata["State"]     = CollectionsTaskState.Collected.ToString();
            blob.Attributes.Metadata["Collected"] = _currentTask.Collected;
            blob.UploadText(_currentTask.ResultsXml);

            //AddQueue
            string message = CollectionsTaskHelper.LocalTaskToMessage(_currentTask);

            StorageClientHelper.AddQueueMessage(QueueType.Processing, SourceType.Twitter, message);
        }
        //Step 3, Outer Loop
        private List <CollectionsTaskLocal> RequestCollectionTasks(int collectionCapacity)
        {
            Logger.Log("RequestCollectionTasks");
            //Hard Limit by Azure
            int messageCount = Math.Min(collectionCapacity, 32);

            //Get Q Messages
            IEnumerable <CloudQueueMessage> queueMessages = StorageClientHelper.GetQueueMessages(QueueType.Collections, _sourceType, messageCount);

            //Convert to Tasks
            List <CollectionsTaskLocal> collectionsTaskList = new List <CollectionsTaskLocal>();

            if (queueMessages == null)
            {
                return(collectionsTaskList);
            }
            else
            {
                Logger.Log("Received #" + queueMessages.Count <CloudQueueMessage>().ToString() + " messages");
            }

            foreach (CloudQueueMessage qm in queueMessages)
            {
                string messag             = qm.AsString;
                CollectionsTaskLocal task = CollectionsTaskHelper.MessageToLocalTask(messag);
                task.QueueMessage = qm;
                collectionsTaskList.Add(task);
            }

            return(collectionsTaskList);
        }
        public static void Delete()
        {
            CloudQueue queue        = AzureClientService.GetCloudQueue(QueueTypeEnum.ProcessingTwitter);
            int        messageCount = queue.RetrieveApproximateMessageCount();

            for (int i = 0; i < messageCount; i++)
            {
                CloudQueueMessage message = queue.GetMessage();
                CollectionsTask   task    = CollectionsTaskHelper.MessageToCollectionsTask(message.AsString);

                //GetBlob
                CloudBlob blob = AzureClientService.GetBlobReference(task.Project, task.BlobName);
#if DEBUG
                blob.DeleteIfExists();
#endif
                queue.DeleteMessage(message);
            }
        }
Esempio n. 7
0
        //2-GetCollectionTask
        protected bool GetCollectionTask()
        {
            Logger.Log(GetContext() + "GetCollectionTask()");


            CloudQueueMessage queueMessage = AzureClientService.GetQueueMessage(_collectionsQueue);

            if (queueMessage != null)
            {
                _currentQueueMessage      = queueMessage;
                _currentTask              = CollectionsTaskHelper.MessageToCollectionsTask(_currentQueueMessage.AsString);
                _currentTask.QueueMessage = _currentQueueMessage;
                return(true);
            }
            else
            {
                return(false);
            }
        }
        public void ProccesCollectionTasks()
        {
            bool loop = true;

            while (loop)
            {
                IEnumerable <CloudQueueMessage> queueMessages = AzureClientService.GetQueueMessages(QueueTypeEnum.ProcessingTwitter, 32);

                int messageCount = queueMessages.Count <CloudQueueMessage>();

                if (messageCount < 1)
                {
                    Console.WriteLine("No Messages");
                    loop = false;
                }
                else
                {
                    Console.WriteLine("GetQueueMessage = " + queueMessages.Count <CloudQueueMessage>());

                    foreach (CloudQueueMessage queueMessage in queueMessages)
                    {
                        //1. Get Queue
                        T_CollectionTask t_newTask = CollectionsTaskHelper.MessageToT_CollectionTask(queueMessage.AsString);
                        //string taskString = CollectionsTaskHelper.T_CollectionsTaskToMessage(newTask);
                        //Console.WriteLine(taskString);

                        //2. Get Blob
                        CloudBlob blob     = AzureClientService.GetBlobReference(t_newTask.Project, t_newTask.BlobName);
                        string    blobText = blob.DownloadText();
                        //TODO: Blob Validate Hash
                        //Console.WriteLine(blobText);

                        //3. Update Database: Results
                        if (t_newTask.Command == TwitterCommandEnum.Followers.ToString())
                        {
                            ProcessTwitterFollowers(blobText, t_newTask);
                        }
                        else if (t_newTask.Command == TwitterCommandEnum.Friends.ToString())
                        {
                            ProcessTwitterFriends(blobText, t_newTask);
                        }
                        else if (t_newTask.Command == TwitterCommandEnum.Profile.ToString())
                        {
                            ProcessTwitterProfile(blobText, t_newTask);
                        }
                        else if (t_newTask.Command == TwitterCommandEnum.Tweets.ToString())
                        {
                            ProcessTwitterTweets(blobText, t_newTask);
                        }
                        else if (t_newTask.Command == TwitterCommandEnum.Search.ToString())
                        {
                            ProcessTwitterSearch(blobText, t_newTask);
                        }
                        else
                        {
                            throw new NotImplementedException();
                        }

                        //4. Update Database: Tasks
                        LinqToSqlAzureHaystackDataContext context = new LinqToSqlAzureHaystackDataContext();
                        T_CollectionTask t_oldTask = (from t in context.T_CollectionTasks
                                                      where t.Id == t_newTask.Id
                                                      orderby t.Id descending
                                                      select t).First <T_CollectionTask>();

                        t_oldTask.State      = StateTypeEnum.Processed.ToString();
                        t_oldTask.Collected  = t_newTask.Collected;
                        t_oldTask.Station    = t_newTask.Station;
                        t_oldTask.Collector  = t_newTask.Collector;
                        t_oldTask.BlobName   = t_newTask.BlobName;
                        t_oldTask.ResultHash = blob.Attributes.Properties.ContentMD5;
                        t_oldTask.Processed  = DateTime.UtcNow;
                        context.SubmitChanges();

                        //5. Update Blob Metadata
                        blob.Attributes.Metadata["State"] = StateTypeEnum.Processed.ToString();

                        //6. Delete Queue
                        AzureClientService.DeleteQueueMessage(QueueTypeEnum.ProcessingTwitter, queueMessage);

                        //7. Print Success
                        string output = CollectionsTaskHelper.T_CollectionsTaskToMessage(t_newTask);
                        Console.WriteLine(output);
                    }
                }
            }
        }