public int IssueCollectionTasks() { int tasksIssued = 0; //Query Collection Tasks LinqToSqlDataContext context = new LinqToSqlDataContext(); IQueryable <HaystackLibrary.CollectionTask> results = from r in context.CollectionTasks where r.State == CollectionsTaskState.Approved.ToString() select r; //Write Queue Message, Update State to "Issued" foreach (HaystackLibrary.CollectionTask r in results) { r.State = CollectionsTaskState.Issued.ToString(); r.Issued = DateTime.UtcNow; string message = CollectionsTaskHelper.DatabaseTaskToMessage(r); StorageClientHelper.AddQueueMessage(QueueType.Collections, SourceType.Twitter, message); tasksIssued++; string task = string.Format("{0} {1} {2} {3} {4} {5} {6} ", r.Id.ToString(), r.State, r.Created.ToString(), r.Project, r.Source, r.Command, r.Target); Console.WriteLine(task); } //Submit Changes context.SubmitChanges(); return(tasksIssued); }
protected CollectionsTaskLocal _currentTask; //used in ExecuteTask and ReportTask //Step 0, Timer Loop, Outer Loop, Inner Loop void _collectionLoopTimer_Elapsed(object sender, ElapsedEventArgs e) { Logger.Log(""); Logger.Log(""); Logger.Log("----------Timer Loop ----------"); PreCollection(); while ((_collectionCapacity > 0) && _queueContainsMessages) { //RequestCollectionTasks Math.Min(_collectionCapacity, 32) List <CollectionsTaskLocal> collectionsTaskList = RequestCollectionTasks(_collectionCapacity); if (collectionsTaskList.Count <CollectionsTaskLocal>() > 0) { foreach (CollectionsTaskLocal taskIn in collectionsTaskList) { _collectionCapacity--; _currentTask = taskIn; ExecuteCollectionTask(); ReportCollectionTask(); StorageClientHelper.DeleteQueueMessage(QueueType.Collections, SourceType.Twitter, _currentTask.QueueMessage); } } else { Logger.Log("Received zero messages"); _queueContainsMessages = false; } } PostCollection(); }
//Step 5, Inner Loop private void ReportCollectionTask() { Logger.Log("ReportCollectionTask: #" + _currentTask.Command); Logger.Log(""); //MetaData _currentTask.CollectionsStation = _collectionsStation; _currentTask.CollectorInstance = _collectorInstance; _currentTask.Collected = CollectionsTaskHelper.Now(); //SaveBlob CloudBlob blob = StorageClientHelper.GetBlobReference(_currentTask.Project, _currentTask.BlobName); blob.Attributes.Properties.ContentMD5 = _currentTask.ResultsHash; blob.Attributes.Metadata["Id"] = _currentTask.Id.ToString(); blob.Attributes.Metadata["Project"] = _currentTask.Project; blob.Attributes.Metadata["State"] = CollectionsTaskState.Collected.ToString(); blob.Attributes.Metadata["Collected"] = _currentTask.Collected; blob.UploadText(_currentTask.ResultsXml); //AddQueue string message = CollectionsTaskHelper.LocalTaskToMessage(_currentTask); StorageClientHelper.AddQueueMessage(QueueType.Processing, SourceType.Twitter, message); }
//Step 3, Outer Loop private List <CollectionsTaskLocal> RequestCollectionTasks(int collectionCapacity) { Logger.Log("RequestCollectionTasks"); //Hard Limit by Azure int messageCount = Math.Min(collectionCapacity, 32); //Get Q Messages IEnumerable <CloudQueueMessage> queueMessages = StorageClientHelper.GetQueueMessages(QueueType.Collections, _sourceType, messageCount); //Convert to Tasks List <CollectionsTaskLocal> collectionsTaskList = new List <CollectionsTaskLocal>(); if (queueMessages == null) { return(collectionsTaskList); } else { Logger.Log("Received #" + queueMessages.Count <CloudQueueMessage>().ToString() + " messages"); } foreach (CloudQueueMessage qm in queueMessages) { string messag = qm.AsString; CollectionsTaskLocal task = CollectionsTaskHelper.MessageToLocalTask(messag); task.QueueMessage = qm; collectionsTaskList.Add(task); } return(collectionsTaskList); }
public void UseBucket() { try { var storageClient = StorageClientHelper.GetStorageClient(); this.TestBucket(FirebaseApp.DefaultInstance.GetProjectId(), storageClient); } catch (Exception ex) { Assert.NotEmpty(ex.Message); } }
public void UseAfterDelete() { try { var app = FirebaseApp.Create(new AppOptions() { Credential = MockCredential }); StorageClient storageClient = StorageClientHelper.GetStorageClient(app); app.Delete(); Assert.Throws <ObjectDisposedException>( () => storageClient.GetBucket("test")); } catch (Exception ex) { Assert.NotEmpty(ex.Message); } }
public void GetStorageClient() { try { var app = FirebaseApp.Create(new AppOptions() { Credential = MockCredential }); StorageClient storageClient = StorageClientHelper.GetStorageClient(app); Assert.Same(storageClient, StorageClientHelper.GetStorageClient(app)); app.Delete(); Assert.Throws <InvalidOperationException>(() => StorageClientHelper.GetStorageClient(app)); } catch (Exception ex) { Assert.NotEmpty(ex.Message); } }
public void GetDefaultStorageClient() { try { var app = FirebaseApp.Create(new AppOptions() { Credential = MockCredential }); StorageClient storageClient = StorageClientHelper.GetStorageClient(); Assert.Same(storageClient, StorageClientHelper.GetStorageClient()); app.Delete(); Assert.Null(StorageClientHelper.GetStorageClient()); } catch (Exception ex) { Assert.NotEmpty(ex.Message); } }
public ServiceRegistration GetRegisteredService <TEntity>(string name, out TEntity[] entities) where TEntity : ServiceEntityBase { return(StorageClientHelper <StorageServiceClient> .GetRegisteredService <TEntity>(GetRegisteredService, name, out entities)); }
public override void RegisterWithStorage(StorageServiceClient storageClient) { Registration = StorageClientHelper <StorageServiceClient> .RegisterWithStorageAndSync <SensorEntity, ISensor>(storageClient, Name, _sensors); }
public override void RegisterWithStorage(StorageServiceClient storageClient) { Registration = StorageClientHelper <StorageServiceClient> .RegisterWithStorageAndSync <ObjectEntity, IObject>(storageClient, Name, _objects); }
public void GetStorageClientWithoutApp() { Assert.Null(StorageClientHelper.GetStorageClient()); }
public void ProccesCollectionTasks() { bool loop = true; while (loop) { IEnumerable <CloudQueueMessage> queueMessages = StorageClientHelper.GetQueueMessages(QueueType.Processing, SourceType.Twitter, 32); int messageCount = queueMessages.Count <CloudQueueMessage>(); if ((messageCount < 1) || (messageCount == null)) { Console.WriteLine("No Messages"); loop = false; } else { Console.WriteLine("GetQueueMessage = " + queueMessages.Count <CloudQueueMessage>()); foreach (CloudQueueMessage queueMessage in queueMessages) { //1. Get Queue HaystackLibrary.CollectionTask newTask = CollectionsTaskHelper.MessageToDatabaseTask(queueMessage.AsString); string taskString = CollectionsTaskHelper.DatabaseTaskString(newTask); Console.WriteLine(taskString); //2. Get Blob CloudBlob blob = StorageClientHelper.GetBlobReference(newTask.Project, newTask.BlobName); string blobText = blob.DownloadText(); //TODO: Blob Validate Hash Console.WriteLine(blobText); //3. Update Database: Results //TODO: Update Database: Results //4. Update Database: Tasks LinqToSqlDataContext context = new LinqToSqlDataContext(); CollectionTask oldTask = (from t in context.CollectionTasks where t.Id == newTask.Id orderby t.Id descending select t).First <CollectionTask>(); oldTask.State = CollectionsTaskState.Processed.ToString(); oldTask.Collected = newTask.Collected; oldTask.Station = newTask.Station; oldTask.Collector = newTask.Collector; oldTask.BlobName = newTask.BlobName; oldTask.BlobHash = blob.Attributes.Properties.ContentMD5; oldTask.Processed = DateTime.UtcNow; context.SubmitChanges(); //5. Update Blob Metadata blob.Attributes.Metadata["State"] = CollectionsTaskState.Processed.ToString(); //6. Delete Queue StorageClientHelper.DeleteQueueMessage(QueueType.Processing, SourceType.Twitter, queueMessage); //7. Print Success string output = CollectionsTaskHelper.DatabaseTaskString(newTask); Console.WriteLine(output); } } } }
public override void RegisterWithStorage(StorageServiceClient storageClient) { Registration = StorageClientHelper <StorageServiceClient> .RegisterWithStorageAndSync <ProcessEntity, IProcess>(storageClient, Name, _processes); }