public int IssueCollectionTasks()
{
int tasksIssued = 0;
//Query Collection Tasks
LinqToSqlDataContext context = new LinqToSqlDataContext();
IQueryable <HaystackLibrary.CollectionTask> results =
from r in context.CollectionTasks
where r.State == CollectionsTaskState.Approved.ToString()
select r;
//Write Queue Message, Update State to "Issued"
foreach (HaystackLibrary.CollectionTask r in results)
{
r.State = CollectionsTaskState.Issued.ToString();
r.Issued = DateTime.UtcNow;
string message = CollectionsTaskHelper.DatabaseTaskToMessage(r);
StorageClientHelper.AddQueueMessage(QueueType.Collections, SourceType.Twitter, message);
tasksIssued++;
string task = string.Format("{0} {1} {2} {3} {4} {5} {6} ",
r.Id.ToString(), r.State, r.Created.ToString(), r.Project, r.Source, r.Command, r.Target);
Console.WriteLine(task);
}
//Submit Changes
context.SubmitChanges();
return(tasksIssued);
}
protected CollectionsTaskLocal _currentTask; //used in ExecuteTask and ReportTask
//Step 0, Timer Loop, Outer Loop, Inner Loop
void _collectionLoopTimer_Elapsed(object sender, ElapsedEventArgs e)
{
Logger.Log("");
Logger.Log("");
Logger.Log("----------Timer Loop ----------");
PreCollection();
while ((_collectionCapacity > 0) && _queueContainsMessages)
{
//RequestCollectionTasks Math.Min(_collectionCapacity, 32)
List <CollectionsTaskLocal> collectionsTaskList = RequestCollectionTasks(_collectionCapacity);
if (collectionsTaskList.Count <CollectionsTaskLocal>() > 0)
{
foreach (CollectionsTaskLocal taskIn in collectionsTaskList)
{
_collectionCapacity--;
_currentTask = taskIn;
ExecuteCollectionTask();
ReportCollectionTask();
StorageClientHelper.DeleteQueueMessage(QueueType.Collections, SourceType.Twitter, _currentTask.QueueMessage);
}
}
else
{
Logger.Log("Received zero messages");
_queueContainsMessages = false;
}
}
PostCollection();
}
//Step 5, Inner Loop
private void ReportCollectionTask()
{
Logger.Log("ReportCollectionTask: #" + _currentTask.Command);
Logger.Log("");
//MetaData
_currentTask.CollectionsStation = _collectionsStation;
_currentTask.CollectorInstance = _collectorInstance;
_currentTask.Collected = CollectionsTaskHelper.Now();
//SaveBlob
CloudBlob blob = StorageClientHelper.GetBlobReference(_currentTask.Project, _currentTask.BlobName);
blob.Attributes.Properties.ContentMD5 = _currentTask.ResultsHash;
blob.Attributes.Metadata["Id"] = _currentTask.Id.ToString();
blob.Attributes.Metadata["Project"] = _currentTask.Project;
blob.Attributes.Metadata["State"] = CollectionsTaskState.Collected.ToString();
blob.Attributes.Metadata["Collected"] = _currentTask.Collected;
blob.UploadText(_currentTask.ResultsXml);
//AddQueue
string message = CollectionsTaskHelper.LocalTaskToMessage(_currentTask);
StorageClientHelper.AddQueueMessage(QueueType.Processing, SourceType.Twitter, message);
}
//Step 3, Outer Loop
private List <CollectionsTaskLocal> RequestCollectionTasks(int collectionCapacity)
{
Logger.Log("RequestCollectionTasks");
//Hard Limit by Azure
int messageCount = Math.Min(collectionCapacity, 32);
//Get Q Messages
IEnumerable <CloudQueueMessage> queueMessages = StorageClientHelper.GetQueueMessages(QueueType.Collections, _sourceType, messageCount);
//Convert to Tasks
List <CollectionsTaskLocal> collectionsTaskList = new List <CollectionsTaskLocal>();
if (queueMessages == null)
{
return(collectionsTaskList);
}
else
{
Logger.Log("Received #" + queueMessages.Count <CloudQueueMessage>().ToString() + " messages");
}
foreach (CloudQueueMessage qm in queueMessages)
{
string messag = qm.AsString;
CollectionsTaskLocal task = CollectionsTaskHelper.MessageToLocalTask(messag);
task.QueueMessage = qm;
collectionsTaskList.Add(task);
}
return(collectionsTaskList);
}
public void UseBucket()
{
try
{
var storageClient = StorageClientHelper.GetStorageClient();
this.TestBucket(FirebaseApp.DefaultInstance.GetProjectId(), storageClient);
}
catch (Exception ex)
{
Assert.NotEmpty(ex.Message);
}
}
public void UseAfterDelete()
{
try
{
var app = FirebaseApp.Create(new AppOptions()
{
Credential = MockCredential
});
StorageClient storageClient = StorageClientHelper.GetStorageClient(app);
app.Delete();
Assert.Throws <ObjectDisposedException>(
() => storageClient.GetBucket("test"));
}
catch (Exception ex)
{
Assert.NotEmpty(ex.Message);
}
}
public void GetStorageClient()
{
try
{
var app = FirebaseApp.Create(new AppOptions()
{
Credential = MockCredential
});
StorageClient storageClient = StorageClientHelper.GetStorageClient(app);
Assert.Same(storageClient, StorageClientHelper.GetStorageClient(app));
app.Delete();
Assert.Throws <InvalidOperationException>(() => StorageClientHelper.GetStorageClient(app));
}
catch (Exception ex)
{
Assert.NotEmpty(ex.Message);
}
}
public void GetDefaultStorageClient()
{
try
{
var app = FirebaseApp.Create(new AppOptions()
{
Credential = MockCredential
});
StorageClient storageClient = StorageClientHelper.GetStorageClient();
Assert.Same(storageClient, StorageClientHelper.GetStorageClient());
app.Delete();
Assert.Null(StorageClientHelper.GetStorageClient());
}
catch (Exception ex)
{
Assert.NotEmpty(ex.Message);
}
}
public ServiceRegistration GetRegisteredService <TEntity>(string name, out TEntity[] entities)
where TEntity : ServiceEntityBase
{
return(StorageClientHelper <StorageServiceClient> .GetRegisteredService <TEntity>(GetRegisteredService, name, out entities));
}
public override void RegisterWithStorage(StorageServiceClient storageClient)
{
Registration = StorageClientHelper <StorageServiceClient> .RegisterWithStorageAndSync <SensorEntity, ISensor>(storageClient, Name, _sensors);
}
public override void RegisterWithStorage(StorageServiceClient storageClient)
{
Registration = StorageClientHelper <StorageServiceClient> .RegisterWithStorageAndSync <ObjectEntity, IObject>(storageClient, Name, _objects);
}
public void GetStorageClientWithoutApp()
{
Assert.Null(StorageClientHelper.GetStorageClient());
}
public void ProccesCollectionTasks()
{
bool loop = true;
while (loop)
{
IEnumerable <CloudQueueMessage> queueMessages = StorageClientHelper.GetQueueMessages(QueueType.Processing, SourceType.Twitter, 32);
int messageCount = queueMessages.Count <CloudQueueMessage>();
if ((messageCount < 1) || (messageCount == null))
{
Console.WriteLine("No Messages");
loop = false;
}
else
{
Console.WriteLine("GetQueueMessage = " + queueMessages.Count <CloudQueueMessage>());
foreach (CloudQueueMessage queueMessage in queueMessages)
{
//1. Get Queue
HaystackLibrary.CollectionTask newTask = CollectionsTaskHelper.MessageToDatabaseTask(queueMessage.AsString);
string taskString = CollectionsTaskHelper.DatabaseTaskString(newTask);
Console.WriteLine(taskString);
//2. Get Blob
CloudBlob blob = StorageClientHelper.GetBlobReference(newTask.Project, newTask.BlobName);
string blobText = blob.DownloadText();
//TODO: Blob Validate Hash
Console.WriteLine(blobText);
//3. Update Database: Results
//TODO: Update Database: Results
//4. Update Database: Tasks
LinqToSqlDataContext context = new LinqToSqlDataContext();
CollectionTask oldTask = (from t in context.CollectionTasks
where t.Id == newTask.Id
orderby t.Id descending
select t).First <CollectionTask>();
oldTask.State = CollectionsTaskState.Processed.ToString();
oldTask.Collected = newTask.Collected;
oldTask.Station = newTask.Station;
oldTask.Collector = newTask.Collector;
oldTask.BlobName = newTask.BlobName;
oldTask.BlobHash = blob.Attributes.Properties.ContentMD5;
oldTask.Processed = DateTime.UtcNow;
context.SubmitChanges();
//5. Update Blob Metadata
blob.Attributes.Metadata["State"] = CollectionsTaskState.Processed.ToString();
//6. Delete Queue
StorageClientHelper.DeleteQueueMessage(QueueType.Processing, SourceType.Twitter, queueMessage);
//7. Print Success
string output = CollectionsTaskHelper.DatabaseTaskString(newTask);
Console.WriteLine(output);
}
}
}
}
public override void RegisterWithStorage(StorageServiceClient storageClient)
{
Registration = StorageClientHelper <StorageServiceClient> .RegisterWithStorageAndSync <ProcessEntity, IProcess>(storageClient, Name, _processes);
}
