public static CollectionTask MessageToDatabaseTask(string message)
{
string[] fields = message.Split('\t');
CollectionTask task = new CollectionTask();
task.Id = int.Parse(fields[0]);
task.Created = DateTime.Parse(fields[1]);
task.Project = fields[2];
task.Source = fields[3];
task.State = fields[4];
task.Issued = DateTime.Parse(fields[5]);
task.Command = fields[6];
task.Target = fields[7];
task.Collected = DateTime.Parse(fields[8]);
task.Station = fields[9];
task.Collector = fields[10];
task.BlobName = fields[11];
task.Processed = null;
return(task);
}
public static string DatabaseTaskString(CollectionTask task)
{
string taskString = string.Format("{0} {1} {2} {3} {4} {5} {6} {7} {8} {9} {10} {11}",
task.Id.ToString(), task.State, task.Created.ToString(), task.Project, task.Source, task.Command, task.Target,
task.Collected.ToString() ?? "", task.Station ?? "", task.BlobName ?? "", task.BlobHash ?? "", task.Processed.ToString() ?? "");
return(taskString);
}
public static string DatabaseTaskToMessage(CollectionTask task)
{
string message = string.Format("{0}\t{1}\t{2}\t{3}\t{4}\t{5}\t{6}\t{7}\t{8}\t{9}\t{10}\t{11}\t{12}",
task.Id,
task.Created,
task.Project,
task.Source,
task.State,
task.Issued,
task.Command,
task.Target,
task.Collected,
task.Station,
task.Collector,
task.BlobName,
task.Processed);
return(message);
}
partial void DeleteCollectionTask(CollectionTask instance);
partial void UpdateCollectionTask(CollectionTask instance);
partial void InsertCollectionTask(CollectionTask instance);
public void ProccesCollectionTasks()
{
bool loop = true;
while (loop)
{
IEnumerable <CloudQueueMessage> queueMessages = StorageClientHelper.GetQueueMessages(QueueType.Processing, SourceType.Twitter, 32);
int messageCount = queueMessages.Count <CloudQueueMessage>();
if ((messageCount < 1) || (messageCount == null))
{
Console.WriteLine("No Messages");
loop = false;
}
else
{
Console.WriteLine("GetQueueMessage = " + queueMessages.Count <CloudQueueMessage>());
foreach (CloudQueueMessage queueMessage in queueMessages)
{
//1. Get Queue
HaystackLibrary.CollectionTask newTask = CollectionsTaskHelper.MessageToDatabaseTask(queueMessage.AsString);
string taskString = CollectionsTaskHelper.DatabaseTaskString(newTask);
Console.WriteLine(taskString);
//2. Get Blob
CloudBlob blob = StorageClientHelper.GetBlobReference(newTask.Project, newTask.BlobName);
string blobText = blob.DownloadText();
//TODO: Blob Validate Hash
Console.WriteLine(blobText);
//3. Update Database: Results
//TODO: Update Database: Results
//4. Update Database: Tasks
LinqToSqlDataContext context = new LinqToSqlDataContext();
CollectionTask oldTask = (from t in context.CollectionTasks
where t.Id == newTask.Id
orderby t.Id descending
select t).First <CollectionTask>();
oldTask.State = CollectionsTaskState.Processed.ToString();
oldTask.Collected = newTask.Collected;
oldTask.Station = newTask.Station;
oldTask.Collector = newTask.Collector;
oldTask.BlobName = newTask.BlobName;
oldTask.BlobHash = blob.Attributes.Properties.ContentMD5;
oldTask.Processed = DateTime.UtcNow;
context.SubmitChanges();
//5. Update Blob Metadata
blob.Attributes.Metadata["State"] = CollectionsTaskState.Processed.ToString();
//6. Delete Queue
StorageClientHelper.DeleteQueueMessage(QueueType.Processing, SourceType.Twitter, queueMessage);
//7. Print Success
string output = CollectionsTaskHelper.DatabaseTaskString(newTask);
Console.WriteLine(output);
}
}
}
}
