| public AddMessage ( Microsoft.WindowsAzure.StorageClient.CloudQueueMessage message ) : void | ||
| message | Microsoft.WindowsAzure.StorageClient.CloudQueueMessage | The message to add. |
| return | void | |
public override bool OnStart()
{
// Set the maximum number of concurrent connections
ServicePointManager.DefaultConnectionLimit = 100;
//Initialize Indexer
storageAccount = CloudStorageAccount.Parse(RoleEnvironment.GetConfigurationSettingValue("CrawlerStorage"));
//Initialize URL Queue
urlQueueClient = storageAccount.CreateCloudQueueClient();
urlQueue = urlQueueClient.GetQueueReference("urlqueue");
if (urlQueue.CreateIfNotExist())
{
//Add first URL to the queue
CloudQueueMessage firstURL = new CloudQueueMessage(startURL);
urlQueue.AddMessage(firstURL);
}
//Initialize Index Queue
indexQueueClient = storageAccount.CreateCloudQueueClient();
indexQueue = indexQueueClient.GetQueueReference("indexqueue");
indexQueue.CreateIfNotExist();
//Initialize Database Blob
databaseClient = storageAccount.CreateCloudBlobClient();
databaseContainer = databaseClient.GetContainerReference("wordfrequencies");
databaseContainer.CreateIfNotExist();
var permission = databaseContainer.GetPermissions();
permission.PublicAccess = BlobContainerPublicAccessType.Container;
databaseContainer.SetPermissions(permission);
return base.OnStart();
}
public BackoffExample(String queueName)
{
CloudStorageAccount cloudStorageAccount = CloudStorageAccount.Parse(ConfigurationManager.AppSettings["DataConnectionString"]);
CloudQueueClient cloudQueueClient = cloudStorageAccount.CreateCloudQueueClient();
cloudQueue = cloudQueueClient.GetQueueReference(queueName);
cloudQueue.CreateIfNotExist();
CloudQueueMessage cloudQueueMessage = new CloudQueueMessage("Some message");
cloudQueue.AddMessage(cloudQueueMessage);
}
//----------------------------------------------------------------------
// ToDo.CountMutualFriends
//
// Input : requestQueue - queue to sent messages to worker roles
// responseQueue - queue to receive messages from worker roles
// Return : A dictionary which contains the number of mutual friends for each user
// Purpose : Count the number of mutual friends for each user
// Note : You should implement the case of arriving messages from worker roles via responseQueue.
// The messages have numbers of mutual friends for each user
// in a way you describe in ToDo.CountMutualFriendsInEachWorkerRole.
// You should check whether there is a message of the counts in the responseQueue,
// and aggregate the counts from worker roles.
//
// * Data type of dictionary which will be returned is Dictionary<int, int>.
// Key (first generic type parameter) of this dictionary indicates user ID.
// Value (second generic type parameter) of this dictionary indicates
// the number of mutual friends of the user.
//----------------------------------------------------------------------
public static Dictionary<int, int> CountMutualFriends(CloudQueue requestQueue, CloudQueue responseQueue)
{
Dictionary<int, int> countMutualFriends = new Dictionary<int, int>();
// request each worker role to count mutual friends it found
for (int i = 0; i < ToDo.NumWorkerRoleInstances; i++)
requestQueue.AddMessage(new CloudQueueMessage("COUNT_MUTUAL_FRIENDS_#" + i));
// wait for finishing all of worker roles
int numFinishedRoles = 0;
while (numFinishedRoles < ToDo.NumWorkerRoleInstances)
{
// retrieve a new message from the queue
CloudQueueMessage msg = responseQueue.GetMessage();
if (msg == null)
{
System.Threading.Thread.Sleep(sleepTime);
continue;
}
// remove message from queue
responseQueue.DeleteMessage(msg);
string msgStr = msg.AsString;
if (msgStr.StartsWith("COMPLETE_COUNTING_MUTUAL_FRIENDS"))
{
numFinishedRoles++;
}
else
{
// YOU MUST IMPLEMENT ADDITIONAL CODE HERE
//
// * To receive the number of friends of users from worker roles, using response queue,
//
// check whether there is a message for the counts in the response queue,
// and aggregate the counts from every worker role.
String[] split = msgStr.Split(new Char[] { ':' });
int key = Convert.ToInt32(split[0]);
int value = Convert.ToInt32(split[1]);
if (countMutualFriends.ContainsKey(key))
countMutualFriends[key] += value;
else
countMutualFriends.Add(key, value);
}
}
return countMutualFriends;
}
public override bool OnStart()
{
// Set the maximum number of concurrent connections
ServicePointManager.DefaultConnectionLimit = 12;
//Initialize Storage
storageAccount = CloudStorageAccount.Parse(RoleEnvironment.GetConfigurationSettingValue("CrawlerStorage"));
//Create Scheduler Queue
queueClient = storageAccount.CreateCloudQueueClient();
queue = queueClient.GetQueueReference("urlschedulerqueue");
queue.CreateIfNotExist();
//Add first URL to the queue
CloudQueueMessage firstURL = new CloudQueueMessage(startURL);
queue.AddMessage(firstURL);
return base.OnStart();
}
public string ComputeICD9MapPlotData(string ICD9Code, DateTime startTime, DateTime endTime)
{
#if DEBUG
account = CloudStorageAccount.DevelopmentStorageAccount;
#else
account = new CloudStorageAccount(accountAndKey, true);
#endif
//client = new CloudQueueClient(account.BlobEndpoint.ToString(), account.Credentials);
qclient = account.CreateCloudQueueClient();
q = qclient.GetQueueReference("icd9mapplotrequests");
q.CreateIfNotExist();
string OperationID = Guid.NewGuid().ToString();
string starttime = startTime.Ticks.ToString();
string endtime = endTime.Ticks.ToString();
q.AddMessage(new CloudQueueMessage(string.Format("{0}|{1}|{2}|{3}",ICD9Code,starttime,endtime,OperationID)));
return OperationID;
}
public JsonModels.UploadReponse UploadPictureElement(int projectId, Stream pictureStream, string fileName, bool isCoverPicture = false)
{
try
{
BlobStorageAccessor blobStorageAccessor = new BlobStorageAccessor();
UploadManager uploadManager = new UploadManager();
ProjectAccessor projectAccessor = new ProjectAccessor();
//initiate queue message
storageAccount = CloudStorageAccount.Parse(RoleEnvironment.GetConfigurationSettingValue("BlobConnectionString"));
queueClient = storageAccount.CreateCloudQueueClient();
queue = queueClient.GetQueueReference(messageQueueName);
queue.CreateIfNotExist();
string imageURI = blobStorageAccessor.uploadImage(pictureStream, false).ToString();
Project p = pa.GetProject(projectId);
if (isCoverPicture)
{
string FileNameThumb1 = Guid.NewGuid().ToString();
string artifactURL1 = string.Format("{0}{1}", FileNameThumb1, ".jpeg");
CloudQueueMessage message3 = new CloudQueueMessage(String.Format("{0},{1},{2},{3},{4},{5},{6},{7}", imageURI, p.id, "thumbnail", "ProjectPicture", 266, 266, "", artifactURL1));
queue.AddMessage(message3);
p.coverPictureThumbnail = RoleEnvironment.GetConfigurationSettingValue("storageAccountUrl").ToString() + "thumbnails/" + artifactURL1;
p.coverPicture = imageURI;
p.dateModified = DateTime.Now;
Project newP = projectAccessor.UpdateProject(p);
return new JsonModels.UploadReponse { id = p.id, fileURL = imageURI, name = fileName, galeriaURL = "noGalleryURL", artifactURL = artifactURL1, description = "default description" };
}
else
{
string FileNameThumb = Guid.NewGuid().ToString();
string artifactURL = string.Format("{0}{1}", FileNameThumb, ".jpeg");
ProjectElement_Picture pe = new ProjectElement_Picture
{
title = GetTitle(fileName),
pictureLocation = imageURI,
pictureThumbnailLocation = RoleEnvironment.GetConfigurationSettingValue("storageAccountUrl").ToString()+"thumbnails/" + artifactURL
};
int projectElementId = pa.AddProjectElement(p, pe);
if (projectElementId == -1)
{
logAccessor.CreateLog(DateTime.Now, "ProjectManager - UploadPictureElement", "problem saving project element - 165 ProjectManager");
return null;
}
//string FileNameGaleria = Guid.NewGuid().ToString();
//string galleryURL = string.Format("{0}{1}", FileNameGaleria, ".jpeg");
CloudQueueMessage message = new CloudQueueMessage(String.Format("{0},{1},{2},{3},{4},{5},{6},{7}", imageURI, projectElementId, "thumbnail", "PictureElement", 635, 397, "", artifactURL));
//CloudQueueMessage message2 = new CloudQueueMessage(String.Format("{0},{1},{2},{3},{4},{5},{6},{7}", imageURI, projectElementId, "thumbnail", "PictureElement_Galleria", 1000, 750, "", galleryURL));
queue.AddMessage(message);
//queue.AddMessage(message2);
return new JsonModels.UploadReponse { id = projectElementId, fileURL = imageURI, name = fileName, galeriaURL = "galleryURL", artifactURL = artifactURL, description = null };
}
}
catch (Exception ex)
{
logAccessor.CreateLog(DateTime.Now, "ProjectManager - UploadPictureElement", ex.StackTrace);
return null;
}
}
public JsonModels.UploadReponse AddDocumentElement(int projectId, string description, Stream fileStream, string fileName, string userName)
{
Project p = pa.GetProject(projectId);
BlobStorageAccessor ba = new BlobStorageAccessor();
string location = null;
string[] s2 = fileName.Split('.');
string extention = s2[s2.Count() - 1].ToLower();
UserAccessor ua = new UserAccessor();
User u = ua.GetUser(userName);
string fullName = u.firstName +" " + u.lastName;
storageAccount = CloudStorageAccount.Parse(RoleEnvironment.GetConfigurationSettingValue("BlobConnectionString"));
queueClient = storageAccount.CreateCloudQueueClient();
queue = queueClient.GetQueueReference(messageQueueName);
queue.CreateIfNotExist();
string documentText = "";
//string PDFLocation = "notset";
if (extention == "pdf")
{
location = ba.uploadPDF(fileStream, false).ToString();
UploadManager um = new UploadManager();
//location = um.stampThatShit(location, fullName, null);
//documentText = um.ExtractText(location);
}
else if (extention == "doc" || extention == "docx")
{
location = ba.uploadDOC(fileStream, false, "."+extention).ToString();
}
else if (extention == "ppt" || extention == "pptx")
{
location = ba.uploadUnknown(fileStream, false, extention).ToString();
}
else if( extention == "xls" || extention == "xlsx")
{
location = ba.uploadUnknown(fileStream,false, extention).ToString();
}
else if (extention == "rtf")
{
location = ba.uploadUnknown(fileStream, false, extention).ToString();
}
else if (extention == "txt")
{
location = ba.uploadUnknown(fileStream, false, extention).ToString();
}
else
{
//upload unknown file type. do this or display message
Uri result = ba.uploadUnknown(fileStream, false, extention);
if (result == null)
{
return null;
}
else
{
location = result.ToString();
}
}
String FileName = Guid.NewGuid().ToString();
string uniqueBlobName = string.Format("{0}{1}", FileName, ".pdf");
string docThumb = "";
if (extention == "pdf")
{
docThumb = location;
}
else
{
docThumb = RoleEnvironment.GetConfigurationSettingValue("storageAccountUrl").ToString()+"pdfs/" + uniqueBlobName;
}
ProjectElement_Document pe = new ProjectElement_Document
{
description = description,
documentLocation = location,
title = fileName,
documentText = documentText,
documentThumbnailLocation = docThumb
};
int projectElementId = pa.AddProjectElement(p, pe);
if (extention == "doc" || extention == "docx")
{
CloudQueueMessage message = new CloudQueueMessage(String.Format("{0},{1},{2},{3},{4},{5},{6},{7}", location, projectElementId, "documentConversion", @"http://do.convertapi.com/Word2Pdf", 0, 0, fullName, uniqueBlobName));
queue.AddMessage(message);
}
else if (extention == "ppt" || extention == "pptx")
{
CloudQueueMessage message = new CloudQueueMessage(String.Format("{0},{1},{2},{3},{4},{5},{6},{7}", location, projectElementId, "documentConversion", @"http://do.convertapi.com/PowerPoint2Pdf", 0, 0, "", uniqueBlobName));
queue.AddMessage(message);
}
else if( extention == "xls" || extention == "xlsx")
{
CloudQueueMessage message = new CloudQueueMessage(String.Format("{0},{1},{2},{3},{4},{5},{6},{7}", location, projectElementId, "documentConversion", @"http://do.convertapi.com/Excel2Pdf", 0, 0, fullName, uniqueBlobName));
queue.AddMessage(message);
}
else if (extention == "rtf")
{
CloudQueueMessage message = new CloudQueueMessage(String.Format("{0},{1},{2},{3},{4},{5},{6},{7}", location, projectElementId, "documentConversion", @"http://do.convertapi.com/RichText2Pdf", 0, 0, fullName, uniqueBlobName));
queue.AddMessage(message);
}
else if (extention == "txt")
{
CloudQueueMessage message = new CloudQueueMessage(String.Format("{0},{1},{2},{3},{4},{5},{6},{7}", location, projectElementId, "documentConversion", @"http://do.convertapi.com/Text2Pdf", 0, 0, fullName, uniqueBlobName));
queue.AddMessage(message);
}
if (extention == "pdf")
{
return new JsonModels.UploadReponse { id = projectElementId, fileURL = location, name = fileName, artifactURL = location };
}
return new JsonModels.UploadReponse { id = projectElementId, fileURL = location, artifactURL = docThumb, name = fileName };
}
private void ProcessMessage(CloudQueue inboundQueue, CloudQueue outboundQueue, CloudQueueMessage message)
{
if (null == message) throw new ArgumentNullException("message");
if (null == inboundQueue) throw new ArgumentNullException("inboundQueue");
if (null == outboundQueue) throw new ArgumentNullException("outboundQueue");
ProcessingTask task = ProcessingTask.FromString(message.AsString);
if (null == task)
{
throw new NullReferenceException("Task is null");
}
if (!string.IsNullOrWhiteSpace(task.InputFilename))
{
DownloadBlob(this.InboundBlobContainer, task.InputFilename, this.WorkingDirectory);
}
Tracer.WriteLine(string.Format("Starting task '{0}'", task), "Information");
task.ProcessingTime = DateTime.UtcNow;
Run(this.WorkingDirectory, this.Command, task.InputFilename);
task.ProcessingDuration = (int)(DateTime.UtcNow - task.ProcessingTime).TotalSeconds;
if (!string.IsNullOrWhiteSpace(task.OutputFilename))
{
UploadBlob(this.OutboundBlobContainer, task.OutputFilename, this.WorkingDirectory);
}
Tracer.WriteLine(string.Format("Finished task '{0}'", task), "Information");
outboundQueue.AddMessage(new CloudQueueMessage(task.ToString()));
inboundQueue.DeleteMessage(message);
Tracer.WriteLine(string.Format("Clearing up task {0}", task), "Information");
if (!string.IsNullOrWhiteSpace(task.InputFilename))
{
this.DeleteFile(task.InputFilename);
}
if (!string.IsNullOrWhiteSpace(task.OutputFilename))
{
this.DeleteFile(task.OutputFilename);
}
Tracer.WriteLine(string.Format("Task complete '{0}'", task), "Information");
}
//----------------------------------------------------------------------
// ToDo.FindMutualFriends
//
// Input : requestQueue - queue data structure to send messages to worker roles
// responseQueue - queue data structure to receive messages from worker roles
// Return : Number of mutual friend pairs
// (summation of the number of mutual friend pairs found from worker roles)
// Purpose : Find every mutual friend pair, that means, do a self-join on follower table.
// Note :
//----------------------------------------------------------------------
public static int FindMutualFriends(CloudQueue requestQueue, CloudQueue responseQueue)
{
int numMutualFriends = 0;
// send a request to each worker role for finding mutual friends
for (int i = 0; i < ToDo.NumWorkerRoleInstances; i++)
requestQueue.AddMessage(new CloudQueueMessage("FIND_MUTUAL_FRIENDS_#" + i));
// wait for finishing every worker role
int numFinishedInstances = 0;
while (numFinishedInstances < ToDo.NumWorkerRoleInstances)
{
// retrieve a new response message from queue
CloudQueueMessage msg = responseQueue.GetMessage();
if (msg == null)
{
System.Threading.Thread.Sleep(sleepTime);
continue;
}
// remove the response message from queue
responseQueue.DeleteMessage(msg);
string msgStr = msg.AsString;
if (msgStr.StartsWith("COMPLETE_FINDING_MUTUAL_FRIENDS "))
{
// a worker role instance finished finding mutual friends
numFinishedInstances++;
numMutualFriends += int.Parse(msgStr.Substring(msgStr.LastIndexOf(" ") + 1));
}
else
{
System.Diagnostics.Trace.TraceError("Unexpected message!!! (" + msgStr + ")");
}
}
return numMutualFriends/2;
}
//----------------------------------------------------------------------
// ToDo.CountMutualFriendsInEachWorkerRole
//
// Input : mutualFriends - mutual friend pairs found by ToDo.FindMutualFriendsInEachWorkerRole
// responseQueue - queue to send count data and a complete message to web role
// Purpose : Count the number of mutual friends for each user within a worker role
// and send it to the web role
// Note :
//
// * To send the number of mutual friends for each user, using response queue,
// you can use any kind of data format to represent the count information, such as,
//
// responseQueue.AddMessage(new CloudQueueMessage(userID + ":" + numMutualFriends));
//----------------------------------------------------------------------
public static void CountMutualFriendsInEachWorkerRole(List<MutualFriend> mutualFriends, CloudQueue responseQueue)
{
Dictionary<int, int> d = new Dictionary<int, int>();
foreach (MutualFriend mf in mutualFriends)
{
if (d.ContainsKey(mf.User1))
d[mf.User1] += 1;
else
d.Add(mf.User1, 1);
if (d.ContainsKey(mf.User2))
d[mf.User2] += 1;
else
d.Add(mf.User2, 1);
}
// Send information about counting.
foreach (var e in d)
{
responseQueue.AddMessage(new CloudQueueMessage(e.Key + ":" + (e.Value/2)));
}
}
