public AddMessage ( Microsoft.WindowsAzure.StorageClient.CloudQueueMessage message ) : void | ||
message | Microsoft.WindowsAzure.StorageClient.CloudQueueMessage | The message to add. |
return | void |
public override bool OnStart() { // Set the maximum number of concurrent connections ServicePointManager.DefaultConnectionLimit = 100; //Initialize Indexer storageAccount = CloudStorageAccount.Parse(RoleEnvironment.GetConfigurationSettingValue("CrawlerStorage")); //Initialize URL Queue urlQueueClient = storageAccount.CreateCloudQueueClient(); urlQueue = urlQueueClient.GetQueueReference("urlqueue"); if (urlQueue.CreateIfNotExist()) { //Add first URL to the queue CloudQueueMessage firstURL = new CloudQueueMessage(startURL); urlQueue.AddMessage(firstURL); } //Initialize Index Queue indexQueueClient = storageAccount.CreateCloudQueueClient(); indexQueue = indexQueueClient.GetQueueReference("indexqueue"); indexQueue.CreateIfNotExist(); //Initialize Database Blob databaseClient = storageAccount.CreateCloudBlobClient(); databaseContainer = databaseClient.GetContainerReference("wordfrequencies"); databaseContainer.CreateIfNotExist(); var permission = databaseContainer.GetPermissions(); permission.PublicAccess = BlobContainerPublicAccessType.Container; databaseContainer.SetPermissions(permission); return base.OnStart(); }
public BackoffExample(String queueName) { CloudStorageAccount cloudStorageAccount = CloudStorageAccount.Parse(ConfigurationManager.AppSettings["DataConnectionString"]); CloudQueueClient cloudQueueClient = cloudStorageAccount.CreateCloudQueueClient(); cloudQueue = cloudQueueClient.GetQueueReference(queueName); cloudQueue.CreateIfNotExist(); CloudQueueMessage cloudQueueMessage = new CloudQueueMessage("Some message"); cloudQueue.AddMessage(cloudQueueMessage); }
//---------------------------------------------------------------------- // ToDo.CountMutualFriends // // Input : requestQueue - queue to sent messages to worker roles // responseQueue - queue to receive messages from worker roles // Return : A dictionary which contains the number of mutual friends for each user // Purpose : Count the number of mutual friends for each user // Note : You should implement the case of arriving messages from worker roles via responseQueue. // The messages have numbers of mutual friends for each user // in a way you describe in ToDo.CountMutualFriendsInEachWorkerRole. // You should check whether there is a message of the counts in the responseQueue, // and aggregate the counts from worker roles. // // * Data type of dictionary which will be returned is Dictionary<int, int>. // Key (first generic type parameter) of this dictionary indicates user ID. // Value (second generic type parameter) of this dictionary indicates // the number of mutual friends of the user. //---------------------------------------------------------------------- public static Dictionary<int, int> CountMutualFriends(CloudQueue requestQueue, CloudQueue responseQueue) { Dictionary<int, int> countMutualFriends = new Dictionary<int, int>(); // request each worker role to count mutual friends it found for (int i = 0; i < ToDo.NumWorkerRoleInstances; i++) requestQueue.AddMessage(new CloudQueueMessage("COUNT_MUTUAL_FRIENDS_#" + i)); // wait for finishing all of worker roles int numFinishedRoles = 0; while (numFinishedRoles < ToDo.NumWorkerRoleInstances) { // retrieve a new message from the queue CloudQueueMessage msg = responseQueue.GetMessage(); if (msg == null) { System.Threading.Thread.Sleep(sleepTime); continue; } // remove message from queue responseQueue.DeleteMessage(msg); string msgStr = msg.AsString; if (msgStr.StartsWith("COMPLETE_COUNTING_MUTUAL_FRIENDS")) { numFinishedRoles++; } else { // YOU MUST IMPLEMENT ADDITIONAL CODE HERE // // * To receive the number of friends of users from worker roles, using response queue, // // check whether there is a message for the counts in the response queue, // and aggregate the counts from every worker role. String[] split = msgStr.Split(new Char[] { ':' }); int key = Convert.ToInt32(split[0]); int value = Convert.ToInt32(split[1]); if (countMutualFriends.ContainsKey(key)) countMutualFriends[key] += value; else countMutualFriends.Add(key, value); } } return countMutualFriends; }
public override bool OnStart() { // Set the maximum number of concurrent connections ServicePointManager.DefaultConnectionLimit = 12; //Initialize Storage storageAccount = CloudStorageAccount.Parse(RoleEnvironment.GetConfigurationSettingValue("CrawlerStorage")); //Create Scheduler Queue queueClient = storageAccount.CreateCloudQueueClient(); queue = queueClient.GetQueueReference("urlschedulerqueue"); queue.CreateIfNotExist(); //Add first URL to the queue CloudQueueMessage firstURL = new CloudQueueMessage(startURL); queue.AddMessage(firstURL); return base.OnStart(); }
public string ComputeICD9MapPlotData(string ICD9Code, DateTime startTime, DateTime endTime) { #if DEBUG account = CloudStorageAccount.DevelopmentStorageAccount; #else account = new CloudStorageAccount(accountAndKey, true); #endif //client = new CloudQueueClient(account.BlobEndpoint.ToString(), account.Credentials); qclient = account.CreateCloudQueueClient(); q = qclient.GetQueueReference("icd9mapplotrequests"); q.CreateIfNotExist(); string OperationID = Guid.NewGuid().ToString(); string starttime = startTime.Ticks.ToString(); string endtime = endTime.Ticks.ToString(); q.AddMessage(new CloudQueueMessage(string.Format("{0}|{1}|{2}|{3}",ICD9Code,starttime,endtime,OperationID))); return OperationID; }
public JsonModels.UploadReponse UploadPictureElement(int projectId, Stream pictureStream, string fileName, bool isCoverPicture = false) { try { BlobStorageAccessor blobStorageAccessor = new BlobStorageAccessor(); UploadManager uploadManager = new UploadManager(); ProjectAccessor projectAccessor = new ProjectAccessor(); //initiate queue message storageAccount = CloudStorageAccount.Parse(RoleEnvironment.GetConfigurationSettingValue("BlobConnectionString")); queueClient = storageAccount.CreateCloudQueueClient(); queue = queueClient.GetQueueReference(messageQueueName); queue.CreateIfNotExist(); string imageURI = blobStorageAccessor.uploadImage(pictureStream, false).ToString(); Project p = pa.GetProject(projectId); if (isCoverPicture) { string FileNameThumb1 = Guid.NewGuid().ToString(); string artifactURL1 = string.Format("{0}{1}", FileNameThumb1, ".jpeg"); CloudQueueMessage message3 = new CloudQueueMessage(String.Format("{0},{1},{2},{3},{4},{5},{6},{7}", imageURI, p.id, "thumbnail", "ProjectPicture", 266, 266, "", artifactURL1)); queue.AddMessage(message3); p.coverPictureThumbnail = RoleEnvironment.GetConfigurationSettingValue("storageAccountUrl").ToString() + "thumbnails/" + artifactURL1; p.coverPicture = imageURI; p.dateModified = DateTime.Now; Project newP = projectAccessor.UpdateProject(p); return new JsonModels.UploadReponse { id = p.id, fileURL = imageURI, name = fileName, galeriaURL = "noGalleryURL", artifactURL = artifactURL1, description = "default description" }; } else { string FileNameThumb = Guid.NewGuid().ToString(); string artifactURL = string.Format("{0}{1}", FileNameThumb, ".jpeg"); ProjectElement_Picture pe = new ProjectElement_Picture { title = GetTitle(fileName), pictureLocation = imageURI, pictureThumbnailLocation = RoleEnvironment.GetConfigurationSettingValue("storageAccountUrl").ToString()+"thumbnails/" + artifactURL }; int projectElementId = pa.AddProjectElement(p, pe); if (projectElementId == -1) { logAccessor.CreateLog(DateTime.Now, "ProjectManager - UploadPictureElement", "problem saving project element - 165 ProjectManager"); return null; } //string FileNameGaleria = Guid.NewGuid().ToString(); //string galleryURL = string.Format("{0}{1}", FileNameGaleria, ".jpeg"); CloudQueueMessage message = new CloudQueueMessage(String.Format("{0},{1},{2},{3},{4},{5},{6},{7}", imageURI, projectElementId, "thumbnail", "PictureElement", 635, 397, "", artifactURL)); //CloudQueueMessage message2 = new CloudQueueMessage(String.Format("{0},{1},{2},{3},{4},{5},{6},{7}", imageURI, projectElementId, "thumbnail", "PictureElement_Galleria", 1000, 750, "", galleryURL)); queue.AddMessage(message); //queue.AddMessage(message2); return new JsonModels.UploadReponse { id = projectElementId, fileURL = imageURI, name = fileName, galeriaURL = "galleryURL", artifactURL = artifactURL, description = null }; } } catch (Exception ex) { logAccessor.CreateLog(DateTime.Now, "ProjectManager - UploadPictureElement", ex.StackTrace); return null; } }
public JsonModels.UploadReponse AddDocumentElement(int projectId, string description, Stream fileStream, string fileName, string userName) { Project p = pa.GetProject(projectId); BlobStorageAccessor ba = new BlobStorageAccessor(); string location = null; string[] s2 = fileName.Split('.'); string extention = s2[s2.Count() - 1].ToLower(); UserAccessor ua = new UserAccessor(); User u = ua.GetUser(userName); string fullName = u.firstName +" " + u.lastName; storageAccount = CloudStorageAccount.Parse(RoleEnvironment.GetConfigurationSettingValue("BlobConnectionString")); queueClient = storageAccount.CreateCloudQueueClient(); queue = queueClient.GetQueueReference(messageQueueName); queue.CreateIfNotExist(); string documentText = ""; //string PDFLocation = "notset"; if (extention == "pdf") { location = ba.uploadPDF(fileStream, false).ToString(); UploadManager um = new UploadManager(); //location = um.stampThatShit(location, fullName, null); //documentText = um.ExtractText(location); } else if (extention == "doc" || extention == "docx") { location = ba.uploadDOC(fileStream, false, "."+extention).ToString(); } else if (extention == "ppt" || extention == "pptx") { location = ba.uploadUnknown(fileStream, false, extention).ToString(); } else if( extention == "xls" || extention == "xlsx") { location = ba.uploadUnknown(fileStream,false, extention).ToString(); } else if (extention == "rtf") { location = ba.uploadUnknown(fileStream, false, extention).ToString(); } else if (extention == "txt") { location = ba.uploadUnknown(fileStream, false, extention).ToString(); } else { //upload unknown file type. do this or display message Uri result = ba.uploadUnknown(fileStream, false, extention); if (result == null) { return null; } else { location = result.ToString(); } } String FileName = Guid.NewGuid().ToString(); string uniqueBlobName = string.Format("{0}{1}", FileName, ".pdf"); string docThumb = ""; if (extention == "pdf") { docThumb = location; } else { docThumb = RoleEnvironment.GetConfigurationSettingValue("storageAccountUrl").ToString()+"pdfs/" + uniqueBlobName; } ProjectElement_Document pe = new ProjectElement_Document { description = description, documentLocation = location, title = fileName, documentText = documentText, documentThumbnailLocation = docThumb }; int projectElementId = pa.AddProjectElement(p, pe); if (extention == "doc" || extention == "docx") { CloudQueueMessage message = new CloudQueueMessage(String.Format("{0},{1},{2},{3},{4},{5},{6},{7}", location, projectElementId, "documentConversion", @"http://do.convertapi.com/Word2Pdf", 0, 0, fullName, uniqueBlobName)); queue.AddMessage(message); } else if (extention == "ppt" || extention == "pptx") { CloudQueueMessage message = new CloudQueueMessage(String.Format("{0},{1},{2},{3},{4},{5},{6},{7}", location, projectElementId, "documentConversion", @"http://do.convertapi.com/PowerPoint2Pdf", 0, 0, "", uniqueBlobName)); queue.AddMessage(message); } else if( extention == "xls" || extention == "xlsx") { CloudQueueMessage message = new CloudQueueMessage(String.Format("{0},{1},{2},{3},{4},{5},{6},{7}", location, projectElementId, "documentConversion", @"http://do.convertapi.com/Excel2Pdf", 0, 0, fullName, uniqueBlobName)); queue.AddMessage(message); } else if (extention == "rtf") { CloudQueueMessage message = new CloudQueueMessage(String.Format("{0},{1},{2},{3},{4},{5},{6},{7}", location, projectElementId, "documentConversion", @"http://do.convertapi.com/RichText2Pdf", 0, 0, fullName, uniqueBlobName)); queue.AddMessage(message); } else if (extention == "txt") { CloudQueueMessage message = new CloudQueueMessage(String.Format("{0},{1},{2},{3},{4},{5},{6},{7}", location, projectElementId, "documentConversion", @"http://do.convertapi.com/Text2Pdf", 0, 0, fullName, uniqueBlobName)); queue.AddMessage(message); } if (extention == "pdf") { return new JsonModels.UploadReponse { id = projectElementId, fileURL = location, name = fileName, artifactURL = location }; } return new JsonModels.UploadReponse { id = projectElementId, fileURL = location, artifactURL = docThumb, name = fileName }; }
private void ProcessMessage(CloudQueue inboundQueue, CloudQueue outboundQueue, CloudQueueMessage message) { if (null == message) throw new ArgumentNullException("message"); if (null == inboundQueue) throw new ArgumentNullException("inboundQueue"); if (null == outboundQueue) throw new ArgumentNullException("outboundQueue"); ProcessingTask task = ProcessingTask.FromString(message.AsString); if (null == task) { throw new NullReferenceException("Task is null"); } if (!string.IsNullOrWhiteSpace(task.InputFilename)) { DownloadBlob(this.InboundBlobContainer, task.InputFilename, this.WorkingDirectory); } Tracer.WriteLine(string.Format("Starting task '{0}'", task), "Information"); task.ProcessingTime = DateTime.UtcNow; Run(this.WorkingDirectory, this.Command, task.InputFilename); task.ProcessingDuration = (int)(DateTime.UtcNow - task.ProcessingTime).TotalSeconds; if (!string.IsNullOrWhiteSpace(task.OutputFilename)) { UploadBlob(this.OutboundBlobContainer, task.OutputFilename, this.WorkingDirectory); } Tracer.WriteLine(string.Format("Finished task '{0}'", task), "Information"); outboundQueue.AddMessage(new CloudQueueMessage(task.ToString())); inboundQueue.DeleteMessage(message); Tracer.WriteLine(string.Format("Clearing up task {0}", task), "Information"); if (!string.IsNullOrWhiteSpace(task.InputFilename)) { this.DeleteFile(task.InputFilename); } if (!string.IsNullOrWhiteSpace(task.OutputFilename)) { this.DeleteFile(task.OutputFilename); } Tracer.WriteLine(string.Format("Task complete '{0}'", task), "Information"); }
//---------------------------------------------------------------------- // ToDo.FindMutualFriends // // Input : requestQueue - queue data structure to send messages to worker roles // responseQueue - queue data structure to receive messages from worker roles // Return : Number of mutual friend pairs // (summation of the number of mutual friend pairs found from worker roles) // Purpose : Find every mutual friend pair, that means, do a self-join on follower table. // Note : //---------------------------------------------------------------------- public static int FindMutualFriends(CloudQueue requestQueue, CloudQueue responseQueue) { int numMutualFriends = 0; // send a request to each worker role for finding mutual friends for (int i = 0; i < ToDo.NumWorkerRoleInstances; i++) requestQueue.AddMessage(new CloudQueueMessage("FIND_MUTUAL_FRIENDS_#" + i)); // wait for finishing every worker role int numFinishedInstances = 0; while (numFinishedInstances < ToDo.NumWorkerRoleInstances) { // retrieve a new response message from queue CloudQueueMessage msg = responseQueue.GetMessage(); if (msg == null) { System.Threading.Thread.Sleep(sleepTime); continue; } // remove the response message from queue responseQueue.DeleteMessage(msg); string msgStr = msg.AsString; if (msgStr.StartsWith("COMPLETE_FINDING_MUTUAL_FRIENDS ")) { // a worker role instance finished finding mutual friends numFinishedInstances++; numMutualFriends += int.Parse(msgStr.Substring(msgStr.LastIndexOf(" ") + 1)); } else { System.Diagnostics.Trace.TraceError("Unexpected message!!! (" + msgStr + ")"); } } return numMutualFriends/2; }
//---------------------------------------------------------------------- // ToDo.CountMutualFriendsInEachWorkerRole // // Input : mutualFriends - mutual friend pairs found by ToDo.FindMutualFriendsInEachWorkerRole // responseQueue - queue to send count data and a complete message to web role // Purpose : Count the number of mutual friends for each user within a worker role // and send it to the web role // Note : // // * To send the number of mutual friends for each user, using response queue, // you can use any kind of data format to represent the count information, such as, // // responseQueue.AddMessage(new CloudQueueMessage(userID + ":" + numMutualFriends)); //---------------------------------------------------------------------- public static void CountMutualFriendsInEachWorkerRole(List<MutualFriend> mutualFriends, CloudQueue responseQueue) { Dictionary<int, int> d = new Dictionary<int, int>(); foreach (MutualFriend mf in mutualFriends) { if (d.ContainsKey(mf.User1)) d[mf.User1] += 1; else d.Add(mf.User1, 1); if (d.ContainsKey(mf.User2)) d[mf.User2] += 1; else d.Add(mf.User2, 1); } // Send information about counting. foreach (var e in d) { responseQueue.AddMessage(new CloudQueueMessage(e.Key + ":" + (e.Value/2))); } }