public override void Run() { try { Trace.TraceInformation("PoP.ServiceTier entry point called"); var queueValetKeyUrl = ConfigurationManager.AppSettings["MediaIngestionQueueValetKeyUrl"]; var queueValet = new QueueValet(queueValetKeyUrl); var blobValetKeyUrl = ConfigurationManager.AppSettings["MediaStorageValetKeyUrl"]; var blobValet = new BlobValet(blobValetKeyUrl); InitStorage(queueValet, blobValet); while (true) { Trace.TraceInformation("Working on the next iteration from PoP.ServiceTier.Run"); NewMediaProcessor.ProcessNextMediaMessage(queueValet, blobValet); Thread.Sleep(TimeSpan.FromSeconds(2)); } } catch (Exception) { Trace.TraceError("PoP.ServiceTier.Run has detected an uncaught exception was thrown - this should never happen", "Error"); throw; } }
private static void InitStorage(QueueValet queueValet, BlobValet blobValet) { queueValet.CreateQueue("media-ingestion"); blobValet.CreateBlobContainer("popmedia", true); var container = blobValet.CreateBlobContainer("popuploads", false); string sasPolicy = BlobValet.CreateBlobSASPolicy( container, "untrusted-uploader", DateTime.UtcNow.AddYears(10), new SharedAccessBlobPermissions[] { SharedAccessBlobPermissions.Write } ); }
public static void ProcessNextMediaMessage(QueueValet queueValet, BlobValet blobValet) { try { var msg = queueValet.GetMessage(TimeSpan.FromMinutes(5)); if (msg != null) { if (msg.DequeueCount <= 1) { Trace.TraceInformation("DequeueCount = {0}", msg.DequeueCount); } else { Trace.TraceWarning("DequeueCount = {0}", msg.DequeueCount); } //TODO: if (msg.DequeueCount > 5) return; // Is it a photo or video? -- only photo supported for now... so assume that // (alternatively, might have a photo queue and a video queue) var mediaUploadInfo = ByteArraySerializer <MediaUploadModel> .Deserialize(msg.AsBytes); var uploadedPhotoUrl = mediaUploadInfo.BlobUrl; var username = mediaUploadInfo.Username; ProcessNextPhotoUpload(username, uploadedPhotoUrl, blobValet); queueValet.DeleteMessage(msg); // TODO: Ensure proper Poison Message Handling } } catch (Exception ex) { // this method CANNOT throw an exception - that's bad manners - even if there is a failure // SEE: Queue-Centric Workflow Pattern (chapter 3) plus Poison Message and Reliable Queue concepts // SEE: Strong Exception Guarantee http://en.wikipedia.org/wiki/Exception_safety // or relate to the NoFail Guaranteed from http://c2.com/cgi/wiki?ExceptionGuarantee var debugMsg = String.Format("Exception in PoP.ServiceTier.NewMediaProcessor.ProcessNextMediaMessage [{0}]\n[{1}]", ex.GetBaseException(), ex); Trace.TraceError(debugMsg); } }
/// <summary> /// Upload the blob and then (if nothing went wrong) drop a message on the queue announcing the blob /// </summary> /// <param name="queueValet"></param> /// <param name="mediaByteStream">Might be from File Upload via web page</param> /// <param name="origFilename"></param> /// <param name="mimeType"></param> /// <param name="byteCount">Count of bytes in the stream. Not used at this time. May be used in future to optimize the upload to blob storage, for telemetry, or to block uploads over a certain size.</param> /// <param name="destinationUrl"></param> /// <param name="blobValet"></param> public static void CaptureUploadedMedia(BlobValet blobValet, QueueValet queueValet, Stream mediaByteStream, string origFilename, string mimeType, int byteCount, string destinationUrl) { try { // TODO: obviate MediaStorageUrlFile.ExtTemplate by basing on MediaStorageValetKeyUrl value --- value="http://127.0.0.1:10000/devstoreaccount1/popmedia/{0}{1}" & "http://127.0.0.1:10000/devstoreaccount1/popmedia?sr=c&si=open-wide-container-access-policy&sig=X0yGw1Ydmu%2BCwk%2FTY7nj5HFgzv%2BIYg%2Bun%2BHQhNMmThk%3D" #if false var destinationUrl = String.Format(ConfigurationManager.AppSettings["MediaStorageUrlFile.ExtTemplate"], Guid.NewGuid(), new FileInfo(origFilename).Extension ); var valetKeyUrl = ConfigurationManager.AppSettings["MediaStorageValetKeyUrl"]; // if that worked, notify via queue var mediaIngestionQueueValetKeyUrl = ConfigurationManager.AppSettings["MediaIngestionQueueValetKeyUrl"]; #endif blobValet.UploadStream(new Uri(destinationUrl), mediaByteStream, mimeType); // TODO: at moment is sync (not async) to avoid race condition mentioned below var info = new MediaUploadModel { BlobUrl = destinationUrl, Username = "******" }; // prep an arbitrary object to send on the queue, not just a string (not rich enough for our use case) var queueMessage = new CloudQueueMessage(ByteArraySerializer <MediaUploadModel> .Serialize(info)); // TODO: race condition when both uploading a BLOB and posting the Queue message - the queue message processing // TODO: ... can begin before the blob upload is complete -- need to sync these // TODO: ... BUT! for now it will still FUNCTION CORRECTLY (if inefficiently) due to Queue-Centric Workflow Pattern retries IF not determined to be a Poison Message // There is no real need for a 50ms delay before the message can appear in queue, but just showing how to do it. // Technique is sometimes useful when there's a reason to delay its processing. You could use it to implement a // scheduler, for example. In the case of PoP, there are no obvious use cases. A made-up use case might be if PoP // introduced a way to make photos show up in the future allowing the user uploading them to indicate PoP should // delay processing for, say, up to 24 hours, and let the user optionally specify a delay within that range. queueValet.AddMessage(queueMessage, initialVisibilityDelay: TimeSpan.FromMilliseconds(50)); } catch (StorageException ex) { System.Diagnostics.Trace.TraceError("Exception thrown in BlobExtensions.UploadFile: " + ex); throw; } }
public ActionResult Upload(IEnumerable <HttpPostedFileBase> files) { var fileList = String.Empty; var firstFile = true; foreach (var file in files.Where(file => file != null && file.ContentLength > 0)) { Contract.Assert(file.FileName == Path.GetFileName(file.FileName)); // browsers should not send path info - but synthetic test could var fileExtension = Path.GetExtension(file.FileName); if (!String.IsNullOrWhiteSpace(fileExtension)) { fileExtension = fileExtension.ToLower(); } var destinationUrl = String.Format(ConfigurationManager.AppSettings["MediaStorageUrlFile.ExtTemplate"], Guid.NewGuid(), fileExtension); var blobValetKeyUrl = ConfigurationManager.AppSettings["MediaStorageValetKeyUrl"]; var queueValetKeyUrl = ConfigurationManager.AppSettings["MediaIngestionQueueValetKeyUrl"]; var blobValet = new BlobValet(blobValetKeyUrl); var queueValet = new QueueValet(queueValetKeyUrl); MediaIngester.CaptureUploadedMedia(blobValet, queueValet, file.InputStream, file.FileName, file.ContentType, file.ContentLength, destinationUrl); if (!firstFile) { fileList += ", "; } fileList += file.FileName; firstFile = false; } if (String.IsNullOrWhiteSpace(fileList)) { return(RedirectToAction("Upload")); } else { var message = String.Format("Upload successful for: {0}", fileList); return(RedirectToAction("Upload", new { msg = message })); } }