/// <summary> /// The main entry point. /// </summary> /// <param name="args"> /// The command line arguments. Not used. /// </param> private static void Main(string[] args) { var credentials = new StorageCredentials("rwwa", "vyMfaSPxURHXZaIhhFJQRg5ZLEN6qDj4yU78r3oeOH+pZzdcf4S86QvGAsB6L8JaPti9qJbB929hy1Y9hipFmw=="); // Retrieve storage account from connection string. var storageAccount = new CloudStorageAccount(credentials, true); CloudQueueClient queueClient = storageAccount.CreateCloudQueueClient(); // Retrieve a reference to a queue CloudQueue queue = queueClient.GetQueueReference("dataqueue"); queue.CreateIfNotExists(); Console.WriteLine("Up and listening to the queue."); while (true) { CloudQueueMessage message; do { message = queue.GetMessage(); if (message != null) { Console.WriteLine("Received Message for BLOB " + message.AsString); var blobUrl = message.AsString; var blockBlob = new CloudBlockBlob(new Uri(blobUrl), credentials); if (blockBlob.Exists()) { // TODO: download and process BLOB /*using (var fileStream = System.IO.File.OpenWrite(@"path\myfile")) { blockBlob.DownloadToStream(fileStream); }*/ blockBlob.Delete(); } queue.DeleteMessage(message); Console.WriteLine("BLOB " + message.AsString + " and queue message deleted."); } } while (message != null); Thread.Sleep(TimeSpan.FromSeconds(5)); } }
/// <summary> /// The main entry point. /// </summary> /// <param name="args"> /// The command line arguments. Not used. /// </param> private static void Main(string[] args) { var credentials = new StorageCredentials("[Enter your account name]", "[Enter your account key]"); // Retrieve storage account from connection string. var storageAccount = new CloudStorageAccount(credentials, true); CloudQueueClient queueClient = storageAccount.CreateCloudQueueClient(); // Retrieve a reference to a queue CloudQueue queue = queueClient.GetQueueReference("dataqueue"); Console.WriteLine("Up and listening to the queue."); while (true) { CloudQueueMessage message; do { message = queue.GetMessage(); if (message != null) { Console.WriteLine("Received Message for BLOB " + message.AsString); var blobUrl = message.AsString; var blockBlob = new CloudBlockBlob(new Uri(blobUrl), credentials); if (blockBlob.Exists()) { // TODO: download and process BLOB /*using (var fileStream = System.IO.File.OpenWrite(@"path\myfile")) { blockBlob.DownloadToStream(fileStream); }*/ blockBlob.Delete(); } queue.DeleteMessage(message); Console.WriteLine("BLOB " + message.AsString + " and queue message deleted."); } } while (message != null); Thread.Sleep(TimeSpan.FromSeconds(5)); } }
public static async Task WaitForBlobAsync(CloudBlockBlob blob) { await TestHelpers.Await(() => { return blob.Exists(); }); }
/// <summary> /// Creates an archived copy of the original package blob if it doesn't already exist. /// </summary> private void ArchiveOriginalPackageBlob(CloudBlockBlob originalPackageBlob, CloudBlockBlob latestPackageBlob) { // Copy the blob to backup only if it isn't already successfully copied if ((!originalPackageBlob.Exists()) || (originalPackageBlob.CopyState != null && originalPackageBlob.CopyState.Status != CopyStatus.Success)) { if (!WhatIf) { Log.Info("Backing up blob: {0} to {1}", latestPackageBlob.Name, originalPackageBlob.Name); originalPackageBlob.StartCopyFromBlob(latestPackageBlob); CopyState state = originalPackageBlob.CopyState; for (int i = 0; (state == null || state.Status == CopyStatus.Pending) && i < SleepTimes.Length; i++) { Log.Info("(sleeping for a copy completion)"); Thread.Sleep(SleepTimes[i]); originalPackageBlob.FetchAttributes(); // To get a refreshed CopyState //refresh state state = originalPackageBlob.CopyState; } if (state.Status != CopyStatus.Success) { string msg = string.Format("Blob copy failed: CopyState={0}", state.StatusDescription); Log.Error("(error) " + msg); throw new BlobBackupFailedException(msg); } } } }
//Delete a blob file public void DeleteBlob(string UserId, string BlobName) { Microsoft.WindowsAzure.Storage.Blob.CloudBlobContainer container = GetCloudBlobContainer(UserId); Microsoft.WindowsAzure.Storage.Blob.CloudBlockBlob blockBlob = container.GetBlockBlobReference(BlobName); if (blockBlob.Exists()) { blockBlob.Delete(); } }
public static async Task<string> WaitForBlobAsync(CloudBlockBlob blob) { await TestHelpers.Await(() => { return blob.Exists(); }); string result = await blob.DownloadTextAsync(Encoding.UTF8, null, new BlobRequestOptions(), new Microsoft.WindowsAzure.Storage.OperationContext()); return result; }
private void DownloadIfExists(CloudBlockBlob blob) { string target = Path.Combine(Root, blob.Name); if (blob.Exists()) { if (!Directory.Exists(Root)) { Directory.CreateDirectory(Root); } using (var file = File.OpenWrite(target)) { blob.DownloadToStream(file); } } }
private static string SetUpInputString(CloudBlockBlob blockBlobReference) { string inputStr; if (blockBlobReference.Exists() == false) { inputStr = ""; } else { using (StreamReader blobReader = new StreamReader(blockBlobReference.OpenRead())) { inputStr = blobReader.ReadToEnd(); } } return inputStr; }
public void Run() { var storageAccount = CloudStorageAccount.Parse(_storageConnectionString); var blobClient = storageAccount.CreateCloudBlobClient(); var container = blobClient.GetContainerReference("mycontainer"); container.CreateIfNotExists(); _blob = container.GetBlockBlobReference("myblob.txt"); if (!_blob.Exists()) { using (var s = new MemoryStream()) { _blob.UploadFromStream(s); } } Task.Factory.StartNew(() => GenerateContentAsync()); Task.Factory.StartNew(() => AppendStreamToBlobAsync()); Task.WaitAll(); }
private DateTime GetLastModified() { Microsoft.WindowsAzure.Storage.CloudStorageAccount storageAccount = Microsoft.WindowsAzure.Storage.CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("CineStorageConStr")); Microsoft.WindowsAzure.Storage.Blob.CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); // Retrieve a reference to a container. Microsoft.WindowsAzure.Storage.Blob.CloudBlobContainer container = blobClient.GetContainerReference("data"); // Retrieve reference to a blob. Microsoft.WindowsAzure.Storage.Blob.CloudBlockBlob cinemasUKBlob = container.GetBlockBlobReference(CinemasUKFileName); if (cinemasUKBlob.Exists()) { DateTimeOffset?dto = cinemasUKBlob.Properties.LastModified; if (dto.HasValue) { return(dto.Value.DateTime); } } return(DateTime.MinValue); }
private void ArchiveOldReport(string reportPath, string reportName, string fileExt, CloudBlockBlob report) { if (!report.Exists()) return; report.FetchAttributes(); string archiveDirString = (archiveBlobPath + ( (archiveBlobPath.EndsWith("/") && reportPath.StartsWith("/")) ? reportPath.Substring(1) : reportPath )).ToLower(); if (!archiveDirString.EndsWith("/")) archiveDirString = archiveDirString + "/"; CloudBlobDirectory archiveDir = container.GetDirectoryReference(archiveDirString); //if the archive path exists, see if we need to delete the oldest blob in the path IEnumerable<IListBlobItem> blobs = archiveDir.ListBlobs(true); Dictionary<DateTimeOffset, string> oldReports = new Dictionary<DateTimeOffset, string>(); DateTimeOffset oldest = DateTimeOffset.MaxValue; ICloudBlob oldestBlob = null; int count = 0; foreach (ICloudBlob blob in blobs) { ++count; if (blob.Properties.LastModified.Value < oldest) { oldest = blob.Properties.LastModified.Value; oldestBlob = blob; } } if (count >= maxNumOfReportsArchived) oldestBlob.Delete(); //copy report to archive string newUri = (archiveDirString + reportName + "_" + report.Properties.LastModified.Value.ToString("u") + fileExt).ToLower(); CloudBlockBlob newReport = container.GetBlockBlobReference(newUri); newReport.StartCopyFromBlob(report); int retries = 3; while (retries > 0) { if (newReport.CopyState.Status == CopyStatus.Success) { Trace.TraceInformation("Blob archiving succeeded: " + newUri); break; } else if (newReport.CopyState.Status == CopyStatus.Aborted || newReport.CopyState.Status == CopyStatus.Failed || newReport.CopyState.Status == CopyStatus.Invalid) { Trace.TraceError("Blob archiving failed: " + newUri); break; } else { Thread.Sleep(1000); --retries; if (retries == 0) Trace.TraceError("Blob archiving timed out: " + newUri); } } }
private void CreateBlob() { var account = CloudStorageAccount.Parse(Config.Get("DeepStorage.OutputConnectionString")); var blobClient = account.CreateCloudBlobClient(); var container = blobClient.GetContainerReference(Config.Get("DeepStorage.OutputContainerName")); container.CreateIfNotExists(); _blob = container.GetBlockBlobReference(_blobPath); if (!_blob.Exists()) { _blob.UploadFromByteArray(new byte[] {},0, 0); } }
/// <summary> /// Uploads Memory Stream to Blob /// </summary> /// <param name="outputBlob"></param> /// <param name="line"></param> private static void UploadBlobStream(CloudBlockBlob outputBlob, string line) { using (MemoryStream ms = new MemoryStream()) { if (outputBlob.Exists()) { outputBlob.DownloadToStream(ms); } byte[] dataToWrite = Encoding.UTF8.GetBytes(line + "\r\n"); ms.Write(dataToWrite, 0, dataToWrite.Length); ms.Position = 0; outputBlob.UploadFromStream(ms); } }
/// <summary> /// Process sample Files for generating test data /// </summary> /// <param name="path"></param> /// <param name="outputStorageAccount"></param> /// <param name="folderPath"></param> /// <param name="outputTableName"></param> /// <param name="logger"></param> public static void ProcessFiles(string path, CloudStorageAccount outputStorageAccount, string folderPath, string outputTableName, IActivityLogger logger) { string[] files = Directory.GetFiles(path); foreach (var file in files) { if (file.Contains(@"artist_data.txt") && outputTableName.ToLower().Contains(@"catalog")) { Uri outputBlobUri = new Uri(outputStorageAccount.BlobEndpoint, folderPath + "artist_data.txt"); CloudBlockBlob outputBlob = new CloudBlockBlob(outputBlobUri, outputStorageAccount.Credentials); if (outputBlob.Exists()) { outputBlob.Delete(); } outputBlob.UploadFromFile(path + "/artist_data.txt", FileMode.Open); } if (file.Contains(@"customers.txt") && outputTableName.ToLower().Contains(@"customers")) { Uri outputBlobUri = new Uri(outputStorageAccount.BlobEndpoint, folderPath + "customers.txt"); CloudBlockBlob outputBlob = new CloudBlockBlob(outputBlobUri, outputStorageAccount.Credentials); if (outputBlob.Exists()) { outputBlob.Delete(); } outputBlob.UploadFromFile(path + "/customers.txt", FileMode.Open); } else if (file.Contains(@"artist_customer_data.txt") && outputTableName.ToLower().Contains(@"rawproducts")) { Uri outputBlobUri = new Uri(outputStorageAccount.BlobEndpoint, folderPath + "artist_customer_data.txt"); CloudBlockBlob outputBlob = new CloudBlockBlob(outputBlobUri, outputStorageAccount.Credentials); List<string> lines = File.ReadAllLines(path + "/artist_customer_data.txt").ToList(); int index = 0; if (outputBlob.Exists() && index == 0) { outputBlob.Delete(); } //add new column value for each row. lines.Skip(0).ToList().ForEach(line => { if (index >= 0 & index <= 1000) { lines[index] += "," + DateTime.UtcNow.AddMonths(-1).ToString("yyyy-MM-dd"); UploadBlobStream(outputBlob, lines[index]); index++; } else if (index >= 1001 & index <= 2000) { lines[index] += "," + DateTime.UtcNow.AddMonths(-2).ToString("yyyy-MM-dd"); UploadBlobStream(outputBlob, lines[index]); index++; } else if (index >= 2001 & index <= 3000) { lines[index] += "," + DateTime.UtcNow.AddMonths(-3).ToString("yyyy-MM-dd"); UploadBlobStream(outputBlob, lines[index]); index++; } else if (index >= 3001 & index <= 4000) { lines[index] += "," + DateTime.UtcNow.AddMonths(-4).ToString("yyyy-MM-dd"); UploadBlobStream(outputBlob, lines[index]); index++; } else if (index >= 4001 & index <= 5000) { lines[index] += "," + DateTime.UtcNow.AddMonths(-5).ToString("yyyy-MM-dd"); UploadBlobStream(outputBlob, lines[index]); index++; } else if (index >= 5001 & index <= 6000) { lines[index] += "," + DateTime.UtcNow.AddMonths(-6).ToString("yyyy-MM-dd"); UploadBlobStream(outputBlob, lines[index]); index++; } else if (index >= 6001 & index <= 7000) { lines[index] += "," + DateTime.UtcNow.AddMonths(-7).ToString("yyyy-MM-dd"); UploadBlobStream(outputBlob, lines[index]); index++; } else if (index >= 7001 & index <= 8000) { lines[index] += "," + DateTime.UtcNow.AddMonths(-8).ToString("yyyy-MM-dd"); UploadBlobStream(outputBlob, lines[index]); index++; } else if (index >= 8001 & index <= 9000) { lines[index] += "," + DateTime.UtcNow.AddMonths(-9).ToString("yyyy-MM-dd"); UploadBlobStream(outputBlob, lines[index]); index++; } else if (index >= 9001 & index <= 10000) { lines[index] += "," + DateTime.UtcNow.AddMonths(-10).ToString("yyyy-MM-dd"); UploadBlobStream(outputBlob, lines[index]); index++; } else if (index >= 10001 & index <= 11000) { lines[index] += "," + DateTime.UtcNow.AddMonths(-11).ToString("yyyy-MM-dd"); UploadBlobStream(outputBlob, lines[index]); index++; } else if (index >= 11001 & index <= 12000) { lines[index] += "," + DateTime.UtcNow.AddMonths(-12).ToString("yyyy-MM-dd"); UploadBlobStream(outputBlob, lines[index]); index++; } Console.WriteLine("Writing blob number: {0}", index); logger.Write(TraceEventType.Information, "Writing blob number: {0}", index); }); } } }
public void DownloadBlob_using_SharedAccessPolicies() { // In this unit test demo there is a timing aspect below where a policy expires. // If run in debug the test might fail var cloudBlockBlob = cloudBlobContainer.GetBlockBlobReference(BlobName); Assert.IsTrue(cloudBlockBlob.Exists(), "If the blob does not exist then execute the test above!"); const string storedPolicyKey = "demostoredpolicy"; // 1) Set the policy in Azure Storage on the container. // Make sure our policy does not already exist. var blobContainerPermissions = cloudBlobContainer.GetPermissions(); if (blobContainerPermissions.SharedAccessPolicies.ContainsKey(storedPolicyKey)) { blobContainerPermissions.SharedAccessPolicies.Remove(storedPolicyKey); } // Or use blobContainerPermissions.SharedAccessPolicies.Clear(); // Set our access policy in Azure to grant read access for a short time given the key. blobContainerPermissions.SharedAccessPolicies.Add( storedPolicyKey, new SharedAccessBlobPolicy { Permissions = SharedAccessBlobPermissions.Read, SharedAccessExpiryTime = DateTime.UtcNow.AddSeconds(5) }); cloudBlobContainer.SetPermissions(blobContainerPermissions); // 2) Use the policy only to // This is the information a server would send to a client: Blob Uri including the SAS key! var blobAbsoluteUri = new Uri(cloudBlockBlob.Uri.AbsoluteUri); var sharedAccessSignature = cloudBlockBlob.GetSharedAccessSignature(null, storedPolicyKey); var blobAccess = blobAbsoluteUri + sharedAccessSignature; // Use the named key to get access // This is typically done on a client that wants access using the policy they have been given var blobAccessUri = new Uri(blobAccess); var blobAccessBySAS = new CloudBlockBlob(blobAccessUri); // Here we use the access by being able to read that the blob does indeed exist Assert.IsTrue(blobAccessBySAS.Exists()); // Allow the policy to expire on the server Task.Delay(TimeSpan.FromSeconds(5)).Wait(); try { blobAccessBySAS.Exists(); Assert.Fail("The policy should have been expired!"); } catch (StorageException sex) { // We have no access to the blob on an expired policy. This is forbidden! const HttpStatusCode expectedStatusCode = HttpStatusCode.Forbidden; var returnedStatusCode = (HttpStatusCode)sex.RequestInformation.HttpStatusCode; Assert.AreEqual(expectedStatusCode, returnedStatusCode); } // Prolong the policy on the server blobContainerPermissions.SharedAccessPolicies[storedPolicyKey].SharedAccessExpiryTime = DateTime.UtcNow.AddSeconds(5); cloudBlobContainer.SetPermissions(blobContainerPermissions); // Access is again granted by the same stored policy as before! We did not need to send a new url to the client! Assert.IsTrue(blobAccessBySAS.Exists()); }
private void InitFormBlob(CloudBlockBlob blob, bool withFileData) { if (blob == null || !blob.Exists()) return; BlobKey = blob.Name; FileName = Uri.UnescapeDataString(blob.Metadata["FileName"]); ContentType = Uri.UnescapeDataString(blob.Metadata["ContentType"]); int contentLength; if (int.TryParse(Uri.UnescapeDataString(blob.Metadata["ContentLength"]), out contentLength)) ContentLength = contentLength; BlobFileType type; if (Enum.TryParse<BlobFileType>(Uri.UnescapeDataString(blob.Metadata["BlobFileType"]), true, out type)) BlobFileType = type; DateTime dtUploaded; if (DateTime.TryParse(Uri.UnescapeDataString(blob.Metadata["DtUploaded"]), out dtUploaded)) UploadedDate = dtUploaded; if (withFileData) { Data = new MemoryStream(); blob.DownloadToStream(Data); Data.Position = 0; } }
/// <summary> /// Adds the diagnostic message to block blob. /// </summary> /// <param name="blob">The cloud blob.</param> /// <param name="message">The message.</param> protected virtual void AddMessageToBlock(CloudBlockBlob blob, string message) { Sitecore.Diagnostics.Assert.ArgumentNotNull(blob, "blob"); Sitecore.Diagnostics.Assert.ArgumentNotNull(message, "message"); var blockIds = new List<string>(); if (blob.Exists()) { blockIds.AddRange(blob.DownloadBlockList().Select(b => b.Name)); } string blockId = Guid.NewGuid().ToString().Replace("-", string.Empty); blockIds.Add(blockId); using (var blockData = new MemoryStream(LogStorageManager.DefaultTextEncoding.GetBytes(message), false)) { blob.PutBlock(blockId, blockData, null); blob.PutBlockList(blockIds); } }