public void BlockBlobReadStreamSeekTest() { byte[] buffer = GetRandomBuffer(3 * 1024 * 1024); CloudBlobContainer container = GetRandomContainerReference(); try { container.Create(); CloudBlockBlob blob = container.GetBlockBlobReference("blob1"); blob.StreamMinimumReadSizeInBytes = 2 * 1024 * 1024; using (MemoryStream wholeBlob = new MemoryStream(buffer)) { blob.UploadFromStream(wholeBlob); } OperationContext opContext = new OperationContext(); using (Stream blobStream = blob.OpenRead(null, null, opContext)) { int attempts = BlobReadStreamSeekTest(blobStream, blob.StreamMinimumReadSizeInBytes, buffer, false); TestHelper.AssertNAttempts(opContext, attempts); } opContext = new OperationContext(); using (Stream blobStream = blob.OpenRead(null, null, opContext)) { int attempts = BlobReadStreamSeekTest(blobStream, blob.StreamMinimumReadSizeInBytes, buffer, true); TestHelper.AssertNAttempts(opContext, attempts); } } finally { container.DeleteIfExists(); } }
public void BlockBlobReadLockToETagTest() { byte[] outBuffer = new byte[1 * 1024 * 1024]; byte[] buffer = GetRandomBuffer(2 * outBuffer.Length); CloudBlobContainer container = GetRandomContainerReference(); try { container.Create(); CloudBlockBlob blob = container.GetBlockBlobReference("blob1"); blob.StreamMinimumReadSizeInBytes = outBuffer.Length; using (MemoryStream wholeBlob = new MemoryStream(buffer)) { blob.UploadFromStream(wholeBlob); } using (Stream blobStream = blob.OpenRead()) { blob.SetMetadata(); blobStream.Read(outBuffer, 0, outBuffer.Length); blob.SetMetadata(); TestHelper.ExpectedException( () => blobStream.Read(outBuffer, 0, outBuffer.Length), "Blob read stream should fail if blob is modified during read", HttpStatusCode.PreconditionFailed); } using (Stream blobStream = blob.OpenRead()) { blob.SetMetadata(); long length = blobStream.Length; blob.SetMetadata(); TestHelper.ExpectedException( () => blobStream.Read(outBuffer, 0, outBuffer.Length), "Blob read stream should fail if blob is modified during read", HttpStatusCode.PreconditionFailed); } AccessCondition accessCondition = AccessCondition.GenerateIfNotModifiedSinceCondition(DateTimeOffset.Now); using (Stream blobStream = blob.OpenRead(accessCondition)) { blob.SetMetadata(); blobStream.Read(outBuffer, 0, outBuffer.Length); blob.SetMetadata(); TestHelper.ExpectedException( () => blobStream.Read(outBuffer, 0, outBuffer.Length), "Blob read stream should fail if blob is modified during read", HttpStatusCode.PreconditionFailed); } } finally { container.DeleteIfExists(); } }
public void BlockBlobReadStreamBasicTest() { byte[] buffer = GetRandomBuffer(5 * 1024 * 1024); CloudBlobContainer container = GetRandomContainerReference(); try { container.Create(); CloudBlockBlob blob = container.GetBlockBlobReference("blob1"); using (MemoryStream wholeBlob = new MemoryStream(buffer)) { blob.UploadFromStream(wholeBlob); } using (MemoryStream wholeBlob = new MemoryStream(buffer)) { using (Stream blobStream = blob.OpenRead()) { TestHelper.AssertStreamsAreEqual(wholeBlob, blobStream); } } } finally { container.DeleteIfExists(); } }
private static XmlLoggingConfiguration ReadLoggingConfiguration(CloudBlockBlob blob) { Logger.Debug("Opening blob for reading."); Stream stream = blob.OpenRead(); Logger.Debug("Reading logging configuration from blob."); return new XmlLoggingConfiguration(new XmlTextReader(stream), null); }
private void LogError(Exception ex, string message) { //ConfigurationManager.ConnectionStrings["CineStorageConStr"].ConnectionString Microsoft.WindowsAzure.Storage.CloudStorageAccount storageAccount = Microsoft.WindowsAzure.Storage.CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("CineStorageConStr")); Microsoft.WindowsAzure.Storage.Blob.CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); // Retrieve a reference to a container. Microsoft.WindowsAzure.Storage.Blob.CloudBlobContainer container = blobClient.GetContainerReference("data"); Microsoft.WindowsAzure.Storage.Blob.CloudBlockBlob logBlob = container.GetBlockBlobReference(String.Format(this.LogFile, DateTime.UtcNow.ToString("dd MMM yyyy"))); try { using (StreamReader sr = new StreamReader(logBlob.OpenRead())) { using (StreamWriter sw = new StreamWriter(logBlob.OpenWrite())) { sw.Write(sr.ReadToEnd()); if (ex != null) { sw.Write(System.Environment.NewLine); sw.WriteLine(ex.Message); sw.WriteLine(ex.StackTrace); sw.Write(System.Environment.NewLine); if (ex.InnerException != null) { sw.Write(System.Environment.NewLine); sw.WriteLine(ex.InnerException.Message); sw.WriteLine(ex.InnerException.StackTrace); sw.Write(System.Environment.NewLine); } } if (message != null) { sw.Write(System.Environment.NewLine); sw.WriteLine(message); sw.Write(System.Environment.NewLine); } } } } catch { } }
public void BlockBlobReadStreamRandomSeekTest() { byte[] buffer = GetRandomBuffer(5 * 1024 * 1024); CloudBlobContainer container = GetRandomContainerReference(); try { container.Create(); CloudBlockBlob blob = container.GetBlockBlobReference("blob1"); using (MemoryStream wholeBlob = new MemoryStream(buffer)) { blob.UploadFromStream(wholeBlob); } Random random = new Random(); byte[] testBuffer = new byte[1024]; using (Stream blobStream = blob.OpenRead()) { for (int count = 0; count < 20; count++) { int newPosition = random.Next(buffer.Length - testBuffer.Length); SeekRandomly(blobStream, newPosition); int readSize = random.Next(1, testBuffer.Length); int totalRead = blobStream.Read(testBuffer, 0, readSize); Assert.IsTrue(totalRead > 0 && totalRead <= readSize, "BlobStream.Read returned " + totalRead + " bytes when we asked for " + readSize + " bytes"); for (int i = 0; i < totalRead; i++) { Assert.AreEqual(buffer[i + newPosition], testBuffer[i]); } } } } finally { container.DeleteIfExists(); } }
/// <summary> /// Test blob reads, expecting success. /// </summary> /// <param name="testBlob">The blob to test.</param> /// <param name="testAccessCondition">The access condition to use.</param> private void BlobReadExpectLeaseSuccess(CloudBlockBlob testBlob, AccessCondition testAccessCondition) { testBlob.FetchAttributes(testAccessCondition, null /* options */); testBlob.Snapshot(null /* metadata */, testAccessCondition, null /* options */).Delete(); DownloadText(testBlob, Encoding.UTF8, testAccessCondition, null /* options */); Stream stream = testBlob.OpenRead(testAccessCondition, null /* options */); stream.ReadByte(); }
/// <summary> /// Test blob reads, expecting lease failure. /// </summary> /// <param name="testBlob">The blob to test.</param> /// <param name="targetBlob">The blob to use for the target of copy operations.</param> /// <param name="testAccessCondition">The failing access condition to use.</param> /// <param name="expectedErrorCode">The expected error code.</param> /// <param name="description">The reason why these calls should fail.</param> private void BlobReadExpectLeaseFailure(CloudBlockBlob testBlob, CloudBlockBlob targetBlob, AccessCondition testAccessCondition, HttpStatusCode expectedStatusCode, string expectedErrorCode, string description) { // FetchAttributes is a HEAD request with no extended error info, so it returns with the generic ConditionFailed error code. TestHelper.ExpectedException( () => testBlob.FetchAttributes(testAccessCondition, null /* options */), description + "(Fetch Attributes)", HttpStatusCode.PreconditionFailed); TestHelper.ExpectedException( () => testBlob.Snapshot(null /* metadata */, testAccessCondition, null /* options */), description + " (Create Snapshot)", expectedStatusCode, expectedErrorCode); TestHelper.ExpectedException( () => DownloadText(testBlob, Encoding.UTF8, testAccessCondition, null /* options */), description + " (Download Text)", expectedStatusCode, expectedErrorCode); TestHelper.ExpectedException( () => testBlob.OpenRead(testAccessCondition, null /* options */), description + " (Read Stream)", expectedStatusCode/*, expectedErrorCode*/); }
public void DisableContentMD5ValidationTest() { byte[] buffer = new byte[1024]; Random random = new Random(); random.NextBytes(buffer); BlobRequestOptions optionsWithNoMD5 = new BlobRequestOptions() { DisableContentMD5Validation = true, StoreBlobContentMD5 = true, }; BlobRequestOptions optionsWithMD5 = new BlobRequestOptions() { DisableContentMD5Validation = false, StoreBlobContentMD5 = true, }; CloudBlobContainer container = GetRandomContainerReference(); try { container.Create(); CloudBlockBlob blockBlob = container.GetBlockBlobReference("blob1"); using (Stream stream = new NonSeekableMemoryStream(buffer)) { blockBlob.UploadFromStream(stream, null, optionsWithMD5); } using (Stream stream = new MemoryStream()) { blockBlob.DownloadToStream(stream, null, optionsWithMD5); blockBlob.DownloadToStream(stream, null, optionsWithNoMD5); using (Stream blobStream = blockBlob.OpenRead(null, optionsWithMD5)) { int read; do { read = blobStream.Read(buffer, 0, buffer.Length); }while (read > 0); } using (Stream blobStream = blockBlob.OpenRead(null, optionsWithNoMD5)) { int read; do { read = blobStream.Read(buffer, 0, buffer.Length); }while (read > 0); } blockBlob.Properties.ContentMD5 = "MDAwMDAwMDA="; blockBlob.SetProperties(); TestHelper.ExpectedException( () => blockBlob.DownloadToStream(stream, null, optionsWithMD5), "Downloading a blob with invalid MD5 should fail", HttpStatusCode.OK); blockBlob.DownloadToStream(stream, null, optionsWithNoMD5); using (Stream blobStream = blockBlob.OpenRead(null, optionsWithMD5)) { TestHelper.ExpectedException <IOException>( () => { int read; do { read = blobStream.Read(buffer, 0, buffer.Length); }while (read > 0); }, "Downloading a blob with invalid MD5 should fail"); } using (Stream blobStream = blockBlob.OpenRead(null, optionsWithNoMD5)) { int read; do { read = blobStream.Read(buffer, 0, buffer.Length); }while (read > 0); } } CloudPageBlob pageBlob = container.GetPageBlobReference("blob2"); using (Stream stream = new MemoryStream(buffer)) { pageBlob.UploadFromStream(stream, null, optionsWithMD5); } using (Stream stream = new MemoryStream()) { pageBlob.DownloadToStream(stream, null, optionsWithMD5); pageBlob.DownloadToStream(stream, null, optionsWithNoMD5); using (Stream blobStream = pageBlob.OpenRead(null, optionsWithMD5)) { int read; do { read = blobStream.Read(buffer, 0, buffer.Length); }while (read > 0); } using (Stream blobStream = pageBlob.OpenRead(null, optionsWithNoMD5)) { int read; do { read = blobStream.Read(buffer, 0, buffer.Length); }while (read > 0); } pageBlob.Properties.ContentMD5 = "MDAwMDAwMDA="; pageBlob.SetProperties(); TestHelper.ExpectedException( () => pageBlob.DownloadToStream(stream, null, optionsWithMD5), "Downloading a blob with invalid MD5 should fail", HttpStatusCode.OK); pageBlob.DownloadToStream(stream, null, optionsWithNoMD5); using (Stream blobStream = pageBlob.OpenRead(null, optionsWithMD5)) { TestHelper.ExpectedException <IOException>( () => { int read; do { read = blobStream.Read(buffer, 0, buffer.Length); }while (read > 0); }, "Downloading a blob with invalid MD5 should fail"); } using (Stream blobStream = pageBlob.OpenRead(null, optionsWithNoMD5)) { int read; do { read = blobStream.Read(buffer, 0, buffer.Length); }while (read > 0); } } } finally { container.DeleteIfExists(); } }
private static string SetUpInputString(CloudBlockBlob blockBlobReference) { string inputStr; if (blockBlobReference.Exists() == false) { inputStr = ""; } else { using (StreamReader blobReader = new StreamReader(blockBlobReference.OpenRead())) { inputStr = blobReader.ReadToEnd(); } } return inputStr; }
public void ProcessTweetBlob(CloudBlockBlob inputBlob, CloudTable outputAzureTable, string folder) { int count = 0; List<TweetSentimentScore> scores = new List<TweetSentimentScore>(); using (var reader = new CsvReader(new StreamReader(inputBlob.OpenRead()))) { while (reader.Read()) { if (count == 0) { logger.Write(TraceEventType.Information, "First line: [{0}]", string.Join(",", reader.CurrentRecord)); } count++; var tweet = reader.GetField(0); //get the tweet var entity = new TweetSentimentScore() { PartitionKey = "tweetsentimentscore", RowKey = Guid.NewGuid().ToString(), Tweet = tweet, SentimentScore = GetScore(url, email, apikey, tweet) }; scores.Add(entity); outputAzureTable.Execute(TableOperation.InsertOrReplace(entity)); //Do it one row at a time for demo output } } var iter = scores.Count() / 100; for (int i = 0; i <= iter; i++) { var batchOp = new TableBatchOperation(); scores.Skip(100 * i).Take(100).ToList().ForEach(a => { batchOp.Add(TableOperation.InsertOrReplace(a)); }); //outputAzureTable.ExecuteBatch(batchOp); //Removed for demo purposes. } logger.Write(TraceEventType.Information, string.Format(CultureInfo.InvariantCulture, "{0},{1},{2},{3},{4}\n", folder, inputBlob.Name, count, Environment.MachineName, DateTime.UtcNow)); }