public void BlockBlobReadStreamSeekTest()
{
byte[] buffer = GetRandomBuffer(3 * 1024 * 1024);
CloudBlobContainer container = GetRandomContainerReference();
try
{
container.Create();
CloudBlockBlob blob = container.GetBlockBlobReference("blob1");
blob.StreamMinimumReadSizeInBytes = 2 * 1024 * 1024;
using (MemoryStream wholeBlob = new MemoryStream(buffer))
{
blob.UploadFromStream(wholeBlob);
}
OperationContext opContext = new OperationContext();
using (Stream blobStream = blob.OpenRead(null, null, opContext))
{
int attempts = BlobReadStreamSeekTest(blobStream, blob.StreamMinimumReadSizeInBytes, buffer, false);
TestHelper.AssertNAttempts(opContext, attempts);
}
opContext = new OperationContext();
using (Stream blobStream = blob.OpenRead(null, null, opContext))
{
int attempts = BlobReadStreamSeekTest(blobStream, blob.StreamMinimumReadSizeInBytes, buffer, true);
TestHelper.AssertNAttempts(opContext, attempts);
}
}
finally
{
container.DeleteIfExists();
}
}
public void BlockBlobReadLockToETagTest()
{
byte[] outBuffer = new byte[1 * 1024 * 1024];
byte[] buffer = GetRandomBuffer(2 * outBuffer.Length);
CloudBlobContainer container = GetRandomContainerReference();
try
{
container.Create();
CloudBlockBlob blob = container.GetBlockBlobReference("blob1");
blob.StreamMinimumReadSizeInBytes = outBuffer.Length;
using (MemoryStream wholeBlob = new MemoryStream(buffer))
{
blob.UploadFromStream(wholeBlob);
}
using (Stream blobStream = blob.OpenRead())
{
blob.SetMetadata();
blobStream.Read(outBuffer, 0, outBuffer.Length);
blob.SetMetadata();
TestHelper.ExpectedException(
() => blobStream.Read(outBuffer, 0, outBuffer.Length),
"Blob read stream should fail if blob is modified during read",
HttpStatusCode.PreconditionFailed);
}
using (Stream blobStream = blob.OpenRead())
{
blob.SetMetadata();
long length = blobStream.Length;
blob.SetMetadata();
TestHelper.ExpectedException(
() => blobStream.Read(outBuffer, 0, outBuffer.Length),
"Blob read stream should fail if blob is modified during read",
HttpStatusCode.PreconditionFailed);
}
AccessCondition accessCondition = AccessCondition.GenerateIfNotModifiedSinceCondition(DateTimeOffset.Now);
using (Stream blobStream = blob.OpenRead(accessCondition))
{
blob.SetMetadata();
blobStream.Read(outBuffer, 0, outBuffer.Length);
blob.SetMetadata();
TestHelper.ExpectedException(
() => blobStream.Read(outBuffer, 0, outBuffer.Length),
"Blob read stream should fail if blob is modified during read",
HttpStatusCode.PreconditionFailed);
}
}
finally
{
container.DeleteIfExists();
}
}
public void BlockBlobReadStreamBasicTest()
{
byte[] buffer = GetRandomBuffer(5 * 1024 * 1024);
CloudBlobContainer container = GetRandomContainerReference();
try
{
container.Create();
CloudBlockBlob blob = container.GetBlockBlobReference("blob1");
using (MemoryStream wholeBlob = new MemoryStream(buffer))
{
blob.UploadFromStream(wholeBlob);
}
using (MemoryStream wholeBlob = new MemoryStream(buffer))
{
using (Stream blobStream = blob.OpenRead())
{
TestHelper.AssertStreamsAreEqual(wholeBlob, blobStream);
}
}
}
finally
{
container.DeleteIfExists();
}
}
private static XmlLoggingConfiguration ReadLoggingConfiguration(CloudBlockBlob blob)
{
Logger.Debug("Opening blob for reading.");
Stream stream = blob.OpenRead();
Logger.Debug("Reading logging configuration from blob.");
return new XmlLoggingConfiguration(new XmlTextReader(stream), null);
}
private void LogError(Exception ex, string message)
{
//ConfigurationManager.ConnectionStrings["CineStorageConStr"].ConnectionString
Microsoft.WindowsAzure.Storage.CloudStorageAccount storageAccount = Microsoft.WindowsAzure.Storage.CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("CineStorageConStr"));
Microsoft.WindowsAzure.Storage.Blob.CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
// Retrieve a reference to a container.
Microsoft.WindowsAzure.Storage.Blob.CloudBlobContainer container = blobClient.GetContainerReference("data");
Microsoft.WindowsAzure.Storage.Blob.CloudBlockBlob logBlob = container.GetBlockBlobReference(String.Format(this.LogFile, DateTime.UtcNow.ToString("dd MMM yyyy")));
try
{
using (StreamReader sr = new StreamReader(logBlob.OpenRead()))
{
using (StreamWriter sw = new StreamWriter(logBlob.OpenWrite()))
{
sw.Write(sr.ReadToEnd());
if (ex != null)
{
sw.Write(System.Environment.NewLine);
sw.WriteLine(ex.Message);
sw.WriteLine(ex.StackTrace);
sw.Write(System.Environment.NewLine);
if (ex.InnerException != null)
{
sw.Write(System.Environment.NewLine);
sw.WriteLine(ex.InnerException.Message);
sw.WriteLine(ex.InnerException.StackTrace);
sw.Write(System.Environment.NewLine);
}
}
if (message != null)
{
sw.Write(System.Environment.NewLine);
sw.WriteLine(message);
sw.Write(System.Environment.NewLine);
}
}
}
}
catch { }
}
public void BlockBlobReadStreamRandomSeekTest()
{
byte[] buffer = GetRandomBuffer(5 * 1024 * 1024);
CloudBlobContainer container = GetRandomContainerReference();
try
{
container.Create();
CloudBlockBlob blob = container.GetBlockBlobReference("blob1");
using (MemoryStream wholeBlob = new MemoryStream(buffer))
{
blob.UploadFromStream(wholeBlob);
}
Random random = new Random();
byte[] testBuffer = new byte[1024];
using (Stream blobStream = blob.OpenRead())
{
for (int count = 0; count < 20; count++)
{
int newPosition = random.Next(buffer.Length - testBuffer.Length);
SeekRandomly(blobStream, newPosition);
int readSize = random.Next(1, testBuffer.Length);
int totalRead = blobStream.Read(testBuffer, 0, readSize);
Assert.IsTrue(totalRead > 0 && totalRead <= readSize, "BlobStream.Read returned " + totalRead + " bytes when we asked for " + readSize + " bytes");
for (int i = 0; i < totalRead; i++)
{
Assert.AreEqual(buffer[i + newPosition], testBuffer[i]);
}
}
}
}
finally
{
container.DeleteIfExists();
}
}
/// <summary>
/// Test blob reads, expecting success.
/// </summary>
/// <param name="testBlob">The blob to test.</param>
/// <param name="testAccessCondition">The access condition to use.</param>
private void BlobReadExpectLeaseSuccess(CloudBlockBlob testBlob, AccessCondition testAccessCondition)
{
testBlob.FetchAttributes(testAccessCondition, null /* options */);
testBlob.Snapshot(null /* metadata */, testAccessCondition, null /* options */).Delete();
DownloadText(testBlob, Encoding.UTF8, testAccessCondition, null /* options */);
Stream stream = testBlob.OpenRead(testAccessCondition, null /* options */);
stream.ReadByte();
}
/// <summary>
/// Test blob reads, expecting lease failure.
/// </summary>
/// <param name="testBlob">The blob to test.</param>
/// <param name="targetBlob">The blob to use for the target of copy operations.</param>
/// <param name="testAccessCondition">The failing access condition to use.</param>
/// <param name="expectedErrorCode">The expected error code.</param>
/// <param name="description">The reason why these calls should fail.</param>
private void BlobReadExpectLeaseFailure(CloudBlockBlob testBlob, CloudBlockBlob targetBlob, AccessCondition testAccessCondition, HttpStatusCode expectedStatusCode, string expectedErrorCode, string description)
{
// FetchAttributes is a HEAD request with no extended error info, so it returns with the generic ConditionFailed error code.
TestHelper.ExpectedException(
() => testBlob.FetchAttributes(testAccessCondition, null /* options */),
description + "(Fetch Attributes)",
HttpStatusCode.PreconditionFailed);
TestHelper.ExpectedException(
() => testBlob.Snapshot(null /* metadata */, testAccessCondition, null /* options */),
description + " (Create Snapshot)",
expectedStatusCode,
expectedErrorCode);
TestHelper.ExpectedException(
() => DownloadText(testBlob, Encoding.UTF8, testAccessCondition, null /* options */),
description + " (Download Text)",
expectedStatusCode,
expectedErrorCode);
TestHelper.ExpectedException(
() => testBlob.OpenRead(testAccessCondition, null /* options */),
description + " (Read Stream)",
expectedStatusCode/*,
expectedErrorCode*/);
}
public void DisableContentMD5ValidationTest()
{
byte[] buffer = new byte[1024];
Random random = new Random();
random.NextBytes(buffer);
BlobRequestOptions optionsWithNoMD5 = new BlobRequestOptions()
{
DisableContentMD5Validation = true,
StoreBlobContentMD5 = true,
};
BlobRequestOptions optionsWithMD5 = new BlobRequestOptions()
{
DisableContentMD5Validation = false,
StoreBlobContentMD5 = true,
};
CloudBlobContainer container = GetRandomContainerReference();
try
{
container.Create();
CloudBlockBlob blockBlob = container.GetBlockBlobReference("blob1");
using (Stream stream = new NonSeekableMemoryStream(buffer))
{
blockBlob.UploadFromStream(stream, null, optionsWithMD5);
}
using (Stream stream = new MemoryStream())
{
blockBlob.DownloadToStream(stream, null, optionsWithMD5);
blockBlob.DownloadToStream(stream, null, optionsWithNoMD5);
using (Stream blobStream = blockBlob.OpenRead(null, optionsWithMD5))
{
int read;
do
{
read = blobStream.Read(buffer, 0, buffer.Length);
}while (read > 0);
}
using (Stream blobStream = blockBlob.OpenRead(null, optionsWithNoMD5))
{
int read;
do
{
read = blobStream.Read(buffer, 0, buffer.Length);
}while (read > 0);
}
blockBlob.Properties.ContentMD5 = "MDAwMDAwMDA=";
blockBlob.SetProperties();
TestHelper.ExpectedException(
() => blockBlob.DownloadToStream(stream, null, optionsWithMD5),
"Downloading a blob with invalid MD5 should fail",
HttpStatusCode.OK);
blockBlob.DownloadToStream(stream, null, optionsWithNoMD5);
using (Stream blobStream = blockBlob.OpenRead(null, optionsWithMD5))
{
TestHelper.ExpectedException <IOException>(
() =>
{
int read;
do
{
read = blobStream.Read(buffer, 0, buffer.Length);
}while (read > 0);
},
"Downloading a blob with invalid MD5 should fail");
}
using (Stream blobStream = blockBlob.OpenRead(null, optionsWithNoMD5))
{
int read;
do
{
read = blobStream.Read(buffer, 0, buffer.Length);
}while (read > 0);
}
}
CloudPageBlob pageBlob = container.GetPageBlobReference("blob2");
using (Stream stream = new MemoryStream(buffer))
{
pageBlob.UploadFromStream(stream, null, optionsWithMD5);
}
using (Stream stream = new MemoryStream())
{
pageBlob.DownloadToStream(stream, null, optionsWithMD5);
pageBlob.DownloadToStream(stream, null, optionsWithNoMD5);
using (Stream blobStream = pageBlob.OpenRead(null, optionsWithMD5))
{
int read;
do
{
read = blobStream.Read(buffer, 0, buffer.Length);
}while (read > 0);
}
using (Stream blobStream = pageBlob.OpenRead(null, optionsWithNoMD5))
{
int read;
do
{
read = blobStream.Read(buffer, 0, buffer.Length);
}while (read > 0);
}
pageBlob.Properties.ContentMD5 = "MDAwMDAwMDA=";
pageBlob.SetProperties();
TestHelper.ExpectedException(
() => pageBlob.DownloadToStream(stream, null, optionsWithMD5),
"Downloading a blob with invalid MD5 should fail",
HttpStatusCode.OK);
pageBlob.DownloadToStream(stream, null, optionsWithNoMD5);
using (Stream blobStream = pageBlob.OpenRead(null, optionsWithMD5))
{
TestHelper.ExpectedException <IOException>(
() =>
{
int read;
do
{
read = blobStream.Read(buffer, 0, buffer.Length);
}while (read > 0);
},
"Downloading a blob with invalid MD5 should fail");
}
using (Stream blobStream = pageBlob.OpenRead(null, optionsWithNoMD5))
{
int read;
do
{
read = blobStream.Read(buffer, 0, buffer.Length);
}while (read > 0);
}
}
}
finally
{
container.DeleteIfExists();
}
}
private static string SetUpInputString(CloudBlockBlob blockBlobReference)
{
string inputStr;
if (blockBlobReference.Exists() == false)
{
inputStr = "";
} else
{
using (StreamReader blobReader = new StreamReader(blockBlobReference.OpenRead()))
{
inputStr = blobReader.ReadToEnd();
}
}
return inputStr;
}
public void ProcessTweetBlob(CloudBlockBlob inputBlob, CloudTable outputAzureTable, string folder)
{
int count = 0;
List<TweetSentimentScore> scores = new List<TweetSentimentScore>();
using (var reader = new CsvReader(new StreamReader(inputBlob.OpenRead())))
{
while (reader.Read())
{
if (count == 0)
{
logger.Write(TraceEventType.Information, "First line: [{0}]", string.Join(",", reader.CurrentRecord));
}
count++;
var tweet = reader.GetField(0); //get the tweet
var entity = new TweetSentimentScore()
{
PartitionKey = "tweetsentimentscore",
RowKey = Guid.NewGuid().ToString(),
Tweet = tweet,
SentimentScore = GetScore(url, email, apikey, tweet)
};
scores.Add(entity);
outputAzureTable.Execute(TableOperation.InsertOrReplace(entity)); //Do it one row at a time for demo output
}
}
var iter = scores.Count() / 100;
for (int i = 0; i <= iter; i++)
{
var batchOp = new TableBatchOperation();
scores.Skip(100 * i).Take(100).ToList().ForEach(a =>
{
batchOp.Add(TableOperation.InsertOrReplace(a));
});
//outputAzureTable.ExecuteBatch(batchOp); //Removed for demo purposes.
}
logger.Write(TraceEventType.Information, string.Format(CultureInfo.InvariantCulture,
"{0},{1},{2},{3},{4}\n",
folder,
inputBlob.Name,
count,
Environment.MachineName,
DateTime.UtcNow));
}
