public static void TaskMain(string[] args)
{
if (args == null || args.Length != 5)
{
throw new Exception("Usage: TopNWordsSample.exe --Task <blobpath> <numtopwords> <storageAccountName> <storageAccountKey>");
}
string blobName = args[1];
int numTopN = int.Parse(args[2]);
string storageAccountName = args[3];
string storageAccountKey = args[4];
// open the cloud blob that contains the book
var storageCred = new StorageCredentials(storageAccountName, storageAccountKey);
CloudBlockBlob blob = new CloudBlockBlob(new Uri(blobName), storageCred);
using (Stream memoryStream = new MemoryStream())
{
blob.DownloadToStream(memoryStream);
memoryStream.Position = 0; //Reset the stream
var sr = new StreamReader(memoryStream);
var myStr = sr.ReadToEnd();
string[] words = myStr.Split(' ');
var topNWords =
words.
Where(word => word.Length > 0).
GroupBy(word => word, (key, group) => new KeyValuePair<String, long>(key, group.LongCount())).
OrderByDescending(x => x.Value).
Take(numTopN).
ToList();
foreach (var pair in topNWords)
{
Console.WriteLine("{0} {1}", pair.Key, pair.Value);
}
}
}
private void init(CloudBlockBlob blob)
{
if (blob == null)
throw new ArgumentNullException("blob");
_server_ref = blob;
Stream = new MemoryStream();
_server_ref.DownloadToStream(Stream);
Stream.Seek(0, SeekOrigin.Begin);
//Automatic Fetches Attributes on DownloadToStream.
ETag = _server_ref.Properties.ETag;
Length = _server_ref.Properties.Length;
}
private static string DownloadBlob(CloudBlockBlob blob)
{
using (var stream = new MemoryStream())
{
StreamReader reader;
try
{
blob.DownloadToStream(stream, options: new BlobRequestOptions()
{
RetryPolicy = new LinearRetry(TimeSpan.FromSeconds(5), 3)
});
}
catch (StorageException se)
{
return "";
}
try
{
stream.Seek(0, 0);
reader = new StreamReader(new GZipStream(stream, CompressionMode.Decompress));
return reader.ReadToEnd();
}
catch
{
stream.Seek(0, 0);
reader = new StreamReader(stream);
return reader.ReadToEnd();
}
}
}
private void DownloadIfExists(CloudBlockBlob blob)
{
string target = Path.Combine(Root, blob.Name);
if (blob.Exists())
{
if (!Directory.Exists(Root))
{
Directory.CreateDirectory(Root);
}
using (var file = File.OpenWrite(target))
{
blob.DownloadToStream(file);
}
}
}
protected override void OnExecuting(CloudBlockBlob workItem)
{
if (workItem == null)
throw new ArgumentNullException("workItem");
if (workItem.Metadata.ContainsKey(CompressedFlag))
return;
using (var blobStream = new MemoryStream())
{
workItem.DownloadToStream(blobStream);
using (var compressedStream = new MemoryStream())
{
CompressStream(compressedStream, blobStream);
SetCompressedFlag(workItem);
workItem.UploadFromStream(compressedStream);
}
}
}
public MemoryStream DownloadBlob(CloudBlockBlob blob)
{
var memorystream = new MemoryStream { Position = 0 };
blob.DownloadToStream(memorystream);
return memorystream;
}
/// <summary>
/// Downloads the BLOB.
/// </summary>
/// <param name="blob">The BLOB.</param>
/// <param name="fetchAttribute">if set to <c>true</c> [fetch attribute].</param>
/// <param name="metaData">The meta data.</param>
/// <returns>System.Byte[].</returns>
public static byte[] DownloadBlob(CloudBlockBlob blob, bool fetchAttribute, out IDictionary<string, string> metaData)
{
try
{
blob.CheckNullObject("blob");
if (fetchAttribute)
{
blob.FetchAttributes();
metaData = blob.Metadata;
}
else
{
metaData = null;
}
var msRead = new MemoryStream { Position = 0 };
using (msRead)
{
blob.DownloadToStream(msRead);
return msRead.ToBytes();
}
}
catch (Exception ex)
{
throw ex.Handle("DownloadBlob", blob.Uri.ToString());
}
}
public void UseTransactionalMD5GetTest()
{
BlobRequestOptions optionsWithNoMD5 = new BlobRequestOptions()
{
UseTransactionalMD5 = false,
};
BlobRequestOptions optionsWithMD5 = new BlobRequestOptions()
{
UseTransactionalMD5 = true,
};
byte[] buffer = GetRandomBuffer(3 * 1024 * 1024);
MD5 hasher = MD5.Create();
string md5 = Convert.ToBase64String(hasher.ComputeHash(buffer));
string lastCheckMD5 = null;
int checkCount = 0;
OperationContext opContextWithMD5Check = new OperationContext();
opContextWithMD5Check.ResponseReceived += (_, args) =>
{
if (args.Response.ContentLength >= buffer.Length)
{
lastCheckMD5 = args.Response.Headers[HttpResponseHeader.ContentMd5];
checkCount++;
}
};
CloudBlobContainer container = GetRandomContainerReference();
try
{
container.Create();
CloudBlockBlob blockBlob = container.GetBlockBlobReference("blob1");
using (Stream blobStream = blockBlob.OpenWrite())
{
blobStream.Write(buffer, 0, buffer.Length);
blobStream.Write(buffer, 0, buffer.Length);
}
checkCount = 0;
using (Stream stream = new MemoryStream())
{
blockBlob.DownloadToStream(stream, null, optionsWithNoMD5, opContextWithMD5Check);
Assert.IsNotNull(lastCheckMD5);
blockBlob.DownloadToStream(stream, null, optionsWithMD5, opContextWithMD5Check);
Assert.IsNotNull(lastCheckMD5);
blockBlob.DownloadRangeToStream(stream, buffer.Length, buffer.Length, null, optionsWithNoMD5, opContextWithMD5Check);
Assert.IsNull(lastCheckMD5);
blockBlob.DownloadRangeToStream(stream, buffer.Length, buffer.Length, null, optionsWithMD5, opContextWithMD5Check);
Assert.AreEqual(md5, lastCheckMD5);
blockBlob.DownloadRangeToStream(stream, 1024, 4 * 1024 * 1024 + 1, null, optionsWithNoMD5, opContextWithMD5Check);
Assert.IsNull(lastCheckMD5);
StorageException storageEx = TestHelper.ExpectedException <StorageException>(
() => blockBlob.DownloadRangeToStream(stream, 1024, 4 * 1024 * 1024 + 1, null, optionsWithMD5, opContextWithMD5Check),
"Downloading more than 4MB with transactional MD5 should not be supported");
Assert.IsInstanceOfType(storageEx.InnerException, typeof(ArgumentOutOfRangeException));
}
Assert.AreEqual(5, checkCount);
CloudPageBlob pageBlob = container.GetPageBlobReference("blob2");
using (Stream blobStream = pageBlob.OpenWrite(buffer.Length * 2))
{
blobStream.Write(buffer, 0, buffer.Length);
blobStream.Write(buffer, 0, buffer.Length);
}
checkCount = 0;
using (Stream stream = new MemoryStream())
{
pageBlob.DownloadToStream(stream, null, optionsWithNoMD5, opContextWithMD5Check);
Assert.IsNull(lastCheckMD5);
StorageException storageEx = TestHelper.ExpectedException <StorageException>(
() => pageBlob.DownloadToStream(stream, null, optionsWithMD5, opContextWithMD5Check),
"Page blob will not have MD5 set by default; with UseTransactional, download should fail");
pageBlob.DownloadRangeToStream(stream, buffer.Length, buffer.Length, null, optionsWithNoMD5, opContextWithMD5Check);
Assert.IsNull(lastCheckMD5);
pageBlob.DownloadRangeToStream(stream, buffer.Length, buffer.Length, null, optionsWithMD5, opContextWithMD5Check);
Assert.AreEqual(md5, lastCheckMD5);
pageBlob.DownloadRangeToStream(stream, 1024, 4 * 1024 * 1024 + 1, null, optionsWithNoMD5, opContextWithMD5Check);
Assert.IsNull(lastCheckMD5);
storageEx = TestHelper.ExpectedException <StorageException>(
() => pageBlob.DownloadRangeToStream(stream, 1024, 4 * 1024 * 1024 + 1, null, optionsWithMD5, opContextWithMD5Check),
"Downloading more than 4MB with transactional MD5 should not be supported");
Assert.IsInstanceOfType(storageEx.InnerException, typeof(ArgumentOutOfRangeException));
}
Assert.AreEqual(5, checkCount);
}
finally
{
container.DeleteIfExists();
}
}
public void BlobIngressEgressCounters()
{
CloudBlobContainer container = GetRandomContainerReference();
container.CreateIfNotExists();
CloudBlockBlob blob = container.GetBlockBlobReference("blob1");
string[] blockIds = new string[] { Convert.ToBase64String(Guid.NewGuid().ToByteArray()), Convert.ToBase64String(Guid.NewGuid().ToByteArray()), Convert.ToBase64String(Guid.NewGuid().ToByteArray()) };
try
{
// 1 byte
TestHelper.ValidateIngressEgress(Selectors.IfUrlContains(blob.Uri.ToString()), () =>
{
OperationContext opContext = new OperationContext();
blob.PutBlock(blockIds[0], new MemoryStream(GetRandomBuffer(1)), null, null, new BlobRequestOptions()
{
RetryPolicy = new RetryPolicies.NoRetry()
}, opContext);
return(opContext.LastResult);
});
// 1024
TestHelper.ValidateIngressEgress(Selectors.IfUrlContains(blob.Uri.ToString()), () =>
{
OperationContext opContext = new OperationContext();
blob.PutBlock(blockIds[1], new MemoryStream(GetRandomBuffer(1024)), null, null, new BlobRequestOptions()
{
RetryPolicy = new RetryPolicies.NoRetry()
}, opContext);
return(opContext.LastResult);
});
// 98765
TestHelper.ValidateIngressEgress(Selectors.IfUrlContains(blob.Uri.ToString()), () =>
{
OperationContext opContext = new OperationContext();
blob.PutBlock(blockIds[2], new MemoryStream(GetRandomBuffer(98765)), null, null, new BlobRequestOptions()
{
RetryPolicy = new RetryPolicies.NoRetry()
}, opContext);
return(opContext.LastResult);
});
// PutBlockList
TestHelper.ValidateIngressEgress(Selectors.IfUrlContains(blob.Uri.ToString()), () =>
{
OperationContext opContext = new OperationContext();
blob.PutBlockList(blockIds, null, new BlobRequestOptions()
{
RetryPolicy = new RetryPolicies.NoRetry()
}, opContext);
return(opContext.LastResult);
});
// GetBlockList
TestHelper.ValidateIngressEgress(Selectors.IfUrlContains(blob.Uri.ToString()), () =>
{
OperationContext opContext = new OperationContext();
blob.DownloadBlockList(BlockListingFilter.All, null, new BlobRequestOptions()
{
RetryPolicy = new RetryPolicies.NoRetry()
}, opContext);
return(opContext.LastResult);
});
// Download
TestHelper.ValidateIngressEgress(Selectors.IfUrlContains(blob.Uri.ToString()), () =>
{
OperationContext opContext = new OperationContext();
blob.DownloadToStream(Stream.Null, null, new BlobRequestOptions()
{
RetryPolicy = new RetryPolicies.NoRetry()
}, opContext);
return(opContext.LastResult);
});
Assert.AreEqual(blob.Properties.Length, 98765 + 1024 + 1);
// Error Case
CloudBlockBlob nullBlob = container.GetBlockBlobReference("null");
OperationContext errorContext = new OperationContext();
try
{
nullBlob.DownloadToStream(Stream.Null, null, new BlobRequestOptions()
{
RetryPolicy = new RetryPolicies.NoRetry()
}, errorContext);
Assert.Fail("Null blob, null stream, no download possible.");
}
catch (StorageException)
{
Assert.IsTrue(errorContext.LastResult.IngressBytes > 0);
}
}
finally
{
container.DeleteIfExists();
}
}
private void DownloadBlobInParallel(CloudBlockBlob blob, Stream stream)
{
try
{
blob.FetchAttributes();
blob.ServiceClient.ParallelOperationThreadCount = NumberOfIOThreads;
blob.DownloadToStream(stream);
stream.Seek(0, SeekOrigin.Begin);
}
catch (StorageException ex)
{
logger.Warn(ex.Message);
}
}
public void DisableContentMD5ValidationTest()
{
byte[] buffer = new byte[1024];
Random random = new Random();
random.NextBytes(buffer);
BlobRequestOptions optionsWithNoMD5 = new BlobRequestOptions()
{
DisableContentMD5Validation = true,
StoreBlobContentMD5 = true,
};
BlobRequestOptions optionsWithMD5 = new BlobRequestOptions()
{
DisableContentMD5Validation = false,
StoreBlobContentMD5 = true,
};
CloudBlobContainer container = GetRandomContainerReference();
try
{
container.Create();
CloudBlockBlob blockBlob = container.GetBlockBlobReference("blob1");
using (Stream stream = new NonSeekableMemoryStream(buffer))
{
blockBlob.UploadFromStream(stream, null, optionsWithMD5);
}
using (Stream stream = new MemoryStream())
{
blockBlob.DownloadToStream(stream, null, optionsWithMD5);
blockBlob.DownloadToStream(stream, null, optionsWithNoMD5);
using (Stream blobStream = blockBlob.OpenRead(null, optionsWithMD5))
{
int read;
do
{
read = blobStream.Read(buffer, 0, buffer.Length);
}while (read > 0);
}
using (Stream blobStream = blockBlob.OpenRead(null, optionsWithNoMD5))
{
int read;
do
{
read = blobStream.Read(buffer, 0, buffer.Length);
}while (read > 0);
}
blockBlob.Properties.ContentMD5 = "MDAwMDAwMDA=";
blockBlob.SetProperties();
TestHelper.ExpectedException(
() => blockBlob.DownloadToStream(stream, null, optionsWithMD5),
"Downloading a blob with invalid MD5 should fail",
HttpStatusCode.OK);
blockBlob.DownloadToStream(stream, null, optionsWithNoMD5);
using (Stream blobStream = blockBlob.OpenRead(null, optionsWithMD5))
{
TestHelper.ExpectedException <IOException>(
() =>
{
int read;
do
{
read = blobStream.Read(buffer, 0, buffer.Length);
}while (read > 0);
},
"Downloading a blob with invalid MD5 should fail");
}
using (Stream blobStream = blockBlob.OpenRead(null, optionsWithNoMD5))
{
int read;
do
{
read = blobStream.Read(buffer, 0, buffer.Length);
}while (read > 0);
}
}
CloudPageBlob pageBlob = container.GetPageBlobReference("blob2");
using (Stream stream = new MemoryStream(buffer))
{
pageBlob.UploadFromStream(stream, null, optionsWithMD5);
}
using (Stream stream = new MemoryStream())
{
pageBlob.DownloadToStream(stream, null, optionsWithMD5);
pageBlob.DownloadToStream(stream, null, optionsWithNoMD5);
using (Stream blobStream = pageBlob.OpenRead(null, optionsWithMD5))
{
int read;
do
{
read = blobStream.Read(buffer, 0, buffer.Length);
}while (read > 0);
}
using (Stream blobStream = pageBlob.OpenRead(null, optionsWithNoMD5))
{
int read;
do
{
read = blobStream.Read(buffer, 0, buffer.Length);
}while (read > 0);
}
pageBlob.Properties.ContentMD5 = "MDAwMDAwMDA=";
pageBlob.SetProperties();
TestHelper.ExpectedException(
() => pageBlob.DownloadToStream(stream, null, optionsWithMD5),
"Downloading a blob with invalid MD5 should fail",
HttpStatusCode.OK);
pageBlob.DownloadToStream(stream, null, optionsWithNoMD5);
using (Stream blobStream = pageBlob.OpenRead(null, optionsWithMD5))
{
TestHelper.ExpectedException <IOException>(
() =>
{
int read;
do
{
read = blobStream.Read(buffer, 0, buffer.Length);
}while (read > 0);
},
"Downloading a blob with invalid MD5 should fail");
}
using (Stream blobStream = pageBlob.OpenRead(null, optionsWithNoMD5))
{
int read;
do
{
read = blobStream.Read(buffer, 0, buffer.Length);
}while (read > 0);
}
}
}
finally
{
container.DeleteIfExists();
}
}
void DownloadBlobInParallel(CloudBlockBlob blob, Stream stream)
{
blob.FetchAttributes();
blob.ServiceClient.ParallelOperationThreadCount = NumberOfIOThreads;
container.ServiceClient.RetryPolicy = new ExponentialRetry(TimeSpan.FromSeconds(BackOffInterval), MaxRetries);
blob.DownloadToStream(stream);
stream.Seek(0, SeekOrigin.Begin);
}
/// <summary>
/// Uploads Memory Stream to Blob
/// </summary>
/// <param name="outputBlob"></param>
/// <param name="line"></param>
private static void UploadBlobStream(CloudBlockBlob outputBlob, string line)
{
using (MemoryStream ms = new MemoryStream())
{
if (outputBlob.Exists())
{
outputBlob.DownloadToStream(ms);
}
byte[] dataToWrite = Encoding.UTF8.GetBytes(line + "\r\n");
ms.Write(dataToWrite, 0, dataToWrite.Length);
ms.Position = 0;
outputBlob.UploadFromStream(ms);
}
}
/// <summary>
/// Down loads JSON log files between 2 dates from the blob storage
/// and exports them into one .CSV file.
/// </summary>
/// <param name="logStart">Begin date and time of the log.</param>
/// <param name="logEnd">End date and time of the log.</param>
private static void GetNetworkSecurityGroupRuleCounters(DateTime logStart, DateTime logEnd)
{
// Creates client.
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(BlobStorageConnectionString);
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
Console.WriteLine($"Getting reference to container {CounterContainerName}");
CloudBlobContainer container = blobClient.GetContainerReference(CounterContainerName);
// Instantiate the URL generator.
StorageURL storageUrl = new StorageURL(container.Uri, SubscriptionID, ResrouceGroupsName, ProviderName, ResrouceTypeName);
List<Log> logs = new List<Log>();
int itemPosition = 0;
// Using the date and time as arguments download all logs from the storage blob.
for (DateTime logTimeStamp = logStart; logTimeStamp <= logEnd; logTimeStamp = logTimeStamp.AddHours(1))
{
Console.WriteLine(logTimeStamp);
Uri storageBlogUrl = storageUrl.GetURL(logTimeStamp);
CloudBlockBlob blockBlob = new CloudBlockBlob(storageBlogUrl, storageAccount.Credentials);
MemoryStream memstream = new MemoryStream();
try
{
blockBlob.DownloadToStream(memstream);
memstream.Position = 0;
JsonSerializer serializer = new JsonSerializer();
using (StreamReader sr = new StreamReader(memstream))
using (JsonTextReader jsonTextReader = new JsonTextReader(sr))
{
// Deserialize JSON.
LogRecords logRecords = serializer.Deserialize<LogRecords>(jsonTextReader);
itemPosition = 0;
foreach (Log logItem in logRecords.records)
{
// Add deserialized logs.
logs.Add(logItem);
itemPosition++;
}
}
}
catch (Exception ex)
{
Console.WriteLine($"{ex.Message} - {storageBlogUrl}");
}
}
// Dump everything in the logs list into a file.
using (StreamWriter file = new StreamWriter(CounterCSVExportNamePath))
{
file.WriteLine("time,systemId,resourceId,operationName,properties.vnetResourceGuid,properties.subnetPrefix"
+ ",properties.macAddress,properties.ruleName,properties.direction,properties.type,properties.matchedConnections");
foreach (Log log in logs)
{
file.WriteLine($"{DateTime.Parse(log.time).ToUniversalTime()}, {log.systemId}, {log.resourceId}, {log.operationName}"
+ $", {log.properties.vnetResourceGuid}, {log.properties.subnetPrefix}, {log.properties.macAddress}"
+ $", {log.properties.ruleName}, {log.properties.direction}, {log.properties.type}, {log.properties.matchedConnections}");
}
}
}
public static void TaskMain(string[] args)
{
if (args == null || args.Length != 4)
{
throw new Exception("Usage: ImageBlur.exe --Task <blobpath> <storageAccountName> <storageAccountKey>");
}
string blobName = args[1];
string storageAccountName = args[2];
string storageAccountKey = args[3];
string workingDirectory = Environment.GetEnvironmentVariable("AZ_BATCH_TASK_WORKING_DIR");
int numberToBlur = 3;
Console.WriteLine();
Console.WriteLine(" blobName: <{0}>", blobName);
Console.WriteLine(" storageAccountName: <{0}>", storageAccountName);
Console.WriteLine(" number to blur: <{0}>", numberToBlur);
Console.WriteLine();
// get source image from cloud blob
var storageCred = new StorageCredentials(storageAccountName, storageAccountKey);
CloudBlockBlob blob = new CloudBlockBlob(new Uri(blobName), storageCred);
using (MemoryStream inStream = new MemoryStream())
{
blob.DownloadToStream(inStream);
Image img = Image.FromStream(inStream);
int imgWidth = img.Width;
int imgHeight = img.Height;
Size size = new Size(imgWidth, imgHeight);
ISupportedImageFormat format = new JpegFormat { Quality = 70 };
// Print image properties to stdout
Console.WriteLine(" Image Properties:");
Console.WriteLine(" Format: {0}", FormatUtilities.GetFormat(inStream));
Console.WriteLine(" Size: {0}", size.ToString());
Console.WriteLine();
for (var i = 0; i < numberToBlur; i++)
{
using (ImageFactory imageFactory = new ImageFactory(preserveExifData: true))
{
int blurIndex = (i * 5) + 10;
imageFactory.Load(inStream);
imageFactory.Resize(size)
.Format(format)
.GaussianBlur(blurIndex)
.Save(workingDirectory + "/resultimage" + i + ".Jpeg");
//.Save(@"C:/Users/jiata/Desktop/imageblur/results/resultimage" + i + ".Jpeg");
}
}
}
// TODO - not working
for (var i = 0; i < numberToBlur; i++)
{
blob.UploadFromFile(workingDirectory + "/resultimage" + i + ".Jpeg", FileMode.Open);
}
Environment.Exit(1);
}
private static void GetLoadBalancerEvents(DateTime logStart, DateTime logEnd)
{
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(BlobStorageConnectionString);
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
Console.WriteLine($"Getting reference to container {EventContainerName}");
CloudBlobContainer container = blobClient.GetContainerReference(EventContainerName);
StorageURL storageUrl = new StorageURL(container.Uri, SubscriptionID, ResrouceGroupsName, ProviderName, ResrouceTypeName, ResourceType.LOADBALANCERS);
List<Log> logs = new List<Log>();
int itemPosition = 0;
// Using the date and time as arguments download all logs from the storage blob.
for (DateTime logTimeStamp = logStart; logTimeStamp <= logEnd; logTimeStamp = logTimeStamp.AddHours(1))
{
Console.WriteLine(logTimeStamp);
Uri storageBlobUrl = storageUrl.GetURL(logTimeStamp);
CloudBlockBlob blockBlob = new CloudBlockBlob(storageBlobUrl, storageAccount.Credentials);
MemoryStream memstream = new MemoryStream();
try
{
blockBlob.DownloadToStream(memstream);
memstream.Position = 0;
JsonSerializer serializer = new JsonSerializer();
using (StreamReader sr = new StreamReader(memstream))
{
using (JsonTextReader jsonTextReader = new JsonTextReader(sr))
{
LogRecords logRecords = serializer.Deserialize<LogRecords>(jsonTextReader);
itemPosition = 0;
foreach (Log logItem in logRecords.records)
{
logs.Add(logItem);
itemPosition++;
}
}
}
}
catch (Exception ex)
{
Console.WriteLine($"{ex.Message} - {storageBlobUrl}");
}
}
using (System.IO.StreamWriter file = new System.IO.StreamWriter(EventCSVExportNamePath))
{
file.WriteLine("time,systemId,category,resourceId,operationName,properties.publicIpAddress"
+ ",properties.port,properties.totalDipCount,properties.dipDownCount,properties.healthPercentage");
foreach (Log log in logs)
{
file.WriteLine($"{DateTime.Parse(log.time).ToUniversalTime()}, {log.systemId}, {log.category}, {log.resourceId}, {log.operationName}"
+ $", {log.properties.publicIpAddress }, {log.properties.port}, {log.properties.totalDipCount}"
+ $", {log.properties.dipDownCount}, {log.properties.healthPercentage}");
}
}
}
public Image blobThumbnail(string filename, string container)
{
try
{
blob = blobClient.GetContainerReference(container);
blob.CreateIfNotExists();
blockBlob = blob.GetBlockBlobReference(@"thumb/" + filename);
using (var outStream = new System.IO.MemoryStream())
{
blockBlob.DownloadToStream(outStream);
return Image.FromStream(outStream);
}
}
catch (Exception e)
{
Console.WriteLine(e.InnerException);
return null;
}
}
private static void DownloadToStream(Stream target, CloudBlockBlob blob)
{
try
{
blob.DownloadToStream(target);
}
catch (StorageException e)
{
throw new BlobException(string.Format(CultureInfo.InvariantCulture, "Could not DownloadToStream from blob named '{0}' in container. See inner exception for details.", blob.Name), e);
}
}
private int DownloadBlob(CloudBlockBlob blockBlob, string targetFolder, int currentCount)
{
string decodedBlockBlobName = System.Net.WebUtility.HtmlDecode(blockBlob.Name).TrimStart(Path.DirectorySeparatorChar);
string targetFileName = string.Format("{0}{1}{2}", targetFolder, Path.DirectorySeparatorChar.ToString(), decodedBlockBlobName);
Logging.Logger.Info(string.Format("Target Folder: {0} Target file: {1} ", targetFolder, targetFileName));
string tempName = Path.Combine(Path.Combine(targetFolder, Guid.NewGuid().ToString()));
if (!File.Exists(targetFileName))
{
Logging.Logger.Info(string.Format("Local file {0} doesn't exist, download size {1}.", targetFileName, blockBlob.Properties.Length));
currentCount++;
if (!SimulationMode)
{
var outFolder = Path.GetDirectoryName(targetFileName);
if(!Directory.Exists(outFolder)){
Directory.CreateDirectory(outFolder);
}
using (var fileStream = System.IO.File.OpenWrite(targetFileName))
{
blockBlob.DownloadToStream(fileStream);
}
File.SetCreationTime(targetFileName, blockBlob.Properties.LastModified.Value.DateTime);
}
else
{
Logging.Logger.Info(string.Format("Download {0} to {1}", blockBlob.Name, targetFolder));
}
}
else
{
FileInfo info = new FileInfo(targetFileName);
if (blockBlob.Properties.Length != info.Length)
{
if (blockBlob.Properties.LastModified.HasValue && blockBlob.Properties.LastModified.Value.DateTime > info.LastWriteTime)
{
Logging.Logger.Info(string.Format("Local file size: {0}, online file size: {1} Blob Updated: {2} FileUpdated: {3} Downloading...", info.Length, blockBlob.Properties.Length, blockBlob.Properties.LastModified.Value.DateTime, info.LastWriteTime));
currentCount++;
if (!SimulationMode)
{
using (var fileStream = System.IO.File.OpenWrite(tempName))
{
blockBlob.DownloadToStream(fileStream);
}
File.Delete(targetFileName);
File.Move(tempName, targetFileName);
File.SetCreationTime(targetFileName, blockBlob.Properties.LastModified.Value.DateTime);
}
else
{
Logging.Logger.Info(string.Format("Download {0} to {1}", blockBlob.Name, targetFolder));
}
}
else
{
Logging.Logger.Info(string.Format("Local file size: {0}, online file size: {1} - Could not determine last updated, local wins.", info.Length, blockBlob.Properties.Length));
}
}
else
{
Logging.Logger.Info(string.Format("Local file size: {0}, online file size: {1} - Skipping...", info.Length, blockBlob.Properties.Length));
}
}
return currentCount;
}
public void CloudBlockBlobGenerateSASForSnapshot()
{
// Client with shared key access.
CloudBlobClient blobClient = GenerateCloudBlobClient();
CloudBlobContainer container = blobClient.GetContainerReference(GetRandomContainerName());
MemoryStream memoryStream = new MemoryStream();
try
{
container.Create();
CloudBlockBlob blob = container.GetBlockBlobReference("Testing");
blob.UploadFromStream(new MemoryStream(GetRandomBuffer(10)));
SharedAccessBlobPolicy policy = new SharedAccessBlobPolicy()
{
SharedAccessStartTime = DateTimeOffset.UtcNow.AddMinutes(-5),
SharedAccessExpiryTime = DateTimeOffset.UtcNow.AddMinutes(30),
Permissions = SharedAccessBlobPermissions.Read | SharedAccessBlobPermissions.Write,
};
CloudBlockBlob snapshot = blob.CreateSnapshot();
string sas = snapshot.GetSharedAccessSignature(policy);
Assert.IsNotNull(sas);
StorageCredentials credentials = new StorageCredentials(sas);
Uri snapshotUri = snapshot.SnapshotQualifiedUri;
CloudBlockBlob blob1 = new CloudBlockBlob(snapshotUri, credentials);
blob1.DownloadToStream(memoryStream);
Assert.IsNotNull(memoryStream);
}
finally
{
container.DeleteIfExists();
memoryStream.Close();
}
}
private static void GetNetworkSecurityGroupEvents(DateTime logStart, DateTime logEnd)
{
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(BlobStorageConnectionString);
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
Console.WriteLine($"Getting reference to container {EventContainerName}");
CloudBlobContainer container = blobClient.GetContainerReference(EventContainerName);
StorageURL storageUrl = new StorageURL(container.Uri, SubscriptionID, ResrouceGroupsName, ProviderName, ResrouceTypeName, ResourceType.NETWORKSECURITYGROUPS);
List<Log> logs = new List<Log>();
int itemPosition = 0;
for (DateTime logTimeStamp = logStart; logTimeStamp <= logEnd; logTimeStamp = logTimeStamp.AddHours(1))
{
Console.WriteLine(logTimeStamp);
Uri storageBlogUrl = storageUrl.GetURL(logTimeStamp);
CloudBlockBlob blockBlob = new CloudBlockBlob(storageBlogUrl, storageAccount.Credentials);
MemoryStream memstream = new MemoryStream();
try
{
blockBlob.DownloadToStream(memstream);
memstream.Position = 0;
JsonSerializer serializer = new JsonSerializer();
using (StreamReader sr = new StreamReader(memstream))
using (JsonTextReader jsonTextReader = new JsonTextReader(sr))
{
LogRecords logRecords = serializer.Deserialize<LogRecords>(jsonTextReader);
itemPosition = 0;
foreach (Log logItem in logRecords.records)
{
logs.Add(logItem);
itemPosition++;
}
}
}
catch (Exception ex)
{
Console.WriteLine($"{ex.Message} - {storageBlogUrl}");
}
}
using (System.IO.StreamWriter file = new System.IO.StreamWriter(EventCSVExportNamePath))
{
file.WriteLine("time,systemId,resourceId,operationName,properties.vnetResourceGuid,properties.subnetPrefix"
+ ",properties.macAddress,properties.ruleName,properties.direction,properties.priority"
+ ",properties.type,properties.conditions.destinationPortRange,properties.conditions.sourcePortRange"
+ ",properties.conditions.sourceIP,properties.conditions.destinationIP,properties.conditions.protocols");
foreach (Log log in logs)
{
file.WriteLine($"{DateTime.Parse(log.time).ToUniversalTime()}, {log.systemId}, {log.resourceId}, {log.operationName}"
+ $", {log.properties.vnetResourceGuid}, {log.properties.subnetPrefix}, {log.properties.macAddress}"
+ $", {log.properties.ruleName}, {log.properties.direction}, {log.properties.priority}, {log.properties.type}"
+ $", {log.properties.conditions.destinationPortRange}, {log.properties.conditions.sourcePortRange}"
+ $", {log.properties.conditions.sourceIP?.Replace(',', ';')}, {log.properties.conditions.destinationIP?.Replace(',', ';')}"
+ $", {(string.IsNullOrWhiteSpace(log.properties.conditions.protocols) ? "*" : log.properties.conditions.protocols?.Replace(',', ';'))}");
}
}
}
private void InitFormBlob(CloudBlockBlob blob, bool withFileData)
{
if (blob == null || !blob.Exists())
return;
BlobKey = blob.Name;
FileName = Uri.UnescapeDataString(blob.Metadata["FileName"]);
ContentType = Uri.UnescapeDataString(blob.Metadata["ContentType"]);
int contentLength;
if (int.TryParse(Uri.UnescapeDataString(blob.Metadata["ContentLength"]), out contentLength))
ContentLength = contentLength;
BlobFileType type;
if (Enum.TryParse<BlobFileType>(Uri.UnescapeDataString(blob.Metadata["BlobFileType"]), true, out type))
BlobFileType = type;
DateTime dtUploaded;
if (DateTime.TryParse(Uri.UnescapeDataString(blob.Metadata["DtUploaded"]), out dtUploaded))
UploadedDate = dtUploaded;
if (withFileData)
{
Data = new MemoryStream();
blob.DownloadToStream(Data);
Data.Position = 0;
}
}
public void TestEventsHelper(CloudBlockBlob blob, int bufferSize, int expectedSending, int expectedResponseReceived, int expectedRequestComplete, int expectedRetry, bool testRetry)
{
byte[] buffer = GetRandomBuffer(bufferSize);
BlobRequestOptions blobOptions = new BlobRequestOptions();
blobOptions.SingleBlobUploadThresholdInBytes = EventFiringTests.SingleBlobUploadThresholdInBytes;
blobOptions.RetryPolicy = new OldStyleAlwaysRetry(new TimeSpan(1), maxRetries);
int sendingRequestFires = 0;
int responseReceivedFires = 0;
int requestCompleteFires = 0;
int retryFires = 0;
OperationContext.GlobalSendingRequest += (sender, args) => sendingRequestFires++;
OperationContext.GlobalResponseReceived += (sender, args) => responseReceivedFires++;
OperationContext.GlobalRequestCompleted += (sender, args) => requestCompleteFires++;
OperationContext.GlobalRetrying += (sender, args) => retryFires++;
if (testRetry)
{
using (MemoryStream wholeBlob = new MemoryStream())
{
try
{
blob.DownloadToStream(wholeBlob, null, blobOptions, null);
}
catch (StorageException e)
{
Assert.AreEqual(404, e.RequestInformation.HttpStatusCode);
}
}
}
else
{
using (MemoryStream wholeBlob = new MemoryStream(buffer))
{
blob.UploadFromStream(wholeBlob, null, blobOptions, null);
}
}
Assert.AreEqual(expectedSending, sendingRequestFires);
Assert.AreEqual(expectedResponseReceived, responseReceivedFires);
Assert.AreEqual(expectedRequestComplete, requestCompleteFires);
Assert.AreEqual(expectedRetry, retryFires);
}
static void UseBlobSAS(string sas)
{
//Try performing blob operations using the SAS provided.
//Return a reference to the blob using the SAS URI.
CloudBlockBlob blob = new CloudBlockBlob(new Uri(sas));
//Write operation: Write a new blob to the container.
try
{
string blobContent = "This blob was created with a shared access signature granting write permissions to the blob. ";
MemoryStream msWrite = new MemoryStream(Encoding.UTF8.GetBytes(blobContent));
msWrite.Position = 0;
using (msWrite)
{
blob.UploadFromStream(msWrite);
}
Console.WriteLine("Write operation succeeded for SAS " + sas);
Console.WriteLine();
}
catch (StorageException e)
{
Console.WriteLine("Write operation failed for SAS " + sas);
Console.WriteLine("Additional error information: " + e.Message);
Console.WriteLine();
}
//Read operation: Read the contents of the blob.
try
{
MemoryStream msRead = new MemoryStream();
using (msRead)
{
blob.DownloadToStream(msRead);
msRead.Position = 0;
using (StreamReader reader = new StreamReader(msRead, true))
{
string line;
while ((line = reader.ReadLine()) != null)
{
Console.WriteLine(line);
}
}
}
Console.WriteLine("Read operation succeeded for SAS " + sas);
Console.WriteLine();
}
catch (StorageException e)
{
Console.WriteLine("Read operation failed for SAS " + sas);
Console.WriteLine("Additional error information: " + e.Message);
Console.WriteLine();
}
//Delete operation: Delete the blob.
try
{
blob.Delete();
Console.WriteLine("Delete operation succeeded for SAS " + sas);
Console.WriteLine();
}
catch (StorageException e)
{
Console.WriteLine("Delete operation failed for SAS " + sas);
Console.WriteLine("Additional error information: " + e.Message);
Console.WriteLine();
}
}
