public static void TaskMain(string[] args) { if (args == null || args.Length != 5) { throw new Exception("Usage: TopNWordsSample.exe --Task <blobpath> <numtopwords> <storageAccountName> <storageAccountKey>"); } string blobName = args[1]; int numTopN = int.Parse(args[2]); string storageAccountName = args[3]; string storageAccountKey = args[4]; // open the cloud blob that contains the book var storageCred = new StorageCredentials(storageAccountName, storageAccountKey); CloudBlockBlob blob = new CloudBlockBlob(new Uri(blobName), storageCred); using (Stream memoryStream = new MemoryStream()) { blob.DownloadToStream(memoryStream); memoryStream.Position = 0; //Reset the stream var sr = new StreamReader(memoryStream); var myStr = sr.ReadToEnd(); string[] words = myStr.Split(' '); var topNWords = words. Where(word => word.Length > 0). GroupBy(word => word, (key, group) => new KeyValuePair<String, long>(key, group.LongCount())). OrderByDescending(x => x.Value). Take(numTopN). ToList(); foreach (var pair in topNWords) { Console.WriteLine("{0} {1}", pair.Key, pair.Value); } } }
private void init(CloudBlockBlob blob) { if (blob == null) throw new ArgumentNullException("blob"); _server_ref = blob; Stream = new MemoryStream(); _server_ref.DownloadToStream(Stream); Stream.Seek(0, SeekOrigin.Begin); //Automatic Fetches Attributes on DownloadToStream. ETag = _server_ref.Properties.ETag; Length = _server_ref.Properties.Length; }
private static string DownloadBlob(CloudBlockBlob blob) { using (var stream = new MemoryStream()) { StreamReader reader; try { blob.DownloadToStream(stream, options: new BlobRequestOptions() { RetryPolicy = new LinearRetry(TimeSpan.FromSeconds(5), 3) }); } catch (StorageException se) { return ""; } try { stream.Seek(0, 0); reader = new StreamReader(new GZipStream(stream, CompressionMode.Decompress)); return reader.ReadToEnd(); } catch { stream.Seek(0, 0); reader = new StreamReader(stream); return reader.ReadToEnd(); } } }
private void DownloadIfExists(CloudBlockBlob blob) { string target = Path.Combine(Root, blob.Name); if (blob.Exists()) { if (!Directory.Exists(Root)) { Directory.CreateDirectory(Root); } using (var file = File.OpenWrite(target)) { blob.DownloadToStream(file); } } }
protected override void OnExecuting(CloudBlockBlob workItem) { if (workItem == null) throw new ArgumentNullException("workItem"); if (workItem.Metadata.ContainsKey(CompressedFlag)) return; using (var blobStream = new MemoryStream()) { workItem.DownloadToStream(blobStream); using (var compressedStream = new MemoryStream()) { CompressStream(compressedStream, blobStream); SetCompressedFlag(workItem); workItem.UploadFromStream(compressedStream); } } }
public MemoryStream DownloadBlob(CloudBlockBlob blob) { var memorystream = new MemoryStream { Position = 0 }; blob.DownloadToStream(memorystream); return memorystream; }
/// <summary> /// Downloads the BLOB. /// </summary> /// <param name="blob">The BLOB.</param> /// <param name="fetchAttribute">if set to <c>true</c> [fetch attribute].</param> /// <param name="metaData">The meta data.</param> /// <returns>System.Byte[].</returns> public static byte[] DownloadBlob(CloudBlockBlob blob, bool fetchAttribute, out IDictionary<string, string> metaData) { try { blob.CheckNullObject("blob"); if (fetchAttribute) { blob.FetchAttributes(); metaData = blob.Metadata; } else { metaData = null; } var msRead = new MemoryStream { Position = 0 }; using (msRead) { blob.DownloadToStream(msRead); return msRead.ToBytes(); } } catch (Exception ex) { throw ex.Handle("DownloadBlob", blob.Uri.ToString()); } }
public void UseTransactionalMD5GetTest() { BlobRequestOptions optionsWithNoMD5 = new BlobRequestOptions() { UseTransactionalMD5 = false, }; BlobRequestOptions optionsWithMD5 = new BlobRequestOptions() { UseTransactionalMD5 = true, }; byte[] buffer = GetRandomBuffer(3 * 1024 * 1024); MD5 hasher = MD5.Create(); string md5 = Convert.ToBase64String(hasher.ComputeHash(buffer)); string lastCheckMD5 = null; int checkCount = 0; OperationContext opContextWithMD5Check = new OperationContext(); opContextWithMD5Check.ResponseReceived += (_, args) => { if (args.Response.ContentLength >= buffer.Length) { lastCheckMD5 = args.Response.Headers[HttpResponseHeader.ContentMd5]; checkCount++; } }; CloudBlobContainer container = GetRandomContainerReference(); try { container.Create(); CloudBlockBlob blockBlob = container.GetBlockBlobReference("blob1"); using (Stream blobStream = blockBlob.OpenWrite()) { blobStream.Write(buffer, 0, buffer.Length); blobStream.Write(buffer, 0, buffer.Length); } checkCount = 0; using (Stream stream = new MemoryStream()) { blockBlob.DownloadToStream(stream, null, optionsWithNoMD5, opContextWithMD5Check); Assert.IsNotNull(lastCheckMD5); blockBlob.DownloadToStream(stream, null, optionsWithMD5, opContextWithMD5Check); Assert.IsNotNull(lastCheckMD5); blockBlob.DownloadRangeToStream(stream, buffer.Length, buffer.Length, null, optionsWithNoMD5, opContextWithMD5Check); Assert.IsNull(lastCheckMD5); blockBlob.DownloadRangeToStream(stream, buffer.Length, buffer.Length, null, optionsWithMD5, opContextWithMD5Check); Assert.AreEqual(md5, lastCheckMD5); blockBlob.DownloadRangeToStream(stream, 1024, 4 * 1024 * 1024 + 1, null, optionsWithNoMD5, opContextWithMD5Check); Assert.IsNull(lastCheckMD5); StorageException storageEx = TestHelper.ExpectedException <StorageException>( () => blockBlob.DownloadRangeToStream(stream, 1024, 4 * 1024 * 1024 + 1, null, optionsWithMD5, opContextWithMD5Check), "Downloading more than 4MB with transactional MD5 should not be supported"); Assert.IsInstanceOfType(storageEx.InnerException, typeof(ArgumentOutOfRangeException)); } Assert.AreEqual(5, checkCount); CloudPageBlob pageBlob = container.GetPageBlobReference("blob2"); using (Stream blobStream = pageBlob.OpenWrite(buffer.Length * 2)) { blobStream.Write(buffer, 0, buffer.Length); blobStream.Write(buffer, 0, buffer.Length); } checkCount = 0; using (Stream stream = new MemoryStream()) { pageBlob.DownloadToStream(stream, null, optionsWithNoMD5, opContextWithMD5Check); Assert.IsNull(lastCheckMD5); StorageException storageEx = TestHelper.ExpectedException <StorageException>( () => pageBlob.DownloadToStream(stream, null, optionsWithMD5, opContextWithMD5Check), "Page blob will not have MD5 set by default; with UseTransactional, download should fail"); pageBlob.DownloadRangeToStream(stream, buffer.Length, buffer.Length, null, optionsWithNoMD5, opContextWithMD5Check); Assert.IsNull(lastCheckMD5); pageBlob.DownloadRangeToStream(stream, buffer.Length, buffer.Length, null, optionsWithMD5, opContextWithMD5Check); Assert.AreEqual(md5, lastCheckMD5); pageBlob.DownloadRangeToStream(stream, 1024, 4 * 1024 * 1024 + 1, null, optionsWithNoMD5, opContextWithMD5Check); Assert.IsNull(lastCheckMD5); storageEx = TestHelper.ExpectedException <StorageException>( () => pageBlob.DownloadRangeToStream(stream, 1024, 4 * 1024 * 1024 + 1, null, optionsWithMD5, opContextWithMD5Check), "Downloading more than 4MB with transactional MD5 should not be supported"); Assert.IsInstanceOfType(storageEx.InnerException, typeof(ArgumentOutOfRangeException)); } Assert.AreEqual(5, checkCount); } finally { container.DeleteIfExists(); } }
public void BlobIngressEgressCounters() { CloudBlobContainer container = GetRandomContainerReference(); container.CreateIfNotExists(); CloudBlockBlob blob = container.GetBlockBlobReference("blob1"); string[] blockIds = new string[] { Convert.ToBase64String(Guid.NewGuid().ToByteArray()), Convert.ToBase64String(Guid.NewGuid().ToByteArray()), Convert.ToBase64String(Guid.NewGuid().ToByteArray()) }; try { // 1 byte TestHelper.ValidateIngressEgress(Selectors.IfUrlContains(blob.Uri.ToString()), () => { OperationContext opContext = new OperationContext(); blob.PutBlock(blockIds[0], new MemoryStream(GetRandomBuffer(1)), null, null, new BlobRequestOptions() { RetryPolicy = new RetryPolicies.NoRetry() }, opContext); return(opContext.LastResult); }); // 1024 TestHelper.ValidateIngressEgress(Selectors.IfUrlContains(blob.Uri.ToString()), () => { OperationContext opContext = new OperationContext(); blob.PutBlock(blockIds[1], new MemoryStream(GetRandomBuffer(1024)), null, null, new BlobRequestOptions() { RetryPolicy = new RetryPolicies.NoRetry() }, opContext); return(opContext.LastResult); }); // 98765 TestHelper.ValidateIngressEgress(Selectors.IfUrlContains(blob.Uri.ToString()), () => { OperationContext opContext = new OperationContext(); blob.PutBlock(blockIds[2], new MemoryStream(GetRandomBuffer(98765)), null, null, new BlobRequestOptions() { RetryPolicy = new RetryPolicies.NoRetry() }, opContext); return(opContext.LastResult); }); // PutBlockList TestHelper.ValidateIngressEgress(Selectors.IfUrlContains(blob.Uri.ToString()), () => { OperationContext opContext = new OperationContext(); blob.PutBlockList(blockIds, null, new BlobRequestOptions() { RetryPolicy = new RetryPolicies.NoRetry() }, opContext); return(opContext.LastResult); }); // GetBlockList TestHelper.ValidateIngressEgress(Selectors.IfUrlContains(blob.Uri.ToString()), () => { OperationContext opContext = new OperationContext(); blob.DownloadBlockList(BlockListingFilter.All, null, new BlobRequestOptions() { RetryPolicy = new RetryPolicies.NoRetry() }, opContext); return(opContext.LastResult); }); // Download TestHelper.ValidateIngressEgress(Selectors.IfUrlContains(blob.Uri.ToString()), () => { OperationContext opContext = new OperationContext(); blob.DownloadToStream(Stream.Null, null, new BlobRequestOptions() { RetryPolicy = new RetryPolicies.NoRetry() }, opContext); return(opContext.LastResult); }); Assert.AreEqual(blob.Properties.Length, 98765 + 1024 + 1); // Error Case CloudBlockBlob nullBlob = container.GetBlockBlobReference("null"); OperationContext errorContext = new OperationContext(); try { nullBlob.DownloadToStream(Stream.Null, null, new BlobRequestOptions() { RetryPolicy = new RetryPolicies.NoRetry() }, errorContext); Assert.Fail("Null blob, null stream, no download possible."); } catch (StorageException) { Assert.IsTrue(errorContext.LastResult.IngressBytes > 0); } } finally { container.DeleteIfExists(); } }
private void DownloadBlobInParallel(CloudBlockBlob blob, Stream stream) { try { blob.FetchAttributes(); blob.ServiceClient.ParallelOperationThreadCount = NumberOfIOThreads; blob.DownloadToStream(stream); stream.Seek(0, SeekOrigin.Begin); } catch (StorageException ex) { logger.Warn(ex.Message); } }
public void DisableContentMD5ValidationTest() { byte[] buffer = new byte[1024]; Random random = new Random(); random.NextBytes(buffer); BlobRequestOptions optionsWithNoMD5 = new BlobRequestOptions() { DisableContentMD5Validation = true, StoreBlobContentMD5 = true, }; BlobRequestOptions optionsWithMD5 = new BlobRequestOptions() { DisableContentMD5Validation = false, StoreBlobContentMD5 = true, }; CloudBlobContainer container = GetRandomContainerReference(); try { container.Create(); CloudBlockBlob blockBlob = container.GetBlockBlobReference("blob1"); using (Stream stream = new NonSeekableMemoryStream(buffer)) { blockBlob.UploadFromStream(stream, null, optionsWithMD5); } using (Stream stream = new MemoryStream()) { blockBlob.DownloadToStream(stream, null, optionsWithMD5); blockBlob.DownloadToStream(stream, null, optionsWithNoMD5); using (Stream blobStream = blockBlob.OpenRead(null, optionsWithMD5)) { int read; do { read = blobStream.Read(buffer, 0, buffer.Length); }while (read > 0); } using (Stream blobStream = blockBlob.OpenRead(null, optionsWithNoMD5)) { int read; do { read = blobStream.Read(buffer, 0, buffer.Length); }while (read > 0); } blockBlob.Properties.ContentMD5 = "MDAwMDAwMDA="; blockBlob.SetProperties(); TestHelper.ExpectedException( () => blockBlob.DownloadToStream(stream, null, optionsWithMD5), "Downloading a blob with invalid MD5 should fail", HttpStatusCode.OK); blockBlob.DownloadToStream(stream, null, optionsWithNoMD5); using (Stream blobStream = blockBlob.OpenRead(null, optionsWithMD5)) { TestHelper.ExpectedException <IOException>( () => { int read; do { read = blobStream.Read(buffer, 0, buffer.Length); }while (read > 0); }, "Downloading a blob with invalid MD5 should fail"); } using (Stream blobStream = blockBlob.OpenRead(null, optionsWithNoMD5)) { int read; do { read = blobStream.Read(buffer, 0, buffer.Length); }while (read > 0); } } CloudPageBlob pageBlob = container.GetPageBlobReference("blob2"); using (Stream stream = new MemoryStream(buffer)) { pageBlob.UploadFromStream(stream, null, optionsWithMD5); } using (Stream stream = new MemoryStream()) { pageBlob.DownloadToStream(stream, null, optionsWithMD5); pageBlob.DownloadToStream(stream, null, optionsWithNoMD5); using (Stream blobStream = pageBlob.OpenRead(null, optionsWithMD5)) { int read; do { read = blobStream.Read(buffer, 0, buffer.Length); }while (read > 0); } using (Stream blobStream = pageBlob.OpenRead(null, optionsWithNoMD5)) { int read; do { read = blobStream.Read(buffer, 0, buffer.Length); }while (read > 0); } pageBlob.Properties.ContentMD5 = "MDAwMDAwMDA="; pageBlob.SetProperties(); TestHelper.ExpectedException( () => pageBlob.DownloadToStream(stream, null, optionsWithMD5), "Downloading a blob with invalid MD5 should fail", HttpStatusCode.OK); pageBlob.DownloadToStream(stream, null, optionsWithNoMD5); using (Stream blobStream = pageBlob.OpenRead(null, optionsWithMD5)) { TestHelper.ExpectedException <IOException>( () => { int read; do { read = blobStream.Read(buffer, 0, buffer.Length); }while (read > 0); }, "Downloading a blob with invalid MD5 should fail"); } using (Stream blobStream = pageBlob.OpenRead(null, optionsWithNoMD5)) { int read; do { read = blobStream.Read(buffer, 0, buffer.Length); }while (read > 0); } } } finally { container.DeleteIfExists(); } }
void DownloadBlobInParallel(CloudBlockBlob blob, Stream stream) { blob.FetchAttributes(); blob.ServiceClient.ParallelOperationThreadCount = NumberOfIOThreads; container.ServiceClient.RetryPolicy = new ExponentialRetry(TimeSpan.FromSeconds(BackOffInterval), MaxRetries); blob.DownloadToStream(stream); stream.Seek(0, SeekOrigin.Begin); }
/// <summary> /// Uploads Memory Stream to Blob /// </summary> /// <param name="outputBlob"></param> /// <param name="line"></param> private static void UploadBlobStream(CloudBlockBlob outputBlob, string line) { using (MemoryStream ms = new MemoryStream()) { if (outputBlob.Exists()) { outputBlob.DownloadToStream(ms); } byte[] dataToWrite = Encoding.UTF8.GetBytes(line + "\r\n"); ms.Write(dataToWrite, 0, dataToWrite.Length); ms.Position = 0; outputBlob.UploadFromStream(ms); } }
/// <summary> /// Down loads JSON log files between 2 dates from the blob storage /// and exports them into one .CSV file. /// </summary> /// <param name="logStart">Begin date and time of the log.</param> /// <param name="logEnd">End date and time of the log.</param> private static void GetNetworkSecurityGroupRuleCounters(DateTime logStart, DateTime logEnd) { // Creates client. CloudStorageAccount storageAccount = CloudStorageAccount.Parse(BlobStorageConnectionString); CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); Console.WriteLine($"Getting reference to container {CounterContainerName}"); CloudBlobContainer container = blobClient.GetContainerReference(CounterContainerName); // Instantiate the URL generator. StorageURL storageUrl = new StorageURL(container.Uri, SubscriptionID, ResrouceGroupsName, ProviderName, ResrouceTypeName); List<Log> logs = new List<Log>(); int itemPosition = 0; // Using the date and time as arguments download all logs from the storage blob. for (DateTime logTimeStamp = logStart; logTimeStamp <= logEnd; logTimeStamp = logTimeStamp.AddHours(1)) { Console.WriteLine(logTimeStamp); Uri storageBlogUrl = storageUrl.GetURL(logTimeStamp); CloudBlockBlob blockBlob = new CloudBlockBlob(storageBlogUrl, storageAccount.Credentials); MemoryStream memstream = new MemoryStream(); try { blockBlob.DownloadToStream(memstream); memstream.Position = 0; JsonSerializer serializer = new JsonSerializer(); using (StreamReader sr = new StreamReader(memstream)) using (JsonTextReader jsonTextReader = new JsonTextReader(sr)) { // Deserialize JSON. LogRecords logRecords = serializer.Deserialize<LogRecords>(jsonTextReader); itemPosition = 0; foreach (Log logItem in logRecords.records) { // Add deserialized logs. logs.Add(logItem); itemPosition++; } } } catch (Exception ex) { Console.WriteLine($"{ex.Message} - {storageBlogUrl}"); } } // Dump everything in the logs list into a file. using (StreamWriter file = new StreamWriter(CounterCSVExportNamePath)) { file.WriteLine("time,systemId,resourceId,operationName,properties.vnetResourceGuid,properties.subnetPrefix" + ",properties.macAddress,properties.ruleName,properties.direction,properties.type,properties.matchedConnections"); foreach (Log log in logs) { file.WriteLine($"{DateTime.Parse(log.time).ToUniversalTime()}, {log.systemId}, {log.resourceId}, {log.operationName}" + $", {log.properties.vnetResourceGuid}, {log.properties.subnetPrefix}, {log.properties.macAddress}" + $", {log.properties.ruleName}, {log.properties.direction}, {log.properties.type}, {log.properties.matchedConnections}"); } } }
public static void TaskMain(string[] args) { if (args == null || args.Length != 4) { throw new Exception("Usage: ImageBlur.exe --Task <blobpath> <storageAccountName> <storageAccountKey>"); } string blobName = args[1]; string storageAccountName = args[2]; string storageAccountKey = args[3]; string workingDirectory = Environment.GetEnvironmentVariable("AZ_BATCH_TASK_WORKING_DIR"); int numberToBlur = 3; Console.WriteLine(); Console.WriteLine(" blobName: <{0}>", blobName); Console.WriteLine(" storageAccountName: <{0}>", storageAccountName); Console.WriteLine(" number to blur: <{0}>", numberToBlur); Console.WriteLine(); // get source image from cloud blob var storageCred = new StorageCredentials(storageAccountName, storageAccountKey); CloudBlockBlob blob = new CloudBlockBlob(new Uri(blobName), storageCred); using (MemoryStream inStream = new MemoryStream()) { blob.DownloadToStream(inStream); Image img = Image.FromStream(inStream); int imgWidth = img.Width; int imgHeight = img.Height; Size size = new Size(imgWidth, imgHeight); ISupportedImageFormat format = new JpegFormat { Quality = 70 }; // Print image properties to stdout Console.WriteLine(" Image Properties:"); Console.WriteLine(" Format: {0}", FormatUtilities.GetFormat(inStream)); Console.WriteLine(" Size: {0}", size.ToString()); Console.WriteLine(); for (var i = 0; i < numberToBlur; i++) { using (ImageFactory imageFactory = new ImageFactory(preserveExifData: true)) { int blurIndex = (i * 5) + 10; imageFactory.Load(inStream); imageFactory.Resize(size) .Format(format) .GaussianBlur(blurIndex) .Save(workingDirectory + "/resultimage" + i + ".Jpeg"); //.Save(@"C:/Users/jiata/Desktop/imageblur/results/resultimage" + i + ".Jpeg"); } } } // TODO - not working for (var i = 0; i < numberToBlur; i++) { blob.UploadFromFile(workingDirectory + "/resultimage" + i + ".Jpeg", FileMode.Open); } Environment.Exit(1); }
private static void GetLoadBalancerEvents(DateTime logStart, DateTime logEnd) { CloudStorageAccount storageAccount = CloudStorageAccount.Parse(BlobStorageConnectionString); CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); Console.WriteLine($"Getting reference to container {EventContainerName}"); CloudBlobContainer container = blobClient.GetContainerReference(EventContainerName); StorageURL storageUrl = new StorageURL(container.Uri, SubscriptionID, ResrouceGroupsName, ProviderName, ResrouceTypeName, ResourceType.LOADBALANCERS); List<Log> logs = new List<Log>(); int itemPosition = 0; // Using the date and time as arguments download all logs from the storage blob. for (DateTime logTimeStamp = logStart; logTimeStamp <= logEnd; logTimeStamp = logTimeStamp.AddHours(1)) { Console.WriteLine(logTimeStamp); Uri storageBlobUrl = storageUrl.GetURL(logTimeStamp); CloudBlockBlob blockBlob = new CloudBlockBlob(storageBlobUrl, storageAccount.Credentials); MemoryStream memstream = new MemoryStream(); try { blockBlob.DownloadToStream(memstream); memstream.Position = 0; JsonSerializer serializer = new JsonSerializer(); using (StreamReader sr = new StreamReader(memstream)) { using (JsonTextReader jsonTextReader = new JsonTextReader(sr)) { LogRecords logRecords = serializer.Deserialize<LogRecords>(jsonTextReader); itemPosition = 0; foreach (Log logItem in logRecords.records) { logs.Add(logItem); itemPosition++; } } } } catch (Exception ex) { Console.WriteLine($"{ex.Message} - {storageBlobUrl}"); } } using (System.IO.StreamWriter file = new System.IO.StreamWriter(EventCSVExportNamePath)) { file.WriteLine("time,systemId,category,resourceId,operationName,properties.publicIpAddress" + ",properties.port,properties.totalDipCount,properties.dipDownCount,properties.healthPercentage"); foreach (Log log in logs) { file.WriteLine($"{DateTime.Parse(log.time).ToUniversalTime()}, {log.systemId}, {log.category}, {log.resourceId}, {log.operationName}" + $", {log.properties.publicIpAddress }, {log.properties.port}, {log.properties.totalDipCount}" + $", {log.properties.dipDownCount}, {log.properties.healthPercentage}"); } } }
public Image blobThumbnail(string filename, string container) { try { blob = blobClient.GetContainerReference(container); blob.CreateIfNotExists(); blockBlob = blob.GetBlockBlobReference(@"thumb/" + filename); using (var outStream = new System.IO.MemoryStream()) { blockBlob.DownloadToStream(outStream); return Image.FromStream(outStream); } } catch (Exception e) { Console.WriteLine(e.InnerException); return null; } }
private static void DownloadToStream(Stream target, CloudBlockBlob blob) { try { blob.DownloadToStream(target); } catch (StorageException e) { throw new BlobException(string.Format(CultureInfo.InvariantCulture, "Could not DownloadToStream from blob named '{0}' in container. See inner exception for details.", blob.Name), e); } }
private int DownloadBlob(CloudBlockBlob blockBlob, string targetFolder, int currentCount) { string decodedBlockBlobName = System.Net.WebUtility.HtmlDecode(blockBlob.Name).TrimStart(Path.DirectorySeparatorChar); string targetFileName = string.Format("{0}{1}{2}", targetFolder, Path.DirectorySeparatorChar.ToString(), decodedBlockBlobName); Logging.Logger.Info(string.Format("Target Folder: {0} Target file: {1} ", targetFolder, targetFileName)); string tempName = Path.Combine(Path.Combine(targetFolder, Guid.NewGuid().ToString())); if (!File.Exists(targetFileName)) { Logging.Logger.Info(string.Format("Local file {0} doesn't exist, download size {1}.", targetFileName, blockBlob.Properties.Length)); currentCount++; if (!SimulationMode) { var outFolder = Path.GetDirectoryName(targetFileName); if(!Directory.Exists(outFolder)){ Directory.CreateDirectory(outFolder); } using (var fileStream = System.IO.File.OpenWrite(targetFileName)) { blockBlob.DownloadToStream(fileStream); } File.SetCreationTime(targetFileName, blockBlob.Properties.LastModified.Value.DateTime); } else { Logging.Logger.Info(string.Format("Download {0} to {1}", blockBlob.Name, targetFolder)); } } else { FileInfo info = new FileInfo(targetFileName); if (blockBlob.Properties.Length != info.Length) { if (blockBlob.Properties.LastModified.HasValue && blockBlob.Properties.LastModified.Value.DateTime > info.LastWriteTime) { Logging.Logger.Info(string.Format("Local file size: {0}, online file size: {1} Blob Updated: {2} FileUpdated: {3} Downloading...", info.Length, blockBlob.Properties.Length, blockBlob.Properties.LastModified.Value.DateTime, info.LastWriteTime)); currentCount++; if (!SimulationMode) { using (var fileStream = System.IO.File.OpenWrite(tempName)) { blockBlob.DownloadToStream(fileStream); } File.Delete(targetFileName); File.Move(tempName, targetFileName); File.SetCreationTime(targetFileName, blockBlob.Properties.LastModified.Value.DateTime); } else { Logging.Logger.Info(string.Format("Download {0} to {1}", blockBlob.Name, targetFolder)); } } else { Logging.Logger.Info(string.Format("Local file size: {0}, online file size: {1} - Could not determine last updated, local wins.", info.Length, blockBlob.Properties.Length)); } } else { Logging.Logger.Info(string.Format("Local file size: {0}, online file size: {1} - Skipping...", info.Length, blockBlob.Properties.Length)); } } return currentCount; }
public void CloudBlockBlobGenerateSASForSnapshot() { // Client with shared key access. CloudBlobClient blobClient = GenerateCloudBlobClient(); CloudBlobContainer container = blobClient.GetContainerReference(GetRandomContainerName()); MemoryStream memoryStream = new MemoryStream(); try { container.Create(); CloudBlockBlob blob = container.GetBlockBlobReference("Testing"); blob.UploadFromStream(new MemoryStream(GetRandomBuffer(10))); SharedAccessBlobPolicy policy = new SharedAccessBlobPolicy() { SharedAccessStartTime = DateTimeOffset.UtcNow.AddMinutes(-5), SharedAccessExpiryTime = DateTimeOffset.UtcNow.AddMinutes(30), Permissions = SharedAccessBlobPermissions.Read | SharedAccessBlobPermissions.Write, }; CloudBlockBlob snapshot = blob.CreateSnapshot(); string sas = snapshot.GetSharedAccessSignature(policy); Assert.IsNotNull(sas); StorageCredentials credentials = new StorageCredentials(sas); Uri snapshotUri = snapshot.SnapshotQualifiedUri; CloudBlockBlob blob1 = new CloudBlockBlob(snapshotUri, credentials); blob1.DownloadToStream(memoryStream); Assert.IsNotNull(memoryStream); } finally { container.DeleteIfExists(); memoryStream.Close(); } }
private static void GetNetworkSecurityGroupEvents(DateTime logStart, DateTime logEnd) { CloudStorageAccount storageAccount = CloudStorageAccount.Parse(BlobStorageConnectionString); CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); Console.WriteLine($"Getting reference to container {EventContainerName}"); CloudBlobContainer container = blobClient.GetContainerReference(EventContainerName); StorageURL storageUrl = new StorageURL(container.Uri, SubscriptionID, ResrouceGroupsName, ProviderName, ResrouceTypeName, ResourceType.NETWORKSECURITYGROUPS); List<Log> logs = new List<Log>(); int itemPosition = 0; for (DateTime logTimeStamp = logStart; logTimeStamp <= logEnd; logTimeStamp = logTimeStamp.AddHours(1)) { Console.WriteLine(logTimeStamp); Uri storageBlogUrl = storageUrl.GetURL(logTimeStamp); CloudBlockBlob blockBlob = new CloudBlockBlob(storageBlogUrl, storageAccount.Credentials); MemoryStream memstream = new MemoryStream(); try { blockBlob.DownloadToStream(memstream); memstream.Position = 0; JsonSerializer serializer = new JsonSerializer(); using (StreamReader sr = new StreamReader(memstream)) using (JsonTextReader jsonTextReader = new JsonTextReader(sr)) { LogRecords logRecords = serializer.Deserialize<LogRecords>(jsonTextReader); itemPosition = 0; foreach (Log logItem in logRecords.records) { logs.Add(logItem); itemPosition++; } } } catch (Exception ex) { Console.WriteLine($"{ex.Message} - {storageBlogUrl}"); } } using (System.IO.StreamWriter file = new System.IO.StreamWriter(EventCSVExportNamePath)) { file.WriteLine("time,systemId,resourceId,operationName,properties.vnetResourceGuid,properties.subnetPrefix" + ",properties.macAddress,properties.ruleName,properties.direction,properties.priority" + ",properties.type,properties.conditions.destinationPortRange,properties.conditions.sourcePortRange" + ",properties.conditions.sourceIP,properties.conditions.destinationIP,properties.conditions.protocols"); foreach (Log log in logs) { file.WriteLine($"{DateTime.Parse(log.time).ToUniversalTime()}, {log.systemId}, {log.resourceId}, {log.operationName}" + $", {log.properties.vnetResourceGuid}, {log.properties.subnetPrefix}, {log.properties.macAddress}" + $", {log.properties.ruleName}, {log.properties.direction}, {log.properties.priority}, {log.properties.type}" + $", {log.properties.conditions.destinationPortRange}, {log.properties.conditions.sourcePortRange}" + $", {log.properties.conditions.sourceIP?.Replace(',', ';')}, {log.properties.conditions.destinationIP?.Replace(',', ';')}" + $", {(string.IsNullOrWhiteSpace(log.properties.conditions.protocols) ? "*" : log.properties.conditions.protocols?.Replace(',', ';'))}"); } } }
private void InitFormBlob(CloudBlockBlob blob, bool withFileData) { if (blob == null || !blob.Exists()) return; BlobKey = blob.Name; FileName = Uri.UnescapeDataString(blob.Metadata["FileName"]); ContentType = Uri.UnescapeDataString(blob.Metadata["ContentType"]); int contentLength; if (int.TryParse(Uri.UnescapeDataString(blob.Metadata["ContentLength"]), out contentLength)) ContentLength = contentLength; BlobFileType type; if (Enum.TryParse<BlobFileType>(Uri.UnescapeDataString(blob.Metadata["BlobFileType"]), true, out type)) BlobFileType = type; DateTime dtUploaded; if (DateTime.TryParse(Uri.UnescapeDataString(blob.Metadata["DtUploaded"]), out dtUploaded)) UploadedDate = dtUploaded; if (withFileData) { Data = new MemoryStream(); blob.DownloadToStream(Data); Data.Position = 0; } }
public void TestEventsHelper(CloudBlockBlob blob, int bufferSize, int expectedSending, int expectedResponseReceived, int expectedRequestComplete, int expectedRetry, bool testRetry) { byte[] buffer = GetRandomBuffer(bufferSize); BlobRequestOptions blobOptions = new BlobRequestOptions(); blobOptions.SingleBlobUploadThresholdInBytes = EventFiringTests.SingleBlobUploadThresholdInBytes; blobOptions.RetryPolicy = new OldStyleAlwaysRetry(new TimeSpan(1), maxRetries); int sendingRequestFires = 0; int responseReceivedFires = 0; int requestCompleteFires = 0; int retryFires = 0; OperationContext.GlobalSendingRequest += (sender, args) => sendingRequestFires++; OperationContext.GlobalResponseReceived += (sender, args) => responseReceivedFires++; OperationContext.GlobalRequestCompleted += (sender, args) => requestCompleteFires++; OperationContext.GlobalRetrying += (sender, args) => retryFires++; if (testRetry) { using (MemoryStream wholeBlob = new MemoryStream()) { try { blob.DownloadToStream(wholeBlob, null, blobOptions, null); } catch (StorageException e) { Assert.AreEqual(404, e.RequestInformation.HttpStatusCode); } } } else { using (MemoryStream wholeBlob = new MemoryStream(buffer)) { blob.UploadFromStream(wholeBlob, null, blobOptions, null); } } Assert.AreEqual(expectedSending, sendingRequestFires); Assert.AreEqual(expectedResponseReceived, responseReceivedFires); Assert.AreEqual(expectedRequestComplete, requestCompleteFires); Assert.AreEqual(expectedRetry, retryFires); }
static void UseBlobSAS(string sas) { //Try performing blob operations using the SAS provided. //Return a reference to the blob using the SAS URI. CloudBlockBlob blob = new CloudBlockBlob(new Uri(sas)); //Write operation: Write a new blob to the container. try { string blobContent = "This blob was created with a shared access signature granting write permissions to the blob. "; MemoryStream msWrite = new MemoryStream(Encoding.UTF8.GetBytes(blobContent)); msWrite.Position = 0; using (msWrite) { blob.UploadFromStream(msWrite); } Console.WriteLine("Write operation succeeded for SAS " + sas); Console.WriteLine(); } catch (StorageException e) { Console.WriteLine("Write operation failed for SAS " + sas); Console.WriteLine("Additional error information: " + e.Message); Console.WriteLine(); } //Read operation: Read the contents of the blob. try { MemoryStream msRead = new MemoryStream(); using (msRead) { blob.DownloadToStream(msRead); msRead.Position = 0; using (StreamReader reader = new StreamReader(msRead, true)) { string line; while ((line = reader.ReadLine()) != null) { Console.WriteLine(line); } } } Console.WriteLine("Read operation succeeded for SAS " + sas); Console.WriteLine(); } catch (StorageException e) { Console.WriteLine("Read operation failed for SAS " + sas); Console.WriteLine("Additional error information: " + e.Message); Console.WriteLine(); } //Delete operation: Delete the blob. try { blob.Delete(); Console.WriteLine("Delete operation succeeded for SAS " + sas); Console.WriteLine(); } catch (StorageException e) { Console.WriteLine("Delete operation failed for SAS " + sas); Console.WriteLine("Additional error information: " + e.Message); Console.WriteLine(); } }