/// <summary> /// Taken from: http://blogs.msdn.com/b/windowsazurestorage/archive/2012/06/12/introducing-asynchronous-cross-account-copy-blob.aspx /// </summary> /// <param name="destContainer">The container to monitor</param> public static void MonitorCopy(CloudBlobContainer destContainer) { var pendingCopy = true; while (pendingCopy) { var destBlobList = destContainer.ListBlobs(true, BlobListingDetails.Copy); foreach (var destBlob in destBlobList.Select(dest => dest as CloudBlob)) { if (destBlob.CopyState == null) { Debug.WriteLine("BlobStorage.MonitorCopy: CopyState is null. Small sleep, then we assume it's done!"); Thread.Sleep(5000); return; } switch (destBlob.CopyState.Status) { case CopyStatus.Failed: case CopyStatus.Aborted: Debug.WriteLine("BlobStorage.MonitorCopy: Copy Failed or Aborted; restarting copy"); destBlob.StartCopyFromBlob(destBlob.CopyState.Source); break; case CopyStatus.Success: pendingCopy = false; break; } } Thread.Sleep(1000); } }
public ContainerTreeNode(CloudBlobContainer container) : this() { Container = container; foreach (var blob in container.ListBlobs().OfType<CloudBlob>()) Nodes.Add(new BlobTreeNode(blob)); Text = container.Name; }
protected void DeleteAllBlobs(CloudBlobContainer container) { foreach ( var blob in container.ListBlobs() ) { if ( blob is CloudBlob ) { ( (CloudBlob)blob ).DeleteIfExists(); } if ( blob is CloudBlobDirectory ) { DeleteAllBlobs((CloudBlobDirectory)blob); } } }
public const int CacheControlOneWeekExpiration = 7 * 24 * 60 * 60; // 1 week #endregion Fields #region Methods /// <summary> /// Finds all js and css files in a container and creates a gzip compressed /// copy of the file with ".gzip" appended to the existing blob name /// </summary> public static void EnsureGzipFiles( CloudBlobContainer container, int cacheControlMaxAgeSeconds) { string cacheControlHeader = "public, max-age=" + cacheControlMaxAgeSeconds.ToString(); var blobInfos = container.ListBlobs( new BlobRequestOptions() { UseFlatBlobListing = true }); Parallel.ForEach(blobInfos, (blobInfo) => { string blobUrl = blobInfo.Uri.ToString(); CloudBlob blob = container.GetBlobReference(blobUrl); // only create gzip copies for css and js files string extension = Path.GetExtension(blobInfo.Uri.LocalPath); if (extension != ".css" && extension != ".js") return; // see if the gzip version already exists string gzipUrl = blobUrl + ".gzip"; CloudBlob gzipBlob = container.GetBlobReference(gzipUrl); if (gzipBlob.Exists()) return; // create a gzip version of the file using (MemoryStream memoryStream = new MemoryStream()) { // push the original blob into the gzip stream using (GZipStream gzipStream = new GZipStream( memoryStream, CompressionMode.Compress, CompressionLevel.BestCompression)) using (BlobStream blobStream = blob.OpenRead()) { blobStream.CopyTo(gzipStream); } // the gzipStream MUST be closed before its safe to read from the memory stream byte[] compressedBytes = memoryStream.ToArray(); // upload the compressed bytes to the new blob gzipBlob.UploadByteArray(compressedBytes); // set the blob headers gzipBlob.Properties.CacheControl = cacheControlHeader; gzipBlob.Properties.ContentType = GetContentType(extension); gzipBlob.Properties.ContentEncoding = "gzip"; gzipBlob.SetProperties(); } }); }
/// <summary> /// Get Items /// </summary> /// <param name="container">Container</param> /// <param name="client">Client</param> /// <param name="path">Path</param> /// <returns>Storage Items</returns> private IEnumerable<IStorageItem> GetItems(CloudBlobContainer container, AmazonS3 client, string path) { if (null != container) { var options = new BlobRequestOptions() { UseFlatBlobListing = true, }; return container.ListBlobs(options).Select(b => new Azure(container, b.Uri.ToString())).Where(c => c.Exists()); } else if (null != client) { var request = new ListObjectsRequest() { BucketName = path, }; using (var response = client.ListObjects(request)) { return response.S3Objects.Select(s3 => new S3(client, path, s3.Key, s3.ETag)); } } else { return this.GetFiles(path, path, new List<IStorageItem>()); } }
static void Sync(CloudBlobContainer container, string rootPath) { // iterate all files from storage foreach (CloudBlob item in container.ListBlobs(new BlobRequestOptions() { UseFlatBlobListing = true })) { try { var localPath = Path.Combine(rootPath, item.Name); if (File.Exists(localPath) && File.GetLastWriteTimeUtc(localPath).ToFileTime() > item.Properties.LastModifiedUtc.ToFileTime()) continue; // local file is more recent if (!Directory.Exists(Path.GetDirectoryName(localPath))) Directory.CreateDirectory(Path.GetDirectoryName(localPath)); for (var i = 0; i < RETRY_COUNT; i++) { try { using (var fs = File.Open(localPath, FileMode.Create)) { item.DownloadToStream(fs); break; // all OK } } catch (IOException) { // retry again later System.Threading.Thread.Sleep(SLEEPMS_ONFAIL); } //catch (UnauthorizedAccessException) { // // ??? just catch this silently otherwise IIS will crash //} } } catch (Exception ex) { NLog.LogManager.GetLogger("optQuick").Error(string.Format("Unable to process path {0} => {1}", rootPath, item.Name)); NLog.LogManager.GetLogger("optQuick").Error(ex); throw; } } }
/// <summary> /// Iterates through each blob in the specified container and adds the /// Cache-Control and ContentType headers /// </summary> public static void EnsureStaticFileHeaders( CloudBlobContainer container, int cacheControlMaxAgeSeconds) { string cacheControlHeader = "public, max-age=" + cacheControlMaxAgeSeconds.ToString(); var blobInfos = container.ListBlobs( new BlobRequestOptions() { UseFlatBlobListing = true }); Parallel.ForEach(blobInfos, (blobInfo) => { // get the blob properties and set headers if necessary CloudBlob blob = container.GetBlobReference(blobInfo.Uri.ToString()); blob.FetchAttributes(); var properties = blob.Properties; bool wasModified = false; // see if a content type is defined for the extension string extension = Path.GetExtension(blobInfo.Uri.LocalPath); string contentType = GetContentType(extension); if (String.IsNullOrEmpty(contentType)) { Trace.TraceWarning("Content type not found for extension:" + extension); } else { if (properties.ContentType != contentType) { properties.ContentType = contentType; wasModified = true; } } if (properties.CacheControl != cacheControlHeader) { properties.CacheControl = cacheControlHeader; wasModified = true; } if (wasModified) { blob.SetProperties(); } }); }
void ClearContainer(CloudBlobContainer c) { BlobRequestOptions opt = new BlobRequestOptions(); opt.UseFlatBlobListing = true; foreach (CloudBlob blob in c.ListBlobs(opt)) blob.Delete(); }
///<summary>Updates the blobs in a storage container.</summary> ///<param name="container">The container to update.</param> ///<param name="newVersion">The new version being uploaded.</param> ///<param name="sourcePath">The directory on the local disk containing the new files.</param> public static void UpdateStorage(CloudBlobContainer container, Version newVersion, string sourcePath) { container.FetchAttributes(); var remoteFiles = container.ListBlobs(Options).Cast<CloudBlob>().ToArray(); var oldBlobs = remoteFiles.ToDictionary( blob => Path.Combine(sourcePath, container.Uri.MakeRelativeUri(blob.Uri).ToString().Replace('/', '\\') ) ); var baseUri = new Uri(sourcePath, UriKind.Absolute); foreach (var file in new DirectoryInfo(sourcePath).EnumerateFiles("*", SearchOption.AllDirectories)) { CloudBlob blob; byte[] hash = file.SHA512Hash(); //If there already is a blob for this file, use it. if (!oldBlobs.TryGetValue(file.FullName, out blob)) { blob = container.GetBlobReference(baseUri.MakeRelativeUri(new Uri(file.FullName, UriKind.Absolute)).ToString()); } else { oldBlobs.Remove(file.FullName); //Remove the blob from the dictionary; all blobs left in the dictionary will be deleted. if (file.Length == blob.Properties.Length && Convert.FromBase64String(blob.Metadata["SHA512"]).SequenceEqual(hash)) continue; //If the blob is identical, don't re-upload it. } blob.Metadata["SHA512"] = Convert.ToBase64String(hash); blob.UploadFile(file.FullName); } foreach (var blob in oldBlobs.Values) blob.Delete(); container.Metadata["Version"] = newVersion.ToString(); container.SetMetadata(); }
public static List<CloudBlob> ListBlobs() { List<IListBlobItem> blobItemList = null; List<CloudBlob> blobList = null; try { string storageAccountConnection = string.Empty; storageAccountConnection = ConfigurationManager.AppSettings["StorageAccount.ConnectionString"].ToString(); Logger.Write("Storage Connection: " + storageAccountConnection); // If you want to use Windows Azure cloud storage account, use the following // code (after uncommenting) instead of the code above. cloudStorageAccount = CloudStorageAccount.Parse(storageAccountConnection); Logger.Write("Blob Uri: " + cloudStorageAccount.BlobEndpoint.AbsoluteUri); // Create the blob client, which provides // authenticated access to the Blob service. blobClient = cloudStorageAccount.CreateCloudBlobClient(); string deploymentPackageFolderString = string.Empty; deploymentPackageFolderString = ConfigurationManager.AppSettings["DeploymentPackageFolder"].ToString(); Logger.Write("Deployment Package Folder: " + deploymentPackageFolderString); // Get the container reference. blobContainer = blobClient.GetContainerReference(deploymentPackageFolderString); // Create the container if it does not exist. blobContainer.CreateIfNotExist(); // Set permissions on the container. containerPermissions = new BlobContainerPermissions(); // This sample sets the container to have public blobs. Your application // needs may be different. See the documentation for BlobContainerPermissions // for more information about blob container permissions. containerPermissions.PublicAccess = BlobContainerPublicAccessType.Blob; blobContainer.SetPermissions(containerPermissions); BlobRequestOptions blobReqOptions = new BlobRequestOptions(); blobReqOptions.BlobListingDetails = BlobListingDetails.All; blobReqOptions.Timeout = new TimeSpan(0, 5, 0); blobReqOptions.UseFlatBlobListing = true; blobItemList = blobContainer.ListBlobs(blobReqOptions).ToList(); if (blobList == null) { blobList = new List<CloudBlob>(); } foreach (IListBlobItem blobItem in blobItemList) { Logger.Write("Blob Uri: " + blobItem.Uri.ToString()); CloudBlob blobEntry = new CloudBlob(blobItem.Uri.ToString()); blobEntry.FetchAttributes(); blobList.Add(blobEntry); } } catch (System.Exception ex) { Logger.Write(string.Format("Error in List<IListBlobItem> ListBlobs() Error: {0}", ex.Message)); blobList = null; } return blobList; }
/// <summary> /// Copy configuration files from Azure Storage to the config VHD /// </summary> /// <param name="dbConfigPath"></param> /// <param name="container"></param> /// <param name="account"></param> /// <returns></returns> public static bool CopyConfigFromAzureStorage(string dbConfigPath, CloudBlobContainer container, CloudStorageAccount account) { try { _account = account; _container = container; _dbConfigPath = dbConfigPath; // We delete the older files Trace.TraceInformation("Deleting the older config files"); try { DeleteFilesAndDirectories(dbConfigPath); } catch (Exception ex) { Trace.TraceError(string.Format("MongoHelper.CopyConfigFromAzureStorage Exception : {0}", ex.Message)); } BlobRequestOptions options = new BlobRequestOptions(); options.UseFlatBlobListing = true; GetBlobs(container.ListBlobs(options)); return true; } catch (Exception ex) { Trace.TraceError(string.Format("CopyConfigFromAzureStorage : Exception while Gettings blobs : {0} ({1}) ", ex.Message, ex.InnerException == null ? "" : ex.InnerException.Message)); return false; } }