/// <summary>
/// Taken from: http://blogs.msdn.com/b/windowsazurestorage/archive/2012/06/12/introducing-asynchronous-cross-account-copy-blob.aspx
/// </summary>
/// <param name="destContainer">The container to monitor</param>
public static void MonitorCopy(CloudBlobContainer destContainer)
{
var pendingCopy = true;
while (pendingCopy)
{
var destBlobList = destContainer.ListBlobs(true, BlobListingDetails.Copy);
foreach (var destBlob in destBlobList.Select(dest => dest as CloudBlob))
{
if (destBlob.CopyState == null)
{
Debug.WriteLine("BlobStorage.MonitorCopy: CopyState is null. Small sleep, then we assume it's done!");
Thread.Sleep(5000);
return;
}
switch (destBlob.CopyState.Status)
{
case CopyStatus.Failed:
case CopyStatus.Aborted:
Debug.WriteLine("BlobStorage.MonitorCopy: Copy Failed or Aborted; restarting copy");
destBlob.StartCopyFromBlob(destBlob.CopyState.Source);
break;
case CopyStatus.Success:
pendingCopy = false;
break;
}
}
Thread.Sleep(1000);
}
}
public ContainerTreeNode(CloudBlobContainer container)
: this()
{
Container = container;
foreach (var blob in container.ListBlobs().OfType<CloudBlob>())
Nodes.Add(new BlobTreeNode(blob));
Text = container.Name;
}
protected void DeleteAllBlobs(CloudBlobContainer container) {
foreach ( var blob in container.ListBlobs() ) {
if ( blob is CloudBlob ) {
( (CloudBlob)blob ).DeleteIfExists();
}
if ( blob is CloudBlobDirectory ) {
DeleteAllBlobs((CloudBlobDirectory)blob);
}
}
}
public const int CacheControlOneWeekExpiration = 7 * 24 * 60 * 60; // 1 week
#endregion Fields
#region Methods
/// <summary>
/// Finds all js and css files in a container and creates a gzip compressed
/// copy of the file with ".gzip" appended to the existing blob name
/// </summary>
public static void EnsureGzipFiles(
CloudBlobContainer container,
int cacheControlMaxAgeSeconds)
{
string cacheControlHeader = "public, max-age=" + cacheControlMaxAgeSeconds.ToString();
var blobInfos = container.ListBlobs(
new BlobRequestOptions() { UseFlatBlobListing = true });
Parallel.ForEach(blobInfos, (blobInfo) =>
{
string blobUrl = blobInfo.Uri.ToString();
CloudBlob blob = container.GetBlobReference(blobUrl);
// only create gzip copies for css and js files
string extension = Path.GetExtension(blobInfo.Uri.LocalPath);
if (extension != ".css" && extension != ".js")
return;
// see if the gzip version already exists
string gzipUrl = blobUrl + ".gzip";
CloudBlob gzipBlob = container.GetBlobReference(gzipUrl);
if (gzipBlob.Exists())
return;
// create a gzip version of the file
using (MemoryStream memoryStream = new MemoryStream())
{
// push the original blob into the gzip stream
using (GZipStream gzipStream = new GZipStream(
memoryStream, CompressionMode.Compress, CompressionLevel.BestCompression))
using (BlobStream blobStream = blob.OpenRead())
{
blobStream.CopyTo(gzipStream);
}
// the gzipStream MUST be closed before its safe to read from the memory stream
byte[] compressedBytes = memoryStream.ToArray();
// upload the compressed bytes to the new blob
gzipBlob.UploadByteArray(compressedBytes);
// set the blob headers
gzipBlob.Properties.CacheControl = cacheControlHeader;
gzipBlob.Properties.ContentType = GetContentType(extension);
gzipBlob.Properties.ContentEncoding = "gzip";
gzipBlob.SetProperties();
}
});
}
/// <summary>
/// Get Items
/// </summary>
/// <param name="container">Container</param>
/// <param name="client">Client</param>
/// <param name="path">Path</param>
/// <returns>Storage Items</returns>
private IEnumerable<IStorageItem> GetItems(CloudBlobContainer container, AmazonS3 client, string path)
{
if (null != container)
{
var options = new BlobRequestOptions()
{
UseFlatBlobListing = true,
};
return container.ListBlobs(options).Select(b => new Azure(container, b.Uri.ToString())).Where(c => c.Exists());
}
else if (null != client)
{
var request = new ListObjectsRequest()
{
BucketName = path,
};
using (var response = client.ListObjects(request))
{
return response.S3Objects.Select(s3 => new S3(client, path, s3.Key, s3.ETag));
}
}
else
{
return this.GetFiles(path, path, new List<IStorageItem>());
}
}
static void Sync(CloudBlobContainer container, string rootPath)
{
// iterate all files from storage
foreach (CloudBlob item in container.ListBlobs(new BlobRequestOptions() { UseFlatBlobListing = true })) {
try {
var localPath = Path.Combine(rootPath, item.Name);
if (File.Exists(localPath) && File.GetLastWriteTimeUtc(localPath).ToFileTime() > item.Properties.LastModifiedUtc.ToFileTime())
continue; // local file is more recent
if (!Directory.Exists(Path.GetDirectoryName(localPath)))
Directory.CreateDirectory(Path.GetDirectoryName(localPath));
for (var i = 0; i < RETRY_COUNT; i++) {
try {
using (var fs = File.Open(localPath, FileMode.Create)) {
item.DownloadToStream(fs);
break; // all OK
}
} catch (IOException) {
// retry again later
System.Threading.Thread.Sleep(SLEEPMS_ONFAIL);
}
//catch (UnauthorizedAccessException) {
// // ??? just catch this silently otherwise IIS will crash
//}
}
} catch (Exception ex) {
NLog.LogManager.GetLogger("optQuick").Error(string.Format("Unable to process path {0} => {1}", rootPath, item.Name));
NLog.LogManager.GetLogger("optQuick").Error(ex);
throw;
}
}
}
/// <summary>
/// Iterates through each blob in the specified container and adds the
/// Cache-Control and ContentType headers
/// </summary>
public static void EnsureStaticFileHeaders(
CloudBlobContainer container,
int cacheControlMaxAgeSeconds)
{
string cacheControlHeader = "public, max-age=" + cacheControlMaxAgeSeconds.ToString();
var blobInfos = container.ListBlobs(
new BlobRequestOptions() { UseFlatBlobListing = true });
Parallel.ForEach(blobInfos, (blobInfo) =>
{
// get the blob properties and set headers if necessary
CloudBlob blob = container.GetBlobReference(blobInfo.Uri.ToString());
blob.FetchAttributes();
var properties = blob.Properties;
bool wasModified = false;
// see if a content type is defined for the extension
string extension = Path.GetExtension(blobInfo.Uri.LocalPath);
string contentType = GetContentType(extension);
if (String.IsNullOrEmpty(contentType))
{
Trace.TraceWarning("Content type not found for extension:" + extension);
}
else
{
if (properties.ContentType != contentType)
{
properties.ContentType = contentType;
wasModified = true;
}
}
if (properties.CacheControl != cacheControlHeader)
{
properties.CacheControl = cacheControlHeader;
wasModified = true;
}
if (wasModified)
{
blob.SetProperties();
}
});
}
void ClearContainer(CloudBlobContainer c)
{
BlobRequestOptions opt = new BlobRequestOptions();
opt.UseFlatBlobListing = true;
foreach (CloudBlob blob in c.ListBlobs(opt))
blob.Delete();
}
///<summary>Updates the blobs in a storage container.</summary>
///<param name="container">The container to update.</param>
///<param name="newVersion">The new version being uploaded.</param>
///<param name="sourcePath">The directory on the local disk containing the new files.</param>
public static void UpdateStorage(CloudBlobContainer container, Version newVersion, string sourcePath)
{
container.FetchAttributes();
var remoteFiles = container.ListBlobs(Options).Cast<CloudBlob>().ToArray();
var oldBlobs = remoteFiles.ToDictionary(
blob => Path.Combine(sourcePath,
container.Uri.MakeRelativeUri(blob.Uri).ToString().Replace('/', '\\')
)
);
var baseUri = new Uri(sourcePath, UriKind.Absolute);
foreach (var file in new DirectoryInfo(sourcePath).EnumerateFiles("*", SearchOption.AllDirectories)) {
CloudBlob blob;
byte[] hash = file.SHA512Hash();
//If there already is a blob for this file, use it.
if (!oldBlobs.TryGetValue(file.FullName, out blob)) {
blob = container.GetBlobReference(baseUri.MakeRelativeUri(new Uri(file.FullName, UriKind.Absolute)).ToString());
} else {
oldBlobs.Remove(file.FullName); //Remove the blob from the dictionary; all blobs left in the dictionary will be deleted.
if (file.Length == blob.Properties.Length
&& Convert.FromBase64String(blob.Metadata["SHA512"]).SequenceEqual(hash))
continue; //If the blob is identical, don't re-upload it.
}
blob.Metadata["SHA512"] = Convert.ToBase64String(hash);
blob.UploadFile(file.FullName);
}
foreach (var blob in oldBlobs.Values)
blob.Delete();
container.Metadata["Version"] = newVersion.ToString();
container.SetMetadata();
}
public static List<CloudBlob> ListBlobs()
{
List<IListBlobItem> blobItemList = null;
List<CloudBlob> blobList = null;
try
{
string storageAccountConnection = string.Empty;
storageAccountConnection = ConfigurationManager.AppSettings["StorageAccount.ConnectionString"].ToString();
Logger.Write("Storage Connection: " + storageAccountConnection);
// If you want to use Windows Azure cloud storage account, use the following
// code (after uncommenting) instead of the code above.
cloudStorageAccount = CloudStorageAccount.Parse(storageAccountConnection);
Logger.Write("Blob Uri: " + cloudStorageAccount.BlobEndpoint.AbsoluteUri);
// Create the blob client, which provides
// authenticated access to the Blob service.
blobClient = cloudStorageAccount.CreateCloudBlobClient();
string deploymentPackageFolderString = string.Empty;
deploymentPackageFolderString = ConfigurationManager.AppSettings["DeploymentPackageFolder"].ToString();
Logger.Write("Deployment Package Folder: " + deploymentPackageFolderString);
// Get the container reference.
blobContainer = blobClient.GetContainerReference(deploymentPackageFolderString);
// Create the container if it does not exist.
blobContainer.CreateIfNotExist();
// Set permissions on the container.
containerPermissions = new BlobContainerPermissions();
// This sample sets the container to have public blobs. Your application
// needs may be different. See the documentation for BlobContainerPermissions
// for more information about blob container permissions.
containerPermissions.PublicAccess = BlobContainerPublicAccessType.Blob;
blobContainer.SetPermissions(containerPermissions);
BlobRequestOptions blobReqOptions = new BlobRequestOptions();
blobReqOptions.BlobListingDetails = BlobListingDetails.All;
blobReqOptions.Timeout = new TimeSpan(0, 5, 0);
blobReqOptions.UseFlatBlobListing = true;
blobItemList = blobContainer.ListBlobs(blobReqOptions).ToList();
if (blobList == null)
{
blobList = new List<CloudBlob>();
}
foreach (IListBlobItem blobItem in blobItemList)
{
Logger.Write("Blob Uri: " + blobItem.Uri.ToString());
CloudBlob blobEntry = new CloudBlob(blobItem.Uri.ToString());
blobEntry.FetchAttributes();
blobList.Add(blobEntry);
}
}
catch (System.Exception ex)
{
Logger.Write(string.Format("Error in List<IListBlobItem> ListBlobs() Error: {0}", ex.Message));
blobList = null;
}
return blobList;
}
/// <summary>
/// Copy configuration files from Azure Storage to the config VHD
/// </summary>
/// <param name="dbConfigPath"></param>
/// <param name="container"></param>
/// <param name="account"></param>
/// <returns></returns>
public static bool CopyConfigFromAzureStorage(string dbConfigPath, CloudBlobContainer container, CloudStorageAccount account)
{
try
{
_account = account;
_container = container;
_dbConfigPath = dbConfigPath;
// We delete the older files
Trace.TraceInformation("Deleting the older config files");
try
{
DeleteFilesAndDirectories(dbConfigPath);
}
catch (Exception ex)
{
Trace.TraceError(string.Format("MongoHelper.CopyConfigFromAzureStorage Exception : {0}", ex.Message));
}
BlobRequestOptions options = new BlobRequestOptions();
options.UseFlatBlobListing = true;
GetBlobs(container.ListBlobs(options));
return true;
}
catch (Exception ex)
{
Trace.TraceError(string.Format("CopyConfigFromAzureStorage : Exception while Gettings blobs : {0} ({1}) ", ex.Message, ex.InnerException == null ? "" : ex.InnerException.Message));
return false;
}
}