// Helper function to allow Storage Client 1.7 (Microsoft.WindowsAzure.StorageClient) to utilize this class.
// Remove this function if only using Storage Client 2.0 (Microsoft.WindowsAzure.Storage).
public void DownloadBlobAsync(Microsoft.WindowsAzure.StorageClient.CloudBlob blob, string LocalFile)
{
Microsoft.WindowsAzure.StorageCredentialsAccountAndKey account = blob.ServiceClient.Credentials as Microsoft.WindowsAzure.StorageCredentialsAccountAndKey;
ICloudBlob blob2 = new Microsoft.WindowsAzure.Storage.Blob.CloudBlockBlob(blob.Attributes.Uri, new Microsoft.WindowsAzure.Storage.Auth.StorageCredentials(blob.ServiceClient.Credentials.AccountName, account.Credentials.ExportBase64EncodedKey()));
DownloadBlobAsync(blob2, LocalFile);
}
public static void Read(BlobRequestOptions mapped, CloudBlob blob, ReaderDelegate reader)
{
blob.FetchAttributes(mapped);
var props = MapFetchedAttrbitues(blob);
var compression = blob.Properties.ContentEncoding ?? "";
var md5 = blob.Metadata[LokadHashFieldName];
switch (compression)
{
case "gzip":
using (var stream = blob.OpenRead(mapped))
{
ReadAndVerifyHash(stream, s =>
{
// important is not to flush the decompression stream
using (var decompress = new GZipStream(s, CompressionMode.Decompress, true))
{
reader(props, decompress);
}
}, md5);
}
break;
case "":
using (var stream = blob.OpenRead(mapped))
{
ReadAndVerifyHash(stream, s => reader(props, s), md5);
}
break;
default:
var error = string.Format("Unsupported ContentEncoding '{0}'", compression);
throw new InvalidOperationException(error);
}
}
/// <summary>
/// Check whether the specified blob exists
/// </summary>
/// <param name="blob">Blob to check</param>
/// <returns>true if specified blob exists, and false if it doesn't</returns>
private static bool BlobExists(CloudBlob blob)
{
try
{
// try fetching Attributes
blob.FetchAttributes();
return true;
}
catch (StorageClientException e)
{
if (e.ErrorCode == StorageErrorCode.ResourceNotFound)
{
// if failed with ResourceNotFound error code, then the blob doesn't exist
return false;
}
else
{
// something else went wrong
//throw;
Console.Write(e);
return false;
}
}
catch (Exception e)
{
Console.Write(e);
return false;
}
}
/// <summary>
/// Copies the full binary contents of the given blob to the given HTTP response.
/// </summary>
private static void CopyContents(CloudBlob blob, HttpResponseBase response, long offset = 0)
{
blob.FetchAttributes();
response.BufferOutput = false;
response.AddHeader("Content-Length", blob.Attributes.Properties.Length.ToString());
response.Flush();
using (var reader = blob.OpenRead())
{
reader.Seek(offset, System.IO.SeekOrigin.Begin);
byte[] buffer = new byte[1024 * 4]; // 4KB buffer
while (reader.CanRead)
{
int numBytes = reader.Read(buffer, 0, buffer.Length);
if (numBytes <= 0)
break;
response.BinaryWrite(buffer);
response.Flush();
}
}
}
static void Main(string[] args)
{
try
{
string BlobURL = "deploy/Testdrivesnapshot.vhd";
CloudStorageAccount storageAccount = CloudStorageAccount.Parse("DefaultEndpointsProtocol=http;AccountName=mongodbwest2;AccountKey=wZcR60wAy+zltHPV7CXJsvBo/rnZHV2FIqg+UA+H1pIhkYl4j0qRZ+GgI5V8IJhngh2DOxI+sS46KddPFWg0Xw==");
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
//Get a reference to a blob.
CloudBlob blob = blobClient.GetBlobReference(BlobURL);
//Take a snapshot of the blob.
CloudBlob snapshot = blob.CreateSnapshot();
//Get the snapshot timestamp.
DateTime timestamp = (DateTime)snapshot.Attributes.Snapshot;
//Use the timestamp to get a second reference to the snapshot.
CloudBlob snapshot2 = new CloudBlob(BlobURL, timestamp, blobClient);
CloudDrive Snapshotdrive = new CloudDrive(snapshot2.Uri, storageAccount.Credentials);
string path = Snapshotdrive.Mount(0, DriveMountOptions.None);
Console.WriteLine("Mounted on " + path);
Console.ReadLine();
Snapshotdrive.Unmount();
}
catch (Exception ex)
{
Console.WriteLine(string.Format("Exception : {0} ({1}) at {2}", ex.Message, ex.InnerException == null ? "" : ex.InnerException.Message, ex.StackTrace));
}
}
public AutoRenewLease(CloudBlob blob)
{
this.blob = blob;
blob.Container.CreateIfNotExist();
try
{
blob.UploadByteArray(new byte[0], new BlobRequestOptions { AccessCondition = AccessCondition.IfNoneMatch("*") });
}
catch (StorageClientException e)
{
if (e.ErrorCode != StorageErrorCode.BlobAlreadyExists
&& e.StatusCode != HttpStatusCode.PreconditionFailed) // 412 from trying to modify a blob that's leased
{
throw;
}
}
leaseId = blob.TryAcquireLease();
if (HasLease)
{
renewalThread = new Thread(() =>
{
Thread.Sleep(TimeSpan.FromSeconds(40));
blob.RenewLease(leaseId);
});
renewalThread.Start();
}
}
public static void DoEvery(CloudBlob blob, TimeSpan interval, Action action)
{
while (true)
{
var lastPerformed = DateTimeOffset.MinValue;
using (var arl = new AutoRenewLease(blob))
{
if (arl.HasLease)
{
blob.FetchAttributes();
DateTimeOffset.TryParseExact(blob.Metadata["lastPerformed"], "R", CultureInfo.CurrentCulture, DateTimeStyles.AdjustToUniversal, out lastPerformed);
if (DateTimeOffset.UtcNow >= lastPerformed + interval)
{
action();
lastPerformed = DateTimeOffset.UtcNow;
blob.Metadata["lastPerformed"] = lastPerformed.ToString("R");
blob.SetMetadata(arl.leaseId);
}
}
}
var timeLeft = (lastPerformed + interval) - DateTimeOffset.UtcNow;
var minimum = TimeSpan.FromSeconds(5); // so we're not polling the leased blob too fast
Thread.Sleep(
timeLeft > minimum
? timeLeft
: minimum);
}
}
public BlobTreeNode(CloudBlob blob)
{
Blob = blob;
ImageKey = "Blob";
SelectedImageKey = "Blob";
Text = blob.Name;
}
private static void DoLeaseOperation(CloudBlob blob, string leaseId, LeaseAction action)
{
var credentials = blob.ServiceClient.Credentials;
var request = BlobRequest.Lease(new Uri(credentials.TransformUri(blob.Uri.AbsoluteUri)), 90,
action, leaseId);
credentials.SignRequest(request);
request.GetResponse().Close();
}
public Project(string projectName, string projectDummyFile, CloudBlob projectDummyFileBlob)
{
this.projectName = projectName;
this.projectDummyFile = projectDummyFile;
this.projectDummyFileBlob = projectDummyFileBlob;
tasksMap = new Dictionary<string, Task>();
projectMetadata = new Dictionary<string, string>();
}
public void SetBlobValuesToSource(CloudBlob blob)
{
SourceLocation = blob.Name;
SourceETag = blob.Properties.ETag;
SourceType = blob.GetBlobInformationType();
SourceMD5 = blob.Properties.ContentMD5;
SourceLastModified = blob.Properties.LastModifiedUtc;
}
private const string MimeTypeName = "text/plain"; // since these are assumed to
#endregion Fields
#region Methods
/// <summary>
/// ugliness of downloading and reuploading whole blob.
/// </summary>
/// <param name="blob"></param>
/// <param name="toAdd"></param>
public static void appendToBlob(CloudBlob blob, string toAdd)
{
string oldLogData = "";
if (Exists(blob))
oldLogData = blob.DownloadText();
blob.DeleteIfExists();
blob.UploadText(oldLogData + "\r\n" + toAdd);
}
private static void DoLeaseOperation(CloudBlob blob, string leaseId, LeaseAction action)
{
var creds = blob.ServiceClient.Credentials;
var transformedUri = new Uri(creds.TransformUri(blob.Uri.ToString()));
var req = BlobRequest.Lease(transformedUri, 60, action, leaseId);
creds.SignRequest(req);
req.GetResponse().Close();
}
public Task(string taskName, string taskDummyFile, CloudBlob taskDummyFileBlob)
{
this.taskName = taskName;
this.taskDummyFile = taskDummyFile;
this.taskDummyFileBlob = taskDummyFileBlob;
this.exes = new List<string>();
this.dataFiles = new List<string>();
taskMetadata = new Dictionary<string, string>();
}
Tuple<string, IHtmlString> Pygmentize(CloudBlob blob)
{
var filename = Uri.UnescapeDataString(blob.Uri.Segments.Last());
var extension = Path.GetExtension(filename).ToLower();
if (extension == ".mdown")
{
return new Tuple<string, IHtmlString>(null, new MvcHtmlString(new Markdown().Transform(blob.DownloadText())));
}
var formatters = new Dictionary<string, string>()
{
{ ".cscfg", "xml" },
{ ".csdef", "xml" },
{ ".config", "xml" },
{ ".xml", "xml" },
{ ".cmd", "bat" },
{ ".rb", "ruby" },
{ ".cs", "csharp" },
{ ".html", "html" },
{ ".cshtml", "html" },
{ ".css", "css" },
{ ".erb", "erb" },
{ ".haml", "haml" },
{ ".js", "javascript" },
{ ".php", "php" },
{ ".py", "python" },
{ ".yaml", "yaml" },
{ ".yml", "yaml" },
{ ".txt", "text" }
};
var executable = "pygmentize";
if (RoleEnvironment.IsAvailable)
{
executable = Path.Combine(RoleEnvironment.GetLocalResource("Python").RootPath, @"scripts\pygmentize.exe");
}
if (!formatters.ContainsKey(extension))
{
extension = ".txt";
}
var startInfo = new ProcessStartInfo(executable, string.Format("-f html -l {0}", formatters[extension]))
{
UseShellExecute = false,
CreateNoWindow = true,
RedirectStandardInput = true,
RedirectStandardOutput = true,
};
var proc = Process.Start(startInfo);
proc.StandardInput.Write(blob.DownloadText().Trim(new char[]{'\uFEFF'}));
proc.StandardInput.Close();
return new Tuple<string, IHtmlString>(filename, new MvcHtmlString(proc.StandardOutput.ReadToEnd()));
}
public TaskRunner CreateUpdateModifiedDateTaskRunner(CloudBlob blob, FileInformation fileInfo)
{
this._statistics.UpdatedModifiedDateCount++;
return new SingleActionTaskRunner(() =>
{
_messageBus.Publish(new FileProgressedMessage(fileInfo.FullPath, 0));
blob.SetFileLastModifiedUtc(fileInfo.LastWriteTimeUtc, true);
_messageBus.Publish(new FileProgressedMessage(fileInfo.FullPath, 1));
});
}
public void SubscribeTargetToSourceChanges(CloudBlob subscriber)
{
if(subscriber.CanContainExternalMetadata() == false)
throw new InvalidDataException("Subscriber candidate cannot contain metadata: " + subscriber.Name);
foreach(var source in CollectionContent.Where(src => src.IsInformationObjectSource))
{
SubscribeSupport.AddSubscriptionToObject(source.SourceLocation, subscriber.Name,
SubscribeSupport.SubscribeType_WebPageToSource);
}
}
/// <summary>
/// Удаляет файл.
/// </summary>
/// <param name="localUri"> Адрес файла </param>
public void Delete(string localUri)
{
CloudBlob blob = new CloudBlob(localUri, _client);
blob.DeleteIfExists(new BlobRequestOptions()
{
RetryPolicy = RetryPolicies.RetryExponential(
RetryPolicies.DefaultClientRetryCount,
RetryPolicies.DefaultClientBackoff)
});
}
public AzureTapeStream(string name, string connectionString, string containerName, ITapeStreamSerializer serializer)
{
_serializer = serializer;
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(connectionString);
blobClient = storageAccount.CreateCloudBlobClient();
container = blobClient.GetContainerReference(containerName);
container.CreateIfNotExist();
_blob = container.GetBlobReference(name);
}
public string AcquireLease(CloudBlob blob, int timeout)
{
var creds = blob.ServiceClient.Credentials;
var transformedUri = new Uri(creds.TransformUri(blob.Uri.ToString()));
var request = BlobRequest.Lease(transformedUri, timeout, LeaseAction.Acquire, null);
blob.ServiceClient.Credentials.SignRequest(request);
using (var response = request.GetResponse())
{
return response.Headers["x-ms-lease-id"];
}
}
public TaskRunner CreateUpdateTaskRunner(CloudBlob blob, FileInformation fileInfo)
{
this._statistics.UpdatedCount++;
return new SingleActionTaskRunner(() =>
{
_messageBus.Publish(new FileProgressedMessage(fileInfo.FullPath, 0));
blob.DownloadToFile(fileInfo.FullPath);
_fileSystem.SetLastWriteTimeUtc(fileInfo.FullPath, blob.GetFileLastModifiedUtc());
_messageBus.Publish(new FileProgressedMessage(fileInfo.FullPath, 1));
});
}
private static async Task DoLeaseOperationAsync(CloudBlob blob, string leaseId, LeaseAction action)
{
var creds = blob.ServiceClient.Credentials;
var transformedUri = new Uri(creds.TransformUri(blob.Uri.ToString()));
var req = BlobRequest.Lease(transformedUri, 90, action, leaseId);
creds.SignRequest(req);
using (var response = await req.GetResponseAsync())
{
response.Close();
}
}
public static CloudBlob MediaContentToBlob(this MediaContent mediaContent, CloudBlob blob)
{
if (mediaContent.Published.HasValue)
{
blob.Metadata["Published"] = mediaContent.Published.Value.ToString();
}
if (!string.IsNullOrEmpty(mediaContent.UserId))
{
blob.Metadata["UserId"] = mediaContent.UserId;
}
if (!string.IsNullOrEmpty(mediaContent.FileName))
{
blob.Metadata["FileName"] = StorageNamesEncoder.EncodeBlobName(mediaContent.FileName);
}
if (mediaContent.ContentFile != null)
{
blob.Metadata["Size"] = mediaContent.ContentFile.Stream.Length.ToString();
}
if (mediaContent.Metadata != null)
{
if (!string.IsNullOrEmpty(mediaContent.Metadata.AlternateText))
{
blob.Metadata["AlternateText"] = StorageNamesEncoder.EncodeBlobName(mediaContent.Metadata.AlternateText);
}
else if(blob.Metadata.AllKeys.Contains("AlternateText"))
{
blob.Metadata.Remove("AlternateText");
}
if (!string.IsNullOrEmpty(mediaContent.Metadata.Description))
{
blob.Metadata["Description"] = StorageNamesEncoder.EncodeBlobName(mediaContent.Metadata.Description);
}
else if (blob.Metadata.AllKeys.Contains("Description"))
{
blob.Metadata.Remove("Description");
}
if (!string.IsNullOrEmpty(mediaContent.Metadata.Title))
{
blob.Metadata["Title"] = StorageNamesEncoder.EncodeBlobName(mediaContent.Metadata.Title);
}
else if (blob.Metadata.AllKeys.Contains("Title"))
{
blob.Metadata.Remove("Title");
}
}
blob.Properties.ContentType = Kooboo.IO.IOUtility.MimeType(mediaContent.FileName);
return blob;
}
/// <summary>
/// Initializes a new instance of the BlobReadStream class.
/// </summary>
/// <param name="blob">The blob used for downloads.</param>
/// <param name="options">Modifiers to be applied to the blob. After first request, the ETag is always applied.</param>
/// <param name="readAheadInBytes">The number of bytes to read ahead.</param>
/// <param name="verifyBlocks">Controls whether block's signatures are verified.</param>
internal BlobReadStream(CloudBlob blob, BlobRequestOptions options, long readAheadInBytes, bool verifyBlocks)
{
CommonUtils.AssertNotNull("blob", blob);
CommonUtils.AssertNotNull("options", options);
CommonUtils.AssertInBounds("readAheadInBytes", readAheadInBytes, 0, Protocol.Constants.MaxBlobSize);
this.Blob = blob;
this.IntegrityControlVerificationEnabled = verifyBlocks;
this.options = options;
this.ReadAheadSize = readAheadInBytes;
this.downloadedBlocksList = new DownloadedBlockCollection();
}
public static InformationSource FromBlob(CloudBlob blob)
{
InformationSource source = CreateDefault();
source.SourceLocation = blob.Name;
source.SourceETag = blob.Properties.ETag;
source.SourceName = "";
source.SourceType = blob.GetBlobInformationType();
source.SourceInformationObjectType = blob.GetBlobInformationObjectType();
source.SourceETag = blob.Properties.ETag;
source.SourceMD5 = blob.Properties.ContentMD5;
source.SourceLastModified = blob.Properties.LastModifiedUtc;
return source;
}
private Uri GetBlobOrCdnUri(CloudBlob blob, string cdnHost)
{
// always use HTTP to avoid Silverlight cross-protocol issues
var ub = new UriBuilder(blob.Uri)
{
Scheme = "http"
};
if (ub.Port == 443) ub.Port = 80; // Adjust to port 80 only if port was already 443, or (below) if we're using a CDN
if (!string.IsNullOrEmpty(cdnHost))
{
ub.Host = cdnHost;
ub.Port = 80;
}
return ub.Uri;
}
/// <summary>
/// Возвращает поток с файлом из хранилища.
/// </summary>
/// <param name="localUri"> Адрес файла </param>
/// <returns> Поток </returns>
public Stream Get(string localUri)
{
CloudBlob blob = new CloudBlob(localUri, _client);
MemoryStream stream = new MemoryStream();
blob.DownloadToStream(stream, new BlobRequestOptions()
{
RetryPolicy = RetryPolicies.RetryExponential(
RetryPolicies.DefaultClientRetryCount,
RetryPolicies.DefaultClientBackoff)
});
stream.Seek(0, SeekOrigin.Begin);
return stream;
}
// The mutex must already exists
public Mutex(CloudBlobContainer container, string mutexName, Exception e)
{
blob = container.GetBlobReference(mutexName);
byte[] b1 = { 1 };
BlobRequestOptions requestOpt = new BlobRequestOptions();
bool keepGoing = true;
string oldEtag = "";
int lastChange = 0;
do
{
byte[] b;
string eTag;
try
{
blob.FetchAttributes();
eTag = blob.Attributes.Properties.ETag;
if (eTag != oldEtag)
{
lastChange = Environment.TickCount;
oldEtag = eTag;
}
b = blob.DownloadByteArray();
}
catch (Exception)
{
throw e;
}
requestOpt.AccessCondition = AccessCondition.IfMatch(eTag);
if (b[0] == 0 || Environment.TickCount - lastChange > 3000) // on ne peut garder un lock plus de 3 s
{
try
{
blob.UploadByteArray(b1, requestOpt);
keepGoing = false;
}
catch (StorageClientException ex)
{
if (ex.ErrorCode != StorageErrorCode.ConditionFailed)
throw;
}
}
else
Thread.Sleep(50); // constante arbitraire
} while (keepGoing);
}
public TaskRunner CreateUpdateTaskRunner(CloudBlob blob, FileInformation fileInfo)
{
this._statistics.UpdatedCount++;
if (fileInfo.SizeInBytes > CloudBlobConstants.FileSizeThresholdInBytes)
{
return new UploadLargeFileTaskRunner(_messageBus, _fileSystem, fileInfo, blob);
}
else
{
return new SingleActionTaskRunner(() =>
{
_messageBus.Publish(new FileProgressedMessage(fileInfo.FullPath, 0));
blob.UploadFile(fileInfo);
_messageBus.Publish(new FileProgressedMessage(fileInfo.FullPath, 1));
});
}
}
public static void DoLeaseOperation(VLogger logger, CloudBlob blob, string leaseId, LeaseAction action, int AzureBlobLeaseTimeout)
{
try
{
if (blob == null || leaseId == null)
return;
var creds = blob.ServiceClient.Credentials;
var transformedUri = new Uri(creds.TransformUri(blob.Uri.ToString()));
var req = BlobRequest.Lease(transformedUri, AzureBlobLeaseTimeout, action, leaseId);
creds.SignRequest(req);
req.GetResponse().Close();
}
catch (WebException e)
{
Utils.structuredLog(logger, "WebException", e.Message + ". DoLeaseOperation, blob: " + blob.Name + ", leaseId: " + leaseId + ", action " + action);
}
}
public bool Exists(CloudBlob blob)
{
try
{
blob.FetchAttributes();
return true;
}
catch (StorageClientException e)
{
if (e.ErrorCode == StorageErrorCode.ResourceNotFound)
{
return false;
}
else
{
throw;
}
}
}
