public async Task Write(ICloudBlob blob, string content)
{
using (Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(content)))
{
await blob.UploadFromStreamAsync(stream);
}
}
/// <summary>
/// Creates a new instance from a ICloudBlob.
/// </summary>
/// <param name="original">The original cloud blob.</param>
/// <returns></returns>
public static AzureBlobInfo Create(ICloudBlob original)
{
var file = original as CloudBlockBlob;
if (file != null)
{
var modified = DateTimeOffset.MinValue;
var modifiedOk = false;
if (file.Metadata.ContainsKey("CbModifiedTime"))
{
modifiedOk = DateTimeOffset.TryParse(file.Metadata["CbModifiedTime"], out modified);
}
return new AzureBlobInfo
{
BlobElement = file,
ContentType = file.Properties.ContentType,
Name = file.Name,
Size = file.Properties.Length,
Location = file.Uri.AbsoluteUri,
ModifiedTime = modifiedOk ? modified : default(DateTimeOffset?)
};
}
var folder = original as CloudBlobDirectory;
if (folder == null)
{
return null;
}
return new AzureBlobInfo
{
Name = folder.Prefix,
Location = folder.Uri.AbsoluteUri
};
}
public AzureIndexInput(AzureDirectory azuredirectory, ICloudBlob blob)
: base(blob.Name)
{
_name = blob.Uri.Segments[blob.Uri.Segments.Length - 1];
_fileMutex = BlobMutexManager.GrabMutex(_name);
_fileMutex.WaitOne();
try
{
_azureDirectory = azuredirectory;
_blobContainer = azuredirectory.BlobContainer;
_blob = blob;
string fileName = _name;
StreamOutput fileStream = _azureDirectory.CreateCachedOutputAsStream(fileName);
// get the blob
_blob.DownloadToStream(fileStream);
fileStream.Flush();
Debug.WriteLine("GET {0} RETREIVED {1} bytes", _name, fileStream.Length);
fileStream.Close();
// and open it as an input
_indexInput = CacheDirectory.openInput(fileName, IOContext.DEFAULT);
}
finally
{
_fileMutex.ReleaseMutex();
}
}
private static async Task CreateBlobIfNoneExists(ICloudBlob updateBlob)
{
if (!await updateBlob.ExistsAsync())
{
await BlobUtils.CreateEmptyBlob(updateBlob);
}
}
public UpdateBlob(ICloudBlob blob, IBlobLeaseFactory blobLeaseFactory)
{
_blob = blob;
_blobLeaseFactory = blobLeaseFactory;
_updateDomain = string.Empty;
_instanceIds = new HashSet<string>();
}
public async Task<string> Read(ICloudBlob blob)
{
using (Stream stream = await blob.OpenReadAsync())
using (StreamReader reader = new StreamReader(stream))
{
return await reader.ReadToEndAsync();
}
}
static async Task UploadTextAsync(ICloudBlob blob, string text, AccessCondition accessCondition)
{
blob.Properties.ContentEncoding = "UTF-8";
blob.Properties.ContentType = "text/plain";
using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(text))) {
await blob.UploadFromStreamAsync(stream, accessCondition, null, null);
}
}
public static string DownloadText(ICloudBlob blob, Encoding encoding, AccessCondition accessCondition = null, BlobRequestOptions options = null, OperationContext operationContext = null)
{
using (MemoryStream stream = new MemoryStream())
{
blob.DownloadToStream(stream, accessCondition, options, operationContext);
return encoding.GetString(stream.ToArray());
}
}
static void UploadText(ICloudBlob blob, string text, AccessCondition accessCondition)
{
blob.Properties.ContentEncoding = "UTF-8";
blob.Properties.ContentType = "text/plain";
using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(text)))
{
blob.UploadFromStream(stream, accessCondition);
}
}
public static async Task<string> DownloadTextAsync(ICloudBlob blob, Encoding encoding, AccessCondition accessCondition = null, BlobRequestOptions options = null, OperationContext operationContext = null)
{
using (MemoryStream stream = new MemoryStream())
{
await blob.DownloadToStreamAsync(stream.AsOutputStream(), accessCondition, options, operationContext);
byte[] buffer = stream.ToArray();
return encoding.GetString(buffer, 0, buffer.Length);
}
}
/// <summary>
/// Return a secret, persistent Download Url for a given Blob
/// </summary>
static string GetDownloadUrlFor(ICloudBlob blob)
{
var signature = blob.GetSharedAccessSignature(new SharedAccessBlobPolicy
{
SharedAccessExpiryTime = new DateTime(2050, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc)
}, DownloadPolicyName);
return string.Format("{0}{1}", blob.Uri.ToString().Replace(" ", "%20"), signature);
}
/// <summary>
/// Creates a job to copy from a blob.
/// </summary>
/// <param name="destBlob">Destination blob to copy to.
/// User should call the method on this object.</param>
/// <param name="sourceBlob">Source blob to copy from.</param>
/// <returns>Job object to do copying.</returns>
public static BlobCopyJob CreateCopyJob(
this ICloudBlob destBlob,
ICloudBlob sourceBlob)
{
return new BlobCopyJob()
{
SourceBlob = sourceBlob,
DestBlob = destBlob
};
}
public static async Task WaitForCopyAsync(ICloudBlob blob)
{
bool copyInProgress = true;
while (copyInProgress)
{
await Task.Delay(1000);
await blob.FetchAttributesAsync();
copyInProgress = (blob.CopyState.Status == CopyStatus.Pending);
}
}
public static void WaitForCopyTask(ICloudBlob blob)
{
bool copyInProgress = true;
while (copyInProgress)
{
Thread.Sleep(1000);
blob.FetchAttributesAsync().Wait();
copyInProgress = (blob.CopyState.Status == CopyStatus.Pending);
}
}
/// <summary>
/// Azure storage blob constructor
/// </summary>
/// <param name="blob">ICloud blob object</param>
public AzureStorageBlob(ICloudBlob blob)
{
Name = blob.Name;
ICloudBlob = blob;
BlobType = blob.BlobType;
Length = blob.Properties.Length;
ContentType = blob.Properties.ContentType;
LastModified = blob.Properties.LastModified;
SnapshotTime = blob.SnapshotTime;
}
public static string GetSasUrl(ICloudBlob blob, SharedAccessBlobPermissions permissions)
{
var policy = new SharedAccessBlobPolicy
{
SharedAccessExpiryTime = DateTime.Now.AddMinutes(30),
Permissions = permissions
};
string sasBlobToken = blob.GetSharedAccessSignature(policy);
//Return the URI string for the container, including the SAS token.
return blob.Uri + sasBlobToken;
}
//https://github.com/Particular/NServiceBus.Azure/blob/e9db29beb21d1fd914191e479cb5948fffd92f3b/src/NServiceBus.Azure/DataBus/Azure/BlobStorage/BlobStorageDataBus.cs#L41
protected override void SetValidUntil(ICloudBlob cloudBlob, TimeSpan timeToBeReceived)
{
if (timeToBeReceived == TimeSpan.MaxValue)
{
cloudBlob.Metadata["ValidUntil"] = TimeSpan.MaxValue.ToString();
}
else
{
cloudBlob.Metadata["ValidUntil"] = (DateTime.Now + timeToBeReceived).ToString();
}
}
private static async Task UpdateBlob(ICloudBlob blob)
{
try
{
byte[] dataBytes = new byte[40];
await blob.UploadFromByteArrayAsync(dataBytes, 0, 40);
}
catch (Exception e)
{
Console.WriteLine(e);
}
}
/// <summary>
/// Initializes a new instance of <see cref="OperationHandle" /> class.
/// </summary>
/// <param name="source">A source blob.</param>
/// <param name="target">A destination blob.</param>
/// <param name="sasToken">A security access token.</param>
public OperationHandle(ICloudBlob source, ICloudBlob target, String sasToken)
{
DeleteIsCompleted = false;
CopyIsCompleted = false;
SourceBlob = source;
TargetBlob = target;
this.sasToken = sasToken;
}
public SelfRenewableBlobLease(ICloudBlob blob, int renewIntervalInSeconds)
{
if (renewIntervalInSeconds > 60)
{
throw new Exception("Blob lease renew interval must be less than 60s");
}
if (renewIntervalInSeconds < 10)
{
throw new Exception("Blob lease renew interval must be at least 10s");
}
_renewIntervalInSeconds = renewIntervalInSeconds;
_blob = blob;
}
/// <summary>
/// Initializes a new instance of the BlobReadStreamBase class.
/// </summary>
/// <param name="blob">Blob reference to read from</param>
/// <param name="accessCondition">An object that represents the access conditions for the blob. If null, no condition is used.</param>
/// <param name="options">An object that specifies any additional options for the request.</param>
/// <param name="operationContext">An <see cref="OperationContext"/> object for tracking the current operation.</param>
protected BlobReadStreamBase(ICloudBlob blob, AccessCondition accessCondition, BlobRequestOptions options, OperationContext operationContext)
{
this.blob = blob;
this.isLengthAvailable = false;
this.currentOffset = 0;
this.buffer = new MemoryStream();
this.accessCondition = accessCondition;
this.lockedToETag = false;
this.options = options;
this.operationContext = operationContext;
this.blobMD5 = options.DisableContentMD5Validation.Value ? null : new MD5Wrapper();
this.lastException = null;
}
private bool UpdateProperty(ICloudBlob blob, string name, string value)
{
var property = blob.Properties.GetType().GetProperty(name.Replace("-", ""));
if (property != null)
{
property.SetValue((object)blob.Properties, (object)value);
var method = blob.GetType().GetMethod("SetProperties");
if (method != null)
{
method.Invoke(blob, new object[]{null,null,null});
return true;
}
}
return false;
}
private static void TestAccessTask(BlobContainerPublicAccessType accessType, CloudBlobContainer container, ICloudBlob inputBlob)
{
StorageCredentials credentials = new StorageCredentials();
container = new CloudBlobContainer(container.Uri, credentials);
CloudPageBlob blob = new CloudPageBlob(inputBlob.Uri, credentials);
if (accessType.Equals(BlobContainerPublicAccessType.Container))
{
blob.FetchAttributesAsync().Wait();
BlobContinuationToken token = null;
do
{
BlobResultSegment results = container.ListBlobsSegmented(token);
results.Results.ToArray();
token = results.ContinuationToken;
}
while (token != null);
container.FetchAttributesAsync().Wait();
}
else if (accessType.Equals(BlobContainerPublicAccessType.Blob))
{
blob.FetchAttributesAsync().Wait();
TestHelper.ExpectedExceptionTask(
container.ListBlobsSegmentedAsync(null),
"List blobs while public access does not allow for listing",
HttpStatusCode.NotFound);
TestHelper.ExpectedExceptionTask(
container.FetchAttributesAsync(),
"Fetch container attributes while public access does not allow",
HttpStatusCode.NotFound);
}
else
{
TestHelper.ExpectedExceptionTask(
blob.FetchAttributesAsync(),
"Fetch blob attributes while public access does not allow",
HttpStatusCode.NotFound);
TestHelper.ExpectedExceptionTask(
container.ListBlobsSegmentedAsync(null),
"List blobs while public access does not allow for listing",
HttpStatusCode.NotFound);
TestHelper.ExpectedExceptionTask(
container.FetchAttributesAsync(),
"Fetch container attributes while public access does not allow",
HttpStatusCode.NotFound);
}
}
internal BlobLeaseManager(ICloudBlob lockBlob, TimeSpan leaseTimeout, string hostId, string instanceId, TraceWriter traceWriter, TimeSpan? renewalInterval = null)
{
_lockBlob = lockBlob;
_leaseTimeout = leaseTimeout;
_traceWriter = traceWriter;
_hostId = hostId;
_instanceId = instanceId;
// Renew the lease three seconds before it expires
_renewalInterval = renewalInterval ?? leaseTimeout.Add(TimeSpan.FromSeconds(-3));
// Attempt to acquire a lease every 5 seconds
_leaseRetryInterval = TimeSpan.FromSeconds(5);
_timer = new Timer(ProcessLeaseTimerTick, null, TimeSpan.Zero, _leaseRetryInterval);
}
public static void AssertAreEqual(ICloudBlob blob1, ICloudBlob blob2)
{
if (blob1 == null)
{
Assert.IsNull(blob2);
}
else
{
Assert.IsNotNull(blob2);
Assert.AreEqual(blob1.BlobType, blob2.BlobType);
Assert.AreEqual(blob1.Uri, blob2.Uri);
Assert.AreEqual(blob1.SnapshotTime, blob2.SnapshotTime);
AssertAreEqual(blob1.Properties, blob2.Properties);
AssertAreEqual(blob1.CopyState, blob2.CopyState);
}
}
private static async Task TestAccessAsync(BlobContainerPublicAccessType accessType, CloudBlobContainer container, ICloudBlob inputBlob)
{
StorageCredentials credentials = new StorageCredentials();
container = new CloudBlobContainer(container.Uri, credentials);
CloudPageBlob blob = new CloudPageBlob(inputBlob.Uri, credentials);
OperationContext context = new OperationContext();
BlobRequestOptions options = new BlobRequestOptions();
if (accessType.Equals(BlobContainerPublicAccessType.Container))
{
await blob.FetchAttributesAsync();
await container.ListBlobsSegmentedAsync(null, true, BlobListingDetails.All, null, null, options, context);
await container.FetchAttributesAsync();
}
else if (accessType.Equals(BlobContainerPublicAccessType.Blob))
{
await blob.FetchAttributesAsync();
await TestHelper.ExpectedExceptionAsync(
async () => await container.ListBlobsSegmentedAsync(null, true, BlobListingDetails.All, null, null, options, context),
context,
"List blobs while public access does not allow for listing",
HttpStatusCode.NotFound);
await TestHelper.ExpectedExceptionAsync(
async () => await container.FetchAttributesAsync(null, options, context),
context,
"Fetch container attributes while public access does not allow",
HttpStatusCode.NotFound);
}
else
{
await TestHelper.ExpectedExceptionAsync(
async () => await blob.FetchAttributesAsync(null, options, context),
context,
"Fetch blob attributes while public access does not allow",
HttpStatusCode.NotFound);
await TestHelper.ExpectedExceptionAsync(
async () => await container.ListBlobsSegmentedAsync(null, true, BlobListingDetails.All, null, null, options, context),
context,
"List blobs while public access does not allow for listing",
HttpStatusCode.NotFound);
await TestHelper.ExpectedExceptionAsync(
async () => await container.FetchAttributesAsync(null, options, context),
context,
"Fetch container attributes while public access does not allow",
HttpStatusCode.NotFound);
}
}
private static async Task RenewLease(ICloudBlob blob, string leaseId)
{
var blobKey = GetKey(blob);
try
{
await blob.RenewLeaseAsync(
new AccessCondition
{
LeaseId = leaseId
});
Trace.TraceInformation("Renewed lease for blob {0}", blobKey);
}
catch (Exception ex)
{
Trace.TraceError("Failed to renew lease for blob {0}. Exception was: {1}", blobKey, ex);
}
}
public static IAsyncCloudBlob FromICloudBlob(ICloudBlob cloudBlob)
{
var block = cloudBlob as CloudBlockBlob;
if (block != null)
{
return new AsyncCloudBlockBlob(block);
}
var page = cloudBlob as CloudPageBlob;
if (page != null)
{
return new AsyncCloudPageBlob(page);
}
// unknown blob type... TODO - it might be better to actually throw an exception here
return new AsyncCloudBlobBase<ICloudBlob>(cloudBlob);
}
public AzureIndexOutput(AzureDirectory azureDirectory, ICloudBlob blob)
{
_fileMutex = BlobMutexManager.GrabMutex(_name);
_fileMutex.WaitOne();
try
{
_azureDirectory = azureDirectory;
_blob = blob;
_name = blob.Uri.Segments[blob.Uri.Segments.Length - 1];
// create the local cache one we will operate against...
_indexOutput = CacheDirectory.createOutput(_name, IOContext.DEFAULT);
}
finally
{
_fileMutex.ReleaseMutex();
}
}
/// <summary>
/// Initializes a new instance of the BlobReadStreamBase class.
/// </summary>
/// <param name="blob">Blob reference to read from</param>
/// <param name="accessCondition">An object that represents the access conditions for the blob. If null, no condition is used.</param>
/// <param name="options">An object that specifies additional options for the request.</param>
/// <param name="operationContext">An <see cref="OperationContext"/> object for tracking the current operation.</param>
protected BlobReadStreamBase(ICloudBlob blob, AccessCondition accessCondition, BlobRequestOptions options, OperationContext operationContext)
{
if (options.UseTransactionalMD5.Value)
{
CommonUtility.AssertInBounds("StreamMinimumReadSizeInBytes", blob.StreamMinimumReadSizeInBytes, 1, Constants.MaxRangeGetContentMD5Size);
}
this.blob = blob;
this.blobProperties = new BlobProperties(blob.Properties);
this.currentOffset = 0;
this.streamMinimumReadSizeInBytes = this.blob.StreamMinimumReadSizeInBytes;
this.internalBuffer = new MemoryStream(this.streamMinimumReadSizeInBytes);
this.accessCondition = accessCondition;
this.options = options;
this.operationContext = operationContext;
this.blobMD5 = (this.options.DisableContentMD5Validation.Value || string.IsNullOrEmpty(this.blobProperties.ContentMD5)) ? null : new MD5Wrapper();
this.lastException = null;
}
public AzureIndexInput(AzureDirectory azuredirectory, ICloudBlob blob)
{
_name = blob.Uri.Segments[blob.Uri.Segments.Length - 1];
#if FULLDEBUG
Debug.WriteLine(String.Format("opening {0} ", _name));
#endif
_fileMutex = BlobMutexManager.GrabMutex(_name);
_fileMutex.WaitOne();
try
{
_azureDirectory = azuredirectory;
_blobContainer = azuredirectory.BlobContainer;
_blob = blob;
string fileName = _name;
bool fFileNeeded = false;
if (!CacheDirectory.FileExists(fileName))
{
fFileNeeded = true;
}
else
{
long cachedLength = CacheDirectory.FileLength(fileName);
long blobLength = blob.Properties.Length;
long.TryParse(blob.Metadata["CachedLength"], out blobLength);
long longLastModified = 0;
DateTime blobLastModifiedUTC = blob.Properties.LastModified.Value.UtcDateTime;
if (long.TryParse(blob.Metadata["CachedLastModified"], out longLastModified))
{
// normalize RAMDirectory and FSDirectory times
if (longLastModified > ticks1970)
{
longLastModified -= ticks1970;
}
blobLastModifiedUTC = new DateTime(longLastModified).ToUniversalTime();
}
if (cachedLength != blobLength)
{
fFileNeeded = true;
}
else
{
// there seems to be an error of 1 tick which happens every once in a while
// for now we will say that if they are within 1 tick of each other and same length
var elapsed = CacheDirectory.FileModified(fileName);
// normalize RAMDirectory and FSDirectory times
if (elapsed > ticks1970)
{
elapsed -= ticks1970;
}
DateTime cachedLastModifiedUTC = new DateTime(elapsed, DateTimeKind.Local).ToUniversalTime();
if (cachedLastModifiedUTC != blobLastModifiedUTC)
{
TimeSpan timeSpan = blobLastModifiedUTC.Subtract(cachedLastModifiedUTC);
if (timeSpan.TotalSeconds > 1)
{
fFileNeeded = true;
}
else
{
#if FULLDEBUG
Debug.WriteLine(timeSpan.TotalSeconds);
#endif
// file not needed
}
}
}
}
// if the file does not exist
// or if it exists and it is older then the lastmodified time in the blobproperties (which always comes from the blob storage)
if (fFileNeeded)
{
#if COMPRESSBLOBS
if (_azureDirectory.ShouldCompressFile(_name))
{
// then we will get it fresh into local deflatedName
// StreamOutput deflatedStream = new StreamOutput(CacheDirectory.CreateOutput(deflatedName));
MemoryStream deflatedStream = new MemoryStream();
// get the deflated blob
_blob.DownloadToStream(deflatedStream);
Debug.WriteLine(string.Format("GET {0} RETREIVED {1} bytes", _name, deflatedStream.Length));
// seek back to begininng
deflatedStream.Seek(0, SeekOrigin.Begin);
// open output file for uncompressed contents
StreamOutput fileStream = _azureDirectory.CreateCachedOutputAsStream(fileName);
// create decompressor
DeflateStream decompressor = new DeflateStream(deflatedStream, CompressionMode.Decompress);
byte[] bytes = new byte[65535];
int nRead = 0;
do
{
nRead = decompressor.Read(bytes, 0, 65535);
if (nRead > 0)
{
fileStream.Write(bytes, 0, nRead);
}
} while (nRead == 65535);
decompressor.Close(); // this should close the deflatedFileStream too
fileStream.Close();
}
else
#endif
{
StreamOutput fileStream = _azureDirectory.CreateCachedOutputAsStream(fileName);
// get the blob
_blob.DownloadToStream(fileStream);
fileStream.Flush();
Debug.WriteLine(string.Format("GET {0} RETREIVED {1} bytes", _name, fileStream.Length));
fileStream.Close();
}
// and open it as an input
_indexInput = CacheDirectory.OpenInput(fileName);
}
else
{
#if FULLDEBUG
Debug.WriteLine(String.Format("Using cached file for {0}", _name));
#endif
// open the file in read only mode
_indexInput = CacheDirectory.OpenInput(fileName);
}
}
finally
{
_fileMutex.ReleaseMutex();
}
}
private static async Task TestAccessAsync(BlobContainerPublicAccessType accessType, CloudBlobContainer container, ICloudBlob inputBlob)
{
StorageCredentials credentials = new StorageCredentials();
container = new CloudBlobContainer(container.Uri, credentials);
CloudPageBlob blob = new CloudPageBlob(inputBlob.Uri, credentials);
OperationContext context = new OperationContext();
BlobRequestOptions options = new BlobRequestOptions();
if (accessType.Equals(BlobContainerPublicAccessType.Container))
{
await blob.FetchAttributesAsync();
await container.ListBlobsSegmentedAsync(null, true, BlobListingDetails.All, null, null, options, context);
await container.FetchAttributesAsync();
}
else if (accessType.Equals(BlobContainerPublicAccessType.Blob))
{
await blob.FetchAttributesAsync();
await TestHelper.ExpectedExceptionAsync(
async() => await container.ListBlobsSegmentedAsync(null, true, BlobListingDetails.All, null, null, options, context),
context,
"List blobs while public access does not allow for listing",
HttpStatusCode.NotFound);
await TestHelper.ExpectedExceptionAsync(
async() => await container.FetchAttributesAsync(null, options, context),
context,
"Fetch container attributes while public access does not allow",
HttpStatusCode.NotFound);
}
else
{
await TestHelper.ExpectedExceptionAsync(
async() => await blob.FetchAttributesAsync(null, options, context),
context,
"Fetch blob attributes while public access does not allow",
HttpStatusCode.NotFound);
await TestHelper.ExpectedExceptionAsync(
async() => await container.ListBlobsSegmentedAsync(null, true, BlobListingDetails.All, null, null, options, context),
context,
"List blobs while public access does not allow for listing",
HttpStatusCode.NotFound);
await TestHelper.ExpectedExceptionAsync(
async() => await container.FetchAttributesAsync(null, options, context),
context,
"Fetch container attributes while public access does not allow",
HttpStatusCode.NotFound);
}
}
private static void SetAttachmentAsContentDisposition(ICloudBlob resizedPhotoCloudBlob,
PictureResizeRequest pictureResizeRequest)
{
resizedPhotoCloudBlob.Properties.ContentDisposition =
$"attachment; filename={pictureResizeRequest.RequiredWidth}x{pictureResizeRequest.RequiredHeight}";
}
/// <summary>
/// Initializes a new instance of the BlobReadStream class.
/// </summary>
/// <param name="blob">Blob reference to read from</param>
/// <param name="accessCondition">An object that represents the access conditions for the blob. If null, no condition is used.</param>
/// <param name="options">An object that specifies additional options for the request.</param>
/// <param name="operationContext">An <see cref="OperationContext"/> object for tracking the current operation.</param>
internal BlobReadStream(ICloudBlob blob, AccessCondition accessCondition, BlobRequestOptions options, OperationContext operationContext)
: base(blob, accessCondition, options, operationContext)
{
}
private async Task StoreElementAsync(ICloudBlob blobRef, XElement element)
{
// holds the last error in case we need to rethrow it
ExceptionDispatchInfo lastError = null;
for (var i = 0; i < ConflictMaxRetries; i++)
{
if (i > 1)
{
// If multiple conflicts occurred, wait a small period of time before retrying
// the operation so that other writers can make forward progress.
await Task.Delay(GetRandomizedBackoffPeriod());
}
if (i > 0)
{
// If at least one conflict occurred, make sure we have an up-to-date
// view of the blob contents.
await GetLatestDataAsync(blobRef);
}
// Merge the new element into the document. If no document exists,
// create a new default document and inject this element into it.
var latestData = Volatile.Read(ref _cachedBlobData);
var doc = (latestData != null)
? CreateDocumentFromBlob(latestData.BlobContents)
: new XDocument(new XElement(RepositoryElementName));
doc.Root.Add(element);
// Turn this document back into a byte[].
var serializedDoc = new MemoryStream();
doc.Save(serializedDoc, SaveOptions.DisableFormatting);
// Generate the appropriate precondition header based on whether or not
// we believe data already exists in storage.
AccessCondition accessCondition;
if (latestData != null)
{
accessCondition = AccessCondition.GenerateIfMatchCondition(blobRef.Properties.ETag);
}
else
{
accessCondition = AccessCondition.GenerateIfNotExistsCondition();
blobRef.Properties.ContentType = "application/xml; charset=utf-8"; // set content type on first write
}
try
{
// Send the request up to the server.
var serializedDocAsByteArray = serializedDoc.ToArray();
await blobRef.UploadFromByteArrayAsync(
buffer : serializedDocAsByteArray,
index : 0,
count : serializedDocAsByteArray.Length,
accessCondition : accessCondition,
options : null,
operationContext : null);
// If we got this far, success!
// We can update the cached view of the remote contents.
Volatile.Write(ref _cachedBlobData, new BlobData()
{
BlobContents = serializedDocAsByteArray,
ETag = blobRef.Properties.ETag // was updated by Upload routine
});
return;
}
catch (StorageException ex)
when(ex.RequestInformation.HttpStatusCode == 409 || ex.RequestInformation.HttpStatusCode == 412)
{
// 409 Conflict
// This error is rare but can be thrown in very special circumstances,
// such as if the blob in the process of being created. We treat it
// as equivalent to 412 for the purposes of retry logic.
// 412 Precondition Failed
// We'll get this error if another writer updated the repository and we
// have an outdated view of its contents. If this occurs, we'll just
// refresh our view of the remote contents and try again up to the max
// retry limit.
lastError = ExceptionDispatchInfo.Capture(ex);
}
}
// if we got this far, something went awry
lastError.Throw();
}
public static void Run([BlobTrigger(BlobPath)] ICloudBlob blob)
{
TaskSource.TrySetResult(blob);
}
/// <summary>
/// Extracts file name from the Cloud BLOB.
/// </summary>
/// <param name="blob">Cloud BLOB</param>
/// <returns>Name of the file.</returns>
private string GetBlobLocalName(ICloudBlob blob)
{
return(blob.Name.Substring(blob.Name.LastIndexOf(BlobPathSeparator) + 1));
}
private static bool NotExists(IEnumerable <ICloudBlob> targetBlobs,
ICloudBlob b)
{
return(targetBlobs.All(tb => tb.Name != b.Name));
}
private static void TestAccess(string sasToken, SharedAccessBlobPermissions permissions, SharedAccessBlobHeaders headers, CloudBlobContainer container, ICloudBlob blob)
{
StorageCredentials credentials = string.IsNullOrEmpty(sasToken) ?
new StorageCredentials() :
new StorageCredentials(sasToken);
if (container != null)
{
container = new CloudBlobContainer(credentials.TransformUri(container.Uri));
if (blob.BlobType == BlobType.BlockBlob)
{
blob = container.GetBlockBlobReference(blob.Name);
}
else
{
blob = container.GetPageBlobReference(blob.Name);
}
}
else
{
if (blob.BlobType == BlobType.BlockBlob)
{
blob = new CloudBlockBlob(credentials.TransformUri(blob.Uri));
}
else
{
blob = new CloudPageBlob(credentials.TransformUri(blob.Uri));
}
}
if (container != null)
{
if ((permissions & SharedAccessBlobPermissions.List) == SharedAccessBlobPermissions.List)
{
container.ListBlobs().ToArray();
}
else
{
TestHelper.ExpectedException(
() => container.ListBlobs().ToArray(),
"List blobs while SAS does not allow for listing",
HttpStatusCode.NotFound);
}
}
if ((permissions & SharedAccessBlobPermissions.Read) == SharedAccessBlobPermissions.Read)
{
blob.FetchAttributes();
// Test headers
if (headers != null)
{
if (headers.CacheControl != null)
{
Assert.AreEqual(headers.CacheControl, blob.Properties.CacheControl);
}
if (headers.ContentDisposition != null)
{
Assert.AreEqual(headers.ContentDisposition, blob.Properties.ContentDisposition);
}
if (headers.ContentEncoding != null)
{
Assert.AreEqual(headers.ContentEncoding, blob.Properties.ContentEncoding);
}
if (headers.ContentLanguage != null)
{
Assert.AreEqual(headers.ContentLanguage, blob.Properties.ContentLanguage);
}
if (headers.ContentType != null)
{
Assert.AreEqual(headers.ContentType, blob.Properties.ContentType);
}
}
}
else
{
TestHelper.ExpectedException(
() => blob.FetchAttributes(),
"Fetch blob attributes while SAS does not allow for reading",
HttpStatusCode.NotFound);
}
if ((permissions & SharedAccessBlobPermissions.Write) == SharedAccessBlobPermissions.Write)
{
blob.SetMetadata();
}
else
{
TestHelper.ExpectedException(
() => blob.SetMetadata(),
"Set blob metadata while SAS does not allow for writing",
HttpStatusCode.NotFound);
}
if ((permissions & SharedAccessBlobPermissions.Delete) == SharedAccessBlobPermissions.Delete)
{
blob.Delete();
}
else
{
TestHelper.ExpectedException(
() => blob.Delete(),
"Delete blob while SAS does not allow for deleting",
HttpStatusCode.NotFound);
}
}
/// <summary>
/// fetch blob attributes
/// </summary>
/// <param name="accessCondition">Access condition</param>
/// <param name="options">blob request options</param>
/// <param name="operationContext">An object that represents the context for the current operation.</param>
public void FetchBlobAttributes(ICloudBlob blob, AccessCondition accessCondition, BlobRequestOptions options, OperationContext operationContext)
{
return;
}
internal OutputFileReference(ICloudBlob blob)
{
CloudBlob = blob;
}
/// <summary>
/// Write ICloudBlob to output using specified service channel
/// </summary>
/// <param name="blob">The output ICloudBlob object</param>
/// <param name="channel">IStorageBlobManagement channel object</param>
internal void WriteICloudBlobObject(long taskId, IStorageBlobManagement channel, ICloudBlob blob, BlobContinuationToken continuationToken = null)
{
AzureStorageBlob azureBlob = new AzureStorageBlob(blob);
azureBlob.Context = channel.StorageContext;
azureBlob.ContinuationToken = continuationToken;
OutputStream.WriteObject(taskId, azureBlob);
}
protected TransferBlobResult TransferBlob(ICloudBlob blob)
{
return(new TransferBlobResult(blob));
}
/// <summary>
/// set azure blob
/// </summary>
/// <param name="fileName">local file name</param>
/// <param name="blob">destination blob</param>
/// <param name="isValidContainer">whether the destination container is validated</param>
/// <returns>null if user cancel the overwrite operation, otherwise return destination blob object</returns>
internal AzureStorageBlob SetAzureBlobContent(string fileName, ICloudBlob blob, bool isValidContainer = false)
{
string filePath = GetFullSendFilePath(fileName);
if (String.IsNullOrEmpty(filePath))
{
return(null);
}
if (null == blob)
{
throw new ArgumentException(String.Format(Resources.ObjectCannotBeNull, typeof(ICloudBlob).Name));
}
if (blob.BlobType == WindowsAzure.Storage.Blob.BlobType.PageBlob)
{
long fileSize = new FileInfo(filePath).Length;
long pageBlobUnit = 512;
long remainder = fileSize % pageBlobUnit;
if (remainder != 0)
{
//the blob size must be a multiple of 512 bytes.
throw new ArgumentException(String.Format(Resources.InvalidPageBlobSize, filePath, fileSize));
}
}
if (!NameUtil.IsValidBlobName(blob.Name))
{
throw new ArgumentException(String.Format(Resources.InvalidBlobName, blob.Name));
}
if (!isValidContainer)
{
ValidatePipelineCloudBlobContainer(blob.Container);
}
AccessCondition accessCondition = null;
BlobRequestOptions requestOptions = RequestOptions;
ICloudBlob blobRef = Channel.GetBlobReferenceFromServer(blob.Container, blob.Name, accessCondition, requestOptions, OperationContext);
if (null != blobRef)
{
if (blob.BlobType != blobRef.BlobType)
{
throw new ArgumentException(String.Format(Resources.BlobTypeMismatch, blobRef.Name, blobRef.BlobType));
}
if (!overwrite)
{
if (!ConfirmOverwrite(blob.Name))
{
return(null);
}
}
}
try
{
Upload2Blob(filePath, blob);
try
{
Channel.FetchBlobAttributes(blob, accessCondition, requestOptions, OperationContext);
}
catch (StorageException se)
{
//Storage credentials only have the write permission, and don't have the read permission.
if (se.IsNotFoundException())
{
WriteVerboseWithTimestamp(Resources.BlobIsNotReadable);
}
else
{
throw;
}
}
}
catch (Exception e)
{
WriteDebugLog(String.Format(Resources.Upload2BlobFailed, e.Message));
throw;
}
return(new AzureStorageBlob(blob));
}
public AzureIndexInput(AzureDirectory azuredirectory, ICloudBlob blob)
{
_name = blob.Uri.Segments[blob.Uri.Segments.Length - 1];
#if FULLDEBUG
Debug.WriteLine(String.Format("opening {0} ", _name));
#endif
_fileMutex = BlobMutexManager.GrabMutex(_name);
_fileMutex.WaitOne();
try
{
_azureDirectory = azuredirectory;
_blobContainer = azuredirectory.BlobContainer;
_blob = blob;
var fileName = _name;
var fFileNeeded = false;
if (!CacheDirectory.FileExists(fileName))
{
fFileNeeded = true;
}
else
{
long cachedLength = CacheDirectory.FileLength(fileName);
string blobLengthMetadata;
bool hasMetadataValue = blob.Metadata.TryGetValue("CachedLength", out blobLengthMetadata);
long blobLength = blob.Properties.Length;
if (hasMetadataValue)
{
long.TryParse(blobLengthMetadata, out blobLength);
}
string blobLastModifiedMetadata;
long longLastModified = 0;
DateTime blobLastModifiedUTC = blob.Properties.LastModified.Value.UtcDateTime;
if (blob.Metadata.TryGetValue("CachedLastModified", out blobLastModifiedMetadata))
{
if (long.TryParse(blobLastModifiedMetadata, out longLastModified))
{
blobLastModifiedUTC = new DateTime(longLastModified).ToUniversalTime();
}
}
if (cachedLength != blobLength)
{
fFileNeeded = true;
}
else
{
// there seems to be an error of 1 tick which happens every once in a while
// for now we will say that if they are within 1 tick of each other and same length
var cachedLastModifiedUTC = new DateTime(CacheDirectory.FileModified(fileName), DateTimeKind.Local).ToUniversalTime();
if (cachedLastModifiedUTC != blobLastModifiedUTC)
{
var timeSpan = blobLastModifiedUTC.Subtract(cachedLastModifiedUTC);
if (timeSpan.TotalSeconds > 1)
{
fFileNeeded = true;
}
else
{
#if FULLDEBUG
Debug.WriteLine(timeSpan.TotalSeconds);
#endif
// file not needed
}
}
}
}
// if the file does not exist
// or if it exists and it is older then the lastmodified time in the blobproperties (which always comes from the blob storage)
if (fFileNeeded)
{
if (_azureDirectory.ShouldCompressFile(_name))
{
InflateStream(fileName);
}
else
{
using (var fileStream = _azureDirectory.CreateCachedOutputAsStream(fileName))
{
// get the blob
_blob.DownloadToStream(fileStream);
fileStream.Flush();
Debug.WriteLine(string.Format("GET {0} RETREIVED {1} bytes", _name, fileStream.Length));
}
}
// and open it as an input
_indexInput = CacheDirectory.OpenInput(fileName);
}
else
{
#if FULLDEBUG
Debug.WriteLine(String.Format("Using cached file for {0}", _name));
#endif
// open the file in read only mode
_indexInput = CacheDirectory.OpenInput(fileName);
}
}
finally
{
_fileMutex.ReleaseMutex();
}
}
/// <summary>
/// Deletes the file.
/// </summary>
/// <param name="bucketName">Name of the bucket.</param>
/// <param name="keyName">Name of the key.</param>
public void DeleteFile(string bucketName, string keyName)
{
ICloudBlob blob = GetCloudBlob(bucketName, keyName);
blob.DeleteIfExists();
}
private void BlobReadStreamReadSizeTestAPM(ICloudBlob blob)
{
IAsyncResult result;
using (AutoResetEvent waitHandle = new AutoResetEvent(false))
{
byte[] buffer = GetRandomBuffer(5 * 1024 * 1024);
using (MemoryStream wholeBlob = new MemoryStream(buffer))
{
result = blob.BeginUploadFromStream(wholeBlob,
ar => waitHandle.Set(),
null);
waitHandle.WaitOne();
blob.EndUploadFromStream(result);
}
TestHelper.ExpectedException <ArgumentOutOfRangeException>(
() => blob.StreamMinimumReadSizeInBytes = 16 * 1024 - 1,
"StreamMinimumReadSizeInBytes should not accept values smaller than 16KB");
blob.StreamMinimumReadSizeInBytes = 4 * 1024 * 1024 + 1;
BlobRequestOptions options = new BlobRequestOptions()
{
UseTransactionalMD5 = true
};
result = blob.BeginOpenRead(null, options, null,
ar => waitHandle.Set(),
null);
waitHandle.WaitOne();
TestHelper.ExpectedException <ArgumentOutOfRangeException>(
() => blob.EndOpenRead(result),
"StreamMinimumReadSizeInBytes should be smaller than 4MB if UseTransactionalMD5 is true");
string range = null;
OperationContext context = new OperationContext();
context.SendingRequest += (sender, e) => range = range ?? e.Request.Headers["x-ms-range"];
blob.StreamMinimumReadSizeInBytes = 4 * 1024 * 1024;
result = blob.BeginOpenRead(null, options, context,
ar => waitHandle.Set(),
null);
waitHandle.WaitOne();
using (Stream blobStream = blob.EndOpenRead(result))
{
blobStream.ReadByte();
Assert.AreEqual("bytes=0-" + (blob.StreamMinimumReadSizeInBytes - 1).ToString(), range);
range = null;
}
blob.StreamMinimumReadSizeInBytes = 6 * 1024 * 1024;
options.UseTransactionalMD5 = false;
result = blob.BeginOpenRead(null, options, context,
ar => waitHandle.Set(),
null);
waitHandle.WaitOne();
using (Stream blobStream = blob.EndOpenRead(result))
{
blobStream.ReadByte();
Assert.AreEqual("bytes=0-" + (buffer.Length - 1).ToString(), range);
range = null;
}
blob.StreamMinimumReadSizeInBytes = 16 * 1024;
result = blob.BeginOpenRead(null, options, context,
ar => waitHandle.Set(),
null);
waitHandle.WaitOne();
using (Stream blobStream = blob.EndOpenRead(result))
{
blobStream.ReadByte();
Assert.AreEqual("bytes=0-" + (blob.StreamMinimumReadSizeInBytes - 1).ToString(), range);
range = null;
}
}
}
public async Task <Stream> GetAsync(string blobName, string containerName = null)
{
ICloudBlob blob = await GetBlobReferenceFromServerAsync(blobName, containerName);
return(await blob.OpenReadAsync(null, null, null));
}
private async Task <Tuple <IDisposable, string> > LockInternal(ICloudBlob blob)
{
string leaseId;
try
{
leaseId = await blob.AcquireLeaseAsync(TimeSpan.FromMinutes(1), null).ConfigureAwait(false);
LeoTrace.WriteLine("Leased Blob: " + blob.Name);
}
catch (StorageException e)
{
// If we have a conflict this blob is already locked...
if (e.RequestInformation.HttpStatusCode == 409)
{
return(null);
}
if (e.RequestInformation.HttpStatusCode == 404)
{
leaseId = null;
}
else
{
throw e.Wrap(blob.Name);
}
}
// May not have had a blob pushed...
if (leaseId == null)
{
try
{
using (var stream = new MemoryStream(new byte[1]))
{
try
{
await blob.UploadFromStreamAsync(stream).ConfigureAwait(false);
}
catch (StorageException) { } // Just eat storage exceptions at this point... something was created obviously
}
leaseId = await blob.AcquireLeaseAsync(TimeSpan.FromMinutes(1), null).ConfigureAwait(false);
LeoTrace.WriteLine("Created new blob and lease (2 calls): " + blob.Name);
}
catch (StorageException e)
{
// If we have a conflict this blob is already locked...
if (e.RequestInformation.HttpStatusCode == 409)
{
return(null);
}
if (e.RequestInformation.HttpStatusCode == 404)
{
return(null);
}
else
{
throw e.Wrap(blob.Name);
}
}
}
var condition = AccessCondition.GenerateLeaseCondition(leaseId);
// Every 30 secs keep the lock renewed
var keepAlive = AsyncEnumerableEx.CreateTimer(TimeSpan.FromSeconds(30))
.Select(t =>
{
LeoTrace.WriteLine("Renewed Lease: " + blob.Name);
return(blob.RenewLeaseAsync(condition));
})
.Unwrap()
.TakeUntilDisposed(null, t =>
{
try
{
// We need to do this to make sure after the dispose the lease is gone
blob.ReleaseLeaseAsync(condition).GetAwaiter().GetResult();
}
catch (Exception e)
{
LeoTrace.WriteLine("Release failed: " + e.Message);
}
});
return(Tuple.Create((IDisposable)keepAlive, leaseId));
}
public ICloudBlob GetReferenceForItem(string itemUrl)
{
ICloudBlob blobRef = Client.GetBlobReferenceFromServer(new Uri(itemUrl));
return(blobRef);
}
private async Task DoUploadDownloadFileAsync(ICloudBlob blob, int fileSize)
{
string inputFileName = Path.GetTempFileName();
string outputFileName = Path.GetTempFileName();
try
{
byte[] buffer = GetRandomBuffer(fileSize);
using (FileStream file = new FileStream(inputFileName, FileMode.Create, FileAccess.Write))
{
await file.WriteAsync(buffer, 0, buffer.Length);
}
await blob.UploadFromFileAsync(inputFileName, FileMode.Open);
OperationContext context = new OperationContext();
await blob.UploadFromFileAsync(inputFileName, FileMode.Open, null, null, context);
Assert.IsNotNull(context.LastResult.ServiceRequestID);
await TestHelper.ExpectedExceptionAsync <IOException>(
async() => await blob.DownloadToFileAsync(outputFileName, FileMode.CreateNew),
"CreateNew on an existing file should fail");
context = new OperationContext();
await blob.DownloadToFileAsync(outputFileName, FileMode.Create, null, null, context);
Assert.IsNotNull(context.LastResult.ServiceRequestID);
using (
FileStream inputFileStream = new FileStream(inputFileName, FileMode.Open, FileAccess.Read),
outputFileStream = new FileStream(outputFileName, FileMode.Open, FileAccess.Read))
{
await TestHelper.AssertStreamsAreEqualAsync(inputFileStream, outputFileStream);
}
await blob.DownloadToFileAsync(outputFileName, FileMode.Append);
using (
FileStream inputFileStream = new FileStream(inputFileName, FileMode.Open, FileAccess.Read),
outputFileStream = new FileStream(outputFileName, FileMode.Open, FileAccess.Read))
{
Assert.AreEqual(2 * fileSize, outputFileStream.Length);
for (int i = 0; i < fileSize; i++)
{
Assert.AreEqual(inputFileStream.ReadByte(), outputFileStream.ReadByte());
}
inputFileStream.Seek(0, SeekOrigin.Begin);
for (int i = 0; i < fileSize; i++)
{
Assert.AreEqual(inputFileStream.ReadByte(), outputFileStream.ReadByte());
}
}
}
finally
{
File.Delete(inputFileName);
File.Delete(outputFileName);
}
}
public void Notify(ICloudBlob blobWritten)
{
_blobWrittenNotifications.Enqueue(blobWritten);
}
/// <summary>
/// Implements getting the stream without specifying a range.
/// </summary>
/// <param name="blob">The blob.</param>
/// <param name="attributes">The attributes.</param>
/// <param name="destStream">The destination stream.</param>
/// <param name="offset">The offset.</param>
/// <param name="length">The length.</param>
/// <param name="accessCondition">An <see cref="AccessCondition"/> object that represents the condition that must be met in order for the request to proceed. If <c>null</c>, no condition is used.</param>
/// <param name="options">A <see cref="BlobRequestOptions"/> object that specifies additional options for the request.</param>
/// <returns>
/// A <see cref="RESTCommand{T}"/> that gets the stream.
/// </returns>
internal static RESTCommand <NullType> GetBlobImpl(ICloudBlob blob, BlobAttributes attributes, Stream destStream, long?offset, long?length, AccessCondition accessCondition, BlobRequestOptions options)
{
string lockedETag = null;
AccessCondition lockedAccessCondition = null;
bool isRangeGet = offset.HasValue;
bool arePropertiesPopulated = false;
string storedMD5 = null;
long startingOffset = offset.HasValue ? offset.Value : 0;
long?startingLength = length;
long?validateLength = null;
RESTCommand <NullType> getCmd = new RESTCommand <NullType>(blob.ServiceClient.Credentials, attributes.StorageUri);
options.ApplyToStorageCommand(getCmd);
getCmd.CommandLocationMode = CommandLocationMode.PrimaryOrSecondary;
getCmd.RetrieveResponseStream = true;
getCmd.DestinationStream = destStream;
getCmd.CalculateMd5ForResponseStream = !options.DisableContentMD5Validation.Value;
getCmd.BuildRequestDelegate = (uri, builder, serverTimeout, ctx) =>
BlobHttpWebRequestFactory.Get(uri, serverTimeout, attributes.SnapshotTime, offset, length, options.UseTransactionalMD5.Value, accessCondition, ctx);
getCmd.SignRequest = blob.ServiceClient.AuthenticationHandler.SignRequest;
getCmd.RecoveryAction = (cmd, ex, ctx) =>
{
if ((lockedAccessCondition == null) && !string.IsNullOrEmpty(lockedETag))
{
lockedAccessCondition = AccessCondition.GenerateIfMatchCondition(lockedETag);
if (accessCondition != null)
{
lockedAccessCondition.LeaseId = accessCondition.LeaseId;
}
}
if (cmd.StreamCopyState != null)
{
offset = startingOffset + cmd.StreamCopyState.Length;
if (startingLength.HasValue)
{
length = startingLength.Value - cmd.StreamCopyState.Length;
}
}
getCmd.BuildRequestDelegate = (uri, builder, serverTimeout, context) =>
BlobHttpWebRequestFactory.Get(uri, serverTimeout, attributes.SnapshotTime, offset, length, options.UseTransactionalMD5.Value && !arePropertiesPopulated, lockedAccessCondition ?? accessCondition, context);
};
getCmd.PreProcessResponse = (cmd, resp, ex, ctx) =>
{
HttpResponseParsers.ProcessExpectedStatusCodeNoException(offset.HasValue ? HttpStatusCode.PartialContent : HttpStatusCode.OK, resp, NullType.Value, cmd, ex);
if (!arePropertiesPopulated)
{
CloudBlobSharedImpl.UpdateAfterFetchAttributes(attributes, resp, isRangeGet);
storedMD5 = resp.Headers[HttpResponseHeader.ContentMd5];
if (!options.DisableContentMD5Validation.Value &&
options.UseTransactionalMD5.Value &&
string.IsNullOrEmpty(storedMD5))
{
throw new StorageException(
cmd.CurrentResult,
SR.MD5NotPresentError,
null)
{
IsRetryable = false
};
}
// If the download fails and Get Blob needs to resume the download, going to the
// same storage location is important to prevent a possible ETag mismatch.
getCmd.CommandLocationMode = cmd.CurrentResult.TargetLocation == StorageLocation.Primary ? CommandLocationMode.PrimaryOnly : CommandLocationMode.SecondaryOnly;
lockedETag = attributes.Properties.ETag;
if (resp.ContentLength >= 0)
{
validateLength = resp.ContentLength;
}
arePropertiesPopulated = true;
}
return(NullType.Value);
};
getCmd.PostProcessResponse = (cmd, resp, ctx) =>
{
HttpResponseParsers.ValidateResponseStreamMd5AndLength(validateLength, storedMD5, cmd);
return(NullType.Value);
};
return(getCmd);
}
/// <summary>
/// Converts the source blob of a copy operation to an appropriate access URI, taking Shared Access Signature credentials into account.
/// </summary>
/// <param name="source">The source blob.</param>
/// <returns>A URI addressing the source blob, using SAS if appropriate.</returns>
internal static Uri SourceBlobToUri(ICloudBlob source)
{
CommonUtility.AssertNotNull("source", source);
return(source.ServiceClient.Credentials.TransformUri(source.SnapshotQualifiedUri));
}
private void Sync()
{
try
{
//we first initialize a new replica in the new server
if (!targetContainer.Exists())
{
targetContainer.Create();
}
}
catch (StorageException ex)
{
//409 is the conflict message.
if (StorageExceptionCode.Conflict(ex))
{
throw ex;
}
}
if (lastModified == null)
{
targetContainer.FetchAttributes();
if (targetContainer.Metadata.ContainsKey("lastsync"))
{
//if no lastmodified time is provided in the constructor, we still try to be fast.
//So, we check to see if by any chance the container previously has synchronized.
lastModified = DateTimeOffset.Parse(targetContainer.Metadata["lastsync"]);
}
}
DateTimeOffset startTime = DateTimeOffset.Now;
try
{
IEnumerable <IListBlobItem> sourceBlobList = null;
if (lastModified != null)
{
sourceBlobList = sourceContainer.ListBlobs().OfType <ICloudBlob>().Where(b => b.Properties.LastModified > lastModified);
}
else
{
//this is the slowest sync path. It needs to go through all blobs.
//it tries to avoid this path as much as possible.
sourceBlobList = sourceContainer.ListBlobs();
}
foreach (ICloudBlob sourceBlob in sourceBlobList)
{
ICloudBlob targetBlob = targetContainer.GetBlockBlobReference(sourceBlob.Name);
//Since its an inter-account transfer, we need access rights to do the transfer.
var sas = sourceContainer.GetSharedAccessSignature(new SharedAccessBlobPolicy()
{
SharedAccessStartTime = DateTime.UtcNow.AddMinutes(-5),
SharedAccessExpiryTime = DateTime.UtcNow.AddMinutes(30),
Permissions = SharedAccessBlobPermissions.Read,
});
AccessCondition ac = new AccessCondition();
Interlocked.Increment(ref concurrentSyncs);
var srcBlockBlobSasUri = string.Format("{0}{1}", sourceBlob.Uri, sas);
targetBlob.BeginStartCopyFromBlob(new Uri(srcBlockBlobSasUri), BlobCopyFinished, null);
}
while (concurrentSyncs > 0)
{
lock (mylock)
Monitor.Wait(mylock, 200);
}
targetContainer.Metadata["lastsync"] = startTime.ToString();
targetContainer.SetMetadata();
}
catch (StorageException se)
{
//if the blob/container does not exist, it means that it is removed by configurator.
//We safely return.
if (StorageExceptionCode.NotFound(se))
{
return;
}
throw se;
}
catch (Exception ex)
{
throw ex;
}
}
static private IAsset CreateAssetFromExistingBlobs(CloudBlobContainer sourceBlobContainer)
{
CloudBlobClient destBlobStorage = _destinationStorageAccount.CreateCloudBlobClient();
// Create a new asset.
IAsset asset = _context.Assets.Create("NewAsset_" + Guid.NewGuid(), AssetCreationOptions.None);
IAccessPolicy writePolicy = _context.AccessPolicies.Create("writePolicy",
TimeSpan.FromHours(24), AccessPermissions.Write);
ILocator destinationLocator =
_context.Locators.CreateLocator(LocatorType.Sas, asset, writePolicy);
// Get the asset container URI and Blob copy from mediaContainer to assetContainer.
CloudBlobContainer destAssetContainer =
destBlobStorage.GetContainerReference((new Uri(destinationLocator.Path)).Segments[1]);
if (destAssetContainer.CreateIfNotExists())
{
destAssetContainer.SetPermissions(new BlobContainerPermissions
{
PublicAccess = BlobContainerPublicAccessType.Blob
});
}
var blobList = sourceBlobContainer.ListBlobs();
foreach (var sourceBlob in blobList)
{
var assetFile = asset.AssetFiles.Create((sourceBlob as ICloudBlob).Name);
ICloudBlob destinationBlob = destAssetContainer.GetBlockBlobReference(assetFile.Name);
CopyBlob(sourceBlob as ICloudBlob, destAssetContainer);
assetFile.ContentFileSize = (sourceBlob as ICloudBlob).Properties.Length;
assetFile.Update();
Console.WriteLine("File {0} is of {1} size", assetFile.Name, assetFile.ContentFileSize);
}
asset.Update();
destinationLocator.Delete();
writePolicy.Delete();
// Set the primary asset file.
// If, for example, we copied a set of Smooth Streaming files,
// set the .ism file to be the primary file.
// If we, for example, copied an .mp4, then the mp4 would be the primary file.
var ismAssetFile = asset.AssetFiles.ToList().
Where(f => f.Name.EndsWith(".ism", StringComparison.OrdinalIgnoreCase)).ToArray().FirstOrDefault();
// The following code assigns the first .ism file as the primary file in the asset.
// An asset should have one .ism file.
if (ismAssetFile != null)
{
ismAssetFile.IsPrimary = true;
ismAssetFile.Update();
}
return(asset);
}
public CloudLock(ICloudBlob blob, TimeSpan acquireTimeSpan) : this(blob, acquireTimeSpan, TimeSpan.FromSeconds(acquireTimeSpan.TotalSeconds / 2))
{
}
public CloudLock(ICloudBlob blob) : this(blob, TimeSpan.FromSeconds(30))
{
}
/// <summary>
/// whether the specified blob is a snapshot
/// </summary>
/// <param name="blob">ICloudBlob object</param>
/// <returns>true if the specified blob is snapshot, otherwise false</returns>
internal bool IsSnapshot(ICloudBlob blob)
{
return(!string.IsNullOrEmpty(blob.Name) && blob.SnapshotTime != null);
}
/// <summary>
/// This renameFile() method renames the file
/// </summary>
/// <param name="username"></param>
/// <param name="path"></param>
public void renameFile(String username, String path, String newname) //path> containerID:filepath
{
try
{
//Break the path <containerID:filepath>
String[] pathTokens = path.Split(':');
DBManager.UserM user = new DBManager.UserM();
Model.UserModel u;
u = user.getUserRecord(connection, username);
String container = pathTokens[0];
String blobfile = pathTokens[1];
DBManager.ResourceM contDetail = new DBManager.ResourceM();
Console.WriteLine("ResourceID:" + container);
Model.AzureContainerModel cont = contDetail.getResourceById(connection, Int32.Parse(container));
if (cont.getOwner() != u.getUid()) //user is not owner
{
Resource res = new Resource();
if (!res.canWrite(u.getUid(), Int32.Parse(container))) //not having write permission
{
throw new DBLikeExceptions.UnauthorizedAccessException();
}
}
container = cont.getContainerName();
//Check if container exists
CloudBlobContainer myContainer = BlobStorageManager.BlobStorage.getCloudBlobContainer(container); //Container ref
Boolean isContainerexists = BlobStorageManager.BlobStorage.isContainerExists(myContainer);
if (isContainerexists == false)
{
Console.WriteLine("Container not found");
throw new DBLikeExceptions.CloudContainerNotFoundException();
}
//Check if blob exists
CloudBlockBlob oldblob = BlobStorageManager.BlobStorage.getCloudBlob(container + '/' + blobfile); //Get reference to blob
Boolean isBlobexists = BlobStorageManager.BlobStorage.isBlobExists(oldblob);
if (isBlobexists == false)
{
Console.WriteLine("Blob not found");
throw new DBLikeExceptions.CloudBlobNotFoundException();
}
ICloudBlob newblob = null;
if (oldblob is CloudBlockBlob)
{
newblob = myContainer.GetBlockBlobReference(newname);
}
else
{
newblob = myContainer.GetPageBlobReference(newname);
}
//CloudBlockBlob newblob = BlobStorageManager.BlobStorage.getCloudBlob(container + '/' + newname); //Get reference to blob
//copy the blob
newblob.StartCopyFromBlob(oldblob.Uri);
while (true)
{
newblob.FetchAttributes();
if (newblob.CopyState.Status != CopyStatus.Pending) //check the copying status
{
break;
}
System.Threading.Thread.Sleep(1000); //sleep for a second
}
//delete old blobfile
oldblob.Delete();
}
catch (Microsoft.WindowsAzure.Storage.StorageException e)
{
Console.WriteLine("File:renameFile says->>" + e.Message);
}
}
