Esempio n. 1
0
		public async Task Write(ICloudBlob blob, string content)
		{
			using (Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(content)))
			{
				await blob.UploadFromStreamAsync(stream);
			}
		}
Esempio n. 2
0
 /// <summary>
 /// Creates a new instance from a ICloudBlob.
 /// </summary>
 /// <param name="original">The original cloud blob.</param>
 /// <returns></returns>
 public static AzureBlobInfo Create(ICloudBlob original)
 {
     var file = original as CloudBlockBlob;
     if (file != null)
     {
         var modified = DateTimeOffset.MinValue;
         var modifiedOk = false;
         if (file.Metadata.ContainsKey("CbModifiedTime"))
         {
             modifiedOk = DateTimeOffset.TryParse(file.Metadata["CbModifiedTime"], out modified);
         }
         return new AzureBlobInfo
         {
             BlobElement = file,
             ContentType = file.Properties.ContentType,
             Name = file.Name,
             Size = file.Properties.Length,
             Location = file.Uri.AbsoluteUri,
             ModifiedTime = modifiedOk ? modified : default(DateTimeOffset?)
         };
     }
     var folder = original as CloudBlobDirectory;
     if (folder == null)
     {
         return null;
     }
     return new AzureBlobInfo
     {                
         Name = folder.Prefix,
         Location = folder.Uri.AbsoluteUri
     };
 }
        public AzureIndexInput(AzureDirectory azuredirectory, ICloudBlob blob)
            : base(blob.Name)
        {
            _name = blob.Uri.Segments[blob.Uri.Segments.Length - 1];

            _fileMutex = BlobMutexManager.GrabMutex(_name);
            _fileMutex.WaitOne();

            try
            {
                _azureDirectory = azuredirectory;
                _blobContainer = azuredirectory.BlobContainer;
                _blob = blob;

                string fileName = _name;
                StreamOutput fileStream = _azureDirectory.CreateCachedOutputAsStream(fileName);

                // get the blob
                _blob.DownloadToStream(fileStream);

                fileStream.Flush();
                Debug.WriteLine("GET {0} RETREIVED {1} bytes", _name, fileStream.Length);

                fileStream.Close();

                // and open it as an input
                _indexInput = CacheDirectory.openInput(fileName, IOContext.DEFAULT);
            }
            finally
            {
                _fileMutex.ReleaseMutex();
            }
        }
Esempio n. 4
0
 private static async Task CreateBlobIfNoneExists(ICloudBlob updateBlob)
 {
     if (!await updateBlob.ExistsAsync())
     {
         await BlobUtils.CreateEmptyBlob(updateBlob);
     }
 }
Esempio n. 5
0
 public UpdateBlob(ICloudBlob blob, IBlobLeaseFactory blobLeaseFactory)
 {
     _blob = blob;
     _blobLeaseFactory = blobLeaseFactory;
     _updateDomain = string.Empty;
     _instanceIds = new HashSet<string>();
 }
Esempio n. 6
0
		public async Task<string> Read(ICloudBlob blob)
		{
			using (Stream stream = await blob.OpenReadAsync())
			using (StreamReader reader = new StreamReader(stream))
			{
				return await reader.ReadToEndAsync();
			}
		}
 static async Task UploadTextAsync(ICloudBlob blob, string text, AccessCondition accessCondition)
 {
     blob.Properties.ContentEncoding = "UTF-8";
     blob.Properties.ContentType = "text/plain";
     using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(text))) {
         await blob.UploadFromStreamAsync(stream, accessCondition, null, null);
     }
 }
 public static string DownloadText(ICloudBlob blob, Encoding encoding, AccessCondition accessCondition = null, BlobRequestOptions options = null, OperationContext operationContext = null)
 {
     using (MemoryStream stream = new MemoryStream())
     {
         blob.DownloadToStream(stream, accessCondition, options, operationContext);
         return encoding.GetString(stream.ToArray());
     }
 }
 static void UploadText(ICloudBlob blob, string text, AccessCondition accessCondition)
 {
     blob.Properties.ContentEncoding = "UTF-8";
     blob.Properties.ContentType = "text/plain";
     using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(text)))
     {
         blob.UploadFromStream(stream, accessCondition);
     }
 }
Esempio n. 10
0
 public static async Task<string> DownloadTextAsync(ICloudBlob blob, Encoding encoding, AccessCondition accessCondition = null, BlobRequestOptions options = null, OperationContext operationContext = null)
 {
     using (MemoryStream stream = new MemoryStream())
     {
         await blob.DownloadToStreamAsync(stream.AsOutputStream(), accessCondition, options, operationContext);
         byte[] buffer = stream.ToArray();
         return encoding.GetString(buffer, 0, buffer.Length);
     }
 }
Esempio n. 11
0
        /// <summary>
        /// Return a secret, persistent Download Url for a given Blob
        /// </summary>
        static string GetDownloadUrlFor(ICloudBlob blob)
        {
            var signature = blob.GetSharedAccessSignature(new SharedAccessBlobPolicy
            {
                SharedAccessExpiryTime = new DateTime(2050, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc)
            }, DownloadPolicyName);

            return string.Format("{0}{1}", blob.Uri.ToString().Replace(" ", "%20"), signature);
        }
 /// <summary>
 /// Creates a job to copy from a blob.
 /// </summary>
 /// <param name="destBlob">Destination blob to copy to. 
 /// User should call the method on this object.</param>
 /// <param name="sourceBlob">Source blob to copy from.</param>
 /// <returns>Job object to do copying.</returns>
 public static BlobCopyJob CreateCopyJob(
     this ICloudBlob destBlob,
     ICloudBlob sourceBlob)
 {
     return new BlobCopyJob()
     {
         SourceBlob = sourceBlob,
         DestBlob = destBlob
     };
 }
 public static async Task WaitForCopyAsync(ICloudBlob blob)
 {
     bool copyInProgress = true;
     while (copyInProgress)
     {
         await Task.Delay(1000);
         await blob.FetchAttributesAsync();
         copyInProgress = (blob.CopyState.Status == CopyStatus.Pending);
     }
 }
 public static void WaitForCopyTask(ICloudBlob blob)
 {
     bool copyInProgress = true;
     while (copyInProgress)
     {
         Thread.Sleep(1000);
         blob.FetchAttributesAsync().Wait();
         copyInProgress = (blob.CopyState.Status == CopyStatus.Pending);
     }
 }
 /// <summary>
 /// Azure storage blob constructor
 /// </summary>
 /// <param name="blob">ICloud blob object</param>
 public AzureStorageBlob(ICloudBlob blob)
 {
     Name = blob.Name;
     ICloudBlob = blob;
     BlobType = blob.BlobType;
     Length = blob.Properties.Length;
     ContentType = blob.Properties.ContentType;
     LastModified = blob.Properties.LastModified;
     SnapshotTime = blob.SnapshotTime;
 }
Esempio n. 16
0
 public static string GetSasUrl(ICloudBlob blob, SharedAccessBlobPermissions permissions)
 {
     var policy = new SharedAccessBlobPolicy
     {
         SharedAccessExpiryTime = DateTime.Now.AddMinutes(30),
         Permissions = permissions
     };
     string sasBlobToken = blob.GetSharedAccessSignature(policy);
     //Return the URI string for the container, including the SAS token.
     return blob.Uri + sasBlobToken;
 }
 //https://github.com/Particular/NServiceBus.Azure/blob/e9db29beb21d1fd914191e479cb5948fffd92f3b/src/NServiceBus.Azure/DataBus/Azure/BlobStorage/BlobStorageDataBus.cs#L41
 protected override void SetValidUntil(ICloudBlob cloudBlob, TimeSpan timeToBeReceived)
 {
     if (timeToBeReceived == TimeSpan.MaxValue)
     {
         cloudBlob.Metadata["ValidUntil"] = TimeSpan.MaxValue.ToString();
     }
     else
     {
         cloudBlob.Metadata["ValidUntil"] = (DateTime.Now + timeToBeReceived).ToString();
     }
 }
 private static async Task UpdateBlob(ICloudBlob blob)
 {
     try
     {
         byte[] dataBytes = new byte[40];
         await blob.UploadFromByteArrayAsync(dataBytes, 0, 40);
     }
     catch (Exception e)
     {
         Console.WriteLine(e);
     }
 }
Esempio n. 19
0
		/// <summary>
		/// Initializes a new instance of <see cref="OperationHandle" /> class.
		/// </summary>
		/// <param name="source">A source blob.</param>
		/// <param name="target">A destination blob.</param>
		/// <param name="sasToken">A security access token.</param>
		public OperationHandle(ICloudBlob source, ICloudBlob target, String sasToken)
		{
			DeleteIsCompleted = false;

			CopyIsCompleted = false;

			SourceBlob = source;

			TargetBlob = target;

			this.sasToken = sasToken;
		}
Esempio n. 20
0
 public SelfRenewableBlobLease(ICloudBlob blob, int renewIntervalInSeconds)
 {
     if (renewIntervalInSeconds > 60)
     {
         throw new Exception("Blob lease renew interval must be less than 60s");
     }
     if (renewIntervalInSeconds < 10)
     {
         throw new Exception("Blob lease renew interval must be at least 10s");
     }
     _renewIntervalInSeconds = renewIntervalInSeconds;
     _blob = blob;
 }
 /// <summary>
 /// Initializes a new instance of the BlobReadStreamBase class.
 /// </summary>
 /// <param name="blob">Blob reference to read from</param>
 /// <param name="accessCondition">An object that represents the access conditions for the blob. If null, no condition is used.</param>
 /// <param name="options">An object that specifies any additional options for the request.</param>
 /// <param name="operationContext">An <see cref="OperationContext"/> object for tracking the current operation.</param>
 protected BlobReadStreamBase(ICloudBlob blob, AccessCondition accessCondition, BlobRequestOptions options, OperationContext operationContext)
 {
     this.blob = blob;
     this.isLengthAvailable = false;
     this.currentOffset = 0;
     this.buffer = new MemoryStream();
     this.accessCondition = accessCondition;
     this.lockedToETag = false;
     this.options = options;
     this.operationContext = operationContext;
     this.blobMD5 = options.DisableContentMD5Validation.Value ? null : new MD5Wrapper();
     this.lastException = null;
 }
Esempio n. 22
0
 private bool UpdateProperty(ICloudBlob blob, string name, string value)
 {
     var property = blob.Properties.GetType().GetProperty(name.Replace("-", ""));
     if (property != null)
     {
         property.SetValue((object)blob.Properties, (object)value);
         var method = blob.GetType().GetMethod("SetProperties");
         if (method != null)
         {
             method.Invoke(blob, new object[]{null,null,null});
             return true;
         }
     }
     return false;
 }
        private static void TestAccessTask(BlobContainerPublicAccessType accessType, CloudBlobContainer container, ICloudBlob inputBlob)
        {
            StorageCredentials credentials = new StorageCredentials();
            container = new CloudBlobContainer(container.Uri, credentials);
            CloudPageBlob blob = new CloudPageBlob(inputBlob.Uri, credentials);

            if (accessType.Equals(BlobContainerPublicAccessType.Container))
            {
                blob.FetchAttributesAsync().Wait();
                BlobContinuationToken token = null;
                do
                {
                    BlobResultSegment results = container.ListBlobsSegmented(token);
                    results.Results.ToArray();
                    token = results.ContinuationToken;
                }
                while (token != null);
                container.FetchAttributesAsync().Wait();
            }
            else if (accessType.Equals(BlobContainerPublicAccessType.Blob))
            {
                blob.FetchAttributesAsync().Wait();

                TestHelper.ExpectedExceptionTask(
                    container.ListBlobsSegmentedAsync(null),
                    "List blobs while public access does not allow for listing",
                    HttpStatusCode.NotFound);
                TestHelper.ExpectedExceptionTask(
                    container.FetchAttributesAsync(),
                    "Fetch container attributes while public access does not allow",
                    HttpStatusCode.NotFound);
            }
            else
            {
                TestHelper.ExpectedExceptionTask(
                    blob.FetchAttributesAsync(),
                    "Fetch blob attributes while public access does not allow",
                    HttpStatusCode.NotFound);
                TestHelper.ExpectedExceptionTask(
                    container.ListBlobsSegmentedAsync(null),
                    "List blobs while public access does not allow for listing",
                    HttpStatusCode.NotFound);
                TestHelper.ExpectedExceptionTask(
                    container.FetchAttributesAsync(),
                    "Fetch container attributes while public access does not allow",
                    HttpStatusCode.NotFound);
            }
        }
        internal BlobLeaseManager(ICloudBlob lockBlob, TimeSpan leaseTimeout, string hostId, string instanceId, TraceWriter traceWriter, TimeSpan? renewalInterval = null)
        {
            _lockBlob = lockBlob;
            _leaseTimeout = leaseTimeout;
            _traceWriter = traceWriter;
            _hostId = hostId;
            _instanceId = instanceId;

            // Renew the lease three seconds before it expires
            _renewalInterval = renewalInterval ?? leaseTimeout.Add(TimeSpan.FromSeconds(-3));

            // Attempt to acquire a lease every 5 seconds
            _leaseRetryInterval = TimeSpan.FromSeconds(5);

            _timer = new Timer(ProcessLeaseTimerTick, null, TimeSpan.Zero, _leaseRetryInterval);
        }
Esempio n. 25
0
 public static void AssertAreEqual(ICloudBlob blob1, ICloudBlob blob2)
 {
     if (blob1 == null)
     {
         Assert.IsNull(blob2);
     }
     else
     {
         Assert.IsNotNull(blob2);
         Assert.AreEqual(blob1.BlobType, blob2.BlobType);
         Assert.AreEqual(blob1.Uri, blob2.Uri);
         Assert.AreEqual(blob1.SnapshotTime, blob2.SnapshotTime);
         AssertAreEqual(blob1.Properties, blob2.Properties);
         AssertAreEqual(blob1.CopyState, blob2.CopyState);
     }
 }
        private static async Task TestAccessAsync(BlobContainerPublicAccessType accessType, CloudBlobContainer container, ICloudBlob inputBlob)
        {
            StorageCredentials credentials = new StorageCredentials();
            container = new CloudBlobContainer(container.Uri, credentials);
            CloudPageBlob blob = new CloudPageBlob(inputBlob.Uri, credentials);
            OperationContext context = new OperationContext();
            BlobRequestOptions options = new BlobRequestOptions();

            if (accessType.Equals(BlobContainerPublicAccessType.Container))
            {
                await blob.FetchAttributesAsync();
                await container.ListBlobsSegmentedAsync(null, true, BlobListingDetails.All, null, null, options, context);
                await container.FetchAttributesAsync();
            }
            else if (accessType.Equals(BlobContainerPublicAccessType.Blob))
            {
                await blob.FetchAttributesAsync();
                await TestHelper.ExpectedExceptionAsync(
                    async () => await container.ListBlobsSegmentedAsync(null, true, BlobListingDetails.All, null, null, options, context),
                    context,
                    "List blobs while public access does not allow for listing",
                    HttpStatusCode.NotFound);
                await TestHelper.ExpectedExceptionAsync(
                    async () => await container.FetchAttributesAsync(null, options, context),
                    context,
                    "Fetch container attributes while public access does not allow",
                    HttpStatusCode.NotFound);
            }
            else
            {
                await TestHelper.ExpectedExceptionAsync(
                    async () => await blob.FetchAttributesAsync(null, options, context),
                    context,
                    "Fetch blob attributes while public access does not allow",
                    HttpStatusCode.NotFound);
                await TestHelper.ExpectedExceptionAsync(
                    async () => await container.ListBlobsSegmentedAsync(null, true, BlobListingDetails.All, null, null, options, context),
                    context,
                    "List blobs while public access does not allow for listing",
                    HttpStatusCode.NotFound);
                await TestHelper.ExpectedExceptionAsync(
                    async () => await container.FetchAttributesAsync(null, options, context),
                    context,
                    "Fetch container attributes while public access does not allow",
                    HttpStatusCode.NotFound);
            }
        }
Esempio n. 27
0
 private static async Task RenewLease(ICloudBlob blob, string leaseId)
 {
     var blobKey = GetKey(blob);
     try
     {
         await blob.RenewLeaseAsync(
             new AccessCondition
             {
                 LeaseId = leaseId
             });
         Trace.TraceInformation("Renewed lease for blob {0}", blobKey);
     }
     catch (Exception ex)
     {
         Trace.TraceError("Failed to renew lease for blob {0}. Exception was: {1}", blobKey, ex);
     }
 }
        public static IAsyncCloudBlob FromICloudBlob(ICloudBlob cloudBlob)
        {
            var block = cloudBlob as CloudBlockBlob;
            if (block != null)
            {
                return new AsyncCloudBlockBlob(block);
            }

            var page = cloudBlob as CloudPageBlob;
            if (page != null)
            {
                return new AsyncCloudPageBlob(page);
            }

            // unknown blob type... TODO - it might be better to actually throw an exception here
            return new AsyncCloudBlobBase<ICloudBlob>(cloudBlob);
        }
        public AzureIndexOutput(AzureDirectory azureDirectory, ICloudBlob blob)
        {
            _fileMutex = BlobMutexManager.GrabMutex(_name);
            _fileMutex.WaitOne();
            try
            {
                _azureDirectory = azureDirectory;
                _blob = blob;
                _name = blob.Uri.Segments[blob.Uri.Segments.Length - 1];

                // create the local cache one we will operate against...
                _indexOutput = CacheDirectory.createOutput(_name, IOContext.DEFAULT);
            }
            finally
            {
                _fileMutex.ReleaseMutex();
            }
        }
        /// <summary>
        /// Initializes a new instance of the BlobReadStreamBase class.
        /// </summary>
        /// <param name="blob">Blob reference to read from</param>
        /// <param name="accessCondition">An object that represents the access conditions for the blob. If null, no condition is used.</param>
        /// <param name="options">An object that specifies additional options for the request.</param>
        /// <param name="operationContext">An <see cref="OperationContext"/> object for tracking the current operation.</param>
        protected BlobReadStreamBase(ICloudBlob blob, AccessCondition accessCondition, BlobRequestOptions options, OperationContext operationContext)
        {
            if (options.UseTransactionalMD5.Value)
            {
                CommonUtility.AssertInBounds("StreamMinimumReadSizeInBytes", blob.StreamMinimumReadSizeInBytes, 1, Constants.MaxRangeGetContentMD5Size);
            }

            this.blob = blob;
            this.blobProperties = new BlobProperties(blob.Properties);
            this.currentOffset = 0;
            this.streamMinimumReadSizeInBytes = this.blob.StreamMinimumReadSizeInBytes;
            this.internalBuffer = new MemoryStream(this.streamMinimumReadSizeInBytes);
            this.accessCondition = accessCondition;
            this.options = options;
            this.operationContext = operationContext;
            this.blobMD5 = (this.options.DisableContentMD5Validation.Value || string.IsNullOrEmpty(this.blobProperties.ContentMD5)) ? null : new MD5Wrapper();
            this.lastException = null;
        }
        public AzureIndexInput(AzureDirectory azuredirectory, ICloudBlob blob)
        {
            _name = blob.Uri.Segments[blob.Uri.Segments.Length - 1];

#if FULLDEBUG
            Debug.WriteLine(String.Format("opening {0} ", _name));
#endif
            _fileMutex = BlobMutexManager.GrabMutex(_name);
            _fileMutex.WaitOne();
            try
            {
                _azureDirectory = azuredirectory;
                _blobContainer  = azuredirectory.BlobContainer;
                _blob           = blob;

                string fileName = _name;

                bool fFileNeeded = false;
                if (!CacheDirectory.FileExists(fileName))
                {
                    fFileNeeded = true;
                }
                else
                {
                    long cachedLength = CacheDirectory.FileLength(fileName);
                    long blobLength   = blob.Properties.Length;
                    long.TryParse(blob.Metadata["CachedLength"], out blobLength);

                    long     longLastModified    = 0;
                    DateTime blobLastModifiedUTC = blob.Properties.LastModified.Value.UtcDateTime;
                    if (long.TryParse(blob.Metadata["CachedLastModified"], out longLastModified))
                    {
                        // normalize RAMDirectory and FSDirectory times
                        if (longLastModified > ticks1970)
                        {
                            longLastModified -= ticks1970;
                        }
                        blobLastModifiedUTC = new DateTime(longLastModified).ToUniversalTime();
                    }

                    if (cachedLength != blobLength)
                    {
                        fFileNeeded = true;
                    }
                    else
                    {
                        // there seems to be an error of 1 tick which happens every once in a while
                        // for now we will say that if they are within 1 tick of each other and same length

                        var elapsed = CacheDirectory.FileModified(fileName);

                        // normalize RAMDirectory and FSDirectory times
                        if (elapsed > ticks1970)
                        {
                            elapsed -= ticks1970;
                        }

                        DateTime cachedLastModifiedUTC = new DateTime(elapsed, DateTimeKind.Local).ToUniversalTime();
                        if (cachedLastModifiedUTC != blobLastModifiedUTC)
                        {
                            TimeSpan timeSpan = blobLastModifiedUTC.Subtract(cachedLastModifiedUTC);
                            if (timeSpan.TotalSeconds > 1)
                            {
                                fFileNeeded = true;
                            }
                            else
                            {
#if FULLDEBUG
                                Debug.WriteLine(timeSpan.TotalSeconds);
#endif
                                // file not needed
                            }
                        }
                    }
                }

                // if the file does not exist
                // or if it exists and it is older then the lastmodified time in the blobproperties (which always comes from the blob storage)
                if (fFileNeeded)
                {
#if COMPRESSBLOBS
                    if (_azureDirectory.ShouldCompressFile(_name))
                    {
                        // then we will get it fresh into local deflatedName
                        // StreamOutput deflatedStream = new StreamOutput(CacheDirectory.CreateOutput(deflatedName));
                        MemoryStream deflatedStream = new MemoryStream();

                        // get the deflated blob
                        _blob.DownloadToStream(deflatedStream);

                        Debug.WriteLine(string.Format("GET {0} RETREIVED {1} bytes", _name, deflatedStream.Length));

                        // seek back to begininng
                        deflatedStream.Seek(0, SeekOrigin.Begin);

                        // open output file for uncompressed contents
                        StreamOutput fileStream = _azureDirectory.CreateCachedOutputAsStream(fileName);

                        // create decompressor
                        DeflateStream decompressor = new DeflateStream(deflatedStream, CompressionMode.Decompress);

                        byte[] bytes = new byte[65535];
                        int    nRead = 0;
                        do
                        {
                            nRead = decompressor.Read(bytes, 0, 65535);
                            if (nRead > 0)
                            {
                                fileStream.Write(bytes, 0, nRead);
                            }
                        } while (nRead == 65535);
                        decompressor.Close(); // this should close the deflatedFileStream too

                        fileStream.Close();
                    }
                    else
#endif
                    {
                        StreamOutput fileStream = _azureDirectory.CreateCachedOutputAsStream(fileName);

                        // get the blob
                        _blob.DownloadToStream(fileStream);

                        fileStream.Flush();
                        Debug.WriteLine(string.Format("GET {0} RETREIVED {1} bytes", _name, fileStream.Length));

                        fileStream.Close();
                    }

                    // and open it as an input
                    _indexInput = CacheDirectory.OpenInput(fileName);
                }
                else
                {
#if FULLDEBUG
                    Debug.WriteLine(String.Format("Using cached file for {0}", _name));
#endif

                    // open the file in read only mode
                    _indexInput = CacheDirectory.OpenInput(fileName);
                }
            }
            finally
            {
                _fileMutex.ReleaseMutex();
            }
        }
        private static async Task TestAccessAsync(BlobContainerPublicAccessType accessType, CloudBlobContainer container, ICloudBlob inputBlob)
        {
            StorageCredentials credentials = new StorageCredentials();

            container = new CloudBlobContainer(container.Uri, credentials);
            CloudPageBlob      blob    = new CloudPageBlob(inputBlob.Uri, credentials);
            OperationContext   context = new OperationContext();
            BlobRequestOptions options = new BlobRequestOptions();

            if (accessType.Equals(BlobContainerPublicAccessType.Container))
            {
                await blob.FetchAttributesAsync();

                await container.ListBlobsSegmentedAsync(null, true, BlobListingDetails.All, null, null, options, context);

                await container.FetchAttributesAsync();
            }
            else if (accessType.Equals(BlobContainerPublicAccessType.Blob))
            {
                await blob.FetchAttributesAsync();

                await TestHelper.ExpectedExceptionAsync(
                    async() => await container.ListBlobsSegmentedAsync(null, true, BlobListingDetails.All, null, null, options, context),
                    context,
                    "List blobs while public access does not allow for listing",
                    HttpStatusCode.NotFound);

                await TestHelper.ExpectedExceptionAsync(
                    async() => await container.FetchAttributesAsync(null, options, context),
                    context,
                    "Fetch container attributes while public access does not allow",
                    HttpStatusCode.NotFound);
            }
            else
            {
                await TestHelper.ExpectedExceptionAsync(
                    async() => await blob.FetchAttributesAsync(null, options, context),
                    context,
                    "Fetch blob attributes while public access does not allow",
                    HttpStatusCode.NotFound);

                await TestHelper.ExpectedExceptionAsync(
                    async() => await container.ListBlobsSegmentedAsync(null, true, BlobListingDetails.All, null, null, options, context),
                    context,
                    "List blobs while public access does not allow for listing",
                    HttpStatusCode.NotFound);

                await TestHelper.ExpectedExceptionAsync(
                    async() => await container.FetchAttributesAsync(null, options, context),
                    context,
                    "Fetch container attributes while public access does not allow",
                    HttpStatusCode.NotFound);
            }
        }
Esempio n. 33
0
 private static void SetAttachmentAsContentDisposition(ICloudBlob resizedPhotoCloudBlob,
                                                       PictureResizeRequest pictureResizeRequest)
 {
     resizedPhotoCloudBlob.Properties.ContentDisposition =
         $"attachment; filename={pictureResizeRequest.RequiredWidth}x{pictureResizeRequest.RequiredHeight}";
 }
 /// <summary>
 /// Initializes a new instance of the BlobReadStream class.
 /// </summary>
 /// <param name="blob">Blob reference to read from</param>
 /// <param name="accessCondition">An object that represents the access conditions for the blob. If null, no condition is used.</param>
 /// <param name="options">An object that specifies additional options for the request.</param>
 /// <param name="operationContext">An <see cref="OperationContext"/> object for tracking the current operation.</param>
 internal BlobReadStream(ICloudBlob blob, AccessCondition accessCondition, BlobRequestOptions options, OperationContext operationContext)
     : base(blob, accessCondition, options, operationContext)
 {
 }
        private async Task StoreElementAsync(ICloudBlob blobRef, XElement element)
        {
            // holds the last error in case we need to rethrow it
            ExceptionDispatchInfo lastError = null;

            for (var i = 0; i < ConflictMaxRetries; i++)
            {
                if (i > 1)
                {
                    // If multiple conflicts occurred, wait a small period of time before retrying
                    // the operation so that other writers can make forward progress.
                    await Task.Delay(GetRandomizedBackoffPeriod());
                }

                if (i > 0)
                {
                    // If at least one conflict occurred, make sure we have an up-to-date
                    // view of the blob contents.
                    await GetLatestDataAsync(blobRef);
                }

                // Merge the new element into the document. If no document exists,
                // create a new default document and inject this element into it.

                var latestData = Volatile.Read(ref _cachedBlobData);
                var doc        = (latestData != null)
                    ? CreateDocumentFromBlob(latestData.BlobContents)
                    : new XDocument(new XElement(RepositoryElementName));
                doc.Root.Add(element);

                // Turn this document back into a byte[].

                var serializedDoc = new MemoryStream();
                doc.Save(serializedDoc, SaveOptions.DisableFormatting);

                // Generate the appropriate precondition header based on whether or not
                // we believe data already exists in storage.

                AccessCondition accessCondition;
                if (latestData != null)
                {
                    accessCondition = AccessCondition.GenerateIfMatchCondition(blobRef.Properties.ETag);
                }
                else
                {
                    accessCondition = AccessCondition.GenerateIfNotExistsCondition();
                    blobRef.Properties.ContentType = "application/xml; charset=utf-8"; // set content type on first write
                }

                try
                {
                    // Send the request up to the server.

                    var serializedDocAsByteArray = serializedDoc.ToArray();

                    await blobRef.UploadFromByteArrayAsync(
                        buffer : serializedDocAsByteArray,
                        index : 0,
                        count : serializedDocAsByteArray.Length,
                        accessCondition : accessCondition,
                        options : null,
                        operationContext : null);

                    // If we got this far, success!
                    // We can update the cached view of the remote contents.

                    Volatile.Write(ref _cachedBlobData, new BlobData()
                    {
                        BlobContents = serializedDocAsByteArray,
                        ETag         = blobRef.Properties.ETag // was updated by Upload routine
                    });

                    return;
                }
                catch (StorageException ex)
                    when(ex.RequestInformation.HttpStatusCode == 409 || ex.RequestInformation.HttpStatusCode == 412)
                    {
                        // 409 Conflict
                        // This error is rare but can be thrown in very special circumstances,
                        // such as if the blob in the process of being created. We treat it
                        // as equivalent to 412 for the purposes of retry logic.

                        // 412 Precondition Failed
                        // We'll get this error if another writer updated the repository and we
                        // have an outdated view of its contents. If this occurs, we'll just
                        // refresh our view of the remote contents and try again up to the max
                        // retry limit.

                        lastError = ExceptionDispatchInfo.Capture(ex);
                    }
            }

            // if we got this far, something went awry
            lastError.Throw();
        }
Esempio n. 36
0
 public static void Run([BlobTrigger(BlobPath)] ICloudBlob blob)
 {
     TaskSource.TrySetResult(blob);
 }
 /// <summary>
 /// Extracts file name from the Cloud BLOB.
 /// </summary>
 /// <param name="blob">Cloud BLOB</param>
 /// <returns>Name of the file.</returns>
 private string GetBlobLocalName(ICloudBlob blob)
 {
     return(blob.Name.Substring(blob.Name.LastIndexOf(BlobPathSeparator) + 1));
 }
 private static bool NotExists(IEnumerable <ICloudBlob> targetBlobs,
                               ICloudBlob b)
 {
     return(targetBlobs.All(tb => tb.Name != b.Name));
 }
Esempio n. 39
0
        private static void TestAccess(string sasToken, SharedAccessBlobPermissions permissions, SharedAccessBlobHeaders headers, CloudBlobContainer container, ICloudBlob blob)
        {
            StorageCredentials credentials = string.IsNullOrEmpty(sasToken) ?
                                             new StorageCredentials() :
                                             new StorageCredentials(sasToken);

            if (container != null)
            {
                container = new CloudBlobContainer(credentials.TransformUri(container.Uri));
                if (blob.BlobType == BlobType.BlockBlob)
                {
                    blob = container.GetBlockBlobReference(blob.Name);
                }
                else
                {
                    blob = container.GetPageBlobReference(blob.Name);
                }
            }
            else
            {
                if (blob.BlobType == BlobType.BlockBlob)
                {
                    blob = new CloudBlockBlob(credentials.TransformUri(blob.Uri));
                }
                else
                {
                    blob = new CloudPageBlob(credentials.TransformUri(blob.Uri));
                }
            }

            if (container != null)
            {
                if ((permissions & SharedAccessBlobPermissions.List) == SharedAccessBlobPermissions.List)
                {
                    container.ListBlobs().ToArray();
                }
                else
                {
                    TestHelper.ExpectedException(
                        () => container.ListBlobs().ToArray(),
                        "List blobs while SAS does not allow for listing",
                        HttpStatusCode.NotFound);
                }
            }

            if ((permissions & SharedAccessBlobPermissions.Read) == SharedAccessBlobPermissions.Read)
            {
                blob.FetchAttributes();

                // Test headers
                if (headers != null)
                {
                    if (headers.CacheControl != null)
                    {
                        Assert.AreEqual(headers.CacheControl, blob.Properties.CacheControl);
                    }

                    if (headers.ContentDisposition != null)
                    {
                        Assert.AreEqual(headers.ContentDisposition, blob.Properties.ContentDisposition);
                    }

                    if (headers.ContentEncoding != null)
                    {
                        Assert.AreEqual(headers.ContentEncoding, blob.Properties.ContentEncoding);
                    }

                    if (headers.ContentLanguage != null)
                    {
                        Assert.AreEqual(headers.ContentLanguage, blob.Properties.ContentLanguage);
                    }

                    if (headers.ContentType != null)
                    {
                        Assert.AreEqual(headers.ContentType, blob.Properties.ContentType);
                    }
                }
            }
            else
            {
                TestHelper.ExpectedException(
                    () => blob.FetchAttributes(),
                    "Fetch blob attributes while SAS does not allow for reading",
                    HttpStatusCode.NotFound);
            }

            if ((permissions & SharedAccessBlobPermissions.Write) == SharedAccessBlobPermissions.Write)
            {
                blob.SetMetadata();
            }
            else
            {
                TestHelper.ExpectedException(
                    () => blob.SetMetadata(),
                    "Set blob metadata while SAS does not allow for writing",
                    HttpStatusCode.NotFound);
            }

            if ((permissions & SharedAccessBlobPermissions.Delete) == SharedAccessBlobPermissions.Delete)
            {
                blob.Delete();
            }
            else
            {
                TestHelper.ExpectedException(
                    () => blob.Delete(),
                    "Delete blob while SAS does not allow for deleting",
                    HttpStatusCode.NotFound);
            }
        }
Esempio n. 40
0
 /// <summary>
 /// fetch blob attributes
 /// </summary>
 /// <param name="accessCondition">Access condition</param>
 /// <param name="options">blob request options</param>
 /// <param name="operationContext">An object that represents the context for the current operation.</param>
 public void FetchBlobAttributes(ICloudBlob blob, AccessCondition accessCondition, BlobRequestOptions options, OperationContext operationContext)
 {
     return;
 }
Esempio n. 41
0
 internal OutputFileReference(ICloudBlob blob)
 {
     CloudBlob = blob;
 }
        /// <summary>
        /// Write ICloudBlob to output using specified service channel
        /// </summary>
        /// <param name="blob">The output ICloudBlob object</param>
        /// <param name="channel">IStorageBlobManagement channel object</param>
        internal void WriteICloudBlobObject(long taskId, IStorageBlobManagement channel, ICloudBlob blob, BlobContinuationToken continuationToken = null)
        {
            AzureStorageBlob azureBlob = new AzureStorageBlob(blob);

            azureBlob.Context           = channel.StorageContext;
            azureBlob.ContinuationToken = continuationToken;
            OutputStream.WriteObject(taskId, azureBlob);
        }
Esempio n. 43
0
 protected TransferBlobResult TransferBlob(ICloudBlob blob)
 {
     return(new TransferBlobResult(blob));
 }
        /// <summary>
        /// set azure blob
        /// </summary>
        /// <param name="fileName">local file name</param>
        /// <param name="blob">destination blob</param>
        /// <param name="isValidContainer">whether the destination container is validated</param>
        /// <returns>null if user cancel the overwrite operation, otherwise return destination blob object</returns>
        internal AzureStorageBlob SetAzureBlobContent(string fileName, ICloudBlob blob, bool isValidContainer = false)
        {
            string filePath = GetFullSendFilePath(fileName);

            if (String.IsNullOrEmpty(filePath))
            {
                return(null);
            }

            if (null == blob)
            {
                throw new ArgumentException(String.Format(Resources.ObjectCannotBeNull, typeof(ICloudBlob).Name));
            }

            if (blob.BlobType == WindowsAzure.Storage.Blob.BlobType.PageBlob)
            {
                long fileSize     = new FileInfo(filePath).Length;
                long pageBlobUnit = 512;
                long remainder    = fileSize % pageBlobUnit;

                if (remainder != 0)
                {
                    //the blob size must be a multiple of 512 bytes.
                    throw new ArgumentException(String.Format(Resources.InvalidPageBlobSize, filePath, fileSize));
                }
            }

            if (!NameUtil.IsValidBlobName(blob.Name))
            {
                throw new ArgumentException(String.Format(Resources.InvalidBlobName, blob.Name));
            }

            if (!isValidContainer)
            {
                ValidatePipelineCloudBlobContainer(blob.Container);
            }

            AccessCondition    accessCondition = null;
            BlobRequestOptions requestOptions  = RequestOptions;
            ICloudBlob         blobRef         = Channel.GetBlobReferenceFromServer(blob.Container, blob.Name, accessCondition, requestOptions, OperationContext);

            if (null != blobRef)
            {
                if (blob.BlobType != blobRef.BlobType)
                {
                    throw new ArgumentException(String.Format(Resources.BlobTypeMismatch, blobRef.Name, blobRef.BlobType));
                }

                if (!overwrite)
                {
                    if (!ConfirmOverwrite(blob.Name))
                    {
                        return(null);
                    }
                }
            }

            try
            {
                Upload2Blob(filePath, blob);

                try
                {
                    Channel.FetchBlobAttributes(blob, accessCondition, requestOptions, OperationContext);
                }
                catch (StorageException se)
                {
                    //Storage credentials only have the write permission, and don't have the read permission.
                    if (se.IsNotFoundException())
                    {
                        WriteVerboseWithTimestamp(Resources.BlobIsNotReadable);
                    }
                    else
                    {
                        throw;
                    }
                }
            }
            catch (Exception e)
            {
                WriteDebugLog(String.Format(Resources.Upload2BlobFailed, e.Message));
                throw;
            }

            return(new AzureStorageBlob(blob));
        }
Esempio n. 45
0
        public AzureIndexInput(AzureDirectory azuredirectory, ICloudBlob blob)
        {
            _name = blob.Uri.Segments[blob.Uri.Segments.Length - 1];

#if FULLDEBUG
            Debug.WriteLine(String.Format("opening {0} ", _name));
#endif
            _fileMutex = BlobMutexManager.GrabMutex(_name);
            _fileMutex.WaitOne();
            try
            {
                _azureDirectory = azuredirectory;
                _blobContainer  = azuredirectory.BlobContainer;
                _blob           = blob;

                var fileName = _name;

                var fFileNeeded = false;
                if (!CacheDirectory.FileExists(fileName))
                {
                    fFileNeeded = true;
                }
                else
                {
                    long   cachedLength = CacheDirectory.FileLength(fileName);
                    string blobLengthMetadata;
                    bool   hasMetadataValue = blob.Metadata.TryGetValue("CachedLength", out blobLengthMetadata);
                    long   blobLength       = blob.Properties.Length;
                    if (hasMetadataValue)
                    {
                        long.TryParse(blobLengthMetadata, out blobLength);
                    }

                    string   blobLastModifiedMetadata;
                    long     longLastModified    = 0;
                    DateTime blobLastModifiedUTC = blob.Properties.LastModified.Value.UtcDateTime;
                    if (blob.Metadata.TryGetValue("CachedLastModified", out blobLastModifiedMetadata))
                    {
                        if (long.TryParse(blobLastModifiedMetadata, out longLastModified))
                        {
                            blobLastModifiedUTC = new DateTime(longLastModified).ToUniversalTime();
                        }
                    }

                    if (cachedLength != blobLength)
                    {
                        fFileNeeded = true;
                    }
                    else
                    {
                        // there seems to be an error of 1 tick which happens every once in a while
                        // for now we will say that if they are within 1 tick of each other and same length
                        var cachedLastModifiedUTC = new DateTime(CacheDirectory.FileModified(fileName), DateTimeKind.Local).ToUniversalTime();
                        if (cachedLastModifiedUTC != blobLastModifiedUTC)
                        {
                            var timeSpan = blobLastModifiedUTC.Subtract(cachedLastModifiedUTC);
                            if (timeSpan.TotalSeconds > 1)
                            {
                                fFileNeeded = true;
                            }
                            else
                            {
#if FULLDEBUG
                                Debug.WriteLine(timeSpan.TotalSeconds);
#endif
                                // file not needed
                            }
                        }
                    }
                }

                // if the file does not exist
                // or if it exists and it is older then the lastmodified time in the blobproperties (which always comes from the blob storage)
                if (fFileNeeded)
                {
                    if (_azureDirectory.ShouldCompressFile(_name))
                    {
                        InflateStream(fileName);
                    }
                    else
                    {
                        using (var fileStream = _azureDirectory.CreateCachedOutputAsStream(fileName))
                        {
                            // get the blob
                            _blob.DownloadToStream(fileStream);

                            fileStream.Flush();
                            Debug.WriteLine(string.Format("GET {0} RETREIVED {1} bytes", _name, fileStream.Length));
                        }
                    }

                    // and open it as an input
                    _indexInput = CacheDirectory.OpenInput(fileName);
                }
                else
                {
#if FULLDEBUG
                    Debug.WriteLine(String.Format("Using cached file for {0}", _name));
#endif

                    // open the file in read only mode
                    _indexInput = CacheDirectory.OpenInput(fileName);
                }
            }
            finally
            {
                _fileMutex.ReleaseMutex();
            }
        }
Esempio n. 46
0
        /// <summary>
        /// Deletes the file.
        /// </summary>
        /// <param name="bucketName">Name of the bucket.</param>
        /// <param name="keyName">Name of the key.</param>
        public void DeleteFile(string bucketName, string keyName)
        {
            ICloudBlob blob = GetCloudBlob(bucketName, keyName);

            blob.DeleteIfExists();
        }
Esempio n. 47
0
        private void BlobReadStreamReadSizeTestAPM(ICloudBlob blob)
        {
            IAsyncResult result;

            using (AutoResetEvent waitHandle = new AutoResetEvent(false))
            {
                byte[] buffer = GetRandomBuffer(5 * 1024 * 1024);
                using (MemoryStream wholeBlob = new MemoryStream(buffer))
                {
                    result = blob.BeginUploadFromStream(wholeBlob,
                                                        ar => waitHandle.Set(),
                                                        null);
                    waitHandle.WaitOne();
                    blob.EndUploadFromStream(result);
                }

                TestHelper.ExpectedException <ArgumentOutOfRangeException>(
                    () => blob.StreamMinimumReadSizeInBytes = 16 * 1024 - 1,
                    "StreamMinimumReadSizeInBytes should not accept values smaller than 16KB");

                blob.StreamMinimumReadSizeInBytes = 4 * 1024 * 1024 + 1;
                BlobRequestOptions options = new BlobRequestOptions()
                {
                    UseTransactionalMD5 = true
                };
                result = blob.BeginOpenRead(null, options, null,
                                            ar => waitHandle.Set(),
                                            null);
                waitHandle.WaitOne();
                TestHelper.ExpectedException <ArgumentOutOfRangeException>(
                    () => blob.EndOpenRead(result),
                    "StreamMinimumReadSizeInBytes should be smaller than 4MB if UseTransactionalMD5 is true");

                string           range   = null;
                OperationContext context = new OperationContext();
                context.SendingRequest += (sender, e) => range = range ?? e.Request.Headers["x-ms-range"];

                blob.StreamMinimumReadSizeInBytes = 4 * 1024 * 1024;
                result = blob.BeginOpenRead(null, options, context,
                                            ar => waitHandle.Set(),
                                            null);
                waitHandle.WaitOne();
                using (Stream blobStream = blob.EndOpenRead(result))
                {
                    blobStream.ReadByte();
                    Assert.AreEqual("bytes=0-" + (blob.StreamMinimumReadSizeInBytes - 1).ToString(), range);
                    range = null;
                }

                blob.StreamMinimumReadSizeInBytes = 6 * 1024 * 1024;
                options.UseTransactionalMD5       = false;
                result = blob.BeginOpenRead(null, options, context,
                                            ar => waitHandle.Set(),
                                            null);
                waitHandle.WaitOne();
                using (Stream blobStream = blob.EndOpenRead(result))
                {
                    blobStream.ReadByte();
                    Assert.AreEqual("bytes=0-" + (buffer.Length - 1).ToString(), range);
                    range = null;
                }

                blob.StreamMinimumReadSizeInBytes = 16 * 1024;
                result = blob.BeginOpenRead(null, options, context,
                                            ar => waitHandle.Set(),
                                            null);
                waitHandle.WaitOne();
                using (Stream blobStream = blob.EndOpenRead(result))
                {
                    blobStream.ReadByte();
                    Assert.AreEqual("bytes=0-" + (blob.StreamMinimumReadSizeInBytes - 1).ToString(), range);
                    range = null;
                }
            }
        }
Esempio n. 48
0
        public async Task <Stream> GetAsync(string blobName, string containerName = null)
        {
            ICloudBlob blob = await GetBlobReferenceFromServerAsync(blobName, containerName);

            return(await blob.OpenReadAsync(null, null, null));
        }
Esempio n. 49
0
        private async Task <Tuple <IDisposable, string> > LockInternal(ICloudBlob blob)
        {
            string leaseId;

            try
            {
                leaseId = await blob.AcquireLeaseAsync(TimeSpan.FromMinutes(1), null).ConfigureAwait(false);

                LeoTrace.WriteLine("Leased Blob: " + blob.Name);
            }
            catch (StorageException e)
            {
                // If we have a conflict this blob is already locked...
                if (e.RequestInformation.HttpStatusCode == 409)
                {
                    return(null);
                }

                if (e.RequestInformation.HttpStatusCode == 404)
                {
                    leaseId = null;
                }
                else
                {
                    throw e.Wrap(blob.Name);
                }
            }

            // May not have had a blob pushed...
            if (leaseId == null)
            {
                try
                {
                    using (var stream = new MemoryStream(new byte[1]))
                    {
                        try
                        {
                            await blob.UploadFromStreamAsync(stream).ConfigureAwait(false);
                        }
                        catch (StorageException) { }  // Just eat storage exceptions at this point... something was created obviously
                    }
                    leaseId = await blob.AcquireLeaseAsync(TimeSpan.FromMinutes(1), null).ConfigureAwait(false);

                    LeoTrace.WriteLine("Created new blob and lease (2 calls): " + blob.Name);
                }
                catch (StorageException e)
                {
                    // If we have a conflict this blob is already locked...
                    if (e.RequestInformation.HttpStatusCode == 409)
                    {
                        return(null);
                    }

                    if (e.RequestInformation.HttpStatusCode == 404)
                    {
                        return(null);
                    }
                    else
                    {
                        throw e.Wrap(blob.Name);
                    }
                }
            }

            var condition = AccessCondition.GenerateLeaseCondition(leaseId);

            // Every 30 secs keep the lock renewed
            var keepAlive = AsyncEnumerableEx.CreateTimer(TimeSpan.FromSeconds(30))
                            .Select(t =>
            {
                LeoTrace.WriteLine("Renewed Lease: " + blob.Name);
                return(blob.RenewLeaseAsync(condition));
            })
                            .Unwrap()
                            .TakeUntilDisposed(null, t =>
            {
                try
                {
                    // We need to do this to make sure after the dispose the lease is gone
                    blob.ReleaseLeaseAsync(condition).GetAwaiter().GetResult();
                }
                catch (Exception e)
                {
                    LeoTrace.WriteLine("Release failed: " + e.Message);
                }
            });


            return(Tuple.Create((IDisposable)keepAlive, leaseId));
        }
        public ICloudBlob GetReferenceForItem(string itemUrl)
        {
            ICloudBlob blobRef = Client.GetBlobReferenceFromServer(new Uri(itemUrl));

            return(blobRef);
        }
        private async Task DoUploadDownloadFileAsync(ICloudBlob blob, int fileSize)
        {
            string inputFileName  = Path.GetTempFileName();
            string outputFileName = Path.GetTempFileName();

            try
            {
                byte[] buffer = GetRandomBuffer(fileSize);
                using (FileStream file = new FileStream(inputFileName, FileMode.Create, FileAccess.Write))
                {
                    await file.WriteAsync(buffer, 0, buffer.Length);
                }

                await blob.UploadFromFileAsync(inputFileName, FileMode.Open);

                OperationContext context = new OperationContext();
                await blob.UploadFromFileAsync(inputFileName, FileMode.Open, null, null, context);

                Assert.IsNotNull(context.LastResult.ServiceRequestID);

                await TestHelper.ExpectedExceptionAsync <IOException>(
                    async() => await blob.DownloadToFileAsync(outputFileName, FileMode.CreateNew),
                    "CreateNew on an existing file should fail");

                context = new OperationContext();
                await blob.DownloadToFileAsync(outputFileName, FileMode.Create, null, null, context);

                Assert.IsNotNull(context.LastResult.ServiceRequestID);

                using (
                    FileStream inputFileStream = new FileStream(inputFileName, FileMode.Open, FileAccess.Read),
                    outputFileStream = new FileStream(outputFileName, FileMode.Open, FileAccess.Read))
                {
                    await TestHelper.AssertStreamsAreEqualAsync(inputFileStream, outputFileStream);
                }

                await blob.DownloadToFileAsync(outputFileName, FileMode.Append);

                using (
                    FileStream inputFileStream = new FileStream(inputFileName, FileMode.Open, FileAccess.Read),
                    outputFileStream = new FileStream(outputFileName, FileMode.Open, FileAccess.Read))
                {
                    Assert.AreEqual(2 * fileSize, outputFileStream.Length);

                    for (int i = 0; i < fileSize; i++)
                    {
                        Assert.AreEqual(inputFileStream.ReadByte(), outputFileStream.ReadByte());
                    }

                    inputFileStream.Seek(0, SeekOrigin.Begin);
                    for (int i = 0; i < fileSize; i++)
                    {
                        Assert.AreEqual(inputFileStream.ReadByte(), outputFileStream.ReadByte());
                    }
                }
            }
            finally
            {
                File.Delete(inputFileName);
                File.Delete(outputFileName);
            }
        }
 public void Notify(ICloudBlob blobWritten)
 {
     _blobWrittenNotifications.Enqueue(blobWritten);
 }
        /// <summary>
        /// Implements getting the stream without specifying a range.
        /// </summary>
        /// <param name="blob">The blob.</param>
        /// <param name="attributes">The attributes.</param>
        /// <param name="destStream">The destination stream.</param>
        /// <param name="offset">The offset.</param>
        /// <param name="length">The length.</param>
        /// <param name="accessCondition">An <see cref="AccessCondition"/> object that represents the condition that must be met in order for the request to proceed. If <c>null</c>, no condition is used.</param>
        /// <param name="options">A <see cref="BlobRequestOptions"/> object that specifies additional options for the request.</param>
        /// <returns>
        /// A <see cref="RESTCommand{T}"/> that gets the stream.
        /// </returns>
        internal static RESTCommand <NullType> GetBlobImpl(ICloudBlob blob, BlobAttributes attributes, Stream destStream, long?offset, long?length, AccessCondition accessCondition, BlobRequestOptions options)
        {
            string          lockedETag            = null;
            AccessCondition lockedAccessCondition = null;

            bool   isRangeGet             = offset.HasValue;
            bool   arePropertiesPopulated = false;
            string storedMD5 = null;

            long startingOffset = offset.HasValue ? offset.Value : 0;
            long?startingLength = length;
            long?validateLength = null;

            RESTCommand <NullType> getCmd = new RESTCommand <NullType>(blob.ServiceClient.Credentials, attributes.StorageUri);

            options.ApplyToStorageCommand(getCmd);
            getCmd.CommandLocationMode           = CommandLocationMode.PrimaryOrSecondary;
            getCmd.RetrieveResponseStream        = true;
            getCmd.DestinationStream             = destStream;
            getCmd.CalculateMd5ForResponseStream = !options.DisableContentMD5Validation.Value;
            getCmd.BuildRequestDelegate          = (uri, builder, serverTimeout, ctx) =>
                                                   BlobHttpWebRequestFactory.Get(uri, serverTimeout, attributes.SnapshotTime, offset, length, options.UseTransactionalMD5.Value, accessCondition, ctx);
            getCmd.SignRequest    = blob.ServiceClient.AuthenticationHandler.SignRequest;
            getCmd.RecoveryAction = (cmd, ex, ctx) =>
            {
                if ((lockedAccessCondition == null) && !string.IsNullOrEmpty(lockedETag))
                {
                    lockedAccessCondition = AccessCondition.GenerateIfMatchCondition(lockedETag);
                    if (accessCondition != null)
                    {
                        lockedAccessCondition.LeaseId = accessCondition.LeaseId;
                    }
                }

                if (cmd.StreamCopyState != null)
                {
                    offset = startingOffset + cmd.StreamCopyState.Length;
                    if (startingLength.HasValue)
                    {
                        length = startingLength.Value - cmd.StreamCopyState.Length;
                    }
                }

                getCmd.BuildRequestDelegate = (uri, builder, serverTimeout, context) =>
                                              BlobHttpWebRequestFactory.Get(uri, serverTimeout, attributes.SnapshotTime, offset, length, options.UseTransactionalMD5.Value && !arePropertiesPopulated, lockedAccessCondition ?? accessCondition, context);
            };

            getCmd.PreProcessResponse = (cmd, resp, ex, ctx) =>
            {
                HttpResponseParsers.ProcessExpectedStatusCodeNoException(offset.HasValue ? HttpStatusCode.PartialContent : HttpStatusCode.OK, resp, NullType.Value, cmd, ex);

                if (!arePropertiesPopulated)
                {
                    CloudBlobSharedImpl.UpdateAfterFetchAttributes(attributes, resp, isRangeGet);
                    storedMD5 = resp.Headers[HttpResponseHeader.ContentMd5];

                    if (!options.DisableContentMD5Validation.Value &&
                        options.UseTransactionalMD5.Value &&
                        string.IsNullOrEmpty(storedMD5))
                    {
                        throw new StorageException(
                                  cmd.CurrentResult,
                                  SR.MD5NotPresentError,
                                  null)
                              {
                                  IsRetryable = false
                              };
                    }

                    // If the download fails and Get Blob needs to resume the download, going to the
                    // same storage location is important to prevent a possible ETag mismatch.
                    getCmd.CommandLocationMode = cmd.CurrentResult.TargetLocation == StorageLocation.Primary ? CommandLocationMode.PrimaryOnly : CommandLocationMode.SecondaryOnly;
                    lockedETag = attributes.Properties.ETag;
                    if (resp.ContentLength >= 0)
                    {
                        validateLength = resp.ContentLength;
                    }

                    arePropertiesPopulated = true;
                }

                return(NullType.Value);
            };

            getCmd.PostProcessResponse = (cmd, resp, ctx) =>
            {
                HttpResponseParsers.ValidateResponseStreamMd5AndLength(validateLength, storedMD5, cmd);
                return(NullType.Value);
            };

            return(getCmd);
        }
 /// <summary>
 /// Converts the source blob of a copy operation to an appropriate access URI, taking Shared Access Signature credentials into account.
 /// </summary>
 /// <param name="source">The source blob.</param>
 /// <returns>A URI addressing the source blob, using SAS if appropriate.</returns>
 internal static Uri SourceBlobToUri(ICloudBlob source)
 {
     CommonUtility.AssertNotNull("source", source);
     return(source.ServiceClient.Credentials.TransformUri(source.SnapshotQualifiedUri));
 }
Esempio n. 55
0
        private void Sync()
        {
            try
            {
                //we first initialize a new replica in the new server
                if (!targetContainer.Exists())
                {
                    targetContainer.Create();
                }
            }
            catch (StorageException ex)
            {
                //409 is the conflict message.
                if (StorageExceptionCode.Conflict(ex))
                {
                    throw ex;
                }
            }

            if (lastModified == null)
            {
                targetContainer.FetchAttributes();
                if (targetContainer.Metadata.ContainsKey("lastsync"))
                {
                    //if no lastmodified time is provided in the constructor, we still try to be fast.
                    //So, we check to see if by any chance the container previously has synchronized.
                    lastModified = DateTimeOffset.Parse(targetContainer.Metadata["lastsync"]);
                }
            }

            DateTimeOffset startTime = DateTimeOffset.Now;

            try
            {
                IEnumerable <IListBlobItem> sourceBlobList = null;
                if (lastModified != null)
                {
                    sourceBlobList = sourceContainer.ListBlobs().OfType <ICloudBlob>().Where(b => b.Properties.LastModified > lastModified);
                }
                else
                {
                    //this is the slowest sync path. It needs to go through all blobs.
                    //it tries to avoid this path as much as possible.
                    sourceBlobList = sourceContainer.ListBlobs();
                }

                foreach (ICloudBlob sourceBlob in sourceBlobList)
                {
                    ICloudBlob targetBlob = targetContainer.GetBlockBlobReference(sourceBlob.Name);

                    //Since its an inter-account transfer, we need access rights to do the transfer.
                    var sas = sourceContainer.GetSharedAccessSignature(new SharedAccessBlobPolicy()
                    {
                        SharedAccessStartTime  = DateTime.UtcNow.AddMinutes(-5),
                        SharedAccessExpiryTime = DateTime.UtcNow.AddMinutes(30),
                        Permissions            = SharedAccessBlobPermissions.Read,
                    });

                    AccessCondition ac = new AccessCondition();
                    Interlocked.Increment(ref concurrentSyncs);
                    var srcBlockBlobSasUri = string.Format("{0}{1}", sourceBlob.Uri, sas);
                    targetBlob.BeginStartCopyFromBlob(new Uri(srcBlockBlobSasUri), BlobCopyFinished, null);
                }

                while (concurrentSyncs > 0)
                {
                    lock (mylock)
                        Monitor.Wait(mylock, 200);
                }

                targetContainer.Metadata["lastsync"] = startTime.ToString();
                targetContainer.SetMetadata();
            }
            catch (StorageException se)
            {
                //if the blob/container does not exist, it means that it is removed by configurator.
                //We safely return.
                if (StorageExceptionCode.NotFound(se))
                {
                    return;
                }
                throw se;
            }
            catch (Exception ex)
            {
                throw ex;
            }
        }
        static private IAsset CreateAssetFromExistingBlobs(CloudBlobContainer sourceBlobContainer)
        {
            CloudBlobClient destBlobStorage = _destinationStorageAccount.CreateCloudBlobClient();

            // Create a new asset.
            IAsset asset = _context.Assets.Create("NewAsset_" + Guid.NewGuid(), AssetCreationOptions.None);

            IAccessPolicy writePolicy = _context.AccessPolicies.Create("writePolicy",
                                                                       TimeSpan.FromHours(24), AccessPermissions.Write);

            ILocator destinationLocator =
                _context.Locators.CreateLocator(LocatorType.Sas, asset, writePolicy);

            // Get the asset container URI and Blob copy from mediaContainer to assetContainer.
            CloudBlobContainer destAssetContainer =
                destBlobStorage.GetContainerReference((new Uri(destinationLocator.Path)).Segments[1]);

            if (destAssetContainer.CreateIfNotExists())
            {
                destAssetContainer.SetPermissions(new BlobContainerPermissions
                {
                    PublicAccess = BlobContainerPublicAccessType.Blob
                });
            }

            var blobList = sourceBlobContainer.ListBlobs();

            foreach (var sourceBlob in blobList)
            {
                var assetFile = asset.AssetFiles.Create((sourceBlob as ICloudBlob).Name);

                ICloudBlob destinationBlob = destAssetContainer.GetBlockBlobReference(assetFile.Name);

                CopyBlob(sourceBlob as ICloudBlob, destAssetContainer);

                assetFile.ContentFileSize = (sourceBlob as ICloudBlob).Properties.Length;
                assetFile.Update();
                Console.WriteLine("File {0} is of {1} size", assetFile.Name, assetFile.ContentFileSize);
            }

            asset.Update();

            destinationLocator.Delete();
            writePolicy.Delete();

            // Set the primary asset file.
            // If, for example, we copied a set of Smooth Streaming files,
            // set the .ism file to be the primary file.
            // If we, for example, copied an .mp4, then the mp4 would be the primary file.
            var ismAssetFile = asset.AssetFiles.ToList().
                               Where(f => f.Name.EndsWith(".ism", StringComparison.OrdinalIgnoreCase)).ToArray().FirstOrDefault();

            // The following code assigns the first .ism file as the primary file in the asset.
            // An asset should have one .ism file.
            if (ismAssetFile != null)
            {
                ismAssetFile.IsPrimary = true;
                ismAssetFile.Update();
            }

            return(asset);
        }
Esempio n. 57
0
 public CloudLock(ICloudBlob blob, TimeSpan acquireTimeSpan) : this(blob, acquireTimeSpan, TimeSpan.FromSeconds(acquireTimeSpan.TotalSeconds / 2))
 {
 }
Esempio n. 58
0
 public CloudLock(ICloudBlob blob) : this(blob, TimeSpan.FromSeconds(30))
 {
 }
 /// <summary>
 /// whether the specified blob is a snapshot
 /// </summary>
 /// <param name="blob">ICloudBlob object</param>
 /// <returns>true if the specified blob is snapshot, otherwise false</returns>
 internal bool IsSnapshot(ICloudBlob blob)
 {
     return(!string.IsNullOrEmpty(blob.Name) && blob.SnapshotTime != null);
 }
Esempio n. 60
0
        /// <summary>
        /// This renameFile() method renames the file
        /// </summary>
        /// <param name="username"></param>
        /// <param name="path"></param>
        public void renameFile(String username, String path, String newname)    //path> containerID:filepath
        {
            try
            {
                //Break the path <containerID:filepath>
                String[] pathTokens = path.Split(':');

                DBManager.UserM user = new DBManager.UserM();
                Model.UserModel u;
                u = user.getUserRecord(connection, username);

                String container = pathTokens[0];
                String blobfile  = pathTokens[1];

                DBManager.ResourceM contDetail = new DBManager.ResourceM();
                Console.WriteLine("ResourceID:" + container);
                Model.AzureContainerModel cont = contDetail.getResourceById(connection, Int32.Parse(container));

                if (cont.getOwner() != u.getUid())      //user is not owner
                {
                    Resource res = new Resource();
                    if (!res.canWrite(u.getUid(), Int32.Parse(container)))       //not having write permission
                    {
                        throw new DBLikeExceptions.UnauthorizedAccessException();
                    }
                }

                container = cont.getContainerName();


                //Check if container exists
                CloudBlobContainer myContainer       = BlobStorageManager.BlobStorage.getCloudBlobContainer(container); //Container ref
                Boolean            isContainerexists = BlobStorageManager.BlobStorage.isContainerExists(myContainer);

                if (isContainerexists == false)
                {
                    Console.WriteLine("Container not found");
                    throw new DBLikeExceptions.CloudContainerNotFoundException();
                }

                //Check if blob exists
                CloudBlockBlob oldblob      = BlobStorageManager.BlobStorage.getCloudBlob(container + '/' + blobfile); //Get reference to blob
                Boolean        isBlobexists = BlobStorageManager.BlobStorage.isBlobExists(oldblob);

                if (isBlobexists == false)
                {
                    Console.WriteLine("Blob not found");
                    throw new DBLikeExceptions.CloudBlobNotFoundException();
                }

                ICloudBlob newblob = null;
                if (oldblob is CloudBlockBlob)
                {
                    newblob = myContainer.GetBlockBlobReference(newname);
                }
                else
                {
                    newblob = myContainer.GetPageBlobReference(newname);
                }

                //CloudBlockBlob newblob = BlobStorageManager.BlobStorage.getCloudBlob(container + '/' + newname); //Get reference to blob

                //copy the blob
                newblob.StartCopyFromBlob(oldblob.Uri);

                while (true)
                {
                    newblob.FetchAttributes();
                    if (newblob.CopyState.Status != CopyStatus.Pending) //check the copying status
                    {
                        break;
                    }
                    System.Threading.Thread.Sleep(1000); //sleep for a second
                }

                //delete old blobfile
                oldblob.Delete();
            }
            catch (Microsoft.WindowsAzure.Storage.StorageException e)
            {
                Console.WriteLine("File:renameFile says->>" + e.Message);
            }
        }