/// <summary> /// Set the metadata of an existing blob if it has not been modified since it was last retrieved. /// </summary> /// <param name="blobProperties">The blob properties object whose metadata is to be updated. /// Typically obtained by a previous call to GetBlob or GetBlobProperties</param> /// <returns>true if the blob metadata was updated. false if it was not updated because the blob /// has been modified</returns> public abstract bool UpdateBlobMetadataIfNotModified(BlobProperties blobProperties);
internal void Assign(BlobProperties other) { Name = other.Name; Uri = other.Uri; ContentEncoding = other.ContentEncoding; ContentLanguage = other.ContentLanguage; ContentLength = other.ContentLength; ContentType = other.ContentType; ETag = other.ETag; LastModifiedTime = other.LastModifiedTime; Metadata = (other.Metadata != null ? new NameValueCollection(other.Metadata) : null) ; }
/// <summary> /// Create a new blob or overwrite an existing blob. /// </summary> /// <param name="blobProperties">The properties of the blob</param> /// <param name="blobContents">The contents of the blob</param> /// <param name="overwrite">Should this request overwrite an existing blob ?</param> /// <returns>true if the blob was created. false if the blob already exists and <paramref name="overwrite"/>was set to false</returns> /// <remarks>The LastModifiedTime property of <paramref name="blobProperties"/> is set as a result of this call. /// This method also has an effect on the ETag values that are managed by the service.</remarks> public abstract bool CreateBlob(BlobProperties blobProperties, BlobContents blobContents, bool overwrite);
/// <summary> /// Gets the blob contents and properties if the blob has not been modified since the time specified. /// Use this method if you have cached the contents of a blob and want to avoid retrieving the blob /// if it has not changed since the last time you retrieved it. /// </summary> /// <param name="blobProperties">The properties of the blob obtained from an earlier call to GetBlob. This /// parameter is updated by the call if the blob has been modified</param> /// <param name="blobContents">Contains the stream to which the contents of the blob are written if it has been /// modified</param> /// <param name="transferAsChunks">Should the blob be gotten in pieces. This requires more round-trips, but will retry smaller pieces in case of failure.</param> /// <returns>true if the blob has been modified, false otherwise</returns> public abstract bool GetBlobIfModified(BlobProperties blobProperties, BlobContents blobContents, bool transferAsChunks);
/// <summary> /// Delete a blob with the given name if the blob has not been modified since it was last obtained. /// Use this method for optimistic concurrency to avoid deleting a blob that has been modified since /// the last time you retrieved it /// </summary> /// <param name="blob">A blob object (typically previously obtained from a GetBlob call)</param> /// <param name="modified">This out parameter is set to true if the blob was not deleted because /// it was modified</param> /// <returns>true if the blob exists and was successfully deleted, false if the blob does not exist or was /// not deleted because the blob was modified.</returns> public override bool DeleteBlobIfNotModified(BlobProperties blob, out bool modified) { return DeleteBlobImpl(blob.Name, blob.ETag, out modified); }
private bool SetBlobMetadataImpl(BlobProperties blobProperties, string eTag) { bool retval = false; RetryPolicy(() => { NameValueCollection queryParams = new NameValueCollection(); queryParams.Add(StorageHttpConstants.QueryParams.QueryParamComp, StorageHttpConstants.CompConstants.Metadata); ResourceUriComponents uriComponents; Uri uri = Utilities.CreateRequestUri( BaseUri, this.UsePathStyleUris, AccountName, ContainerName, blobProperties.Name, Timeout, queryParams, out uriComponents ); HttpWebRequest request = Utilities.CreateHttpRequest(uri, StorageHttpConstants.HttpMethod.Put, Timeout); if (blobProperties.Metadata != null) { Utilities.AddMetadataHeaders(request, blobProperties.Metadata); } if (eTag != null) { request.Headers.Add(StorageHttpConstants.HeaderNames.IfMatch, eTag); } credentials.SignRequest(request, uriComponents); try { using (HttpWebResponse response = (HttpWebResponse)request.GetResponse()) { if (response.StatusCode == HttpStatusCode.OK) { retval = true; } else if ((response.StatusCode == HttpStatusCode.PreconditionFailed || response.StatusCode == HttpStatusCode.NotModified) && eTag != null) { retval = false; } else { Utilities.ProcessUnexpectedStatusCode(response); retval = false; } blobProperties.LastModifiedTime = response.LastModified.ToUniversalTime(); blobProperties.ETag = response.Headers[StorageHttpConstants.HeaderNames.ETag]; response.Close(); } } catch (IOException ioe) { throw new StorageServerException(StorageErrorCode.TransportError, "The connection may be lost", default(HttpStatusCode), ioe); } catch (System.TimeoutException te) { throw new StorageServerException(StorageErrorCode.ServiceTimeout, "Timeout during blob metadata upload", HttpStatusCode.RequestTimeout, te); } catch (WebException we) { if (we.Response != null) { HttpWebResponse response = (HttpWebResponse)we.Response; if (eTag != null && (response.StatusCode == HttpStatusCode.PreconditionFailed || response.StatusCode == HttpStatusCode.NotModified)) { retval = false; return; } } throw Utilities.TranslateWebException(we); } }); return retval; }
private bool UploadData(BlobProperties blobProperties, Stream stream, long length, bool overwrite, string eTag, NameValueCollection nvc) { ResourceUriComponents uriComponents; Uri blobUri = Utilities.CreateRequestUri(BaseUri, this.UsePathStyleUris, AccountName, ContainerName, blobProperties.Name, Timeout, nvc, out uriComponents); HttpWebRequest request = CreateHttpRequestForPutBlob( blobUri, StorageHttpConstants.HttpMethod.Put, blobProperties, length, overwrite, eTag ); credentials.SignRequest(request, uriComponents); bool retval = false; try { using (Stream requestStream = request.GetRequestStream()) { Utilities.CopyStream(stream, requestStream, length); using (HttpWebResponse response = (HttpWebResponse)request.GetResponse()) { if (response.StatusCode == HttpStatusCode.Created) { retval = true; } else if (!overwrite && (response.StatusCode == HttpStatusCode.PreconditionFailed || response.StatusCode == HttpStatusCode.NotModified)) { retval = false; } else { retval = false; Utilities.ProcessUnexpectedStatusCode(response); } blobProperties.LastModifiedTime = response.LastModified.ToUniversalTime(); blobProperties.ETag = response.Headers[StorageHttpConstants.HeaderNames.ETag]; response.Close(); } requestStream.Close(); } } catch (IOException ioe) { throw new StorageServerException( StorageErrorCode.TransportError, "The connection may be lost", default(HttpStatusCode), null, ioe ); } catch (System.TimeoutException te) { throw new StorageServerException( StorageErrorCode.ServiceTimeout, "Timeout during blob upload", HttpStatusCode.RequestTimeout, null, te ); } catch (WebException we) { if (we.Response != null) { HttpWebResponse response = (HttpWebResponse)we.Response; if ((response.StatusCode == HttpStatusCode.PreconditionFailed || response.StatusCode == HttpStatusCode.NotModified) && (!overwrite || eTag != null)) { retval = false; return retval; } } throw Utilities.TranslateWebException(we); } return retval; }
/// <summary> /// Uploads a blob in chunks. /// </summary> /// <param name="blobProperties"></param> /// <param name="stream"></param> /// <param name="overwrite"></param> /// <param name="eTag"></param> /// <returns></returns> private bool PutLargeBlobImpl(BlobProperties blobProperties, Stream stream, bool overwrite, string eTag) { bool retval = false; // Since we got a large block, chunk it into smaller pieces called blocks long blockSize = StorageHttpConstants.BlobBlockConstants.BlockSize; long startPosition = stream.Position; long length = stream.Length - startPosition; int numBlocks = (int)Math.Ceiling((double)length / blockSize); string[] blockIds = new string[numBlocks]; //We can retry only if the stream supports seeking. An alternative is to buffer the data in memory //but we do not do this currently. RetryPolicy R = (stream.CanSeek ? this.RetryPolicy : RetryPolicies.NoRetry); //Upload each of the blocks, retrying any failed uploads for (int i = 0; i < numBlocks; ++i) { string blockId = Convert.ToBase64String(System.BitConverter.GetBytes(i)); blockIds[i] = blockId; R(() => { // Rewind the stream to appropriate location in case this is a retry if (stream.CanSeek) stream.Position = startPosition + i * blockSize; NameValueCollection nvc = new NameValueCollection(); nvc.Add(QueryParams.QueryParamComp, CompConstants.Block); nvc.Add(QueryParams.QueryParamBlockId, blockId); // The block naming should be more elaborate to give more meanings on GetBlockList long blockLength = Math.Min(blockSize, length - stream.Position); retval = UploadData(blobProperties, stream, blockLength, overwrite, eTag, nvc); }); } // Now commit the list // First create the output using (MemoryStream buffer = new MemoryStream()) { // construct our own XML segment with correct blockId's XmlTextWriter writer = new XmlTextWriter(buffer, Encoding.UTF8); writer.WriteStartDocument(); writer.WriteStartElement(XmlElementNames.BlockList); foreach (string id in blockIds) { writer.WriteElementString(XmlElementNames.Block, id); } writer.WriteEndElement(); writer.WriteEndDocument(); writer.Flush(); buffer.Position = 0; //Rewind NameValueCollection nvc = new NameValueCollection(); nvc.Add(QueryParams.QueryParamComp, CompConstants.BlockList); retval = UploadData(blobProperties, buffer, buffer.Length, overwrite, eTag, nvc); } return retval; }
private bool PutBlobImpl(BlobProperties blobProperties, Stream stream, bool overwrite, string eTag) { // If the blob is large, we should use blocks to upload it in pieces. // This will ensure that a broken connection will only impact a single piece long originalPosition = stream.Position; long length = stream.Length - stream.Position; if (length > StorageHttpConstants.BlobBlockConstants.MaximumBlobSizeBeforeTransmittingAsBlocks) return PutLargeBlobImpl(blobProperties, stream, overwrite, eTag); bool retval = false; RetryPolicy R = stream.CanSeek ? this.RetryPolicy : RetryPolicies.NoRetry; R(() => { if (stream.CanSeek) stream.Position = originalPosition; retval = UploadData(blobProperties, stream, length, overwrite, eTag, new NameValueCollection()); }); return retval; }
internal bool UploadStream(string blobName, Stream output, bool overwrite) { BlobContainer container = GetContainer(); try { output.Position = 0; //Rewind to start Log.Write(EventKind.Information, "Uploading contents of blob {0}", ContainerUrl + _PathSeparator + blobName); BlobProperties properties = new BlobProperties(blobName); return container.CreateBlob(properties, new BlobContents(output), overwrite); } catch (StorageException se) { Log.Write(EventKind.Error, "Error uploading blob {0}: {1}", ContainerUrl + _PathSeparator + blobName, se.Message); throw; } }
internal bool GetBlobContentsWithoutInitialization(string blobName, Stream outputStream, out BlobProperties properties) { Debug.Assert(outputStream != null); BlobContainer container = GetContainer(); try { properties = container.GetBlob(blobName, new BlobContents(outputStream), false); Log.Write(EventKind.Information, "Getting contents of blob {0}", _info.BaseUri + _PathSeparator + _containerName + _PathSeparator + blobName); return true; } catch (StorageClientException sc) { if (sc.ErrorCode == StorageErrorCode.ResourceNotFound || sc.ErrorCode == StorageErrorCode.BlobNotFound) { properties = null; return false; } else throw; } }
/// <summary> /// Gets the blob contents and properties if the blob has not been modified since the time specified. /// Use this method if you have cached the contents of a blob and want to avoid retrieving the blob /// if it has not changed since the last time you retrieved it. /// </summary> /// <param name="blobProperties">The properties of the blob obtained from an earlier call to GetBlob. This /// parameter is updated by the call if the blob has been modified</param> /// <param name="blobContents">Contains the stream to which the contents of the blob are written if it has been /// modified</param> /// <param name="transferAsChunks">Should the blob be gotten in pieces. This requires more round-trips, but will retry smaller piecs in case of failure.</param> /// <returns>true if the blob has been modified, false otherwise</returns> public override bool GetBlobIfModified(BlobProperties blobProperties, BlobContents blobContents, bool transferAsChunks) { bool modified = true; BlobProperties newProperties = GetBlobImpl(blobProperties.Name, blobContents.AsStream, blobProperties.ETag, transferAsChunks, out modified); if (modified) blobProperties.Assign(newProperties); return modified; }
/// <summary> /// Copies a Blob from source to destination /// </summary> /// <param name="sourceProperties">Blob Properties of the source blob</param> /// <param name="sourceBlobName">Name of the source blob in the format /accountName/containerName/blobName</param> /// <param name="destinationBlobName">destination blob name</param> /// <param name="versionString">version string of the API version</param> /// <returns></returns> public abstract bool CopyBlob(BlobProperties sourceProperties, string sourceBlobName, string destinationBlobName, string versionString);
/// <summary> /// Updates an existing blob if it has not been modified since the specified time which is typically /// the last modified time of the blob when you retrieved it. /// Use this method to implement optimistic concurrency by avoiding clobbering changes to the blob /// made by another writer. /// </summary> /// <param name="blobProperties">The properties of the blob. This object should be one previously /// obtained from a call to GetBlob or GetBlobProperties and have its LastModifiedTime property set.</param> /// <param name="contents">The contents of the blob. The contents of the blob should be readable</param> /// <returns>true if the blob was updated. false if the blob has changed since the last time</returns> /// <remarks>The LastModifiedTime property of <paramref name="properties"/> is set as a result of this call</remarks> public override bool UpdateBlobIfNotModified(BlobProperties blobProperties, BlobContents contents) { return PutBlobImpl(blobProperties, contents.AsStream, true, blobProperties.ETag); }
/// <summary> /// Updates an existing blob if it has not been modified since the specified time which is typically /// the last modified time of the blob when you retrieved it. /// Use this method to implement optimistic concurrency by avoiding clobbering changes to the blob /// made by another writer. /// </summary> /// <param name="blob">The properties of the blob. This object should be one previously /// obtained from a call to GetBlob or GetBlobProperties and have its LastModifiedTime property set.</param> /// <param name="contents">The contents of the blob. The contents of the blob should be readable</param> /// <returns>true if the blob was updated. false if the blob has changed since the last time</returns> /// <remarks>The LastModifiedTime property of <paramref name="blob"/> is set as a result of this call. /// This method also has an effect on the ETag values that are managed by the service if the update was /// successful.</remarks> public abstract bool UpdateBlobIfNotModified(BlobProperties blob, BlobContents contents);
/// <summary> /// Set the metadata of an existing blob. /// </summary> /// <param name="blobProperties">The blob properties object whose metadata is to be updated</param> public override void UpdateBlobMetadata(BlobProperties blobProperties) { SetBlobMetadataImpl(blobProperties, null); }
/// <summary> /// Set the metadata of an existing blob. /// </summary> /// <param name="blobProperties">The blob properties object whose metadata is to be updated</param> public abstract void UpdateBlobMetadata(BlobProperties blobProperties);
/// <summary> /// Set the metadata of an existing blob if it has not been modified since it was last retrieved. /// </summary> /// <param name="blobProperties">The blob properties object whose metadata is to be updated. /// Typically obtained by a previous call to GetBlob or GetBlobProperties</param> public override bool UpdateBlobMetadataIfNotModified(BlobProperties blobProperties) { return SetBlobMetadataImpl(blobProperties, blobProperties.ETag); }
/// <summary> /// Delete a blob with the given name if the blob has not been modified since it was last obtained. /// Use this method for optimistic concurrency to avoid deleting a blob that has been modified since /// the last time you retrieved it /// </summary> /// <param name="blob">A blob object (typically previously obtained from a GetBlob call)</param> /// <param name="modified">This out parameter is set to true if the blob was not deleted because /// it was modified</param> /// <returns>true if the blob exists and was successfully deleted, false if the blob does not exist or was /// not deleted because the blob was modified.</returns> public abstract bool DeleteBlobIfNotModified(BlobProperties blob, out bool modified);
private static BlobProperties BlobPropertiesFromResponse( string blobName, Uri blobUri, HttpWebResponse response ) { BlobProperties blobProperties = new BlobProperties(blobName); blobProperties.Uri = blobUri; blobProperties.ContentEncoding = response.Headers[StorageHttpConstants.HeaderNames.ContentEncoding]; blobProperties.LastModifiedTime = response.LastModified.ToUniversalTime(); blobProperties.ETag = response.Headers[StorageHttpConstants.HeaderNames.ETag]; blobProperties.ContentLanguage = response.Headers[StorageHttpConstants.HeaderNames.ContentLanguage]; blobProperties.ContentLength = response.ContentLength; blobProperties.ContentType = response.ContentType; NameValueCollection metadataEntries = MetadataFromHeaders(response.Headers); if (metadataEntries.Count > 0) blobProperties.Metadata = metadataEntries; return blobProperties; }
private static ListBlobsResult ListBlobsResultFromResponse(Stream responseBody) { List<BlobProperties> blobs = new List<BlobProperties>(); List<string> commonPrefixes = new List<string>(); string nextMarker = null; XmlDocument doc = new XmlDocument(); try { doc.Load(responseBody); } catch (XmlException xe) { throw new StorageServerException(StorageErrorCode.ServiceBadResponse, "The result of a ListBlobs operation could not be parsed", default(HttpStatusCode), xe); } // Get the commonPrefixes XmlNodeList prefixNodes = doc.SelectNodes(XPathQueryHelper.CommonPrefixQuery); foreach (XmlNode prefixNode in prefixNodes) { string blobPrefix = XPathQueryHelper.LoadSingleChildStringValue( prefixNode, StorageHttpConstants.XmlElementNames.BlobPrefixName, false); commonPrefixes.Add(blobPrefix); } // Get all the blobs returned as the listing results XmlNodeList blobNodes = doc.SelectNodes(XPathQueryHelper.BlobQuery); foreach (XmlNode blobNode in blobNodes) { DateTime? lastModified = XPathQueryHelper.LoadSingleChildDateTimeValue( blobNode, StorageHttpConstants.XmlElementNames.LastModified, false); string eTag = XPathQueryHelper.LoadSingleChildStringValue( blobNode, StorageHttpConstants.XmlElementNames.Etag, false); string contentType = XPathQueryHelper.LoadSingleChildStringValue( blobNode, StorageHttpConstants.XmlElementNames.ContentType, false); string contentEncoding = XPathQueryHelper.LoadSingleChildStringValue( blobNode, StorageHttpConstants.XmlElementNames.ContentEncoding, false); string contentLanguage = XPathQueryHelper.LoadSingleChildStringValue( blobNode, StorageHttpConstants.XmlElementNames.ContentLanguage, false); long? blobSize = XPathQueryHelper.LoadSingleChildLongValue( blobNode, StorageHttpConstants.XmlElementNames.Size, false); string blobUrl = XPathQueryHelper.LoadSingleChildStringValue( blobNode, StorageHttpConstants.XmlElementNames.Url, true); string blobName = XPathQueryHelper.LoadSingleChildStringValue( blobNode, StorageHttpConstants.XmlElementNames.BlobName, true); BlobProperties properties = new BlobProperties(blobName); properties.Uri = new Uri(blobUrl); if (lastModified.HasValue) properties.LastModifiedTime = lastModified.Value; properties.ContentEncoding = contentEncoding; properties.ContentLanguage = contentLanguage; properties.ETag = eTag; properties.ContentLength = (blobSize == null ? 0 : blobSize.Value); properties.ContentType = contentType; blobs.Add(properties); } // Get the nextMarker XmlNode nextMarkerNode = doc.SelectSingleNode(XPathQueryHelper.NextMarkerQuery); if (nextMarkerNode != null && nextMarkerNode.FirstChild != null) { nextMarker = nextMarkerNode.FirstChild.Value; } return new ListBlobsResult(blobs, commonPrefixes, nextMarker); }
private HttpWebRequest CreateHttpRequestForPutBlob(Uri uri, string httpMethod, BlobProperties blobProperties, long contentLength, bool overwrite, string eTag) { HttpWebRequest request = Utilities.CreateHttpRequest(uri, httpMethod, Timeout); if (blobProperties.ContentEncoding != null) request.Headers.Add(StorageHttpConstants.HeaderNames.ContentEncoding, blobProperties.ContentEncoding); if (blobProperties.ContentLanguage != null) request.Headers.Add(StorageHttpConstants.HeaderNames.ContentLanguage, blobProperties.ContentLanguage); if (blobProperties.ContentType != null) request.ContentType = blobProperties.ContentType; if (eTag != null) request.Headers.Add(StorageHttpConstants.HeaderNames.IfMatch, eTag); if (blobProperties.Metadata != null && blobProperties.Metadata.Count > 0) { Utilities.AddMetadataHeaders(request, blobProperties.Metadata); } request.ContentLength = contentLength; if (!overwrite) { request.Headers.Set(StorageHttpConstants.HeaderNames.IfNoneMatch, "*"); } return request; }
/// <summary> /// Create a new blob or overwrite an existing blob. /// </summary> /// /// <param name="blobProperties">The properties of the blob</param> /// <param name="blobContents">The contents of the blob</param> /// <param name="overwrite">Should this request overwrite an existing blob ?</param> /// <returns>true if the blob was created. false if the blob already exists and <paramref name="overwrite"/>was set to false</returns> /// <remarks>The LastModifiedTime property of <paramref name="blobProperties"/> is set as a result of this call</remarks> public override bool CreateBlob(BlobProperties blobProperties, BlobContents blobContents, bool overwrite) { return PutBlobImpl(blobProperties, blobContents.AsStream, overwrite, null); }
public override bool Write(string key, byte[] data, bool overwrite) { BlobProperties properties = new BlobProperties(key); BlobContents contents = new BlobContents(data); return blobContainer.CreateBlob(properties, contents, overwrite); }
internal MemoryStream GetBlobContent(string blobName, out BlobProperties properties) { MemoryStream blobContent = new MemoryStream(); properties = GetBlobContent(blobName, blobContent); blobContent.Seek(0, SeekOrigin.Begin); return blobContent; }