public async Task<UploadResult> UploadAsync(UploadRequest request) { // validate input _pathBuilder.Validate(request.PathFormat); // initialize request.Trace.Write("Initializing..."); var context = new CloudContext(request.ConnectionString, request.Container); await context.BlobContainer.CreateIfNotExistsAsync( BlobContainerPublicAccessType.Blob, new BlobRequestOptions(), null); request.Trace.WriteLine(" done."); // set the direct var result = new UploadResult(); CloudBlockBlob directBlob = null; if (request.UploadDirect) { if (request.Type == UploadRequestType.Number) { directBlob = await UploadDirectNumberAsync(request, context, result); } else { directBlob = await UploadDirectTimestampAsync(request, context); } } // set the latest CloudBlockBlob latestBlob = null; if (request.UploadLatest) { var latestPath = _pathBuilder.GetLatest(request.PathFormat); if (directBlob == null) { latestBlob = await UploadBlobAsync(context, request, latestPath, direct: false); } else { request.Trace.Write($"Copying the direct blob to the latest blob at '{latestPath}'..."); latestBlob = context.BlobContainer.GetBlockBlobReference(latestPath); var sourceAccessCondition = new AccessCondition {IfMatchETag = directBlob.Properties.ETag}; var destAccessCondition = new AccessCondition {IfMatchETag = request.ETag}; await latestBlob.StartCopyAsync(directBlob, sourceAccessCondition, destAccessCondition, null, null); while (latestBlob.CopyState.Status == CopyStatus.Pending) { await Task.Delay(100); await latestBlob.ExistsAsync(); } request.Trace.WriteLine(" done."); } } request.Trace.WriteLine(); if (directBlob != null) { result.DirectUri = directBlob.Uri; result.DirectETag = directBlob.Properties.ETag; request.Trace.WriteLine($"Direct: {directBlob.Uri}"); } if (latestBlob != null) { result.LatestUri = latestBlob.Uri; result.LatestETag = latestBlob.Properties.ETag; request.Trace.WriteLine($"Latest: {latestBlob.Uri}"); } return result; }
public Uri GetLatestUri(GetLatestRequest request) { var context = new CloudContext(request.ConnectionString, request.Container); var latestPath = _pathBuilder.GetLatest(request.PathFormat); var latestBlob = context.BlobContainer.GetBlockBlobReference(latestPath); return latestBlob.Uri; }
public async Task<UriResult> GetLatestUriResultAsync(GetLatestRequest request) { var context = new CloudContext(request.ConnectionString, request.Container); var latestPath = _pathBuilder.GetLatest(request.PathFormat); var latestBlob = context.BlobContainer.GetBlockBlobReference(latestPath); if (!await latestBlob.ExistsAsync()) { return null; } return new UriResult { Uri = latestBlob.Uri, ETag = latestBlob.Properties.ETag }; }
private static async Task<CloudBlockBlob> UploadBlobAsync(CloudContext context, UploadRequest request, string blobPath, bool direct) { request.Trace.Write($"Uploading the blob at '{blobPath}'..."); var blob = context.BlobContainer.GetBlockBlobReference(blobPath); AccessCondition accessCondition; if (!direct && !request.UseETags) { accessCondition = null; } if (direct || request.ETag == null) { accessCondition = AccessCondition.GenerateIfNoneMatchCondition("*"); } else { accessCondition = AccessCondition.GenerateIfMatchCondition(request.ETag); } // upload the blob await blob.UploadFromStreamAsync(request.Stream, accessCondition, options: null, operationContext: null); request.Trace.WriteLine(" done."); // set the content type if (!string.IsNullOrWhiteSpace(request.ContentType)) { request.Trace.Write($"Setting the content type of '{blobPath}'..."); blob.Properties.ContentType = request.ContentType; await blob.SetPropertiesAsync(); request.Trace.WriteLine(" done."); } return blob; }
public async Task<StreamResult> GetLatestStreamAsync(GetLatestRequest request) { var context = new CloudContext(request.ConnectionString, request.Container); var latestPath = _pathBuilder.GetLatest(request.PathFormat); var latestBlob = context.BlobContainer.GetBlockBlobReference(latestPath); try { var stream = await latestBlob.OpenReadAsync(); return new StreamResult { Stream = stream, ETag = latestBlob.Properties.ETag, ContentMD5 = latestBlob.Properties.ContentMD5 }; } catch (StorageException e) { if (e.RequestInformation.HttpStatusCode == 404) { request.Trace.WriteLine($"The stream could not be found due to the following error: {e.RequestInformation.HttpStatusMessage}"); return null; } throw; } }
private async Task<CloudBlockBlob> UploadDirectNumberAsync(UploadRequest request, CloudContext context, UploadResult result) { var latestNumberPath = _pathBuilder.GetDirect(request.PathFormat, 0); var latestNumberBlob = context.BlobContainer.GetBlockBlobReference(latestNumberPath); string latestNumberEtag = null; var latestNumber = 0; if (request.LatestNumber.HasValue) { // Use the provided version number latestNumberEtag = request.LatestNumberETag; latestNumber = request.LatestNumber.Value; } else { // Determine what version number what last used try { await latestNumberBlob.FetchAttributesAsync(); latestNumberEtag = latestNumberBlob.Properties.ETag; var latestNumberString = latestNumberBlob.Metadata[LatestNumberMetadataKey]; latestNumber = int.Parse(latestNumberString); } catch (StorageException e) { if (e.RequestInformation.HttpStatusCode != 404) { throw; } } latestNumber++; } // Upload the stream var latestPath = _pathBuilder.GetDirect(request.PathFormat, latestNumber); var directBlob = await UploadBlobAsync(context, request, latestPath, direct: true); // Update the latest version record AccessCondition accessCondition; if (!request.UseETags) { accessCondition = null; } else if (latestNumberEtag != null) { accessCondition = AccessCondition.GenerateIfMatchCondition(latestNumberEtag); } else { accessCondition = AccessCondition.GenerateIfNoneMatchCondition("*"); } latestNumberBlob.Metadata[LatestNumberMetadataKey] = latestNumber.ToString(); await latestNumberBlob.UploadFromByteArrayAsync(new byte[0], 0, 0, accessCondition, options: null, operationContext: null); // Set the latest etag and number on the result result.LatestNumberETag = latestNumberBlob.Properties.ETag; result.LatestNumber = latestNumber; return directBlob; }
private async Task<CloudBlockBlob> UploadDirectTimestampAsync(UploadRequest request, CloudContext context) { var latestPath = _pathBuilder.GetDirect(request.PathFormat, _systemTime.UtcNow); var directBlob = await UploadBlobAsync(context, request, latestPath, direct: true); return directBlob; }
public TestContext() { // data UtcNow = new DateTimeOffset(2015, 1, 2, 3, 4, 5, 6, TimeSpan.Zero); Content = "foobar"; Container = TestSupport.GetTestContainer(); Prefix = "testpath"; UploadRequest = new UploadRequest { ConnectionString = TestSupport.ConnectionString, Container = Container, ContentType = "text/plain", PathFormat = Prefix + "/{0}.txt", UploadDirect = true, UploadLatest = true, Stream = new MemoryStream(Encoding.UTF8.GetBytes(Content)), Trace = TextWriter.Null }; CollapseRequest = new CollapseRequest { ConnectionString = UploadRequest.ConnectionString, Container = UploadRequest.Container, PathFormat = UploadRequest.PathFormat, Comparer = new OrdinalCollapserComparer(), Trace = TextWriter.Null }; CloudContext = new CloudContext(UploadRequest.ConnectionString, UploadRequest.Container); // dependencies SystemTime = new Mock<ISystemTime>(); PathBuilder = new PathBuilder(); Client = new Client(SystemTime.Object, PathBuilder); // setup SystemTime .Setup(x => x.UtcNow) .Returns(() => UtcNow).Callback(() => UtcNow = UtcNow.AddSeconds(1)); // target Target = new Collapser(PathBuilder); }
public static void DeleteContainer(string container) { var context = new CloudContext(ConnectionString, container); context.BlobContainer.DeleteIfExistsAsync().Wait(); }
public async Task CollapseAsync(CollapseRequest request) { _pathBuilder.Validate(request.PathFormat); var context = new CloudContext(request.ConnectionString, request.Container); if (!await context.BlobContainer.ExistsAsync()) { request.Trace.WriteLine($"The container {request.Container} does not exist so no collapsing is necessary."); return; } // determine the prefix var placeholderIndex = request.PathFormat.IndexOf("{0}", StringComparison.Ordinal); var prefix = request.PathFormat.Substring(0, placeholderIndex); var suffix = request.PathFormat.Substring(placeholderIndex + "{0}".Length); var latestPath = _pathBuilder.GetLatest(request.PathFormat); // collect and sort all of the blob names var blobNames = new List<string>(); var token = (BlobContinuationToken) null; do { var segment = await context.BlobContainer.ListBlobsSegmentedAsync(prefix, true, BlobListingDetails.All, null, token, null, null); token = segment.ContinuationToken; // filter out packages that don't match the path format and the latest var segmentBlobNames = segment .Results .OfType<ICloudBlob>() .Select(x => x.Name) .Where(x => x.EndsWith(suffix) && x != latestPath); int before = blobNames.Count; blobNames.AddRange(segmentBlobNames); int added = blobNames.Count - before; request.Trace.WriteLine($"Fetched {added} blobs."); } while (token != null); blobNames.Sort(request.Comparer); request.Trace.WriteLine($"{blobNames.Count} blobs were sorted."); // collapse the blobs int indexX = 0; int indexY = 1; while(indexX < blobNames.Count - 1 && indexY < blobNames.Count) { var nameX = blobNames[indexX]; var nameY = blobNames[indexY]; var blobX = context.BlobContainer.GetBlockBlobReference(nameX); var blobY = context.BlobContainer.GetBlockBlobReference(nameY); using (var streamX = await blobX.OpenReadAsync()) using (var streamY = await blobY.OpenReadAsync()) { if (blobX.Properties.ContentMD5 == blobY.Properties.ContentMD5 || await request.Comparer.EqualsAsync(nameX, streamX, nameY, streamY, CancellationToken.None)) { request.Trace.WriteLine($"Deleting '{nameY}'."); await blobY.DeleteAsync(); indexY++; } else { indexX = indexY; indexY++; } } } }