public TestContext() { // data UtcNow = new DateTimeOffset(2015, 1, 2, 3, 4, 5, 6, TimeSpan.Zero); Content = "foobar"; Container = TestSupport.GetTestContainer(); Prefix = "testpath"; UploadRequest = new UploadRequest { ConnectionString = TestSupport.ConnectionString, Container = Container, ContentType = "text/plain", PathFormat = Prefix + "/{0}.txt", UploadDirect = true, UploadLatest = true, Stream = new MemoryStream(Encoding.UTF8.GetBytes(Content)), Trace = TextWriter.Null }; CollapseRequest = new CollapseRequest { ConnectionString = UploadRequest.ConnectionString, Container = UploadRequest.Container, PathFormat = UploadRequest.PathFormat, Comparer = new OrdinalCollapserComparer(), Trace = TextWriter.Null }; CloudContext = new CloudContext(UploadRequest.ConnectionString, UploadRequest.Container); // dependencies SystemTime = new Mock<ISystemTime>(); PathBuilder = new PathBuilder(); Client = new Client(SystemTime.Object, PathBuilder); // setup SystemTime .Setup(x => x.UtcNow) .Returns(() => UtcNow).Callback(() => UtcNow = UtcNow.AddSeconds(1)); // target Target = new Collapser(PathBuilder); }
public async Task CollapseAsync(CollapseRequest request) { _pathBuilder.Validate(request.PathFormat); var context = new CloudContext(request.ConnectionString, request.Container); if (!await context.BlobContainer.ExistsAsync()) { request.Trace.WriteLine($"The container {request.Container} does not exist so no collapsing is necessary."); return; } // determine the prefix var placeholderIndex = request.PathFormat.IndexOf("{0}", StringComparison.Ordinal); var prefix = request.PathFormat.Substring(0, placeholderIndex); var suffix = request.PathFormat.Substring(placeholderIndex + "{0}".Length); var latestPath = _pathBuilder.GetLatest(request.PathFormat); // collect and sort all of the blob names var blobNames = new List<string>(); var token = (BlobContinuationToken) null; do { var segment = await context.BlobContainer.ListBlobsSegmentedAsync(prefix, true, BlobListingDetails.All, null, token, null, null); token = segment.ContinuationToken; // filter out packages that don't match the path format and the latest var segmentBlobNames = segment .Results .OfType<ICloudBlob>() .Select(x => x.Name) .Where(x => x.EndsWith(suffix) && x != latestPath); int before = blobNames.Count; blobNames.AddRange(segmentBlobNames); int added = blobNames.Count - before; request.Trace.WriteLine($"Fetched {added} blobs."); } while (token != null); blobNames.Sort(request.Comparer); request.Trace.WriteLine($"{blobNames.Count} blobs were sorted."); // collapse the blobs int indexX = 0; int indexY = 1; while(indexX < blobNames.Count - 1 && indexY < blobNames.Count) { var nameX = blobNames[indexX]; var nameY = blobNames[indexY]; var blobX = context.BlobContainer.GetBlockBlobReference(nameX); var blobY = context.BlobContainer.GetBlockBlobReference(nameY); using (var streamX = await blobX.OpenReadAsync()) using (var streamY = await blobY.OpenReadAsync()) { if (blobX.Properties.ContentMD5 == blobY.Properties.ContentMD5 || await request.Comparer.EqualsAsync(nameX, streamX, nameY, streamY, CancellationToken.None)) { request.Trace.WriteLine($"Deleting '{nameY}'."); await blobY.DeleteAsync(); indexY++; } else { indexX = indexY; indexY++; } } } }