public async Task<UploadResult> UploadAsync(UploadRequest request)
{
// validate input
_pathBuilder.Validate(request.PathFormat);
// initialize
request.Trace.Write("Initializing...");
var context = new CloudContext(request.ConnectionString, request.Container);
await context.BlobContainer.CreateIfNotExistsAsync(
BlobContainerPublicAccessType.Blob,
new BlobRequestOptions(),
null);
request.Trace.WriteLine(" done.");
// set the direct
var result = new UploadResult();
CloudBlockBlob directBlob = null;
if (request.UploadDirect)
{
if (request.Type == UploadRequestType.Number)
{
directBlob = await UploadDirectNumberAsync(request, context, result);
}
else
{
directBlob = await UploadDirectTimestampAsync(request, context);
}
}
// set the latest
CloudBlockBlob latestBlob = null;
if (request.UploadLatest)
{
var latestPath = _pathBuilder.GetLatest(request.PathFormat);
if (directBlob == null)
{
latestBlob = await UploadBlobAsync(context, request, latestPath, direct: false);
}
else
{
request.Trace.Write($"Copying the direct blob to the latest blob at '{latestPath}'...");
latestBlob = context.BlobContainer.GetBlockBlobReference(latestPath);
var sourceAccessCondition = new AccessCondition {IfMatchETag = directBlob.Properties.ETag};
var destAccessCondition = new AccessCondition {IfMatchETag = request.ETag};
await latestBlob.StartCopyAsync(directBlob, sourceAccessCondition, destAccessCondition, null, null);
while (latestBlob.CopyState.Status == CopyStatus.Pending)
{
await Task.Delay(100);
await latestBlob.ExistsAsync();
}
request.Trace.WriteLine(" done.");
}
}
request.Trace.WriteLine();
if (directBlob != null)
{
result.DirectUri = directBlob.Uri;
result.DirectETag = directBlob.Properties.ETag;
request.Trace.WriteLine($"Direct: {directBlob.Uri}");
}
if (latestBlob != null)
{
result.LatestUri = latestBlob.Uri;
result.LatestETag = latestBlob.Properties.ETag;
request.Trace.WriteLine($"Latest: {latestBlob.Uri}");
}
return result;
}
public Uri GetLatestUri(GetLatestRequest request)
{
var context = new CloudContext(request.ConnectionString, request.Container);
var latestPath = _pathBuilder.GetLatest(request.PathFormat);
var latestBlob = context.BlobContainer.GetBlockBlobReference(latestPath);
return latestBlob.Uri;
}
public async Task<UriResult> GetLatestUriResultAsync(GetLatestRequest request)
{
var context = new CloudContext(request.ConnectionString, request.Container);
var latestPath = _pathBuilder.GetLatest(request.PathFormat);
var latestBlob = context.BlobContainer.GetBlockBlobReference(latestPath);
if (!await latestBlob.ExistsAsync())
{
return null;
}
return new UriResult
{
Uri = latestBlob.Uri,
ETag = latestBlob.Properties.ETag
};
}
private static async Task<CloudBlockBlob> UploadBlobAsync(CloudContext context, UploadRequest request, string blobPath, bool direct)
{
request.Trace.Write($"Uploading the blob at '{blobPath}'...");
var blob = context.BlobContainer.GetBlockBlobReference(blobPath);
AccessCondition accessCondition;
if (!direct && !request.UseETags)
{
accessCondition = null;
}
if (direct || request.ETag == null)
{
accessCondition = AccessCondition.GenerateIfNoneMatchCondition("*");
}
else
{
accessCondition = AccessCondition.GenerateIfMatchCondition(request.ETag);
}
// upload the blob
await blob.UploadFromStreamAsync(request.Stream, accessCondition, options: null, operationContext: null);
request.Trace.WriteLine(" done.");
// set the content type
if (!string.IsNullOrWhiteSpace(request.ContentType))
{
request.Trace.Write($"Setting the content type of '{blobPath}'...");
blob.Properties.ContentType = request.ContentType;
await blob.SetPropertiesAsync();
request.Trace.WriteLine(" done.");
}
return blob;
}
public async Task<StreamResult> GetLatestStreamAsync(GetLatestRequest request)
{
var context = new CloudContext(request.ConnectionString, request.Container);
var latestPath = _pathBuilder.GetLatest(request.PathFormat);
var latestBlob = context.BlobContainer.GetBlockBlobReference(latestPath);
try
{
var stream = await latestBlob.OpenReadAsync();
return new StreamResult
{
Stream = stream,
ETag = latestBlob.Properties.ETag,
ContentMD5 = latestBlob.Properties.ContentMD5
};
}
catch (StorageException e)
{
if (e.RequestInformation.HttpStatusCode == 404)
{
request.Trace.WriteLine($"The stream could not be found due to the following error: {e.RequestInformation.HttpStatusMessage}");
return null;
}
throw;
}
}
private async Task<CloudBlockBlob> UploadDirectNumberAsync(UploadRequest request, CloudContext context, UploadResult result)
{
var latestNumberPath = _pathBuilder.GetDirect(request.PathFormat, 0);
var latestNumberBlob = context.BlobContainer.GetBlockBlobReference(latestNumberPath);
string latestNumberEtag = null;
var latestNumber = 0;
if (request.LatestNumber.HasValue)
{
// Use the provided version number
latestNumberEtag = request.LatestNumberETag;
latestNumber = request.LatestNumber.Value;
}
else
{
// Determine what version number what last used
try
{
await latestNumberBlob.FetchAttributesAsync();
latestNumberEtag = latestNumberBlob.Properties.ETag;
var latestNumberString = latestNumberBlob.Metadata[LatestNumberMetadataKey];
latestNumber = int.Parse(latestNumberString);
}
catch (StorageException e)
{
if (e.RequestInformation.HttpStatusCode != 404)
{
throw;
}
}
latestNumber++;
}
// Upload the stream
var latestPath = _pathBuilder.GetDirect(request.PathFormat, latestNumber);
var directBlob = await UploadBlobAsync(context, request, latestPath, direct: true);
// Update the latest version record
AccessCondition accessCondition;
if (!request.UseETags)
{
accessCondition = null;
}
else if (latestNumberEtag != null)
{
accessCondition = AccessCondition.GenerateIfMatchCondition(latestNumberEtag);
}
else
{
accessCondition = AccessCondition.GenerateIfNoneMatchCondition("*");
}
latestNumberBlob.Metadata[LatestNumberMetadataKey] = latestNumber.ToString();
await latestNumberBlob.UploadFromByteArrayAsync(new byte[0], 0, 0, accessCondition, options: null, operationContext: null);
// Set the latest etag and number on the result
result.LatestNumberETag = latestNumberBlob.Properties.ETag;
result.LatestNumber = latestNumber;
return directBlob;
}
private async Task<CloudBlockBlob> UploadDirectTimestampAsync(UploadRequest request, CloudContext context)
{
var latestPath = _pathBuilder.GetDirect(request.PathFormat, _systemTime.UtcNow);
var directBlob = await UploadBlobAsync(context, request, latestPath, direct: true);
return directBlob;
}
public TestContext()
{
// data
UtcNow = new DateTimeOffset(2015, 1, 2, 3, 4, 5, 6, TimeSpan.Zero);
Content = "foobar";
Container = TestSupport.GetTestContainer();
Prefix = "testpath";
UploadRequest = new UploadRequest
{
ConnectionString = TestSupport.ConnectionString,
Container = Container,
ContentType = "text/plain",
PathFormat = Prefix + "/{0}.txt",
UploadDirect = true,
UploadLatest = true,
Stream = new MemoryStream(Encoding.UTF8.GetBytes(Content)),
Trace = TextWriter.Null
};
CollapseRequest = new CollapseRequest
{
ConnectionString = UploadRequest.ConnectionString,
Container = UploadRequest.Container,
PathFormat = UploadRequest.PathFormat,
Comparer = new OrdinalCollapserComparer(),
Trace = TextWriter.Null
};
CloudContext = new CloudContext(UploadRequest.ConnectionString, UploadRequest.Container);
// dependencies
SystemTime = new Mock<ISystemTime>();
PathBuilder = new PathBuilder();
Client = new Client(SystemTime.Object, PathBuilder);
// setup
SystemTime
.Setup(x => x.UtcNow)
.Returns(() => UtcNow).Callback(() => UtcNow = UtcNow.AddSeconds(1));
// target
Target = new Collapser(PathBuilder);
}
public static void DeleteContainer(string container)
{
var context = new CloudContext(ConnectionString, container);
context.BlobContainer.DeleteIfExistsAsync().Wait();
}
public async Task CollapseAsync(CollapseRequest request)
{
_pathBuilder.Validate(request.PathFormat);
var context = new CloudContext(request.ConnectionString, request.Container);
if (!await context.BlobContainer.ExistsAsync())
{
request.Trace.WriteLine($"The container {request.Container} does not exist so no collapsing is necessary.");
return;
}
// determine the prefix
var placeholderIndex = request.PathFormat.IndexOf("{0}", StringComparison.Ordinal);
var prefix = request.PathFormat.Substring(0, placeholderIndex);
var suffix = request.PathFormat.Substring(placeholderIndex + "{0}".Length);
var latestPath = _pathBuilder.GetLatest(request.PathFormat);
// collect and sort all of the blob names
var blobNames = new List<string>();
var token = (BlobContinuationToken) null;
do
{
var segment = await context.BlobContainer.ListBlobsSegmentedAsync(prefix, true, BlobListingDetails.All, null, token, null, null);
token = segment.ContinuationToken;
// filter out packages that don't match the path format and the latest
var segmentBlobNames = segment
.Results
.OfType<ICloudBlob>()
.Select(x => x.Name)
.Where(x => x.EndsWith(suffix) && x != latestPath);
int before = blobNames.Count;
blobNames.AddRange(segmentBlobNames);
int added = blobNames.Count - before;
request.Trace.WriteLine($"Fetched {added} blobs.");
}
while (token != null);
blobNames.Sort(request.Comparer);
request.Trace.WriteLine($"{blobNames.Count} blobs were sorted.");
// collapse the blobs
int indexX = 0;
int indexY = 1;
while(indexX < blobNames.Count - 1 && indexY < blobNames.Count)
{
var nameX = blobNames[indexX];
var nameY = blobNames[indexY];
var blobX = context.BlobContainer.GetBlockBlobReference(nameX);
var blobY = context.BlobContainer.GetBlockBlobReference(nameY);
using (var streamX = await blobX.OpenReadAsync())
using (var streamY = await blobY.OpenReadAsync())
{
if (blobX.Properties.ContentMD5 == blobY.Properties.ContentMD5 ||
await request.Comparer.EqualsAsync(nameX, streamX, nameY, streamY, CancellationToken.None))
{
request.Trace.WriteLine($"Deleting '{nameY}'.");
await blobY.DeleteAsync();
indexY++;
}
else
{
indexX = indexY;
indexY++;
}
}
}
}
