private async Task DeleteFile(string fileName)
{
var blockBlob = GetBlockBlobForFileName(fileName);
_log.LogInformation($"Sending delete request for blob file: [{fileName}]");
await blockBlob.DeleteAsync(DeleteSnapshotsOption.IncludeSnapshots, AccessCondition.GenerateEmptyCondition(), _blobRequestOptions, new OperationContext());
}
private void BeginUploadStream(
BlobTransferContext transferContext,
KeyValuePair <long, int> startAndLength,
MemoryStream memoryStream,
byte[] streamBuffer)
{
if (transferContext.CancellationToken.IsCancellationRequested)
{
return;
}
memoryStream.Seek(0, SeekOrigin.Begin);
OperationContext operationContext = new OperationContext();
operationContext.ClientRequestID = transferContext.ClientRequestId;
Interlocked.Increment(ref transferContext.NumInProgressUploadDownloads);
string blockId =
Convert.ToBase64String(Encoding.ASCII.GetBytes(string.Format(CultureInfo.InvariantCulture, "BlockId{0:d7}", (startAndLength.Key / transferContext.BlockSize))));
memoryStream.SetLength(startAndLength.Value);
transferContext.Blob.BeginPutBlock(
blockId,
memoryStream,
null,
AccessCondition.GenerateEmptyCondition(),
transferContext.BlobRequestOptions,
operationContext,
ar =>
{
SuccessfulOrRetryableResult wasWriteSuccessful = EndPutBlock(transferContext, ar);
Interlocked.Decrement(ref transferContext.NumInProgressUploadDownloads);
if (wasWriteSuccessful.IsRetryable)
{
BeginUploadStream(transferContext, startAndLength, memoryStream, streamBuffer);
return;
}
transferContext.MemoryManager.ReleaseBuffer(streamBuffer);
if (!wasWriteSuccessful.IsSuccessful)
{
return;
}
Interlocked.Add(ref transferContext.BytesBlobIOCompleted, startAndLength.Value);
InvokeProgressCallback(transferContext, transferContext.BytesBlobIOCompleted, startAndLength.Value);
if (transferContext.BytesBlobIOCompleted >= transferContext.Length)
{
BeginPutBlockList(transferContext);
}
},
state: null);
}
public void GenerateEmptyConditionReturnsEmptyAccessCondition()
{
AccessCondition result = AccessCondition.GenerateEmptyCondition();
Assert.Null(result.IfMatch);
Assert.Null(result.IfNoneMatch);
}
private async Task <T> LoadAuxiliaryFileAsync <T>(
string blobName,
Func <JsonReader, T> loadData) where T : class
{
_logger.LogInformation(
"Attempted to load blob {BlobName} as {TypeName}.",
blobName,
typeof(T).FullName);
var stopwatch = Stopwatch.StartNew();
var blob = Container.GetBlobReference(blobName);
using (var stream = await blob.OpenReadAsync(AccessCondition.GenerateEmptyCondition()))
using (var textReader = new StreamReader(stream))
using (var jsonReader = new JsonTextReader(textReader))
{
var data = loadData(jsonReader);
stopwatch.Stop();
_telemetryService.TrackAuxiliaryFileDownloaded(blobName, stopwatch.Elapsed);
_logger.LogInformation(
"Loaded blob {BlobName}. Took {Duration}.",
blobName,
stopwatch.Elapsed);
return(data);
};
}
public async Task <string> ReadAllTextAsync()
{
using (var reader = new StreamReader(await cloudBlob.OpenReadAsync(AccessCondition.GenerateEmptyCondition(), new BlobRequestOptions(), new OperationContext())))
{
return(await reader.ReadToEndAsync());
}
}
private async Task UploadAuditMetadata(string folder, AuditMetadata metadata, CancellationToken cancellation)
{
var metadataPath = $"{folder}/meta";
Logger.Information("Uploading metadata to {MetadataPath}", metadataPath);
try
{
var metadataUrl = new Uri($"{this.config.AzureBlobBaseUrl}/{metadataPath}?{this.config.AzureBlobSasToken}");
var auditMetaBlob = new CloudBlockBlob(metadataUrl);
var stringMetadata = JsonConvert.SerializeObject(metadata);
await auditMetaBlob.UploadTextAsync(
stringMetadata,
Encoding.UTF8,
AccessCondition.GenerateEmptyCondition(),
new BlobRequestOptions
{
RetryPolicy = new ExponentialRetry(TimeSpan.FromSeconds(10), 3),
},
new OperationContext(),
cancellation);
}
catch (Exception ex)
{
Logger.Warning(ex, "Audit metadata upload failed");
}
}
/// <summary>
/// Loads the metadata stored with the given ID
/// </summary>
public async Task <Dictionary <string, string> > ReadMetadata(string id)
{
var blobName = GetBlobName(id);
try
{
var container = _client.GetContainerReference(_containerName);
var blob = await container.GetBlobReferenceFromServerAsync(
blobName : blobName,
accessCondition : AccessCondition.GenerateEmptyCondition(),
options : new BlobRequestOptions {
RetryPolicy = new ExponentialRetry()
},
operationContext : new OperationContext()
);
var metadata = new Dictionary <string, string>(blob.Metadata)
{
[MetadataKeys.Length] = blob.Properties.Length.ToString()
};
return(metadata);
}
catch (StorageException exception) when(exception.IsStatus(HttpStatusCode.NotFound))
{
throw new ArgumentException($"Could not find blob named '{blobName}' in the '{_containerName}' container", exception);
}
}
private async Task WriteToAppendBlobAsync(IList <AzureAppendLogDto> logList)
{
foreach (var containerGroup in logList.GroupBy(s => s.ContainerRef.Name))
{
var containerFirst = containerGroup.First();
await containerFirst.ContainerRef.CreateIfNotExistsAsync();
foreach (var blobGroup in containerGroup.GroupBy(s => s.BlobRef.Name))
{
var sb = new StringBuilder();
var blobFirst = blobGroup.First();
if (!await blobFirst.BlobRef.ExistsAsync())
{
blobFirst.BlobRef.Properties.ContentType = "text/plain";
await blobFirst.BlobRef.CreateOrReplaceAsync(AccessCondition.GenerateEmptyCondition(), null,
null);
await blobFirst.BlobRef.SetPropertiesAsync();
}
foreach (var azureLogDto in blobGroup)
{
sb.AppendLine(azureLogDto.LogEvent.FormattedMessage);
}
using (var ms = new MemoryStream(Encoding.UTF8.GetBytes(sb.ToString())))
{
await blobFirst.BlobRef.AppendBlockAsync(ms);
}
}
}
}
public static async Task <HttpResponseMessage> Run([HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = null)] HttpRequestMessage req,
TraceWriter log)
{
Stream photoStream;
try
{
string imagePath = req.GetQueryStrings()["path"];
var photoBlob = await Utils.BlobClient.GetBlobReferenceFromServerAsync(new Uri(imagePath, UriKind.Absolute));
photoStream = await photoBlob.OpenReadAsync(AccessCondition.GenerateEmptyCondition(),
new BlobRequestOptions(), new OperationContext());
}
catch (Exception e)
{
Console.WriteLine(e.Message);
return(new HttpResponseMessage(HttpStatusCode.NotFound));
}
var result = new HttpResponseMessage(HttpStatusCode.OK);
result.Content = new StreamContent(photoStream);
result.Content.Headers.ContentType = new MediaTypeHeaderValue("image/jpeg");
return(result);
}
protected override async Task SaveAsync(SnapshotMetadata metadata, object snapshot)
{
var blob = Container.GetBlockBlobReference(metadata.ToSnapshotBlobId());
var snapshotData = _serialization.SnapshotToBytes(new Serialization.Snapshot(snapshot));
using (var cts = new CancellationTokenSource(_settings.RequestTimeout))
{
blob.Metadata.Add(TimeStampMetaDataKey, metadata.Timestamp.Ticks.ToString());
/*
* N.B. No need to convert the key into the Journal format we use here.
* The blobs themselves don't have their sort order affected by
* the presence of this metadata, so we should just save the SeqNo
* in a format that can be easily deserialized later.
*/
blob.Metadata.Add(SeqNoMetaDataKey, metadata.SequenceNr.ToString());
await blob.UploadFromByteArrayAsync(
snapshotData,
0,
snapshotData.Length,
AccessCondition.GenerateEmptyCondition(),
GenerateOptions(),
new OperationContext(),
cts.Token);
}
}
private async Task UploadFile(BackupFileUploadJob job)
{
var g = Guid.NewGuid().ToString().Substring(0, 8);
try
{
_logger.LogDebug($"{{{g}}} Uploading {job.UploadPath} on thread {Thread.CurrentThread.ManagedThreadId}");
var account = GetStorageAccount(job);
var client = account.CreateCloudBlobClient();
var container = client.GetContainerReference("blobblaze");
await container.CreateIfNotExistsAsync();
var blob = container.GetBlockBlobReference(job.UploadPath);
_logger.LogDebug($"{{{g}}} Using Cloud Storage Account {account.BlobStorageUri.PrimaryUri}");
_logger.LogDebug($"{{{g}}} Uploading to {container.Name}/{job.UploadPath}");
_logger.LogInformation(
$"{{{g}}} Beginning Upload of {job.UploadPath}. File Size is {ByteSize.FromBytes(job.LocalFile.Length).ToString()}");
var startTime = DateTime.UtcNow;
using (var fs = File.OpenRead(job.LocalFile.FullName))
{
await blob.UploadFromStreamAsync(fs, AccessCondition.GenerateEmptyCondition(), null, null, _cancellationTokenSource.Token);
}
job.Status = BackupFileUploadJobStatus.Successful;
var secondsTime = (DateTime.UtcNow - startTime).TotalSeconds;
_logger.LogInformation(
$"{{{g}}} Finished uploading {job.UploadPath} in {secondsTime:0.##}s. Average Speed was {ByteSize.FromBytes(job.LocalFile.Length / secondsTime).ToString()}/s.");
await TrackFile(job);
job.ParentJob.IncrementComplete();
await SetArchiveStatus(job, container);
// set to archive
}
catch (Exception e)
{
_logger.LogError($"{{{g}}} Exception occurred while uploading file {e.GetType().Name}: {e.Message} {e.StackTrace}");
await LogError(job, e);
if (job.RetryCount < MaxNumRetries)
{
_logger.LogInformation($"{{{g}}} Retrying job... {job.RetryCount} of {MaxNumRetries}");
_jobQueue.Add(job);
job.RetryCount++;
}
else
{
job.ParentJob.IncrementErrored();
}
}
finally
{
_uploadTaskSemaphore.Release();
}
}
private async Task <string> UploadZipToStorage(Stream zip, IDictionary <string, string> appSettings)
{
const string containerName = "function-releases";
const string blobNameFormat = "{0}-{1}.zip";
var storageConnection = appSettings["AzureWebJobsStorage"];
var storageAccount = CloudStorageAccount.Parse(storageConnection);
var blobClient = storageAccount.CreateCloudBlobClient();
var blobContainer = blobClient.GetContainerReference(containerName);
await blobContainer.CreateIfNotExistsAsync();
var releaseName = Guid.NewGuid().ToString();
var blob = blobContainer.GetBlockBlobReference(string.Format(blobNameFormat, DateTimeOffset.UtcNow.ToString("yyyyMMddHHmmss"), releaseName));
using (var progress = new StorageProgressBar($"Uploading {Utilities.BytesToHumanReadable(zip.Length)}", zip.Length))
{
await blob.UploadFromStreamAsync(zip,
AccessCondition.GenerateEmptyCondition(),
new BlobRequestOptions(),
new OperationContext(),
progress,
new CancellationToken());
}
var sasConstraints = new SharedAccessBlobPolicy();
sasConstraints.SharedAccessStartTime = DateTimeOffset.UtcNow.AddMinutes(-5);
sasConstraints.SharedAccessExpiryTime = DateTimeOffset.UtcNow.AddYears(10);
sasConstraints.Permissions = SharedAccessBlobPermissions.Read;
var blobToken = blob.GetSharedAccessSignature(sasConstraints);
return(blob.Uri + blobToken);
}
/// <summary>
/// Uploads a file.
/// </summary>
/// <param name="file">File to upload.</param>
/// <param name="preferedFilename">Prefered filename.</param>
/// <returns>Final filename.</returns>
public static string Upload(string containerName, HttpPostedFileBase file, string preferedFilename)
{
var container = GetContainer(containerName);
var extension = System.IO.Path.GetExtension(preferedFilename).ToLower();
// Concats GUID to filename to avoid name collision.
var filename = string.Format("{0}_{1}{2}",
System.IO.Path.GetFileNameWithoutExtension(preferedFilename),
Guid.NewGuid(),
System.IO.Path.GetExtension(preferedFilename));
var blob = container.GetBlockBlobReference(filename);
// ContentType from Extension
switch (extension)
{
case ".png":
blob.Properties.ContentType = "image/png";
break;
case ".jpg":
case ".jpeg":
blob.Properties.ContentType = "image/jpeg";
break;
case ".gif":
blob.Properties.ContentType = "image/gif";
break;
}
// Uploads (Fails if GUIDs collide. Almost no chance.)
blob.UploadFromStream(file.InputStream, AccessCondition.GenerateEmptyCondition());
return(filename);
}
public async Task <Tuple <AzureBenchmarkResult[], string> > GetAzureExperimentResults(ExperimentID experimentId)
{
AzureBenchmarkResult[] results;
string blobName = GetResultBlobName(experimentId);
var blob = resultsContainer.GetBlobReference(blobName);
try
{
using (MemoryStream zipStream = new MemoryStream(4 << 20))
{
await blob.DownloadToStreamAsync(zipStream,
AccessCondition.GenerateEmptyCondition(),
new Microsoft.WindowsAzure.Storage.Blob.BlobRequestOptions
{
RetryPolicy = new Microsoft.WindowsAzure.Storage.RetryPolicies.ExponentialRetry(TimeSpan.FromMilliseconds(100), 10)
}, null);
zipStream.Position = 0;
using (ZipArchive zip = new ZipArchive(zipStream, ZipArchiveMode.Read))
{
var entry = zip.GetEntry(GetResultsFileName(experimentId));
using (var tableStream = entry.Open())
{
results = AzureBenchmarkResult.LoadBenchmarks(experimentId, tableStream);
return(Tuple.Create(results, blob.Properties.ETag));
}
}
}
}
catch (StorageException ex) when(ex.RequestInformation.HttpStatusCode == 404) // Not found == no results
{
return(Tuple.Create(new AzureBenchmarkResult[0], (string)null));
}
}
private async Task <bool> SetupBlockCurrentBlobAsync(string blobNamePrefix, int sequenceId, CancellationToken cancellationToken)
{
var blockBlob = GetBlob(blobNamePrefix, sequenceId);
var blockIds = new List <string>();
var sizeRemaining = _rollSizeBytes;
string blobEtag = null;
bool isNewBlob = true;
if (await blockBlob.ExistsAsync(cancellationToken).ConfigureAwait(false))
{
var items =
await
blockBlob.DownloadBlockListAsync(
BlockListingFilter.Committed,
AccessCondition.GenerateEmptyCondition(),
new BlobRequestOptions(),
null,
cancellationToken);
blockIds.AddRange(items.Select(b => b.Name));
sizeRemaining -= blockBlob.Properties.Length;
blobEtag = blockBlob.Properties.ETag;
isNewBlob = false;
}
InitializeCurrentWriterState(blobNamePrefix, sequenceId, blockBlob, blockIds, sizeRemaining, blobEtag);
return(isNewBlob);
}
private void AssertAccessCondition(Uri resourceUri, AccessCondition accessCondition)
{
Content.TryGetValue(resourceUri, out var existingContent);
if (IsAccessCondition(AccessCondition.GenerateEmptyCondition(), accessCondition))
{
return;
}
if (IsAccessCondition(AccessCondition.GenerateIfNotExistsCondition(), accessCondition))
{
Assert.Null(existingContent);
return;
}
if (IsAccessCondition(AccessCondition.GenerateIfExistsCondition(), accessCondition))
{
Assert.NotNull(existingContent);
return;
}
if (existingContent is StringStorageContentWithETag eTagContent)
{
var eTag = eTagContent.ETag;
if (IsAccessCondition(AccessCondition.GenerateIfMatchCondition(eTag), accessCondition))
{
return;
}
}
throw new InvalidOperationException("Could not validate access condition!");
}
public static async Task <FaceOnPhoto> Run([ActivityTrigger] PhotoUriWithPersonGroup photoUriWithPersonGroup)
{
var photoBlob =
await SharedResources.BlobClient.GetBlobReferenceFromServerAsync(new Uri(photoUriWithPersonGroup.PhotoUri));
var photoStream = await photoBlob.OpenReadAsync(AccessCondition.GenerateEmptyCondition(),
new BlobRequestOptions(), new OperationContext());
var faces = await SharedResources.FaceServiceClient.DetectAsync(photoStream);
if (!faces.Any())
{
return new FaceOnPhoto()
{
IsFaceIdentifyOnPhoto = false, PhotoUri = photoUriWithPersonGroup.PhotoUri
}
}
;
var facesIds = faces.Select(e => e.FaceId);
IdentifyResult[] identifyResults = await SharedResources.FaceServiceClient.IdentifyAsync(facesIds.ToArray(),
photoUriWithPersonGroup.PersonGroupId);
FaceOnPhoto result = new FaceOnPhoto()
{
IsFaceIdentifyOnPhoto = identifyResults.Any(e => e.Candidates.Any()),
PhotoUri = photoUriWithPersonGroup.PhotoUri
};
return(result);
}
}
public async Task <IReadOnlyDictionary <string, string> > GetExecutableMetadata(string name)
{
var blob = binContainer.GetBlobReference(name);
await blob.FetchAttributesAsync(AccessCondition.GenerateEmptyCondition(), new BlobRequestOptions { RetryPolicy = retryPolicy }, null);
return(new System.Collections.ObjectModel.ReadOnlyDictionary <string, string>(blob.Metadata));
}
public virtual Uri UploadFile(
string storageName,
Uri blobEndpointUri,
string storageKey,
string filePath,
BlobRequestOptions blobRequestOptions)
{
StorageCredentials credentials = new StorageCredentials(storageName, storageKey);
CloudBlobClient client = new CloudBlobClient(blobEndpointUri, credentials);
string blobName = string.Format(
CultureInfo.InvariantCulture,
"{0}_{1}",
DateTime.UtcNow.ToString("yyyyMMdd_HHmmss", CultureInfo.InvariantCulture),
Path.GetFileName(filePath));
CloudBlobContainer container = client.GetContainerReference(ContainerName);
container.CreateIfNotExists();
CloudBlockBlob blob = container.GetBlockBlobReference(blobName);
BlobRequestOptions uploadRequestOption = blobRequestOptions ?? new BlobRequestOptions();
if (!uploadRequestOption.ServerTimeout.HasValue)
{
uploadRequestOption.ServerTimeout = TimeSpan.FromMinutes(30);
}
using (FileStream readStream = File.OpenRead(filePath))
{
blob.UploadFromStream(readStream, AccessCondition.GenerateEmptyCondition(), uploadRequestOption);
}
return(new Uri(string.Format(CultureInfo.InvariantCulture, "{0}{1}{2}{3}", client.BaseUri, ContainerName, client.DefaultDelimiter, blobName)));
}
/// <summary>
/// If the blob was successfully uploaded, returns true.
/// If the blob already exists, returns false.
/// Otherwise throws an exception.
/// </summary>
private async Task <bool> TryUploadNewExecutableAsBlob(Stream source, string blobName, string creator, string originalFileName)
{
if (source == null)
{
throw new ArgumentNullException("source");
}
if (blobName == null)
{
throw new ArgumentNullException("blobName");
}
CloudBlockBlob blob = binContainer.GetBlockBlobReference(blobName);
try
{
await blob.UploadFromStreamAsync(source, AccessCondition.GenerateIfNotExistsCondition(),
new BlobRequestOptions()
{
RetryPolicy = retryPolicy
}, null);
blob.Metadata.Add(KeyCreator, StripNonAscii(creator));
blob.Metadata.Add(KeyFileName, StripNonAscii(originalFileName));
await blob.SetMetadataAsync(AccessCondition.GenerateEmptyCondition(), new BlobRequestOptions { RetryPolicy = retryPolicy }, null);
return(true);
}
catch (StorageException ex) when(ex.RequestInformation?.HttpStatusCode == (int)HttpStatusCode.Conflict)
{
return(false);
}
}
/// <summary>
/// Increment the sequence number of the event stream
/// </summary>
private async Task IncrementSequence(string writeStreamLeaseId = "")
{
if (null != EventStreamBlob)
{
bool exists = await EventStreamBlob.ExistsAsync();
if (exists)
{
await EventStreamBlob.FetchAttributesAsync();
int sequenceNumber;
if (int.TryParse(EventStreamBlob.Metadata[METADATA_SEQUENCE], out sequenceNumber))
{
sequenceNumber += 1;
EventStreamBlob.Metadata[METADATA_SEQUENCE] = $"{sequenceNumber }";
// and commit it back
AccessCondition condition = AccessCondition.GenerateEmptyCondition();
if (!string.IsNullOrWhiteSpace(writeStreamLeaseId))
{
condition.LeaseId = writeStreamLeaseId;
}
await EventStreamBlob.SetMetadataAsync(condition, null, new Microsoft.Azure.Storage.OperationContext());
}
}
}
}
public async Task WhenNewRequestFindsUpdatedBlob_ThenRequestReturnsFalse()
{
var payloadStringA = new string('a', 50);
var payloadStringB = new string('b', 50);
var payloadStringC = new string('c', 50);
var payloadStringD = new string('d', 50);
Assert.True(
await _sut.WriteAsync(
new[]
{
TestUtilities.CreateBlockData(payloadStringA, BlockSize),
TestUtilities.CreateBlockData(payloadStringB, BlockSize)
},
CancellationToken.None));
await AzureStorageHelpers.TouchBlobAsync(GetContainer().GetBlockBlobReference(Prefix + "0"));
Assert.False(
await _sut.WriteAsync(
new[]
{
TestUtilities.CreateBlockData(payloadStringC, BlockSize),
TestUtilities.CreateBlockData(payloadStringD, BlockSize)
},
CancellationToken.None));
Assert.Equal(
payloadStringA + payloadStringB,
await GetContainer()
.GetBlockBlobReference(Prefix + "0")
.DownloadTextAsync(Encoding.UTF8, AccessCondition.GenerateEmptyCondition(), null, null));
}
private static HttpWebRequest BlobGetRequest(KeyValuePair <long, int> blockOffsetAndLength, CloudBlockBlob blob)
{
StorageCredentials credentials = blob.ServiceClient.Credentials;
var transformedUri = credentials.TransformUri(blob.Uri);
// Prepare the HttpWebRequest to download data from the chunk.
HttpWebRequest blobGetRequest = BlobHttpWebRequestFactory.Get(
transformedUri,
Timeout,
snapshot: null,
offset: blockOffsetAndLength.Key,
count: blockOffsetAndLength.Value,
rangeContentMD5: false,
accessCondition: AccessCondition.GenerateEmptyCondition(),
operationContext: new OperationContext());
if (credentials.IsSharedKey)
{
IAuthenticationHandler authenticationHandler = new SharedKeyAuthenticationHandler(
SharedKeyCanonicalizer.Instance,
credentials,
credentials.AccountName);
authenticationHandler.SignRequest(blobGetRequest, new OperationContext());
}
return(blobGetRequest);
}
public async Task WhenIssuingMultipleWriteRequests_ThenBlockAreWrittenInOrder()
{
var payloadStringA = new string('a', 50);
var payloadStringB = new string('b', 50);
var payloadStringC = new string('c', 50);
var payloadStringD = new string('d', 50);
Assert.True(
await _sut.WriteAsync(
new[]
{
TestUtilities.CreateBlockData(payloadStringA, BlockSize),
TestUtilities.CreateBlockData(payloadStringB, BlockSize)
},
CancellationToken.None));
Assert.True(
await _sut.WriteAsync(
new[]
{
TestUtilities.CreateBlockData(payloadStringC, BlockSize),
TestUtilities.CreateBlockData(payloadStringD, BlockSize)
},
CancellationToken.None));
Assert.Equal(
payloadStringA + payloadStringB + payloadStringC + payloadStringD,
await GetContainer()
.GetBlockBlobReference(Prefix + "0")
.DownloadTextAsync(Encoding.UTF8, AccessCondition.GenerateEmptyCondition(), null, null));
}
public async Task <ResultAndAccessCondition <VersionListData> > ReadAsync(string id)
{
var blobReference = Container.GetBlobReference(GetFileName(id));
_logger.LogInformation("Reading the version list for package ID {PackageId}.", id);
VersionListData data;
IAccessCondition accessCondition;
try
{
using (var stream = await blobReference.OpenReadAsync(AccessCondition.GenerateEmptyCondition()))
using (var streamReader = new StreamReader(stream))
using (var jsonTextReader = new JsonTextReader(streamReader))
{
data = Serializer.Deserialize <VersionListData>(jsonTextReader);
}
accessCondition = AccessConditionWrapper.GenerateIfMatchCondition(blobReference.ETag);
}
catch (StorageException ex) when(ex.RequestInformation.HttpStatusCode == (int)HttpStatusCode.NotFound)
{
data = new VersionListData(new Dictionary <string, VersionPropertiesData>());
accessCondition = AccessConditionWrapper.GenerateIfNotExistsCondition();
}
return(new ResultAndAccessCondition <VersionListData>(data, accessCondition));
}
public async Task <CloudBlobStream> OpenAppendBlobWriteStream(CloudAppendBlob blob)
{
BlobRequestOptions requestOptions = new BlobRequestOptions();
OperationContext ctx = new OperationContext();
return(await blob.OpenWriteAsync(true, AccessCondition.GenerateEmptyCondition(), requestOptions, ctx, Engine.CT));
}
public async Task RunWithLeaseAsync(Func <ILeasedExecutedMigrationsStorage, Task> action, CancellationToken cancellationToken = default)
{
await CreateIfNotExistsAsync(cancellationToken);
string leaseId = null;
try
{
leaseId = await blob.AcquireLeaseAsync(
null,
null,
AccessCondition.GenerateEmptyCondition(),
new BlobRequestOptions(),
new OperationContext(),
cancellationToken);
}
catch (StorageException e)
when(e.RequestInformation.HttpStatusCode == (int)HttpStatusCode.Conflict)
{
throw new InvalidOperationException($"The lease for blob {blob.Uri.AbsolutePath} is already leased.");
}
try
{
var leasedStorage = new AzureBlobLeasedExecutedMigrationsStorage(blob, leaseId);
await action(leasedStorage);
}
finally
{
await blob.ReleaseLeaseAsync(AccessCondition.GenerateLeaseCondition(leaseId));
}
}
public async Task <ResultAndAccessCondition <SortedDictionary <string, SortedSet <string> > > > ReadLatestIndexedAsync()
{
var stopwatch = Stopwatch.StartNew();
var blobName = GetLatestIndexedBlobName();
var blobReference = Container.GetBlobReference(blobName);
_logger.LogInformation("Reading the latest indexed owners from {BlobName}.", blobName);
var builder = new PackageIdToOwnersBuilder(_logger);
IAccessCondition accessCondition;
try
{
using (var stream = await blobReference.OpenReadAsync(AccessCondition.GenerateEmptyCondition()))
{
accessCondition = AccessConditionWrapper.GenerateIfMatchCondition(blobReference.ETag);
ReadStream(stream, builder.Add);
}
}
catch (StorageException ex) when(ex.RequestInformation.HttpStatusCode == (int)HttpStatusCode.NotFound)
{
accessCondition = AccessConditionWrapper.GenerateIfNotExistsCondition();
_logger.LogInformation("The blob {BlobName} does not exist.", blobName);
}
var output = new ResultAndAccessCondition <SortedDictionary <string, SortedSet <string> > >(
builder.GetResult(),
accessCondition);
stopwatch.Stop();
_telemetryService.TrackReadLatestIndexedOwners(output.Result.Count, stopwatch.Elapsed);
return(output);
}
/// <summary>
/// Opens the data stored under the given ID for reading
/// </summary>
public async Task <Stream> Read(string id)
{
var blobName = GetBlobName(id);
try
{
var container = _client.GetContainerReference(_containerName);
var blob = await container.GetBlobReferenceFromServerAsync(blobName);
await UpdateLastReadTime(blob);
return(AsyncHelpers.GetResult(() =>
{
var accessCondition = AccessCondition.GenerateEmptyCondition();
var requestOptions = new BlobRequestOptions {
RetryPolicy = new ExponentialRetry()
};
var operationContext = new OperationContext();
return blob.OpenReadAsync(accessCondition, requestOptions, operationContext);
}));
}
catch (StorageException exception) when(exception.IsStatus(HttpStatusCode.NotFound))
{
throw new ArgumentException($"Could not find blob named '{blobName}' in the '{_containerName}' container", exception);
}
}
/// <summary>
/// Opens the data stored under the given ID for reading
/// </summary>
public async Task <Stream> Read(string id)
{
var blobName = GetBlobName(id);
try
{
var blob = await _container.GetBlobReferenceFromServerAsync(
blobName : blobName,
accessCondition : AccessCondition.GenerateEmptyCondition(),
options : new BlobRequestOptions {
RetryPolicy = new ExponentialRetry()
},
operationContext : new OperationContext()
);
if (_options.UpdateLastReadTime)
{
await UpdateLastReadTime(blob);
}
return(await blob.OpenReadAsync(
accessCondition : AccessCondition.GenerateEmptyCondition(),
options : new BlobRequestOptions {
RetryPolicy = new ExponentialRetry()
},
operationContext : new OperationContext()
));
}
catch (StorageException exception) when(exception.IsStatus(HttpStatusCode.NotFound))
{
throw new ArgumentException(
$"Could not find blob named '{blobName}' in the '{_containerName}' container", exception);
}
}
