/// <summary>
/// Downloads the next block.
/// </summary>
private async Task DownloadBlock(bool async, CancellationToken cancellationToken)
{
Response <BlobDownloadStreamingResult> response;
HttpRange range = new HttpRange(_offset, _blockSize);
response = async
? await _blobClient.DownloadStreamingAsync(range, cancellationToken : cancellationToken).ConfigureAwait(false)
: _blobClient.DownloadStreaming(range, cancellationToken: cancellationToken);
_stream = response.Value.Content;
_offset += response.Value.Details.ContentLength;
_lastDownloadBytes = response.Value.Details.ContentLength;
_blobLength = GetBlobLength(response);
}
public async Task UploadBlobFromStream()
{
string data = "hello world";
BlobContainerClient containerClient = new BlobContainerClient(ConnectionString, Randomize("sample-container"));
try
{
await containerClient.CreateAsync();
string blobName = Randomize("sample-blob");
string localFilePath = this.CreateTempPath();
FileStream fs = File.OpenWrite(localFilePath);
var bytes = Encoding.UTF8.GetBytes(data);
await fs.WriteAsync(bytes, 0, bytes.Length);
await fs.FlushAsync();
fs.Close();
#region Snippet:SampleSnippetsBlobMigration_UploadBlobFromStream
BlobClient blobClient = containerClient.GetBlobClient(blobName);
using Stream stream = File.OpenRead(localFilePath);
await blobClient.UploadAsync(stream, overwrite : true);
#endregion
Stream downloadStream = (await blobClient.DownloadStreamingAsync()).Value.Content;
string downloadedData = await new StreamReader(downloadStream).ReadToEndAsync();
downloadStream.Close();
Assert.AreEqual(data, downloadedData);
}
finally
{
await containerClient.DeleteIfExistsAsync();
}
}
public async Task ExpectedDownloadStreamingStreamTypeReturned(TransactionalHashAlgorithm algorithm, bool isBuffered)
{
await using var test = await GetDisposingContainerAsync();
// Arrange
var data = GetRandomBuffer(Constants.KB);
BlobClient blob = InstrumentClient(test.Container.GetBlobClient(GetNewResourceName()));
using (var stream = new MemoryStream(data))
{
await blob.UploadAsync(stream);
}
// don't make options instance at all for no hash request
DownloadTransactionalHashingOptions hashingOptions = algorithm == TransactionalHashAlgorithm.None
? default
: new DownloadTransactionalHashingOptions {
Algorithm = algorithm
};
// Act
Response <BlobDownloadStreamingResult> response = await blob.DownloadStreamingAsync(new BlobDownloadOptions
{
TransactionalHashingOptions = hashingOptions,
Range = new HttpRange(length: data.Length)
});
// Assert
if (isBuffered)
{
Assert.AreEqual(typeof(MemoryStream), response.Value.Content.GetType());
}
// actual unbuffered stream type is private; just check we didn't get back a buffered stream
else
{
Assert.AreNotEqual(typeof(MemoryStream), response.Value.Content.GetType());
}
}
#pragma warning disable CA1822 // Does not acces instance data can be marked static.
public virtual async Task <Segment> BuildSegment(
#pragma warning restore CA1822 // Can't mock static methods in MOQ.
bool async,
string manifestPath,
SegmentCursor cursor = default)
{
// Models we need for later
List <Shard> shards = new List <Shard>();
DateTimeOffset dateTime = BlobChangeFeedExtensions.ToDateTimeOffset(manifestPath).Value;
// Download segment manifest
BlobClient blobClient = _containerClient.GetBlobClient(manifestPath);
BlobDownloadStreamingResult blobDownloadStreamingResult;
if (async)
{
blobDownloadStreamingResult = await blobClient.DownloadStreamingAsync().ConfigureAwait(false);
}
else
{
blobDownloadStreamingResult = blobClient.DownloadStreaming();
}
// Parse segment manifest
JsonDocument jsonManifest;
if (async)
{
jsonManifest = await JsonDocument.ParseAsync(blobDownloadStreamingResult.Content).ConfigureAwait(false);
}
else
{
jsonManifest = JsonDocument.Parse(blobDownloadStreamingResult.Content);
}
foreach (JsonElement shardJsonElement in jsonManifest.RootElement.GetProperty("chunkFilePaths").EnumerateArray())
{
string shardPath = shardJsonElement.ToString().Substring("$blobchangefeed/".Length);
var shardCursor = cursor?.ShardCursors?.Find(x => x.CurrentChunkPath.StartsWith(shardPath, StringComparison.InvariantCulture));
Shard shard = await _shardFactory.BuildShard(
async,
shardPath,
shardCursor)
.ConfigureAwait(false);
if (shard.HasNext())
{
shards.Add(shard);
}
}
int shardIndex = 0;
string currentShardPath = cursor?.CurrentShardPath;
if (!string.IsNullOrWhiteSpace(currentShardPath))
{
shardIndex = shards.FindIndex(s => s.ShardPath == currentShardPath);
if (shardIndex < 0)
{
// Either shard doesn't exist or cursor is pointing to end of shard. So start from beginning.
shardIndex = 0;
}
}
return(new Segment(
shards,
shardIndex,
dateTime,
manifestPath));
}
public async Task <ChangeFeed> BuildChangeFeed(
DateTimeOffset?startTime,
DateTimeOffset?endTime,
string continuation,
bool async,
CancellationToken cancellationToken)
{
DateTimeOffset lastConsumable;
Queue <string> years = new Queue <string>();
Queue <string> segments = new Queue <string>();
ChangeFeedCursor cursor = null;
// Create cursor
if (continuation != null)
{
cursor = JsonSerializer.Deserialize <ChangeFeedCursor>(continuation);
ValidateCursor(_containerClient, cursor);
startTime = BlobChangeFeedExtensions.ToDateTimeOffset(cursor.CurrentSegmentCursor.SegmentPath).Value;
endTime = cursor.EndTime;
}
// Round start and end time if we are not using the cursor.
else
{
startTime = startTime.RoundDownToNearestHour();
endTime = endTime.RoundUpToNearestHour();
}
// Check if Change Feed has been abled for this account.
bool changeFeedContainerExists;
if (async)
{
changeFeedContainerExists = await _containerClient.ExistsAsync(cancellationToken : cancellationToken).ConfigureAwait(false);
}
else
{
changeFeedContainerExists = _containerClient.Exists(cancellationToken: cancellationToken);
}
if (!changeFeedContainerExists)
{
throw new ArgumentException("Change Feed hasn't been enabled on this account, or is currently being enabled.");
}
// Get last consumable
BlobClient blobClient = _containerClient.GetBlobClient(Constants.ChangeFeed.MetaSegmentsPath);
BlobDownloadStreamingResult blobDownloadInfo;
try
{
if (async)
{
blobDownloadInfo = await blobClient.DownloadStreamingAsync(cancellationToken : cancellationToken).ConfigureAwait(false);
}
else
{
blobDownloadInfo = blobClient.DownloadStreaming(cancellationToken: cancellationToken);
}
}
catch (RequestFailedException e) when(e.ErrorCode == BlobErrorCode.BlobNotFound)
{
return(ChangeFeed.Empty());
}
JsonDocument jsonMetaSegment;
if (async)
{
jsonMetaSegment = await JsonDocument.ParseAsync(
blobDownloadInfo.Content,
cancellationToken : cancellationToken
).ConfigureAwait(false);
}
else
{
jsonMetaSegment = JsonDocument.Parse(blobDownloadInfo.Content);
}
lastConsumable = jsonMetaSegment.RootElement.GetProperty("lastConsumable").GetDateTimeOffset();
// Get year paths
years = await GetYearPathsInternal(
async,
cancellationToken).ConfigureAwait(false);
// Dequeue any years that occur before start time
if (startTime.HasValue)
{
while (years.Count > 0 &&
BlobChangeFeedExtensions.ToDateTimeOffset(years.Peek()) < startTime.RoundDownToNearestYear())
{
years.Dequeue();
}
}
// There are no years.
if (years.Count == 0)
{
return(ChangeFeed.Empty());
}
while (segments.Count == 0 && years.Count > 0)
{
// Get Segments for year
segments = await BlobChangeFeedExtensions.GetSegmentsInYearInternal(
containerClient : _containerClient,
yearPath : years.Dequeue(),
startTime : startTime,
endTime : BlobChangeFeedExtensions.MinDateTime(lastConsumable, endTime),
async : async,
cancellationToken : cancellationToken)
.ConfigureAwait(false);
}
// We were on the last year, and there were no more segments.
if (segments.Count == 0)
{
return(ChangeFeed.Empty());
}
Segment currentSegment = await _segmentFactory.BuildSegment(
async,
segments.Dequeue(),
cursor?.CurrentSegmentCursor)
.ConfigureAwait(false);
return(new ChangeFeed(
_containerClient,
_segmentFactory,
years,
segments,
currentSegment,
lastConsumable,
startTime,
endTime));
}
public async Task <string> PublishFileAsync(FileMapping fileMap, CancellationToken ct)
{
Uri result = null;
int retriesLeft = MaxFullLoopRetries;
TimeSpan loopDelay = FullLoopRetryDelay;
bool completed = false;
do
{
_logger.LogInformation($"Attempting to publish {fileMap.RelativeOutputPath}, {retriesLeft} tries left.");
try
{
BlobContainerClient client = await GetClient(ct);
if (client == null)
{
// client creation failed, return
return(null);
}
using var srcStream = new FileStream(fileMap.LocalSourcePath, FileMode.Open, FileAccess.Read);
BlobClient blobClient = client.GetBlobClient(GetBlobName(_releaseName, fileMap.RelativeOutputPath));
await blobClient.UploadAsync(srcStream, overwrite : true, ct);
BlobSasBuilder sasBuilder = new BlobSasBuilder()
{
BlobContainerName = client.Name,
BlobName = blobClient.Name,
Identifier = AccessPolicyDownloadId,
Protocol = SasProtocol.Https
};
Uri accessUri = blobClient.GenerateSasUri(sasBuilder);
using BlobDownloadStreamingResult blobStream = (await blobClient.DownloadStreamingAsync(cancellationToken: ct)).Value;
srcStream.Position = 0;
completed = await VerifyFileStreamsMatchAsync(srcStream, blobStream, ct);
result = accessUri;
}
catch (IOException ioEx) when(!(ioEx is PathTooLongException))
{
_logger.LogWarning(ioEx, $"Failed to publish {fileMap.LocalSourcePath}, retries remaining: {retriesLeft}.");
/* Retry IO exceptions */
retriesLeft--;
loopDelay *= 2;
if (retriesLeft > 0)
{
await Task.Delay(loopDelay, ct);
}
}
catch (Exception ex)
{
// Azure errors have their own built-in retry logic, so just abort if we got an AzureResponseException
_logger.LogWarning(ex, $"Failed to publish {fileMap.LocalSourcePath}, unexpected error, aborting.");
return(null);
}
} while (retriesLeft > 0 && !completed);
return(result?.OriginalString);
}
