public BlobReadOnlyContentSession(
IAbsFileSystem fileSystem,
string name,
ImplicitPin implicitPin,
IBlobStoreHttpClient blobStoreHttpClient,
TimeSpan timeToKeepContent,
bool downloadBlobsThroughBlobStore,
CounterTracker?counterTracker = null)
: base(name, counterTracker)
{
Contract.Requires(fileSystem != null);
Contract.Requires(name != null);
Contract.Requires(blobStoreHttpClient != null);
ImplicitPin = implicitPin;
BlobStoreHttpClient = blobStoreHttpClient;
TimeToKeepContent = timeToKeepContent;
_downloadBlobsThroughBlobStore = downloadBlobsThroughBlobStore;
_parallelSegmentDownloadConfig = ParallelHttpDownload.Configuration.GetParallelSegmentDownloadConfig(EnvironmentVariablePrefix);
TempDirectory = new DisposableDirectory(fileSystem);
_counters = CounterTracker.CreateCounterCollection <BackingContentStore.SessionCounters>(counterTracker);
_blobCounters = CounterTracker.CreateCounterCollection <Counters>(counterTracker);
}
/// <summary>
/// Initializes a new instance of the <see cref="BlobContentSession" /> class.
/// </summary>
/// <param name="configuration">Configuration.</param>
/// <param name="name">Session name.</param>
/// <param name="implicitPin">Policy determining whether or not content should be automatically pinned on adds or gets.</param>
/// <param name="blobStoreHttpClient">Backing BlobStore http client.</param>
/// <param name="counterTracker">Parent counters to track the session.</param>
public BlobContentSession(
BackingContentStoreConfiguration configuration,
string name,
ImplicitPin implicitPin,
IBlobStoreHttpClient blobStoreHttpClient,
CounterTracker counterTracker)
: base(configuration, name, implicitPin, blobStoreHttpClient, counterTracker)
{
}
/// <summary>
/// Initializes a new instance of the <see cref="BlobContentSession" /> class.
/// </summary>
/// <param name="fileSystem">Filesystem used to read/write files.</param>
/// <param name="name">Session name.</param>
/// <param name="implicitPin">Policy determining whether or not content should be automatically pinned on adds or gets.</param>
/// <param name="blobStoreHttpClient">Backing BlobStore http client.</param>
/// <param name="timeToKeepContent">Minimum time-to-live for accessed content.</param>
/// <param name="downloadBlobsThroughBlobStore">
/// If true, gets blobs through BlobStore. If false, gets blobs from the Azure
/// Uri.
/// </param>
public BlobContentSession(
IAbsFileSystem fileSystem,
string name,
ImplicitPin implicitPin,
IBlobStoreHttpClient blobStoreHttpClient,
TimeSpan timeToKeepContent,
bool downloadBlobsThroughBlobStore)
: base(fileSystem, name, implicitPin, blobStoreHttpClient, timeToKeepContent, downloadBlobsThroughBlobStore)
{
}
/// <summary>
/// Initializes a new instance of the <see cref="BlobReadOnlyContentSession"/> class.
/// </summary>
/// <param name="fileSystem">Filesystem used to read/write files.</param>
/// <param name="name">Session name.</param>
/// <param name="implicitPin">Policy determining whether or not content should be automatically pinned on adds or gets.</param>
/// <param name="blobStoreHttpClient">Backing BlobStore http client.</param>
/// <param name="timeToKeepContent">Minimum time-to-live for accessed content.</param>
/// <param name="downloadBlobsThroughBlobStore">If true, gets blobs through BlobStore. If false, gets blobs from the Azure Uri.</param>
/// <param name="counterTracker">Parent counters to track the session.</param>
public BlobReadOnlyContentSession(
IAbsFileSystem fileSystem,
string name,
ImplicitPin implicitPin,
IBlobStoreHttpClient blobStoreHttpClient,
TimeSpan timeToKeepContent,
bool downloadBlobsThroughBlobStore,
CounterTracker counterTracker = null)
: base(name, counterTracker)
{
Contract.Requires(fileSystem != null);
Contract.Requires(name != null);
Contract.Requires(blobStoreHttpClient != null);
ImplicitPin = implicitPin;
BlobStoreHttpClient = blobStoreHttpClient;
TimeToKeepContent = timeToKeepContent;
_downloadBlobsThroughBlobStore = downloadBlobsThroughBlobStore;
_parallelSegmentDownloadConfig = new ParallelHttpDownload
.Configuration(
segmentDownloadTimeout: TimeSpan.FromMinutes(int.Parse(
Environment.GetEnvironmentVariable(EnvironmentVariablePrefix + "SegmentDownloadTimeoutInMinutes") ??
DefaultSegmentDownloadTimeoutInMinutes.ToString())),
segmentSizeInBytes: int.Parse(
Environment.GetEnvironmentVariable(EnvironmentVariablePrefix + "ParallelDownloadSegmentSizeInBytes") ??
DefaultParallelDownloadSegmentSizeInBytes.ToString()),
maxParallelSegmentDownloadsPerFile: int.Parse(
Environment.GetEnvironmentVariable(EnvironmentVariablePrefix + "MaxParallelSegmentDownloadsPerFile") ??
DefaultMaxParallelSegmentDownloadsPerFile.ToString()),
maxSegmentDownloadRetries:
int.Parse(
Environment.GetEnvironmentVariable(EnvironmentVariablePrefix + "MaxSegmentDownloadRetries") ??
DefaultMaxSegmentDownloadRetries.ToString()));
TempDirectory = new DisposableDirectory(fileSystem);
Native.IO.FileUtilities.CreateDirectory(TempDirectory.Path.Path);
_counters = CounterTracker.CreateCounterCollection <BackingContentStore.SessionCounters>(counterTracker);
_blobCounters = CounterTracker.CreateCounterCollection <Counters>(counterTracker);
}
public BlobReadOnlyContentSession(
BackingContentStoreConfiguration configuration,
string name,
ImplicitPin implicitPin,
IBlobStoreHttpClient blobStoreHttpClient,
CounterTracker?counterTracker = null)
: base(name, counterTracker)
{
Contract.Requires(configuration != null);
Contract.Requires(configuration.FileSystem != null);
Contract.Requires(name != null);
Contract.Requires(blobStoreHttpClient != null);
Configuration = configuration;
ImplicitPin = implicitPin;
BlobStoreHttpClient = blobStoreHttpClient;
_parallelSegmentDownloadConfig = ParallelHttpDownload.DownloadConfiguration.ReadFromEnvironment(EnvironmentVariablePrefix);
TempDirectory = new DisposableDirectory(configuration.FileSystem);
_counters = CounterTracker.CreateCounterCollection <BackingContentStore.SessionCounters>(counterTracker);
_blobCounters = CounterTracker.CreateCounterCollection <Counters>(counterTracker);
}
/// <summary>
/// Alternative for UploadAndReferenceBlobAsync that uses WalkBlocksAsync instead of the synchronous WalkBlocks.
/// Also utilizes the AsyncHttpRetryHelper to mitigate transient exceptions.
/// </summary>
public static Task UploadAndReferenceBlobWithRetriesAsync(
this IBlobStoreHttpClient client,
BlobIdentifier blobId,
Stream stream,
BlobReference reference,
Context context,
CancellationToken cts)
{
Contract.Requires(stream != null);
var attempt = 0;
return(AsyncHttpRetryHelper.InvokeVoidAsync(
async() =>
{
bool blobUploaded = false;
stream.Position = 0;
attempt++;
await VsoHash.WalkBlocksAsync(
stream,
blockActionSemaphore: null,
multiBlocksInParallel: true,
singleBlockCallback:
async(block, blockLength, blockHash) =>
{
await client.PutSingleBlockBlobAndReferenceAsync(
blobId, block, blockLength, reference, cts).ConfigureAwait(false);
blobUploaded = true;
},
multiBlockCallback:
(block, blockLength, blockHash, isFinalBlock) =>
client.PutBlobBlockAsync(blobId, block, blockLength, cts),
multiBlockSealCallback: async blobIdWithBlocks =>
{
var failedRefs = await client.TryReferenceWithBlocksAsync(
new Dictionary <BlobIdentifierWithBlocks, IEnumerable <BlobReference> >
{
{ blobIdWithBlocks, new[] { reference } }
},
cancellationToken: cts).ConfigureAwait(false);
blobUploaded = !failedRefs.Any();
}
);
if (!blobUploaded)
{
throw new AsyncHttpRetryHelper.RetryableException($"Could not upload blob on attempt {attempt}.");
}
},
maxRetries: 5,
tracer: new AppTraceSourceContextAdapter(context, "BlobStoreHttpClient", SourceLevels.All),
canRetryDelegate: exception =>
{
// HACK HACK: This is an additional layer of retries specifically to catch the SSL exceptions seen in DM.
// Newer versions of Artifact packages have this retry automatically, but the packages deployed with the M119.0 release do not.
// Once the next release is completed, these retries can be removed.
if (exception is HttpRequestException && exception.InnerException is WebException)
{
return true;
}
while (exception != null)
{
if (exception is TimeoutException)
{
return true;
}
exception = exception.InnerException;
}
return false;
},
cancellationToken: cts,
continueOnCapturedContext: false,
context: context.Id.ToString()));
}
