public AzureStorageFactory(
CloudStorageAccount account,
string containerName,
TimeSpan maxExecutionTime,
TimeSpan serverTimeout,
string path,
Uri baseAddress,
bool useServerSideCopy,
bool compressContent,
bool verbose,
bool initializeContainer,
IThrottle throttle) : base(throttle)
{
_account = account;
_containerName = containerName;
_path = null;
_maxExecutionTime = maxExecutionTime;
_serverTimeout = serverTimeout;
_useServerSideCopy = useServerSideCopy;
_initializeContainer = initializeContainer;
if (path != null)
{
_path = path.Trim('/') + '/';
}
_differentBaseAddress = baseAddress;
var blobEndpointBuilder = new UriBuilder(account.BlobEndpoint)
{
Scheme = "http", // Convert base address to http. 'https' can be used for communication but is not part of the names.
Port = 80
};
if (baseAddress == null)
{
BaseAddress = new Uri(blobEndpointBuilder.Uri, containerName + "/" + _path ?? string.Empty);
}
else
{
Uri newAddress = baseAddress;
if (path != null)
{
newAddress = new Uri(baseAddress, path + "/");
}
BaseAddress = newAddress;
}
// Beautify the destination URL.
blobEndpointBuilder.Scheme = "https";
blobEndpointBuilder.Port = -1;
DestinationAddress = new Uri(blobEndpointBuilder.Uri, containerName + "/" + _path ?? string.Empty);
CompressContent = compressContent;
Verbose = verbose;
}
public static CatalogStorageFactory CreateStorageFactory(
IDictionary <string, string> arguments,
bool verbose,
IThrottle throttle = null)
{
return(CreateStorageFactoryImpl(
arguments,
ArgumentNames,
verbose,
compressed: false,
throttle: throttle));
}
public ThrottledResponse(IThrottle throttle, HttpResponseMessage response)
{
if (throttle == null)
{
throw new ArgumentNullException(nameof(throttle));
}
if (response == null)
{
throw new ArgumentNullException(nameof(response));
}
_throttle = throttle;
Response = response;
}
public AzureStorage(
Uri storageBaseUri,
TimeSpan maxExecutionTime,
TimeSpan serverTimeout,
bool useServerSideCopy,
bool compressContent,
bool verbose,
IThrottle throttle)
: this(GetCloudBlobDirectoryUri(storageBaseUri), storageBaseUri, maxExecutionTime, serverTimeout, initializeContainer : false)
{
_useServerSideCopy = useServerSideCopy;
_compressContent = compressContent;
_throttle = throttle ?? NullThrottle.Instance;
Verbose = verbose;
}
public HiveStorage(
ICloudBlobClient cloudBlobClient,
RegistrationUrlBuilder urlBuilder,
IEntityBuilder entityBuilder,
IThrottle throttle,
IOptionsSnapshot <Catalog2RegistrationConfiguration> options,
ILogger <HiveStorage> logger)
{
_cloudBlobClient = cloudBlobClient ?? throw new ArgumentNullException(nameof(cloudBlobClient));
_urlBuilder = urlBuilder ?? throw new ArgumentNullException(nameof(urlBuilder));
_entityBuilder = entityBuilder ?? throw new ArgumentNullException(nameof(entityBuilder));
_throttle = throttle ?? throw new ArgumentNullException(nameof(throttle));
_options = options ?? throw new ArgumentNullException(nameof(options));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public static HttpSource Create(SourceRepository source, IThrottle throttle)
{
if (source == null)
{
throw new ArgumentNullException(nameof(source));
}
if (throttle == null)
{
throw new ArgumentNullException(nameof(throttle));
}
Func <Task <HttpHandlerResource> > factory = () => source.GetResourceAsync <HttpHandlerResource>();
return(new HttpSource(source.PackageSource, factory, throttle));
}
public static void AdjustSpeed(float speed, float desiredSpeed, float majorThreshold, float minorThreshold, IGearBox gearBox, IThrottle throttle, IBreaks breaks)
{
float diff = speed - desiredSpeed;
if (diff < -majorThreshold)
{
gearBox.DownShift();
throttle.Accellerate();
}
else if (diff < -minorThreshold)
{
throttle.Accellerate();
}
else if (diff > minorThreshold)
{
breaks.Engage();
}
}
public HttpSourceTestHost(
PackageSource packageSource,
Func <Task <HttpHandlerResource> > messageHandlerFactory,
IThrottle throttle,
HttpClient client)
: base(packageSource, messageHandlerFactory, throttle)
{
var prop = typeof(HttpSource).GetField("_httpClient",
System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance);
prop.SetValue(this, client);
this.HttpCacheDirectory = Path.Combine(Path.GetTempPath(), System.Guid.NewGuid().ToString("N"));
var cacheDir = new DirectoryInfo(this.HttpCacheDirectory);
if (cacheDir.Exists)
{
foreach (var dir in cacheDir.EnumerateDirectories())
{
dir.Delete(true);
}
}
}
private TestableHttpSource(
PackageSource packageSource,
Func <Task <HttpHandlerResource> > messageHandlerFactory,
IThrottle throttle,
HttpClient httpClient,
string httpCacheDirectory)
: base(packageSource, messageHandlerFactory, throttle)
{
// Set the HTTP client on the parent's private field.
var flags = BindingFlags.NonPublic | BindingFlags.Instance;
var field = typeof(HttpSource).GetField("_httpClient", flags);
if (field == null)
{
throw new InvalidOperationException(
$"Could not find field '_httpClient' with flags '{flags}' " +
$"on {nameof(HttpSource)}");
}
field.SetValue(this, httpClient);
HttpCacheDirectory = httpCacheDirectory;
}
public AzureStorage(
CloudStorageAccount account,
string containerName,
string path,
Uri baseAddress,
TimeSpan maxExecutionTime,
TimeSpan serverTimeout,
bool useServerSideCopy,
bool compressContent,
bool verbose,
bool initializeContainer,
IThrottle throttle) : this(
new CloudBlobDirectoryWrapper(account.CreateCloudBlobClient().GetContainerReference(containerName).GetDirectoryReference(path)),
baseAddress,
maxExecutionTime,
serverTimeout,
initializeContainer)
{
_useServerSideCopy = useServerSideCopy;
_compressContent = compressContent;
_throttle = throttle ?? NullThrottle.Instance;
Verbose = verbose;
}
public StorageFactory(IThrottle throttle)
{
Throttle = throttle ?? NullThrottle.Instance;
}
private static CatalogStorageFactory CreateStorageFactoryImpl(
IDictionary <string, string> arguments,
IDictionary <string, string> argumentNameMap,
bool verbose,
bool compressed,
IThrottle throttle = null)
{
Uri storageBaseAddress = null;
var storageBaseAddressStr = arguments.GetOrDefault <string>(argumentNameMap[Arguments.StorageBaseAddress]);
if (!string.IsNullOrEmpty(storageBaseAddressStr))
{
storageBaseAddressStr = storageBaseAddressStr.TrimEnd('/') + "/";
storageBaseAddress = new Uri(storageBaseAddressStr);
}
var storageType = arguments.GetOrThrow <string>(Arguments.StorageType);
if (storageType.Equals(Arguments.FileStorageType, StringComparison.InvariantCultureIgnoreCase))
{
var storagePath = arguments.GetOrThrow <string>(argumentNameMap[Arguments.StoragePath]);
if (storageBaseAddress != null)
{
return(new CatalogFileStorageFactory(storageBaseAddress, storagePath, verbose));
}
TraceRequiredArgument(argumentNameMap[Arguments.StorageBaseAddress]);
return(null);
}
if (Arguments.AzureStorageType.Equals(storageType, StringComparison.InvariantCultureIgnoreCase))
{
var storageAccountName = arguments.GetOrThrow <string>(argumentNameMap[Arguments.StorageAccountName]);
var storageKeyValue = arguments.GetOrThrow <string>(argumentNameMap[Arguments.StorageKeyValue]);
var storageContainer = arguments.GetOrThrow <string>(argumentNameMap[Arguments.StorageContainer]);
var storagePath = arguments.GetOrDefault <string>(argumentNameMap[Arguments.StoragePath]);
var storageSuffix = arguments.GetOrDefault <string>(argumentNameMap[Arguments.StorageSuffix]);
var storageOperationMaxExecutionTime = MaxExecutionTime(arguments.GetOrDefault <int>(argumentNameMap[Arguments.StorageOperationMaxExecutionTimeInSeconds]));
var storageServerTimeout = MaxExecutionTime(arguments.GetOrDefault <int>(argumentNameMap[Arguments.StorageServerTimeoutInSeconds]));
var storageUseServerSideCopy = arguments.GetOrDefault <bool>(argumentNameMap[Arguments.StorageUseServerSideCopy]);
var credentials = new StorageCredentials(storageAccountName, storageKeyValue);
var account = string.IsNullOrEmpty(storageSuffix) ?
new CloudStorageAccount(credentials, useHttps: true) :
new CloudStorageAccount(credentials, storageSuffix, useHttps: true);
return(new CatalogAzureStorageFactory(
account,
storageContainer,
storageOperationMaxExecutionTime,
storageServerTimeout,
storagePath,
storageBaseAddress,
storageUseServerSideCopy,
compressed,
verbose,
initializeContainer: true,
throttle: throttle ?? NullThrottle.Instance));
}
throw new ArgumentException($"Unrecognized storageType \"{storageType}\"");
}
public void ThenRun(Action toRun)
{
throttle = throttleFactory.CreateThrottle(toRun, throttleTime);
}
public void ThenRunOnMainThread(Action toRun)
{
throttle = throttleFactory.CreateMainThreadMarshalledThrottle(toRun, throttleTime);
}
public RequestThrottle(IThrottle throttle)
{
_throttle = throttle;
}
/// <summary>
/// Initializes the HTTP range reader. The provided <see cref="HttpClient"/> is not disposed by this instance.
/// </summary>
/// <param name="httpClient">The HTTP client used to make the HTTP requests.</param>
/// <param name="requestUri">The URL to request bytes from.</param>
/// <param name="length">The length of the request URI, a size in bytes.</param>
/// <param name="etag">The optional etag header to be used for follow-up requests.</param>
/// <param name="requireContentRange">Whether or not to require and validate the Content-Range header.</param>
/// <param name="httpThrottle">The throttle to use for HTTP operations.</param>
public HttpRangeReader(HttpClient httpClient, Uri requestUri, long length, string etag, bool requireContentRange, IThrottle httpThrottle)
: this(httpClient, requestUri, length, etag, requireContentRange, sendXMsVersionHeader : false, allowETagVariants : false, httpThrottle : httpThrottle)
{
}
/// <summary>
/// Initializes the HTTP range reader. The provided <see cref="HttpClient"/> is not disposed by this instance.
/// </summary>
/// <param name="httpClient">The HTTP client used to make the HTTP requests.</param>
/// <param name="requestUri">The URL to request bytes from.</param>
/// <param name="length">The length of the request URI, a size in bytes.</param>
/// <param name="etag">The optional etag header to be used for follow-up requests.</param>
/// <param name="requireContentRange">Whether or not to require and validate the Content-Range header.</param>
/// <param name="sendXMsVersionHeader">Whether or not to send the <c>x-ms-version: 2013-08-15</c> request header.</param>
/// <param name="allowETagVariants">Whether or not to allow different variants of the etag.</param>
/// <param name="httpThrottle">The throttle to use for HTTP operations.</param>
public HttpRangeReader(HttpClient httpClient, Uri requestUri, long length, string etag, bool requireContentRange, bool sendXMsVersionHeader, bool allowETagVariants, IThrottle httpThrottle)
{
_httpClient = httpClient;
_requestUri = requestUri;
_length = length;
_etag = etag;
if (etag != null && allowETagVariants)
{
if (etag.StartsWith("\"") && etag.EndsWith("\""))
{
_etagQuotes = etag;
_etagNoQuotes = etag.Substring(1, etag.Length - 2);
}
else
{
_etagQuotes = "\"" + etag + "\"";
_etagNoQuotes = etag;
}
}
_requireContentRange = requireContentRange;
_sendXMsVersionHeader = sendXMsVersionHeader;
_allowETagVariants = allowETagVariants;
_httpThrottle = httpThrottle ?? NullThrottle.Instance;
}
/// <summary>
/// Initializes an HTTP ZIP provider. This instance does not dispose the provided <see cref="HttpClient"/>.
/// </summary>
/// <param name="httpClient">The HTTP client to use for HTTP requests.</param>
/// <param name="httpThrottle">The throttle to use for HTTP operations.</param>
public HttpZipProvider(HttpClient httpClient, IThrottle httpThrottle)
{
_httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient));
_httpThrottle = httpThrottle ?? NullThrottle.Instance;
}
