private async Task ExecuteCatalog2DnxAsync(PerBatchContext context, IReadOnlyList <PerPackageContext> packageContexts, Uri catalogIndexUri)
{
var serviceProvider = GetServiceProvider(
context,
context.Global.FlatContainerContainerName,
context.Process.FlatContainerStoragePath);
var storageFactory = serviceProvider.GetRequiredService <StorageFactory>();
IAzureStorage preferredPackageSourceStorage = null;
var httpClientTimeout = TimeSpan.FromMinutes(10);
var maxDegreeOfParallelism = ServicePointManager.DefaultConnectionLimit;
var collector = new DnxCatalogCollector(
catalogIndexUri,
storageFactory,
preferredPackageSourceStorage,
context.Global.ContentBaseAddress,
serviceProvider.GetRequiredService <ITelemetryService>(),
serviceProvider.GetRequiredService <ILogger>(),
maxDegreeOfParallelism,
() => serviceProvider.GetRequiredService <HttpMessageHandler>(),
httpClientTimeout);
var lowercasePackageIds = packageContexts.Select(x => x.PackageId.ToLowerInvariant());
using (await _stringLocker.AcquireAsync(lowercasePackageIds, TimeSpan.FromMinutes(5)))
{
await collector.RunAsync(CancellationToken.None);
}
}
public CromwellOnAzureEnvironment(ILoggerFactory loggerFactory, IAzureStorage storage, ICromwellApiClient cromwellApiClient)
{
logger = loggerFactory.CreateLogger <AzureStorage>();
this.storage = storage;
this.cromwellApiClient = cromwellApiClient;
logger.LogInformation($"Cromwell URL: {cromwellApiClient.GetUrl()}");
}
public SeleniumService(IOptions <SeleniumConfiguration> options, ILogger <SeleniumService> logger, IDomainProvider domainServiceProvider, IAzureStorage azureStorage)
{
this.options = options;
this.logger = logger;
this.domainServiceProvider = domainServiceProvider;
this.azureStorage = azureStorage;
}
public DnxCatalogCollector(
Uri index,
StorageFactory storageFactory,
IAzureStorage preferredPackageSourceStorage,
Uri contentBaseAddress,
ITelemetryService telemetryService,
ILogger logger,
int maxDegreeOfParallelism,
Func <HttpMessageHandler> handlerFunc = null,
TimeSpan?httpClientTimeout = null)
: base(index, telemetryService, handlerFunc, httpClientTimeout)
{
_storageFactory = storageFactory ?? throw new ArgumentNullException(nameof(storageFactory));
_sourceStorage = preferredPackageSourceStorage;
_contentBaseAddress = contentBaseAddress;
_dnxMaker = new DnxMaker(storageFactory);
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
if (maxDegreeOfParallelism < 1)
{
throw new ArgumentOutOfRangeException(
nameof(maxDegreeOfParallelism),
string.Format(Strings.ArgumentOutOfRange, 1, int.MaxValue));
}
// Find two factors which are close or equal to each other.
var squareRoot = Math.Sqrt(maxDegreeOfParallelism);
// If the max degree of parallelism is a perfect square, great.
// Otherwise, prefer a greater degree of parallelism in batches than commit items within a batch.
_maxConcurrentBatches = Convert.ToInt32(Math.Ceiling(squareRoot));
_maxConcurrentCommitItemsWithinBatch = Convert.ToInt32(maxDegreeOfParallelism / _maxConcurrentBatches);
ServicePointManager.DefaultConnectionLimit = _maxConcurrentBatches * _maxConcurrentCommitItemsWithinBatch;
}
public DnxCatalogCollector(
Uri index,
StorageFactory storageFactory,
IAzureStorage preferredPackageSourceStorage,
Uri contentBaseAddress,
ITelemetryService telemetryService,
ILogger logger,
int maxDegreeOfParallelism,
Func <HttpMessageHandler> handlerFunc = null,
TimeSpan?httpClientTimeout = null)
: base(index, telemetryService, handlerFunc, httpClientTimeout)
{
_storageFactory = storageFactory ?? throw new ArgumentNullException(nameof(storageFactory));
_sourceStorage = preferredPackageSourceStorage;
_contentBaseAddress = contentBaseAddress;
_dnxMaker = new DnxMaker(storageFactory);
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
if (maxDegreeOfParallelism < 1)
{
throw new ArgumentOutOfRangeException(
nameof(maxDegreeOfParallelism),
string.Format(Strings.ArgumentOutOfRange, 1, int.MaxValue));
}
_maxDegreeOfParallelism = maxDegreeOfParallelism;
}
public async Task <System.IO.Stream> GenerateGif(List <Snap> files, IAzureStorage storage)
{
System.IO.MemoryStream ms = new System.IO.MemoryStream();
using (MagickImageCollection collection = new MagickImageCollection())
{
for (int i = 0; i < files.Count; i++)
{
var azstream = await storage.DownloadFromStorage(files[i].FileName);
MagickImage mImage = new MagickImage(azstream);
mImage.Format = MagickFormat.Jpg;
collection.Add(mImage);
collection[i].AnimationDelay = 13;
}
// Optionally reduce colors
QuantizeSettings settings = new QuantizeSettings();
settings.Colors = 256;
collection.Quantize(settings);
// Optionally optimize the images (images should have the same size).
collection.Optimize();
// Save gif
collection.Write(ms, MagickFormat.Gif);
ms.Position = 0;
return(ms);
}
}
public void Setup()
{
var random = new Random();
var name = string.Format("a{0}b", random.Next());
queue = new BusQueue(name, Configuration.ConnectionString);
}
public Lab(ILoggerFactory loggerFactory, string name, IAzureStorage storage, ICromwellApiClient cromwellApiClient)
{
logger = loggerFactory.CreateLogger <AzureStorage>();
Name = name;
this.storage = storage;
this.cromwellApiClient = cromwellApiClient;
logger.LogInformation($"Cromwell URL: {cromwellApiClient.GetUrl()}");
}
public void Setup()
{
var random = new Random();
var name = string.Format("a{0}b", random.Next());
queue = new BusQueue(name, connection);
queue.CreateIfNotExists().Wait();
}
/// <summary>
/// Intialize class with Storage to create
/// </summary>
/// <param name="storage">Storage</param>
public InitializeStorageTask(IAzureStorage storage)
{
if (null == storage)
{
throw new ArgumentNullException("storage");
}
this.storage = storage;
}
public FtpDownlaoder(
IAzureStorage azureStorage,
IFtpWrapperFactory ftpWrapperFactory,
ISettingsProvider settingsProvider)
{
this.settingsProvider = settingsProvider;
this.ftpWrapperFactory = ftpWrapperFactory;
this.azureStorage = azureStorage;
}
/// <summary>
/// Intialize class with Storage to create
/// </summary>
/// <param name="storage">Storage</param>
public InitializeStorageTask(IAzureStorage storage)
{
if (null == storage)
{
throw new ArgumentNullException("storage");
}
this.storage = storage;
}
public WriteTumblrCommand(IAzureStorage azureStorage, ICommand uploadBlobCommand, ICommand generateLCommand, ICommand generateMCommand, ICommand generateSCommand, ICommand createTumblrCommand, ICommand createTableEntitiesCommand)
{
_azureStorage = azureStorage;
_uploadBlobCommand = uploadBlobCommand;
_generateLCommand = generateLCommand;
_generateMCommand = generateMCommand;
_generateSCommand = generateSCommand;
_createTumblrCommand = createTumblrCommand;
_createTableEntitiesCommand = createTableEntitiesCommand;
}
public ExameServicoAplicacao(
IMapper mapper,
IExameServico exameServico,
IAzureStorage azureStorage,
IEmailResultadoEnviadoServicoAplicacao emailResultadoEnviadoServicoAplicacao) : base(mapper, exameServico)
{
_exameServico = exameServico;
_azureStorage = azureStorage;
_emailResultadoEnviadoServicoAplicacao = emailResultadoEnviadoServicoAplicacao;
}
private PackageCatalogItemCreator(
HttpClient httpClient,
ITelemetryService telemetryService,
ILogger logger,
IStorage storage)
{
_httpClient = httpClient;
_telemetryService = telemetryService;
_logger = logger;
_storage = storage as IAzureStorage;
}
public TumblrService(
IAzureStorage azureStorage,
ICommand readEntityCommand,
ICommand readEntitiesCommand,
ISasService sasService)
{
_azureStorage = azureStorage;
_readEntityCommand = readEntityCommand;
_readEntitiesCommand = readEntitiesCommand;
_sasService = sasService;
}
public DatabankService(
ISettingsProvider settingsProvider,
IAzureStorage azureStorage,
IDbConnection dbConnection,
IDataTableCreator dataTableCreator)
{
this.settingsProvider = settingsProvider;
this.azureStorage = azureStorage;
this.dbConnection = dbConnection;
this.dataTableCreator = dataTableCreator;
}
public FakeHelper()
{
//_azureStorage = new AzureStoragePool(
// new AzureStorageDb(CloudStorageAccount.DevelopmentStorageAccount),
// new LvMemoryCache(),
// new CacheKeyFactory());
_azureStorage = new AzureStoragePool(
new AzureStorageDb(CloudStorageAccount.Parse("DefaultEndpointsProtocol=https;AccountName=qingyulustg;AccountKey=W3MKiTHPCNBroHyWbOMYaRF7V91LNgstG3qBTh7W8Z5Pn5ZlmukhR/uTV9+VsenNpifn/dpXd8kFtkLw0ZffDA==")),
new LvMemoryCache(),
new CacheKeyFactory());
}
private async Task <Stream> GetPackageStreamAsync(
IAzureStorage sourceStorage,
string packageId,
string normalizedPackageVersion,
CancellationToken cancellationToken)
{
var packageFileName = PackageUtility.GetPackageFileName(packageId, normalizedPackageVersion);
var sourceUri = sourceStorage.ResolveUri(packageFileName);
var packageSourceBlob = await sourceStorage.GetCloudBlockBlobReferenceAsync(sourceUri);
return(await packageSourceBlob.GetStreamAsync(cancellationToken));
}
public FavoriteService(
IAzureStorage azureStorage,
ICommand readEntityCommand,
ICommand createFavoriteCommand,
ICommand deleteFavoriteCommand,
IUriFactory uriFactory)
{
_azureStorage = azureStorage;
_readEntityCommand = readEntityCommand;
_createFavoriteCommand = createFavoriteCommand;
_deleteFavoriteCommand = deleteFavoriteCommand;
_uriFactory = uriFactory;
}
public void Setup()
{
var random = new Random();
name = string.Format("a{0}b", random.Next());
topic = new BusTopic(name, connection);
topic.CreateIfNotExists().Wait();
var s = new BusTopicSubscription(topic.Name, connection, "testing");
s.CreateIfNotExists().Wait();
}
public SasCommentService(
IAzureStorage azureStorage,
ICommand createCommentCommand,
ITumblrService tumblrService,
IUriFactory uriFactory,
ISasService sasService)
{
_azureStorage = azureStorage;
_createCommentCommand = createCommentCommand;
_tumblrService = tumblrService;
_uriFactory = uriFactory;
_sasService = sasService;
}
public BusShard(IAzureStorage resource, IQueueObject sender)
{
if (null == resource)
{
throw new ArgumentNullException("resource");
}
if (null == sender)
{
throw new ArgumentNullException("sender");
}
this.resource = resource;
this.sender = sender;
}
public CatalogLeafDataProcessor(
IAzureStorage packageStorage,
IIconProcessor iconProcessor,
IExternalIconContentProvider externalIconContentProvider,
IIconCopyResultCache iconCopyResultCache,
ITelemetryService telemetryService,
ILogger <CatalogLeafDataProcessor> logger)
{
_packageStorage = packageStorage ?? throw new ArgumentNullException(nameof(packageStorage));
_iconProcessor = iconProcessor ?? throw new ArgumentNullException(nameof(iconProcessor));
_externalIconContentProvider = externalIconContentProvider ?? throw new ArgumentNullException(nameof(externalIconContentProvider));
_iconCopyResultCache = iconCopyResultCache ?? throw new ArgumentNullException(nameof(iconCopyResultCache));
_telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
protected override void Init(IDictionary <string, string> arguments, CancellationToken cancellationToken)
{
var source = arguments.GetOrThrow <string>(Arguments.Source);
var verbose = arguments.GetOrDefault(Arguments.Verbose, false);
var contentBaseAddress = arguments.GetOrDefault <string>(Arguments.ContentBaseAddress);
var storageFactory = CommandHelpers.CreateStorageFactory(arguments, verbose);
var httpClientTimeoutInSeconds = arguments.GetOrDefault <int?>(Arguments.HttpClientTimeoutInSeconds);
var httpClientTimeout = httpClientTimeoutInSeconds.HasValue ? (TimeSpan?)TimeSpan.FromSeconds(httpClientTimeoutInSeconds.Value) : null;
StorageFactory preferredPackageSourceStorageFactory = null;
IAzureStorage preferredPackageSourceStorage = null;
var preferAlternatePackageSourceStorage = arguments.GetOrDefault(Arguments.PreferAlternatePackageSourceStorage, defaultValue: false);
if (preferAlternatePackageSourceStorage)
{
preferredPackageSourceStorageFactory = CommandHelpers.CreateSuffixedStorageFactory("PreferredPackageSourceStorage", arguments, verbose);
preferredPackageSourceStorage = preferredPackageSourceStorageFactory.Create() as IAzureStorage;
}
Logger.LogInformation("CONFIG source: \"{ConfigSource}\" storage: \"{Storage}\" preferred package source storage: \"{PreferredPackageSourceStorage}\"",
source,
storageFactory,
preferredPackageSourceStorageFactory);
Logger.LogInformation("HTTP client timeout: {Timeout}", httpClientTimeout);
MaxDegreeOfParallelism = 256;
_collector = new DnxCatalogCollector(
new Uri(source),
storageFactory,
preferredPackageSourceStorage,
contentBaseAddress == null ? null : new Uri(contentBaseAddress),
TelemetryService,
Logger,
MaxDegreeOfParallelism,
httpClient => new CatalogClient(new SimpleHttpClient(httpClient, LoggerFactory.CreateLogger <SimpleHttpClient>()), LoggerFactory.CreateLogger <CatalogClient>()),
CommandHelpers.GetHttpMessageHandlerFactory(TelemetryService, verbose),
httpClientTimeout);
var storage = storageFactory.Create();
_front = new DurableCursor(storage.ResolveUri("cursor.json"), storage, MemoryCursor.MinValue);
_back = MemoryCursor.CreateMax();
_destination = storageFactory.BaseAddress;
TelemetryService.GlobalDimensions[TelemetryConstants.Destination] = _destination.AbsoluteUri;
}
public async Task <DnxEntry> AddPackageAsync(
IAzureStorage sourceStorage,
string nuspec,
string packageId,
string normalizedPackageVersion,
string iconFilename,
CancellationToken cancellationToken)
{
if (sourceStorage == null)
{
throw new ArgumentNullException(nameof(sourceStorage));
}
if (string.IsNullOrEmpty(nuspec))
{
throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(nuspec));
}
if (string.IsNullOrEmpty(packageId))
{
throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(packageId));
}
if (string.IsNullOrEmpty(normalizedPackageVersion))
{
throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(normalizedPackageVersion));
}
cancellationToken.ThrowIfCancellationRequested();
var destinationStorage = _storageFactory.Create(packageId);
var nuspecUri = await SaveNuspecAsync(destinationStorage, packageId, normalizedPackageVersion, nuspec, cancellationToken);
if (!string.IsNullOrWhiteSpace(iconFilename))
{
await CopyIconFromAzureStorageIfExistAsync(sourceStorage, destinationStorage, packageId, normalizedPackageVersion, iconFilename, cancellationToken);
}
else
{
_logger.LogInformation("Package {PackageId} {PackageVersion} doesn't have icon file specified in Azure Storage stream case",
packageId,
normalizedPackageVersion);
}
var nupkgUri = await CopyNupkgAsync(sourceStorage, destinationStorage, packageId, normalizedPackageVersion, cancellationToken);
return(new DnxEntry(nupkgUri, nuspecUri));
}
public static async Task <(IEnumerable <IAzureStorage>, IAzureStorage)> GetStorageAccountsUsingMsiAsync(string accountName)
{
IAzureStorage defaultAzureStorage = default;
(var accounts, var defaultAccount) = await GetCloudStorageAccountsUsingMsiAsync(accountName);
return(accounts.Select(GetAzureStorage).ToList(), defaultAzureStorage ?? throw new Exception($"Azure Storage account with name: {accountName} not found in list of {accounts.Count} storage accounts."));
IAzureStorage GetAzureStorage(CloudStorageAccount cloudStorage)
{
var azureStorage = new AzureStorage(cloudStorage, new HttpClient());
if (cloudStorage.Equals(defaultAccount))
{
defaultAzureStorage = azureStorage;
}
return(azureStorage);
}
}
private async Task CopyIconFromAzureStorageIfExistAsync(
IAzureStorage sourceStorage,
Storage destinationStorage,
string packageId,
string normalizedPackageVersion,
string iconFilename,
CancellationToken cancellationToken)
{
using (var packageStream = await GetPackageStreamAsync(sourceStorage, packageId, normalizedPackageVersion, cancellationToken))
{
await CopyIconAsync(
packageStream,
iconFilename,
destinationStorage,
packageId,
normalizedPackageVersion,
cancellationToken);
}
}
public AzureMediaService(ConfigWrapper settings, IAzureStorage storage, IAzureMediaServicesClient client)
{
_settings = settings;
_storage = storage;
_client = client;
}
public AzureFileUpload(IAzureStorage azurestorage)
{
_azurestorage = azurestorage;
}
public AzureStoragePool(IAzureStorage azureStorage, ICache cache, ICacheKeyFactory cacheKeyFactory)
{
_azureStorage = azureStorage;
_cache = cache;
_cacheKeyFactory = cacheKeyFactory;
}
public TodoItemManager (IAzureStorage storage)
{
this.storage = storage;
}
