/// <summary>
/// Initializes a new instance of the <see cref="StorageServiceTask" /> class.
/// </summary>
/// <param name="cosmos">The options.</param>
public StorageServiceTask(CosmosOptions cosmos)
{
this.Endpoint = cosmos.Endpoint;
this.Key = cosmos.Key;
this.Database = cosmos.Database;
this.Collection = cosmos.DatasetsCollection;
this.CosmosOptions = cosmos;
}
public CosmosContext(CosmosClient cosmosClient, IOptions <CosmosOptions> options, ILogger <CosmosContext> logger)
{
_cosmosClient = cosmosClient;
_logger = logger;
_cosmosOptions = options.Value;
_cosmosOptions.Guard();
SetupAsync().Wait();
}
// This method gets called by the runtime. Use this method to add services to the container.
// For more information on how to configure your application, visit https://go.microsoft.com/fwlink/?LinkID=398940
public void ConfigureServices(IServiceCollection services)
{
var cosmosOptions = new CosmosOptions();
_configuration.Bind("Cosmos", cosmosOptions);
services.AddHttpClientsWithCaching(cosmosOptions);
services.AddControllers();
}
public DatasetDeleteTask(
SearchOptions searchOptions,
CosmosOptions cosmosOptions,
string datasetId)
{
SearchOptions = searchOptions;
CosmosOptions = cosmosOptions;
DatasetId = datasetId;
}
public DataInitTask(
CosmosOptions cosmosOptions,
StorageOptions storageOptions,
DataInitTypes selectedTypes)
{
CosmosOptions = cosmosOptions;
StorageOptions = storageOptions;
SelectedTypes = selectedTypes;
}
public NominationDeleteTask(
SearchOptions searchOptions,
CosmosOptions cosmosOptions,
string nominationId)
{
SearchOptions = searchOptions;
CosmosOptions = cosmosOptions;
NominationId = nominationId;
}
public DatasetViewTask(
CosmosOptions cosmosOptions,
StorageOptions storageOptions,
string datasetId)
{
CosmosOptions = cosmosOptions;
StorageOptions = storageOptions;
DatasetId = datasetId;
}
public AnalysisRepository(IOptions <CosmosOptions> cosmosOptions, ILogger <AnalysisRepository> logger)
{
if (string.IsNullOrWhiteSpace(cosmosOptions?.Value?.ConnectionString))
{
throw new ArgumentException("Incorrect value provided for Cosmos connectionstring");
}
_cosmosOptions = cosmosOptions.Value;
_logger = logger;
}
public DatasetNominateTask(
CosmosOptions cosmosOptions,
ContactInfoOptions contactOptions,
BatchOptions batchOptions,
string datasetId,
bool queueJob)
{
CosmosOptions = cosmosOptions;
ContactOptions = contactOptions;
BatchOptions = batchOptions;
DatasetId = datasetId;
QueueJob = queueJob;
}
public BatchInitTask(
BatchOptions batchOptions,
SearchOptions searchOptions,
CosmosOptions cosmosOptions,
StorageOptions storageOptions,
VaultOptions vaultOptions)
{
BatchOptions = batchOptions;
SearchOptions = searchOptions;
CosmosOptions = cosmosOptions;
StorageOptions = storageOptions;
VaultOptions = vaultOptions;
}
public DatasetImportTask(
CosmosOptions cosmosOptions,
StorageOptions storageOptions,
ContactInfoOptions contactOptions,
string datasetUrl,
string storageName)
{
CosmosOptions = cosmosOptions;
StorageOptions = storageOptions;
ContactOptions = contactOptions;
DatasetUrl = datasetUrl;
StorageName = storageName;
}
public DatasetVerifyTask(
CosmosOptions cosmosOptions,
StorageOptions storageOptions,
SearchOptions searchOptions,
IndexOptions indexOptions,
bool fixDatasets)
{
CosmosOptions = cosmosOptions;
StorageOptions = storageOptions;
SearchOptions = searchOptions;
IndexOptions = indexOptions;
FixDatasets = fixDatasets;
}
public static void Configure(CommandLineApplication command)
{
command.Description = "Creates initial data for application.";
command.SetDefaultHelp();
var typesOption = command.Option(
"--types | -t <types>", $"The comma-separated list of types to initialize ({AllDataTypes}) or '*' for all.",
CommandOptionType.SingleValue);
command.OnExecute(async() =>
{
var cosmos = new CosmosOptions();
var storage = new StorageOptions();
DataInitTypes selectedTypes = DataInitTypes.None;
string typesText = typesOption.Value();
if (typesText == "*")
{
selectedTypes =
DataInitTypes.Domains |
DataInitTypes.Licenses |
DataInitTypes.FAQs |
DataInitTypes.Email |
DataInitTypes.ARM |
DataInitTypes.DatasetOwners;
}
else if (!string.IsNullOrWhiteSpace(typesText))
{
selectedTypes = typesText
.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries)
.Select(s => Enum.Parse(typeof(DataInitTypes), s.Trim(), true))
.Aggregate(DataInitTypes.None, (selected, v) => selected | (DataInitTypes)v);
}
if (selectedTypes == DataInitTypes.None)
{
throw new ArgumentException("No data types specified.");
}
if (command.HasAllRequiredParameters(new[]
{
cosmos.Endpoint,
cosmos.Database,
storage.Account
}))
{
await new Admin.Data.DataInitTask(cosmos, storage, selectedTypes).ExecuteAsync();
}
return(0);
});
}
public static void Configure(CommandLineApplication command)
{
command.Description = "Initializes the Azure Batch configuration";
command.SetDefaultHelp();
command.OnExecute(async() =>
{
var batch = new BatchOptions();
var search = new SearchOptions();
var cosmos = new CosmosOptions();
var storage = new StorageOptions();
var vault = new VaultOptions();
if (command.HasAllRequiredParameters(new[]
{
batch.Account,
batch.Url,
batch.Key,
batch.StorageName,
batch.StorageKey,
search.Name,
search.Key,
cosmos.Endpoint,
cosmos.RawKey,
cosmos.Database,
cosmos.DatasetsCollection,
cosmos.UserDataCollection,
storage.Account,
storage.Key,
vault.VaultUrl
}))
{
await new Admin.Batch.BatchInitTask(batch, search, cosmos, storage, vault).ExecuteAsync();
}
return(0);
});
}
public static void Configure(CommandLineApplication command)
{
command.Description = "Imports a dataset from another site as a nomination.";
command.SetDefaultHelp();
var datasetUrlOpt = command.Option(
"--datasetUrl | -u <datasetUrl>",
"The API url of the dataset",
CommandOptionType.SingleValue);
var storageNameOpt = command.Option(
"--storageName | -n <storageName>",
"The name of the storage container where the dataset contents reside.",
CommandOptionType.SingleValue);
command.OnExecute(async() =>
{
var cosmos = new CosmosOptions();
var storage = new StorageOptions();
var contact = new ContactInfoOptions();
var datasetUrl = datasetUrlOpt.Value();
var storageName = storageNameOpt.Value();
if (command.HasAllRequiredParameters(new[]
{
cosmos.Endpoint,
cosmos.Database,
storage.Account,
storage.Key,
contact.Name,
contact.Email,
datasetUrl,
storageName,
}))
{
await new Admin.Dataset.DatasetImportTask(cosmos, storage, contact, datasetUrl, storageName).ExecuteAsync();
}
return(0);
});
}
public static void Configure(CommandLineApplication command)
{
command.Description = "Creates the initial database and collection in an account";
command.SetDefaultHelp();
command.OnExecute(async() =>
{
var cosmos = new CosmosOptions();
if (command.HasAllRequiredParameters(new[]
{
cosmos.DatasetsCollection,
cosmos.Database,
cosmos.Endpoint,
cosmos.RawKey
}))
{
await new Storage.CreateDatabaseTask(cosmos).ExecuteAsync();
}
return(0);
});
}
public static void Configure(CommandLineApplication command)
{
command.Description = "Initializes the Azure Search configuration";
command.SetDefaultHelp();
var indexesOption = command.Option(
"--indexes | -i <indexes>", $"The comma-separated list of indexes ({IndexOptions.AllIndexNames})",
CommandOptionType.SingleValue);
command.OnExecute(async() =>
{
var search = new SearchOptions();
var index = new IndexOptions(indexesOption.Value());
var cosmos = new CosmosOptions();
if (command.HasAllRequiredParameters(new[]
{
search.Key,
search.Name,
cosmos.DatasetsCollection,
cosmos.Database,
cosmos.Endpoint,
cosmos.RawKey,
index.DatasetIndexer,
index.DatasetDataSource,
index.DatasetIndex,
index.FileIndexer,
index.FileIndex,
index.FileDataSource
}))
{
await new Admin.Search.CreateSearchTask(search, index, cosmos).ExecuteAsync();
}
return(0);
});
}
private static CosmosClient CreateCosmosClient(string connectionString, CosmosOptions cosmosOptions)
{
GuardEmptyString(connectionString, "cosmosDbConnectionString");
cosmosOptions.Guard();
var builder = new CosmosClientBuilder(connectionString);
// ConnectionPolicy のDefault値は、SDKのConnectionPolicy.csで設定されています。
// ## Connection Mode について
// Default で ConnectionMode.Direct/Protocol.Tcp で接続されます。
// もしGateway(ConnectionMode.Gateway/Protocol.Https) を使いたければ、以下メソッドを呼ぶ
// builder.UseConnectionModeGateway(maxConnectionLimit: ??);
// Default: CamelCase Serialize/Deserialize and ignore Readonly property
// TODO: 設定変更用のconfigは未実装
//var settings = JsonSerializerSettingsFactory.CreateForReadonlyIgnoreAndCamelCase();
var settings = JsonSerializerSettingsFactory.CreateForCamelCase();
builder.WithCustomSerializer(new CustomizableCaseJsonSerializer(settings));
if (cosmosOptions.ThrottlingRetryOptions != null)
{
builder.WithThrottlingRetryOptions(
cosmosOptions.ThrottlingRetryOptions.MaxRetryWaitTimeOnThrottledRequests,
cosmosOptions.ThrottlingRetryOptions.MaxRetryAttemptsOnThrottledRequests);
}
// multi-master support
if (!string.IsNullOrEmpty(cosmosOptions.CurrentRegion))
{
builder.WithApplicationRegion(cosmosOptions.CurrentRegion);
}
return(builder.Build());
}
public static IServiceCollection AddHttpClientsWithCaching(this IServiceCollection services, CosmosOptions cosmosOptions)
{
// Register a typed http client
services.AddHttpClient <JsonPlaceholderClient>(client =>
{
client.Timeout = TimeSpan.FromSeconds(10);
client.BaseAddress = new Uri("https://jsonplaceholder.typicode.com/");
});
// Register cosmos cache as a distributed cache
services.AddCosmosCache((CosmosCacheOptions cacheOptions) =>
{
cacheOptions.ContainerName = cosmosOptions.ContainerName;
cacheOptions.DatabaseName = cosmosOptions.DatabaseName;
cacheOptions.ClientBuilder = new CosmosClientBuilder(cosmosOptions.ServiceEndpoint.OriginalString, cosmosOptions.AuthKey);
cacheOptions.CreateIfNotExists = true;
});
// Register AsyncCacheProvider for use by Polly
services.AddSingleton(serviceProvider => serviceProvider.GetRequiredService <IDistributedCache>().AsAsyncCacheProvider <string>());
// Register a Polly cache policy for caching responses received by the JsonPlaceholderClient, using above distributed cache provider.
services.AddSingleton <IReadOnlyPolicyRegistry <string>, PolicyRegistry>((serviceProvider) =>
{
var logger = serviceProvider.GetRequiredService <ILogger <JsonPlaceholderClient> >();
PolicyRegistry registry = new PolicyRegistry();
registry.Add("jsonPlaceHolderClientCachePolicy", Policy.CacheAsync(serviceProvider.GetRequiredService <IAsyncCacheProvider <string> >(), TimeSpan.FromMinutes(1),
onCachePut: (context, key) => logger.LogInformation($"Caching '{key}'."),
onCacheGet: (context, key) => logger.LogInformation($"Retrieving '{key}' from the cache."),
onCachePutError: (context, key, exception) => logger.LogWarning(exception, $"Cannot add '{key}' to the cache."),
onCacheGetError: (context, key, exception) => logger.LogWarning(exception, $"Cannot retrieve '{key}' from the cache."),
onCacheMiss: (context, key) => logger.LogInformation($"Cache miss for '{key}'.")
));
return(registry);
});
return(services);
}
public static IServiceCollection AddCosmosContext(this IServiceCollection services, string connectionString, CosmosOptions cosmosOptions)
{
GuardNotNull(services, nameof(services));
services.AddSingleton(_ = CreateCosmosClient(connectionString, cosmosOptions));
services.AddSingleton <CosmosContext>();
return(services);
}
/// <summary>
/// Initializes a new instance of the <see cref="CreateDatabaseTask" /> class.
/// </summary>
/// <param name="cosmos">The Cosmos DB configuration.</param>
public CreateDatabaseTask(CosmosOptions cosmos)
: base(cosmos)
{
}
/// <summary>
/// Initializes a new instance of the <see cref="CreateSearchTask" /> class.
/// </summary>
/// <param name="search">The Azure search configuration options.</param>
/// <param name="cosmos">The Cosmos configuration options.</param>
public CreateSearchTask(SearchOptions search, IndexOptions index, CosmosOptions cosmos)
: base(search, index)
{
this.CosmosOptions = cosmos;
}