public async Task StartAsync()
{
monitorCollection = CosmosDbConfig.GetMonitorConfig();
leaseCollection = CosmosDbConfig.GetLeaseConfig();
destinationCollection = CosmosDbConfig.GetDestinationConfig();
monitorClientExtension = new SqlClientExtension(
monitorCollection,
ConsistencyLevel.Session,
ConnectionPolicy.Default);
leaseClientExtension = new SqlClientExtension(
leaseCollection,
ConsistencyLevel.Session,
ConnectionPolicy.Default);
destClientExtension = new SqlClientExtension(
destinationCollection,
ConsistencyLevel.Session,
ConnectionPolicy.Default);
// Not advised to use this code in production.
this.CreateCollectionsInDevMode();
await this.RunChangeFeedHostAsync();
}
private static async Task ExportToCosmosDb(ConsumeContext <CompanyRequest> context)
{
var config = new CosmosDbConfig();
var repo = new DocumentDbRepository <Company>(config, Constant.CosmosDbCollections.Companies);
var query = repo.Client.CreateDocumentQuery <Company>(repo.CollectionUri, new SqlQuerySpec
{
QueryText = @"SELECT * FROM Companies c WHERE CONTAINS(c.CompanyName, @CompanyName)",
Parameters = new SqlParameterCollection
{
new SqlParameter("@CompanyName", context.Message.Company.CompanyName.ToUpperInvariant())
}
}, repo.DefaultOptions);
var exists = await repo.GetItemAsync(context.Message.Company.Id.ToString());
if (!query.ToList().Any() && exists == null)
{
Log.Debug("company insert try {company}", context.Message.Company.CompanyName);
await repo.CreateItemAsync(context.Message.Company);
Log.Information("company insert ok {company}", context.Message.Company.CompanyName);
}
else
{
Log.Debug("company update try {searchphrase}", context.Message.Company.CompanyName);
await repo.UpdateItemAsync(context.Message.Company.Id.ToString(), context.Message.Company);
Log.Information("company update ok {searchphrase}", context.Message.Company);
}
}
public Task InitializeAsync(CosmosDbConfig cosmosDbConfig, bool disasterRecoverySupported)
{
if (string.IsNullOrWhiteSpace(cosmosDbConfig.ConnectionString))
{
throw new ArgumentException(nameof(cosmosDbConfig.ConnectionString));
}
if (string.IsNullOrWhiteSpace(cosmosDbConfig.TableName))
{
throw new ArgumentException(nameof(cosmosDbConfig.TableName));
}
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(cosmosDbConfig.ConnectionString);
CloudTableClient tableClient = storageAccount.CreateCloudTableClient();
var tableConnectionPolicy = new TableConnectionPolicy();
tableConnectionPolicy.MaxRetryWaitTimeInSeconds = cosmosDbConfig.RetryIntervalInSeconds;
tableConnectionPolicy.MaxConnectionLimit = cosmosDbConfig.MaxConnectionLimit;
tableConnectionPolicy.EnableEndpointDiscovery = disasterRecoverySupported;
tableConnectionPolicy.MaxRetryAttemptsOnThrottledRequests = cosmosDbConfig.RetryCountOnThrottling;
tableConnectionPolicy.UseTcpProtocol = true;
// Create a table client for interacting with the table service
this.cloudTableClient = storageAccount.CreateCloudTableClient(tableConnectionPolicy,
Microsoft.Azure.CosmosDB.ConsistencyLevel.Session);
this.IsInitialized = true;
cloudTable = cloudTableClient.GetTableReference(cosmosDbConfig.TableName);
var cloudtable = (CloudTable)cloudTable;
return(cloudtable.CreateIfNotExistsAsync());
}
// TODO - set false
/// <summary>
/// Configuration constructor
/// </summary>
/// <param name="serviceId"></param>
/// <param name="configuration"></param>
public Config(string serviceId, IConfigurationRoot configuration) :
base(configuration)
{
_tasks = new TaskProcessorConfig(configuration);
_db = new CosmosDbConfig(configuration);
_storage = new StorageConfig(configuration);
_hub = new IoTHubConfig(configuration, serviceId);
}
/// <summary>
/// Configuration constructor
/// </summary>
/// <param name="configuration"></param>
public Config(IConfiguration configuration) :
base(configuration)
{
_tasks = new TaskProcessorConfig(configuration);
_db = new CosmosDbConfig(configuration);
_storage = new StorageConfig(configuration);
_hub = new IoTHubConfig(configuration);
}
// TODO - set false
/// <summary>
/// Configuration constructor
/// </summary>
/// <param name="configuration"></param>
public Config(IConfigurationRoot configuration) :
base(configuration)
{
_tasks = new TaskProcessorConfig(configuration);
_db = new CosmosDbConfig(configuration);
_storage = new StorageConfig(configuration);
_hub = new IoTHubConfig(configuration);
_ai = new ApplicationInsightsConfig(configuration);
}
/// <summary>
/// Configuration constructor
/// </summary>
/// <param name="configuration"></param>
public Config(IConfiguration configuration) :
base(configuration)
{
_hub = new IoTHubConfig(configuration);
_sb = new ServiceBusConfig(configuration);
_ep = new EventProcessorConfig(configuration);
_eh = new IoTHubEventConfig(configuration);
_cosmos = new CosmosDbConfig(configuration);
}
/// <summary>
/// Configuration constructor
/// </summary>
/// <param name="configuration"></param>
public Config(IConfiguration configuration) :
base(configuration)
{
_openApi = new OpenApiConfig(configuration);
_auth = new AuthConfig(configuration);
_host = new HostConfig(configuration);
_hub = new IoTHubConfig(configuration);
_cors = new CorsConfig(configuration);
_cosmos = new CosmosDbConfig(configuration);
}
/// <summary>
/// Configuration constructor
/// </summary>
/// <param name="configuration"></param>
public Config(IConfigurationRoot configuration) :
base(configuration)
{
_swagger = new SwaggerConfig(configuration);
_auth = new AuthConfig(configuration);
_hub = new IoTHubConfig(configuration);
_cors = new CorsConfig(configuration);
_sb = new ServiceBusConfig(configuration);
_cosmos = new CosmosDbConfig(configuration);
_ai = new ApplicationInsightsConfig(configuration);
}
/// <summary>
/// Configuration constructor
/// </summary>
/// <param name="configuration"></param>
public Config(IConfiguration configuration) :
base(configuration)
{
_openApi = new OpenApiConfig(configuration);
_host = new WebHostConfig(configuration);
_hub = new IoTHubConfig(configuration);
_cors = new CorsConfig(configuration);
_cosmos = new CosmosDbConfig(configuration);
_jobs = new JobOrchestratorConfig(configuration);
_fh = new ForwardedHeadersConfig(configuration);
}
protected override void Load(ContainerBuilder builder)
{
builder.RegisterGeneric(typeof(DocumentDbRepository <>)).As(typeof(IDocumentDbRepository <>));
builder.Register <CosmosDbConfig>((context, parameters) =>
{
var cosmosDbConfig = new CosmosDbConfig();
var configuration = context.Resolve <IConfiguration>();
configuration.GetSection(nameof(CosmosDbConfig)).Bind(cosmosDbConfig);
return(cosmosDbConfig);
});
}
/// <summary>
/// Configuration constructor
/// </summary>
/// <param name="configuration"></param>
public Config(IConfiguration configuration) :
base(configuration)
{
_openApi = new OpenApiConfig(configuration);
_auth = new AuthConfig(configuration);
_host = new HostConfig(configuration);
_hub = new IoTHubConfig(configuration);
_cors = new CorsConfig(configuration);
_cosmos = new CosmosDbConfig(configuration);
_jobs = new JobOrchestratorConfig(configuration);
_edge = new JobOrchestratorApiConfig(configuration);
}
/// <summary>
/// Configuration constructor
/// </summary>
/// <param name="configuration"></param>
public Config(IConfiguration configuration) :
base(configuration)
{
_openApi = new OpenApiConfig(configuration);
_auth = new AuthConfig(configuration);
_host = new HostConfig(configuration);
_hub = new IoTHubConfig(configuration);
_cors = new CorsConfig(configuration);
_sb = new ServiceBusConfig(configuration);
_cosmos = new CosmosDbConfig(configuration);
_fh = new ForwardedHeadersConfig(configuration);
}
/// <summary>
/// Configuration constructor
/// </summary>
/// <param name="configuration"></param>
public Config(IConfiguration configuration) :
base(configuration)
{
_openApi = new OpenApiConfig(configuration);
_host = new WebHostConfig(configuration);
_hub = new IoTHubConfig(configuration);
_cors = new CorsConfig(configuration);
_api = new ApiConfig(configuration);
_cosmos = new CosmosDbConfig(configuration);
_fh = new ForwardedHeadersConfig(configuration);
_cr = new ContainerRegistryConfig(configuration);
}
/// <summary>
/// Get database
/// </summary>
/// <param name="options"></param>
/// <returns></returns>
private static async Task <IDatabase> GetDatabaseAsync(CliOptions options)
{
var logger = ConsoleLogger.Create();
var config = new ConfigurationBuilder()
.AddFromDotEnvFile()
.AddFromKeyVault()
.Build();
var configuration = new CosmosDbConfig(config);
var server = new CosmosDbServiceClient(configuration, logger);
return(await server.OpenAsync(
options.GetValueOrDefault("-d", "--db", "default"), null));
}
/// <summary>
/// Configuration constructor
/// </summary>
/// <param name="configuration"></param>
public Config(IConfiguration configuration) :
base(configuration)
{
_openApi = new OpenApiConfig(configuration);
_host = new WebHostConfig(configuration);
_hub = new IoTHubConfig(configuration);
_cors = new CorsConfig(configuration);
_sb = new ServiceBusConfig(configuration);
_cosmos = new CosmosDbConfig(configuration);
_fh = new ForwardedHeadersConfig(configuration);
_cr = new ContainerRegistryConfig(configuration);
_lwc = new LogWorkspaceConfig(configuration);
}
/// <summary>
/// Configuration constructor
/// </summary>
/// <param name="configuration"></param>
internal Config(IConfiguration configuration) :
base(configuration)
{
_vault = new VaultConfig(configuration);
_keyVault = new KeyVaultConfig(configuration);
_cosmos = new CosmosDbConfig(configuration);
_openApi = new OpenApiConfig(configuration);
_host = new WebHostConfig(configuration);
_cors = new CorsConfig(configuration);
_sb = new ServiceBusConfig(configuration);
_hub = new IoTHubConfig(configuration);
_registry = new RegistryConfig(configuration);
_fh = new ForwardedHeadersConfig(configuration);
}
/// <summary>
/// Configuration constructor
/// </summary>
/// <param name="configuration"></param>
internal Config(IConfigurationRoot configuration) :
base(configuration)
{
_vault = new VaultConfig(configuration);
_keyVault = new KeyVaultConfig(configuration);
_cosmos = new CosmosDbConfig(configuration);
_swagger = new SwaggerConfig(configuration);
_auth = new AuthConfig(configuration);
_cors = new CorsConfig(configuration);
_sb = new ServiceBusConfig(configuration);
_hub = new IoTHubConfig(configuration);
_registry = new ApiConfig(configuration);
_ai = new ApplicationInsightsConfig(configuration);
}
public DbCollectionCreatorClient(CosmosDbConfig cosmosConfig)
{
this.cosmosConfig = cosmosConfig;
this.documentSqlClient = new DocumentClient(new Uri(cosmosConfig.CosmosDbSqlEndpoint), cosmosConfig.KeySql,
new ConnectionPolicy {
ConnectionMode = ConnectionMode.Direct, ConnectionProtocol = Protocol.Tcp, MaxConnectionLimit = 1000
});
this.documentGraphClient = new DocumentClient(new Uri(cosmosConfig.CosmosDbGraphEndpoint), cosmosConfig.KeyGraph,
new ConnectionPolicy {
ConnectionMode = ConnectionMode.Direct, ConnectionProtocol = Protocol.Tcp, MaxConnectionLimit = 1000
});
this.documentSqlClient.OpenAsync();
this.documentGraphClient.OpenAsync();
}
public SqlClientExtension(
CosmosDbConfig config,
ConsistencyLevel consistencyLevel,
ConnectionPolicy connectionPolicy)
{
this.endPoint = config.AccountEndPoint;
this.masterKey = config.Key;
this.databaseName = config.DbName;
this.collectionName = config.CollectionName;
this.throughput = config.Throughput;
this.partitionKey = config.PartitionKey;
this.includePaths = config.IncludePaths;
this.ttlInDays = config.TtlInDays;
this.initClient(consistencyLevel, connectionPolicy);
}
public async Task Consume(ConsumeContext <ChargeSeedRequest> context)
{
using (LogContext.PushProperty("RequestId", context.RequestId))
using (LogContext.PushProperty("CorrelationId", context.CorrelationId))
{
LogVerbose($"received: {context.SentTime}");
try
{
var config = new CosmosDbConfig();
var repo = new DocumentDbRepository <Charge>(config, Constant.CosmosDbCollections.Charges);
var query = repo.Client.CreateDocumentQuery <Charge>(repo.CollectionUri, new SqlQuerySpec
{
QueryText = @"SELECT * FROM Charges c WHERE CONTAINS(c.SearchPhrase, @SearchPhrase)",
Parameters = new SqlParameterCollection
{
new SqlParameter("@SearchPhrase", context.Message.Charge.SearchPhrase)
//new SqlParameter("@Id", context.Message.Charge.Id)
}
}, repo.DefaultOptions);
var exists = await repo.GetItemAsync(context.Message.Charge.Id.ToString());
if (!query.ToList().Any() && exists == null)
{
LogDebug($"insert try {context.Message.Charge.SearchPhrase}");
await repo.CreateItemAsync(context.Message.Charge);
LogInfo($"insert ok {context.Message.Charge.SearchPhrase}");
}
else
{
LogDebug($"update try {context.Message.Charge.SearchPhrase}");
await repo.UpdateItemAsync(context.Message.Charge.Id.ToString(), context.Message.Charge);
LogInfo("update ok {context.Message.Charge.SearchPhrase}");
}
}
catch (Exception e)
{
Console.WriteLine(e);
LogError(e, "{context.Message.Charge.SearchPhrase}");
// throw;
}
}
await Task.Delay(TimeSpan.FromSeconds(3));
}
public CosmosDBFixture()
{
_config = new ConfigurationBuilder()
.AddJsonFile("appsettings.json", optional: false)
.AddEnvironmentVariables()
.Build();
CosmosDbConfig = new CosmosDbConfig();
_config.GetSection("CosmosDbConfig").Bind(CosmosDbConfig);
// Initializing collection with configuration
_client = new DocumentClient(
new Uri(CosmosDbConfig.EndPoint),
CosmosDbConfig.AuthKey);
}
public QueryRunner(CosmosDbConfig cosmosConfig)
{
this.cosmosConfig = cosmosConfig;
var connectionPolicy = new ConnectionPolicy
{
ConnectionMode = ConnectionMode.Direct,
ConnectionProtocol = Protocol.Tcp
};
connectionPolicy.PreferredLocations.Add(LocationNames.AustraliaEast);
connectionPolicy.PreferredLocations.Add(LocationNames.AustraliaSoutheast);
this.documentClient = new DocumentClient(new Uri(cosmosConfig.CosmosDbSqlEndpoint), cosmosConfig.KeySql, connectionPolicy);
this.documentClient.OpenAsync();
}
public CosmosDBDataAccess(IOptions <CosmosDbConfig> config)
{
var jsonSerializerSettings = new JsonSerializerSettings()
{
Converters = new List <JsonConverter>()
{
new StringEnumConverter(),
new CosmosDbJsonConverter()
},
NullValueHandling = NullValueHandling.Ignore,
DateTimeZoneHandling = DateTimeZoneHandling.Utc
};
_config = config.Value;
_client = new DocumentClient(
new Uri(_config.Endpoint),
_config.AuthKey,
jsonSerializerSettings);
}
public async Task <IChangeFeedProcessor> StartAsync()
{
monitorCollection = CosmosDbConfig.GetMonitorConfig(this.logger);
leaseCollection = CosmosDbConfig.GetLeaseConfig(this.logger);
monitorClientExtension = new SqlClientExtension(
monitorCollection,
ConsistencyLevel.Session,
ConnectionPolicy.Default);
leaseClientExtension = new SqlClientExtension(
leaseCollection,
ConsistencyLevel.Session,
ConnectionPolicy.Default);
DestinationType?destinationType = ConfigHelper.GetDestinationType(this.logger);
if (destinationType == DestinationType.CosmosDB)
{
destinationCollection = CosmosDbConfig.GetDestinationConfig(this.logger);
destClientExtension = new SqlClientExtension(
destinationCollection,
ConsistencyLevel.Session,
ConnectionPolicy.Default);
}
else if (destinationType == DestinationType.MongoDB)
{
string destConnectionString =
ConfigurationManager.AppSettings["dest-conn"];
MongoClientSettings destSettings = MongoClientSettings.FromUrl(
new MongoUrl(destConnectionString)
);
destMongoClient = new MongoClient(destSettings);
destDatabase = destMongoClient.GetDatabase(destDbName);
destDocStoreCollection = destDatabase.GetCollection <BsonDocument>(destCollectionName);
}
// Not advised to use this code in production.
this.CreateCollectionsInDevMode();
return(await this.RunChangeFeedHostAsync());
}
public async Task GenerateTimeSeriesData()
{
CosmosDbConfig config = CosmosDbConfig.GetCosmosDbConfig();
SqlClientExtension clientExtension = new SqlClientExtension(
config,
ConsistencyLevel.Session,
connectionPolicy);
await clientExtension.CreateCollectionIfNotExistsAsync();
while (true)
{
SensorReading reading = SensorReading.GetSampleReading();
await clientExtension.DocumentClient.UpsertDocumentAsync(
UriFactory.CreateDocumentCollectionUri(clientExtension.DatabaseName, clientExtension.CollectionName),
reading);
System.Threading.Thread.Sleep(new TimeSpan(0, 0, 0, 1));
}
}
public async Task Consume(ConsumeContext <ChargeRefreshRequest> context)
{
using (LogContext.PushProperty("RequestId", context.RequestId))
using (LogContext.PushProperty("CorrelationId", context.CorrelationId))
{
LogVerbose($"received: {context.SentTime}");
try
{
var config = new CosmosDbConfig();
var repo = new DocumentDbRepository <Charge>(config, Constant.CosmosDbCollections.Charges);
var charge = await repo.GetItemAsync(context.Message.Id.ToString());
var place = await GetPlace(charge);
if (place.Status.HasValue)
{
if (place.Status == Status.Ok)
{
charge.PlaceLocationTypes = GetPlaceTypes(place.Results).ToList();
// charge.PlaceId = place.Results.ToList();
LogDebug($"update try {charge.Id}");
await repo.UpdateItemAsync(context.Message.Id.ToString(), charge);
LogInfo($"update ok {charge.Id}");
}
}
}
catch (Exception e)
{
Console.WriteLine(e);
Log.Error(e, "error {err}", context.Message.Id);
// throw;
}
}
await Task.Delay(TimeSpan.FromSeconds(3));
}
private static async Task ExportToCosmosDb(ConsumeContext <ClassifierRequest> context)
{
var config = new CosmosDbConfig();
var repo = new DocumentDbRepository <Contracts.Classifier>(config,
Constant.CosmosDbCollections.KeywordClassifiers);
var query = repo.Client.CreateDocumentQuery <Contracts.Classifier>(repo.CollectionUri, new SqlQuerySpec
{
QueryText = @"SELECT * FROM KeywordClassifiers c WHERE CONTAINS(c.SubCategory, @SubCategory)",
Parameters = new SqlParameterCollection
{
new SqlParameter("@SubCategory", context.Message.Identifier.SubCategory.ToUpperInvariant())
}
}, repo.DefaultOptions);
if (context.Message.Identifier.Id == Guid.Empty)
{
await Insert(context, repo);
}
else
{
await Upsert(context, repo, query);
}
}
/// <summary>
/// Configuration constructor
/// </summary>
/// <param name="configuration"></param>
public VaultConfig(IConfiguration configuration) :
base(configuration)
{
_cosmos = new CosmosDbConfig(configuration);
}
public static async Task <DataSource> GetOrCreateCosmosDataSource(ISearchServiceClient serviceClient, string name, DataSourceType dataSourceType, CosmosDbConfig config, string query = "")
{
if (await serviceClient.DataSources.ExistsAsync(name))
{
return(await serviceClient.DataSources.GetAsync(name));
}
var dataSource = new DataSource
{
Name = name,
Type = dataSourceType,
Credentials = new DataSourceCredentials($"{config.ConnectionString};Database={config.DatabaseId}"),
Container = new DataContainer(config.ContainerId, "SELECT * FROM c WHERE c._ts > @HighWaterMark ORDER BY c._ts")
};
return(await serviceClient.DataSources.CreateAsync(dataSource));
}
