private static ILeaseStore CreateLeaseStore(IServiceCollection services, CosmosDataStoreOptions cosmosDataStoreOptions)
{
var cosmosDbConnectionString = new CosmosDbConnectionString(cosmosDataStoreOptions.ConnectionString);
var documentClient = new DocumentClient(cosmosDbConnectionString.ServiceEndpoint, cosmosDbConnectionString.AuthKey);
var serviceProvider = services.BuildServiceProvider();
return(new CosmosDbLeaseStore(documentClient, serviceProvider.GetService <IBigBrother>(),
Options.Create(cosmosDataStoreOptions)));
}
/// <summary>
/// Retrieves a Cosmos DB database and a container with the specified partition key.
/// </summary>
/// <returns></returns>
private static CosmosDbService InitializeCosmosClientInstance(IConfiguration configuration)
{
var databaseName = configuration["DatabaseName"];
var containerName = configuration["ContainerName"];
var connectionString = configuration["CosmosDBConnection"];
var cosmosDbConnectionString = new CosmosDbConnectionString(connectionString);
// TODO 8: Use the CosmosClientBuilder to create the CosmosClient and instantiate a new CosmosDbService so it can be returned.
// Complete: CosmosClientBuilder clientBuilder = ...; CosmosClient client = ...; CosmosDbService cosmosDbService = ...;
return(cosmosDbService);
}
/// <summary>
/// Retrieves a Cosmos DB database and a container with the specified partition key.
/// </summary>
/// <returns></returns>
private static CosmosDbService InitializeCosmosClientInstance(IConfiguration configuration)
{
var databaseName = configuration["DatabaseName"];
var containerName = configuration["ContainerName"];
var connectionString = configuration["CosmosDBConnection"];
var cosmosDbConnectionString = new CosmosDbConnectionString(connectionString);
CosmosClientBuilder clientBuilder = new CosmosClientBuilder(cosmosDbConnectionString.ServiceEndpoint.OriginalString, cosmosDbConnectionString.AuthKey);
CosmosClient client = clientBuilder
.WithConnectionModeDirect()
.Build();
CosmosDbService cosmosDbService = new CosmosDbService(client, databaseName, containerName);
return(cosmosDbService);
}
public static IServiceCollection AddCosmosDocumentClient(this IServiceCollection services, string connectionString)
{
services.TryAddSingleton <IDocumentClient>(provider =>
{
var cosmosDBConnectionString = new CosmosDbConnectionString(connectionString);
return(new DocumentClient(cosmosDBConnectionString.ServiceEndpoint, cosmosDBConnectionString.AuthKey,
new JsonSerializerSettings
{
ContractResolver = new CamelCasePropertyNamesContractResolver()
}));
});
return(services);
}
public override void Configure(IFunctionsHostBuilder builder)
{
// Add a new HttpClientFactory that can be injected into the functions.
// We add resilience and transient fault-handling capabilities to the HttpClient instances that the factory creates
// by adding a Polly Retry policy with a very brief back-off starting at quarter-of-a-second to two seconds.
// We want the HTTP requests that are sent to the downstream Logic App service to wait before attempting to try
// sending the message, giving it some "breathing room" in case the service is overwhelmed. We chose to make
// the time between retries relatively brief so as not to disrupt Cosmos DB message processing for too long, but
// enough time to hopefully allow the downstream service to recover.
// See the following for more information:
// https://docs.microsoft.com/en-us/dotnet/architecture/microservices/implement-resilient-applications/implement-http-call-retries-exponential-backoff-polly
builder.Services.AddHttpClient(NamedHttpClients.LogicAppClient, client =>
{
client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
})
.AddTransientHttpErrorPolicy(policyBuilder => policyBuilder.WaitAndRetryAsync(new[]
{
TimeSpan.FromMilliseconds(250),
TimeSpan.FromMilliseconds(500),
TimeSpan.FromMilliseconds(1000),
TimeSpan.FromMilliseconds(2000)
}));
// Register the Cosmos DB client as a Singleton.
builder.Services.AddSingleton((s) => {
var connectionString = configuration["CosmosDBConnection"];
var cosmosDbConnectionString = new CosmosDbConnectionString(connectionString);
if (string.IsNullOrEmpty(connectionString))
{
throw new ArgumentNullException("Please specify a value for CosmosDBConnection in the local.settings.json file or your Azure Functions Settings.");
}
CosmosClientBuilder configurationBuilder = new CosmosClientBuilder(cosmosDbConnectionString.ServiceEndpoint.OriginalString, cosmosDbConnectionString.AuthKey);
return(configurationBuilder
.Build());
});
// Create Azure Storage queue client reference and create any queues that do not yet exist, through the
// CreateKnownAzureQueues method.
builder.Services.Configure <AzureStorageSettings>(configuration);
StorageQueuesHelper.CreateKnownAzureQueues(configuration["ColdStorageAccount"]);
builder.Services.AddTransient <IQueueResolver, QueueResolver>();
}
/// <summary>
/// Adds <see cref="DistributedLock"/> to the services collection
/// </summary>
/// <param name="builder"></param>
/// <param name="configuration"></param>
public static void AddCosmosDistributedLock(this ContainerBuilder builder, IConfigurationRoot configuration)
{
var cosmosDataStoreOptions = Options.Create(configuration.BindSection <CosmosDataStoreOptions>(CosmosDataStoreOptions,
m => { m.AddMapping(x => x.ConnectionString, CosmosConnectionKeyVaultKey); }));
builder.Register(ctx => cosmosDataStoreOptions)
.As <IOptions <CosmosDataStoreOptions> >().SingleInstance();
var cosmosDbConnectionString = new CosmosDbConnectionString(cosmosDataStoreOptions.Value.ConnectionString);
var documentClient = new DocumentClient(cosmosDbConnectionString.ServiceEndpoint, cosmosDbConnectionString.AuthKey);
builder.Register(ctx =>
new CosmosDistributedLockStore(
documentClient,
cosmosDataStoreOptions,
ctx.Resolve <IBigBrother>())).As <ICosmosDistributedLockStore>().SingleInstance();
builder.RegisterType <DistributedLock>().As <IDistributedLock>();
}
/// <summary>
/// Retrieves a Cosmos DB database and a container with the specified partition key.
/// </summary>
/// <returns></returns>
private static CosmosDbService InitializeCosmosClientInstance(IConfiguration configuration)
{
var databaseName = configuration["DatabaseName"];
var containerName = configuration["ContainerName"];
var alertsContainerName = configuration["AlertsContainerName"];
var connectionString = configuration["CosmosDBConnection"];
var cosmosDbConnectionString = new CosmosDbConnectionString(connectionString);
// Define the container name collection. The first container name listed is the default container
// the Cosmos DB service class uses if no container name is provided within its methods.
var containerNames = new List <string> {
containerName, alertsContainerName
};
CosmosClientBuilder clientBuilder = new CosmosClientBuilder(cosmosDbConnectionString.ServiceEndpoint.OriginalString, cosmosDbConnectionString.AuthKey);
CosmosClient client = clientBuilder
.WithConnectionModeDirect()
.Build();
CosmosDbService cosmosDbService = new CosmosDbService(client, databaseName, containerNames);
return(cosmosDbService);
}
/// <summary>
/// Uses the vehicle telemetry contained in Cosmos DB to train the Anomaly Detector model.
/// </summary>
private static async Task <bool> TrainAnomalyDetectorModel(AppConfig appConfig)
{
var anomalyDetectorTrainUri = $"{appConfig.AnomalyDetector.Endpoint}anomalydetector/v1.0/timeseries/entire/detect";
var timeSeriesData = new Series
{
maxAnomalyRatio = 0.25F,
sensitivity = 95,
granularity = "minutely"
};
// Retrieve data from Cosmos DB
var cosmosDbConnectionString = new CosmosDbConnectionString(appConfig.CosmosDb.ConnectionString);
// Set the Cosmos DB connection policy.
var connectionPolicy = new ConnectionPolicy
{
ConnectionMode = ConnectionMode.Direct,
ConnectionProtocol = Protocol.Tcp
};
using (var cosmosClient = new DocumentClient(cosmosDbConnectionString.ServiceEndpoint, cosmosDbConnectionString.AuthKey, connectionPolicy))
{
var containerUri = UriFactory.CreateDocumentCollectionUri(appConfig.CosmosDb.DatabaseId, appConfig.CosmosDb.ContainerId);
var query = cosmosClient.CreateDocumentQuery <EngineTempRecord>(containerUri.ToString(),
new Documents.SqlQuerySpec("SELECT TOP 5000 c.engineTemperature FROM c"),
new FeedOptions {
MaxItemCount = -1, EnableCrossPartitionQuery = true
})
.ToList();
var models = new List <AnomalyModel>();
var startDateTime = DateTime.Now.AddDays(-6);
var i = 0;
foreach (var temp in query)
{
models.Add(new AnomalyModel {
timestamp = startDateTime.AddMinutes(i), value = temp.engineTemperature
});
i++;
}
timeSeriesData.series = models;
}
var requestBody = JsonConvert.SerializeObject(timeSeriesData);
// Send the training request to the Anomaly Detector batch detect endpoint.
using (var client = new HttpClient())
{
using (var request = new HttpRequestMessage())
{
request.Method = HttpMethod.Post;
request.RequestUri = new Uri(anomalyDetectorTrainUri);
request.Content = new StringContent(requestBody, Encoding.UTF8, "application/json");
request.Headers.Add("Ocp-Apim-Subscription-Key", appConfig.AnomalyDetector.Key);
var response = await client.SendAsync(request).ConfigureAwait(false);
var responseBody = await response.Content.ReadAsStringAsync();
Console.ForegroundColor = response.IsSuccessStatusCode ? ConsoleColor.Green : ConsoleColor.Red;
Console.WriteLine(responseBody);
Console.ResetColor();
return(response.IsSuccessStatusCode);
}
}
}
static async Task Main(string[] args)
{
// Setup configuration to either read from the appsettings.json file (if present) or environment variables.
var builder = new ConfigurationBuilder()
.SetBasePath(Directory.GetCurrentDirectory())
.AddJsonFile("appsettings.json", optional: true, reloadOnChange: true);
_configuration = builder.Build();
var appConfig = new AppConfig();
_configuration.Bind(appConfig);
var statistics = new Statistic[0];
var progress = new Progress <Progress>();
progress.ProgressChanged += (sender, progressArgs) =>
{
foreach (var message in progressArgs.Messages)
{
WriteLineInColor(message.Message, message.Color.ToConsoleColor());
}
statistics = progressArgs.Statistics;
};
while (true)
{
Console.WriteLine("Data Generator sends generated data into Cosmos DB.");
Console.WriteLine("** Enter 1 to generate tweets.");
Console.WriteLine("** Enter 2 to generate vehicle telemetry.");
Console.WriteLine("** Enter X to exit the console application.");
Console.WriteLine("");
var userInput = "";
while (true)
{
Console.Write("Enter the number of the operation you would like to perform > ");
var input = Console.ReadLine();
if (input.Equals("1", StringComparison.InvariantCultureIgnoreCase) ||
input.Equals("2", StringComparison.InvariantCultureIgnoreCase) ||
input.Equals("X", StringComparison.InvariantCultureIgnoreCase))
{
userInput = input.Trim();
break;
}
else
{
Console.WriteLine("Invalid input entered. Please enter either 1, 2 or X.");
}
}
ThreadPool.SetMinThreads(100, 100);
// Set an optional timeout for the generator.
var cancellationSource = appConfig.CosmosDb.MillisecondsToRun == 0 ? new CancellationTokenSource() : new CancellationTokenSource(appConfig.CosmosDb.MillisecondsToRun);
var cancellationToken = cancellationSource.Token;
// Handle Control+C or Control+Break.
Console.CancelKeyPress += (o, e) =>
{
WriteLineInColor("Stopped generator. No more data is being sent.", ConsoleColor.Yellow);
cancellationSource.Cancel();
// Allow the main thread to continue and exit...
WaitHandle.Set();
OutputStatistics(statistics);
};
Console.WriteLine(string.Empty);
Console.WriteLine("======");
WriteLineInColor("Press Ctrl+C or Ctrl+Break to cancel.", ConsoleColor.Cyan);
Console.WriteLine(string.Empty);
var dataType = "";
var cosmosDbConnectionString = new CosmosDbConnectionString(appConfig.CosmosDb.ConnectionString);
// Set the Cosmos DB connection policy.
var connectionPolicy = new ConnectionPolicy
{
ConnectionMode = ConnectionMode.Direct,
ConnectionProtocol = Protocol.Tcp
};
using (_cosmosDbClient = new DocumentClient(cosmosDbConnectionString.ServiceEndpoint, cosmosDbConnectionString.AuthKey, connectionPolicy))
{
switch (userInput.ToLower())
{
case "1":
await GenerateTweets(appConfig, cancellationToken, progress);
dataType = "Tweet";
break;
case "2":
await GenerateVehicleTelemetry(appConfig, cancellationToken, progress);
dataType = "Vehicle Telemetry";
break;
default:
// Exit the application
cancellationSource.Cancel();
return;
}
}
Console.WriteLine();
WriteLineInColor($"Done sending generated {dataType} data", ConsoleColor.Cyan);
Console.WriteLine();
Console.WriteLine();
OutputStatistics(statistics);
}
}
