internal static async Task ExampleAsync(NameValueCollection settings)
{
const int pooledSessions = 3;
ServicePointManager.DefaultConnectionLimit = pooledSessions * 3;
using (var client = new HdInsightClient(settings["ClusterName"], settings["ClusterUsername"], settings["ClusterPassword"]))
using (var cosmos = new CosmosDbLivyObjectPooledSession(client, CosmosSettings.GetSettings(settings), CosmosExampleSessionConfiguration.GetConfiguration(), pooledSessions))
{
const string sql = "SELECT contactIdentifier AS ContactIdentifier, COUNT(*) AS Count FROM cosmos GROUP BY contactIdentifier ORDER BY COUNT(*) DESC LIMIT 20";
var tasks = new[]
{
cosmos.QuerySparkSqlAsync <Result>(sql),
cosmos.QuerySparkSqlAsync <Result>(sql),
cosmos.QuerySparkSqlAsync <Result>(sql),
cosmos.QuerySparkSqlAsync <Result>(sql),
cosmos.QuerySparkSqlAsync <Result>(sql),
cosmos.QuerySparkSqlAsync <Result>(sql),
cosmos.QuerySparkSqlAsync <Result>(sql),
cosmos.QuerySparkSqlAsync <Result>(sql),
cosmos.QuerySparkSqlAsync <Result>(sql),
cosmos.QuerySparkSqlAsync <Result>(sql)
};
var results = await Task.WhenAll(tasks);
results.SelectMany(r => r).ToList().ForEach(t => Console.WriteLine($"{t.ContactIdentifier}:{t.Count}"));
}
Console.ReadKey();
}
internal static async Task ExampleAsync(NameValueCollection settings)
{
const int pooledSessions = 20;
ServicePointManager.DefaultConnectionLimit = pooledSessions * 3;
using (var client = new HdInsightClient(settings["ClusterName"], settings["ClusterUsername"], settings["ClusterPassword"]))
using (var cosmos = new CosmosDbLivyObjectPooledSession(client, CosmosSettings.GetSettings(settings), CosmosExampleSessionConfiguration.GetConfiguration(), pooledSessions))
{
await CloseAllSessions(client);
await InitializeAllPoolSessionsAsync(cosmos, pooledSessions);
const string baseQuery = "SELECT c.contactIdentifier, c.json.products FROM c";
const string sql = "SELECT * FROM cosmos LIMIT 1";
var query = (Func <Task <IEnumerable <dynamic> > >)(() => cosmos.QuerySparkSqlAsync <dynamic>(sql, baseQuery));
var tasks = new List <(int index, Task <(TimeSpan Elapsed, IEnumerable <dynamic> Result)> task)>();
for (var i = 0; i < 20; i++)
{
tasks.Add((i, Task.Run(async() =>
{
try
{
return(await query.RunAndTimeExecution());
}
catch (Exception e)
{
Console.WriteLine(e.Message);
return(TimeSpan.Zero, Enumerable.Empty <dynamic>());
}
})));
await Task.Delay(200); // 300/min => 5/sec => 1/200ms
}
await tasks.Select(t => t.task);
tasks.Select(t => (t.index, t.task.Result.Elapsed))
.OrderBy(t => t.Item1)
.Select(t => $"[{t.Item2}] - {t.Item1}")
.ToList()
.ForEach(Console.WriteLine);
Console.WriteLine("Press any key to dispose the sessions...");
Console.ReadKey();
}
Console.WriteLine("Press any key to close...");
Console.ReadKey();
}
internal static async Task ExampleAsync(NameValueCollection settings)
{
using (var client = new HdInsightClient(settings["ClusterName"], settings["ClusterUsername"], settings["ClusterPassword"]))
using (var cosmos = new CosmosDbLivySession(client, CosmosSettings.GetSettings(settings), CosmosExampleSessionConfiguration.GetConfiguration()))
{
const string sql = "SELECT contactIdentifier AS ContactIdentifier, COUNT(*) AS Count FROM cosmos GROUP BY contactIdentifier ORDER BY COUNT(*) DESC LIMIT 20";
var results = await cosmos.QuerySparkSqlAsync <Result>(sql);
results.ToList().ForEach(t => Console.WriteLine($"{t.ContactIdentifier}:{t.Count}"));
}
Console.ReadKey();
}
internal static async Task ExampleAsync(NameValueCollection settings)
{
using (var client = new HdInsightClient(settings["ClusterName"], settings["ClusterUsername"], settings["ClusterPassword"]))
using (var cosmos = new CosmosDbLivySession(client, CosmosSettings.GetSettings(settings), CosmosExampleSessionConfiguration.GetConfiguration()))
{
const string sparkSql = "SELECT contactIdentifier AS Item1, COUNT(*) AS Item2 FROM cosmos GROUP BY contactIdentifier ORDER BY COUNT(*) DESC LIMIT 20";
const string cosmosSqlQuery = "SELECT cosmos.contactIdentifier FROM cosmos";
// Create session and warm up
await cosmos.QuerySparkSqlAsync <(int, decimal)>(sparkSql, cosmosSqlQuery);
var query = (Func <Task <IEnumerable <(int, decimal)> > >)(() => cosmos.QuerySparkSqlAsync <(int, decimal)>(sparkSql, cosmosSqlQuery));
var result = await query.RunAndTimeExecution();
Console.WriteLine($"Elpsed: {result.Elapsed}");
result.Result.ToList().ForEach(t => Console.WriteLine($"{t.Item1}:{t.Item2}"));
}
Console.ReadKey();
}
internal static async Task ExampleAsync(NameValueCollection settings)
{
const int pooledSessions = 5;
ServicePointManager.DefaultConnectionLimit = pooledSessions * 3;
using (var client = new HdInsightClient(settings["ClusterName"], settings["ClusterUsername"], settings["ClusterPassword"]))
using (var cosmos = new CosmosDbLivyObjectPooledSession(client, CosmosSettings.GetSettings(settings), CosmosExampleSessionConfiguration.GetConfiguration(), pooledSessions))
{
const string baseQuery = "SELECT c.contactIdentifier, c.json FROM c";
const string sql = @"
SELECT contactIdentifier, SUM(price)
FROM cosmos
LATERAL VIEW explode(json.products.price) tab AS price
GROUP BY contactIdentifier
HAVING SUM(price) > 5000.00
";
var tasks = new[]
{
cosmos.QuerySparkSqlAsync <Result>(sql, baseQuery),
cosmos.QuerySparkSqlAsync <Result>(sql, baseQuery),
cosmos.QuerySparkSqlAsync <Result>(sql, baseQuery),
cosmos.QuerySparkSqlAsync <Result>(sql, baseQuery),
cosmos.QuerySparkSqlAsync <Result>(sql, baseQuery),
cosmos.QuerySparkSqlAsync <Result>(sql, baseQuery),
cosmos.QuerySparkSqlAsync <Result>(sql, baseQuery),
cosmos.QuerySparkSqlAsync <Result>(sql, baseQuery),
cosmos.QuerySparkSqlAsync <Result>(sql, baseQuery),
cosmos.QuerySparkSqlAsync <Result>(sql, baseQuery)
};
var results = await Task.WhenAll(tasks);
results.SelectMany(r => r).ToList().ForEach(t => Console.WriteLine($"{t.ContactIdentifier}:{t.Count}"));
}
Console.ReadKey();
}