/// <summary>
/// Parse results from exchange API (Ticker format) to BsonDocument and add it to a list.
/// </summary>
/// <param name="task">Bulk of RestApi.GetTickerHistory tasks to run.</param>
/// <param name="inputs">Input parameters for RestApi.GetTickerHistory</param>
/// <returns>List of BsonDocuments and List of all exceptions occured.</returns>
private (List <BsonDocument> bson, Exception ex) GetListBson(Task <List <Ticker> > task, TickerHistoryParameters inputs)
{
string sym = inputs.symbol, per = inputs.period.ToKey();
Exception res_exception = null;
List <BsonDocument> bson_format = new List <BsonDocument>();
if (task.IsCompletedSuccessfully)
{
try
{
bson_format = (from elt in task.Result
where elt.Timestamp > _last_ts.GetValueOrDefault(Tuple.Create(sym, per))
select new
{
insts = DateTime.UtcNow,
ts = (UInt64)elt.Timestamp,
ccy = sym,
period = per,
o = elt.Open,
c = elt.Close,
h = elt.High,
l = elt.Low,
vol = elt.Vol,
amt = elt.Amount,
ct = elt.Count
}.ToBsonDocument()).ToList();
}
catch (Exception ex)
{
res_exception = new Exception($"Error while parsing Ticker object to anonymous object to BsonDocument: {ex.Message}.");
}
}
else
{
res_exception = new Exception($"Error while calling exchange API with parameters ({sym}, {per}, {inputs.size}). Task status: {task.Status}. " +
$"Task exception: {task.Exception.Message}");
}
return(bson : bson_format, ex : res_exception);
}
/// <summary>
/// Request data from exchange API and add them to CosmosDB (NoSql MongoDB).
/// </summary>
/// <param name="requests_info">Dictionary of requested crypto currencies and their timeframe.</param>
/// <returns>The execution message and the list of all exceptions to log.</returns>
public (string message, IEnumerable <Exception> errors) Run(Dictionary <string, List <TickerPeriod> > requests_info)
{
int chunksize = exchange_api_parallel_tasks, pos = 0;
// Initialize the list of NoSql requests to send in one time
var bson_format_stack = new ConcurrentStack <BsonDocument>();
var exceptions_stack = new ConcurrentStack <Exception>();
try
{
if (chunksize <= 0)
{
throw new ArgumentOutOfRangeException("chunksize");
}
if (pos < 0)
{
throw new ArgumentOutOfRangeException("pos");
}
// Compute timestamp and diff
var elapsed_time = DateTime.UtcNow - new DateTime(1970, 1, 1, 0, 0, 0, 0);
var ts_today = BigInteger.Parse(elapsed_time.TotalSeconds.ToString().Split(".")[0]);
// Get all parameters for RestApi.GetTickerHistory
var requests = GetRequestsInfos(requests_info ?? new Dictionary <string, List <TickerPeriod> >(), ts_today);
// Create ConcurrentStack for all requests parameters
var all_param_stack = new ConcurrentStack <TickerHistoryParameters>(requests);
log("{0} exchange requests to be sent.", all_param_stack.Count);
// Execute async requests in chunks because the exchange API does not accept many requests at once
int previous_count = all_param_stack.Count, count = 1, max_retry_count = exchange_api_max_retry_count;
while (!all_param_stack.IsEmpty && count <= max_retry_count)
{
int n = Math.Min(chunksize, all_param_stack.Count);
var sub_list = new TickerHistoryParameters[n];
var nb_elt = all_param_stack.TryPopRange(sub_list, 0, n);
if (nb_elt > 0)
{
var sub_tasks = sub_list.Select(elt => RestApi.GetTickerHistory(elt.symbol, elt.period, elt.size)).ToArray();
try
{
Task.WaitAll(sub_tasks);
}
catch
{ }
// Fill in bson_format_stack and exceptions_stack in parallel
var parallel_result = Parallel.For(0, sub_tasks.Count(), i =>
{
if (sub_tasks[i].IsCompletedSuccessfully || count == max_retry_count)
{
// Get list of BsonDocument for current bulk
var(bson, ex) = GetListBson(sub_tasks[i], sub_list[i]);
// Concatenate all BsonDocuments
if (bson.Count > 0)
{
bson_format_stack.PushRange(bson.ToArray());
}
// Concatenate all Exceptions
if (ex != null)
{
exceptions_stack.Push(ex);
}
}
else
{
all_param_stack.Push(sub_list[i]);
}
});
}
count = all_param_stack.Count != previous_count ? 1 : count + 1;
previous_count = all_param_stack.Count;
}
var total_elts = bson_format_stack.Count;
InsertQuotations(bson_format_stack);
if (exceptions_stack.Count <= 0)
{
return(message : $"Success: {total_elts} new documents inserted.", errors : null);
}
else
{
return(message : "Partially Failed", errors : exceptions_stack);
}
}
catch (Exception ex)
{
exceptions_stack.Push(ex);
return(message : "Fail", errors : exceptions_stack);
}
}