/// <summary> /// Parse results from exchange API (Ticker format) to BsonDocument and add it to a list. /// </summary> /// <param name="task">Bulk of RestApi.GetTickerHistory tasks to run.</param> /// <param name="inputs">Input parameters for RestApi.GetTickerHistory</param> /// <returns>List of BsonDocuments and List of all exceptions occured.</returns> private (List <BsonDocument> bson, Exception ex) GetListBson(Task <List <Ticker> > task, TickerHistoryParameters inputs) { string sym = inputs.symbol, per = inputs.period.ToKey(); Exception res_exception = null; List <BsonDocument> bson_format = new List <BsonDocument>(); if (task.IsCompletedSuccessfully) { try { bson_format = (from elt in task.Result where elt.Timestamp > _last_ts.GetValueOrDefault(Tuple.Create(sym, per)) select new { insts = DateTime.UtcNow, ts = (UInt64)elt.Timestamp, ccy = sym, period = per, o = elt.Open, c = elt.Close, h = elt.High, l = elt.Low, vol = elt.Vol, amt = elt.Amount, ct = elt.Count }.ToBsonDocument()).ToList(); } catch (Exception ex) { res_exception = new Exception($"Error while parsing Ticker object to anonymous object to BsonDocument: {ex.Message}."); } } else { res_exception = new Exception($"Error while calling exchange API with parameters ({sym}, {per}, {inputs.size}). Task status: {task.Status}. " + $"Task exception: {task.Exception.Message}"); } return(bson : bson_format, ex : res_exception); }
/// <summary> /// Request data from exchange API and add them to CosmosDB (NoSql MongoDB). /// </summary> /// <param name="requests_info">Dictionary of requested crypto currencies and their timeframe.</param> /// <returns>The execution message and the list of all exceptions to log.</returns> public (string message, IEnumerable <Exception> errors) Run(Dictionary <string, List <TickerPeriod> > requests_info) { int chunksize = exchange_api_parallel_tasks, pos = 0; // Initialize the list of NoSql requests to send in one time var bson_format_stack = new ConcurrentStack <BsonDocument>(); var exceptions_stack = new ConcurrentStack <Exception>(); try { if (chunksize <= 0) { throw new ArgumentOutOfRangeException("chunksize"); } if (pos < 0) { throw new ArgumentOutOfRangeException("pos"); } // Compute timestamp and diff var elapsed_time = DateTime.UtcNow - new DateTime(1970, 1, 1, 0, 0, 0, 0); var ts_today = BigInteger.Parse(elapsed_time.TotalSeconds.ToString().Split(".")[0]); // Get all parameters for RestApi.GetTickerHistory var requests = GetRequestsInfos(requests_info ?? new Dictionary <string, List <TickerPeriod> >(), ts_today); // Create ConcurrentStack for all requests parameters var all_param_stack = new ConcurrentStack <TickerHistoryParameters>(requests); log("{0} exchange requests to be sent.", all_param_stack.Count); // Execute async requests in chunks because the exchange API does not accept many requests at once int previous_count = all_param_stack.Count, count = 1, max_retry_count = exchange_api_max_retry_count; while (!all_param_stack.IsEmpty && count <= max_retry_count) { int n = Math.Min(chunksize, all_param_stack.Count); var sub_list = new TickerHistoryParameters[n]; var nb_elt = all_param_stack.TryPopRange(sub_list, 0, n); if (nb_elt > 0) { var sub_tasks = sub_list.Select(elt => RestApi.GetTickerHistory(elt.symbol, elt.period, elt.size)).ToArray(); try { Task.WaitAll(sub_tasks); } catch { } // Fill in bson_format_stack and exceptions_stack in parallel var parallel_result = Parallel.For(0, sub_tasks.Count(), i => { if (sub_tasks[i].IsCompletedSuccessfully || count == max_retry_count) { // Get list of BsonDocument for current bulk var(bson, ex) = GetListBson(sub_tasks[i], sub_list[i]); // Concatenate all BsonDocuments if (bson.Count > 0) { bson_format_stack.PushRange(bson.ToArray()); } // Concatenate all Exceptions if (ex != null) { exceptions_stack.Push(ex); } } else { all_param_stack.Push(sub_list[i]); } }); } count = all_param_stack.Count != previous_count ? 1 : count + 1; previous_count = all_param_stack.Count; } var total_elts = bson_format_stack.Count; InsertQuotations(bson_format_stack); if (exceptions_stack.Count <= 0) { return(message : $"Success: {total_elts} new documents inserted.", errors : null); } else { return(message : "Partially Failed", errors : exceptions_stack); } } catch (Exception ex) { exceptions_stack.Push(ex); return(message : "Fail", errors : exceptions_stack); } }