//---------------------------------------------------------------------------------------- internal static object GetCbnTimeSeriesDataCell(string id, string type, DateTime date, string field, object[] filterValues = null, bool throwException = false) //---------------------------------------------------------------------------------------- { var client = GetCarbonClient(); try { if (id == null) throw new ArgumentNullException("id"); if (type == null) throw new ArgumentNullException("type"); var filterValueStrings = filterValues != null ? filterValues.Select(filterValue => filterValue.ToString()).ToArray() : null; var option = new TimeSeriesOptions { Fields = new[] { field } }; return (client.GetTimeSeriesAsync(id, type,date,date, option) .ContinueWith(r => GetValueFromTimeSeriesList(r.Result, field, filterValueStrings)).Result); } catch (Exception ex) { SLog.log.ErrorFormat("Error when getting time series data cell: {0}", ex); if (throwException) throw; } return null; }
public CarbonTimeSeriesRequest(IReadOnlyList<string> ids, string type, DateTime startDate, DateTime endDate, TimeSeriesOptions option) { mIds = ids; mType = type; mStartDate = startDate; mEndDate = endDate; mOption = option; }
public IUnifiedEntityTestOperation Build(string targetDatabaseId, BsonDocument arguments) { var database = _entityMap.GetDatabase(targetDatabaseId); string name = null; string viewOn = null; PipelineDefinition <BsonDocument, BsonDocument> pipeline = null; IClientSessionHandle session = null; TimeSpan? expireAfter = null; TimeSeriesOptions timeSeriesOptions = null; ClusteredIndexOptions <BsonDocument> clusteredIndex = null; foreach (var argument in arguments) { switch (argument.Name) { case "clusteredIndex": var clusteredIndexSpecification = argument.Value.AsBsonDocument; clusteredIndex = new ClusteredIndexOptions <BsonDocument> { Key = clusteredIndexSpecification["key"].AsBsonDocument, Unique = clusteredIndexSpecification["unique"].AsBoolean, Name = clusteredIndexSpecification["name"].AsString }; break; case "collection": name = argument.Value.AsString; break; case "expireAfterSeconds": expireAfter = TimeSpan.FromSeconds(argument.Value.ToInt64()); break; case "pipeline": pipeline = new EmptyPipelineDefinition <BsonDocument>(); foreach (var stage in argument.Value.AsBsonArray) { pipeline = pipeline.AppendStage <BsonDocument, BsonDocument, BsonDocument>(stage.AsBsonDocument); } break; case "session": var sessionId = argument.Value.AsString; session = _entityMap.GetSession(sessionId); break; case "timeseries": var timeseries = argument.Value.AsBsonDocument; var timeField = timeseries["timeField"].AsString; var metaField = timeseries.TryGetValue("metaField", out var metaFieldValue) ? metaFieldValue.AsString : null; TimeSeriesGranularity?granularity = null; if (timeseries.TryGetValue("granularity", out var granularityValue)) { granularity = (TimeSeriesGranularity)Enum.Parse(typeof(TimeSeriesGranularity), granularityValue.AsString, true); } timeSeriesOptions = new TimeSeriesOptions(timeField, metaField, granularity); break; case "viewOn": viewOn = argument.Value.AsString; break; default: throw new FormatException($"Invalid CreateCollectionOperation argument name: '{argument.Name}'."); } } if (viewOn == null && pipeline == null) { var options = new CreateCollectionOptions <BsonDocument> { ExpireAfter = expireAfter, TimeSeriesOptions = timeSeriesOptions, ClusteredIndex = clusteredIndex }; return(new UnifiedCreateCollectionOperation(session, database, name, options)); } if (viewOn != null && expireAfter == null && timeSeriesOptions == null && clusteredIndex == null) { var options = new CreateViewOptions <BsonDocument>(); return(new UnifiedCreateViewOperation(session, database, name, viewOn, pipeline, options)); } var invalidArguments = string.Join(",", arguments.Elements.Select(x => x.Name)); throw new InvalidOperationException($"Invalid combination of CreateCollectionOperation arguments: {invalidArguments}"); }
private void AggregateRequests(IReadOnlyCollection<TimeSeriesCellResponseEvent> requests) { var dictByDate = new Dictionary<string, List<CarbonTimeSeriesCellRequest>>(); var dictById = new Dictionary<string, List<CarbonTimeSeriesCellRequest>>(); var dateKeys = new Dictionary<string, TimeSeriesCellRequestAggregateByDateKey>(); var idKeys = new Dictionary<string, TimeSeriesCellRequestAggregatByIdKey>(); foreach (var request in requests) { List<CarbonTimeSeriesCellRequest> byDateRequests; if (!dictByDate.TryGetValue(request.SeriesCellRequest.ByDateKey.Key, out byDateRequests)) { byDateRequests = new List<CarbonTimeSeriesCellRequest>(); dictByDate[request.SeriesCellRequest.ByDateKey.Key] = byDateRequests; dateKeys[request.SeriesCellRequest.ByDateKey.Key] = request.SeriesCellRequest.ByDateKey; } byDateRequests.Add(request.SeriesCellRequest); List<CarbonTimeSeriesCellRequest> byIdRequests; if (!dictById.TryGetValue(request.SeriesCellRequest.ByIdKey.Key, out byIdRequests)) { byIdRequests = new List<CarbonTimeSeriesCellRequest>(); dictById[request.SeriesCellRequest.ByIdKey.Key] = byIdRequests; idKeys[request.SeriesCellRequest.ByIdKey.Key] = request.SeriesCellRequest.ByIdKey; } byIdRequests.Add(request.SeriesCellRequest); } //SLog.log.InfoFormat("Conflate {0} cell requests to {1} by date requests to Carbon", requests.Count, dictByDate.Count); //SLog.log.InfoFormat("Conflate {0} cell requests to {1} by id requests to Carbon", requests.Count, dictById.Count); if (dictById.Count < dictByDate.Count) { // Request by id foreach (var kvp in dictById) { var max = ToUtc(kvp.Value.Max(r => r.Date)); var min = ToUtc(kvp.Value.Min(r => r.Date)); var option = new TimeSeriesOptions { Filters = idKeys[kvp.Key].Filters, Fields = kvp.Value.Select(v => v.Field).Concat(new[]{"date"}).Distinct().ToList() }; var request = new CarbonTimeSeriesRequest(new[] { idKeys[kvp.Key].Identifier }, idKeys[kvp.Key].RequestType, min, max, option); mRequestToDispatch.Add(new CarbonRequestHandle(request, requests)); //Task.Run(() => RequestDataFromCarbon(new CarbonRequestHandle(request, requests))); } } else { // Request by dates foreach (var kvp in dictByDate) { var option = new TimeSeriesOptions { Filters = dateKeys[kvp.Key].Filters, Fields = kvp.Value.Select(v => v.Field).Concat(new[] { "date" }).Distinct().ToList() }; var date = ToUtc(dateKeys[kvp.Key].Date); var request = new CarbonTimeSeriesRequest(kvp.Value.Select(v => v.Identifier).Distinct().ToList(), dateKeys[kvp.Key].RequestType, date, date, option); mRequestToDispatch.Add(new CarbonRequestHandle(request, requests)); //Task.Run(() => RequestDataFromCarbon(new CarbonRequestHandle(request, requests))); } } }
public void ShouldGetMinuteDataFromCarbon() { using (var carbonClient = new CarbonClient("PRD")) { var atTime = new TimeSpan(0, 14, 15); var options = new TimeSeriesOptions { AtTime = atTime }; // specify explicitly UTC time var start = new DateTime(2012, 4, 1, 0, 0, 0, DateTimeKind.Utc); var end = new DateTime(2016, 3, 1, 0, 0, 0, DateTimeKind.Utc); DateTime now = DateTime.Now; var intra = carbonClient.GetTimeSeriesAsync("FVH6 Comdty", "intra", start, end, options).Result; //var intra = carbonClient.GetTimeSeriesAsync(new[]{"FVU6 Comdty","FVM6 Comdty","FVH6 Comdty","FVZ15 Comdty","FVU15 Comdty","FVM15 Comdty","FVH15 Comdty"}, "intra", start, end, options).Result; Console.WriteLine(string.Format("get time series intra taken {0}s", (DateTime.Now - now).TotalSeconds)); now = DateTime.Now; intra = carbonClient.GetTimeSeriesAsync("FVZ15 Comdty", "intra", start, end, options).Result; //intra = carbonClient.GetTimeSeriesAsync(new[] { "FVU6 Comdty", "FVM6 Comdty", "FVH6 Comdty", "FVZ15 Comdty", "FVU15 Comdty", "FVM15 Comdty", "FVH15 Comdty" }, "intra", start, end, options).Result; Console.WriteLine(string.Format("get time series intra taken {0}s", (DateTime.Now - now).TotalSeconds)); now = DateTime.Now; intra = carbonClient.GetTimeSeriesAsync("FVH6 Comdty", "intra", start, end, options).Result; Console.WriteLine(string.Format("get time series intra taken {0}s", (DateTime.Now - now).TotalSeconds)); foreach (var row in intra) { var ticker = row.Identifier; foreach (var series in row.Series) { // these are nullable and only present if data was provided var bid = series.GetDouble("highBid"); var ask = series.GetDouble("highAsk"); var time = series.GetDateTime("timeStamp"); } } } }