public async Task <SquashedEntityResult[]> SquashAsync(
EntityReference[] entityReferences,
AggregatesRequest aggregatesRequest,
string[] fields,
bool live,
DateTime?pointInTime,
Profile profile,
CancellationToken cancellationToken)
{
var entityProfile = profile.Entities.SingleOrDefault(x => x.Name.Equals(typeof(T).Name, StringComparison.InvariantCultureIgnoreCase));
if (entityProfile == null)
{
throw new ProfileMisconfiguredException(profile.Name, typeof(T));
}
_logger.Info($"Getting {entityReferences.Length} references from adapters");
var adapterResults = await GetEntitiesFromAdapters(entityReferences, aggregatesRequest, fields, live, pointInTime, cancellationToken);
_logger.Info($"Collating {adapterResults.Count} adapter results");
var candidates = CollateAdapterResults(entityReferences, adapterResults);
var squashed = Squash(candidates, aggregatesRequest, fields, entityProfile);
return(squashed);
}
private void SquashAggregationsForEntity(
PropertyInfo property,
T entity,
SourceSystemEntity <T>[] nonErroredSources,
string[] sources,
AggregatesRequest aggregatesRequest)
{
var aggregations = new List <Aggregation>();
foreach (var aggregationName in aggregatesRequest.AggregateQueries.Keys)
{
var candidateAggregations = nonErroredSources
.Where(c => sources.Any(s => s.Equals(c.SourceName, StringComparison.InvariantCultureIgnoreCase)))
.SelectMany(c => (Aggregation[])property.GetValue(c.Entity))
.Where(x => x != null && x.Name.Equals(aggregationName, StringComparison.InvariantCultureIgnoreCase))
.ToArray();
// Currently only supports count. Though this should work for others such as sum
var value = candidateAggregations.Sum(x => x.Value);
aggregations.Add(new Aggregation
{
Name = aggregationName,
Value = value,
});
}
property.SetValue(entity, aggregations.ToArray());
}
private AggregatesRequest DeserializeAggregationRequests <TContext>(
ResolveFieldContext <TContext> context)
{
var aggregations = context.FieldAst.SelectionSet.Selections
.Select(x => (Field)x)
.SingleOrDefault(f => f.Name == "_aggregations");
if (aggregations == null)
{
return(null);
}
var definitionsArgument = aggregations.Arguments.SingleOrDefault(a => a.Name == "definitions");
if (definitionsArgument == null)
{
return(null);
}
var definitions = (List <object>)definitionsArgument.Value.Value;
var request = new AggregatesRequest();
foreach (Dictionary <string, object> definition in definitions)
{
var name = (string)definition["name"];
var conditions = (List <object>)definition["conditions"];
var dataFilters = new List <DataFilter>();
foreach (Dictionary <string, object> condition in conditions)
{
var conditionOperator = DataOperator.Equals;
if (condition.ContainsKey("operator"))
{
var specifiedOperator = ((string)condition["operator"]).Replace("_", "").ToUpper();
if (DataOperators.ContainsKey(specifiedOperator))
{
conditionOperator = DataOperators[specifiedOperator];
}
}
dataFilters.Add(new DataFilter
{
Field = (string)condition["field"],
Operator = conditionOperator,
Value = (string)condition["value"],
});
}
request.AggregateQueries.Add(name, new AggregateQuery {
DataFilters = dataFilters.ToArray()
});
}
return(request);
}
public async Task Update()
{
int period = IsReady ? 1 : Period;
var aggregatesRequest = new AggregatesRequest(Symbol, AggregationPeriod);
var from = RunManager.TradingCalendars.SubtractTradingDays(period);
aggregatesRequest.SetInclusiveTimeInterval(from, DateTime.UtcNow);
var history = await RunManager.PolygonDataClient.ListAggregatesAsync(aggregatesRequest);
foreach (var agg in history.Items)
{
queue.Enqueue(agg);
}
}
private RequestParameters GetRequestParameters(int numberOfEntities = 1, bool forceBothSources = false)
{
var entityReferences = new EntityReference[numberOfEntities];
for (var i = 0; i < entityReferences.Length; i++)
{
var sourceSelection = forceBothSources ? 3 : _random.Next(1, 3); // 1=Source1, 2=Source2, 3=Both
entityReferences[i] = GetEntityReference(sourceSelection == 1 || sourceSelection == 3, sourceSelection == 2 || sourceSelection == 3);
}
AggregatesRequest aggregatesRequest = null;
var fields = new string[0];
var live = _random.Next(0, 100) >= 50;
var daysAgo = _random.Next(0, 100);
var pointInTime = daysAgo < 10 ? null : (DateTime?)DateTime.Today.AddDays(-daysAgo);
var profile = new Profile
{
Name = $"ProfileName-{Guid.NewGuid().ToString()}",
Entities = new[]
{
new EntityProfile
{
Name = nameof(TestEntity),
Sources = new[] { "Source1", "Source2" },
Fields = new[]
{
new EntityFieldProfile {
Name = "Name"
},
new EntityFieldProfile {
Name = "Address"
},
new EntityFieldProfile {
Name = "NumberOfPupils"
},
new EntityFieldProfile {
Name = "IsOpen"
},
new EntityFieldProfile {
Name = "OpenDate"
},
new EntityFieldProfile {
Name = "ContactNumbers"
},
}
},
}
};
return(new RequestParameters
{
EntityReferences = entityReferences,
AggregatesRequest = aggregatesRequest,
Fields = fields,
Live = live,
PointInTime = pointInTime,
Profile = profile,
});
}
private async Task <Dictionary <string, DataAdapterResult <T>[]> > GetEntitiesFromAdapters(
EntityReference[] entityReferences,
AggregatesRequest aggregatesRequest,
string[] fields,
bool live,
DateTime?pointInTime,
CancellationToken cancellationToken)
{
// Split request for adapters
var adapterReferences =
entityReferences
.SelectMany(er => er.AdapterRecordReferences)
.GroupBy(ar => ar.SourceSystemName.ToUpper())
.Select(grp => new
{
SourceName = grp.Key,
Identifiers = grp
.Where(ar => !string.IsNullOrEmpty(ar.SourceSystemId))
.Select(ar => ar.SourceSystemId)
.Distinct()
.ToArray(),
})
.ToArray();
// Start calls to adapters
var tasks = new Task <DataAdapterResult <T>[]> [adapterReferences.Length];
for (var i = 0; i < adapterReferences.Length; i++)
{
var adapterName = adapterReferences[i].SourceName;
var adapterIdentifiers = adapterReferences[i].Identifiers;
var adapter = _adapters.SingleOrDefault(a => a.SourceName.Equals(adapterName, StringComparison.InvariantCultureIgnoreCase));
if (adapter == null)
{
tasks[i] = Task.FromException <DataAdapterResult <T>[]>(
new DataAdapterException($"Cannot find adapter for {adapterName}")
{
AdapterName = adapterName,
RequestedIds = adapterIdentifiers,
RequestedFields = fields,
});
continue;
}
tasks[i] = adapter.GetEntitiesAsync(adapterIdentifiers, aggregatesRequest?.AggregateQueries, fields, live, pointInTime, cancellationToken);
}
// Collate results from adapters
var results = new Dictionary <string, DataAdapterResult <T>[]>();
for (var i = 0; i < adapterReferences.Length; i++)
{
var adapterName = adapterReferences[i].SourceName;
var task = tasks[i];
DataAdapterResult <T>[] adapterResults;
try
{
adapterResults = await task;
}
catch (DataAdapterException ex)
{
_logger.Warning($"{adapterName} adapter returned error status {(int) ex.HttpStatusCode}:\n{ex.HttpErrorBody}");
var adapterIdentifiers = adapterReferences[i].Identifiers;
adapterResults = adapterIdentifiers
.Select(id =>
new DataAdapterResult <T>
{
Identifier = id,
AdapterError = ex,
})
.ToArray();
}
catch (Exception ex)
{
_logger.Warning($"{adapterName} adapter encountered an unexpected error: {ex.Message}");
var adapterIdentifiers = adapterReferences[i].Identifiers;
adapterResults = adapterIdentifiers
.Select(id =>
new DataAdapterResult <T>
{
Identifier = id,
AdapterError = new DataAdapterException("Unexpected error calling adapter", ex)
{
AdapterName = adapterName,
RequestedIds = adapterIdentifiers,
RequestedFields = fields,
},
})
.ToArray();
}
results.Add(adapterName, adapterResults);
}
return(results);
}
private SquashedEntityResult[] Squash(EntityReferenceSourceData <T>[] candidates, AggregatesRequest aggregatesRequest, string[] fields,
EntityProfile entityProfile)
{
var results = new SquashedEntityResult[candidates.Length];
var requiredProperties = fields.Length == 0
? _typeProperties
: _typeProperties
.Where(p => fields.Any(f => f.Equals(p.Name, StringComparison.InvariantCultureIgnoreCase)))
.ToArray();
_logger.Info($"Squashing {candidates.Length} entities with {requiredProperties.Length} properties");
_logger.Debug("Required properties: " + requiredProperties.Select(x => x.Name).Aggregate((x, y) => $"{x}, {y}"));
for (var i = 0; i < candidates.Length; i++)
{
var candidate = candidates[i];
var nonErroredSources = candidate.SourceEntities.Where(e => e.AdapterError == null).ToArray();
T entity = null;
if (nonErroredSources.Length > 0)
{
entity = new T();
var entityBase = entity as EntityBase;
var isLineageRequired = entity != null &&
(fields.Length == 0 || fields.Any(x => x.Equals("_lineage", StringComparison.InvariantCultureIgnoreCase)));
if (isLineageRequired)
{
entityBase._Lineage = new Dictionary <string, LineageEntry>();
}
_logger.Debug($"Squashing {candidate.EntityReference} {(isLineageRequired ? "with" : "without")} lineage");
foreach (var property in requiredProperties)
{
var fieldProfile = entityProfile.Fields.SingleOrDefault(f => f.Name.Equals(property.Name, StringComparison.InvariantCultureIgnoreCase));
var sources = fieldProfile?.Sources != null && fieldProfile.Sources.Length > 0
? fieldProfile.Sources
: entityProfile.Sources;
var treatWhitespaceAsNull = fieldProfile?.TreatWhitespaceAsNull ?? false;
if (property.Name.Equals("_aggregations", StringComparison.InvariantCultureIgnoreCase))
{
SquashAggregationsForEntity(property, entity, nonErroredSources, sources, aggregatesRequest);
}
else
{
SquashPropertyForEntity(property, entity, entityBase, nonErroredSources, sources, treatWhitespaceAsNull, isLineageRequired);
}
}
}
results[i] = new SquashedEntityResult
{
EntityReference = candidate.EntityReference,
SquashedEntity = entity,
EntityAdapterErrorDetails = candidate.SourceEntities
.Where(e => e.AdapterError != null)
.Select(e =>
new EntityAdapterErrorDetail
{
AdapterName = e.SourceName,
RequestedId = e.SourceId,
RequestedFields = fields,
RequestedEntityName = e.AdapterError.RequestedEntityName,
HttpStatusCode = e.AdapterError.HttpStatusCode,
HttpErrorBody = e.AdapterError.HttpErrorBody
})
.ToArray(),
};
}
return(results);
}
