// GET: RelatedEntities
public ActionResult Index()
{
DatabaseClient db = new DatabaseClient();
var REInfo = db.GetAllDocuments("related_entities");
List <RelatedEntities> results = new List <RelatedEntities>();
foreach (BsonDocument info in REInfo)
{
RelatedEntities relatedEntities = new RelatedEntities();
try
{
relatedEntities.Keyword = info["keyword"].AsString;
BsonArray companies = info["companies"].AsBsonArray;
foreach (BsonDocument c in companies)
{
relatedEntities.StockSymbols.Add(c["stockSymbol"].AsString);
relatedEntities.Occurrences.Add(c["occurrences"].AsInt32);
relatedEntities.Verified.Add(c["verified"].AsBoolean);
}
} catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
results.Add(relatedEntities);
}
ViewBag.Results = results;
return(View());
}
public async Task <ActionResult <EntitiesResponse <LineForSave> > > Generate([FromRoute] int lineDefId, [FromQuery] Dictionary <string, string> args, CancellationToken cancellation)
{
var serverTime = DateTimeOffset.UtcNow;
var(lines, accounts, resources, agents, entryTypes, centers, currencies, units) = await GetService().Generate(lineDefId, args, cancellation);
// Related entitiess
var relatedEntities = new RelatedEntities
{
Account = accounts,
Resource = resources,
Agent = agents,
EntryType = entryTypes,
Center = centers,
Currency = currencies,
Unit = units
};
// Prepare the result in a response object
var response = new EntitiesResponse <LineForSave>
{
Result = lines,
RelatedEntities = relatedEntities,
CollectionName = "", // Not important
ServerTime = serverTime,
};
// Return
return(Ok(response));
}
/// <summary>
/// Recursive function that parses a single data row (string array) into 0 or more entities that are each
/// added in the correct list in the <see cref="MappingInfo"/> tree. Any errors will be added to the <see cref="ImportErrors"/>.
/// Entities referenced by user keys should be loaded in advance and passed to this function as a <see cref="RelatedEntities"/> dictionary.
/// This function is the opposite of <see cref="DataComposer.ComposeDataRowsFromEntity(List{string[]}, EntityWithKey, MappingInfo, int, int)"/>
/// </summary>
/// <param name="dataRow">An array of strings representing a single row in a CSV or XSLX file</param>
/// <param name="rowNumber">The number of the current row, for the purpose of error reporting</param>
/// <param name="mapping">The <see cref="MappingInfo"/> to rely on for constructing the <see cref="Entity"/> objects</param>
/// <param name="entities">All related entities that are referenced by user keys in the raw data</param>
/// <param name="errors">Any validation errors are added to this collection</param>
/// <returns>False if the <see cref="ImportErrors"/> dictionary has been maxed out, true otherwise.</returns>
private bool ParseRow(string[] dataRow, int rowNumber, MappingInfo mapping, RelatedEntities entities, ImportErrors errors, int selfRefPropertiesCount = 0)
{
bool entityCreated = false;
foreach (var prop in mapping.SimpleProperties)
{
var stringField = dataRow[prop.Index];
if (!string.IsNullOrEmpty(stringField))
{
if (!entityCreated)
{
mapping.Entity = mapping.CreateBaseEntity(rowNumber);
mapping.List.Add(mapping.Entity);
entityCreated = true;
}
var entity = prop.GetTerminalEntityForSave(mapping.Entity);
// Hydrate the property
if (prop is ForeignKeyMappingInfo fkProp && fkProp.NotUsingIdAsKey)
{
// Get the user key value (usually the code or the name)
object userKeyValue = fkProp.KeyType switch
{
KeyType.String => stringField?.Trim(),
KeyType.Int => int.Parse(stringField),
_ => null
};
// Get the entity from the dictionary
var dic = entities[(fkProp.TargetType, fkProp.TargetDefId, fkProp.KeyPropertyMetadata.Descriptor.Name)];
internal IEnumerable <ObjectRepresentation> AllRelated(List <ObjectRepresentation> evaluatedObjects)
{
var items = RelatedEntities.ToList();
foreach (var objectRepresentationBase in RelatedEntities)
{
if (evaluatedObjects.Contains(objectRepresentationBase))
{
continue;
}
evaluatedObjects.Add(objectRepresentationBase);
items.AddRange(objectRepresentationBase.AllRelated(evaluatedObjects));
}
return(items);
}
/// <summary>
/// Inspects the data and mapping and loads all related entities from the API, that are referenced by custom use keys like Code and Name.
/// The purpose is to use the Ids of these entities in the constructed Entity objects that are being imported.
/// </summary>
private async Task <RelatedEntities> LoadRelatedEntities(IEnumerable <string[]> dataWithoutHeader, MappingInfo mapping, ImportErrors errors)
{
if (errors is null)
{
throw new ArgumentNullException(nameof(errors));
}
// Each set of foreign keys will result in an API query to retrieve the corresponding Ids for FK hydration
// So we group foreign keys by the target type, the target definition Id and the key property (e.g. Code or Name)
var queryInfos = new List <(Type navType, int?navDefId, PropertyDescriptor keyPropMeta, HashSet <object> keysSet)>();
foreach (var g in mapping.GetForeignKeys().Where(p => p.NotUsingIdAsKey)
.GroupBy(fk => (fk.TargetType, fk.TargetDefId, fk.KeyPropertyMetadata.Descriptor)))
{
var(navType, navDefId, keyPropDesc) = g.Key;
var keysSet = new HashSet <object>();
foreach (var fkMapping in g)
{
int rowNumber = 2;
foreach (var row in dataWithoutHeader)
{
string stringKey = row[fkMapping.Index];
if (string.IsNullOrEmpty(stringKey))
{
continue;
}
switch (fkMapping.KeyType)
{
case KeyType.String:
keysSet.Add(stringKey);
break;
case KeyType.Int:
if (int.TryParse(stringKey, out int intKey))
{
keysSet.Add(intKey);
}
else if (!errors.AddImportError(rowNumber, fkMapping.ColumnNumber, _localizer[$"Error_TheValue0IsNotAValidInteger", stringKey]))
{
// This means the validation errors are at maximum capacity, pointless to keep going.
return(null);
}
break;
default:
throw new InvalidOperationException("Bug: Only int and string Ids are supported.");
}
rowNumber++;
}
}
if (keysSet.Any())
{
// Actual API calls are delayed till the end in case there are any errors
queryInfos.Add((navType, navDefId, keyPropDesc, keysSet));
}
}
if (!errors.IsValid)
{
return(null);
}
var result = new RelatedEntities();
if (queryInfos.Any())
{
// Load all related entities in parallel
await Task.WhenAll(queryInfos.Select(async queryInfo =>
{
var(navType, navDefId, keyPropDesc, keysSet) = queryInfo; // Deconstruct the queryInfo
var keyPropName = keyPropDesc.Name;
var data = await _client.GetEntitiesByPropertyValues(
collection: navType.Name,
definitionId: navDefId,
propName: keyPropName,
values: keysSet,
cancellation: default);
/// <summary>
/// Takes a list of <see cref="Entity"/>'s, and for every entity it inspects the navigation properties, if a navigation property
/// contains an <see cref="Entity"/> with a strong type, it sets that property to null, and moves the strong entity into a separate
/// "relatedEntities" hash set, this has several advantages: <br/>
/// 1 - The JSON serializer does not have to deal with circular references <br/>
/// 2 - Every strong entity is present once in the JSON response (smaller response size) <br/>
/// 3 - It makes it easier for clients to store and track entities in a central workspace <br/>
/// </summary>
/// <returns>A dictionary mapping every type name to an <see cref="IEnumerable"/> of related entities of that type (excluding the result entities).</returns>
public static RelatedEntities Flatten <TEntity>(IEnumerable <TEntity> resultEntities, CancellationToken cancellation)
where TEntity : Entity
{
// If the result is empty, nothing to do
var relatedEntities = new RelatedEntities();
if (resultEntities == null || !resultEntities.Any())
{
return(relatedEntities);
}
var resultHash = resultEntities.Cast <Entity>().ToHashSet();
var addedAleady = new HashSet <EntityWithKey>();
void FlattenInner(Entity entity, TypeDescriptor typeDesc)
{
if (entity.EntityMetadata.Flattened)
{
// This has already been flattened before
return;
}
// Mark the entity as flattened
entity.EntityMetadata.Flattened = true;
// Recursively go over the nav properties
foreach (var prop in typeDesc.NavigationProperties)
{
if (prop.GetValue(entity) is EntityWithKey relatedEntity)
{
prop.SetValue(entity, null);
if (!resultHash.Contains(relatedEntity) && addedAleady.Add(relatedEntity))
{
// Unless it is part of the main result, add it to relatedEntities (only once)
relatedEntities.AddEntity(relatedEntity);
}
FlattenInner(relatedEntity, prop.TypeDescriptor);
}
}
// Recursively go over every entity in the nav collection properties
foreach (var prop in typeDesc.CollectionProperties)
{
var collectionType = prop.CollectionTypeDescriptor;
if (prop.GetValue(entity) is IList collection)
{
foreach (var obj in collection)
{
if (obj is Entity relatedEntity)
{
FlattenInner(relatedEntity, collectionType);
}
}
}
}
}
// Flatten every entity in the main list
var typeDesc = TypeDescriptor.Get <TEntity>();
foreach (var entity in resultEntities)
{
if (entity != null)
{
FlattenInner(entity, typeDesc);
cancellation.ThrowIfCancellationRequested();
}
}
// Return the result
return(relatedEntities);
}
public RelationshipBuilder RelateTo(Id id)
{
RelatedEntities.Add(id);
return(this);
}
public Associate With <TRelatedEntity>(TRelatedEntity entity) where TRelatedEntity : CrmPlusPlusEntity, new()
{
RelatedEntities.Add(entity.Id, EntityNameAttribute.GetFromType <TRelatedEntity>());
return(this);
}
/// <summary>
/// Inspects the data and mapping and loads all related entities from the API, that are referenced by custom use keys like Code and Name.
/// The purpose is to use the Ids of these entities in the constructed Entity objects that are being imported
/// </summary>
private async Task <RelatedEntities> LoadRelatedEntities(IEnumerable <string[]> dataWithoutHeader, MappingInfo mapping, ImportErrors errors)
{
if (errors is null)
{
throw new ArgumentNullException(nameof(errors));
}
// Each set of foreign keys will result in an API query to retrieve the corresponding Ids for FK hydration
// So we group foreign keys by the target type, the target definition Id and the key property (e.g. Code or Name)
var queryInfos = new List <(Type navType, int?navDefId, PropertyMetadata keyPropMeta, HashSet <object> keysSet)>();
foreach (var g in mapping.GetForeignKeys().Where(p => p.NotUsingIdAsKey)
.GroupBy(fk => (fk.TargetType, fk.TargetDefId, fk.KeyPropertyMetadata)))
{
var(navType, navDefId, keyPropMetadata) = g.Key;
HashSet <object> keysSet = new HashSet <object>();
foreach (var fkMapping in g)
{
int rowNumber = 2;
foreach (var row in dataWithoutHeader)
{
string stringKey = row[fkMapping.Index];
if (string.IsNullOrEmpty(stringKey))
{
continue;
}
switch (fkMapping.KeyType)
{
case KeyType.String:
keysSet.Add(stringKey);
break;
case KeyType.Int:
if (int.TryParse(stringKey, out int intKey))
{
keysSet.Add(intKey);
}
else if (!errors.AddImportError(rowNumber, fkMapping.ColumnNumber, _localizer[$"Error_TheValue0IsNotAValidInteger", stringKey]))
{
// This means the validation errors are at maximum capacity, pointless to keep going.
return(null);
}
break;
default:
throw new InvalidOperationException("Bug: Only int and string IDs are supported");
}
rowNumber++;
}
}
if (keysSet.Any())
{
// Actual API calls are delayed till the end in case there are any errors
queryInfos.Add((navType, navDefId, keyPropMetadata, keysSet));
}
}
if (!errors.IsValid)
{
return(null);
}
using var _1 = _instrumentation.Block("Loading Related Entities");
using var _2 = _instrumentation.Disable();
var result = new RelatedEntities();
if (queryInfos.Any())
{
// Load all related entities in parallel
// If they're the same type we load them in sequence because it will be the same controller instance and it might cause issues
await Task.WhenAll(queryInfos.GroupBy(e => e.navType).Select(async g =>
{
var navType = g.Key;
foreach (var queryInfo in g)
{
var(_, navDefId, keyPropMeta, keysSet) = queryInfo; // Deconstruct the queryInfo
var service = _sp.FactWithIdServiceByEntityType(navType.Name, navDefId);
var keyPropDesc = keyPropMeta.Descriptor;
var keyPropName = keyPropDesc.Name;
var args = new SelectExpandArguments {
Select = keyPropName
};
var(data, _) = await service.GetByPropertyValues(keyPropName, keysSet, args, cancellation: default);
var grouped = data.GroupBy(e => keyPropDesc.GetValue(e)).ToDictionary(g => g.Key, g => (IEnumerable <EntityWithKey>)g);
result.TryAdd((navType, navDefId, keyPropName), grouped);
}
// The code above might override the definition Id of the service calling it, here we fix that
_sp.FactWithIdServiceByEntityType(navType.Name, mapping.MetadataForSave.DefinitionId);
}));
}
return(result);
}