private async Task ImportEntity(IOspSession session, Common.Shared.Exchange.RtEntity modelRtEntity)
{
var progress = Interlocked.Increment(ref _entityProgressCount);
if (progress > Max)
{
Logger.Info($"{_importEntityQueue.Count} entities (total imports of {_entityImportIds.Count}) imported.");
Interlocked.Exchange(ref _entityProgressCount, 1);
await ImportToDatabase(session);
}
var entityCacheItem = _tenantContext.CkCache.GetEntityCacheItem(modelRtEntity.CkId);
var rtEntity = _tenantContext.Repository.CreateTransientRtEntity(modelRtEntity.CkId);
rtEntity.RtId = modelRtEntity.Id.ToObjectId();
rtEntity.WellKnownName = modelRtEntity.WellKnownName;
if (_entityImportIds.Contains(rtEntity.RtId))
{
Logger.Error($"{rtEntity.RtId} already imported.");
return;
}
_entityImportIds.Add(rtEntity.RtId);
foreach (var modelAttribute in modelRtEntity.Attributes)
{
var attributeCacheItem = entityCacheItem.Attributes.Values.FirstOrDefault(a => a.AttributeId == modelAttribute.Id);
if (attributeCacheItem == null)
{
Logger.Error($"'{modelAttribute.Id}' does not exit on type $'{entityCacheItem.CkId}'");
return;
}
rtEntity.SetAttributeValue(attributeCacheItem.AttributeName, attributeCacheItem.AttributeValueType, modelAttribute.Value);
}
_importEntityQueue.Enqueue(rtEntity);
if (modelRtEntity.Associations != null && modelRtEntity.Associations.Count > 0)
{
var originId = rtEntity.RtId;
foreach (var association in modelRtEntity.Associations)
{
var rtAssociation = new RtAssociation
{
AssociationRoleId = association.RoleId,
OriginRtId = originId,
OriginCkId = rtEntity.CkId,
TargetRtId = association.TargetRtId.ToObjectId(),
TargetCkId = association.TargetCkId
};
_importAssociationQueue.Enqueue(rtAssociation);
Interlocked.Increment(ref _associationsCount);
}
}
}
private async Task DeleteRtEntityAsync(IOspSession session, IReadOnlyList <RtEntity> rtEntities)
{
foreach (var rtEntity in rtEntities.AsParallel())
{
await _databaseContext.GetRtCollection <RtEntity>(rtEntity.CkId).DeleteOneAsync(session, rtEntity.RtId);
}
}
public async Task SetConfigurationAsync <TValueType>(IOspSession systemSession, string key, TValueType value)
where TValueType : struct
{
ArgumentValidation.ValidateString(nameof(key), key);
await SetConfigAsync(systemSession, key, value);
}
private async Task <long> ExecuteAutoIncrementAsync(IOspSession session, RtSystemAutoIncrement autoIncrement)
{
var end = autoIncrement.End;
if (!autoIncrement.CurrentValue.HasValue)
{
throw new AutoIncrementFailedException(
$"'{autoIncrement.RtId}' cannot be incremented because current value was null.");
}
var currentValue = autoIncrement.CurrentValue.Value;
currentValue++;
if (currentValue > end)
{
throw new AutoIncrementFailedException(
$"'{autoIncrement.RtId}' cannot be incremented because end value is reached.");
}
autoIncrement.CurrentValue = currentValue;
await _databaseContext.GetRtCollection <RtEntity>(autoIncrement.CkId)
.ReplaceByIdAsync(session, autoIncrement.RtId, autoIncrement);
return(currentValue);
}
private async Task ValidateAssociationsToCreate(IOspSession session, IEnumerable <AssociationUpdateInfo> createAssociations, GraphRuleEngineResult graphRuleEngineResult)
{
foreach (var associationUpdateInfo in createAssociations)
{
var origin = associationUpdateInfo.Origin;
var target = associationUpdateInfo.Target;
var rtAssociation = await _tenantRepository.GetRtAssociationAsync(session,
origin,
target,
associationUpdateInfo.RoleId);
if (rtAssociation != null)
{
throw new OperationFailedException($"Association from '{origin}' to '{target}' in role '{associationUpdateInfo.RoleId}' already exits.");
}
graphRuleEngineResult.RtAssociationsToCreate.Add(new RtAssociation
{
OriginRtId = associationUpdateInfo.Origin.RtId.ToObjectId(),
OriginCkId = associationUpdateInfo.Origin.CkId,
TargetRtId = associationUpdateInfo.Target.RtId.ToObjectId(),
TargetCkId = associationUpdateInfo.Target.CkId,
AssociationRoleId = associationUpdateInfo.RoleId
});
}
}
public async Task UpdateAutoCompleteTexts(IOspSession session, string ckId, string attributeName,
IEnumerable <string> autoCompleteTexts)
{
ArgumentValidation.ValidateString(nameof(ckId), ckId);
ArgumentValidation.ValidateString(nameof(attributeName), attributeName);
ArgumentValidation.Validate(nameof(autoCompleteTexts), autoCompleteTexts);
var ckEntity =
await _databaseContext.CkEntities.FindSingleOrDefaultAsync(session, x => x.CkId == ckId);
if (ckEntity == null)
{
throw new EntityNotFoundException($"'{ckId}' does not exist in database.");
}
var attribute = ckEntity.Attributes.FirstOrDefault(x => x.AttributeName == attributeName);
if (attribute == null)
{
throw new InvalidAttributeException(
$"Attribute with name '{attributeName}' does not exist on type '{ckId}'");
}
attribute.AutoCompleteTexts = autoCompleteTexts.ToList();
try
{
await _databaseContext.CkEntities.ReplaceByIdAsync(session, ckEntity.CkId, ckEntity);
}
catch (Exception e)
{
throw new OperationFailedException("An error occurred during import: " + e.Message, e);
}
}
private async Task ValidateCkModel(IOspSession session, GraphRuleEngineResult graphRuleEngineResult, IReadOnlyList <EntityUpdateInfo> entityUpdateInfoList, IReadOnlyList <AssociationUpdateInfo> associationUpdateInfoList)
{
await ValidateOrigin(session, entityUpdateInfoList, associationUpdateInfoList);
await ValidateTarget(session, entityUpdateInfoList, associationUpdateInfoList);
// Ensure that all associations exists when creating an entity
// Currently, the only mandatory association has multiplicity of One
foreach (var entityUpdateInfo in entityUpdateInfoList.Where(x => x.ModOption == EntityModOptions.Create))
{
var cacheItem = _ckCache.GetEntityCacheItem(entityUpdateInfo.RtEntity.CkId);
var inboundAssociationCacheItems = cacheItem.InboundAssociations.Values.SelectMany(x => x.Where(a => a.InboundMultiplicity == Multiplicities.One));
foreach (var inboundAssociationCacheItem in inboundAssociationCacheItems)
{
if (!associationUpdateInfoList.Any(x => x.ModOption == AssociationModOptionsDto.Create && x.RoleId == inboundAssociationCacheItem.RoleId))
{
throw new CkModelViolationException($"Inbound association '{inboundAssociationCacheItem.RoleId}' has multiplicity of 'One', but create statement is missing. Error occurred at CK type '{entityUpdateInfo.RtEntity.CkId}' (from RtId '{entityUpdateInfo.RtEntity.RtId}').");
}
}
}
// Delete all corresponding associations if an entity is deleted
foreach (var entityUpdateInfo in entityUpdateInfoList.Where(x => x.ModOption == EntityModOptions.Delete))
{
var result = await _tenantRepository.GetRtAssociationsAsync(session, entityUpdateInfo.RtEntity.RtId.ToString(), GraphDirections.Any);
graphRuleEngineResult.RtAssociationsToDelete.AddRange(result);
}
}
private async Task <ITenantContextInternal> CreateOrGetTenantContextInternal(IOspSession systemSession,
string tenantId)
{
if (TryGetCkCache(tenantId, out var ckCache))
{
return(await CreateTenantContextAsync(systemSession, ckCache));
}
try
{
await _semaphoreSlim.WaitAsync();
if (TryGetCkCache(tenantId, out ckCache))
{
return(await CreateTenantContextAsync(systemSession, ckCache));
}
var databaseContext = await CreateDatabaseContextByTenantAsync(systemSession, tenantId);
ckCache = new CkCache(tenantId);
await ckCache.Initialize(databaseContext);
var key = tenantId.MakeKey();
_ckCaches[key] = ckCache;
return(await CreateTenantContextAsync(systemSession, ckCache));
}
finally
{
_semaphoreSlim.Release();
}
}
private async Task CreateSystemSchemaAsync(IOspSession systemSession)
{
await OspSystemDatabase.CreateCollectionIfNotExistsAsync <SystemEntities.OspTenant>();
await OspSystemDatabase.CreateCollectionIfNotExistsAsync <OspConfiguration>();
await OspSystemDatabase.CreateCollectionIfNotExistsAsync <OspUser>();
await OspSystemDatabase.CreateCollectionIfNotExistsAsync <OspRole>();
await OspSystemDatabase.CreateCollectionIfNotExistsAsync <OspPermission>();
await OspSystemDatabase.CreateCollectionIfNotExistsAsync <OspPermissionRole>();
await OspSystemDatabase.CreateCollectionIfNotExistsAsync <OspClient>();
await OspSystemDatabase.CreateCollectionIfNotExistsAsync <OspApiResource>();
await OspSystemDatabase.CreateCollectionIfNotExistsAsync <OspApiScope>();
await OspSystemDatabase.CreateCollectionIfNotExistsAsync <OspIdentityResource>();
await OspSystemDatabase.CreateCollectionIfNotExistsAsync <OspPersistedGrant>();
await OspSystemDatabase.CreateCollectionIfNotExistsAsync <OspIdentityProvider>();
await SetConfigAsync(systemSession, Constants.SystemSchemaVersion, Constants.SystemSchemaVersionValue);
}
public async Task <ResultSet <RtEntity> > GetRtEntitiesByIdAsync(IOspSession session, string ckId,
IReadOnlyList <ObjectId> rtIds,
DataQueryOperation dataQueryOperation, int?skip = null, int?take = null)
{
ArgumentValidation.ValidateString(nameof(ckId), ckId);
ArgumentValidation.Validate(nameof(rtIds), rtIds);
ArgumentValidation.Validate(nameof(dataQueryOperation), dataQueryOperation);
if (!rtIds.Any())
{
return(new ResultSet <RtEntity>(new List <RtEntity>(), 0));
}
var resultSet = new List <RtEntity>();
long totalCount = 0;
var entityCacheItem = GetEntityCacheItem(ckId);
var statementCreator =
new RtStatementCreator(entityCacheItem, _databaseContext, dataQueryOperation.Language);
statementCreator.AddFieldFilters(dataQueryOperation.FieldFilters);
statementCreator.AddIdFilter(rtIds);
statementCreator.AddTextSearchFilter(dataQueryOperation.TextSearchFilter);
statementCreator.AddAttributeSearchFilter(dataQueryOperation.AttributeSearchFilter);
statementCreator.AddSort(dataQueryOperation.SortOrders);
var tempResultSet = await statementCreator.ExecuteQuery(session, skip, take);
resultSet.AddRange(tempResultSet.Result);
totalCount += tempResultSet.TotalCount;
return(new ResultSet <RtEntity>(resultSet, totalCount));
}
protected override IAggregateFluent <RtEntity> CreateAggregate(IOspSession ospSession)
{
var targetRtIdAttribute = nameof(RtAssociation.TargetRtId);
var matchFilter = Builders <RtAssociation> .Filter.And(
Builders <RtAssociation> .Filter.Eq(x => x.OriginRtId, _rtId),
Builders <RtAssociation> .Filter.Eq(x => x.TargetCkId, _targetCkId));
var targetCollection = _databaseContext.GetRtCollection <RtEntity>(_targetCkId);
if (_graphDirections.HasFlag(GraphDirections.Inbound))
{
matchFilter = Builders <RtAssociation> .Filter.And(
Builders <RtAssociation> .Filter.Eq(x => x.TargetRtId, _rtId),
Builders <RtAssociation> .Filter.Eq(x => x.OriginCkId, _targetCkId));
targetRtIdAttribute = nameof(RtAssociation.OriginRtId);
}
var aggregate = _databaseContext.RtAssociations.Aggregate(ospSession)
.Match(matchFilter).Lookup(targetCollection.GetMongoCollection().CollectionNamespace.CollectionName,
targetRtIdAttribute.ToCamelCase(), "_id", "children")
.ReplaceRoot <RtEntity>(new BsonDocument("$arrayElemAt", new BsonArray().Add("$children").Add(0)));
return(aggregate);
}
public async Task ImportText(IOspSession session, string jsonText, ScopeIds scopeId, CancellationToken?cancellationToken = null)
{
Logger.Info("Reading CK model....");
var model = CkSerializer.Deserialize(jsonText);
await ExecuteImport(session, model, scopeId, cancellationToken);
}
public async Task <BulkImportResult> BulkRtAssociationsAsync(IOspSession session,
IEnumerable <RtAssociation> rtAssociations)
{
ArgumentValidation.Validate(nameof(rtAssociations), rtAssociations);
return(await _databaseContext.RtAssociations.BulkImportAsync(session, rtAssociations));
}
/// <summary>
/// Finds all documents matching a given expression
/// </summary>
/// <param name="session">Session object to manage the transaction</param>
/// <param name="filterDefinition">The filter definition</param>
/// <param name="sort">Sort definition</param>
/// <param name="skip">The number of documents to skip in the query</param>
/// <param name="take">The maximal amount of documents to return. The skip is applied before the limit restriction</param>
/// <returns>Returns a cursor</returns>
public async Task <ICollection <TDocument> > FindManyAsync(IOspSession session,
FilterDefinition <TDocument> filterDefinition,
SortDefinition <TDocument> sort = null, int?skip = null, int?take = null)
{
ArgumentValidation.Validate(nameof(session), session);
ArgumentValidation.Validate(nameof(filterDefinition), filterDefinition);
try
{
var cursor = await _documentCollection.FindAsync(((IOspSessionInternal)session).SessionHandle,
filterDefinition, new FindOptions <TDocument>
{
Sort = sort,
Skip = skip,
Limit = take
});
return(await cursor.ToListAsync());
}
catch (Exception e)
{
Console.WriteLine(e);
throw;
}
}
/// <summary>
/// Removes the stale persisted grants.
/// </summary>
/// <returns></returns>
private async Task RemoveGrantsAsync(IOspSession session)
{
var found = Int32.MaxValue;
while (found >= TokenCleanupBatchSize)
{
var query = await _persistentGrantCollection.FindManyAsync(session,
grant => grant.Expiration < DateTime.UtcNow,
0, TokenCleanupBatchSize);
var expiredGrants = query.OrderBy(x => x.Key)
.ToList();
found = expiredGrants.Count;
Logger.Info($"Removing {found} grants");
if (found > 0)
{
try
{
foreach (var persistedGrant in expiredGrants)
{
await _persistentGrantCollection.DeleteOneAsync(session, persistedGrant.Id);
}
}
catch (MongoException ex)
{
// we get this if/when someone else already deleted the records
// we want to essentially ignore this, and keep working
Logger.Debug($"Concurrency exception removing expired grants: {ex.Message}");
}
}
}
}
private async Task ValidateTarget(IOspSession session, IReadOnlyList <EntityUpdateInfo> entityUpdateInfoList, IReadOnlyList <AssociationUpdateInfo> associationUpdateInfoList)
{
var targetList = associationUpdateInfoList.Select(a => a.Target).Distinct();
foreach (var targetRtId in targetList)
{
var targetEntity = await GetEntity(session, entityUpdateInfoList, targetRtId);
if (targetEntity == null)
{
throw new CkModelViolationException($"Target entity '{targetRtId}' does not exist.");
}
var targetCacheItem = _ckCache.GetEntityCacheItem(targetEntity.CkId);
foreach (var associationUpdateInfosByRoleId in associationUpdateInfoList.Where(a => a.Target == targetRtId).GroupBy(a => a.RoleId))
{
var inboundAssociationCacheItem = targetCacheItem.InboundAssociations.Values.SelectMany(x => x.Where(a => a.RoleId == associationUpdateInfosByRoleId.Key)).FirstOrDefault();
if (inboundAssociationCacheItem == null)
{
throw new CkModelViolationException($"Inbound association '{associationUpdateInfosByRoleId.Key}' is not allowed at CK type '{targetCacheItem.CkId}' (from RtId '{targetRtId}').");
}
foreach (var associationUpdateInfo in associationUpdateInfosByRoleId)
{
var originEntity = await GetEntity(session, entityUpdateInfoList, associationUpdateInfo.Origin);
var originCacheItem = _ckCache.GetEntityCacheItem(originEntity.CkId);
if (!inboundAssociationCacheItem.AllowedTypes.Contains(originCacheItem))
{
throw new CkModelViolationException($"Inbound association '{associationUpdateInfosByRoleId.Key}' does not allow CK type '{targetCacheItem.CkId}' (from RtId '{targetRtId}').");
}
}
var storedTargetAssociations = await _tenantRepository.GetCurrentRtAssociationMultiplicityAsync(session, targetRtId, associationUpdateInfosByRoleId.Key, GraphDirections.Inbound);
var createCount = associationUpdateInfosByRoleId.Count(x => x.ModOption == AssociationModOptionsDto.Create);
var deleteCount = associationUpdateInfosByRoleId.Count(x => x.ModOption == AssociationModOptionsDto.Delete);
var changeDelta = createCount - deleteCount;
if (changeDelta < 0)
{
if (storedTargetAssociations == CurrentMultiplicity.One && inboundAssociationCacheItem.InboundMultiplicity == Multiplicities.One)
{
throw new CkModelViolationException($"Inbound association '{associationUpdateInfosByRoleId.Key}' has multiplicity of 'One'. Association deletion violates the model (from RtId '{targetRtId}').");
}
}
if (changeDelta > 0)
{
if (storedTargetAssociations == CurrentMultiplicity.One && (inboundAssociationCacheItem.InboundMultiplicity == Multiplicities.One || inboundAssociationCacheItem.InboundMultiplicity == Multiplicities.ZeroOrOne))
{
throw new CkModelViolationException($"Inbound association '{associationUpdateInfosByRoleId.Key}' has multiplicity of 'One'. Adding another association violates the model (from RtId '{targetRtId}').");
}
}
}
}
}
public IAsyncCursor <TOutput> Aggregate <TOutput>(IOspSession session,
PipelineDefinition <TDocument, TOutput> pipelineDefinition)
{
ArgumentValidation.Validate(nameof(session), session);
ArgumentValidation.Validate(nameof(pipelineDefinition), pipelineDefinition);
return(_documentCollection.Aggregate(((IOspSessionInternal)session).SessionHandle, pipelineDefinition));
}
public async Task <long> GetTotalCountAsync(IOspSession session, Expression <Func <TDocument, bool> > expression)
{
ArgumentValidation.Validate(nameof(session), session);
ArgumentValidation.Validate(nameof(expression), expression);
return(await _documentCollection.CountDocumentsAsync(((IOspSessionInternal)session).SessionHandle,
expression));
}
public async Task ImportRtModelAsTextAsync(IOspSession session, string jsonText)
{
ArgumentValidation.Validate(nameof(session), session);
ArgumentValidation.ValidateString(nameof(jsonText), jsonText);
var importer = new ImportRtModel(this);
await importer.ImportText(session, jsonText);
}
public async Task <CkTypeInfo> GetCkTypeInfoAsync(IOspSession session, CkEntity ckId)
{
var filter = Builders <CkEntity> .Filter.BuildIdFilter(ckId.CkId);
var aggregate = CkEntities.Aggregate(session).Match(filter);
return(await AggregateCkTypeInfo(aggregate).SingleOrDefaultAsync());
}
public async Task <long> GetTotalCountAsync(IOspSession session, FilterDefinition <TDocument> filterDefinition)
{
ArgumentValidation.Validate(nameof(session), session);
ArgumentValidation.Validate(nameof(filterDefinition), filterDefinition);
return(await _documentCollection.CountDocumentsAsync(((IOspSessionInternal)session).SessionHandle,
filterDefinition));
}
public async Task <ResultSet <TEntity> > GetRtEntitiesByTypeAsync <TEntity>(IOspSession session,
DataQueryOperation dataQueryOperation,
int?skip = null, int?take = null) where TEntity : RtEntity, new()
{
var ckId = GetCkId <TEntity>();
return(await GetRtEntitiesByTypeAsync <TEntity>(session, ckId, dataQueryOperation, skip, take));
}
public async Task ImportRtModelAsync(IOspSession session, string filePath, CancellationToken?cancellationToken)
{
ArgumentValidation.Validate(nameof(session), session);
ArgumentValidation.ValidateFilePath(nameof(filePath), filePath);
var importer = new ImportRtModel(this);
await importer.Import(session, filePath, cancellationToken);
}
private async Task <ITenantContextInternal> CreateTenantContextAsync(IOspSession systemSession,
ICkCache ckCache)
{
var databaseContext = await CreateDatabaseContextByTenantAsync(systemSession, ckCache.TenantId);
var tenantRepository = new TenantRepository(ckCache, databaseContext);
return(new TenantContext(ckCache.TenantId, tenantRepository, ckCache));
}
// ReSharper disable once MemberCanBePrivate.Global
public async Task <bool> IsTenantExistingAsync(IOspSession systemSession, string tenantId)
{
ArgumentValidation.Validate(nameof(systemSession), systemSession);
ArgumentValidation.ValidateString(nameof(tenantId), tenantId);
var ospDatabase = await GetOspDatabaseFromTenantAsync(systemSession, tenantId);
return(ospDatabase != null);
}
private async Task RestoreTenantSystemCkModelAsync(IOspSession systemSession, SystemEntities.OspTenant ospTenant)
{
var ckModelFilePath = Path.Combine(Helper.AssemblyDirectory, "CKModel.json");
Logger.Info("Importing construction kit model '{0}'", ckModelFilePath);
await ImportCkModelAsync(systemSession, ospTenant.TenantId, ScopeIds.System, ckModelFilePath, null);
Logger.Info("Construction kit model imported.");
}
public IAggregateFluent <TDocument> Aggregate(IOspSession session)
{
ArgumentValidation.Validate(nameof(session), session);
return(_documentCollection.Aggregate(((IOspSessionInternal)session).SessionHandle,
new AggregateOptions {
AllowDiskUse = true
}));
}
private async Task ImportToDatabase(IOspSession session)
{
Logger.Info($"Importing {_importEntityQueue.Count} to database.");
try
{
var importEntities = new List <RtEntity>();
var importAssociations = new List <RtAssociation>();
var entityMax = _importEntityQueue.Count;
var associationsMax = _importAssociationQueue.Count;
for (int i = 0; i < entityMax; i++)
{
if (_importEntityQueue.TryDequeue(out RtEntity tmp))
{
importEntities.Add(tmp);
}
else
{
break;
}
}
for (int i = 0; i < associationsMax; i++)
{
if (_importAssociationQueue.TryDequeue(out RtAssociation tmp))
{
importAssociations.Add(tmp);
}
else
{
break;
}
}
if (importEntities.Any())
{
Logger.Info("Adding entities...");
await _tenantContext.InternalRepository.BulkInsertRtEntitiesAsync(session, importEntities);
}
if (importAssociations.Any())
{
Logger.Info("Adding associations...");
await _tenantContext.InternalRepository.BulkRtAssociationsAsync(session, importAssociations);
}
Logger.Info("Add to database completed.");
}
catch (Exception e)
{
throw new ModelImportException("Import of model failed during import to database.", e);
}
}
public async Task UpdateCollectionsAsync(IOspSession session)
{
var ckEntities = (await CkEntities.GetAsync(session)).ToList();
foreach (var ckEntity in ckEntities)
{
var suffix = ckEntity.CkId.Replace(".", "_");
await _repository.CreateCollectionIfNotExistsAsync <RtEntity>(suffix);
}
}
public async Task <TEntity> GetRtEntityAsync <TEntity>(IOspSession session, RtEntityId rtEntityId)
where TEntity : RtEntity, new()
{
ArgumentValidation.Validate(nameof(rtEntityId), rtEntityId);
ArgumentValidation.ValidateString(nameof(rtEntityId.CkId), rtEntityId.CkId);
ArgumentValidation.Validate(nameof(rtEntityId.RtId), rtEntityId.RtId);
return(await _databaseContext.GetRtCollection <TEntity>(rtEntityId.CkId)
.DocumentAsync(session, rtEntityId.RtId.ToObjectId()));
}
