private static FilterDefinition <MongoContentEntity> CreateFilter(DomainId appId, IEnumerable <DomainId>?schemaIds, HashSet <DomainId> ids)
{
var filters = new List <FilterDefinition <MongoContentEntity> >();
var documentIds = ids.Select(x => DomainId.Combine(appId, x)).ToList();
if (documentIds.Count > 1)
{
filters.Add(
Filter.Or(
Filter.In(x => x.DocumentId, documentIds)));
}
else
{
var first = documentIds.First();
filters.Add(
Filter.Or(
Filter.Eq(x => x.DocumentId, first)));
}
if (schemaIds != null)
{
filters.Add(Filter.In(x => x.IndexedSchemaId, schemaIds));
}
filters.Add(Filter.Ne(x => x.IsDeleted, true));
return(Filter.And(filters));
}
public Task SetAsync(DomainId appId, TextContentState state)
{
var documentId = DomainId.Combine(appId, state.ContentId).ToString();
var document = new MongoTextIndexState(documentId, state);
return Collection.ReplaceOneAsync(x => x.DocumentId == documentId, document, UpsertReplace);
}
private void Create(ContentEvent @event, ContentData data)
{
var uniqueId = DomainId.Combine(@event.AppId, @event.ContentId);
var state = new TextContentState
{
UniqueContentId = uniqueId
};
state.GenerateDocIdCurrent();
Index(@event,
new UpsertIndexEntry
{
ContentId = @event.ContentId,
DocId = state.DocIdCurrent,
GeoObjects = data.ToGeo(jsonSerializer),
ServeAll = true,
ServePublished = false,
Texts = data.ToTexts(),
IsNew = true
});
states[state.UniqueContentId] = state;
updates[state.UniqueContentId] = state;
}
private async Task <IRuleEntity?> GetRuleAsync(DomainId appId, DomainId id)
{
using (Profiler.TraceMethod <RulesIndex>())
{
return(await GetRuleCoreAsync(DomainId.Combine(appId, id)));
}
}
private static FilterDefinition <MongoContentEntity> CreateFilter(DomainId appId, ICollection <DomainId> ids)
{
var filters = new List <FilterDefinition <MongoContentEntity> >
{
Filter.Ne(x => x.IsDeleted, true)
};
if (ids != null && ids.Count > 0)
{
var documentIds = ids.Select(x => DomainId.Combine(appId, x)).ToList();
if (ids.Count > 1)
{
filters.Add(
Filter.Or(
Filter.In(x => x.DocumentId, documentIds)));
}
else
{
var first = documentIds.First();
filters.Add(
Filter.Or(
Filter.Eq(x => x.DocumentId, first)));
}
}
return(Filter.And(filters));
}
public async Task <ISchemaEntity?> GetSchemaAsync(DomainId appId, DomainId id, bool canCache,
CancellationToken ct = default)
{
using (Telemetry.Activities.StartActivity("SchemasIndex/GetSchemaAsync"))
{
var cacheKey = GetCacheKey(appId, id);
if (canCache)
{
if (grainCache.TryGetValue(cacheKey, out var v) && v is ISchemaEntity cachedSchema)
{
return(cachedSchema);
}
}
var schema = await GetSchemaCoreAsync(DomainId.Combine(appId, id));
if (schema != null)
{
await CacheItAsync(schema);
}
return(schema);
}
}
private static FilterDefinition <MongoContentEntity> CreateFilter(DomainId appId, IEnumerable <DomainId> schemaIds, ICollection <DomainId>?ids, ClrQuery?query, DomainId referenced)
{
var filters = new List <FilterDefinition <MongoContentEntity> >
{
Filter.Eq(x => x.IndexedAppId, appId),
Filter.In(x => x.IndexedSchemaId, schemaIds),
Filter.Ne(x => x.IsDeleted, true)
};
if (ids != null && ids.Count > 0)
{
var documentIds = ids.Select(x => DomainId.Combine(appId, x)).ToList();
filters.Add(
Filter.Or(
Filter.AnyIn(x => x.ReferencedIds, documentIds),
Filter.In(x => x.DocumentId, documentIds)));
}
if (query?.Filter != null)
{
filters.Add(query.Filter.BuildFilter <MongoContentEntity>());
}
if (referenced != default)
{
filters.Add(Filter.AnyEq(x => x.ReferencedIds, referenced));
}
return(Filter.And(filters));
}
private ISchemaEntity SetupSchema(long version = 0, bool isDeleted = false)
{
var schemaEntity = A.Fake <ISchemaEntity>();
A.CallTo(() => schemaEntity.SchemaDef)
.Returns(new Schema(schemaId.Name));
A.CallTo(() => schemaEntity.Id)
.Returns(schemaId.Id);
A.CallTo(() => schemaEntity.AppId)
.Returns(appId);
A.CallTo(() => schemaEntity.Version)
.Returns(version);
A.CallTo(() => schemaEntity.IsDeleted)
.Returns(isDeleted);
var schemaGrain = A.Fake <ISchemaGrain>();
A.CallTo(() => schemaGrain.GetStateAsync())
.Returns(J.Of(schemaEntity));
var key = DomainId.Combine(appId, schemaId.Id).ToString();
A.CallTo(() => grainFactory.GetGrain <ISchemaGrain>(key, null))
.Returns(schemaGrain);
return(schemaEntity);
}
private void DeleteDraft(ContentEvent @event)
{
var uniqueId = DomainId.Combine(@event.AppId, @event.ContentId);
if (states.TryGetValue(uniqueId, out var state) && state.DocIdNew != null)
{
Index(@event,
new UpdateIndexEntry
{
DocId = state.DocIdCurrent,
ServeAll = true,
ServePublished = true
});
Index(@event,
new DeleteIndexEntry
{
DocId = state.DocIdNew
});
state.DocIdNew = null;
updates[state.UniqueContentId] = state;
}
}
private async Task <IRuleEntity> GetRuleInternalAsync(DomainId appId, DomainId id)
{
var key = DomainId.Combine(appId, id).ToString();
var rule = await grainFactory.GetGrain <IRuleGrain>(key).GetStateAsync();
return(rule.Value);
}
public async Task<TextContentState?> GetAsync(DomainId appId, DomainId contentId)
{
var documentId = DomainId.Combine(appId, contentId).ToString();
var result = await Collection.Find(x => x.DocumentId == documentId).FirstOrDefaultAsync()!;
return result?.ToState();
}
private static FilterDefinition <MongoAssetEntity> BuildFilter(DomainId appId, HashSet <DomainId> ids)
{
var documentIds = ids.Select(x => DomainId.Combine(appId, x));
return(Filter.And(
Filter.In(x => x.DocumentId, documentIds),
Filter.Ne(x => x.IsDeleted, true)));
}
public StreamMapper(RestoreContext context)
{
Guard.NotNull(context, nameof(context));
this.context = context;
brokenAppId = DomainId.Combine(context.PreviousAppId, context.PreviousAppId);
}
public async Task UpdateAsync()
{
const int SizeOfBatch = 1000;
const int SizeOfQueue = 20;
var collectionOld = database.GetCollection <BsonDocument>("Events");
var collectionNew = database.GetCollection <BsonDocument>("Events2");
var batchBlock = new BatchBlock <BsonDocument>(SizeOfBatch, new GroupingDataflowBlockOptions
{
BoundedCapacity = SizeOfQueue * SizeOfBatch
});
var writeOptions = new BulkWriteOptions
{
IsOrdered = false
};
var actionBlock = new ActionBlock <BsonDocument[]>(async batch =>
{
try
{
var writes = new List <WriteModel <BsonDocument> >();
foreach (var document in batch)
{
var eventStream = document["EventStream"].AsString;
if (TryGetAppId(document, out var appId))
{
if (!eventStream.StartsWith("app-", StringComparison.OrdinalIgnoreCase))
{
var indexOfType = eventStream.IndexOf('-');
var indexOfId = indexOfType + 1;
var indexOfOldId = eventStream.LastIndexOf("--", StringComparison.OrdinalIgnoreCase);
if (indexOfOldId > 0)
{
indexOfId = indexOfOldId + 2;
}
var domainType = eventStream.Substring(0, indexOfType);
var domainId = eventStream[indexOfId..];
var newDomainId = DomainId.Combine(DomainId.Create(appId), DomainId.Create(domainId)).ToString();
var newStreamName = $"{domainType}-{newDomainId}";
document["EventStream"] = newStreamName;
foreach (var @event in document["Events"].AsBsonArray)
{
var metadata = @event["Metadata"].AsBsonDocument;
metadata["AggregateId"] = newDomainId;
}
}
public AssetLoaderTests()
{
var key = DomainId.Combine(appId, id).ToString();
A.CallTo(() => grainFactory.GetGrain <IAssetGrain>(key, null))
.Returns(grain);
sut = new AssetLoader(grainFactory);
}
private Envelope <ContentEvent> ContentEvent(ContentEvent @event)
{
@event.AppId = appId;
var envelope = Envelope.Create(@event);
envelope.SetAggregateId(DomainId.Combine(appId.Id, @event.ContentId));
return(envelope);
}
public async Task Should_restore_states_for_all_contents()
{
var me = new RefToken(RefTokenType.Subject, "123");
var schemaId1 = NamedId.Of(DomainId.NewGuid(), "my-schema1");
var schemaId2 = NamedId.Of(DomainId.NewGuid(), "my-schema2");
var contentId1 = DomainId.NewGuid();
var contentId2 = DomainId.NewGuid();
var contentId3 = DomainId.NewGuid();
var context = new RestoreContext(appId.Id, new UserMapping(me), A.Fake <IBackupReader>(), DomainId.NewGuid());
await sut.RestoreEventAsync(ContentEvent(new ContentCreated
{
ContentId = contentId1,
SchemaId = schemaId1
}), context);
await sut.RestoreEventAsync(ContentEvent(new ContentCreated
{
ContentId = contentId2,
SchemaId = schemaId1
}), context);
await sut.RestoreEventAsync(ContentEvent(new ContentCreated
{
ContentId = contentId3,
SchemaId = schemaId2
}), context);
await sut.RestoreEventAsync(ContentEvent(new ContentDeleted
{
ContentId = contentId2,
SchemaId = schemaId1
}), context);
await sut.RestoreEventAsync(Envelope.Create(new SchemaDeleted
{
SchemaId = schemaId2
}), context);
var rebuildContents = new HashSet <DomainId>();
A.CallTo(() => rebuilder.InsertManyAsync <ContentDomainObject, ContentState>(A <IEnumerable <DomainId> > ._, A <CancellationToken> ._))
.Invokes((IEnumerable <DomainId> source, CancellationToken _) => rebuildContents.AddRange(source));
await sut.RestoreAsync(context);
Assert.Equal(new HashSet <DomainId>
{
DomainId.Combine(appId.Id, contentId1),
DomainId.Combine(appId.Id, contentId2)
}, rebuildContents);
}
private void CreateDraft(ContentEvent @event)
{
var uniqueId = DomainId.Combine(@event.AppId, @event.ContentId);
if (states.TryGetValue(uniqueId, out var state))
{
state.GenerateDocIdNew();
updates[state.UniqueContentId] = state;
}
}
public static ISchemaEntity Schema(NamedId <DomainId> appId, NamedId <DomainId> schemaId, Schema?schemaDef = null)
{
var schema = A.Fake <ISchemaEntity>();
A.CallTo(() => schema.Id).Returns(schemaId.Id);
A.CallTo(() => schema.AppId).Returns(appId);
A.CallTo(() => schema.SchemaDef).Returns(schemaDef ?? new Schema(schemaId.Name));
A.CallTo(() => schema.UniqueId).Returns(DomainId.Combine(appId, schemaId.Id));
return(schema);
}
public Task UpdateAsync()
{
return(stateForAssets.ReadAllAsync(async(state, version) =>
{
state.Slug = state.FileName.ToAssetSlug();
var key = DomainId.Combine(state.AppId.Id, state.Id).ToString();
await stateForAssets.WriteAsync(key, state, version, version);
}));
}
public async Task UpdateAsync(
CancellationToken ct)
{
await foreach (var(state, version) in stateForAssets.ReadAllAsync(ct))
{
state.Slug = state.FileName.ToAssetSlug();
var key = DomainId.Combine(state.AppId.Id, state.Id);
await stateForAssets.WriteAsync(key, state, version, version, ct);
}
}
public override FilterNode <ClrValue> Visit(CompareFilter <ClrValue> nodeIn, Args args)
{
var result = nodeIn;
var(path, op, value) = nodeIn;
var clrValue = value.Value;
if (string.Equals(path[0], "id", StringComparison.OrdinalIgnoreCase))
{
path = "_id";
if (clrValue is List <string> idList)
{
value = idList.Select(x => DomainId.Combine(args.AppId, DomainId.Create(x)).ToString()).ToList();
}
else if (clrValue is string id)
{
value = DomainId.Combine(args.AppId, DomainId.Create(id)).ToString();
}
else if (clrValue is List <Guid> guidIdList)
{
value = guidIdList.Select(x => DomainId.Combine(args.AppId, DomainId.Create(x)).ToString()).ToList();
}
else if (clrValue is Guid guidId)
{
value = DomainId.Combine(args.AppId, DomainId.Create(guidId)).ToString();
}
}
else
{
path = Adapt.MapPath(path);
if (clrValue is List <Guid> guidList)
{
value = guidList.Select(x => x.ToString()).ToList();
}
else if (clrValue is Guid guid)
{
value = guid.ToString();
}
else if (clrValue is Instant &&
!string.Equals(path[0], "mt", StringComparison.OrdinalIgnoreCase) &&
!string.Equals(path[0], "ct", StringComparison.OrdinalIgnoreCase))
{
value = clrValue.ToString();
}
}
return(result with {
Path = path, Value = value
});
}
public async Task <List <IRuleEntity> > GetRulesAsync(DomainId appId)
{
using (Profiler.TraceMethod <RulesIndex>())
{
var ids = await GetRuleIdsAsync(appId);
var rules =
await Task.WhenAll(
ids.Select(id => GetRuleCoreAsync(DomainId.Combine(appId, id))));
return(rules.NotNull().ToList());
}
}
public async Task <IAssetEntity?> FindAssetAsync(DomainId appId, DomainId id)
{
using (Profiler.TraceMethod <MongoAssetRepository>())
{
var documentId = DomainId.Combine(appId, id);
var assetEntity =
await Collection.Find(x => x.DocumentId == documentId && !x.IsDeleted)
.FirstOrDefaultAsync();
return(assetEntity);
}
}
public async Task <IAssetFolderEntity?> FindAssetFolderAsync(DomainId appId, DomainId id,
CancellationToken ct = default)
{
using (Telemetry.Activities.StartActivity("MongoAssetFolderRepository/FindAssetFolderAsync"))
{
var documentId = DomainId.Combine(appId, id);
var assetFolderEntity =
await Collection.Find(x => x.DocumentId == documentId && !x.IsDeleted)
.FirstOrDefaultAsync(ct);
return(assetFolderEntity);
}
}
public async Task <List <IRuleEntity> > GetRulesAsync(DomainId appId,
CancellationToken ct = default)
{
using (Telemetry.Activities.StartActivity("RulesIndex/GetRulesAsync"))
{
var ids = await GetRuleIdsAsync(appId);
var rules =
await Task.WhenAll(
ids.Select(id => GetRuleCoreAsync(DomainId.Combine(appId, id))));
return(rules.NotNull().ToList());
}
}
public async Task <bool> DoAsync(DomainId appId, DomainId contentId)
{
var currentId = DomainId.Combine(appId, contentId);
var filter =
Filter.And(
Filter.AnyEq(x => x.ReferencedIds, appId),
Filter.Eq(x => x.IndexedAppId, appId),
Filter.Ne(x => x.IsDeleted, true),
Filter.Ne(x => x.Id, currentId));
var hasReferrerAsync =
await Collection.Find(filter).Only(x => x.Id)
.AnyAsync();
return(hasReferrerAsync);
}
protected override (string Description, FastlyJob Data) CreateJob(EnrichedEvent @event, FastlyAction action)
{
var id = string.Empty;
if (@event is IEnrichedEntityEvent entityEvent)
{
id = DomainId.Combine(@event.AppId.Id, entityEvent.Id).ToString();
}
var ruleJob = new FastlyJob
{
Key = id,
FastlyApiKey = action.ApiKey,
FastlyServiceID = action.ServiceId
};
return(Description, ruleJob);
}
public GrainEnvironment()
{
var indexGrain = A.Fake <ISchemasByAppIndexGrain>();
A.CallTo(() => indexGrain.GetIdAsync(AppId.Name))
.Returns(AppId.Id);
var schemaGrain = A.Fake <ISchemaGrain>();
A.CallTo(() => schemaGrain.GetStateAsync())
.ReturnsLazily(() => CreateEntity().AsJ());
A.CallTo(() => GrainFactory.GetGrain <ISchemaGrain>(DomainId.Combine(AppId.Id, SchemaId.Id).ToString(), null))
.Returns(schemaGrain);
A.CallTo(() => GrainFactory.GetGrain <ISchemasByAppIndexGrain>(SingleGrain.Id, null))
.Returns(indexGrain);
}
private void Update(ContentEvent @event, ContentData data)
{
var uniqueId = DomainId.Combine(@event.AppId, @event.ContentId);
if (states.TryGetValue(uniqueId, out var state))
{
if (state.DocIdNew != null)
{
Index(@event,
new UpsertIndexEntry
{
ContentId = @event.ContentId,
DocId = state.DocIdNew,
GeoObjects = data.ToGeo(jsonSerializer),
ServeAll = true,
ServePublished = false,
Texts = data.ToTexts()
});
Index(@event,
new UpdateIndexEntry
{
DocId = state.DocIdCurrent,
ServeAll = false,
ServePublished = true
});
}
else
{
var isPublished = state.DocIdCurrent == state.DocIdForPublished;
Index(@event,
new UpsertIndexEntry
{
ContentId = @event.ContentId,
DocId = state.DocIdCurrent,
GeoObjects = data.ToGeo(jsonSerializer),
ServeAll = true,
ServePublished = isPublished,
Texts = data.ToTexts()
});
}
}
}