public static bool GenerateIndexes <T>(out IndexKeysDefinition <T> value)
where T : BaseEntity
{
var properties = typeof(T).GetProperties();
if (!properties.Any())
{
value = null;
return(false);
}
else
{
value = Builders <T> .IndexKeys.Descending(x => x.Id);
foreach (var property in properties)
{
if (property.HasAttribute <BsonPropertyAttribute>(out var attr))
{
if (attr.BsonDirection == BsonDirection.DESC)
{
value = value.Descending(property.Name.ToRegular());
}
else if (attr.BsonDirection == BsonDirection.ASC)
{
value = value.Ascending(property.Name.ToRegular());
}
}
}
return(true);
}
}
/// <summary>
/// Verify the provided <paramref name="mongoIndex"/> is defined and ready to go.
/// </summary>
protected virtual void VerifyIndex(MongoDbIndex <MongoDbEventData> mongoIndex)
{
IndexKeysDefinitionBuilder <MongoDbEventData> indexKeysBuilder = Builders <MongoDbEventData> .IndexKeys;
IndexKeysDefinition <MongoDbEventData> indexKey = null;
IList <Expression <Func <MongoDbEventData, object> > > selectors = mongoIndex.Selectors.ToList();
for (int i = 0; i < selectors.Count; i++)
{
Expression <Func <MongoDbEventData, object> > expression = selectors[i];
if (mongoIndex.IsAcending)
{
if (i == 0)
{
indexKey = indexKeysBuilder.Ascending(expression);
}
else
{
indexKey = indexKey.Ascending(expression);
}
}
else
{
if (i == 0)
{
indexKey = indexKeysBuilder.Descending(expression);
}
else
{
indexKey = indexKey.Descending(expression);
}
}
}
bool throwExceptions;
if (!bool.TryParse(ConfigurationManager.GetSetting("Cqrs.MongoDb.EventStore.ThrowExceptionsOnIndexPreparation"), out throwExceptions))
{
throwExceptions = true;
}
try
{
MongoCollection.Indexes.CreateOne
(
indexKey,
new CreateIndexOptions
{
Unique = mongoIndex.IsUnique,
Name = mongoIndex.Name
}
);
}
catch
{
if (throwExceptions)
{
throw;
}
}
}
/// <summary>
/// Create unique index
/// </summary>
/// <param name="context">work context</param>
/// <param name="collectionIndex">collection index information</param>
/// <returns>Task</returns>
public async Task CreateIndex(IWorkContext context, CollectionIndex collectionIndex)
{
Verify.IsNotNull(nameof(context), context);
Verify.IsNotNull(nameof(collectionIndex), collectionIndex);
Verify.IsNotEmpty(nameof(collectionIndex.Name), collectionIndex.Name);
Verify.Assert(collectionIndex.Sparse == false || (collectionIndex?.Keys?.Count() == 0), "sparse index does not support compound indexes");
Verify.Assert(collectionIndex.Keys?.Count > 0, "requires key definitions");
context = context.WithTag(_tag);
var options = new CreateIndexOptions
{
Name = collectionIndex.Name,
Version = 1,
Unique = collectionIndex.Unique,
Sparse = collectionIndex.Sparse,
};
IndexKeysDefinition <TDocument> keys = null;
foreach (var key in collectionIndex.Keys)
{
if (key.Descending)
{
keys = keys?.Descending(key.FieldName) ?? Builders <TDocument> .IndexKeys.Descending(key.FieldName);
}
else
{
keys = keys?.Ascending(key.FieldName) ?? Builders <TDocument> .IndexKeys.Ascending(key.FieldName);
}
}
MongoDbEventSource.Log.Info(context, $"Creating index={collectionIndex.Name}");
await Parent.MongoCollection.Indexes.CreateOneAsync(keys, options, context.CancellationToken);
}
private static IndexKeysDefinition <T> CreateIndexDefinition <T>(IndexKeysDefinition <T> index, IIndexField <T> field)
{
switch (field.SortOrder)
{
case IndexSortOrder.Desc:
return(index.Descending(field.Field));
default:
case IndexSortOrder.Asc:
return(index.Ascending(field.Field));
}
}
protected virtual void VerifyIndexes <TEntity>(IMongoCollection <TEntity> collection)
{
Type entityType = typeof(TEntity);
if (IndexTypesByEntityType.ContainsKey(entityType))
{
foreach (object untypedIndexType in IndexTypesByEntityType[entityType])
{
var mongoIndex = (MongoDbIndex <TEntity>)untypedIndexType;
IndexKeysDefinitionBuilder <TEntity> indexKeysBuilder = Builders <TEntity> .IndexKeys;
IndexKeysDefinition <TEntity> indexKey = null;
IList <Expression <Func <TEntity, object> > > selectors = mongoIndex.Selectors.ToList();
for (int i = 0; i < selectors.Count; i++)
{
Expression <Func <TEntity, object> > expression = selectors[i];
if (mongoIndex.IsAcending)
{
if (i == 0)
{
indexKey = indexKeysBuilder.Ascending(expression);
}
else
{
indexKey = indexKey.Ascending(expression);
}
}
else
{
if (i == 0)
{
indexKey = indexKeysBuilder.Descending(expression);
}
else
{
indexKey = indexKey.Descending(expression);
}
}
}
collection.Indexes.CreateOne
(
indexKey,
new CreateIndexOptions
{
Unique = mongoIndex.IsUnique,
Name = mongoIndex.Name
}
);
}
}
}
protected virtual void VerifyIndex(MongoDbIndex <EventData> mongoIndex)
{
IndexKeysDefinitionBuilder <EventData> indexKeysBuilder = Builders <EventData> .IndexKeys;
IndexKeysDefinition <EventData> indexKey = null;
IList <Expression <Func <EventData, object> > > selectors = mongoIndex.Selectors.ToList();
for (int i = 0; i < selectors.Count; i++)
{
Expression <Func <EventData, object> > expression = selectors[i];
if (mongoIndex.IsAcending)
{
if (i == 0)
{
indexKey = indexKeysBuilder.Ascending(expression);
}
else
{
indexKey = indexKey.Ascending(expression);
}
}
else
{
if (i == 0)
{
indexKey = indexKeysBuilder.Descending(expression);
}
else
{
indexKey = indexKey.Descending(expression);
}
}
}
MongoCollection.Indexes.CreateOne
(
indexKey,
new CreateIndexOptions
{
Unique = mongoIndex.IsUnique,
Name = mongoIndex.Name
}
);
}
/// <summary>
/// Ensures that the desired indexes exist and creates them if they don't exist.
/// </summary>
/// <param name="keynames">The indexed fields.</param>
/// <param name="descending">Set to true to make index descending, false for ascending.</param>
/// <param name="unique">Set to true to ensure index enforces unique values.</param>
/// <param name="sparse">Set to true to specify the index is sparse.</param>
/// <param name="cancellationToken">Optional threading cancellation token</param>
/// <remarks>
/// This is a convenience method for EnsureIndexes(IMongoIndexKeys keys, IMongoIndexOptions options).
/// </remarks>
public async virtual Task <string> EnsureIndexAsync(IEnumerable <Expression <Func <T, object> > > keynames, bool descending = false, bool unique = false, bool sparse = false, CancellationToken cancellationToken = default(CancellationToken))
{
Condition.Requires(keynames, "keynames").IsNotNull();
var options = new CreateIndexOptions {
Unique = unique, Sparse = sparse
};
var builder = Builders <T> .IndexKeys;
IndexKeysDefinition <T> indexKeysDefinition = null;
foreach (var k in keynames)
{
if (descending)
{
if (null == indexKeysDefinition)
{
indexKeysDefinition = builder.Descending(k);
}
else
{
indexKeysDefinition = indexKeysDefinition.Descending(k);
}
}
else
if (null == indexKeysDefinition)
{
indexKeysDefinition = builder.Ascending(k);
}
else
{
indexKeysDefinition = indexKeysDefinition.Ascending(k);
}
}
return(await this.collection.Indexes.CreateOneAsync(indexKeysDefinition, options, cancellationToken));
}
/// <summary> Reseeds a database using the given descriptor. </summary>
/// <param name="reseedInfo"> JsonObject containing description of how to reseed the database. </param>
public void Reseed(JsonObject reseedInfo, string topDir = null)
{
if (reseedInfo.Has <JsonArray>("drop"))
{
Drop(reseedInfo.Get <JsonArray>("drop"));
}
if (reseedInfo.Has <JsonObject>("index"))
{
Index(reseedInfo.Get <JsonObject>("index"));
}
if (reseedInfo.Has <JsonObject>("insert"))
{
Insert(reseedInfo.Get <JsonObject>("insert"), topDir);
}
void Drop(JsonArray databases)
{
foreach (var dbname in databases)
{
if (dbname.isString)
{
dbClient.DropDatabase(dbname.stringVal);
}
}
}
void Index(JsonObject descriptor)
{
string database = descriptor.Pull("database", dbName);
string collection = descriptor.Pull("collection", "Garbage");
if (database == "$default")
{
database = dbName;
}
JsonObject fields = descriptor.Pull <JsonObject>("fields");
MDB db = dbClient.GetDatabase(database);
List <CreateIndexModel <BDoc> > indexes = new List <CreateIndexModel <BDoc> >();
IndexKeysDefinition <BDoc> index = null;
foreach (var pair in fields)
{
string fieldName = pair.Key;
int order = pair.Value;
if (order > 0)
{
index = index?.Ascending(fieldName) ?? Builders <BDoc> .IndexKeys.Ascending(fieldName);
}
else
{
index = index?.Descending(fieldName) ?? Builders <BDoc> .IndexKeys.Descending(fieldName);
}
}
var model = new CreateIndexModel <BDoc>(index);
db.GetCollection <BDoc>(collection).Indexes.CreateOne(model);
}
void Insert(JsonObject descriptor, string dir)
{
string database = descriptor.Pull("database", dbName);
string collection = descriptor.Pull("collection", "Garbage");
string[] files = descriptor.Pull <string[]>("files");
if (files == null)
{
files = new string[] { collection };
}
dir = ForwardSlashPath(dir);
if (!dir.EndsWith("/"))
{
dir += "/";
}
foreach (var file in files)
{
string json = null;
string fpath = dir + file;
if (file.EndsWith("/**"))
{
string directory = fpath.Replace("/**", "");
Glob(database, collection, directory);
}
else
{
try { json = json ?? File.ReadAllText(fpath); } catch (Exception) { }
try { json = json ?? File.ReadAllText(fpath + ".json"); } catch (Exception) { }
try { json = json ?? File.ReadAllText(fpath + ".wtf"); } catch (Exception) { }
if (json == null)
{
Log.Warning($"Seeder could not find file {{{ForwardSlashPath(file)}}} under {{{dir}}}");
continue;
}
JsonValue data = Json.Parse(json);
if (data == null || !(data is JsonObject) && !(data is JsonArray))
{
Log.Warning($"Seeder cannot use {{{ForwardSlashPath(file)}}} under {{{dir}}}, it is not an object or array.");
continue;
}
if (data is JsonObject)
{
data["filename"] = UpToLast(FromLast(ForwardSlashPath(fpath), "/"), ".");
InsertData(database, collection, data as JsonObject);
}
else if (data is JsonArray)
{
InsertData(database, collection, data as JsonArray);
}
}
}
}
void Glob(string database, string collection, string directory)
{
List <string> files = AllFilesInDirectory(directory);
foreach (string file in files)
{
string json = null;
try {
json = json ?? File.ReadAllText(file);
} catch (Exception e) {
Log.Warning($"Seeder could not find {{{file}}}.", e);
}
try {
JsonValue data = Json.Parse(json);
if (data == null || !(data is JsonObject) && !(data is JsonArray))
{
Log.Warning($"Seeder cannot use {{{ForwardSlashPath(file)}}}, it is not an object or array.");
continue;
}
if (data is JsonObject)
{
data["filename"] = UpToLast(FromLast(ForwardSlashPath(file), "/"), ".");
InsertData(database, collection, data as JsonObject);
}
else if (data is JsonArray)
{
InsertData(database, collection, data as JsonArray);
}
} catch (Exception e) {
Log.Warning($"Seeder could not parse {{{ForwardSlashPath(file)}}}.", e);
}
}
}
}
public static void Main(string[] args)
{
// ***** START OF CONFIG
Dictionary <String, String> connections = new Dictionary <String, String>();
connections.Add("main", "mongodb://*****:*****@"C:\projects\cmd\github\Content-Store\src\Backoffice\configuration";
Dictionary <String, IMongoDatabase> databases = new Dictionary <String, IMongoDatabase>();
foreach (KeyValuePair <String, String> connection in connections)
{
MongoUrl url = new MongoUrl(connection.Value);
MongoClient client = new MongoClient(url);
databases.Add(connection.Key, client.GetDatabase(url.DatabaseName));
}
Boolean reinit = true;
IContainerParser parser = new ContentStore.JsonSettings.JsonContainerParser();
IReadonlyContainerStore store = new ContentStore.LocalFileSystem.ContainerStore(null, parser, solutionConfigurationRoot);
// ***** END OF CONFIG
String[] containerConfigurations = Directory.GetFiles(Path.Combine(solutionConfigurationRoot, "containers"), "*.json", SearchOption.AllDirectories);
foreach (String path in containerConfigurations)
{
IContainer container = store.Get(Path.GetFileNameWithoutExtension(path));
IMongoDatabase db = databases[String.IsNullOrWhiteSpace(container.Database) ? "main" : container.Database];
List <BsonDocument> collections = db.ListCollections().ToList();
Boolean createCollection = true;
if (collections.Any(c => c["name"].AsString == container.Name) && reinit == false)
{
createCollection = false;
}
if (createCollection)
{
db.CreateCollection(container.Name, new CreateCollectionOptions {
AutoIndexId = true
});
if (container.Indexes != null && container.Indexes.Any())
{
List <CreateIndexModel <BsonDocument> > indexes = new List <CreateIndexModel <BsonDocument> >();
foreach (IIndex index in container.Indexes)
{
IndexKeysDefinition <BsonDocument> idx = null;
CreateIndexOptions options = null;
if (index.Fields != null && index.Fields.Any())
{
if (index.Order == Order.Ascending)
{
idx = Builders <BsonDocument> .IndexKeys.Ascending(index.Fields.First());
foreach (String field in index.Fields.Skip(1))
{
idx = idx.Ascending(field);
}
}
else
{
idx = Builders <BsonDocument> .IndexKeys.Descending(index.Fields.First());
foreach (String field in index.Fields.Skip(1))
{
idx = idx.Descending(field);
}
}
if (index.Unique.HasValue && index.Unique.Value == true)
{
options = new CreateIndexOptions {
Unique = true
};
}
}
else if (!String.IsNullOrWhiteSpace(index.Field))
{
if (index.Order == Order.Ascending)
{
idx = Builders <BsonDocument> .IndexKeys.Ascending(index.Field);
}
else
{
idx = Builders <BsonDocument> .IndexKeys.Descending(index.Field);
}
if (index.Unique.HasValue && index.Unique.Value == true)
{
options = new CreateIndexOptions {
Unique = true
};
}
}
else
{
// TODO:
throw new Exception("");
}
indexes.Add(new CreateIndexModel <BsonDocument>(idx, options));
}
IMongoCollection <BsonDocument> collection = db.GetCollection <BsonDocument>(container.Name);
collection.Indexes.CreateMany(indexes);
}
}
}
String[] treeConfigurations = Directory.GetFiles(Path.Combine(solutionConfigurationRoot, "trees"), "*.json", SearchOption.AllDirectories);
foreach (String path in containerConfigurations)
{
}
}
//--- PRIVATE
/// <summary>
/// Returned object holds two collection references - one for the base
/// type of all records and the other for the record type specified
/// as generic parameter.
///
/// The need to hold two collection arises from the requirement
/// that query for a derived type takes into account that another
/// record with the same key and later dataset or object timestamp
/// may exist. For this reason, the typed collection is used for
/// LINQ constraints and base collection is used to iterate over
/// objects.
///
/// This method also creates indices if they do not exist. The
/// two default indices are always created: one for optimizing
/// loading by key and the other by query.
///
/// Additional indices may be created using class attribute
/// [IndexElements] for further performance optimization.
/// </summary>
private TemporalMongoCollection <TRecord> GetOrCreateCollection <TRecord>()
where TRecord : Record
{
// Check if collection object has already been cached
// for this type and return cached result if found
if (collectionDict_.TryGetValue(typeof(TRecord), out object collectionObj))
{
var cachedResult = collectionObj.CastTo <TemporalMongoCollection <TRecord> >();
return(cachedResult);
}
// Check that hierarchical discriminator convention is set for TRecord
var discriminatorConvention = BsonSerializer.LookupDiscriminatorConvention(typeof(TRecord));
if (!discriminatorConvention.Is <HierarchicalDiscriminatorConvention>())
{
throw new Exception(
$"Hierarchical discriminator convention is not set for type {typeof(TRecord).Name}. " +
$"The convention should have been set set in the static constructor of " +
$"MongoDataSource");
}
// Collection name is root class name of the record without prefix
string collectionName = DataTypeInfo.GetOrCreate <TRecord>().GetCollectionName();
// Get interfaces to base and typed collections for the same name
var baseCollection = Db.GetCollection <Record>(collectionName);
var typedCollection = Db.GetCollection <TRecord>(collectionName);
//--- Load standard index types
// Each data type has an index for optimized loading by key.
// This index consists of Key in ascending order, followed by
// DataSet and ID in descending order.
var loadIndexKeys = Builders <TRecord> .IndexKeys
.Ascending(new StringFieldDefinition <TRecord>("_key")) // .Key
.Descending(new StringFieldDefinition <TRecord>("_dataset")) // .DataSet
.Descending(new StringFieldDefinition <TRecord>("_id")); // .Id
// Use index definition convention to specify the index name
var loadIndexName = "Key-DataSet-Id";
var loadIndexModel = new CreateIndexModel <TRecord>(loadIndexKeys, new CreateIndexOptions {
Name = loadIndexName
});
typedCollection.Indexes.CreateOne(loadIndexModel);
//--- Load custom index types
// Additional indices are provided using IndexAttribute for the class.
// Get a sorted dictionary of (definition, name) pairs
// for the inheritance chain of the specified type.
var indexDict = IndexElementsAttribute.GetAttributesDict <TRecord>();
// Iterate over the dictionary to define the index
foreach (var indexInfo in indexDict)
{
string indexDefinition = indexInfo.Key;
string indexName = indexInfo.Value;
// Parse index definition to get a list of (ElementName,SortOrder) tuples
List <(string, int)> indexTokens = IndexElementsAttribute.ParseDefinition <TRecord>(indexDefinition);
var indexKeysBuilder = Builders <TRecord> .IndexKeys;
IndexKeysDefinition <TRecord> indexKeys = null;
// Iterate over (ElementName,SortOrder) tuples
foreach (var indexToken in indexTokens)
{
(string elementName, int sortOrder) = indexToken;
if (indexKeys == null)
{
// Create from builder for the first element
if (sortOrder == 1)
{
indexKeys = indexKeysBuilder.Ascending(new StringFieldDefinition <TRecord>(elementName));
}
else if (sortOrder == -1)
{
indexKeys = indexKeysBuilder.Descending(new StringFieldDefinition <TRecord>(elementName));
}
else
{
throw new Exception("Sort order must be 1 or -1.");
}
}
else
{
// Chain to the previous list of index keys for the remaining elements
if (sortOrder == 1)
{
indexKeys = indexKeys.Ascending(new StringFieldDefinition <TRecord>(elementName));
}
else if (sortOrder == -1)
{
indexKeys = indexKeys.Descending(new StringFieldDefinition <TRecord>(elementName));
}
else
{
throw new Exception("Sort order must be 1 or -1.");
}
}
}
if (indexName == null)
{
throw new Exception("Index name cannot be null.");
}
var indexModel = new CreateIndexModel <TRecord>(indexKeys, new CreateIndexOptions {
Name = indexName
});
// Add to indexes for the collection
typedCollection.Indexes.CreateOne(indexModel);
}
// Create result that holds both base and typed collections
TemporalMongoCollection <TRecord> result = new TemporalMongoCollection <TRecord>(this, baseCollection, typedCollection);
// Add the result to the collection dictionary and return
collectionDict_.TryAdd(typeof(TRecord), result);
return(result);
}