public void Read_Profile_Ok()
{
string json = LoadProfile("SampleProfile.json");
IDataProfileSerializer serializer = new JsonDataProfileSerializer();
DataProfile profile = serializer.Read(json);
// facets
Assert.Single(profile.Facets);
FacetDefinition facetDef = profile.Facets[0];
Assert.Equal("facet-default", facetDef.Id);
Assert.Equal("default", facetDef.Label);
Assert.Equal("The default facet", facetDef.Description);
Assert.Equal("FF0000", facetDef.ColorKey);
Assert.Equal(7, facetDef.PartDefinitions.Count);
// TODO: check each facet definition
// flags
Assert.Single(profile.Flags);
FlagDefinition flagDef = profile.Flags[0];
Assert.Equal(1, flagDef.Id);
Assert.Equal("to revise", flagDef.Label);
Assert.Equal("The item must be revised.", flagDef.Description);
Assert.Equal("F08080", flagDef.ColorKey);
// thesauri
Assert.Equal(2, profile.Thesauri.Length);
Thesaurus thesaurus = Array.Find(profile.Thesauri,
t => t.Id == "categories@en");
Assert.NotNull(thesaurus);
Assert.Equal(16, thesaurus.GetEntries().Count);
// TODO: check each entry
thesaurus = Array.Find(profile.Thesauri,
t => t.Id == "languages@en");
Assert.NotNull(thesaurus);
Assert.Equal(8, thesaurus.GetEntries().Count);
// TODO: check each entry
}
private static async Task SeedCadmusDatabaseAsync(
IServiceProvider serviceProvider,
IConfiguration config,
ILogger logger)
{
// build connection string
string connString = config.GetConnectionString("Default");
string databaseName = config["DatabaseNames:Data"];
if (string.IsNullOrEmpty(databaseName))
{
return;
}
connString = string.Format(connString, databaseName);
// nope if database exists
IDatabaseManager manager = new MongoDatabaseManager();
if (manager.DatabaseExists(connString))
{
return;
}
// load seed profile (nope if no profile)
string profileSource = config["Seed:ProfileSource"];
if (string.IsNullOrEmpty(profileSource))
{
return;
}
Console.WriteLine($"Loading seed profile from {profileSource}...");
logger.LogInformation($"Loading seed profile from {profileSource}...");
ResourceLoaderService loaderService =
new ResourceLoaderService(serviceProvider);
Stream stream = await loaderService.LoadAsync(profileSource);
if (stream == null)
{
Console.WriteLine("Error: seed profile could not be loaded");
return;
}
// deserialize the profile
string profileContent;
using (StreamReader reader = new StreamReader(stream, Encoding.UTF8))
{
profileContent = reader.ReadToEnd();
}
IDataProfileSerializer serializer = new JsonDataProfileSerializer();
DataProfile profile = serializer.Read(profileContent);
// issue warning on invalid profile
Console.WriteLine("Validating profile...");
string error = profile.Validate();
if (error != null)
{
logger.LogWarning(error);
}
// create database
Console.WriteLine("Creating database...");
logger.LogInformation($"Creating database {connString}...");
manager.CreateDatabase(connString, profile);
Console.WriteLine("Database created.");
logger.LogInformation("Database created.");
// get the required services
Console.WriteLine("Creating seeders factory...");
Serilog.Log.Information("Creating seeders factory...");
IPartSeederFactoryProvider seederService =
serviceProvider.GetService <IPartSeederFactoryProvider>();
PartSeederFactory factory = seederService.GetFactory(
loaderService.ResolvePath(profileSource));
Console.WriteLine("Creating repository...");
Serilog.Log.Information("Creating repository...");
IRepositoryProvider repositoryProvider =
serviceProvider.GetService <IRepositoryProvider>();
ICadmusRepository repository =
repositoryProvider.CreateRepository(databaseName);
// get seed count
int count = 0;
string configCount = config["Seed:ItemCount"];
if (configCount != null && int.TryParse(configCount, out int n))
{
count = n;
}
// if required, seed data
if (count > 0)
{
Console.WriteLine($"Seeding {count} items...");
CadmusSeeder seeder = new CadmusSeeder(factory);
foreach (IItem item in seeder.GetItems(count))
{
Console.WriteLine($"{item}: {item.Parts.Count} parts");
repository.AddItem(item, true);
foreach (IPart part in item.Parts)
{
repository.AddPart(part, true);
}
}
Console.WriteLine("Seeding completed.");
}
// import data if required
IList <string> sources = factory.GetImports();
if (sources?.Count > 0)
{
PartImporter importer = new PartImporter(repository);
foreach (string source in sources)
{
foreach (string resolved in SourceRangeResolver.Resolve(source))
{
Console.WriteLine($"Importing from {resolved}...");
using Stream jsonStream =
await loaderService.LoadAsync(resolved);
importer.Import(jsonStream);
}
}
}
}
public Task Run()
{
Console.WriteLine("IMPORT DATABASE\n" +
$"Input directory: {_inputDir}\n" +
$"Database: {_database}\n" +
$"Profile file: {_profileText}\n" +
$"Preflight: {(_preflight? "yes" : "no")}");
Serilog.Log.Information("IMPORT DATABASE: " +
$"Input directory: {_inputDir}, " +
$"Database: {_database}, " +
$"Profile file: {_profileText}, " +
$"Preflight: {_preflight}");
try
{
// create database if not exists
string connection = string.Format(CultureInfo.InvariantCulture,
_config.GetConnectionString("Mongo"),
_database);
IDatabaseManager manager = new MongoDatabaseManager();
string profileContent = LoadProfile(_profileText);
IDataProfileSerializer serializer = new JsonDataProfileSerializer();
_profile = serializer.Read(profileContent);
if (!_preflight)
{
if (!manager.DatabaseExists(connection))
{
Console.WriteLine("Creating database...");
Serilog.Log.Information($"Creating database {_database}...");
manager.CreateDatabase(connection, _profile);
Console.WriteLine("Database created.");
Serilog.Log.Information("Database created.");
}
}
Console.WriteLine("Creating repository...");
Serilog.Log.Information("Creating repository...");
ICadmusRepository repository = _repositoryService.CreateRepository(_database);
foreach (string lexFile in Directory.GetFiles(_inputDir, "??.xml").OrderBy(s => s))
{
XDocument doc = XDocument.Load(lexFile, LoadOptions.PreserveWhitespace);
if (doc.Root == null)
{
continue;
}
foreach (XElement itemElement in doc.Root.Elements("item"))
{
// read essential metadata
string id = itemElement.Attribute("id").Value;
string sid = itemElement.Attribute("sid").Value;
string lemma = itemElement.Element("lemma").Value;
int hom = GetHomographNumber(itemElement);
int flags = itemElement.ReadOptionalAttribute("flags", 0);
// TODO parent...
// item
Item item = new Item
{
Id = id,
Title = BuildWordKey(lemma, hom),
Description = BuildItemDescription(itemElement),
FacetId = "facet-lex-word",
SortKey = sid,
UserId = USERID,
Flags = flags
};
Console.WriteLine(item.Title);
// read parts
// first remove all the _bm so we don't accidentally include metatext
XElement filtered = XElement.Parse(itemElement.ToString(SaveOptions.DisableFormatting),
LoadOptions.PreserveWhitespace);
foreach (XElement bm in filtered.Descendants("_bm").ToList())
{
bm.Remove();
}
ReadParts(filtered, item);
if (!_preflight)
{
repository.AddItem(item, false);
foreach (IPart part in item.Parts)
{
repository.AddPart(part, false);
}
}
}
}
Console.WriteLine(" completed.");
Serilog.Log.Information("Import completed");
return(Task.FromResult(0));
}
catch (Exception ex)
{
Serilog.Log.Error(ex, ex.Message);
throw;
}
}
public Task Run()
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("IMPORT JSON TEXT AND APPARATUS\n");
Console.ResetColor();
Console.WriteLine(
$"Text dir: {_txtFileDir}\n" +
$"Text mask: {_txtFileMask}\n" +
$"Apparatus dir: {_appFileDir}\n" +
$"Profile file: {_profilePath}\n" +
$"Database: {_database}\n" +
$"Dry run: {_dry}\n");
ILoggerFactory loggerFactory = new LoggerFactory();
loggerFactory.AddSerilog(Log.Logger);
Log.Logger.Information("IMPORT JSON TEXT AND APPARATUS");
if (!_dry)
{
// create database if not exists
string connection = string.Format(CultureInfo.InvariantCulture,
_config.GetConnectionString("Mongo"),
_database);
IDatabaseManager manager = new MongoDatabaseManager();
string profileContent = LoadProfile(_profilePath);
IDataProfileSerializer serializer = new JsonDataProfileSerializer();
DataProfile profile = serializer.Read(profileContent);
if (!manager.DatabaseExists(connection))
{
Console.WriteLine("Creating database...");
Log.Information($"Creating database {_database}...");
manager.CreateDatabase(connection, profile);
Console.WriteLine("Database created.");
Log.Information("Database created.");
}
}
else
{
if (!File.Exists(_profilePath))
{
string error = "Profile path not found: " + _profilePath;
Console.WriteLine(error);
Log.Error(error);
return(Task.CompletedTask);
}
}
ICadmusRepository repository =
_repositoryService.CreateRepository(_database);
JsonImporter importer = new JsonImporter(repository)
{
Logger = loggerFactory.CreateLogger("json-importer"),
IsDry = _dry
};
int inputFileCount = 0;
// 1) import text
string[] files = FileEnumerator.Enumerate(
_txtFileDir, _txtFileMask, _regexMask).ToArray();
HashSet <string> fileNames = new HashSet <string>();
Console.WriteLine($"Importing text from {files.Length} file(s)...");
foreach (string txtFilePath in files)
{
fileNames.Add(
StripFileNameNr(
Path.GetFileNameWithoutExtension(txtFilePath)));
Console.WriteLine(txtFilePath);
inputFileCount++;
using (Stream stream = new FileStream(txtFilePath, FileMode.Open,
FileAccess.Read, FileShare.Read))
{
importer.ImportText(stream);
}
}
// 2) import apparatus
Console.WriteLine("Importing apparatus...");
foreach (string fileName in fileNames)
{
Console.WriteLine(fileName);
foreach (string appFilePath in Directory.EnumerateFiles(
_appFileDir, fileName + "-app_*.json"))
{
Console.WriteLine(" " + appFilePath);
using (Stream stream = new FileStream(appFilePath, FileMode.Open,
FileAccess.Read, FileShare.Read))
{
importer.ImportApparatus(stream);
}
}
}
return(Task.CompletedTask);
}
public Task Run()
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("SEED DATABASE\n");
Console.ResetColor();
Console.WriteLine($"Database: {_database}\n" +
$"Profile file: {_profilePath}\n" +
$"Count: {_count}\n" +
$"Dry run: {_dry}\n" +
$"History: {_history}\n");
Serilog.Log.Information("SEED DATABASE: " +
$"Database: {_database}, " +
$"Profile file: {_profilePath}, " +
$"Count: {_count}, " +
$"Dry: {_dry}, " +
$"History: {_history}");
if (!_dry)
{
// create database if not exists
string connection = string.Format(CultureInfo.InvariantCulture,
_config.GetConnectionString("Mongo"),
_database);
IDatabaseManager manager = new MongoDatabaseManager();
string profileContent = LoadProfile(_profilePath);
IDataProfileSerializer serializer = new JsonDataProfileSerializer();
DataProfile profile = serializer.Read(profileContent);
if (!manager.DatabaseExists(connection))
{
Console.WriteLine("Creating database...");
Serilog.Log.Information($"Creating database {_database}...");
manager.CreateDatabase(connection, profile);
Console.WriteLine("Database created.");
Serilog.Log.Information("Database created.");
}
}
Console.WriteLine("Creating repository...");
Serilog.Log.Information("Creating repository...");
ICadmusRepository repository = _dry
? null : _repositoryService.CreateRepository(_database);
Console.WriteLine("Seeding items");
PartSeederFactory factory = _seederService.GetFactory(_profilePath);
CadmusSeeder seeder = new CadmusSeeder(factory);
foreach (IItem item in seeder.GetItems(_count))
{
Console.WriteLine($"{item}: {item.Parts.Count} parts");
if (!_dry)
{
repository.AddItem(item, _history);
foreach (IPart part in item.Parts)
{
repository.AddPart(part, _history);
}
}
}
Console.WriteLine("Completed.");
return(Task.CompletedTask);
}