/// <summary>
/// Imports the specified text stream.
/// </summary>
/// <param name="stream">The text stream.</param>
/// <exception cref="ArgumentNullException">txtStream or appStream</exception>
public void ImportText(Stream stream)
{
if (stream == null)
{
throw new ArgumentNullException(nameof(stream));
}
JsonDocument doc = JsonDocument.Parse(stream, _options);
// for each item
foreach (JsonElement itemElem in doc.RootElement.EnumerateArray())
{
// read its metadata
IItem item = ReadItem(itemElem);
// import it
Logger?.LogInformation("Importing item {ItemId}: {Title}",
item.Id, item.Title);
if (!IsDry)
{
_repository.AddItem(item);
}
// import its parts
foreach (JsonElement partElem in itemElem.GetProperty("parts")
.EnumerateArray())
{
if (!IsDry)
{
_repository.AddPartFromContent(partElem.ToString());
}
}
}
}
public IActionResult AddItem(
[FromRoute] string database,
[FromBody] ItemBindingModel model)
{
if (!ModelState.IsValid)
{
return(BadRequest(ModelState));
}
Item item = new Item
{
Title = model.Title,
Description = model.Description,
FacetId = model.FacetId,
SortKey = model.SortKey,
Flags = model.Flags,
// override the user ID
UserId = User.Identity.Name,
};
// set the creator ID if not specified
if (string.IsNullOrEmpty(item.CreatorId))
{
item.CreatorId = User.Identity.Name;
}
// set the item's ID if specified, else go with the default
// newly generated one
if (!string.IsNullOrEmpty(model.Id) && _guidRegex.IsMatch(model.Id))
{
item.Id = model.Id;
}
_logger.Information("User {UserName} saving item {ItemId} from {IP}",
User.Identity.Name,
item.Id,
HttpContext.Connection.RemoteIpAddress);
ICadmusRepository repository =
_repositoryProvider.CreateRepository(database);
repository.AddItem(item);
return(CreatedAtRoute("GetItem", new
{
database,
id = item.Id,
parts = false
}, item));
}
private int ImportItemsWithParts(JsonDocument doc)
{
int count = 0;
// for each item
foreach (JsonElement itemElem in doc.RootElement.EnumerateArray())
{
// read its metadata
IItem item = ReadItem(itemElem);
// import it
_repository.AddItem(item);
// import its parts
foreach (JsonElement partElem in itemElem.GetProperty("parts")
.EnumerateArray())
{
_repository.AddPartFromContent(partElem.ToString());
count++;
}
}
return(count);
}
private static async Task SeedCadmusDatabaseAsync(
IServiceProvider serviceProvider,
IConfiguration config,
ILogger logger)
{
// build connection string
string connString = config.GetConnectionString("Default");
string databaseName = config["DatabaseNames:Data"];
if (string.IsNullOrEmpty(databaseName))
{
return;
}
connString = string.Format(connString, databaseName);
// nope if database exists
IDatabaseManager manager = new MongoDatabaseManager();
if (manager.DatabaseExists(connString))
{
return;
}
// load seed profile (nope if no profile)
string profileSource = config["Seed:ProfileSource"];
if (string.IsNullOrEmpty(profileSource))
{
return;
}
Console.WriteLine($"Loading seed profile from {profileSource}...");
logger.LogInformation($"Loading seed profile from {profileSource}...");
ResourceLoaderService loaderService =
new ResourceLoaderService(serviceProvider);
Stream stream = await loaderService.LoadAsync(profileSource);
if (stream == null)
{
Console.WriteLine("Error: seed profile could not be loaded");
return;
}
// deserialize the profile
string profileContent;
using (StreamReader reader = new StreamReader(stream, Encoding.UTF8))
{
profileContent = reader.ReadToEnd();
}
IDataProfileSerializer serializer = new JsonDataProfileSerializer();
DataProfile profile = serializer.Read(profileContent);
// issue warning on invalid profile
Console.WriteLine("Validating profile...");
string error = profile.Validate();
if (error != null)
{
logger.LogWarning(error);
}
// create database
Console.WriteLine("Creating database...");
logger.LogInformation($"Creating database {connString}...");
manager.CreateDatabase(connString, profile);
Console.WriteLine("Database created.");
logger.LogInformation("Database created.");
// get the required services
Console.WriteLine("Creating seeders factory...");
Serilog.Log.Information("Creating seeders factory...");
IPartSeederFactoryProvider seederService =
serviceProvider.GetService <IPartSeederFactoryProvider>();
PartSeederFactory factory = seederService.GetFactory(
loaderService.ResolvePath(profileSource));
Console.WriteLine("Creating repository...");
Serilog.Log.Information("Creating repository...");
IRepositoryProvider repositoryProvider =
serviceProvider.GetService <IRepositoryProvider>();
ICadmusRepository repository =
repositoryProvider.CreateRepository(databaseName);
// get seed count
int count = 0;
string configCount = config["Seed:ItemCount"];
if (configCount != null && int.TryParse(configCount, out int n))
{
count = n;
}
// if required, seed data
if (count > 0)
{
Console.WriteLine($"Seeding {count} items...");
CadmusSeeder seeder = new CadmusSeeder(factory);
foreach (IItem item in seeder.GetItems(count))
{
Console.WriteLine($"{item}: {item.Parts.Count} parts");
repository.AddItem(item, true);
foreach (IPart part in item.Parts)
{
repository.AddPart(part, true);
}
}
Console.WriteLine("Seeding completed.");
}
// import data if required
IList <string> sources = factory.GetImports();
if (sources?.Count > 0)
{
PartImporter importer = new PartImporter(repository);
foreach (string source in sources)
{
foreach (string resolved in SourceRangeResolver.Resolve(source))
{
Console.WriteLine($"Importing from {resolved}...");
using Stream jsonStream =
await loaderService.LoadAsync(resolved);
importer.Import(jsonStream);
}
}
}
}
public Task Run()
{
Console.WriteLine("IMPORT DATABASE\n" +
$"Input directory: {_inputDir}\n" +
$"Database: {_database}\n" +
$"Profile file: {_profileText}\n" +
$"Preflight: {(_preflight? "yes" : "no")}");
Serilog.Log.Information("IMPORT DATABASE: " +
$"Input directory: {_inputDir}, " +
$"Database: {_database}, " +
$"Profile file: {_profileText}, " +
$"Preflight: {_preflight}");
try
{
// create database if not exists
string connection = string.Format(CultureInfo.InvariantCulture,
_config.GetConnectionString("Mongo"),
_database);
IDatabaseManager manager = new MongoDatabaseManager();
string profileContent = LoadProfile(_profileText);
IDataProfileSerializer serializer = new JsonDataProfileSerializer();
_profile = serializer.Read(profileContent);
if (!_preflight)
{
if (!manager.DatabaseExists(connection))
{
Console.WriteLine("Creating database...");
Serilog.Log.Information($"Creating database {_database}...");
manager.CreateDatabase(connection, _profile);
Console.WriteLine("Database created.");
Serilog.Log.Information("Database created.");
}
}
Console.WriteLine("Creating repository...");
Serilog.Log.Information("Creating repository...");
ICadmusRepository repository = _repositoryService.CreateRepository(_database);
foreach (string lexFile in Directory.GetFiles(_inputDir, "??.xml").OrderBy(s => s))
{
XDocument doc = XDocument.Load(lexFile, LoadOptions.PreserveWhitespace);
if (doc.Root == null)
{
continue;
}
foreach (XElement itemElement in doc.Root.Elements("item"))
{
// read essential metadata
string id = itemElement.Attribute("id").Value;
string sid = itemElement.Attribute("sid").Value;
string lemma = itemElement.Element("lemma").Value;
int hom = GetHomographNumber(itemElement);
int flags = itemElement.ReadOptionalAttribute("flags", 0);
// TODO parent...
// item
Item item = new Item
{
Id = id,
Title = BuildWordKey(lemma, hom),
Description = BuildItemDescription(itemElement),
FacetId = "facet-lex-word",
SortKey = sid,
UserId = USERID,
Flags = flags
};
Console.WriteLine(item.Title);
// read parts
// first remove all the _bm so we don't accidentally include metatext
XElement filtered = XElement.Parse(itemElement.ToString(SaveOptions.DisableFormatting),
LoadOptions.PreserveWhitespace);
foreach (XElement bm in filtered.Descendants("_bm").ToList())
{
bm.Remove();
}
ReadParts(filtered, item);
if (!_preflight)
{
repository.AddItem(item, false);
foreach (IPart part in item.Parts)
{
repository.AddPart(part, false);
}
}
}
}
Console.WriteLine(" completed.");
Serilog.Log.Information("Import completed");
return(Task.FromResult(0));
}
catch (Exception ex)
{
Serilog.Log.Error(ex, ex.Message);
throw;
}
}
public Task Run()
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("SEED DATABASE\n");
Console.ResetColor();
Console.WriteLine($"Database: {_database}\n" +
$"Profile file: {_profilePath}\n" +
$"Count: {_count}\n" +
$"Dry run: {_dry}\n" +
$"History: {_history}\n");
Serilog.Log.Information("SEED DATABASE: " +
$"Database: {_database}, " +
$"Profile file: {_profilePath}, " +
$"Count: {_count}, " +
$"Dry: {_dry}, " +
$"History: {_history}");
if (!_dry)
{
// create database if not exists
string connection = string.Format(CultureInfo.InvariantCulture,
_config.GetConnectionString("Mongo"),
_database);
IDatabaseManager manager = new MongoDatabaseManager();
string profileContent = LoadProfile(_profilePath);
IDataProfileSerializer serializer = new JsonDataProfileSerializer();
DataProfile profile = serializer.Read(profileContent);
if (!manager.DatabaseExists(connection))
{
Console.WriteLine("Creating database...");
Serilog.Log.Information($"Creating database {_database}...");
manager.CreateDatabase(connection, profile);
Console.WriteLine("Database created.");
Serilog.Log.Information("Database created.");
}
}
Console.WriteLine("Creating repository...");
Serilog.Log.Information("Creating repository...");
ICadmusRepository repository = _dry
? null : _repositoryService.CreateRepository(_database);
Console.WriteLine("Seeding items");
PartSeederFactory factory = _seederService.GetFactory(_profilePath);
CadmusSeeder seeder = new CadmusSeeder(factory);
foreach (IItem item in seeder.GetItems(_count))
{
Console.WriteLine($"{item}: {item.Parts.Count} parts");
if (!_dry)
{
repository.AddItem(item, _history);
foreach (IPart part in item.Parts)
{
repository.AddPart(part, _history);
}
}
}
Console.WriteLine("Completed.");
return(Task.CompletedTask);
}