public void Read_Profile_Ok()
        {
            string json = LoadProfile("SampleProfile.json");
            IDataProfileSerializer serializer = new JsonDataProfileSerializer();

            DataProfile profile = serializer.Read(json);

            // facets
            Assert.Single(profile.Facets);
            FacetDefinition facetDef = profile.Facets[0];

            Assert.Equal("facet-default", facetDef.Id);
            Assert.Equal("default", facetDef.Label);
            Assert.Equal("The default facet", facetDef.Description);
            Assert.Equal("FF0000", facetDef.ColorKey);
            Assert.Equal(7, facetDef.PartDefinitions.Count);

            // TODO: check each facet definition

            // flags
            Assert.Single(profile.Flags);
            FlagDefinition flagDef = profile.Flags[0];

            Assert.Equal(1, flagDef.Id);
            Assert.Equal("to revise", flagDef.Label);
            Assert.Equal("The item must be revised.", flagDef.Description);
            Assert.Equal("F08080", flagDef.ColorKey);

            // thesauri
            Assert.Equal(2, profile.Thesauri.Length);
            Thesaurus thesaurus = Array.Find(profile.Thesauri,
                                             t => t.Id == "categories@en");

            Assert.NotNull(thesaurus);
            Assert.Equal(16, thesaurus.GetEntries().Count);
            // TODO: check each entry

            thesaurus = Array.Find(profile.Thesauri,
                                   t => t.Id == "languages@en");
            Assert.NotNull(thesaurus);
            Assert.Equal(8, thesaurus.GetEntries().Count);
            // TODO: check each entry
        }
        private static async Task SeedCadmusDatabaseAsync(
            IServiceProvider serviceProvider,
            IConfiguration config,
            ILogger logger)
        {
            // build connection string
            string connString   = config.GetConnectionString("Default");
            string databaseName = config["DatabaseNames:Data"];

            if (string.IsNullOrEmpty(databaseName))
            {
                return;
            }
            connString = string.Format(connString, databaseName);

            // nope if database exists
            IDatabaseManager manager = new MongoDatabaseManager();

            if (manager.DatabaseExists(connString))
            {
                return;
            }

            // load seed profile (nope if no profile)
            string profileSource = config["Seed:ProfileSource"];

            if (string.IsNullOrEmpty(profileSource))
            {
                return;
            }

            Console.WriteLine($"Loading seed profile from {profileSource}...");
            logger.LogInformation($"Loading seed profile from {profileSource}...");

            ResourceLoaderService loaderService =
                new ResourceLoaderService(serviceProvider);

            Stream stream = await loaderService.LoadAsync(profileSource);

            if (stream == null)
            {
                Console.WriteLine("Error: seed profile could not be loaded");
                return;
            }

            // deserialize the profile
            string profileContent;

            using (StreamReader reader = new StreamReader(stream, Encoding.UTF8))
            {
                profileContent = reader.ReadToEnd();
            }

            IDataProfileSerializer serializer = new JsonDataProfileSerializer();
            DataProfile            profile    = serializer.Read(profileContent);

            // issue warning on invalid profile
            Console.WriteLine("Validating profile...");
            string error = profile.Validate();

            if (error != null)
            {
                logger.LogWarning(error);
            }

            // create database
            Console.WriteLine("Creating database...");
            logger.LogInformation($"Creating database {connString}...");

            manager.CreateDatabase(connString, profile);

            Console.WriteLine("Database created.");
            logger.LogInformation("Database created.");

            // get the required services
            Console.WriteLine("Creating seeders factory...");
            Serilog.Log.Information("Creating seeders factory...");
            IPartSeederFactoryProvider seederService =
                serviceProvider.GetService <IPartSeederFactoryProvider>();
            PartSeederFactory factory = seederService.GetFactory(
                loaderService.ResolvePath(profileSource));

            Console.WriteLine("Creating repository...");
            Serilog.Log.Information("Creating repository...");
            IRepositoryProvider repositoryProvider =
                serviceProvider.GetService <IRepositoryProvider>();
            ICadmusRepository repository =
                repositoryProvider.CreateRepository(databaseName);

            // get seed count
            int    count       = 0;
            string configCount = config["Seed:ItemCount"];

            if (configCount != null && int.TryParse(configCount, out int n))
            {
                count = n;
            }

            // if required, seed data
            if (count > 0)
            {
                Console.WriteLine($"Seeding {count} items...");
                CadmusSeeder seeder = new CadmusSeeder(factory);

                foreach (IItem item in seeder.GetItems(count))
                {
                    Console.WriteLine($"{item}: {item.Parts.Count} parts");
                    repository.AddItem(item, true);
                    foreach (IPart part in item.Parts)
                    {
                        repository.AddPart(part, true);
                    }
                }
                Console.WriteLine("Seeding completed.");
            }

            // import data if required
            IList <string> sources = factory.GetImports();

            if (sources?.Count > 0)
            {
                PartImporter importer = new PartImporter(repository);

                foreach (string source in sources)
                {
                    foreach (string resolved in SourceRangeResolver.Resolve(source))
                    {
                        Console.WriteLine($"Importing from {resolved}...");
                        using Stream jsonStream =
                                  await loaderService.LoadAsync(resolved);

                        importer.Import(jsonStream);
                    }
                }
            }
        }
Beispiel #3
0
        public Task Run()
        {
            Console.WriteLine("IMPORT DATABASE\n" +
                              $"Input directory: {_inputDir}\n" +
                              $"Database: {_database}\n" +
                              $"Profile file: {_profileText}\n" +
                              $"Preflight: {(_preflight? "yes" : "no")}");
            Serilog.Log.Information("IMPORT DATABASE: " +
                                    $"Input directory: {_inputDir}, " +
                                    $"Database: {_database}, " +
                                    $"Profile file: {_profileText}, " +
                                    $"Preflight: {_preflight}");

            try
            {
                // create database if not exists
                string connection = string.Format(CultureInfo.InvariantCulture,
                                                  _config.GetConnectionString("Mongo"),
                                                  _database);

                IDatabaseManager manager        = new MongoDatabaseManager();
                string           profileContent = LoadProfile(_profileText);

                IDataProfileSerializer serializer = new JsonDataProfileSerializer();
                _profile = serializer.Read(profileContent);

                if (!_preflight)
                {
                    if (!manager.DatabaseExists(connection))
                    {
                        Console.WriteLine("Creating database...");
                        Serilog.Log.Information($"Creating database {_database}...");

                        manager.CreateDatabase(connection, _profile);

                        Console.WriteLine("Database created.");
                        Serilog.Log.Information("Database created.");
                    }
                }

                Console.WriteLine("Creating repository...");
                Serilog.Log.Information("Creating repository...");

                ICadmusRepository repository = _repositoryService.CreateRepository(_database);

                foreach (string lexFile in Directory.GetFiles(_inputDir, "??.xml").OrderBy(s => s))
                {
                    XDocument doc = XDocument.Load(lexFile, LoadOptions.PreserveWhitespace);
                    if (doc.Root == null)
                    {
                        continue;
                    }

                    foreach (XElement itemElement in doc.Root.Elements("item"))
                    {
                        // read essential metadata
                        string id    = itemElement.Attribute("id").Value;
                        string sid   = itemElement.Attribute("sid").Value;
                        string lemma = itemElement.Element("lemma").Value;
                        int    hom   = GetHomographNumber(itemElement);
                        int    flags = itemElement.ReadOptionalAttribute("flags", 0);
                        // TODO parent...

                        // item
                        Item item = new Item
                        {
                            Id          = id,
                            Title       = BuildWordKey(lemma, hom),
                            Description = BuildItemDescription(itemElement),
                            FacetId     = "facet-lex-word",
                            SortKey     = sid,
                            UserId      = USERID,
                            Flags       = flags
                        };
                        Console.WriteLine(item.Title);

                        // read parts
                        // first remove all the _bm so we don't accidentally include metatext
                        XElement filtered = XElement.Parse(itemElement.ToString(SaveOptions.DisableFormatting),
                                                           LoadOptions.PreserveWhitespace);
                        foreach (XElement bm in filtered.Descendants("_bm").ToList())
                        {
                            bm.Remove();
                        }

                        ReadParts(filtered, item);

                        if (!_preflight)
                        {
                            repository.AddItem(item, false);
                            foreach (IPart part in item.Parts)
                            {
                                repository.AddPart(part, false);
                            }
                        }
                    }
                }

                Console.WriteLine(" completed.");
                Serilog.Log.Information("Import completed");
                return(Task.FromResult(0));
            }
            catch (Exception ex)
            {
                Serilog.Log.Error(ex, ex.Message);
                throw;
            }
        }
Beispiel #4
0
        public Task Run()
        {
            Console.ForegroundColor = ConsoleColor.Red;
            Console.WriteLine("IMPORT JSON TEXT AND APPARATUS\n");
            Console.ResetColor();
            Console.WriteLine(
                $"Text dir:  {_txtFileDir}\n" +
                $"Text mask: {_txtFileMask}\n" +
                $"Apparatus dir: {_appFileDir}\n" +
                $"Profile file: {_profilePath}\n" +
                $"Database: {_database}\n" +
                $"Dry run: {_dry}\n");

            ILoggerFactory loggerFactory = new LoggerFactory();

            loggerFactory.AddSerilog(Log.Logger);
            Log.Logger.Information("IMPORT JSON TEXT AND APPARATUS");

            if (!_dry)
            {
                // create database if not exists
                string connection = string.Format(CultureInfo.InvariantCulture,
                                                  _config.GetConnectionString("Mongo"),
                                                  _database);

                IDatabaseManager manager = new MongoDatabaseManager();

                string profileContent             = LoadProfile(_profilePath);
                IDataProfileSerializer serializer = new JsonDataProfileSerializer();
                DataProfile            profile    = serializer.Read(profileContent);

                if (!manager.DatabaseExists(connection))
                {
                    Console.WriteLine("Creating database...");
                    Log.Information($"Creating database {_database}...");

                    manager.CreateDatabase(connection, profile);

                    Console.WriteLine("Database created.");
                    Log.Information("Database created.");
                }
            }
            else
            {
                if (!File.Exists(_profilePath))
                {
                    string error = "Profile path not found: " + _profilePath;
                    Console.WriteLine(error);
                    Log.Error(error);
                    return(Task.CompletedTask);
                }
            }

            ICadmusRepository repository =
                _repositoryService.CreateRepository(_database);

            JsonImporter importer = new JsonImporter(repository)
            {
                Logger = loggerFactory.CreateLogger("json-importer"),
                IsDry  = _dry
            };

            int inputFileCount = 0;

            // 1) import text
            string[] files = FileEnumerator.Enumerate(
                _txtFileDir, _txtFileMask, _regexMask).ToArray();
            HashSet <string> fileNames = new HashSet <string>();

            Console.WriteLine($"Importing text from {files.Length} file(s)...");

            foreach (string txtFilePath in files)
            {
                fileNames.Add(
                    StripFileNameNr(
                        Path.GetFileNameWithoutExtension(txtFilePath)));
                Console.WriteLine(txtFilePath);
                inputFileCount++;

                using (Stream stream = new FileStream(txtFilePath, FileMode.Open,
                                                      FileAccess.Read, FileShare.Read))
                {
                    importer.ImportText(stream);
                }
            }

            // 2) import apparatus
            Console.WriteLine("Importing apparatus...");

            foreach (string fileName in fileNames)
            {
                Console.WriteLine(fileName);

                foreach (string appFilePath in Directory.EnumerateFiles(
                             _appFileDir, fileName + "-app_*.json"))
                {
                    Console.WriteLine("  " + appFilePath);
                    using (Stream stream = new FileStream(appFilePath, FileMode.Open,
                                                          FileAccess.Read, FileShare.Read))
                    {
                        importer.ImportApparatus(stream);
                    }
                }
            }

            return(Task.CompletedTask);
        }
        public Task Run()
        {
            Console.ForegroundColor = ConsoleColor.Red;
            Console.WriteLine("SEED DATABASE\n");
            Console.ResetColor();

            Console.WriteLine($"Database: {_database}\n" +
                              $"Profile file: {_profilePath}\n" +
                              $"Count: {_count}\n" +
                              $"Dry run: {_dry}\n" +
                              $"History: {_history}\n");
            Serilog.Log.Information("SEED DATABASE: " +
                                    $"Database: {_database}, " +
                                    $"Profile file: {_profilePath}, " +
                                    $"Count: {_count}, " +
                                    $"Dry: {_dry}, " +
                                    $"History: {_history}");

            if (!_dry)
            {
                // create database if not exists
                string connection = string.Format(CultureInfo.InvariantCulture,
                                                  _config.GetConnectionString("Mongo"),
                                                  _database);

                IDatabaseManager manager = new MongoDatabaseManager();

                string profileContent             = LoadProfile(_profilePath);
                IDataProfileSerializer serializer = new JsonDataProfileSerializer();
                DataProfile            profile    = serializer.Read(profileContent);

                if (!manager.DatabaseExists(connection))
                {
                    Console.WriteLine("Creating database...");
                    Serilog.Log.Information($"Creating database {_database}...");

                    manager.CreateDatabase(connection, profile);

                    Console.WriteLine("Database created.");
                    Serilog.Log.Information("Database created.");
                }
            }

            Console.WriteLine("Creating repository...");
            Serilog.Log.Information("Creating repository...");

            ICadmusRepository repository = _dry
                ? null : _repositoryService.CreateRepository(_database);

            Console.WriteLine("Seeding items");
            PartSeederFactory factory = _seederService.GetFactory(_profilePath);
            CadmusSeeder      seeder  = new CadmusSeeder(factory);

            foreach (IItem item in seeder.GetItems(_count))
            {
                Console.WriteLine($"{item}: {item.Parts.Count} parts");
                if (!_dry)
                {
                    repository.AddItem(item, _history);
                    foreach (IPart part in item.Parts)
                    {
                        repository.AddPart(part, _history);
                    }
                }
            }
            Console.WriteLine("Completed.");

            return(Task.CompletedTask);
        }