public void ConvertFromTest()
{
// Arrange
SetupCalls();
var socSkillsMatrixConverter =
new SocCodeConverter(fakeDynamicContentExtensions);
//Act
socSkillsMatrixConverter.ConvertFrom(fakeDynamicContentItem);
//Assert
A.CallTo(() => fakeDynamicContentExtensions.GetFieldValue <Lstring>(A <DynamicContent> ._, A <string> ._))
.MustHaveHappened(4, Times.Exactly);
}
static async Task Main(string[] args)
{
string timestamp = $"{DateTime.UtcNow:O}";
IConfigurationRoot config = new ConfigurationBuilder()
.AddJsonFile($"appsettings.Development.json", optional: true)
.Build();
string jobProfilesToImport = config["JobProfilesToImport"];
bool createTestFiles = bool.Parse(config["CreateTestFiles"] ?? "False");
string[] socCodeList = !createTestFiles ? new string[] { } : config["TestSocCodes"].Split(',');
string[] oNetCodeList = !createTestFiles ? new string[] { } : config["TestONetCodes"].Split(',');
string[] apprenticeshipStandardsRefList = !createTestFiles ? new string[] { } : config["TestApprenticeshipStandardReferences"].Split(',');
string filenamePrefix = createTestFiles ? "TestData_" : "";
var socCodeConverter = new SocCodeConverter(socCodeList);
var socCodeDictionary = socCodeConverter.Go(timestamp);
using var reader = new StreamReader(@"SeedData\job_profiles_updated.xlsx");
var jobProfileWorkbook = new XSSFWorkbook(reader.BaseStream);
var oNetConverter = new ONetConverter(oNetCodeList);
var oNetDictionary = oNetConverter.Go(jobProfileWorkbook, timestamp);
var titleOptionsLookup = new TitleOptionsImporter().Import(jobProfileWorkbook);
//use these knobs to work around rate - limiting
const int skip = 0;
const int take = 0;
const int napTimeMs = 5500;
// max number of contentitems in an import recipe
const int batchSize = 400;
const int jobProfileBatchSize = 200;
const int occupationLabelsBatchSize = 1000;
const int occupationsBatchSize = 400;
const int skillBatchSize = 400;
const int skillLabelsBatchSize = 1000;
var httpClient = new HttpClient
{
BaseAddress = new Uri("https://pp.api.nationalcareers.service.gov.uk/job-profiles/"),
DefaultRequestHeaders =
{
{ "version", "v1" },
{ "Ocp-Apim-Subscription-Key", config["Ocp-Apim-Subscription-Key"] }
}
};
var dysacImporter = new DysacImporter(oNetConverter.ONetOccupationalCodeToSocCodeDictionary, oNetConverter.ONetOccupationalCodeContentItems);
using var dysacJobProfileReader = new StreamReader(@"SeedData\dysac_job_profile_mappings.xlsx");
var dysacJobProfileWorkbook = new XSSFWorkbook(dysacJobProfileReader.BaseStream);
var client = new RestHttpClient.RestHttpClient(httpClient);
var converter = new JobProfileConverter(client, socCodeDictionary, oNetDictionary, titleOptionsLookup, timestamp);
await converter.Go(skip, take, napTimeMs, jobProfilesToImport);
var jobProfiles = converter.JobProfiles.ToArray();
List <string> mappedOccupationUris = new EscoJobProfileMapper().Map(jobProfiles);
var jobCategoryImporter = new JobCategoryImporter();
jobCategoryImporter.Import(jobProfileWorkbook, timestamp, jobProfiles);
var qcfLevelBuilder = new QCFLevelBuilder();
qcfLevelBuilder.Build(timestamp);
var apprenticeshipStandardImporter = new ApprenticeshipStandardImporter(apprenticeshipStandardsRefList);
apprenticeshipStandardImporter.Import(jobProfileWorkbook, timestamp, qcfLevelBuilder.QCFLevelDictionary, jobProfiles);
using var dysacReader = new StreamReader(@"SeedData\dysac.xlsx");
var dysacWorkbook = new XSSFWorkbook(dysacReader.BaseStream);
dysacImporter.ImportONetSkillRank(jobProfileWorkbook);
dysacImporter.ImportTraits(jobCategoryImporter.JobCategoryContentItemIdDictionary, dysacWorkbook, timestamp);
dysacImporter.ImportShortQuestions(dysacWorkbook, timestamp);
dysacImporter.ImportQuestionSet(timestamp);
dysacImporter.BuildONetOccupationalSkills(dysacJobProfileWorkbook);
const string cypherCommandRecipesPath = "CypherCommandRecipes";
string whereClause = "";
string occupationMatch = "";
int totalOccupations = 2942;
int totalOccupationLabels = int.Parse(config["totalOccupationLabels"] ?? "33036");
int totalSkillLabels = int.Parse(config["totalSkillLabels"] ?? "97816");
int totalSkills = int.Parse(config["totalSkills"] ?? "13485");
if (!string.IsNullOrWhiteSpace(jobProfilesToImport) && jobProfilesToImport != "*")
{
string uriList = string.Join(',', mappedOccupationUris.Select(u => $"'{u}'"));
whereClause = $"where o.uri in [{uriList}]";
totalOccupations = mappedOccupationUris.Count;
occupationMatch = " (o:esco__Occupation) --> ";
}
IDictionary <string, string> tokens = new Dictionary <string, string>
{
{ "whereClause", whereClause },
{ "occupationMatch", occupationMatch }
};
NewMasterRecipe("main");
bool excludeGraphContentMutators = bool.Parse(config["ExcludeGraphContentMutators"] ?? "False");
if (!(excludeGraphContentMutators || createTestFiles))
{
await CopyRecipeWithTokenisation(cypherCommandRecipesPath, "CreateOccupationLabelNodes", tokens);
await CopyRecipeWithTokenisation(cypherCommandRecipesPath, "CreateOccupationPrefLabelNodes", tokens);
await CopyRecipeWithTokenisation(cypherCommandRecipesPath, "CreateSkillLabelNodes", tokens);
await CopyRecipe(cypherCommandRecipesPath, "CleanUpEscoData");
}
bool excludeGraphIndexMutators = bool.Parse(config["ExcludeGraphIndexMutators"] ?? "False");
if (!(excludeGraphIndexMutators || createTestFiles))
{
await CopyRecipe(cypherCommandRecipesPath, "CreateFullTextSearchIndexes");
}
const string cypherToContentRecipesPath = "CypherToContentRecipes";
await BatchRecipes(cypherToContentRecipesPath, "CreateOccupationLabelContentItems", occupationLabelsBatchSize, "OccupationLabels", totalOccupationLabels, tokens);
await BatchRecipes(cypherToContentRecipesPath, "CreateSkillLabelContentItems", skillLabelsBatchSize, "SkillLabels", totalSkillLabels, tokens);
await BatchRecipes(cypherToContentRecipesPath, "CreateSkillContentItems", skillBatchSize, "Skills", totalSkills, tokens);
await BatchRecipes(cypherToContentRecipesPath, "CreateOccupationContentItems", occupationsBatchSize, "Occupations", totalOccupations, tokens);
ProcessJobProfileSpreadsheet(jobProfileWorkbook);
converter.UpdateRouteItemsWithSharedNames();
// remove calls to pages related recipe file generation
// comment out as may need to re-generate later
const string contentRecipesPath = "ContentRecipes";
await BatchSerializeToFiles(qcfLevelBuilder.QCFLevelContentItems, batchSize, $"{filenamePrefix}QCFLevels");
await BatchSerializeToFiles(apprenticeshipStandardImporter.ApprenticeshipStandardRouteContentItems, batchSize, $"{filenamePrefix}ApprenticeshipStandardRoutes");
await BatchSerializeToFiles(apprenticeshipStandardImporter.ApprenticeshipStandardContentItems, batchSize, $"{filenamePrefix}ApprenticeshipStandards");
await BatchSerializeToFiles(RouteFactory.RequirementsPrefixes.IdLookup.Select(r => new RequirementsPrefixContentItem(r.Key, timestamp, r.Key, r.Value)), batchSize, $"{filenamePrefix}RequirementsPrefixes");
await BatchSerializeToFiles(converter.ApprenticeshipRoutes.Links.IdLookup.Select(r => new ApprenticeshipLinkContentItem(GetTitle("ApprenticeshipLink", r.Key), r.Key, timestamp, r.Value)), batchSize, $"{filenamePrefix}ApprenticeshipLinks");
await BatchSerializeToFiles(converter.ApprenticeshipRoutes.Requirements.IdLookup.Select(r => new ApprenticeshipRequirementContentItem(GetTitle("ApprenticeshipRequirement", r.Key), timestamp, r.Key, r.Value)), batchSize, $"{filenamePrefix}ApprenticeshipRequirements");
await BatchSerializeToFiles(converter.CollegeRoutes.Links.IdLookup.Select(r => new CollegeLinkContentItem(GetTitle("CollegeLink", r.Key), r.Key, timestamp, r.Value)), batchSize, $"{filenamePrefix}CollegeLinks");
await BatchSerializeToFiles(converter.CollegeRoutes.Requirements.IdLookup.Select(r => new CollegeRequirementContentItem(GetTitle("CollegeRequirement", r.Key), timestamp, r.Key, r.Value)), batchSize, $"{filenamePrefix}CollegeRequirements");
await BatchSerializeToFiles(converter.UniversityRoutes.Links.IdLookup.Select(r => new UniversityLinkContentItem(GetTitle("UniversityLink", r.Key), r.Key, timestamp, r.Value)), batchSize, $"{filenamePrefix}UniversityLinks");
await BatchSerializeToFiles(converter.UniversityRoutes.Requirements.IdLookup.Select(r => new UniversityRequirementContentItem(GetTitle("UniversityRequirement", r.Key), timestamp, r.Key, r.Value)), batchSize, $"{filenamePrefix}UniversityRequirements");
await BatchSerializeToFiles(converter.DayToDayTasks.Select(x => new DayToDayTaskContentItem(x.Key, timestamp, x.Key, x.Value.id)), batchSize, $"{filenamePrefix}DayToDayTasks");
await BatchSerializeToFiles(converter.OtherRequirements.IdLookup.Select(r => new OtherRequirementContentItem(r.Key, timestamp, r.Key, r.Value)), batchSize, $"{filenamePrefix}OtherRequirements");
await BatchSerializeToFiles(converter.Registrations.IdLookup.Select(r => new RegistrationContentItem(GetTitle("Registration", r.Key), timestamp, r.Key, r.Value)), batchSize, $"{filenamePrefix}Registrations");
await BatchSerializeToFiles(converter.Restrictions.IdLookup.Select(r => new RestrictionContentItem(GetTitle("Restriction", r.Key), timestamp, r.Key, r.Value)), batchSize, $"{filenamePrefix}Restrictions");
await BatchSerializeToFiles(socCodeConverter.SocCodeContentItems, batchSize, $"{filenamePrefix}SocCodes");
await CopyRecipe(contentRecipesPath, "ONetSkill");
await BatchSerializeToFiles(oNetConverter.ONetOccupationalCodeContentItems, batchSize, $"{filenamePrefix}ONetOccupationalCodes", CSharpContentStep.StepName);
await CopyRecipeWithTokenisation(cypherCommandRecipesPath, "ONetSkillMappings", new Dictionary <string, string>
{
{ "commandText", string.Join($"{Environment.NewLine},", dysacImporter.ONetSkillCypherCommands) }
});
await BatchSerializeToFiles(converter.WorkingEnvironments.IdLookup.Select(x => new WorkingEnvironmentContentItem(GetTitle("Environment", x.Key), timestamp, x.Key, x.Value)), batchSize, $"{filenamePrefix}WorkingEnvironments");
await BatchSerializeToFiles(converter.WorkingLocations.IdLookup.Select(x => new WorkingLocationContentItem(GetTitle("Location", x.Key), timestamp, x.Key, x.Value)), batchSize, $"{filenamePrefix}WorkingLocations");
await BatchSerializeToFiles(converter.WorkingUniforms.IdLookup.Select(x => new WorkingUniformContentItem(GetTitle("Uniform", x.Key), timestamp, x.Key, x.Value)), batchSize, $"{filenamePrefix}WorkingUniforms");
await BatchSerializeToFiles(converter.ApprenticeshipRoute.ItemToCompositeName.Keys, batchSize, $"{filenamePrefix}ApprenticeshipRoutes");
await BatchSerializeToFiles(converter.CollegeRoute.ItemToCompositeName.Keys, batchSize, $"{filenamePrefix}CollegeRoutes");
await BatchSerializeToFiles(converter.UniversityRoute.ItemToCompositeName.Keys, batchSize, $"{filenamePrefix}UniversityRoutes");
await BatchSerializeToFiles(converter.DirectRoute.ItemToCompositeName.Keys, batchSize, $"{filenamePrefix}DirectRoutes");
await BatchSerializeToFiles(converter.OtherRoute.ItemToCompositeName.Keys, batchSize, $"{filenamePrefix}OtherRoutes");
await BatchSerializeToFiles(converter.VolunteeringRoute.ItemToCompositeName.Keys, batchSize, $"{filenamePrefix}VolunteeringRoutes");
await BatchSerializeToFiles(converter.WorkRoute.ItemToCompositeName.Keys, batchSize, $"{filenamePrefix}WorkRoutes");
await BatchSerializeToFiles(jobProfiles, jobProfileBatchSize, $"{filenamePrefix}JobProfiles", CSharpContentStep.StepName);
await BatchSerializeToFiles(jobCategoryImporter.JobCategoryContentItems, batchSize, $"{filenamePrefix}JobCategories");
await BatchSerializeToFiles(dysacImporter.PersonalityTraitContentItems, batchSize, $"{filenamePrefix}PersonalityTrait", CSharpContentStep.StepName);
await BatchSerializeToFiles(dysacImporter.PersonalityShortQuestionContentItems, batchSize, $"{filenamePrefix}PersonalityShortQuestion", CSharpContentStep.StepName);
await BatchSerializeToFiles(dysacImporter.PersonalityQuestionSetContentItems, batchSize, $"{filenamePrefix}PersonalityQuestionSet", CSharpContentStep.StepName);
await CopyRecipe(contentRecipesPath, "PersonalityFilteringQuestion");
string masterRecipeName = config["MasterRecipeName"] ?? "master";
await WriteMasterRecipesFiles(masterRecipeName);
await File.WriteAllTextAsync($"{OutputBasePath}content items count_{_executionId}.txt", @$ "{_importFilesReport}# Totals
{_importTotalsReport}");
