/// <summary>
/// Configures and returns a manage-courses dbContext
/// </summary>
public static ManageCoursesDbContext GetDbContext(IConfiguration config, bool enableRetryOnFailure = true)
{
var mcConfig = new McConfig(config);
var connectionString = mcConfig.BuildConnectionString();
const int maxRetryCount = 1; // don't actually allow retry for tests
const int maxRetryDelaySeconds = 1;
var postgresErrorCodesToConsiderTransient = new List <string>(); // ref: https://github.com/npgsql/Npgsql.EntityFrameworkCore.PostgreSQL/blob/16c8d07368cb92e10010b646098b562ecd5815d6/src/EFCore.PG/NpgsqlRetryingExecutionStrategy.cs#L99
// Configured to be similar to context setup in real app
// Importantly the retry is enabled as that is what the production code uses and that is incompatible with the normal transaction pattern.
// This will allow us to catch any re-introduction of following error before the code ships: "The configured execution strategy 'NpgsqlRetryingExecutionStrategy' does not support user initiated transactions. Use the execution strategy returned by 'DbContext.Database.CreateExecutionStrategy()' to execute all the operations in the transaction as a retriable unit."
var options = new DbContextOptionsBuilder <ManageCoursesDbContext>()
.UseNpgsql(connectionString, b =>
{
b.MigrationsAssembly((typeof(ManageCoursesDbContext).Assembly).ToString());
if (enableRetryOnFailure)
{
b.EnableRetryOnFailure(maxRetryCount, TimeSpan.FromSeconds(maxRetryDelaySeconds), postgresErrorCodesToConsiderTransient);
}
})
.Options;
return(new ManageCoursesDbContext(options));
}
private static ManageCoursesDbContext GetDbContext(McConfig config)
{
var connectionString = config.BuildConnectionString();
var options = new DbContextOptionsBuilder <ManageCoursesDbContext>()
.UseNpgsql(connectionString)
.Options;
return(new ManageCoursesDbContext(options));
}
public void SetUp()
{
var config = new Mock <IConfiguration>();
config.SetupGet(c => c["SETTINGS:MANAGE_BACKEND:SECRET"]).Returns("SETTINGS:MANAGE_BACKEND:SECRET");
var _mcConfig = new McConfig(config.Object);
_httpContextAccessor = new Mock <IHttpContextAccessor>();
var claims = new List <Claim>()
{
new Claim(ClaimTypes.NameIdentifier, "manage_courses_api"),
new Claim(ClaimTypes.Email, "*****@*****.**")
};
var identity = new ClaimsIdentity(
claims,
BearerTokenDefaults.AuthenticationScheme
);
_httpContextAccessor.Setup(x => x.HttpContext.User.Identity).Returns(identity);
_manageCoursesBackendJwtService = new ManageCoursesBackendJwtService(_httpContextAccessor.Object, _mcConfig);
}
public ManageCoursesBackendJwtService(IHttpContextAccessor httpContextAccessor, McConfig mcConfig)
{
_httpContextAccessor = httpContextAccessor;
var key = new SymmetricSecurityKey(Encoding.UTF8.GetBytes(mcConfig.ManageCoursesBackendKey));
_signingCredentials = new SigningCredentials(key, SecurityAlgorithms.HmacSha256);
}
public Publisher(IConfiguration configuration)
{
_logger = GetLogger(configuration);
_mcConfig = GetMcConfig(configuration);
}
private static McConfig GetMcConfig(IConfiguration configurationRoot)
{
var mcConfig = new McConfig(configurationRoot);
return(mcConfig);
}
static void Main(string[] args)
{
var configuration = GetConfiguration();
var telemetryClient = new TelemetryClient()
{
InstrumentationKey = configuration["APPINSIGHTS_INSTRUMENTATIONKEY"]
};
var logger = new LoggerConfiguration()
.ReadFrom.Configuration(configuration)
.WriteTo
.ApplicationInsightsTraces(configuration["APPINSIGHTS_INSTRUMENTATIONKEY"])
.Enrich.WithProperty("WebJob", "UcasCourseImporter")
.Enrich.WithProperty("WebJob_Identifer", Guid.NewGuid())
.Enrich.WithProperty("WebJob_Triggered_Date", DateTime.UtcNow)
.CreateLogger();
var folder = Path.Combine(Path.GetTempPath(), "ucasfiles", Guid.NewGuid().ToString());
try
{
logger.Information("UcasCourseImporter started.");
var configOptions = new UcasCourseImporterConfigurationOptions(configuration);
configOptions.Validate();
var mcConfig = new McConfig(configuration);
Directory.CreateDirectory(folder);
var downloadAndExtractor = new DownloaderAndExtractor(logger, folder, configOptions.AzureUrl,
configOptions.AzureSignature);
var unzipFolder = downloadAndExtractor.DownloadAndExtractLatest("NetupdateExtract");
var unzipFolderProfiles = downloadAndExtractor.DownloadAndExtractLatest("EntryProfilesExtract_test");
var xlsReader = new XlsReader(logger);
// only used to avoid importing orphaned data
// i.e. we do not import institutions but need them to determine which campuses to import
var subjects = xlsReader.ReadSubjects("data");
// entry profile data - used to correct institution data
var institutionProfiles = ReadInstitutionProfiles(unzipFolderProfiles);
// data to import
var institutions = xlsReader.ReadInstitutions(unzipFolder);
UpdateContactDetails(institutions, institutionProfiles);
var campuses = xlsReader.ReadCampuses(unzipFolder, institutions);
var courses = xlsReader.ReadCourses(unzipFolder, campuses);
var courseSubjects = xlsReader.ReadCourseSubjects(unzipFolder, courses, subjects);
var courseNotes = xlsReader.ReadCourseNotes(unzipFolder);
var noteTexts = xlsReader.ReadNoteText(unzipFolder);
var payload = new UcasPayload
{
Institutions = new List <Xls.Domain.UcasInstitution>(institutions),
Courses = new List <UcasCourse>(courses),
CourseSubjects = new List <UcasCourseSubject>(courseSubjects),
Campuses = new List <UcasCampus>(campuses),
CourseNotes = new List <UcasCourseNote>(courseNotes),
NoteTexts = new List <UcasNoteText>(noteTexts),
Subjects = new List <UcasSubject>(subjects)
};
var context = GetDbContext(mcConfig);
var ucasDataMigrator = new UcasDataMigrator(context, logger, payload);
ucasDataMigrator.UpdateUcasData();
}
catch (Exception e)
{
logger.Error(e, "UcasCourseImporter error.");
}
finally
{
CleanupTempData(folder, logger);
logger.Information("UcasCourseImporter finished.");
// flush logs and wait for them to be written. https://github.com/serilog/serilog-sinks-applicationinsights#how-when-and-why-to-flush-messages-manually
telemetryClient.Flush();
Thread.Sleep(5000);
}
}
