private static ILifetimeScope RegisterContainer(IConfiguration configuration)
{
var builder = new ContainerBuilder();
builder.RegisterInstance(configuration)
.As <IConfiguration>()
.As <IConfigurationRoot>()
.SingleInstance();
builder.RegisterInstance(BulkLoadClientConfiguration.Create(configuration))
.As <IApiConfiguration>()
.As <IHashCacheConfiguration>()
.As <IDataConfiguration>()
.As <IOAuthTokenConfiguration>()
.As <IApiMetadataConfiguration>()
.As <IXsdConfiguration>()
.As <IInterchangeOrderConfiguration>()
.As <IThrottleConfiguration>()
.AsSelf()
.SingleInstance();
builder.RegisterModule(new LoadToolsModule());
if (configuration.GetValue <bool>("IncludeStats"))
{
builder.RegisterModule(new IncludeStatsModule());
}
return(builder.Build());
}
public XsdFilesRetriever(BulkLoadClientConfiguration configuration,
IXsdMetadataInformationProvider xsdMetadataInformationProvider,
IXsdMetadataFilesProvider xsdMetadataFilesProvider,
IRemoteFileDownloader remoteFileDownloader,
OdsVersionInformation odsVersionInformation)
{
_configuration = configuration;
_xsdMetadataInformationProvider = xsdMetadataInformationProvider;
_xsdMetadataFilesProvider = xsdMetadataFilesProvider;
_remoteFileDownloader = remoteFileDownloader;
_odsVersionInformation = odsVersionInformation;
_httpClient = new HttpClient {
Timeout = new TimeSpan(0, 0, 5, 0)
};
_xsdMedataUrl = _configuration.XsdMetadataUrl;
_extension = _configuration.Extension;
_xsdFolder = Path.GetFullPath(_configuration.XsdFolder);
_workingFolder = Path.GetFullPath(_configuration.WorkingFolder);
ValidateXsdFolder();
void ValidateXsdFolder()
{
// create the xsd folder if it does not exists so that the dependencies work.
if (!Directory.Exists(_xsdFolder))
{
Directory.CreateDirectory(_xsdFolder);
}
}
}
public static async Task <int> Run(IConfiguration configuration)
{
int exitCode;
try
{
var odsVersionInformation = await SetOdsEndpoints(configuration);
var bulkLoadClientConfiguration = BulkLoadClientConfiguration.Create(configuration);
var container = RegisterContainer(configuration, odsVersionInformation, bulkLoadClientConfiguration);
await using var scope = container.BeginLifetimeScope();
var validator = container.Resolve <IBulkLoadClientConfigurationValidator>();
if (!validator.IsValid())
{
throw new InvalidOperationException(validator.ErrorText);
}
await LoadXsdFiles(bulkLoadClientConfiguration, odsVersionInformation);
// run application
exitCode = await container.Resolve <IApiLoaderApplication>().Run();
}
catch (Exception ex)
{
exitCode = 1;
_log.Error(ex);
}
return(exitCode);
}
private static async Task LoadXsdFiles(BulkLoadClientConfiguration bulkLoadClientConfiguration,
OdsVersionInformation odsVersionInformation)
{
using var xsdFilesRetriever = new XsdFilesRetriever(
bulkLoadClientConfiguration,
new XsdMetadataInformationProvider(),
new XsdMetadataFilesProvider(),
new RemoteFileDownloader(),
odsVersionInformation);
await xsdFilesRetriever.DownloadXsdFilesAsync();
}
public static async Task <BulkLoadValidationResult> ValidateXmlFile(IConfiguration configuration,
string[] unsupportedInterchanges)
{
var odsVersionInformation = await SetOdsEndpoints(configuration);
var xsdConfiguration = BulkLoadClientConfiguration.Create(configuration);
await LoadXsdFiles(xsdConfiguration, odsVersionInformation);
var validationErrors = new BulkLoadValidationResult();
var streamsRetriever = new XsdStreamsRetriever(xsdConfiguration);
var factory = new SchemaSetFactory(streamsRetriever);
var schemaSet = factory.GetSchemaSet();
foreach (var fileToImport in GetFilesToImport(xsdConfiguration))
{
var xmlReaderSettings = new XmlReaderSettings
{
ValidationType = ValidationType.Schema,
Schemas = schemaSet,
ValidationFlags = XmlSchemaValidationFlags.AllowXmlAttributes |
XmlSchemaValidationFlags.ReportValidationWarnings |
XmlSchemaValidationFlags.ProcessIdentityConstraints
};
xmlReaderSettings.ValidationEventHandler += (s, e) => { validationErrors.Add(e.Message); };
await using var fileStream = new FileStream(fileToImport, FileMode.Open, FileAccess.Read);
using (var reader = XmlReader.Create(fileStream, xmlReaderSettings))
{
while (reader.Read())
{
if (reader.NodeType == XmlNodeType.Element && unsupportedInterchanges.Any(
i => i.Equals(reader.Name, StringComparison.InvariantCultureIgnoreCase)))
{
validationErrors.Add($"Import of data for {reader.Name} is not supported");
}
}
reader.Close();
}
fileStream.Close();
}
return(validationErrors);
}
public void OneTimeSetUp()
{
var configRoot = new ConfigurationBuilder()
.SetBasePath(Directory.GetParent(AppContext.BaseDirectory).FullName)
.AddJsonFile("appsettings.json", optional: true, reloadOnChange: true)
.Build();
_workingFolder = TestContext.CurrentContext.WorkDirectory;
_xsdFolder = Path.Combine(_workingFolder, "xsd");
configRoot["Folders:Working"] = _workingFolder;
configRoot["Folders:Xsd"] = _xsdFolder;
_configuration = BulkLoadClientConfiguration.Create(configRoot);
if (Directory.Exists(_xsdFolder))
{
Directory.Delete(_xsdFolder, true);
}
}
public void ShouldReplaceYearSuccessfully()
{
int year = DateTime.Now.Year;
var args = new[]
{
"-m",
"https://api-stage.ed-fi.org/YearSpecific_v5.2.0/api/metadata/1999",
"-a",
"https://api-stage.ed-fi.org/YearSpecific_v5.2.0/api/data/v3/1999",
"-g",
"https://api-stage.ed-fi.org/YearSpecific_v5.2.0/api/metadata/data/v3/1999/dependencies",
"-o",
"https://api-stage.ed-fi.org/YearSpecific_v5.2.0/api/oauth/token",
"-y",
year.ToString(),
};
var config = new ConfigurationBuilder()
.AddCommandLine(
args, new Dictionary <string, string>(StringComparer.InvariantCultureIgnoreCase)
{
{ "-a", "OdsApi:ApiUrl" },
{ "-y", "OdsApi:SchoolYear" },
{ "-m", "OdsApi:MetadataUrl" },
{ "-o", "OdsApi:OAuthUrl" },
{ "-g", "OdsApi:DependenciesUrl" }
}).
Build();
config["OdsApi:ApiMode"] = "Year Specific";
var result = BulkLoadClientConfiguration.Create(config);
result.ApiUrl.ShouldBe($"https://api-stage.ed-fi.org/YearSpecific_v5.2.0/api/data/v3/{year}");
result.MetadataUrl.ShouldBe($"https://api-stage.ed-fi.org/YearSpecific_v5.2.0/api/metadata/{year}");
result.DependenciesUrl.ShouldBe($"https://api-stage.ed-fi.org/YearSpecific_v5.2.0/api/metadata/data/v3/{year}/dependencies");
result.OauthUrl.ShouldBe($"https://api-stage.ed-fi.org/YearSpecific_v5.2.0/api/oauth/token");
}
private static IEnumerable <string> GetFilesToImport(BulkLoadClientConfiguration configuration)
{
return(Directory.EnumerateFiles(configuration.DataFolder));
}
