public void Check(ICheckNotifier notifier)
{
Catalogue[] catalogueMetadatas = null;
try
{
//check there are catalogues and we can retrieve them
catalogueMetadatas = _loadMetadata.GetAllCatalogues().Cast <Catalogue>().ToArray();
}
catch (Exception e)
{
notifier.OnCheckPerformed(new CheckEventArgs("Crashed trying to fetch Catalogues for metadata",
CheckResult.Fail, e));
return;
}
//there no catalogues
if (catalogueMetadatas.Length == 0)
{
notifier.OnCheckPerformed(new CheckEventArgs("There are no Catalogues associated with this metadata",
CheckResult.Fail, null));
}
List <ITableInfo> tablesFound = new List <ITableInfo>();
//check each catalogue is sufficiently configured to perform a migration
foreach (Catalogue catalogue in catalogueMetadatas)
{
notifier.OnCheckPerformed(new CheckEventArgs("Found Catalogue:" + catalogue, CheckResult.Success, null));
ITableInfo[] tableInfos = catalogue.GetTableInfoList(true).Distinct().ToArray();
if (tableInfos.Length == 0)
{
notifier.OnCheckPerformed(new CheckEventArgs(
"Catalogue " + catalogue.Name + " does not have any TableInfos", CheckResult.Fail, null));
}
tablesFound.AddRange(tableInfos.Where(tableInfo => !tablesFound.Contains(tableInfo)));
}
//check regular tables
foreach (TableInfo regularTable in tablesFound)
{
notifier.OnCheckPerformed(new CheckEventArgs("About To check configuration of TableInfo:" + regularTable, CheckResult.Success, null));
CheckTableInfo(regularTable, notifier);
//check anonymisation
AnonymisationChecks anonymisationChecks = new AnonymisationChecks(regularTable);
anonymisationChecks.Check(notifier);
}
}
public HICDataLoadFactory(ILoadMetadata loadMetadata, HICDatabaseConfiguration databaseConfiguration, HICLoadConfigurationFlags loadConfigurationFlags, ICatalogueRepository repository, ILogManager logManager)
{
_databaseConfiguration = databaseConfiguration;
_loadConfigurationFlags = loadConfigurationFlags;
_repository = repository;
_logManager = logManager;
LoadMetadata = loadMetadata;
// If we are not supplied any catalogues to load, it is expected that we will load all catalogues associated with the provided ILoadMetadata
_cataloguesToLoad = LoadMetadata.GetAllCatalogues().ToList();
if (!_cataloguesToLoad.Any())
{
throw new InvalidOperationException("LoadMetadata " + LoadMetadata.ID + " is not related to any Catalogues, there is nothing to load");
}
}
public void Check(ICheckNotifier notifier)
{
_notifier = notifier;
//For each table in load can we reach it and is it a valid table type
foreach (ITableInfo ti in _loadMetadata.GetAllCatalogues().SelectMany(c => c.GetTableInfoList(true)).Distinct())
{
DiscoveredTable tbl;
try
{
tbl = ti.Discover(DataAccessContext.DataLoad);
}
catch (Exception e)
{
HardFail = true;
notifier.OnCheckPerformed(new CheckEventArgs("Could not reach table in load '" + ti.Name + "'", CheckResult.Fail, e));
return;
}
if (!tbl.Exists())
{
HardFail = true;
notifier.OnCheckPerformed(new CheckEventArgs("Table '" + ti.Name + "' does not exist", CheckResult.Fail));
}
if (tbl.TableType != TableType.Table)
{
HardFail = true;
notifier.OnCheckPerformed(new CheckEventArgs("Table '" + ti + "' is a " + tbl.TableType, CheckResult.Fail));
}
}
if (HardFail)
{
return;
}
AtLeastOneTaskCheck();
PreExecutionStagingDatabaseCheck(false);
PreExecutionDatabaseCheck();
}
public DataLoadJob(IRDMPPlatformRepositoryServiceLocator repositoryLocator, string description, ILogManager logManager, ILoadMetadata loadMetadata, ILoadDirectory directory, IDataLoadEventListener listener, HICDatabaseConfiguration configuration)
{
_logManager = logManager;
RepositoryLocator = repositoryLocator;
LoadMetadata = loadMetadata;
LoadDirectory = directory;
Configuration = configuration;
_listener = listener;
Description = description;
List <ICatalogue> catalogues = LoadMetadata.GetAllCatalogues().ToList();
if (LoadMetadata != null)
{
_loggingTask = GetLoggingTask(catalogues);
}
RegularTablesToLoad = catalogues.SelectMany(catalogue => catalogue.GetTableInfoList(false)).Distinct().ToList();
LookupTablesToLoad = catalogues.SelectMany(catalogue => catalogue.GetLookupTableInfoList()).Distinct().ToList();
}
public void Check(ICheckNotifier notifier)
{
var catalogues = _loadMetadata.GetAllCatalogues().ToArray();
//if there are no logging tasks defined on any Catalogues
if (catalogues.Any() && catalogues.All(c => string.IsNullOrWhiteSpace(c.LoggingDataTask)))
{
string proposedName;
bool fix;
if (catalogues.Length == 1)
{
proposedName = "Loading '" + catalogues[0] + "'";
fix = notifier.OnCheckPerformed(
new CheckEventArgs(
"Catalogue " + catalogues[0] + " does not have a logging task specified",
CheckResult.Fail, null, "Create a new Logging Task called '" + proposedName + "'?"));
}
else
{
proposedName = _loadMetadata.Name;
fix =
notifier.OnCheckPerformed(
new CheckEventArgs(
"Catalogues " + string.Join(",", catalogues.Select(c => c.Name)) + " do not have a logging task specified",
CheckResult.Fail, null, "Create a new Logging Task called '" + proposedName + "'?"));
}
if (fix)
{
CreateNewLoggingTaskFor(notifier, catalogues, proposedName);
}
else
{
return;
}
}
#region Fix missing LoggingDataTask
var missingTasks = catalogues.Where(c => string.IsNullOrWhiteSpace(c.LoggingDataTask)).ToArray();
var potentialTasks = catalogues.Except(missingTasks).Select(c => c.LoggingDataTask).Distinct().ToArray();
//If any Catalogues are missing tasks
if (missingTasks.Any())
{
//but there is consensus for those that are not missing tasks
if (potentialTasks.Length == 1)
{
var fix = notifier.OnCheckPerformed(new CheckEventArgs("Some catalogues have NULL LoggingDataTasks", CheckResult.Fail, null, $"Set task to {potentialTasks.Single()}"));
if (fix)
{
foreach (var cata in missingTasks)
{
cata.LoggingDataTask = potentialTasks.Single();
cata.SaveToDatabase();
}
}
}
}
#endregion
#region Fix missing LiveLoggingServer_ID
var missingServer = catalogues.Where(c => c.LiveLoggingServer_ID == null).ToArray();
var potentialServer = catalogues.Except(missingServer).Select(c => c.LiveLoggingServer_ID).Distinct().ToArray();
if (missingServer.Any())
{
if (potentialServer.Length == 1)
{
var fix = notifier.OnCheckPerformed(new CheckEventArgs("Some catalogues have NULL LiveLoggingServer_ID", CheckResult.Fail, null, $"Set LiveLoggingServer_ID to {potentialServer.Single()}"));
if (fix)
{
foreach (var cata in missingServer)
{
cata.LiveLoggingServer_ID = potentialServer.Single();
cata.SaveToDatabase();
}
}
}
else
{
var defaults = _loadMetadata.CatalogueRepository.GetServerDefaults();
var defaultLoggingServer = defaults.GetDefaultFor(PermissableDefaults.LiveLoggingServer_ID);
if (defaultLoggingServer != null)
{
var fix = notifier.OnCheckPerformed(new CheckEventArgs("Some catalogues have NULL LiveLoggingServer_ID", CheckResult.Fail, null, $"Set LiveLoggingServer_ID to '{defaultLoggingServer}' (the default)"));
if (fix)
{
foreach (var cata in missingServer)
{
cata.LiveLoggingServer_ID = defaultLoggingServer.ID;
cata.SaveToDatabase();
}
}
}
}
}
#endregion
string distinctLoggingTask = null;
try
{
distinctLoggingTask = _loadMetadata.GetDistinctLoggingTask();
notifier.OnCheckPerformed(new CheckEventArgs("All Catalogues agreed on a single Logging Task:" + distinctLoggingTask, CheckResult.Success, null));
}
catch (Exception e)
{
notifier.OnCheckPerformed(new CheckEventArgs("Catalogues could not agreed on a single Logging Task", CheckResult.Fail, e));
}
try
{
var settings = _loadMetadata.GetDistinctLoggingDatabase();
settings.TestConnection();
notifier.OnCheckPerformed(new CheckEventArgs("Connected to logging architecture successfully", CheckResult.Success, null));
if (distinctLoggingTask != null)
{
LogManager lm = new LogManager(settings);
string[] dataTasks = lm.ListDataTasks();
if (dataTasks.Contains(distinctLoggingTask))
{
notifier.OnCheckPerformed(new CheckEventArgs("Found Logging Task " + distinctLoggingTask + " in Logging database", CheckResult.Success, null));
}
else
{
var fix = notifier.OnCheckPerformed(new CheckEventArgs("Could not find Logging Task " + distinctLoggingTask + " in Logging database", CheckResult.Fail, null, "Create Logging Task '" + distinctLoggingTask + "'"));
if (fix)
{
lm.CreateNewLoggingTaskIfNotExists(distinctLoggingTask);
}
}
}
}
catch (Exception e)
{
notifier.OnCheckPerformed(new CheckEventArgs("Could reach default logging server", CheckResult.Fail, e));
}
}
public PreExecutionChecker(ILoadMetadata loadMetadata, HICDatabaseConfiguration overrideDatabaseConfiguration)
{
_loadMetadata = loadMetadata;
_databaseConfiguration = overrideDatabaseConfiguration ?? new HICDatabaseConfiguration(loadMetadata);
_cataloguesToLoad = _loadMetadata.GetAllCatalogues().ToList();
}
public void Check(ICheckNotifier notifier)
{
var catalogues = _loadMetadata.GetAllCatalogues().ToArray();
//if there are no logging tasks defined on any Catalogues
if (catalogues.Any() && catalogues.All(c => string.IsNullOrWhiteSpace(c.LoggingDataTask)))
{
string proposedName;
bool fix;
if (catalogues.Length == 1)
{
proposedName = "Loading '" + catalogues[0] + "'";
fix = notifier.OnCheckPerformed(
new CheckEventArgs(
"Catalogue " + catalogues[0] + " does not have a logging task specified",
CheckResult.Fail, null, "Create a new Logging Task called '" + proposedName + "'?"));
}
else
{
proposedName = _loadMetadata.Name;
fix =
notifier.OnCheckPerformed(
new CheckEventArgs(
"Catalogues " + string.Join(",", catalogues.Select(c => c.Name)) + " do not have a logging task specified",
CheckResult.Fail, null, "Create a new Logging Task called '" + proposedName + "'?"));
}
if (fix)
{
CreateNewLoggingTaskFor(notifier, catalogues, proposedName);
}
else
{
return;
}
}
string distinctLoggingTask = null;
try
{
distinctLoggingTask = _loadMetadata.GetDistinctLoggingTask();
notifier.OnCheckPerformed(new CheckEventArgs("All Catalogues agreed on a single Logging Task:" + distinctLoggingTask, CheckResult.Success, null));
}
catch (Exception e)
{
notifier.OnCheckPerformed(new CheckEventArgs("Catalogues could not agreed on a single Logging Task", CheckResult.Fail, e));
}
try
{
var settings = _loadMetadata.GetDistinctLoggingDatabase();
settings.TestConnection();
notifier.OnCheckPerformed(new CheckEventArgs("Connected to logging architecture successfully", CheckResult.Success, null));
if (distinctLoggingTask != null)
{
LogManager lm = new LogManager(settings);
string[] dataTasks = lm.ListDataTasks();
if (dataTasks.Contains(distinctLoggingTask))
{
notifier.OnCheckPerformed(new CheckEventArgs("Found Logging Task " + distinctLoggingTask + " in Logging database", CheckResult.Success, null));
}
else
{
var fix = notifier.OnCheckPerformed(new CheckEventArgs("Could not find Logging Task " + distinctLoggingTask + " in Logging database", CheckResult.Fail, null, "Create Logging Task '" + distinctLoggingTask + "'"));
if (fix)
{
lm.CreateNewLoggingTaskIfNotExists(distinctLoggingTask);
}
}
}
}
catch (Exception e)
{
notifier.OnCheckPerformed(new CheckEventArgs("Could reach default logging server", CheckResult.Fail, e));
}
}
public void FetchDataFromDQE(ICheckNotifier notifier)
{
CataloguesWithDQERuns = new Dictionary <Catalogue, Evaluation>();
CataloguesMissingDQERuns = new HashSet <Catalogue>();
//tell them about the missing evaluations catalogues
foreach (Catalogue catalogue in _loadMetadata.GetAllCatalogues().Cast <Catalogue>())
{
var evaluation = dqeRepository.GetMostRecentEvaluationFor(catalogue);
if (evaluation == null)
{
//Catalogue has never been run in the DQE
CataloguesMissingDQERuns.Add(catalogue);
if (notifier != null)
{
notifier.OnCheckPerformed(
new CheckEventArgs(
"Catalogue '" + catalogue +
"' does not have any DQE evaluations on it in the DQE Repository. You should run the DQE on the dataset",
CheckResult.Warning));
}
}
else
{
CataloguesWithDQERuns.Add(catalogue, evaluation);
}
}
//The following code uses an epic pivot to produce something like:
/*YearMonth Year Month 6429 6430
* 1970-1 1970 1 4 0
* 1970-2 1970 2 2 0
* 1970-3 1970 3 6 1
* 1970-4 1970 4 4 0
* 1970-6 1970 6 2 0*/
if (!CataloguesWithDQERuns.Any())
{
throw new Exception("There are no Catalogues that have had DQE run on them in this LoadMetadata");
}
if (notifier != null)
{
foreach (var catalogue in CataloguesWithDQERuns)
{
notifier.OnCheckPerformed(
new CheckEventArgs(
"Found DQE Evaluations for Catalogue '" + catalogue + "'",
CheckResult.Success));
}
}
using (var con = dqeRepository.GetConnection())
{
CataloguesPeriodictiyData = new DataTable();
var cmd = dqeRepository.DiscoveredServer.GetCommand(GetTotalsByMonthSQL(CataloguesWithDQERuns.Keys.ToArray()), con);
var da = dqeRepository.DiscoveredServer.GetDataAdapter(cmd);
da.Fill(CataloguesPeriodictiyData);
}
//Now rename the columns from ID to the catalogue name
foreach (DataColumn col in CataloguesPeriodictiyData.Columns)
{
int cataId;
if (int.TryParse(col.ColumnName, out cataId))
{
col.ColumnName = CataloguesWithDQERuns.Keys.Single(c => c.ID == cataId).Name;
}
}
//Now extend the X axis up to the cache fill location
var cacheProgress = _loadProgress.CacheProgress;
if (cacheProgress != null && cacheProgress.CacheFillProgress != null)
{
ExtendXAxisTill(cacheProgress.CacheFillProgress.Value);
}
}
