public void Check(ICheckNotifier notifier) { Catalogue[] catalogueMetadatas = null; try { //check there are catalogues and we can retrieve them catalogueMetadatas = _loadMetadata.GetAllCatalogues().Cast <Catalogue>().ToArray(); } catch (Exception e) { notifier.OnCheckPerformed(new CheckEventArgs("Crashed trying to fetch Catalogues for metadata", CheckResult.Fail, e)); return; } //there no catalogues if (catalogueMetadatas.Length == 0) { notifier.OnCheckPerformed(new CheckEventArgs("There are no Catalogues associated with this metadata", CheckResult.Fail, null)); } List <ITableInfo> tablesFound = new List <ITableInfo>(); //check each catalogue is sufficiently configured to perform a migration foreach (Catalogue catalogue in catalogueMetadatas) { notifier.OnCheckPerformed(new CheckEventArgs("Found Catalogue:" + catalogue, CheckResult.Success, null)); ITableInfo[] tableInfos = catalogue.GetTableInfoList(true).Distinct().ToArray(); if (tableInfos.Length == 0) { notifier.OnCheckPerformed(new CheckEventArgs( "Catalogue " + catalogue.Name + " does not have any TableInfos", CheckResult.Fail, null)); } tablesFound.AddRange(tableInfos.Where(tableInfo => !tablesFound.Contains(tableInfo))); } //check regular tables foreach (TableInfo regularTable in tablesFound) { notifier.OnCheckPerformed(new CheckEventArgs("About To check configuration of TableInfo:" + regularTable, CheckResult.Success, null)); CheckTableInfo(regularTable, notifier); //check anonymisation AnonymisationChecks anonymisationChecks = new AnonymisationChecks(regularTable); anonymisationChecks.Check(notifier); } }
public HICDataLoadFactory(ILoadMetadata loadMetadata, HICDatabaseConfiguration databaseConfiguration, HICLoadConfigurationFlags loadConfigurationFlags, ICatalogueRepository repository, ILogManager logManager) { _databaseConfiguration = databaseConfiguration; _loadConfigurationFlags = loadConfigurationFlags; _repository = repository; _logManager = logManager; LoadMetadata = loadMetadata; // If we are not supplied any catalogues to load, it is expected that we will load all catalogues associated with the provided ILoadMetadata _cataloguesToLoad = LoadMetadata.GetAllCatalogues().ToList(); if (!_cataloguesToLoad.Any()) { throw new InvalidOperationException("LoadMetadata " + LoadMetadata.ID + " is not related to any Catalogues, there is nothing to load"); } }
public void Check(ICheckNotifier notifier) { _notifier = notifier; //For each table in load can we reach it and is it a valid table type foreach (ITableInfo ti in _loadMetadata.GetAllCatalogues().SelectMany(c => c.GetTableInfoList(true)).Distinct()) { DiscoveredTable tbl; try { tbl = ti.Discover(DataAccessContext.DataLoad); } catch (Exception e) { HardFail = true; notifier.OnCheckPerformed(new CheckEventArgs("Could not reach table in load '" + ti.Name + "'", CheckResult.Fail, e)); return; } if (!tbl.Exists()) { HardFail = true; notifier.OnCheckPerformed(new CheckEventArgs("Table '" + ti.Name + "' does not exist", CheckResult.Fail)); } if (tbl.TableType != TableType.Table) { HardFail = true; notifier.OnCheckPerformed(new CheckEventArgs("Table '" + ti + "' is a " + tbl.TableType, CheckResult.Fail)); } } if (HardFail) { return; } AtLeastOneTaskCheck(); PreExecutionStagingDatabaseCheck(false); PreExecutionDatabaseCheck(); }
public DataLoadJob(IRDMPPlatformRepositoryServiceLocator repositoryLocator, string description, ILogManager logManager, ILoadMetadata loadMetadata, ILoadDirectory directory, IDataLoadEventListener listener, HICDatabaseConfiguration configuration) { _logManager = logManager; RepositoryLocator = repositoryLocator; LoadMetadata = loadMetadata; LoadDirectory = directory; Configuration = configuration; _listener = listener; Description = description; List <ICatalogue> catalogues = LoadMetadata.GetAllCatalogues().ToList(); if (LoadMetadata != null) { _loggingTask = GetLoggingTask(catalogues); } RegularTablesToLoad = catalogues.SelectMany(catalogue => catalogue.GetTableInfoList(false)).Distinct().ToList(); LookupTablesToLoad = catalogues.SelectMany(catalogue => catalogue.GetLookupTableInfoList()).Distinct().ToList(); }
public void Check(ICheckNotifier notifier) { var catalogues = _loadMetadata.GetAllCatalogues().ToArray(); //if there are no logging tasks defined on any Catalogues if (catalogues.Any() && catalogues.All(c => string.IsNullOrWhiteSpace(c.LoggingDataTask))) { string proposedName; bool fix; if (catalogues.Length == 1) { proposedName = "Loading '" + catalogues[0] + "'"; fix = notifier.OnCheckPerformed( new CheckEventArgs( "Catalogue " + catalogues[0] + " does not have a logging task specified", CheckResult.Fail, null, "Create a new Logging Task called '" + proposedName + "'?")); } else { proposedName = _loadMetadata.Name; fix = notifier.OnCheckPerformed( new CheckEventArgs( "Catalogues " + string.Join(",", catalogues.Select(c => c.Name)) + " do not have a logging task specified", CheckResult.Fail, null, "Create a new Logging Task called '" + proposedName + "'?")); } if (fix) { CreateNewLoggingTaskFor(notifier, catalogues, proposedName); } else { return; } } #region Fix missing LoggingDataTask var missingTasks = catalogues.Where(c => string.IsNullOrWhiteSpace(c.LoggingDataTask)).ToArray(); var potentialTasks = catalogues.Except(missingTasks).Select(c => c.LoggingDataTask).Distinct().ToArray(); //If any Catalogues are missing tasks if (missingTasks.Any()) { //but there is consensus for those that are not missing tasks if (potentialTasks.Length == 1) { var fix = notifier.OnCheckPerformed(new CheckEventArgs("Some catalogues have NULL LoggingDataTasks", CheckResult.Fail, null, $"Set task to {potentialTasks.Single()}")); if (fix) { foreach (var cata in missingTasks) { cata.LoggingDataTask = potentialTasks.Single(); cata.SaveToDatabase(); } } } } #endregion #region Fix missing LiveLoggingServer_ID var missingServer = catalogues.Where(c => c.LiveLoggingServer_ID == null).ToArray(); var potentialServer = catalogues.Except(missingServer).Select(c => c.LiveLoggingServer_ID).Distinct().ToArray(); if (missingServer.Any()) { if (potentialServer.Length == 1) { var fix = notifier.OnCheckPerformed(new CheckEventArgs("Some catalogues have NULL LiveLoggingServer_ID", CheckResult.Fail, null, $"Set LiveLoggingServer_ID to {potentialServer.Single()}")); if (fix) { foreach (var cata in missingServer) { cata.LiveLoggingServer_ID = potentialServer.Single(); cata.SaveToDatabase(); } } } else { var defaults = _loadMetadata.CatalogueRepository.GetServerDefaults(); var defaultLoggingServer = defaults.GetDefaultFor(PermissableDefaults.LiveLoggingServer_ID); if (defaultLoggingServer != null) { var fix = notifier.OnCheckPerformed(new CheckEventArgs("Some catalogues have NULL LiveLoggingServer_ID", CheckResult.Fail, null, $"Set LiveLoggingServer_ID to '{defaultLoggingServer}' (the default)")); if (fix) { foreach (var cata in missingServer) { cata.LiveLoggingServer_ID = defaultLoggingServer.ID; cata.SaveToDatabase(); } } } } } #endregion string distinctLoggingTask = null; try { distinctLoggingTask = _loadMetadata.GetDistinctLoggingTask(); notifier.OnCheckPerformed(new CheckEventArgs("All Catalogues agreed on a single Logging Task:" + distinctLoggingTask, CheckResult.Success, null)); } catch (Exception e) { notifier.OnCheckPerformed(new CheckEventArgs("Catalogues could not agreed on a single Logging Task", CheckResult.Fail, e)); } try { var settings = _loadMetadata.GetDistinctLoggingDatabase(); settings.TestConnection(); notifier.OnCheckPerformed(new CheckEventArgs("Connected to logging architecture successfully", CheckResult.Success, null)); if (distinctLoggingTask != null) { LogManager lm = new LogManager(settings); string[] dataTasks = lm.ListDataTasks(); if (dataTasks.Contains(distinctLoggingTask)) { notifier.OnCheckPerformed(new CheckEventArgs("Found Logging Task " + distinctLoggingTask + " in Logging database", CheckResult.Success, null)); } else { var fix = notifier.OnCheckPerformed(new CheckEventArgs("Could not find Logging Task " + distinctLoggingTask + " in Logging database", CheckResult.Fail, null, "Create Logging Task '" + distinctLoggingTask + "'")); if (fix) { lm.CreateNewLoggingTaskIfNotExists(distinctLoggingTask); } } } } catch (Exception e) { notifier.OnCheckPerformed(new CheckEventArgs("Could reach default logging server", CheckResult.Fail, e)); } }
public PreExecutionChecker(ILoadMetadata loadMetadata, HICDatabaseConfiguration overrideDatabaseConfiguration) { _loadMetadata = loadMetadata; _databaseConfiguration = overrideDatabaseConfiguration ?? new HICDatabaseConfiguration(loadMetadata); _cataloguesToLoad = _loadMetadata.GetAllCatalogues().ToList(); }
public void Check(ICheckNotifier notifier) { var catalogues = _loadMetadata.GetAllCatalogues().ToArray(); //if there are no logging tasks defined on any Catalogues if (catalogues.Any() && catalogues.All(c => string.IsNullOrWhiteSpace(c.LoggingDataTask))) { string proposedName; bool fix; if (catalogues.Length == 1) { proposedName = "Loading '" + catalogues[0] + "'"; fix = notifier.OnCheckPerformed( new CheckEventArgs( "Catalogue " + catalogues[0] + " does not have a logging task specified", CheckResult.Fail, null, "Create a new Logging Task called '" + proposedName + "'?")); } else { proposedName = _loadMetadata.Name; fix = notifier.OnCheckPerformed( new CheckEventArgs( "Catalogues " + string.Join(",", catalogues.Select(c => c.Name)) + " do not have a logging task specified", CheckResult.Fail, null, "Create a new Logging Task called '" + proposedName + "'?")); } if (fix) { CreateNewLoggingTaskFor(notifier, catalogues, proposedName); } else { return; } } string distinctLoggingTask = null; try { distinctLoggingTask = _loadMetadata.GetDistinctLoggingTask(); notifier.OnCheckPerformed(new CheckEventArgs("All Catalogues agreed on a single Logging Task:" + distinctLoggingTask, CheckResult.Success, null)); } catch (Exception e) { notifier.OnCheckPerformed(new CheckEventArgs("Catalogues could not agreed on a single Logging Task", CheckResult.Fail, e)); } try { var settings = _loadMetadata.GetDistinctLoggingDatabase(); settings.TestConnection(); notifier.OnCheckPerformed(new CheckEventArgs("Connected to logging architecture successfully", CheckResult.Success, null)); if (distinctLoggingTask != null) { LogManager lm = new LogManager(settings); string[] dataTasks = lm.ListDataTasks(); if (dataTasks.Contains(distinctLoggingTask)) { notifier.OnCheckPerformed(new CheckEventArgs("Found Logging Task " + distinctLoggingTask + " in Logging database", CheckResult.Success, null)); } else { var fix = notifier.OnCheckPerformed(new CheckEventArgs("Could not find Logging Task " + distinctLoggingTask + " in Logging database", CheckResult.Fail, null, "Create Logging Task '" + distinctLoggingTask + "'")); if (fix) { lm.CreateNewLoggingTaskIfNotExists(distinctLoggingTask); } } } } catch (Exception e) { notifier.OnCheckPerformed(new CheckEventArgs("Could reach default logging server", CheckResult.Fail, e)); } }
public void FetchDataFromDQE(ICheckNotifier notifier) { CataloguesWithDQERuns = new Dictionary <Catalogue, Evaluation>(); CataloguesMissingDQERuns = new HashSet <Catalogue>(); //tell them about the missing evaluations catalogues foreach (Catalogue catalogue in _loadMetadata.GetAllCatalogues().Cast <Catalogue>()) { var evaluation = dqeRepository.GetMostRecentEvaluationFor(catalogue); if (evaluation == null) { //Catalogue has never been run in the DQE CataloguesMissingDQERuns.Add(catalogue); if (notifier != null) { notifier.OnCheckPerformed( new CheckEventArgs( "Catalogue '" + catalogue + "' does not have any DQE evaluations on it in the DQE Repository. You should run the DQE on the dataset", CheckResult.Warning)); } } else { CataloguesWithDQERuns.Add(catalogue, evaluation); } } //The following code uses an epic pivot to produce something like: /*YearMonth Year Month 6429 6430 * 1970-1 1970 1 4 0 * 1970-2 1970 2 2 0 * 1970-3 1970 3 6 1 * 1970-4 1970 4 4 0 * 1970-6 1970 6 2 0*/ if (!CataloguesWithDQERuns.Any()) { throw new Exception("There are no Catalogues that have had DQE run on them in this LoadMetadata"); } if (notifier != null) { foreach (var catalogue in CataloguesWithDQERuns) { notifier.OnCheckPerformed( new CheckEventArgs( "Found DQE Evaluations for Catalogue '" + catalogue + "'", CheckResult.Success)); } } using (var con = dqeRepository.GetConnection()) { CataloguesPeriodictiyData = new DataTable(); var cmd = dqeRepository.DiscoveredServer.GetCommand(GetTotalsByMonthSQL(CataloguesWithDQERuns.Keys.ToArray()), con); var da = dqeRepository.DiscoveredServer.GetDataAdapter(cmd); da.Fill(CataloguesPeriodictiyData); } //Now rename the columns from ID to the catalogue name foreach (DataColumn col in CataloguesPeriodictiyData.Columns) { int cataId; if (int.TryParse(col.ColumnName, out cataId)) { col.ColumnName = CataloguesWithDQERuns.Keys.Single(c => c.ID == cataId).Name; } } //Now extend the X axis up to the cache fill location var cacheProgress = _loadProgress.CacheProgress; if (cacheProgress != null && cacheProgress.CacheFillProgress != null) { ExtendXAxisTill(cacheProgress.CacheFillProgress.Value); } }