private void SetupLogging(ICatalogueRepository repository)
{
//if we have already setup logging successfully then don't worry about doing it again
if (_loggingServer != null && _logManager != null && _loggingTask != null)
{
return;
}
_loggingServer = repository.GetServerDefaults().GetDefaultFor(PermissableDefaults.LiveLoggingServer_ID);
if (_loggingServer != null)
{
_logManager = new LogManager(_loggingServer);
_loggingTask = _logManager.ListDataTasks().SingleOrDefault(task => task.ToLower().Equals("dqe"));
if (_loggingTask == null)
{
_logManager.CreateNewLoggingTaskIfNotExists("DQE");
_loggingTask = "DQE";
}
}
else
{
throw new NotSupportedException(
"You must set a Default LiveLoggingServer so we can audit the DQE run, do this through the ManageExternalServers dialog");
}
}
public ExecuteCommandCreateNewANOTable(IBasicActivateItems activator) : base(activator)
{
_anoStoreServer = BasicActivator.ServerDefaults.GetDefaultFor(PermissableDefaults.ANOStore);
if (_anoStoreServer == null)
{
SetImpossible("No default ANOStore has been set");
}
}
/// <summary>
/// Runs the DLE using a custom names for RAW/STAGING. Pass in the load to execute and the files/directories to process
/// in the batch.
/// </summary>
/// <param name="lmd"></param>
/// <param name="payload"></param>
/// <returns>The exit code of the data load after it completes</returns>
private ExitCodeType RunDLE(LoadMetadata lmd, object payload)
{
var catalogueRepository = (CatalogueRepository)lmd.Repository;
//ensures that RAW/STAGING always have unique names
_configuration = new HICDatabaseConfiguration(lmd, _namer);
_configuration.UpdateButDoNotDiff = new Regex("^MessageGuid");
var logManager = catalogueRepository.GetDefaultLogManager();
logManager.CreateNewLoggingTaskIfNotExists(lmd.GetDistinctLoggingTask());
// Create the pipeline to pass into the DataLoadProcess object
var dataLoadFactory = new HICDataLoadFactory(lmd, _configuration, new HICLoadConfigurationFlags(),
catalogueRepository, logManager);
var stagingCreator = _namer as ICreateAndDestroyStagingDuringLoads;
if (stagingCreator != null)
{
stagingCreator.CreateStaging(lmd.GetDistinctLiveDatabaseServer());
}
var listener = new NLogThrowerDataLoadEventListener(NLog.LogManager.GetCurrentClassLogger());
IDataLoadExecution execution = dataLoadFactory.Create(listener);
IExternalDatabaseServer raw = catalogueRepository.GetServerDefaults().GetDefaultFor(PermissableDefaults.RAWDataLoadServer);
DiscoveredServer liveDb = lmd.GetDistinctLiveDatabaseServer();
//do we want to try to cut down the time it takes to do RAW=>STAGING by using INSERT INTO instead of running anonymisation/migration pipeline
if (_useInsertIntoForRawMigration)
{
//if it is on the same server swap out the migration engine for INSERT INTO
if (raw == null || (raw.Server != null && raw.Server.Equals(liveDb.Name) && raw.DatabaseType == liveDb.DatabaseType))
{
listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "SWAPPING RAW=>STAGING migration strategy to INSERT INTO"));
SwapMigrateRAWToStagingComponent(execution.Components);
}
else
{
//Cannot use because different servers / DatabaseTypes.
listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, "CANNOT SWAP RAW=>STAGING migration strategy to INSERT INTO because RAW is on '" + raw.Server + "' (" + raw.DatabaseType + ") and STAGING is on '" + liveDb.Name + "' (" + liveDb.DatabaseType + ")"));
}
}
else
{
listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Flag is false for SWAP RAW=>STAGING migration strategy to INSERT INTO So won't do it"));
}
var procedure = new DataLoadProcess(_repositoryLocator, lmd, null, logManager, listener, execution, _configuration);
ExitCodeType exitCode = procedure.Run(new GracefulCancellationToken(), payload);
return(exitCode);
}
public DQEMenuItem(IActivateItems activator, Catalogue catalogue) : base(activator, "Data Quality Engine...")
{
_catalogue = catalogue;
_dqeServer = activator.RepositoryLocator.CatalogueRepository.GetServerDefaults().GetDefaultFor(PermissableDefaults.DQE);
Image = activator.CoreIconProvider.GetImage(RDMPConcept.DQE);
Text = _dqeServer == null ? "Create DQE Database..." : "Data Quality Engine...";
}
public ExecuteCommandBulkImportTableInfos(IActivateItems activator) : base(activator)
{
_loggingServer = Activator.ServerDefaults.GetDefaultFor(PermissableDefaults.LiveLoggingServer_ID);
if (_loggingServer == null)
{
SetImpossible("There is no default logging server configured");
}
UseTripleDotSuffix = true;
}
/// <inheritdoc cref="GetDistinctLoggingDatabase()"/>
public DiscoveredServer GetDistinctLoggingDatabase(out IExternalDatabaseServer serverChosen)
{
var loggingServers = GetLoggingServers();
var loggingServer = loggingServers.FirstOrDefault();
//get distinct connection
var toReturn = DataAccessPortal.GetInstance().ExpectDistinctServer(loggingServers, DataAccessContext.Logging, true);
serverChosen = (IExternalDatabaseServer)loggingServer;
return(toReturn);
}
private void InsertNewValue(PermissableDefaults toChange, IExternalDatabaseServer externalDatabaseServer)
{
if (toChange == PermissableDefaults.None)
{
throw new ArgumentException("toChange cannot be None", "toChange");
}
_repository.Insert(
"INSERT INTO ServerDefaults(DefaultType,ExternalDatabaseServer_ID) VALUES (@DefaultType,@ExternalDatabaseServer_ID)",
new Dictionary <string, object>()
{
{ "DefaultType", StringExpansionDictionary[toChange] },
{ "ExternalDatabaseServer_ID", externalDatabaseServer.ID }
});
}
public ExecuteCommandCreateNewImagingDatasetSuite(IRDMPPlatformRepositoryServiceLocator repositoryLocator, DiscoveredDatabase databaseToCreateInto, DirectoryInfo projectDirectory)
{
_repositoryLocator = repositoryLocator;
_catalogueRepository = repositoryLocator.CatalogueRepository;
_databaseToCreateInto = databaseToCreateInto;
_projectDirectory = projectDirectory;
NewCataloguesCreated = new List <Catalogue>();
_loggingServer = _catalogueRepository.GetServerDefaults().GetDefaultFor(PermissableDefaults.LiveLoggingServer_ID);
if (_loggingServer == null)
{
SetImpossible("No default logging server has been configured in your Catalogue database");
}
CreateLoad = true;
}
/// <inheritdoc/>
public void SetDefault(PermissableDefaults toChange, IExternalDatabaseServer externalDatabaseServer)
{
if (toChange == PermissableDefaults.None)
{
throw new ArgumentException("toChange cannot be None", "toChange");
}
var oldValue = GetDefaultFor(toChange);
if (oldValue == null)
{
InsertNewValue(toChange, externalDatabaseServer);
}
else
{
UpdateExistingValue(toChange, externalDatabaseServer);
}
}
private void UpdateExistingValue(PermissableDefaults toChange, IExternalDatabaseServer externalDatabaseServer)
{
if (toChange == PermissableDefaults.None)
{
throw new ArgumentException("toChange cannot be None", "toChange");
}
string sql =
"UPDATE ServerDefaults set ExternalDatabaseServer_ID = @ExternalDatabaseServer_ID where DefaultType=@DefaultType";
int affectedRows = _repository.Update(sql, new Dictionary <string, object>()
{
{ "DefaultType", StringExpansionDictionary[toChange] },
{ "ExternalDatabaseServer_ID", externalDatabaseServer.ID }
});
if (affectedRows != 1)
{
throw new Exception("We were asked to update default for " + toChange + " but the query '" + sql + "' did not result in 1 affected rows (it resulted in " + affectedRows + ")");
}
}
public DiscoveredServer GetDistinctLoggingDatabase(out IExternalDatabaseServer serverChosen)
{
serverChosen = CatalogueRepository.GetServerDefaults().GetDefaultFor(PermissableDefaults.LiveLoggingServer_ID);
return(GetDistinctLoggingDatabase());
}
public CachedAggregateConfigurationResultsManager(IExternalDatabaseServer server)
{
_server = DataAccessPortal.GetInstance().ExpectServer(server, DataAccessContext.InternalDataProcessing);
_database = DataAccessPortal.GetInstance().ExpectDatabase(server, DataAccessContext.InternalDataProcessing);
}
public DiscoveredServer GetDistinctLoggingDatabase(out IExternalDatabaseServer serverChosen)
{
serverChosen = GetDistinctLoggingServer(false);
return(serverChosen.Discover(DataAccessContext.Logging).Server);
}
/// <summary>
/// Constructor for use in tests, if possible use the LoadMetadata constructor instead
/// </summary>
/// <param name="liveServer">The live server where the data is held, IMPORTANT: this must contain InitialCatalog parameter</param>
/// <param name="namer">optionally lets you specify how to pick database names for the temporary bubbles STAGING and RAW</param>
/// <param name="defaults">optionally specifies the location to get RAW default server from</param>
/// <param name="overrideRAWServer">optionally specifies an explicit server to use for RAW</param>
public HICDatabaseConfiguration(DiscoveredServer liveServer, INameDatabasesAndTablesDuringLoads namer = null, IServerDefaults defaults = null, IExternalDatabaseServer overrideRAWServer = null)
{
//respects the override of LIVE server
var liveDatabase = liveServer.GetCurrentDatabase();
if (liveDatabase == null)
{
throw new Exception("Cannot load live without having a unique live named database");
}
// Default namer
if (namer == null)
{
if (liveServer.DatabaseType == DatabaseType.PostgreSql)
{
//create the DLE tables on the live database because postgres can't handle cross database references
namer = new FixedStagingDatabaseNamer(liveDatabase.GetRuntimeName(), liveDatabase.GetRuntimeName());
}
else
{
namer = new FixedStagingDatabaseNamer(liveDatabase.GetRuntimeName());
}
}
//if there are defaults
if (overrideRAWServer == null && defaults != null)
{
overrideRAWServer = defaults.GetDefaultFor(PermissableDefaults.RAWDataLoadServer);//get the raw default if there is one
}
DiscoveredServer rawServer;
//if there was defaults and a raw default server
if (overrideRAWServer != null)
{
rawServer = DataAccessPortal.GetInstance().ExpectServer(overrideRAWServer, DataAccessContext.DataLoad, false); //get the raw server connection
}
else
{
rawServer = liveServer; //there is no raw override so we will have to use the live server for RAW too.
}
//populates the servers -- note that an empty rawServer value passed to this method makes it the localhost
DeployInfo = new StandardDatabaseHelper(liveServer.GetCurrentDatabase(), namer, rawServer);
RequiresStagingTableCreation = true;
}
