public ExecuteCommandCreateNewProcessTask(IActivateItems activator, ProcessTaskType taskType, LoadMetadata loadMetadata, LoadStage loadStage, FileInfo file = null) : base(activator)
{
_taskType = taskType;
_loadMetadata = loadMetadata;
_loadStage = loadStage;
try
{
_LoadDirectory = new LoadDirectory(_loadMetadata.LocationOfFlatFiles);
}
catch (Exception)
{
SetImpossible("Could not construct LoadDirectory");
}
if (taskType == ProcessTaskType.SQLFile)
{
_image = activator.CoreIconProvider.GetImage(RDMPConcept.SQL, OverlayKind.Add);
}
else if (taskType == ProcessTaskType.Executable)
{
_image = new IconOverlayProvider().GetOverlayNoCache(CatalogueIcons.Exe, OverlayKind.Add);
}
else
{
SetImpossible("Only SQLFile and Executable task types are supported by this command");
}
if (!ProcessTask.IsCompatibleStage(taskType, loadStage))
{
SetImpossible("You cannot run " + taskType + " in " + loadStage);
}
_file = file;
}
protected IStageArgs CreateLoadArgs(LoadStage loadStage)
{
return
(new StageArgs(loadStage,
_dbDeployInfo[loadStage.ToLoadBubble()]
, new LoadDirectory(_loadMetadata.LocationOfFlatFiles.TrimEnd(new[] { '\\' }))));
}
public ExecuteCommandCreateNewClassBasedProcessTask(IBasicActivateItems activator, LoadMetadata loadMetadata, LoadStage loadStage,
[DemandsInitialization("Class to execute, must be an attacher, mutilater etc", TypeOf = typeof(IDisposeAfterDataLoad))]
Type type) : base(activator)
{
_loadMetadata = loadMetadata;
_loadStage = loadStage;
_type = type;
if (typeof(IAttacher).IsAssignableFrom(_type))
{
_processTaskType = ProcessTaskType.Attacher;
}
else
if (typeof(IDataProvider).IsAssignableFrom(_type))
{
_processTaskType = ProcessTaskType.DataProvider;
}
else
if (typeof(IMutilateDataTables).IsAssignableFrom(_type))
{
_processTaskType = ProcessTaskType.MutilateDataTable;
}
else
{
SetImpossible($"Type '{_type}' was not a compatible one e.g. IAttacher, IDataProvider or IMutilateDataTables");
}
}
public void Initialize(DiscoveredDatabase dbInfo, LoadStage loadStage)
{
if (loadStage != LoadStage.AdjustStaging && loadStage != LoadStage.PostLoad)
{
throw new Exception(typeof(BackupDatabaseMutilation).Name + " can only be done in AdjustStaging or PostLoad (this minimises redundant backups that would otherwise be created while you attempt to fix RAW / constraint related load errors)");
}
}
/// <summary>
/// Gets the DataType adjusted for the stage at which the ColumnInfo is at, this is almost always the same as Data_type. The only
/// time it is different is when there is an ANOTable involved e.g. ANOLocation could be a varchar(6) like 'AB10_L' after anonymisation
/// but if the LoadStage is AdjustRaw then it would have a value like 'NH10' (varchar(4) - the unanonymised state).
/// </summary>
/// <param name="loadStage"></param>
/// <returns></returns>
public string GetRuntimeDataType(LoadStage loadStage)
{
if (loadStage <= LoadStage.AdjustRaw)
{
//if it has an ANO transform
if (ANOTable_ID != null)
{
return(ANOTable.GetRuntimeDataType(loadStage)); //get the datatype from the ANOTable because ColumnInfo is of mutable type depending on whether it has been anonymised yet
}
//it doesn't have an ANOtransform but it might be the subject of dilution
var discard = TableInfo.PreLoadDiscardedColumns.SingleOrDefault(c => c.GetRuntimeName().Equals(GetRuntimeName(), StringComparison.InvariantCultureIgnoreCase));
//The column exists both in the live database and in the identifier dump. This is because it goes through horrendous bitcrushing operations e.g. Load RAW with full
//postcode varchar(8) and ship postcode off to identifier dump but also let it go through to live but only as the first 4 letters varchar(4). so the datatype of the column
//in RAW is varchar(8) but in Live is varchar(4)
if (discard != null)
{
return(discard.Data_type);
}
return(Data_type);
}
//The user is asking about a stage other than RAW so tell them about the final column type state
return(Data_type);
}
public ExecuteCommandCreateNewFileBasedProcessTask(IBasicActivateItems activator, ProcessTaskType taskType, LoadMetadata loadMetadata, LoadStage loadStage, FileInfo file = null) : base(activator)
{
_taskType = taskType;
_loadMetadata = loadMetadata;
_loadStage = loadStage;
try
{
_LoadDirectory = new LoadDirectory(_loadMetadata.LocationOfFlatFiles);
}
catch (Exception)
{
SetImpossible("Could not construct LoadDirectory");
}
if (!(taskType == ProcessTaskType.SQLFile || taskType == ProcessTaskType.Executable))
{
SetImpossible("Only SQLFile and Executable task types are supported by this command");
}
if (!ProcessTask.IsCompatibleStage(taskType, loadStage))
{
SetImpossible("You cannot run " + taskType + " in " + loadStage);
}
_file = file;
}
public void Initialize(DiscoveredDatabase dbInfo, LoadStage loadStage)
{
if (loadStage != LoadStage.AdjustRaw)
{
throw new Exception("Primary key collisions can only be resolved in a RAW environment, current load stage is:" + loadStage + " (The reason for this is because there should be primary keys in the database level in STAGING and LIVE making primary key collisions IMPOSSIBE)");
}
_dbInfo = dbInfo;
}
public void EmptyClassPath(string path, ProcessTaskType typeThatRequiresMEF, LoadStage stage)
{
_task.ProcessTaskType = typeThatRequiresMEF;
_task.Path = path;
_task.LoadStage = stage;
_task.SaveToDatabase();
var ex = Assert.Throws <ArgumentException>(() => _checker.Check(new ThrowImmediatelyCheckNotifier()));
Assert.IsTrue(Regex.IsMatch(ex.Message, "Path is blank for ProcessTask 'New Process.*' - it should be a class name of type"));
}
public void Initialize(DiscoveredDatabase dbInfo, LoadStage loadStage)
{
_raw = dbInfo;
_syntaxHelper = _raw.Server.GetQuerySyntaxHelper();
if (loadStage != LoadStage.AdjustRaw)
{
throw new Exception("This component should only run in AdjustRaw");
}
}
public void Initialize(DiscoveredDatabase dbInfo, LoadStage loadStage)
{
if (_allowedStages != null && !_allowedStages.Contains(loadStage))
{
throw new NotSupportedException("Mutilation " + GetType() + " is not allowed at stage " + loadStage);
}
_loadStage = loadStage;
DbInfo = dbInfo;
}
/// <inheritdoc/>
public string GetDatabaseRuntimeName(LoadStage loadStage, INameDatabasesAndTablesDuringLoads namer = null)
{
var baseName = GetDatabaseRuntimeName();
if (namer == null)
{
namer = new FixedStagingDatabaseNamer(baseName);
}
return(namer.GetDatabaseName(baseName, loadStage.ToLoadBubble()));
}
public ExecuteCommandCreateNewClassBasedProcessTask(IBasicActivateItems activator, LoadMetadata loadMetadata, LoadStage loadStage,
[DemandsInitialization("Class to execute, must be an attacher, mutilater etc", TypeOf = typeof(IDisposeAfterDataLoad))]
Type type) : base(activator)
{
_loadMetadata = loadMetadata;
_loadStage = loadStage;
if (type != null)
{
SetType(type);
}
}
/// <summary>
/// Creates a new operation in the data load (e.g. copy files from A to B, load all CSV files to RAW table B etc)
/// </summary>
/// <param name="repository"></param>
/// <param name="parent"></param>
/// <param name="stage"></param>
public ProcessTask(ICatalogueRepository repository, ILoadMetadata parent, LoadStage stage)
{
var order = repository.GetAllObjectsWithParent <ProcessTask>(parent).Select(t => t.Order).DefaultIfEmpty().Max() + 1;
repository.InsertAndHydrate(this, new Dictionary <string, object>
{
{ "LoadMetadata_ID", parent.ID },
{ "ProcessTaskType", ProcessTaskType.Executable.ToString() },
{ "LoadStage", stage },
{ "Name", "New Process" + Guid.NewGuid() },
{ "Order", order }
});
}
/// <summary>Get the data fields to read/write for save data.</summary>
/// <param name="stage">The current load stage.</param>
private IEnumerable <IDictionary <string, string> > GetDataFields(LoadStage stage)
{
if (stage == LoadStage.None)
{
yield break;
}
yield return(Game1.CustomData);
if (SaveGame.loaded != null)
{
yield return(SaveGame.loaded.CustomData);
}
}
/// <summary>
/// Anonymisation with an <see cref="ANOTable"/> happens during data load. This means that the column goes from identifiable in RAW to anonymous in STAGING/LIVE. This means
/// that the datatype of the column changes depending on the <see cref="LoadStage"/>.
///
/// <para>Returns the appropriate datatype for the <see cref="LoadStage"/>. This is done by connecting to the mapping table and retrieving the mapping table types</para>
/// </summary>
/// <param name="loadStage"></param>
/// <returns></returns>
public string GetRuntimeDataType(LoadStage loadStage)
{
//cache answers
if (_identifiableDataType == null)
{
var server = DataAccessPortal.GetInstance().ExpectServer(Server, DataAccessContext.DataLoad);
DiscoveredColumn[] columnsFoundInANO = server.GetCurrentDatabase().ExpectTable(TableName).DiscoverColumns();
string expectedIdentifiableName = TableName.Substring("ANO".Length);
DiscoveredColumn anonymous = columnsFoundInANO.SingleOrDefault(c => c.GetRuntimeName().Equals(TableName));
DiscoveredColumn identifiable = columnsFoundInANO.SingleOrDefault(c => c.GetRuntimeName().Equals(expectedIdentifiableName));
if (anonymous == null)
{
throw new Exception("Could not find a column called " + TableName + " in table " + TableName + " on server " + Server + " (Columns found were " + string.Join(",", columnsFoundInANO.Select(c => c.GetRuntimeName()).ToArray()) + ")");
}
if (identifiable == null)
{
throw new Exception("Could not find a column called " + expectedIdentifiableName + " in table " + TableName + " on server " + Server + " (Columns found were " + string.Join(",", columnsFoundInANO.Select(c => c.GetRuntimeName()).ToArray()) + ")");
}
_identifiableDataType = identifiable.DataType.SQLType;
_anonymousDataType = anonymous.DataType.SQLType;
}
//return cached answer
switch (loadStage)
{
case LoadStage.GetFiles:
return(_identifiableDataType);
case LoadStage.Mounting:
return(_identifiableDataType);
case LoadStage.AdjustRaw:
return(_identifiableDataType);
case LoadStage.AdjustStaging:
return(_anonymousDataType);
case LoadStage.PostLoad:
return(_anonymousDataType);
default:
throw new ArgumentOutOfRangeException("loadStage");
}
}
public LoadState(Game game, Func <State> nextState) : base(game)
{
m_stage = LoadStage.Startup;
m_nextStateFunc = nextState;
m_loadTask = null;
m_backdrop = new Image(Texture.Black, Game.Screen.Width, Game.Screen.Height);
m_backdrop.Anchor = Anchor.CentreMiddle;
m_backdrop.LocalPosition = new Vector2(-0.5f * m_backdrop.Width, -0.5f * m_backdrop.Height);
m_widget = new Image(Texture.Get("gui/loading.png", true), GetWidgetQuad(0.0f), 64.0f, 64.0f);
m_widget.Anchor = Anchor.CentreMiddle;
m_widget.LocalPosition = new Vector2(-0.5f * m_widget.Width, -0.5f * m_widget.Height);
m_timeInStage = 0.0f;
}
public CompositeDataLoadComponent CreateCompositeDataLoadComponentFor(LoadStage loadStage, string descriptionForComponent)
{
RuntimeTaskFactory factory = new RuntimeTaskFactory(_repository);
var tasks = new List <IDataLoadComponent>();
foreach (var task in GetRuntimeTasksForStage(loadStage))
{
tasks.Add(factory.Create(task.ProcessTask, _loadArgsDictionary[loadStage]));
}
return(new CompositeDataLoadComponent(tasks)
{
Description = descriptionForComponent
});
}
///<inheritdoc/>
public string GetRuntimeName(LoadStage stage)
{
string finalName = this.GetRuntimeName();
if (stage <= LoadStage.AdjustRaw)
{
//see if it has an ANO Transform on it
if (ANOTable_ID != null && finalName.StartsWith("ANO"))
{
return(finalName.Substring("ANO".Length));
}
}
//any other stage will be the regular final name
return(finalName);
}
public override ExitCodeType Run(IDataLoadJob job, GracefulCancellationToken cancellationToken)
{
var db = RuntimeArguments.StageSpecificArguments.DbInfo;
_loadStage = RuntimeArguments.StageSpecificArguments.LoadStage;
if (!Exists())
{
throw new Exception("The sql file " + Filepath + " does not exist");
}
string commandText;
try
{
commandText = File.ReadAllText(Filepath);
// Any string arguments refer to tokens that are to be replaced in the SQL file
foreach (var kvp in RuntimeArguments.GetAllArgumentsOfType <string>())
{
var value = kvp.Value;
if (value.Contains("<DatabaseServer>"))
{
value = value.Replace("<DatabaseServer>", RuntimeArguments.StageSpecificArguments.DbInfo.Server.Name);
}
if (value.Contains("<DatabaseName>"))
{
value = value.Replace("<DatabaseName>", RuntimeArguments.StageSpecificArguments.DbInfo.GetRuntimeName());
}
commandText = commandText.Replace("##" + kvp.Key + "##", value);
}
}
catch (Exception e)
{
throw new Exception("Could not read the sql file at " + Filepath + ": " + e);
}
job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Executing script " + Filepath + " (" + db.DescribeDatabase() + ")"));
var executer = new ExecuteSqlInDleStage(job, _loadStage);
return(executer.Execute(commandText, db));
}
public void Add(ITableInfo tableInfo, LoadStage loadStage)
{
//we already have it or it is not setup properly
if (items.Any(i => i.Tag.Equals(tableInfo)) || string.IsNullOrWhiteSpace(tableInfo.Database) || string.IsNullOrWhiteSpace(tableInfo.Server))
{
return;
}
var runtimeName = tableInfo.GetRuntimeName(loadStage);
var dbName = tableInfo.GetDatabaseRuntimeName(loadStage);
var syntaxHelper = tableInfo.GetQuerySyntaxHelper();
var fullSql = syntaxHelper.EnsureFullyQualified(dbName, null, runtimeName);
var snip = new SubstringAutocompleteItem(tableInfo.GetRuntimeName());
snip.MenuText = runtimeName; //name of table
snip.Text = fullSql; //full SQL
snip.Tag = tableInfo; //record object for future reference
snip.ImageIndex = GetIndexFor(tableInfo, RDMPConcept.TableInfo.ToString());
foreach (IHasStageSpecificRuntimeName o in tableInfo.GetColumnsAtStage(loadStage))
{
var preDiscarded = o as PreLoadDiscardedColumn;
var columnInfo = o as ColumnInfo;
if (preDiscarded != null)
{
Add(preDiscarded, tableInfo, dbName);
}
else
if (columnInfo != null)
{
Add(columnInfo, tableInfo, dbName, loadStage, syntaxHelper);
}
else
{
throw new Exception("Expected IHasStageSpecificRuntimeName returned by TableInfo.GetColumnsAtStage to return only ColumnInfos and PreLoadDiscardedColumns. It returned a '" + o.GetType().Name + "'");
}
}
AddUnlessDuplicate(snip);
}
public IEbicsClient Create(
Func <string, byte[]> readBytes,
Action <string, byte[]> writeBytes,
string password, string address, string hostId, string partnerId, string userId,
LoadStage loadStage = LoadStage.All)
{
var signCert = loadStage >= LoadStage.Sign?new System.Security.Cryptography.X509Certificates.X509Certificate2(readBytes("sign.p12"), password):null;
var authCert = loadStage >= LoadStage.Auth ? new System.Security.Cryptography.X509Certificates.X509Certificate2(readBytes("auth.p12"), password):null;
var encCert = loadStage >= LoadStage.Enc ? new System.Security.Cryptography.X509Certificates.X509Certificate2(readBytes("enc.p12"), password):null;
var cfg = new EbicsConfig
{
readBytes = readBytes,
writeBytes = writeBytes,
Address = address,
User = new UserParams
{
HostId = hostId,
PartnerId = partnerId,
UserId = userId,
SignKeys = new SignKeyPair
{
Version = SignVersion.A005,
Certificate = signCert
},
AuthKeys = new AuthKeyPair
{
Version = AuthVersion.X002,
Certificate = authCert
},
CryptKeys = new CryptKeyPair
{
Version = CryptVersion.E002,
Certificate = encCert
}
}
};
if (loadStage >= LoadStage.Bank)
{
cfg.LoadBank();
}
return(_ctor(cfg));
}
/// <summary>
/// Creates a new copy of the processTask and all it's arguments in the database, this clone is then hooked up to the
/// new LoadMetadata at the specified stage
/// </summary>
/// <param name="loadMetadata">The new LoadMetadata parent for the clone</param>
/// <param name="loadStage">The new load stage to put the clone in </param>
/// <returns>the new ProcessTask (the clone has a different ID to the parent)</returns>
public ProcessTask CloneToNewLoadMetadataStage(LoadMetadata loadMetadata, LoadStage loadStage)
{
var cataRepository = ((CatalogueRepository)Repository);
//clone only accepts sql connections so make sure we aren't in mysql land or something
using (cataRepository.BeginNewTransactedConnection())
{
try
{
//get list of arguments to also clone (will happen outside of transaction
ProcessTaskArgument[] toCloneArguments = ProcessTaskArguments.ToArray();
//create a new transaction for all the cloning - note that once all objects are cloned the transaction is committed then all the objects are adjusted outside the transaction
ProcessTask clone = new ProcessTask(CatalogueRepository, LoadMetadata, loadStage);
CopyShallowValuesTo(clone);
//foreach of our child arguments
foreach (ProcessTaskArgument argument in toCloneArguments)
{
//clone it but rewire it to the proper ProcessTask parent (the clone)
argument.ShallowClone(clone);
}
//the values passed into parameter
clone.LoadMetadata_ID = loadMetadata.ID;
clone.LoadStage = loadStage;
clone.SaveToDatabase();
//it worked
cataRepository.EndTransactedConnection(true);
//return the clone
return(clone);
}
catch (Exception)
{
cataRepository.EndTransactedConnection(false);
throw;
}
}
}
public List <IRuntimeTask> GetRuntimeTasksForStage(LoadStage loadStage)
{
var runtimeTasks = new List <IRuntimeTask>();
var tasksForThisLoadStage = ProcessTasks.Where(task => task.LoadStage == loadStage).ToList();
if (!tasksForThisLoadStage.Any())
{
return(runtimeTasks);
}
var factory = new Runtime.RuntimeTaskFactory(_repository);
foreach (var processTask in tasksForThisLoadStage)
{
runtimeTasks.Add(factory.Create(processTask, _loadArgsDictionary[processTask.LoadStage]));
}
runtimeTasks = runtimeTasks.OrderBy(task => task.ProcessTask.Order).ToList();
return(runtimeTasks);
}
internal ProcessTask(ICatalogueRepository repository, DbDataReader r)
: base(repository, r)
{
LoadMetadata_ID = int.Parse(r["LoadMetaData_ID"].ToString());
if (r["RelatesSolelyToCatalogue_ID"] != DBNull.Value)
{
_relatesSolelyToCatalogueID = int.Parse(r["RelatesSolelyToCatalogue_ID"].ToString());
}
Path = r["Path"] as string;
Name = r["Name"] as string;
Order = int.Parse(r["Order"].ToString());
ProcessTaskType processTaskType;
if (ProcessTaskType.TryParse(r["ProcessTaskType"] as string, out processTaskType))
{
ProcessTaskType = processTaskType;
}
else
{
throw new Exception("Could not parse ProcessTaskType:" + r["ProcessTaskType"]);
}
LoadStage loadStage;
if (LoadStage.TryParse(r["LoadStage"] as string, out loadStage))
{
LoadStage = loadStage;
}
else
{
throw new Exception("Could not parse LoadStage:" + r["LoadStage"]);
}
IsDisabled = Convert.ToBoolean(r["IsDisabled"]);
}
private Bitmap GetImageForStage(LoadStage loadStage)
{
switch (loadStage)
{
case LoadStage.GetFiles:
return(_iconProvider.GetImage(RDMPConcept.GetFilesStage));
case LoadStage.Mounting:
return(_iconProvider.GetImage(RDMPConcept.LoadBubbleMounting));
case LoadStage.AdjustRaw:
return(_iconProvider.GetImage(RDMPConcept.LoadBubble));
case LoadStage.AdjustStaging:
return(_iconProvider.GetImage(RDMPConcept.LoadBubble));
case LoadStage.PostLoad:
return(_iconProvider.GetImage(RDMPConcept.LoadFinalDatabase));
default:
throw new ArgumentOutOfRangeException();
}
}
/// <summary>
/// Returns true if the <see cref="ProcessTaskType"/> is allowed to happen during the given <see cref="LoadStage"/> (e.g. you can't use an IAttacher to
/// load data into STAGING/LIVE - only RAW).
/// </summary>
/// <param name="type"></param>
/// <param name="stage"></param>
/// <returns></returns>
public static bool IsCompatibleStage(ProcessTaskType type, LoadStage stage)
{
switch (type)
{
case ProcessTaskType.Executable:
return(true);
case ProcessTaskType.SQLFile:
return(stage != LoadStage.GetFiles);
case ProcessTaskType.Attacher:
return(stage == LoadStage.Mounting);
case ProcessTaskType.DataProvider:
return(true);
case ProcessTaskType.MutilateDataTable:
return(stage != LoadStage.GetFiles);
default:
throw new ArgumentOutOfRangeException("type");
}
}
/// <summary>
/// Converts a <see cref="LoadStage"/> into a <see cref="LoadBubble"/>
/// </summary>
/// <param name="loadStage"></param>
/// <returns></returns>
public static LoadBubble ToLoadBubble(this LoadStage loadStage)
{
switch (loadStage)
{
case LoadStage.GetFiles:
return(LoadBubble.Raw);
case LoadStage.Mounting:
return(LoadBubble.Raw);
case LoadStage.AdjustRaw:
return(LoadBubble.Raw);
case LoadStage.AdjustStaging:
return(LoadBubble.Staging);
case LoadStage.PostLoad:
return(LoadBubble.Live);
default:
throw new ArgumentOutOfRangeException("Unknown value for LoadStage: " + loadStage);
}
}
/// <inheritdoc/>
public IEnumerable <IHasStageSpecificRuntimeName> GetColumnsAtStage(LoadStage loadStage)
{
//if it is AdjustRaw then it will also have the pre load discarded columns
if (loadStage <= LoadStage.AdjustRaw)
{
foreach (PreLoadDiscardedColumn discardedColumn in PreLoadDiscardedColumns.Where(c => c.Destination != DiscardedColumnDestination.Dilute))
{
yield return(discardedColumn);
}
}
//also add column infos
foreach (ColumnInfo c in ColumnInfos)
{
if (loadStage <= LoadStage.AdjustRaw && SpecialFieldNames.IsHicPrefixed(c))
{
continue;
}
else
if (loadStage <= LoadStage.AdjustStaging && c.IsAutoIncrement) //auto increment columns do not get created in RAW/STAGING
{
continue;
}
else
if (loadStage == LoadStage.AdjustStaging &&
//these two do not appear in staging
(c.GetRuntimeName().Equals(SpecialFieldNames.DataLoadRunID) || c.GetRuntimeName().Equals(SpecialFieldNames.ValidFrom))
)
{
continue;
}
else
{
yield return(c);
}
}
}
public ExecuteSqlInDleStage(IDataLoadJob job, LoadStage loadStage)
{
_job = job;
_loadStage = loadStage;
}
/// <inheritdoc/>
public string GetRuntimeName(LoadStage stage, INameDatabasesAndTablesDuringLoads tableNamingScheme = null)
{
return(GetRuntimeName(stage.ToLoadBubble(), tableNamingScheme));
}
