public ExecuteCommandCreateNewProcessTask(IActivateItems activator, ProcessTaskType taskType, LoadMetadata loadMetadata, LoadStage loadStage, FileInfo file = null) : base(activator) { _taskType = taskType; _loadMetadata = loadMetadata; _loadStage = loadStage; try { _LoadDirectory = new LoadDirectory(_loadMetadata.LocationOfFlatFiles); } catch (Exception) { SetImpossible("Could not construct LoadDirectory"); } if (taskType == ProcessTaskType.SQLFile) { _image = activator.CoreIconProvider.GetImage(RDMPConcept.SQL, OverlayKind.Add); } else if (taskType == ProcessTaskType.Executable) { _image = new IconOverlayProvider().GetOverlayNoCache(CatalogueIcons.Exe, OverlayKind.Add); } else { SetImpossible("Only SQLFile and Executable task types are supported by this command"); } if (!ProcessTask.IsCompatibleStage(taskType, loadStage)) { SetImpossible("You cannot run " + taskType + " in " + loadStage); } _file = file; }
protected IStageArgs CreateLoadArgs(LoadStage loadStage) { return (new StageArgs(loadStage, _dbDeployInfo[loadStage.ToLoadBubble()] , new LoadDirectory(_loadMetadata.LocationOfFlatFiles.TrimEnd(new[] { '\\' })))); }
public ExecuteCommandCreateNewClassBasedProcessTask(IBasicActivateItems activator, LoadMetadata loadMetadata, LoadStage loadStage, [DemandsInitialization("Class to execute, must be an attacher, mutilater etc", TypeOf = typeof(IDisposeAfterDataLoad))] Type type) : base(activator) { _loadMetadata = loadMetadata; _loadStage = loadStage; _type = type; if (typeof(IAttacher).IsAssignableFrom(_type)) { _processTaskType = ProcessTaskType.Attacher; } else if (typeof(IDataProvider).IsAssignableFrom(_type)) { _processTaskType = ProcessTaskType.DataProvider; } else if (typeof(IMutilateDataTables).IsAssignableFrom(_type)) { _processTaskType = ProcessTaskType.MutilateDataTable; } else { SetImpossible($"Type '{_type}' was not a compatible one e.g. IAttacher, IDataProvider or IMutilateDataTables"); } }
public void Initialize(DiscoveredDatabase dbInfo, LoadStage loadStage) { if (loadStage != LoadStage.AdjustStaging && loadStage != LoadStage.PostLoad) { throw new Exception(typeof(BackupDatabaseMutilation).Name + " can only be done in AdjustStaging or PostLoad (this minimises redundant backups that would otherwise be created while you attempt to fix RAW / constraint related load errors)"); } }
/// <summary> /// Gets the DataType adjusted for the stage at which the ColumnInfo is at, this is almost always the same as Data_type. The only /// time it is different is when there is an ANOTable involved e.g. ANOLocation could be a varchar(6) like 'AB10_L' after anonymisation /// but if the LoadStage is AdjustRaw then it would have a value like 'NH10' (varchar(4) - the unanonymised state). /// </summary> /// <param name="loadStage"></param> /// <returns></returns> public string GetRuntimeDataType(LoadStage loadStage) { if (loadStage <= LoadStage.AdjustRaw) { //if it has an ANO transform if (ANOTable_ID != null) { return(ANOTable.GetRuntimeDataType(loadStage)); //get the datatype from the ANOTable because ColumnInfo is of mutable type depending on whether it has been anonymised yet } //it doesn't have an ANOtransform but it might be the subject of dilution var discard = TableInfo.PreLoadDiscardedColumns.SingleOrDefault(c => c.GetRuntimeName().Equals(GetRuntimeName(), StringComparison.InvariantCultureIgnoreCase)); //The column exists both in the live database and in the identifier dump. This is because it goes through horrendous bitcrushing operations e.g. Load RAW with full //postcode varchar(8) and ship postcode off to identifier dump but also let it go through to live but only as the first 4 letters varchar(4). so the datatype of the column //in RAW is varchar(8) but in Live is varchar(4) if (discard != null) { return(discard.Data_type); } return(Data_type); } //The user is asking about a stage other than RAW so tell them about the final column type state return(Data_type); }
public ExecuteCommandCreateNewFileBasedProcessTask(IBasicActivateItems activator, ProcessTaskType taskType, LoadMetadata loadMetadata, LoadStage loadStage, FileInfo file = null) : base(activator) { _taskType = taskType; _loadMetadata = loadMetadata; _loadStage = loadStage; try { _LoadDirectory = new LoadDirectory(_loadMetadata.LocationOfFlatFiles); } catch (Exception) { SetImpossible("Could not construct LoadDirectory"); } if (!(taskType == ProcessTaskType.SQLFile || taskType == ProcessTaskType.Executable)) { SetImpossible("Only SQLFile and Executable task types are supported by this command"); } if (!ProcessTask.IsCompatibleStage(taskType, loadStage)) { SetImpossible("You cannot run " + taskType + " in " + loadStage); } _file = file; }
public void Initialize(DiscoveredDatabase dbInfo, LoadStage loadStage) { if (loadStage != LoadStage.AdjustRaw) { throw new Exception("Primary key collisions can only be resolved in a RAW environment, current load stage is:" + loadStage + " (The reason for this is because there should be primary keys in the database level in STAGING and LIVE making primary key collisions IMPOSSIBE)"); } _dbInfo = dbInfo; }
public void EmptyClassPath(string path, ProcessTaskType typeThatRequiresMEF, LoadStage stage) { _task.ProcessTaskType = typeThatRequiresMEF; _task.Path = path; _task.LoadStage = stage; _task.SaveToDatabase(); var ex = Assert.Throws <ArgumentException>(() => _checker.Check(new ThrowImmediatelyCheckNotifier())); Assert.IsTrue(Regex.IsMatch(ex.Message, "Path is blank for ProcessTask 'New Process.*' - it should be a class name of type")); }
public void Initialize(DiscoveredDatabase dbInfo, LoadStage loadStage) { _raw = dbInfo; _syntaxHelper = _raw.Server.GetQuerySyntaxHelper(); if (loadStage != LoadStage.AdjustRaw) { throw new Exception("This component should only run in AdjustRaw"); } }
public void Initialize(DiscoveredDatabase dbInfo, LoadStage loadStage) { if (_allowedStages != null && !_allowedStages.Contains(loadStage)) { throw new NotSupportedException("Mutilation " + GetType() + " is not allowed at stage " + loadStage); } _loadStage = loadStage; DbInfo = dbInfo; }
/// <inheritdoc/> public string GetDatabaseRuntimeName(LoadStage loadStage, INameDatabasesAndTablesDuringLoads namer = null) { var baseName = GetDatabaseRuntimeName(); if (namer == null) { namer = new FixedStagingDatabaseNamer(baseName); } return(namer.GetDatabaseName(baseName, loadStage.ToLoadBubble())); }
public ExecuteCommandCreateNewClassBasedProcessTask(IBasicActivateItems activator, LoadMetadata loadMetadata, LoadStage loadStage, [DemandsInitialization("Class to execute, must be an attacher, mutilater etc", TypeOf = typeof(IDisposeAfterDataLoad))] Type type) : base(activator) { _loadMetadata = loadMetadata; _loadStage = loadStage; if (type != null) { SetType(type); } }
/// <summary> /// Creates a new operation in the data load (e.g. copy files from A to B, load all CSV files to RAW table B etc) /// </summary> /// <param name="repository"></param> /// <param name="parent"></param> /// <param name="stage"></param> public ProcessTask(ICatalogueRepository repository, ILoadMetadata parent, LoadStage stage) { var order = repository.GetAllObjectsWithParent <ProcessTask>(parent).Select(t => t.Order).DefaultIfEmpty().Max() + 1; repository.InsertAndHydrate(this, new Dictionary <string, object> { { "LoadMetadata_ID", parent.ID }, { "ProcessTaskType", ProcessTaskType.Executable.ToString() }, { "LoadStage", stage }, { "Name", "New Process" + Guid.NewGuid() }, { "Order", order } }); }
/// <summary>Get the data fields to read/write for save data.</summary> /// <param name="stage">The current load stage.</param> private IEnumerable <IDictionary <string, string> > GetDataFields(LoadStage stage) { if (stage == LoadStage.None) { yield break; } yield return(Game1.CustomData); if (SaveGame.loaded != null) { yield return(SaveGame.loaded.CustomData); } }
/// <summary> /// Anonymisation with an <see cref="ANOTable"/> happens during data load. This means that the column goes from identifiable in RAW to anonymous in STAGING/LIVE. This means /// that the datatype of the column changes depending on the <see cref="LoadStage"/>. /// /// <para>Returns the appropriate datatype for the <see cref="LoadStage"/>. This is done by connecting to the mapping table and retrieving the mapping table types</para> /// </summary> /// <param name="loadStage"></param> /// <returns></returns> public string GetRuntimeDataType(LoadStage loadStage) { //cache answers if (_identifiableDataType == null) { var server = DataAccessPortal.GetInstance().ExpectServer(Server, DataAccessContext.DataLoad); DiscoveredColumn[] columnsFoundInANO = server.GetCurrentDatabase().ExpectTable(TableName).DiscoverColumns(); string expectedIdentifiableName = TableName.Substring("ANO".Length); DiscoveredColumn anonymous = columnsFoundInANO.SingleOrDefault(c => c.GetRuntimeName().Equals(TableName)); DiscoveredColumn identifiable = columnsFoundInANO.SingleOrDefault(c => c.GetRuntimeName().Equals(expectedIdentifiableName)); if (anonymous == null) { throw new Exception("Could not find a column called " + TableName + " in table " + TableName + " on server " + Server + " (Columns found were " + string.Join(",", columnsFoundInANO.Select(c => c.GetRuntimeName()).ToArray()) + ")"); } if (identifiable == null) { throw new Exception("Could not find a column called " + expectedIdentifiableName + " in table " + TableName + " on server " + Server + " (Columns found were " + string.Join(",", columnsFoundInANO.Select(c => c.GetRuntimeName()).ToArray()) + ")"); } _identifiableDataType = identifiable.DataType.SQLType; _anonymousDataType = anonymous.DataType.SQLType; } //return cached answer switch (loadStage) { case LoadStage.GetFiles: return(_identifiableDataType); case LoadStage.Mounting: return(_identifiableDataType); case LoadStage.AdjustRaw: return(_identifiableDataType); case LoadStage.AdjustStaging: return(_anonymousDataType); case LoadStage.PostLoad: return(_anonymousDataType); default: throw new ArgumentOutOfRangeException("loadStage"); } }
public LoadState(Game game, Func <State> nextState) : base(game) { m_stage = LoadStage.Startup; m_nextStateFunc = nextState; m_loadTask = null; m_backdrop = new Image(Texture.Black, Game.Screen.Width, Game.Screen.Height); m_backdrop.Anchor = Anchor.CentreMiddle; m_backdrop.LocalPosition = new Vector2(-0.5f * m_backdrop.Width, -0.5f * m_backdrop.Height); m_widget = new Image(Texture.Get("gui/loading.png", true), GetWidgetQuad(0.0f), 64.0f, 64.0f); m_widget.Anchor = Anchor.CentreMiddle; m_widget.LocalPosition = new Vector2(-0.5f * m_widget.Width, -0.5f * m_widget.Height); m_timeInStage = 0.0f; }
public CompositeDataLoadComponent CreateCompositeDataLoadComponentFor(LoadStage loadStage, string descriptionForComponent) { RuntimeTaskFactory factory = new RuntimeTaskFactory(_repository); var tasks = new List <IDataLoadComponent>(); foreach (var task in GetRuntimeTasksForStage(loadStage)) { tasks.Add(factory.Create(task.ProcessTask, _loadArgsDictionary[loadStage])); } return(new CompositeDataLoadComponent(tasks) { Description = descriptionForComponent }); }
///<inheritdoc/> public string GetRuntimeName(LoadStage stage) { string finalName = this.GetRuntimeName(); if (stage <= LoadStage.AdjustRaw) { //see if it has an ANO Transform on it if (ANOTable_ID != null && finalName.StartsWith("ANO")) { return(finalName.Substring("ANO".Length)); } } //any other stage will be the regular final name return(finalName); }
public override ExitCodeType Run(IDataLoadJob job, GracefulCancellationToken cancellationToken) { var db = RuntimeArguments.StageSpecificArguments.DbInfo; _loadStage = RuntimeArguments.StageSpecificArguments.LoadStage; if (!Exists()) { throw new Exception("The sql file " + Filepath + " does not exist"); } string commandText; try { commandText = File.ReadAllText(Filepath); // Any string arguments refer to tokens that are to be replaced in the SQL file foreach (var kvp in RuntimeArguments.GetAllArgumentsOfType <string>()) { var value = kvp.Value; if (value.Contains("<DatabaseServer>")) { value = value.Replace("<DatabaseServer>", RuntimeArguments.StageSpecificArguments.DbInfo.Server.Name); } if (value.Contains("<DatabaseName>")) { value = value.Replace("<DatabaseName>", RuntimeArguments.StageSpecificArguments.DbInfo.GetRuntimeName()); } commandText = commandText.Replace("##" + kvp.Key + "##", value); } } catch (Exception e) { throw new Exception("Could not read the sql file at " + Filepath + ": " + e); } job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Executing script " + Filepath + " (" + db.DescribeDatabase() + ")")); var executer = new ExecuteSqlInDleStage(job, _loadStage); return(executer.Execute(commandText, db)); }
public void Add(ITableInfo tableInfo, LoadStage loadStage) { //we already have it or it is not setup properly if (items.Any(i => i.Tag.Equals(tableInfo)) || string.IsNullOrWhiteSpace(tableInfo.Database) || string.IsNullOrWhiteSpace(tableInfo.Server)) { return; } var runtimeName = tableInfo.GetRuntimeName(loadStage); var dbName = tableInfo.GetDatabaseRuntimeName(loadStage); var syntaxHelper = tableInfo.GetQuerySyntaxHelper(); var fullSql = syntaxHelper.EnsureFullyQualified(dbName, null, runtimeName); var snip = new SubstringAutocompleteItem(tableInfo.GetRuntimeName()); snip.MenuText = runtimeName; //name of table snip.Text = fullSql; //full SQL snip.Tag = tableInfo; //record object for future reference snip.ImageIndex = GetIndexFor(tableInfo, RDMPConcept.TableInfo.ToString()); foreach (IHasStageSpecificRuntimeName o in tableInfo.GetColumnsAtStage(loadStage)) { var preDiscarded = o as PreLoadDiscardedColumn; var columnInfo = o as ColumnInfo; if (preDiscarded != null) { Add(preDiscarded, tableInfo, dbName); } else if (columnInfo != null) { Add(columnInfo, tableInfo, dbName, loadStage, syntaxHelper); } else { throw new Exception("Expected IHasStageSpecificRuntimeName returned by TableInfo.GetColumnsAtStage to return only ColumnInfos and PreLoadDiscardedColumns. It returned a '" + o.GetType().Name + "'"); } } AddUnlessDuplicate(snip); }
public IEbicsClient Create( Func <string, byte[]> readBytes, Action <string, byte[]> writeBytes, string password, string address, string hostId, string partnerId, string userId, LoadStage loadStage = LoadStage.All) { var signCert = loadStage >= LoadStage.Sign?new System.Security.Cryptography.X509Certificates.X509Certificate2(readBytes("sign.p12"), password):null; var authCert = loadStage >= LoadStage.Auth ? new System.Security.Cryptography.X509Certificates.X509Certificate2(readBytes("auth.p12"), password):null; var encCert = loadStage >= LoadStage.Enc ? new System.Security.Cryptography.X509Certificates.X509Certificate2(readBytes("enc.p12"), password):null; var cfg = new EbicsConfig { readBytes = readBytes, writeBytes = writeBytes, Address = address, User = new UserParams { HostId = hostId, PartnerId = partnerId, UserId = userId, SignKeys = new SignKeyPair { Version = SignVersion.A005, Certificate = signCert }, AuthKeys = new AuthKeyPair { Version = AuthVersion.X002, Certificate = authCert }, CryptKeys = new CryptKeyPair { Version = CryptVersion.E002, Certificate = encCert } } }; if (loadStage >= LoadStage.Bank) { cfg.LoadBank(); } return(_ctor(cfg)); }
/// <summary> /// Creates a new copy of the processTask and all it's arguments in the database, this clone is then hooked up to the /// new LoadMetadata at the specified stage /// </summary> /// <param name="loadMetadata">The new LoadMetadata parent for the clone</param> /// <param name="loadStage">The new load stage to put the clone in </param> /// <returns>the new ProcessTask (the clone has a different ID to the parent)</returns> public ProcessTask CloneToNewLoadMetadataStage(LoadMetadata loadMetadata, LoadStage loadStage) { var cataRepository = ((CatalogueRepository)Repository); //clone only accepts sql connections so make sure we aren't in mysql land or something using (cataRepository.BeginNewTransactedConnection()) { try { //get list of arguments to also clone (will happen outside of transaction ProcessTaskArgument[] toCloneArguments = ProcessTaskArguments.ToArray(); //create a new transaction for all the cloning - note that once all objects are cloned the transaction is committed then all the objects are adjusted outside the transaction ProcessTask clone = new ProcessTask(CatalogueRepository, LoadMetadata, loadStage); CopyShallowValuesTo(clone); //foreach of our child arguments foreach (ProcessTaskArgument argument in toCloneArguments) { //clone it but rewire it to the proper ProcessTask parent (the clone) argument.ShallowClone(clone); } //the values passed into parameter clone.LoadMetadata_ID = loadMetadata.ID; clone.LoadStage = loadStage; clone.SaveToDatabase(); //it worked cataRepository.EndTransactedConnection(true); //return the clone return(clone); } catch (Exception) { cataRepository.EndTransactedConnection(false); throw; } } }
public List <IRuntimeTask> GetRuntimeTasksForStage(LoadStage loadStage) { var runtimeTasks = new List <IRuntimeTask>(); var tasksForThisLoadStage = ProcessTasks.Where(task => task.LoadStage == loadStage).ToList(); if (!tasksForThisLoadStage.Any()) { return(runtimeTasks); } var factory = new Runtime.RuntimeTaskFactory(_repository); foreach (var processTask in tasksForThisLoadStage) { runtimeTasks.Add(factory.Create(processTask, _loadArgsDictionary[processTask.LoadStage])); } runtimeTasks = runtimeTasks.OrderBy(task => task.ProcessTask.Order).ToList(); return(runtimeTasks); }
internal ProcessTask(ICatalogueRepository repository, DbDataReader r) : base(repository, r) { LoadMetadata_ID = int.Parse(r["LoadMetaData_ID"].ToString()); if (r["RelatesSolelyToCatalogue_ID"] != DBNull.Value) { _relatesSolelyToCatalogueID = int.Parse(r["RelatesSolelyToCatalogue_ID"].ToString()); } Path = r["Path"] as string; Name = r["Name"] as string; Order = int.Parse(r["Order"].ToString()); ProcessTaskType processTaskType; if (ProcessTaskType.TryParse(r["ProcessTaskType"] as string, out processTaskType)) { ProcessTaskType = processTaskType; } else { throw new Exception("Could not parse ProcessTaskType:" + r["ProcessTaskType"]); } LoadStage loadStage; if (LoadStage.TryParse(r["LoadStage"] as string, out loadStage)) { LoadStage = loadStage; } else { throw new Exception("Could not parse LoadStage:" + r["LoadStage"]); } IsDisabled = Convert.ToBoolean(r["IsDisabled"]); }
private Bitmap GetImageForStage(LoadStage loadStage) { switch (loadStage) { case LoadStage.GetFiles: return(_iconProvider.GetImage(RDMPConcept.GetFilesStage)); case LoadStage.Mounting: return(_iconProvider.GetImage(RDMPConcept.LoadBubbleMounting)); case LoadStage.AdjustRaw: return(_iconProvider.GetImage(RDMPConcept.LoadBubble)); case LoadStage.AdjustStaging: return(_iconProvider.GetImage(RDMPConcept.LoadBubble)); case LoadStage.PostLoad: return(_iconProvider.GetImage(RDMPConcept.LoadFinalDatabase)); default: throw new ArgumentOutOfRangeException(); } }
/// <summary> /// Returns true if the <see cref="ProcessTaskType"/> is allowed to happen during the given <see cref="LoadStage"/> (e.g. you can't use an IAttacher to /// load data into STAGING/LIVE - only RAW). /// </summary> /// <param name="type"></param> /// <param name="stage"></param> /// <returns></returns> public static bool IsCompatibleStage(ProcessTaskType type, LoadStage stage) { switch (type) { case ProcessTaskType.Executable: return(true); case ProcessTaskType.SQLFile: return(stage != LoadStage.GetFiles); case ProcessTaskType.Attacher: return(stage == LoadStage.Mounting); case ProcessTaskType.DataProvider: return(true); case ProcessTaskType.MutilateDataTable: return(stage != LoadStage.GetFiles); default: throw new ArgumentOutOfRangeException("type"); } }
/// <summary> /// Converts a <see cref="LoadStage"/> into a <see cref="LoadBubble"/> /// </summary> /// <param name="loadStage"></param> /// <returns></returns> public static LoadBubble ToLoadBubble(this LoadStage loadStage) { switch (loadStage) { case LoadStage.GetFiles: return(LoadBubble.Raw); case LoadStage.Mounting: return(LoadBubble.Raw); case LoadStage.AdjustRaw: return(LoadBubble.Raw); case LoadStage.AdjustStaging: return(LoadBubble.Staging); case LoadStage.PostLoad: return(LoadBubble.Live); default: throw new ArgumentOutOfRangeException("Unknown value for LoadStage: " + loadStage); } }
/// <inheritdoc/> public IEnumerable <IHasStageSpecificRuntimeName> GetColumnsAtStage(LoadStage loadStage) { //if it is AdjustRaw then it will also have the pre load discarded columns if (loadStage <= LoadStage.AdjustRaw) { foreach (PreLoadDiscardedColumn discardedColumn in PreLoadDiscardedColumns.Where(c => c.Destination != DiscardedColumnDestination.Dilute)) { yield return(discardedColumn); } } //also add column infos foreach (ColumnInfo c in ColumnInfos) { if (loadStage <= LoadStage.AdjustRaw && SpecialFieldNames.IsHicPrefixed(c)) { continue; } else if (loadStage <= LoadStage.AdjustStaging && c.IsAutoIncrement) //auto increment columns do not get created in RAW/STAGING { continue; } else if (loadStage == LoadStage.AdjustStaging && //these two do not appear in staging (c.GetRuntimeName().Equals(SpecialFieldNames.DataLoadRunID) || c.GetRuntimeName().Equals(SpecialFieldNames.ValidFrom)) ) { continue; } else { yield return(c); } } }
public ExecuteSqlInDleStage(IDataLoadJob job, LoadStage loadStage) { _job = job; _loadStage = loadStage; }
/// <inheritdoc/> public string GetRuntimeName(LoadStage stage, INameDatabasesAndTablesDuringLoads tableNamingScheme = null) { return(GetRuntimeName(stage.ToLoadBubble(), tableNamingScheme)); }