public DataTable ProcessPipelineData(DataTable toProcess, IDataLoadEventListener listener, GracefulCancellationToken cancellationToken) { bool didAno = false; stopwatch_TimeSpentTransforming.Start(); if (!_bInitialized) { throw new Exception("Not Initialized yet"); } recordsProcessedSoFar += toProcess.Rows.Count; var missingColumns = columnsToAnonymise.Keys.Where(k => !toProcess.Columns.Cast <DataColumn>().Any(c => c.ColumnName.Equals(k))).ToArray(); if (missingColumns.Any()) { throw new KeyNotFoundException("The following columns (which have ANO Transforms on them) were missing from the DataTable:" + Environment.NewLine + string.Join(Environment.NewLine, missingColumns) + Environment.NewLine + "The columns found in the DataTable were:" + Environment.NewLine + string.Join(Environment.NewLine, toProcess.Columns.Cast <DataColumn>().Select(c => c.ColumnName))); } //Dump Identifiers stopwatch_TimeSpentDumping.Start(); _dumper.DumpAllIdentifiersInTable(toProcess); //do the dumping of all the rest of the columns (those that must disapear from pipeline as opposed to those above which were substituted for ANO versions) stopwatch_TimeSpentDumping.Stop(); if (_dumper.HaveDumpedRecords) { listener.OnProgress(this, new ProgressEventArgs("Dump Identifiers", new ProgressMeasurement(recordsProcessedSoFar, ProgressType.Records), stopwatch_TimeSpentDumping.Elapsed));//time taken to dump identifiers } //Process ANO Identifier Substitutions //for each column with an ANOTrasformer foreach (KeyValuePair <string, ANOTransformer> kvp in columnsToAnonymise) { didAno = true; var column = kvp.Key; ANOTransformer transformer = kvp.Value; //add an ANO version DataColumn ANOColumn = new DataColumn(ANOTable.ANOPrefix + column); toProcess.Columns.Add(ANOColumn); //populate ANO version transformer.Transform(toProcess, toProcess.Columns[column], ANOColumn); //drop the non ANO version toProcess.Columns.Remove(column); } stopwatch_TimeSpentTransforming.Stop(); if (didAno) { listener.OnProgress(this, new ProgressEventArgs("Anonymise Identifiers", new ProgressMeasurement(recordsProcessedSoFar, ProgressType.Records), stopwatch_TimeSpentTransforming.Elapsed)); //time taken to swap ANO identifiers } return(toProcess); }
private void MigrateExistingData(Func <string, bool> shouldApplySql, DbConnection con, ICheckNotifier notifier, DiscoveredTable tbl) { string from = _colToNuke.GetRuntimeName(LoadStage.PostLoad); string to = _newANOColumnInfo.GetRuntimeName(LoadStage.PostLoad); //create an empty table for the anonymised data DbCommand cmdCreateTempMap = DatabaseCommandHelper.GetCommand(string.Format("SELECT top 0 {0},{1} into TempANOMap from {2}", from, to, tbl.GetFullyQualifiedName()), con); if (!shouldApplySql(cmdCreateTempMap.CommandText)) { throw new Exception("User decided not to create the TempANOMap table"); } cmdCreateTempMap.ExecuteNonQuery(); try { //get the existing data DbCommand cmdGetExistingData = DatabaseCommandHelper.GetCommand(string.Format("SELECT {0},{1} from {2}", from, to, tbl.GetFullyQualifiedName()), con); DbDataAdapter da = DatabaseCommandHelper.GetDataAdapter(cmdGetExistingData); DataTable dt = new DataTable(); da.Fill(dt);//into memory //transform it in memory ANOTransformer transformer = new ANOTransformer(_toConformTo, new FromCheckNotifierToDataLoadEventListener(notifier)); transformer.Transform(dt, dt.Columns[0], dt.Columns[1]); var tempAnoMapTbl = tbl.Database.ExpectTable("TempANOMap"); using (var insert = tempAnoMapTbl.BeginBulkInsert()) { insert.Upload(dt); } //create an empty table for the anonymised data DbCommand cmdUpdateMainTable = DatabaseCommandHelper.GetCommand(string.Format("UPDATE source set source.{1} = map.{1} from {2} source join TempANOMap map on source.{0}=map.{0}", from, to, tbl.GetFullyQualifiedName()), con); if (!shouldApplySql(cmdUpdateMainTable.CommandText)) { throw new Exception("User decided not to perform update on table"); } cmdUpdateMainTable.ExecuteNonQuery(); } finally { //always drop the temp anomap DbCommand dropMappingTable = DatabaseCommandHelper.GetCommand("DROP TABLE TempANOMap", con); dropMappingTable.ExecuteNonQuery(); } }