public DataTable ProcessPipelineData(DataTable toProcess, IDataLoadEventListener listener, GracefulCancellationToken cancellationToken)
{
bool didAno = false;
stopwatch_TimeSpentTransforming.Start();
if (!_bInitialized)
{
throw new Exception("Not Initialized yet");
}
recordsProcessedSoFar += toProcess.Rows.Count;
var missingColumns = columnsToAnonymise.Keys.Where(k => !toProcess.Columns.Cast <DataColumn>().Any(c => c.ColumnName.Equals(k))).ToArray();
if (missingColumns.Any())
{
throw new KeyNotFoundException("The following columns (which have ANO Transforms on them) were missing from the DataTable:" + Environment.NewLine
+ string.Join(Environment.NewLine, missingColumns) + Environment.NewLine + "The columns found in the DataTable were:" + Environment.NewLine
+ string.Join(Environment.NewLine, toProcess.Columns.Cast <DataColumn>().Select(c => c.ColumnName)));
}
//Dump Identifiers
stopwatch_TimeSpentDumping.Start();
_dumper.DumpAllIdentifiersInTable(toProcess); //do the dumping of all the rest of the columns (those that must disapear from pipeline as opposed to those above which were substituted for ANO versions)
stopwatch_TimeSpentDumping.Stop();
if (_dumper.HaveDumpedRecords)
{
listener.OnProgress(this, new ProgressEventArgs("Dump Identifiers", new ProgressMeasurement(recordsProcessedSoFar, ProgressType.Records), stopwatch_TimeSpentDumping.Elapsed));//time taken to dump identifiers
}
//Process ANO Identifier Substitutions
//for each column with an ANOTrasformer
foreach (KeyValuePair <string, ANOTransformer> kvp in columnsToAnonymise)
{
didAno = true;
var column = kvp.Key;
ANOTransformer transformer = kvp.Value;
//add an ANO version
DataColumn ANOColumn = new DataColumn(ANOTable.ANOPrefix + column);
toProcess.Columns.Add(ANOColumn);
//populate ANO version
transformer.Transform(toProcess, toProcess.Columns[column], ANOColumn);
//drop the non ANO version
toProcess.Columns.Remove(column);
}
stopwatch_TimeSpentTransforming.Stop();
if (didAno)
{
listener.OnProgress(this, new ProgressEventArgs("Anonymise Identifiers", new ProgressMeasurement(recordsProcessedSoFar, ProgressType.Records), stopwatch_TimeSpentTransforming.Elapsed)); //time taken to swap ANO identifiers
}
return(toProcess);
}
private void MigrateExistingData(Func <string, bool> shouldApplySql, DbConnection con, ICheckNotifier notifier, DiscoveredTable tbl)
{
string from = _colToNuke.GetRuntimeName(LoadStage.PostLoad);
string to = _newANOColumnInfo.GetRuntimeName(LoadStage.PostLoad);
//create an empty table for the anonymised data
DbCommand cmdCreateTempMap = DatabaseCommandHelper.GetCommand(string.Format("SELECT top 0 {0},{1} into TempANOMap from {2}", from, to, tbl.GetFullyQualifiedName()), con);
if (!shouldApplySql(cmdCreateTempMap.CommandText))
{
throw new Exception("User decided not to create the TempANOMap table");
}
cmdCreateTempMap.ExecuteNonQuery();
try
{
//get the existing data
DbCommand cmdGetExistingData = DatabaseCommandHelper.GetCommand(string.Format("SELECT {0},{1} from {2}", from, to, tbl.GetFullyQualifiedName()), con);
DbDataAdapter da = DatabaseCommandHelper.GetDataAdapter(cmdGetExistingData);
DataTable dt = new DataTable();
da.Fill(dt);//into memory
//transform it in memory
ANOTransformer transformer = new ANOTransformer(_toConformTo, new FromCheckNotifierToDataLoadEventListener(notifier));
transformer.Transform(dt, dt.Columns[0], dt.Columns[1]);
var tempAnoMapTbl = tbl.Database.ExpectTable("TempANOMap");
using (var insert = tempAnoMapTbl.BeginBulkInsert())
{
insert.Upload(dt);
}
//create an empty table for the anonymised data
DbCommand cmdUpdateMainTable = DatabaseCommandHelper.GetCommand(string.Format("UPDATE source set source.{1} = map.{1} from {2} source join TempANOMap map on source.{0}=map.{0}", from, to, tbl.GetFullyQualifiedName()), con);
if (!shouldApplySql(cmdUpdateMainTable.CommandText))
{
throw new Exception("User decided not to perform update on table");
}
cmdUpdateMainTable.ExecuteNonQuery();
}
finally
{
//always drop the temp anomap
DbCommand dropMappingTable = DatabaseCommandHelper.GetCommand("DROP TABLE TempANOMap", con);
dropMappingTable.ExecuteNonQuery();
}
}
