Пример #1
0
        public DataTable ProcessPipelineData(DataTable toProcess, IDataLoadEventListener listener, GracefulCancellationToken cancellationToken)
        {
            bool didAno = false;

            stopwatch_TimeSpentTransforming.Start();

            if (!_bInitialized)
            {
                throw new Exception("Not Initialized yet");
            }

            recordsProcessedSoFar += toProcess.Rows.Count;

            var missingColumns = columnsToAnonymise.Keys.Where(k => !toProcess.Columns.Cast <DataColumn>().Any(c => c.ColumnName.Equals(k))).ToArray();

            if (missingColumns.Any())
            {
                throw new KeyNotFoundException("The following columns (which have ANO Transforms on them) were missing from the DataTable:" + Environment.NewLine
                                               + string.Join(Environment.NewLine, missingColumns) + Environment.NewLine + "The columns found in the DataTable were:" + Environment.NewLine
                                               + string.Join(Environment.NewLine, toProcess.Columns.Cast <DataColumn>().Select(c => c.ColumnName)));
            }

            //Dump Identifiers
            stopwatch_TimeSpentDumping.Start();
            _dumper.DumpAllIdentifiersInTable(toProcess); //do the dumping of all the rest of the columns (those that must disapear from pipeline as opposed to those above which were substituted for ANO versions)
            stopwatch_TimeSpentDumping.Stop();

            if (_dumper.HaveDumpedRecords)
            {
                listener.OnProgress(this, new ProgressEventArgs("Dump Identifiers", new ProgressMeasurement(recordsProcessedSoFar, ProgressType.Records), stopwatch_TimeSpentDumping.Elapsed));//time taken to dump identifiers
            }
            //Process ANO Identifier Substitutions
            //for each column with an ANOTrasformer
            foreach (KeyValuePair <string, ANOTransformer> kvp in columnsToAnonymise)
            {
                didAno = true;

                var            column      = kvp.Key;
                ANOTransformer transformer = kvp.Value;

                //add an ANO version
                DataColumn ANOColumn = new DataColumn(ANOTable.ANOPrefix + column);
                toProcess.Columns.Add(ANOColumn);

                //populate ANO version
                transformer.Transform(toProcess, toProcess.Columns[column], ANOColumn);

                //drop the non ANO version
                toProcess.Columns.Remove(column);
            }

            stopwatch_TimeSpentTransforming.Stop();

            if (didAno)
            {
                listener.OnProgress(this, new ProgressEventArgs("Anonymise Identifiers", new ProgressMeasurement(recordsProcessedSoFar, ProgressType.Records), stopwatch_TimeSpentTransforming.Elapsed)); //time taken to swap ANO identifiers
            }
            return(toProcess);
        }
Пример #2
0
        private void MigrateExistingData(Func <string, bool> shouldApplySql, DbConnection con, ICheckNotifier notifier, DiscoveredTable tbl)
        {
            string from = _colToNuke.GetRuntimeName(LoadStage.PostLoad);
            string to   = _newANOColumnInfo.GetRuntimeName(LoadStage.PostLoad);


            //create an empty table for the anonymised data
            DbCommand cmdCreateTempMap = DatabaseCommandHelper.GetCommand(string.Format("SELECT top 0 {0},{1} into TempANOMap from {2}", from, to, tbl.GetFullyQualifiedName()), con);

            if (!shouldApplySql(cmdCreateTempMap.CommandText))
            {
                throw new Exception("User decided not to create the TempANOMap table");
            }

            cmdCreateTempMap.ExecuteNonQuery();
            try
            {
                //get the existing data
                DbCommand cmdGetExistingData = DatabaseCommandHelper.GetCommand(string.Format("SELECT {0},{1} from {2}", from, to, tbl.GetFullyQualifiedName()), con);

                DbDataAdapter da = DatabaseCommandHelper.GetDataAdapter(cmdGetExistingData);

                DataTable dt = new DataTable();
                da.Fill(dt);//into memory

                //transform it in memory
                ANOTransformer transformer = new ANOTransformer(_toConformTo, new FromCheckNotifierToDataLoadEventListener(notifier));
                transformer.Transform(dt, dt.Columns[0], dt.Columns[1]);

                var tempAnoMapTbl = tbl.Database.ExpectTable("TempANOMap");

                using (var insert = tempAnoMapTbl.BeginBulkInsert())
                {
                    insert.Upload(dt);
                }

                //create an empty table for the anonymised data
                DbCommand cmdUpdateMainTable = DatabaseCommandHelper.GetCommand(string.Format("UPDATE source set source.{1} = map.{1} from {2} source join TempANOMap map on source.{0}=map.{0}", from, to, tbl.GetFullyQualifiedName()), con);

                if (!shouldApplySql(cmdUpdateMainTable.CommandText))
                {
                    throw new Exception("User decided not to perform update on table");
                }
                cmdUpdateMainTable.ExecuteNonQuery();
            }
            finally
            {
                //always drop the temp anomap
                DbCommand dropMappingTable = DatabaseCommandHelper.GetCommand("DROP TABLE TempANOMap", con);
                dropMappingTable.ExecuteNonQuery();
            }
        }