public void DumpAllIdentifiersInTable_Passes()
        {
            var preDiscardedColumn1 = new PreLoadDiscardedColumn(CatalogueRepository, tableInfoCreated, "surname")
            {
                Destination = DiscardedColumnDestination.StoreInIdentifiersDump,
                SqlDataType = "varchar(20)"
            };

            preDiscardedColumn1.SaveToDatabase();

            //give it the correct server
            tableInfoCreated.IdentifierDumpServer_ID = IdentifierDump_ExternalDatabaseServer.ID;
            tableInfoCreated.SaveToDatabase();

            IdentifierDumper dumper = new IdentifierDumper(tableInfoCreated);

            var chiToSurnameDictionary = new Dictionary <string, HashSet <string> >();

            try
            {
                dumper.Check(new AcceptAllCheckNotifier());

                DataTable dt = _bulkData.GetDataTable(1000);

                Assert.AreEqual(1000, dt.Rows.Count);
                Assert.IsTrue(dt.Columns.Contains("surname"));

                //for checking the final ID table has the correct values in
                foreach (DataRow row in dt.Rows)
                {
                    var chi = row["chi"].ToString();

                    if (!chiToSurnameDictionary.ContainsKey(chi))
                    {
                        chiToSurnameDictionary.Add(chi, new HashSet <string>());
                    }

                    chiToSurnameDictionary[chi].Add(row["surname"] as string);
                }

                dumper.CreateSTAGINGTable();
                dumper.DumpAllIdentifiersInTable(dt);
                dumper.DropStaging();

                //confirm that the surname column is no longer in the pipeline
                Assert.IsFalse(dt.Columns.Contains("surname"));

                //now look at the ids in the identifier dump and make sure they match what was in the pipeline before we sent it
                var server = IdentifierDump_Database.Server;
                using (var con = server.GetConnection())
                {
                    con.Open();

                    var cmd = server.GetCommand("Select * from " + "ID_" + BulkTestsData.BulkDataTable, con);
                    var r   = cmd.ExecuteReader();

                    //make sure the values in the ID table match the ones we originally had in the pipeline
                    while (r.Read())
                    {
                        if (!chiToSurnameDictionary[r["chi"].ToString()].Any())
                        {
                            Assert.IsTrue(r["surname"] == DBNull.Value);
                        }
                        else
                        {
                            Assert.IsTrue(chiToSurnameDictionary[r["chi"].ToString()].Contains(r["surname"] as string), "Dictionary did not contain expected surname:" + r["surname"]);
                        }
                    }
                    r.Close();

                    //leave the identifier dump in the way we found it (empty)
                    var tbl = IdentifierDump_Database.ExpectTable("ID_" + BulkTestsData.BulkDataTable);

                    if (tbl.Exists())
                    {
                        tbl.Drop();
                    }

                    tbl = IdentifierDump_Database.ExpectTable("ID_" + BulkTestsData.BulkDataTable + "_Archive");

                    if (tbl.Exists())
                    {
                        tbl.Drop();
                    }
                }
            }
            finally
            {
                preDiscardedColumn1.DeleteInDatabase();
                tableInfoCreated.IdentifierDumpServer_ID = null;//reset it back to how it was when we found it
                tableInfoCreated.SaveToDatabase();
            }
        }