public void SubstituteANOIdentifiers_PreviewWithoutPush()
{
var anoTable = GetANOTable();
anoTable.NumberOfCharactersToUseInAnonymousRepresentation = 0;
anoTable.NumberOfIntegersToUseInAnonymousRepresentation = 10;
DiscoveredTable ANOtable = ANOStore_Database.ExpectTable(anoTable.TableName);
//should not exist yet
Assert.False(ANOtable.Exists());
DataTable dt = new DataTable();
dt.Columns.Add("CHI");
dt.Columns.Add("ANOCHI");
dt.Rows.Add("0101010101", DBNull.Value);
ANOTransformer transformer = new ANOTransformer(anoTable, new ThrowImmediatelyDataLoadEventListener());
transformer.Transform(dt, dt.Columns["CHI"], dt.Columns["ANOCHI"], true);
Assert.IsTrue(_anochiPattern.IsMatch((string)dt.Rows[0][1]));//should be 10 digits and then _A
//still not exist yet
Assert.False(ANOtable.Exists());
anoTable.DeleteInDatabase();
}
private void ddExternalDatabaseServer_SelectedIndexChanged(object sender, EventArgs e)
{
var server = ddExternalDatabaseServer.SelectedItem as ExternalDatabaseServer;
if (server == null)
{
return;
}
ANOTransformer.ConfirmDependencies(DataAccessPortal.GetInstance().ExpectDatabase(server, DataAccessContext.DataLoad), checksUI1);
}
private void GeneratePreviews()
{
if (preview == null)
{
preview = new DataTable();
preview.Columns.Add(_columnInfo.GetRuntimeName(LoadStage.PostLoad));
preview.Columns.Add(ANOTable.ANOPrefix + _columnInfo.GetRuntimeName(LoadStage.PostLoad));
var server = DataAccessPortal.GetInstance().ExpectServer(ColumnInfo.TableInfo, DataAccessContext.DataLoad);
using (var con = server.GetConnection())
{
con.Open();
lblPreviewDataIsFictional.Visible = false;
var qb = new QueryBuilder(null, null, new[] { ColumnInfo.TableInfo });
qb.AddColumn(new ColumnInfoToIColumn(new MemoryRepository(), _columnInfo));
qb.TopX = 10;
bool rowsRead = false;
using (DbCommand cmd = server.GetCommand(qb.SQL, con))
{
cmd.CommandTimeout = Convert.ToInt32(ntimeout.Value);
using (var r = cmd.ExecuteReader())
{
while (r.Read())
{
preview.Rows.Add(r[_columnInfo.GetRuntimeName(LoadStage.PostLoad)], DBNull.Value);
rowsRead = true;
}
}
}
if (!rowsRead)
{
lblPreviewDataIsFictional.Visible = true;
if (_columnInfo.GetRuntimeDataType(LoadStage.AdjustRaw).ToLower().Contains("char"))
{
preview.Rows.Add("?", DBNull.Value);
preview.Rows.Add("?", DBNull.Value);
preview.Rows.Add("?", DBNull.Value);
preview.Rows.Add("?", DBNull.Value);
}
else if (_columnInfo.GetRuntimeDataType(LoadStage.AdjustRaw).ToLower().Contains("date"))
{
preview.Rows.Add("1977-08-16", DBNull.Value);
preview.Rows.Add("1977-08-16", DBNull.Value);
preview.Rows.Add("1977-08-16", DBNull.Value);
preview.Rows.Add("1977-08-16", DBNull.Value);
}
else
{
preview.Rows.Add("-1", DBNull.Value);
preview.Rows.Add("-1", DBNull.Value);
preview.Rows.Add("-1", DBNull.Value);
preview.Rows.Add("-1", DBNull.Value);
}
}
con.Close();
}
}
if (ANOTable != null)
{
try
{
if (preview.Rows.Count != 0)
{
checksUI1.Clear();
ANOTransformer transformer = new ANOTransformer(ANOTable, new FromCheckNotifierToDataLoadEventListener(checksUI1));
transformer.Transform(preview, preview.Columns[0], preview.Columns[1], true);
}
}
catch (Exception e)
{
checksUI1.OnCheckPerformed(new CheckEventArgs(e.Message, CheckResult.Fail, e));
}
}
dgPreview.DataSource = preview;
}
public void SubstituteANOIdentifiers_BulkTest()
{
const int batchSize = 10000;
var anoTable = GetANOTable();
anoTable.NumberOfCharactersToUseInAnonymousRepresentation = 0;
anoTable.NumberOfIntegersToUseInAnonymousRepresentation = 10;
anoTable.PushToANOServerAsNewTable("varchar(10)", new ThrowImmediatelyCheckNotifier());
Stopwatch sw = new Stopwatch();
sw.Start();
DataTable dt = new DataTable();
dt.Columns.Add("CHI");
dt.Columns.Add("ANOCHI");
Random r = new Random();
HashSet <string> uniqueSourceSet = new HashSet <string>();
for (int i = 0; i < batchSize; i++)
{
var val = r.NextDouble() * 9999999999;
val = Math.Round(val);
string valAsString = val.ToString();
while (valAsString.Length < 10)
{
valAsString = "0" + valAsString;
}
if (!uniqueSourceSet.Contains(valAsString))
{
uniqueSourceSet.Add(valAsString);
}
dt.Rows.Add(valAsString, DBNull.Value);//duplicates
}
Console.WriteLine("Time to allocate in C# memory:" + sw.Elapsed);
Console.WriteLine("Allocated " + dt.Rows.Count + " identifiers (" + uniqueSourceSet.Count() + " unique ones)");
sw.Reset();
sw.Start();
ANOTransformer transformer = new ANOTransformer(anoTable, new ThrowImmediatelyDataLoadEventListener());
transformer.Transform(dt, dt.Columns["CHI"], dt.Columns["ANOCHI"]);
Console.WriteLine("Time to perform SQL transform and allocation:" + sw.Elapsed);
sw.Reset();
sw.Start();
HashSet <string> uniqueSet = new HashSet <string>();
foreach (DataRow row in dt.Rows)
{
var ANOid = row["ANOCHI"].ToString();
if (!uniqueSet.Contains(ANOid))
{
uniqueSet.Add(ANOid);
}
Assert.IsTrue(_anochiPattern.IsMatch(ANOid));
}
Console.WriteLine("Allocated " + uniqueSet.Count + " anonymous identifiers");
var server = ANOStore_Database.Server;
using (var con = server.GetConnection())
{
con.Open();
var cmd = server.GetCommand("Select count(*) from ANOMyTable", con);
int numberOfRows = Convert.ToInt32(cmd.ExecuteScalar());
//should be the same number of unique identifiers in memory as in the database
Assert.AreEqual(uniqueSet.Count, numberOfRows);
Console.WriteLine("Found " + numberOfRows + " unique ones");
var cmdNulls = server.GetCommand("select count(*) from ANOMyTable where ANOMyTable is null", con);
int nulls = Convert.ToInt32(cmdNulls.ExecuteScalar());
Assert.AreEqual(0, nulls);
Console.WriteLine("Found " + nulls + " null ANO identifiers");
con.Close();
}
sw.Stop();
Console.WriteLine("Time to evaluate results:" + sw.Elapsed);
TruncateANOTable(anoTable);
anoTable.DeleteInDatabase();
}
public void SubstituteANOIdentifiers_2CHINumbers()
{
var anoTable = GetANOTable();
anoTable.NumberOfCharactersToUseInAnonymousRepresentation = 0;
anoTable.NumberOfIntegersToUseInAnonymousRepresentation = 10;
anoTable.PushToANOServerAsNewTable("varchar(10)", new ThrowImmediatelyCheckNotifier());
DataTable dt = new DataTable();
dt.Columns.Add("CHI");
dt.Columns.Add("ANOCHI");
dt.Rows.Add("0101010101", DBNull.Value); //duplicates
dt.Rows.Add("0101010102", DBNull.Value);
dt.Rows.Add("0101010101", DBNull.Value); //duplicates
ANOTransformer transformer = new ANOTransformer(anoTable, new ThrowImmediatelyDataLoadEventListener());
transformer.Transform(dt, dt.Columns["CHI"], dt.Columns["ANOCHI"]);
Assert.IsTrue((string)dt.Rows[0][0] == "0101010101");
Assert.IsTrue(_anochiPattern.IsMatch((string)dt.Rows[0][1])); //should be 10 digits and then _A
Assert.AreEqual(dt.Rows[0][1], dt.Rows[2][1]); //because of duplication these should both be the same
Console.WriteLine("ANO identifiers created were:" + dt.Rows[0][1] + "," + dt.Rows[1][1]);
TruncateANOTable(anoTable);
//now test previews
transformer.Transform(dt, dt.Columns["CHI"], dt.Columns["ANOCHI"], true);
var val1 = dt.Rows[0][1];
transformer.Transform(dt, dt.Columns["CHI"], dt.Columns["ANOCHI"], true);
var val2 = dt.Rows[0][1];
transformer.Transform(dt, dt.Columns["CHI"], dt.Columns["ANOCHI"], true);
var val3 = dt.Rows[0][1];
//should always be different
Assert.AreNotEqual(val1, val2);
Assert.AreNotEqual(val1, val3);
//now test repeatability
transformer.Transform(dt, dt.Columns["CHI"], dt.Columns["ANOCHI"], false);
var val4 = dt.Rows[0][1];
transformer.Transform(dt, dt.Columns["CHI"], dt.Columns["ANOCHI"], false);
var val5 = dt.Rows[0][1];
transformer.Transform(dt, dt.Columns["CHI"], dt.Columns["ANOCHI"], false);
var val6 = dt.Rows[0][1];
Assert.AreEqual(val4, val5);
Assert.AreEqual(val4, val6);
TruncateANOTable(anoTable);
anoTable.DeleteInDatabase();
}