Exemple #1
0
        public void SubstituteANOIdentifiers_PreviewWithoutPush()
        {
            var anoTable = GetANOTable();

            anoTable.NumberOfCharactersToUseInAnonymousRepresentation = 0;
            anoTable.NumberOfIntegersToUseInAnonymousRepresentation   = 10;

            DiscoveredTable ANOtable = ANOStore_Database.ExpectTable(anoTable.TableName);

            //should not exist yet
            Assert.False(ANOtable.Exists());

            DataTable dt = new DataTable();

            dt.Columns.Add("CHI");
            dt.Columns.Add("ANOCHI");
            dt.Rows.Add("0101010101", DBNull.Value);
            ANOTransformer transformer = new ANOTransformer(anoTable, new ThrowImmediatelyDataLoadEventListener());

            transformer.Transform(dt, dt.Columns["CHI"], dt.Columns["ANOCHI"], true);

            Assert.IsTrue(_anochiPattern.IsMatch((string)dt.Rows[0][1]));//should be 10 digits and then _A

            //still not exist yet
            Assert.False(ANOtable.Exists());

            anoTable.DeleteInDatabase();
        }
Exemple #2
0
        private void GeneratePreviews()
        {
            if (preview == null)
            {
                preview = new DataTable();
                preview.Columns.Add(_columnInfo.GetRuntimeName(LoadStage.PostLoad));
                preview.Columns.Add(ANOTable.ANOPrefix + _columnInfo.GetRuntimeName(LoadStage.PostLoad));

                var server = DataAccessPortal.GetInstance().ExpectServer(ColumnInfo.TableInfo, DataAccessContext.DataLoad);

                using (var con = server.GetConnection())
                {
                    con.Open();

                    lblPreviewDataIsFictional.Visible = false;

                    var qb = new QueryBuilder(null, null, new[] { ColumnInfo.TableInfo });
                    qb.AddColumn(new ColumnInfoToIColumn(new MemoryRepository(), _columnInfo));
                    qb.TopX = 10;

                    bool rowsRead = false;

                    using (DbCommand cmd = server.GetCommand(qb.SQL, con))
                    {
                        cmd.CommandTimeout = Convert.ToInt32(ntimeout.Value);
                        using (var r = cmd.ExecuteReader())
                        {
                            while (r.Read())
                            {
                                preview.Rows.Add(r[_columnInfo.GetRuntimeName(LoadStage.PostLoad)], DBNull.Value);
                                rowsRead = true;
                            }
                        }
                    }

                    if (!rowsRead)
                    {
                        lblPreviewDataIsFictional.Visible = true;
                        if (_columnInfo.GetRuntimeDataType(LoadStage.AdjustRaw).ToLower().Contains("char"))
                        {
                            preview.Rows.Add("?", DBNull.Value);
                            preview.Rows.Add("?", DBNull.Value);
                            preview.Rows.Add("?", DBNull.Value);
                            preview.Rows.Add("?", DBNull.Value);
                        }
                        else if (_columnInfo.GetRuntimeDataType(LoadStage.AdjustRaw).ToLower().Contains("date"))
                        {
                            preview.Rows.Add("1977-08-16", DBNull.Value);
                            preview.Rows.Add("1977-08-16", DBNull.Value);
                            preview.Rows.Add("1977-08-16", DBNull.Value);
                            preview.Rows.Add("1977-08-16", DBNull.Value);
                        }
                        else
                        {
                            preview.Rows.Add("-1", DBNull.Value);
                            preview.Rows.Add("-1", DBNull.Value);
                            preview.Rows.Add("-1", DBNull.Value);
                            preview.Rows.Add("-1", DBNull.Value);
                        }
                    }

                    con.Close();
                }
            }

            if (ANOTable != null)
            {
                try
                {
                    if (preview.Rows.Count != 0)
                    {
                        checksUI1.Clear();
                        ANOTransformer transformer = new ANOTransformer(ANOTable, new FromCheckNotifierToDataLoadEventListener(checksUI1));
                        transformer.Transform(preview, preview.Columns[0], preview.Columns[1], true);
                    }
                }
                catch (Exception e)
                {
                    checksUI1.OnCheckPerformed(new CheckEventArgs(e.Message, CheckResult.Fail, e));
                }
            }

            dgPreview.DataSource = preview;
        }
Exemple #3
0
        public void SubstituteANOIdentifiers_BulkTest()
        {
            const int batchSize = 10000;

            var anoTable = GetANOTable();

            anoTable.NumberOfCharactersToUseInAnonymousRepresentation = 0;
            anoTable.NumberOfIntegersToUseInAnonymousRepresentation   = 10;
            anoTable.PushToANOServerAsNewTable("varchar(10)", new ThrowImmediatelyCheckNotifier());


            Stopwatch sw = new Stopwatch();

            sw.Start();

            DataTable dt = new DataTable();

            dt.Columns.Add("CHI");
            dt.Columns.Add("ANOCHI");

            Random r = new Random();

            HashSet <string> uniqueSourceSet = new HashSet <string>();


            for (int i = 0; i < batchSize; i++)
            {
                var val = r.NextDouble() * 9999999999;
                val = Math.Round(val);
                string valAsString = val.ToString();

                while (valAsString.Length < 10)
                {
                    valAsString = "0" + valAsString;
                }

                if (!uniqueSourceSet.Contains(valAsString))
                {
                    uniqueSourceSet.Add(valAsString);
                }

                dt.Rows.Add(valAsString, DBNull.Value);//duplicates
            }
            Console.WriteLine("Time to allocate in C# memory:" + sw.Elapsed);
            Console.WriteLine("Allocated " + dt.Rows.Count + " identifiers (" + uniqueSourceSet.Count() + " unique ones)");

            sw.Reset();
            sw.Start();

            ANOTransformer transformer = new ANOTransformer(anoTable, new ThrowImmediatelyDataLoadEventListener());

            transformer.Transform(dt, dt.Columns["CHI"], dt.Columns["ANOCHI"]);
            Console.WriteLine("Time to perform SQL transform and allocation:" + sw.Elapsed);

            sw.Reset();
            sw.Start();
            HashSet <string> uniqueSet = new HashSet <string>();

            foreach (DataRow row in dt.Rows)
            {
                var ANOid = row["ANOCHI"].ToString();
                if (!uniqueSet.Contains(ANOid))
                {
                    uniqueSet.Add(ANOid);
                }

                Assert.IsTrue(_anochiPattern.IsMatch(ANOid));
            }

            Console.WriteLine("Allocated " + uniqueSet.Count + " anonymous identifiers");


            var server = ANOStore_Database.Server;

            using (var con = server.GetConnection())
            {
                con.Open();

                var cmd          = server.GetCommand("Select count(*) from ANOMyTable", con);
                int numberOfRows = Convert.ToInt32(cmd.ExecuteScalar());

                //should be the same number of unique identifiers in memory as in the database
                Assert.AreEqual(uniqueSet.Count, numberOfRows);
                Console.WriteLine("Found " + numberOfRows + " unique ones");

                var cmdNulls = server.GetCommand("select count(*) from ANOMyTable where ANOMyTable is null", con);
                int nulls    = Convert.ToInt32(cmdNulls.ExecuteScalar());
                Assert.AreEqual(0, nulls);
                Console.WriteLine("Found " + nulls + " null ANO identifiers");

                con.Close();
            }
            sw.Stop();
            Console.WriteLine("Time to evaluate results:" + sw.Elapsed);
            TruncateANOTable(anoTable);

            anoTable.DeleteInDatabase();
        }
Exemple #4
0
        public void SubstituteANOIdentifiers_2CHINumbers()
        {
            var anoTable = GetANOTable();

            anoTable.NumberOfCharactersToUseInAnonymousRepresentation = 0;
            anoTable.NumberOfIntegersToUseInAnonymousRepresentation   = 10;
            anoTable.PushToANOServerAsNewTable("varchar(10)", new ThrowImmediatelyCheckNotifier());


            DataTable dt = new DataTable();

            dt.Columns.Add("CHI");
            dt.Columns.Add("ANOCHI");

            dt.Rows.Add("0101010101", DBNull.Value); //duplicates
            dt.Rows.Add("0101010102", DBNull.Value);
            dt.Rows.Add("0101010101", DBNull.Value); //duplicates

            ANOTransformer transformer = new ANOTransformer(anoTable, new ThrowImmediatelyDataLoadEventListener());

            transformer.Transform(dt, dt.Columns["CHI"], dt.Columns["ANOCHI"]);

            Assert.IsTrue((string)dt.Rows[0][0] == "0101010101");
            Assert.IsTrue(_anochiPattern.IsMatch((string)dt.Rows[0][1])); //should be 10 digits and then _A
            Assert.AreEqual(dt.Rows[0][1], dt.Rows[2][1]);                //because of duplication these should both be the same

            Console.WriteLine("ANO identifiers created were:" + dt.Rows[0][1] + "," + dt.Rows[1][1]);

            TruncateANOTable(anoTable);

            //now test previews
            transformer.Transform(dt, dt.Columns["CHI"], dt.Columns["ANOCHI"], true);
            var val1 = dt.Rows[0][1];

            transformer.Transform(dt, dt.Columns["CHI"], dt.Columns["ANOCHI"], true);
            var val2 = dt.Rows[0][1];

            transformer.Transform(dt, dt.Columns["CHI"], dt.Columns["ANOCHI"], true);
            var val3 = dt.Rows[0][1];

            //should always be different
            Assert.AreNotEqual(val1, val2);
            Assert.AreNotEqual(val1, val3);

            //now test repeatability
            transformer.Transform(dt, dt.Columns["CHI"], dt.Columns["ANOCHI"], false);
            var val4 = dt.Rows[0][1];

            transformer.Transform(dt, dt.Columns["CHI"], dt.Columns["ANOCHI"], false);
            var val5 = dt.Rows[0][1];

            transformer.Transform(dt, dt.Columns["CHI"], dt.Columns["ANOCHI"], false);
            var val6 = dt.Rows[0][1];

            Assert.AreEqual(val4, val5);
            Assert.AreEqual(val4, val6);

            TruncateANOTable(anoTable);

            anoTable.DeleteInDatabase();
        }