示例#1
0
        static void TestBlum(string path)
        {
            var buf = File.ReadAllBytes(path);

            var tests = LoadTestFiles(spath_tests_folder);

            // load
            int maxStringLen = BitConverter.ToInt32(DataPack.Unpack(buf, 0), 0);
            //int count = BitConverter.ToInt32(DataPack.Unpack(buf, 0),sizeof(int));
            int filterBufLen = BitConverter.ToInt32(DataPack.Unpack(buf, 0), sizeof(int));

            byte[] arr = DataPack.Unpack(buf, 1);

            var excs2 = SplitString(ASCIIEncoding.ASCII.GetString(DataPack.Unpack(buf, 2)));
            var ends  = SplitString(ASCIIEncoding.ASCII.GetString(DataPack.Unpack(buf, 3)));
            var d2b   = SplitString(ASCIIEncoding.ASCII.GetString(DataPack.Unpack(buf, 4)));

            var bf = new BlumFilter(filterBufLen);

            bf.SetArray(arr);

            var flt = new PreFilter(excs2, ends, d2b, maxStringLen);

            int errcount = 0;

            foreach (var t in tests)
            {
                bool   isWord = false;
                var    s      = t.Key.ToLower().Trim();
                string sb;

                var sf = flt.WordStatus(s, out sb);
                if (sf == 0)
                {
                    isWord = false;
                }
                else if (sf == 1)
                {
                    isWord = true;
                }
                else
                {
                    isWord = bf.Contains(sb);
                }

                if (isWord && !t.Value)
                {
                    errcount++;
                }
                if (!isWord && t.Value)
                {
                    errcount++;
                }
            }

            Console.WriteLine("Errors:{0} from {1}, {2}",
                              errcount, tests.Count, (float)errcount / (float)(tests.Count) * 100.0F);
        }
示例#2
0
        static void Main(string[] args)
        {
            // loadTests(spath_tests_folder);
            // CombineTests(spath_tests_folder, spath_tests_2);

            int    filterBufLen = 506464;
            int    maxStringLen = 12;
            int    endingsCount = 23;
            double err          = 0.04;

            var sz = LoadAndOrderAndSave(spath0, spath2, spathe, maxStringLen);

            SaveAvailable2Beg(spath2, spath2e, spath3, err);

            TreeAnalyze(spath3, spath_ends, maxStringLen, filterBufLen, endingsCount);

            var buf = File.ReadAllLines(spath3);

            var excs2 = File.ReadAllLines(spathe).ToList();
            var ends  = File.ReadAllLines(spath_ends).ToList();
            var d2b   = File.ReadAllLines(spath2e).ToList();

            Console.WriteLine("d2b:{0}", d2b.Count);

            var flt = new PreFilter(excs2, ends, d2b, maxStringLen);

            Dictionary <string, bool> dicf = new Dictionary <string, bool>();

            List <string> buf2 = new List <string>();

            Console.WriteLine("Words before filtering:{0}", buf.Length);
            // filter buffer
            foreach (var sb in buf)
            {
                string sbase;
                if (flt.WordStatus(sb, out sbase) == 2)
                {
                    dicf[sbase] = true;
                }
            }

            buf2 = dicf.Keys.ToList();
            buf2.Sort();

            Console.WriteLine("Words after {0} level filtering:{1}", ends.Count, buf2.Count);

            var bf = new BlumFilter(filterBufLen);

            bf.AddRange(buf2);

            if (File.Exists(spath_filter))
            {
                File.Delete(spath_filter);
            }

            File.WriteAllBytes(spath_filter, bf.GetArray());

            // pack data
            DataPack.Pack(maxStringLen, buf2.Count, filterBufLen, spath_data, spath_filter, spathe, spath_ends, spath2e);

            TestBlum(spath_data);
            var cs = CalcSources();

            Console.WriteLine("Packet size:{0} Kb, overflow:{1} bytes", cs / 1024.0, cs - 64 * 1024);
            Console.ReadLine();

            return;
        }