static void TestBlum(string path)
{
var buf = File.ReadAllBytes(path);
var tests = LoadTestFiles(spath_tests_folder);
// load
int maxStringLen = BitConverter.ToInt32(DataPack.Unpack(buf, 0), 0);
//int count = BitConverter.ToInt32(DataPack.Unpack(buf, 0),sizeof(int));
int filterBufLen = BitConverter.ToInt32(DataPack.Unpack(buf, 0), sizeof(int));
byte[] arr = DataPack.Unpack(buf, 1);
var excs2 = SplitString(ASCIIEncoding.ASCII.GetString(DataPack.Unpack(buf, 2)));
var ends = SplitString(ASCIIEncoding.ASCII.GetString(DataPack.Unpack(buf, 3)));
var d2b = SplitString(ASCIIEncoding.ASCII.GetString(DataPack.Unpack(buf, 4)));
var bf = new BlumFilter(filterBufLen);
bf.SetArray(arr);
var flt = new PreFilter(excs2, ends, d2b, maxStringLen);
int errcount = 0;
foreach (var t in tests)
{
bool isWord = false;
var s = t.Key.ToLower().Trim();
string sb;
var sf = flt.WordStatus(s, out sb);
if (sf == 0)
{
isWord = false;
}
else if (sf == 1)
{
isWord = true;
}
else
{
isWord = bf.Contains(sb);
}
if (isWord && !t.Value)
{
errcount++;
}
if (!isWord && t.Value)
{
errcount++;
}
}
Console.WriteLine("Errors:{0} from {1}, {2}",
errcount, tests.Count, (float)errcount / (float)(tests.Count) * 100.0F);
}
static void Main(string[] args)
{
// loadTests(spath_tests_folder);
// CombineTests(spath_tests_folder, spath_tests_2);
int filterBufLen = 506464;
int maxStringLen = 12;
int endingsCount = 23;
double err = 0.04;
var sz = LoadAndOrderAndSave(spath0, spath2, spathe, maxStringLen);
SaveAvailable2Beg(spath2, spath2e, spath3, err);
TreeAnalyze(spath3, spath_ends, maxStringLen, filterBufLen, endingsCount);
var buf = File.ReadAllLines(spath3);
var excs2 = File.ReadAllLines(spathe).ToList();
var ends = File.ReadAllLines(spath_ends).ToList();
var d2b = File.ReadAllLines(spath2e).ToList();
Console.WriteLine("d2b:{0}", d2b.Count);
var flt = new PreFilter(excs2, ends, d2b, maxStringLen);
Dictionary <string, bool> dicf = new Dictionary <string, bool>();
List <string> buf2 = new List <string>();
Console.WriteLine("Words before filtering:{0}", buf.Length);
// filter buffer
foreach (var sb in buf)
{
string sbase;
if (flt.WordStatus(sb, out sbase) == 2)
{
dicf[sbase] = true;
}
}
buf2 = dicf.Keys.ToList();
buf2.Sort();
Console.WriteLine("Words after {0} level filtering:{1}", ends.Count, buf2.Count);
var bf = new BlumFilter(filterBufLen);
bf.AddRange(buf2);
if (File.Exists(spath_filter))
{
File.Delete(spath_filter);
}
File.WriteAllBytes(spath_filter, bf.GetArray());
// pack data
DataPack.Pack(maxStringLen, buf2.Count, filterBufLen, spath_data, spath_filter, spathe, spath_ends, spath2e);
TestBlum(spath_data);
var cs = CalcSources();
Console.WriteLine("Packet size:{0} Kb, overflow:{1} bytes", cs / 1024.0, cs - 64 * 1024);
Console.ReadLine();
return;
}
