public void testMerge() { BloomFilter bf = new BloomFilter(10000); string val = "bloo"; string val1 = "bloom fil"; string val2 = "bloom filter"; string val3 = "cuckoo filter"; bf.addString(val); bf.addString(val1); bf.addString(val2); bf.addString(val3); BloomFilter bf2 = new BloomFilter(10000); string v = "2_bloo"; string v1 = "2_bloom fil"; string v2 = "2_bloom filter"; string v3 = "2_cuckoo filter"; bf2.addString(v); bf2.addString(v1); bf2.addString(v2); bf2.addString(v3); Assert.Equal(true, bf.testString(val)); Assert.Equal(true, bf.testString(val1)); Assert.Equal(true, bf.testString(val2)); Assert.Equal(true, bf.testString(val3)); Assert.Equal(false, bf.testString(v)); Assert.Equal(false, bf.testString(v1)); Assert.Equal(false, bf.testString(v2)); Assert.Equal(false, bf.testString(v3)); bf.merge(bf2); Assert.Equal(true, bf.testString(val)); Assert.Equal(true, bf.testString(val1)); Assert.Equal(true, bf.testString(val2)); Assert.Equal(true, bf.testString(val3)); Assert.Equal(true, bf.testString(v)); Assert.Equal(true, bf.testString(v1)); Assert.Equal(true, bf.testString(v2)); Assert.Equal(true, bf.testString(v3)); }
public void testBloomFilterString() { BloomFilter bf = new BloomFilter(100000); string val = "bloo"; string val1 = "bloom fil"; string val2 = "bloom filter"; string val3 = "cuckoo filter"; Assert.Equal(false, bf.testString(val)); Assert.Equal(false, bf.testString(val1)); Assert.Equal(false, bf.testString(val2)); Assert.Equal(false, bf.testString(val3)); bf.addString(val); Assert.Equal(true, bf.testString(val)); Assert.Equal(false, bf.testString(val1)); Assert.Equal(false, bf.testString(val2)); Assert.Equal(false, bf.testString(val3)); bf.addString(val1); Assert.Equal(true, bf.testString(val)); Assert.Equal(true, bf.testString(val1)); Assert.Equal(false, bf.testString(val2)); Assert.Equal(false, bf.testString(val3)); bf.addString(val2); Assert.Equal(true, bf.testString(val)); Assert.Equal(true, bf.testString(val1)); Assert.Equal(true, bf.testString(val2)); Assert.Equal(false, bf.testString(val3)); bf.addString(val3); Assert.Equal(true, bf.testString(val)); Assert.Equal(true, bf.testString(val1)); Assert.Equal(true, bf.testString(val2)); Assert.Equal(true, bf.testString(val3)); long randVal = 0; for (int i = 0; i < COUNT; i++) { randVal = rand.NextLong(); bf.addString(randVal.ToString(CultureInfo.InvariantCulture)); } // last value should be present Assert.Equal(true, bf.testString(randVal.ToString(CultureInfo.InvariantCulture))); // most likely this value should not exist Assert.Equal(false, bf.testString((-120L).ToString(CultureInfo.InvariantCulture))); Assert.Equal(77944, bf.sizeInBytes()); }
public void testNullsInBloomFilter() { List<object> args = new List<object>(); args.Add(HiveDecimal.Parse("15")); args.Add(null); args.Add(HiveDecimal.Parse("19")); PredicateLeaf pred = TestSearchArgumentImpl.createPredicateLeaf (PredicateLeaf.Operator.IN, PredicateLeaf.Type.DECIMAL, "x", null, args); BloomFilter bf = new BloomFilter(10000); for (int i = 20; i < 1000; i++) { bf.addString(HiveDecimal.create(i).ToString()); } ColumnStatistics cs = ColumnStatisticsImpl.deserialize(createDecimalStats("10", "200", false)); // hasNull is false, so bloom filter should return NO Assert.Equal(TruthValue.NO, RecordReaderImpl.evaluatePredicate(cs, pred, bf)); cs = ColumnStatisticsImpl.deserialize(createDecimalStats("10", "200", true)); // hasNull is true, so bloom filter should return YES_NO_NULL Assert.Equal(TruthValue.YES_NO_NULL, RecordReaderImpl.evaluatePredicate(cs, pred, bf)); bf.addString(HiveDecimal.create(19).ToString()); Assert.Equal(TruthValue.YES_NO_NULL, RecordReaderImpl.evaluatePredicate(cs, pred, bf)); bf.addString(HiveDecimal.create(15).ToString()); Assert.Equal(TruthValue.YES_NO_NULL, RecordReaderImpl.evaluatePredicate(cs, pred, bf)); }
public void testDecimalNullSafeEqualsBloomFilter() { PredicateLeaf pred = TestSearchArgumentImpl.createPredicateLeaf( PredicateLeaf.Operator.NULL_SAFE_EQUALS, PredicateLeaf.Type.DECIMAL, "x", HiveDecimal.Parse("15"), null); BloomFilter bf = new BloomFilter(10000); for (int i = 20; i < 1000; i++) { bf.addString(HiveDecimal.create(i).ToString()); } ColumnStatistics cs = ColumnStatisticsImpl.deserialize(createDecimalStats("10", "200")); Assert.Equal(TruthValue.NO, RecordReaderImpl.evaluatePredicate(cs, pred, bf)); bf.addString(HiveDecimal.create(15).ToString()); Assert.Equal(TruthValue.YES_NO, RecordReaderImpl.evaluatePredicate(cs, pred, bf)); }
public void testStringNullSafeEqualsBloomFilter() { PredicateLeaf pred = TestSearchArgumentImpl.createPredicateLeaf( PredicateLeaf.Operator.NULL_SAFE_EQUALS, PredicateLeaf.Type.STRING, "x", "str_15", null); BloomFilter bf = new BloomFilter(10000); for (int i = 20; i < 1000; i++) { bf.addString("str_" + i); } ColumnStatistics cs = ColumnStatisticsImpl.deserialize(createStringStats("str_10", "str_200")); Assert.Equal(TruthValue.NO, RecordReaderImpl.evaluatePredicate(cs, pred, bf)); bf.addString("str_15"); Assert.Equal(TruthValue.YES_NO, RecordReaderImpl.evaluatePredicate(cs, pred, bf)); }
public void testStringInBloomFilter() { List<object> args = new List<object>(); args.Add("str_15"); args.Add("str_19"); PredicateLeaf pred = TestSearchArgumentImpl.createPredicateLeaf (PredicateLeaf.Operator.IN, PredicateLeaf.Type.STRING, "x", null, args); BloomFilter bf = new BloomFilter(10000); for (int i = 20; i < 1000; i++) { bf.addString("str_" + i); } ColumnStatistics cs = ColumnStatisticsImpl.deserialize(createStringStats("str_10", "str_200")); Assert.Equal(TruthValue.NO_NULL, RecordReaderImpl.evaluatePredicate(cs, pred, bf)); bf.addString("str_19"); Assert.Equal(TruthValue.YES_NO_NULL, RecordReaderImpl.evaluatePredicate(cs, pred, bf)); bf.addString("str_15"); Assert.Equal(TruthValue.YES_NO_NULL, RecordReaderImpl.evaluatePredicate(cs, pred, bf)); }