public override bool distance(Embeddable struct1, Embeddable struct2, Key key) { BloomFilter bf1 = (BloomFilter)struct1; BloomFilter bf2 = (BloomFilter)struct2; int t = ((HammingKey)key).t; BitSet bs1 = bf1.getBitSet(); BitSet bs2 = bf2.getBitSet(); int d = distance(bs1, bs2); return(d <= t); }
public void setHashKeys(String id, Embeddable emb, String keyFieldName) { bool isKeyed = this.getConfiguration().isKeyed; String[] keyFieldNames = this.getConfiguration().keyFieldNames; IStoreEngine hashKeys = keys; if (isKeyed) { hashKeys = this.getKeyMap(keyFieldName); } Dictionary <String, Object> cache = this.getCacheMap(keyFieldName); Key key = this.getConfiguration().getKey(keyFieldName); for (int j = 0; j < key.L; j++) { String hashKey = buildHashKey(j, emb, keyFieldName); if (cache.ContainsKey(hashKey)) { List <String> arr = (List <String>)cache[hashKey]; arr.Add(id); if (arr.Count >= CACHEENTRYLIMIT) { long tt = incId(); hashKeys.set(hashKey + "_" + tt, arr); cache.Remove(hashKey); } } else { List <String> arr = new List <String>(); arr.Add(id); cache[hashKey] = arr; } if (cache.Count >= CACHENOLIMIT) { persistCache(keyFieldName); } } }
public override String buildHashKey(int j, Embeddable emb, String keyFieldName) { BloomFilter bf = (BloomFilter)emb; BitSet bs = bf.getBitSet(); String hashKey = ""; Key key = hConf.getKey(keyFieldName); for (int k = 0; k < key.k; k++) { int position = ((HammingKey)key).samples[j][k]; if (bs.get(position)) { hashKey = hashKey + "1"; } else { hashKey = hashKey + "0"; } } return("L" + j + "_" + hashKey); }
public void insert(Record rec) { if (this.getConfiguration().isPrivate) { Embeddable emb = (Embeddable)rec.get(Record.PRIVATE_STRUCTURE); data.set(rec.getId(), emb); setHashKeys(rec.getId(), emb, Configuration.RECORD_LEVEL); return; } bool isKeyed = this.getConfiguration().isKeyed; String[] keyFieldNames = this.getConfiguration().keyFieldNames; Dictionary <String, Embeddable[]> embMap = buildEmbeddableMap(rec); if (isKeyed) { for (int i = 0; i < keyFieldNames.Length; i++) { String keyFieldName = keyFieldNames[i]; Embeddable[] embs = embMap[keyFieldName]; for (int j = 0; j < embs.Length; j++) { Embeddable emb = embs[j]; setHashKeys(rec.getId() + Key.KEYFIELD + j, emb, keyFieldName); this.getDataMap(keyFieldName).set(rec.getId() + Key.KEYFIELD + j, emb); } } } else { data.set(rec.getId(), ((Embeddable[])embMap[Configuration.RECORD_LEVEL])[0]); setHashKeys(rec.getId(), ((Embeddable[])embMap[Configuration.RECORD_LEVEL])[0], Configuration.RECORD_LEVEL); } records.set(rec.getId(), rec); }
public HammingKey(String keyFieldName, int k, double delta, int t, bool tokenized, bool performComparisons, Embeddable emb) { this.keyFieldName = keyFieldName; this.k = k; this.delta = delta; this.t = t; this.size = emb.getSize(); optimizeL(); this.tokenized = tokenized; this.samples = new int[this.L][]; for (int i = 0; i < this.L; i++) { this.samples[i] = new int[this.k]; } initSamples(); this.performComparisons = performComparisons; this.emb = emb; Console.WriteLine("Number of hash tables generated L=" + this.L + " using k=" + this.k + " and size=" + this.size); }
public abstract bool distance(Embeddable struct1, Embeddable struct2, Key key);
/* * Opens a HammingLSH store * found in specified @target. * @throws StoreInitExcoetion */ // public static DataStore open(String storeName){ // // Config conf = new Config(Config.CONFIG_FILE); // // StoreConfigurationParams c = conf.get(Config.CONFIG_STORE, storeName); // try { // DataStore ds = this.dataStoreFactory.create(c.getTarget(), storeName, c.getLSHStore(), c.getEngine(), null, true); // return ds; // } catch (Exception ex) { // Console.Error.WriteLine("Initialization error of data store " + storeName, ex); // return null; // } // } public abstract String buildHashKey(int j, Embeddable structure, String keyFieldName);
public Dictionary <String, Embeddable[]> buildEmbeddableMap(Record rec) { Dictionary <String, Embeddable[]> embMap = new Dictionary <String, Embeddable[]>(); bool isKeyed = this.getConfiguration().isKeyed; String[] keyFieldNames = this.getConfiguration().keyFieldNames; List <String> fieldNames = rec.getFieldNames(); Embeddable embRec = null; if ((!isKeyed) && (this.getConfiguration().getKey(Configuration.RECORD_LEVEL) != null)) { embRec = this.getConfiguration().getKey(Configuration.RECORD_LEVEL).emb.freshCopy(); } for (int i = 0; i < fieldNames.Count; i++) { String fieldName = fieldNames[i]; bool isNotIndexedField = rec.isNotIndexedField(fieldName); String s = (String)rec.get(fieldName); if (isKeyed) { for (int j = 0; j < keyFieldNames.Length; j++) { String keyFieldName = keyFieldNames[j]; if (keyFieldName.Equals(fieldName)) { Key key = this.getConfiguration().getKey(keyFieldName); bool isTokenized = key.tokenized; if (!isTokenized) { Embeddable emb = key.emb.freshCopy(); emb.embed(s); embMap[keyFieldName] = new Embeddable[] { emb }; } else { String[] keyValues = (String[])rec.get(keyFieldName + Key.TOKENS); Embeddable[] bfs = new Embeddable[keyValues.Length]; for (int k = 0; k < bfs.Length; k++) { String v = keyValues[k]; Embeddable emb = key.emb.freshCopy(); emb.embed(v); bfs[k] = emb; } embMap[keyFieldName] = bfs; } } } } else if (!isNotIndexedField) { if (embRec != null) { embRec.embed(s); } else { Console.Error.WriteLine("Although no key fields are specified, a record-level embeddable is missing."); } } } if (!isKeyed) { embMap[Configuration.RECORD_LEVEL] = new Embeddable[] { embRec }; } return(embMap); }