public IEnumerable <int> screenSim(SqlConnection conn, byte[] fp, int?next_after_storate_id, getBoundsDelegate boundsDelegate) { screenInBlockDelegate screenBlockSim = new screenInBlockDelegate( (List <int> fp_ones, Block block, byte[] chunk, byte[] chunk2, SqlConnection conn2) => _screenInBlockSim(fp_ones, block, chunk, chunk2, conn2, boundsDelegate, next_after_storate_id)); return(_screen(conn, fp, next_after_storate_id, screenBlockSim)); }
private List<int> _screenInBlockSim (List<int> fp_ones, Block block, byte[] chunk, byte[] chunk2, SqlConnection conn, getBoundsDelegate boundsDelegate, int? next_after_storate_id) { List<int> passed_screening = new List<int>(); if (next_after_storate_id.HasValue && block.maximum_index < next_after_storate_id.Value) return passed_screening; int min_storate_id_bound = -1; if (next_after_storate_id.HasValue) min_storate_id_bound = next_after_storate_id.Value; BingoTimer timer = new BingoTimer("fingerprints.screening_sim"); int[] max_common_ones = new int[block.indices.Count]; int[] min_common_ones = new int[block.indices.Count]; int[] one_counters = new int[block.indices.Count]; // Calculate max and min bounds BingoTimer timer2 = new BingoTimer("fingerprints.screening_bounds"); boundsDelegate(block.indices, ref min_common_ones, ref max_common_ones, conn); timer2.end(); List<int> passed_screening_tmp = new List<int>(); BingoCore.lib.bingoProfIncCounter("fingerprints.bits_total", fp_ones.Count); if (fp_ones.Count == 0) { for (int i = 0; i < block.indices.Count; i++) { if (block.indices[i] <= min_storate_id_bound) continue; if (min_common_ones[i] == 0) passed_screening.Add(i); } timer.end(); return passed_screening; } int iteration = 0; foreach (BitChunk bit_chunk in bitChunksReaderGrouped(conn, block, fp_ones, _fp_sim_bits_group)) { chunk = bit_chunk.chunk; BingoTimer timer3 = new BingoTimer("fingerprints.screening_one_counters"); // Calculate ones count int max_byte_index = (block.indices.Count + 7) / 8; for (int i = 0; i < max_byte_index; i++) { byte b = chunk[i]; if (b == 0) continue; for (int j = 0; j < 8; j++) if ((b & (1 << j)) != 0) one_counters[8 * i + j]++; } timer3.end(); BingoTimer timer4 = new BingoTimer("fingerprints.screening_process"); if (iteration == 0) { for (int i = 0; i < block.indices.Count; i++) { if (block.indices[i] <= min_storate_id_bound) continue; int min_possible_ones = one_counters[i]; int max_possible_ones = one_counters[i] + fp_ones.Count; if (min_possible_ones <= max_common_ones[i] && max_possible_ones >= min_common_ones[i]) passed_screening.Add(i); } } else { passed_screening_tmp.Clear(); foreach (int i in passed_screening) { int min_possible_ones = one_counters[i]; int max_possible_ones = one_counters[i] + fp_ones.Count - iteration; if (min_possible_ones <= max_common_ones[i] && max_possible_ones >= min_common_ones[i]) passed_screening_tmp.Add(i); } // Swap then List<int> tmp = passed_screening; passed_screening = passed_screening_tmp; passed_screening_tmp = tmp; } timer4.end(); iteration++; if (passed_screening.Count < _sim_screening_pass_mark) break; } BingoCore.lib.bingoProfIncCounter("fingerprints.bits_used", iteration); for (int i = 0; i < passed_screening.Count; i++) passed_screening[i] = block.indices[passed_screening[i]]; timer.end(); return passed_screening; }
public IEnumerable<int> screenSim (SqlConnection conn, byte[] fp, int? next_after_storate_id, getBoundsDelegate boundsDelegate) { screenInBlockDelegate screenBlockSim = new screenInBlockDelegate( (List<int> fp_ones, Block block, byte[] chunk, byte[] chunk2, SqlConnection conn2) => _screenInBlockSim(fp_ones, block, chunk, chunk2, conn2, boundsDelegate, next_after_storate_id)); return _screen(conn, fp, next_after_storate_id, screenBlockSim); }
private List <int> _screenInBlockSim(List <int> fp_ones, Block block, byte[] chunk, byte[] chunk2, SqlConnection conn, getBoundsDelegate boundsDelegate, int?next_after_storate_id) { List <int> passed_screening = new List <int>(); if (next_after_storate_id.HasValue && block.maximum_index < next_after_storate_id.Value) { return(passed_screening); } int min_storate_id_bound = -1; if (next_after_storate_id.HasValue) { min_storate_id_bound = next_after_storate_id.Value; } BingoTimer timer = new BingoTimer("fingerprints.screening_sim"); int[] max_common_ones = new int[block.indices.Count]; int[] min_common_ones = new int[block.indices.Count]; int[] one_counters = new int[block.indices.Count]; // Calculate max and min bounds BingoTimer timer2 = new BingoTimer("fingerprints.screening_bounds"); boundsDelegate(block.indices, ref min_common_ones, ref max_common_ones, conn); timer2.end(); List <int> passed_screening_tmp = new List <int>(); BingoCore.lib.bingoProfIncCounter("fingerprints.bits_total", fp_ones.Count); if (fp_ones.Count == 0) { for (int i = 0; i < block.indices.Count; i++) { if (block.indices[i] <= min_storate_id_bound) { continue; } if (min_common_ones[i] == 0) { passed_screening.Add(i); } } timer.end(); return(passed_screening); } int iteration = 0; foreach (BitChunk bit_chunk in bitChunksReaderGrouped(conn, block, fp_ones, _fp_sim_bits_group)) { chunk = bit_chunk.chunk; BingoTimer timer3 = new BingoTimer("fingerprints.screening_one_counters"); // Calculate ones count int max_byte_index = (block.indices.Count + 7) / 8; for (int i = 0; i < max_byte_index; i++) { byte b = chunk[i]; if (b == 0) { continue; } for (int j = 0; j < 8; j++) { if ((b & (1 << j)) != 0) { one_counters[8 * i + j]++; } } } timer3.end(); BingoTimer timer4 = new BingoTimer("fingerprints.screening_process"); if (iteration == 0) { for (int i = 0; i < block.indices.Count; i++) { if (block.indices[i] <= min_storate_id_bound) { continue; } int min_possible_ones = one_counters[i]; int max_possible_ones = one_counters[i] + fp_ones.Count; if (min_possible_ones <= max_common_ones[i] && max_possible_ones >= min_common_ones[i]) { passed_screening.Add(i); } } } else { passed_screening_tmp.Clear(); foreach (int i in passed_screening) { int min_possible_ones = one_counters[i]; int max_possible_ones = one_counters[i] + fp_ones.Count - iteration; if (min_possible_ones <= max_common_ones[i] && max_possible_ones >= min_common_ones[i]) { passed_screening_tmp.Add(i); } } // Swap then List <int> tmp = passed_screening; passed_screening = passed_screening_tmp; passed_screening_tmp = tmp; } timer4.end(); iteration++; if (passed_screening.Count < _sim_screening_pass_mark) { break; } } BingoCore.lib.bingoProfIncCounter("fingerprints.bits_used", iteration); for (int i = 0; i < passed_screening.Count; i++) { passed_screening[i] = block.indices[passed_screening[i]]; } timer.end(); return(passed_screening); }