private void simGetMinMaxBounds(IList <int> storage_ids, ref int[] min_common_ones, ref int[] max_common_ones, SqlConnection conn) { BingoTimer timer2 = new BingoTimer("fingerprints.screening_bounds_ones"); int cache_index = 0; int[] targets_ones = new int[storage_ids.Count]; for (int i = 0; i < storage_ids.Count; i++) { int storage_id = storage_ids[i]; targets_ones[i] = _index_data.storage.getShort(storage_id, 4, conn, ref cache_index); } timer2.end(); BingoTimer timer3 = new BingoTimer("fingerprints.screening_bounds_get"); IntPtr min_common_ones_ptr, max_common_ones_ptr; BingoCore.lib.mangoSimilarityGetBitMinMaxBoundsArray(targets_ones.Length, targets_ones, out min_common_ones_ptr, out max_common_ones_ptr); if (min_common_ones.Length != storage_ids.Count || max_common_ones.Length != storage_ids.Count) { throw new Exception("Internal error in simGetMinMaxBounds: array lengths mismatch"); } Marshal.Copy(min_common_ones_ptr, min_common_ones, 0, targets_ones.Length); Marshal.Copy(max_common_ones_ptr, max_common_ones, 0, targets_ones.Length); timer3.end(); }
private void simGetMinMaxBounds (IList<int> storage_ids, ref int[] min_common_ones, ref int[] max_common_ones, SqlConnection conn) { BingoTimer timer2 = new BingoTimer("fingerprints.screening_bounds_ones"); int cache_index = 0; int[] targets_ones = new int[storage_ids.Count]; for (int i = 0; i < storage_ids.Count; i++) { int storage_id = storage_ids[i]; targets_ones[i] = _index_data.storage.getShort(storage_id, 4, conn, ref cache_index); } timer2.end(); BingoTimer timer3 = new BingoTimer("fingerprints.screening_bounds_get"); IntPtr min_common_ones_ptr, max_common_ones_ptr; BingoCore.lib.mangoSimilarityGetBitMinMaxBoundsArray(targets_ones.Length, targets_ones, out min_common_ones_ptr, out max_common_ones_ptr); if (min_common_ones.Length != storage_ids.Count || max_common_ones.Length != storage_ids.Count) throw new Exception("Internal error in simGetMinMaxBounds: array lengths mismatch"); Marshal.Copy(min_common_ones_ptr, min_common_ones, 0, targets_ones.Length); Marshal.Copy(max_common_ones_ptr, max_common_ones, 0, targets_ones.Length); timer3.end(); }
private void _flushShadowTable(SqlConnection conn) { if (shadow_datatable == null || (shadow_datatable.Rows.Count == 0 && components_datatable.Rows.Count == 0)) { return; } if (conn.ConnectionString == "context connection=true") { // SqlBulkInsert cannot be used in the context connection _flushShadowTableInContext(conn); return; } BingoTimer timer = new BingoTimer("shadow_table.flush"); using (SqlTransaction transaction = conn.BeginTransaction()) { // Copy shadow table using (SqlBulkCopy bulkCopy = new SqlBulkCopy(conn, SqlBulkCopyOptions.TableLock, transaction)) { bulkCopy.DestinationTableName = shadowTable; foreach (DataColumn dc in shadow_datatable.Columns) { bulkCopy.ColumnMappings.Add(dc.ColumnName, dc.ColumnName); } bulkCopy.BatchSize = shadow_datatable.Rows.Count; bulkCopy.BulkCopyTimeout = 3600; bulkCopy.WriteToServer(shadow_datatable); } shadow_datatable.Rows.Clear(); // Copy components table using (SqlBulkCopy bulkCopy = new SqlBulkCopy(conn, SqlBulkCopyOptions.TableLock, transaction)) { bulkCopy.DestinationTableName = componentsTable; foreach (DataColumn dc in components_datatable.Columns) { bulkCopy.ColumnMappings.Add(dc.ColumnName, dc.ColumnName); } bulkCopy.BatchSize = components_datatable.Rows.Count; bulkCopy.BulkCopyTimeout = 3600; bulkCopy.WriteToServer(components_datatable); } components_datatable.Rows.Clear(); transaction.Commit(); } timer.end(); }
IEnumerable <BitChunk> bitChunksReaderGrouped(SqlConnection conn, Block block, List <int> bits, int size) { int offset = 0; while (offset < bits.Count) { int cur_size = size; if (offset + cur_size >= bits.Count) { cur_size = bits.Count - offset; } StringBuilder indices = new StringBuilder(); for (int i = offset; i < offset + cur_size; i++) { if (indices.Length != 0) { indices.Append(", "); } indices.AppendFormat("{0}", bits[i]); } string command_text = String.Format("select bit, bits_chunk from {0} where part={1} and bit in ({2})", _index_data.fingerprintBitsTable, block.part, indices.ToString()); using (SqlCommand command = new SqlCommand(command_text, conn)) { command.CommandTimeout = 3600; BingoTimer timer = new BingoTimer("fingerprints.read_grouped_exec"); using (SqlDataReader reader = command.ExecuteReader()) { timer.end(); while (true) { timer = new BingoTimer("fingerprints.read_grouped_read"); bool ret = reader.Read(); if (!ret) { break; } BitChunk bit_chunk = new BitChunk(); int bit = (int)reader[0]; bit_chunk.bit_index = bit; bit_chunk.chunk = (byte[])reader[1]; timer.end(); yield return(bit_chunk); } } } offset += size; } }
private void _getBitChunk(Block block, int bit_index, SqlConnection conn, ref byte[] chunk) { BingoTimer timer = new BingoTimer("fingerprints.read"); if (block.bits != null) { Buffer.BlockCopy(block.bits[bit_index], 0, chunk, 0, chunk.Length); } else { chunk = (byte[])BingoSqlUtils.ExecObjQuery(conn, "SELECT bits_chunk from {0} where part = {1} and bit = {2}", _index_data.fingerprintBitsTable, block.part, bit_index); } timer.end(); }
private void _flushShadowTable (SqlConnection conn) { if (shadow_datatable == null || (shadow_datatable.Rows.Count == 0 && components_datatable.Rows.Count == 0)) return; if (conn.ConnectionString == "context connection=true") { // SqlBulkInsert cannot be used in the context connection _flushShadowTableInContext(conn); return; } BingoTimer timer = new BingoTimer("shadow_table.flush"); using (SqlTransaction transaction = conn.BeginTransaction()) { // Copy shadow table using (SqlBulkCopy bulkCopy = new SqlBulkCopy(conn, SqlBulkCopyOptions.TableLock, transaction)) { bulkCopy.DestinationTableName = shadowTable; foreach (DataColumn dc in shadow_datatable.Columns) bulkCopy.ColumnMappings.Add(dc.ColumnName, dc.ColumnName); bulkCopy.BatchSize = shadow_datatable.Rows.Count; bulkCopy.BulkCopyTimeout = 3600; bulkCopy.WriteToServer(shadow_datatable); } shadow_datatable.Rows.Clear(); // Copy components table using (SqlBulkCopy bulkCopy = new SqlBulkCopy(conn, SqlBulkCopyOptions.TableLock, transaction)) { bulkCopy.DestinationTableName = componentsTable; foreach (DataColumn dc in components_datatable.Columns) bulkCopy.ColumnMappings.Add(dc.ColumnName, dc.ColumnName); bulkCopy.BatchSize = components_datatable.Rows.Count; bulkCopy.BulkCopyTimeout = 3600; bulkCopy.WriteToServer(components_datatable); } components_datatable.Rows.Clear(); transaction.Commit(); } timer.end(); }
private void _validateBlock(short block_index, SqlConnection conn) { if (_blockLoaded(block_index)) { return; } lock (_sync_object) { // Double-checked locking if (_blockLoaded(block_index)) { return; } BingoTimer timer = new BingoTimer("storage.validate_block"); BingoLog.logMessage("Loading storage block {0} for table {1}...", block_index, _index_data.id.InformationName()); string text = "SELECT [data], [first_index], [offsets], [lengths], [count] from " + _index_data.storageTable + " where id = " + block_index; using (SqlCommand command = new SqlCommand(text, conn)) { command.CommandTimeout = 3600; using (SqlDataReader reader = command.ExecuteReader()) { if (!reader.Read()) { throw new Exception("Block cannot be found"); } while (_blocks.Count <= block_index) { _blocks.Add(new _Block()); } if (_blocks[block_index] == null) { _blocks[block_index] = new _Block(); } _Block block = _blocks[block_index]; block.block_index = block_index; block.data = (byte[])reader["data"]; block.first_index = (int)reader["first_index"]; int count = (int)reader["count"]; block.offsets = new int[count]; MemoryStream mem_stream = new MemoryStream((byte[])reader["offsets"]); BinaryReader bin_reader = new BinaryReader(mem_stream); for (int i = 0; i < count; i++) { block.offsets[i] = bin_reader.ReadInt32(); } block.lengths = new short[count]; mem_stream = new MemoryStream((byte[])reader["lengths"]); bin_reader = new BinaryReader(mem_stream); for (int i = 0; i < count; i++) { block.lengths[i] = bin_reader.ReadInt16(); } block.end_index = block.first_index + count; } } BingoLog.logMessage(" Done."); timer.end(); } }
// Method for executing abstract operation with BingoIndexData private static void _ExecuteBingoOperation(SqlString bingo_schema, bingoOperationDelegate operationDelegate, bingoGetIndexDataDelegate getBingoDataDelegate, BingoOp op_flags) { string table_name = "<Undef>"; BingoIndexID id = null; SqlConnection ext_conn = null; try { using (SqlConnection ctx_conn = new SqlConnection("context connection=true")) { ctx_conn.Open(); SqlConnection conn = ctx_conn; if ((op_flags & BingoOp.NON_CONTEXT_CONN) != 0) { string conn_string = String.Format( "server={0};integrated security=true;database={1};enlist=false", getServername(ctx_conn), ctx_conn.Database); ext_conn = new SqlConnection(conn_string); ext_conn.Open(); conn = ext_conn; } using (BingoSession session = new BingoSession()) { BingoCore.lib.bingoProfilingReset(false); BingoTimer timer = new BingoTimer("total"); BingoIndexData index_data = getBingoDataDelegate(ctx_conn, conn, bingo_schema); table_name = index_data.id.FullTableName(ctx_conn); id = index_data.id; if (index_data.locked) { BingoLog.logMessage("Attempt to get locked index for the table {0}", table_name); throw new Exception("Chemical index for the table '" + table_name + "' is locked"); } if ((op_flags & BingoOp.LOCK_INDEX) != 0) index_data.locked = true; ContextFlags flags = ContextFlags.NTHREADS | ContextFlags.FINGERPRINTS; if ((op_flags & BingoOp.LOAD_TAU_RULES) != 0) flags |= ContextFlags.TAU_RULES; if ((op_flags & BingoOp.LOAD_CMF) != 0) flags |= ContextFlags.CMF; prepareContext(ctx_conn, bingo_schema.Value, index_data.id.table_id, flags); index_data.syncContextParameters(ctx_conn, bingo_schema.Value); try { operationDelegate(ctx_conn, conn, index_data); } catch (Exception ex) { if ((Thread.CurrentThread.ThreadState & ThreadState.AbortRequested) != 0) Thread.ResetAbort(); BingoLog.logMessage("Exception {0} in {1}:\n{2}", ex.Message, ex.Source, ex.StackTrace); if ((op_flags & BingoOp.LOCK_INDEX) != 0) index_data.locked = false; if ((op_flags & BingoOp.DROP_ON_EXCEPTION) != 0) BingoIndexData.DropIndexData(conn, bingo_schema.Value, id, false); throw ex; } if ((op_flags & BingoOp.LOCK_INDEX) != 0) index_data.locked = false; timer.end(); if ((op_flags & BingoOp.NEED_STAT) != 0) BingoLog.logMessage("Statistics for table {0}:\n{1}\n", table_name, BingoCore.bingoProfilingGetStatistics(false)); } } } finally { if ((Thread.CurrentThread.ThreadState & ThreadState.AbortRequested) != 0) Thread.ResetAbort(); if (ext_conn != null) ext_conn.Close(); } }
private List<int> _screenInBlockSim (List<int> fp_ones, Block block, byte[] chunk, byte[] chunk2, SqlConnection conn, getBoundsDelegate boundsDelegate, int? next_after_storate_id) { List<int> passed_screening = new List<int>(); if (next_after_storate_id.HasValue && block.maximum_index < next_after_storate_id.Value) return passed_screening; int min_storate_id_bound = -1; if (next_after_storate_id.HasValue) min_storate_id_bound = next_after_storate_id.Value; BingoTimer timer = new BingoTimer("fingerprints.screening_sim"); int[] max_common_ones = new int[block.indices.Count]; int[] min_common_ones = new int[block.indices.Count]; int[] one_counters = new int[block.indices.Count]; // Calculate max and min bounds BingoTimer timer2 = new BingoTimer("fingerprints.screening_bounds"); boundsDelegate(block.indices, ref min_common_ones, ref max_common_ones, conn); timer2.end(); List<int> passed_screening_tmp = new List<int>(); BingoCore.lib.bingoProfIncCounter("fingerprints.bits_total", fp_ones.Count); if (fp_ones.Count == 0) { for (int i = 0; i < block.indices.Count; i++) { if (block.indices[i] <= min_storate_id_bound) continue; if (min_common_ones[i] == 0) passed_screening.Add(i); } timer.end(); return passed_screening; } int iteration = 0; foreach (BitChunk bit_chunk in bitChunksReaderGrouped(conn, block, fp_ones, _fp_sim_bits_group)) { chunk = bit_chunk.chunk; BingoTimer timer3 = new BingoTimer("fingerprints.screening_one_counters"); // Calculate ones count int max_byte_index = (block.indices.Count + 7) / 8; for (int i = 0; i < max_byte_index; i++) { byte b = chunk[i]; if (b == 0) continue; for (int j = 0; j < 8; j++) if ((b & (1 << j)) != 0) one_counters[8 * i + j]++; } timer3.end(); BingoTimer timer4 = new BingoTimer("fingerprints.screening_process"); if (iteration == 0) { for (int i = 0; i < block.indices.Count; i++) { if (block.indices[i] <= min_storate_id_bound) continue; int min_possible_ones = one_counters[i]; int max_possible_ones = one_counters[i] + fp_ones.Count; if (min_possible_ones <= max_common_ones[i] && max_possible_ones >= min_common_ones[i]) passed_screening.Add(i); } } else { passed_screening_tmp.Clear(); foreach (int i in passed_screening) { int min_possible_ones = one_counters[i]; int max_possible_ones = one_counters[i] + fp_ones.Count - iteration; if (min_possible_ones <= max_common_ones[i] && max_possible_ones >= min_common_ones[i]) passed_screening_tmp.Add(i); } // Swap then List<int> tmp = passed_screening; passed_screening = passed_screening_tmp; passed_screening_tmp = tmp; } timer4.end(); iteration++; if (passed_screening.Count < _sim_screening_pass_mark) break; } BingoCore.lib.bingoProfIncCounter("fingerprints.bits_used", iteration); for (int i = 0; i < passed_screening.Count; i++) passed_screening[i] = block.indices[passed_screening[i]]; timer.end(); return passed_screening; }
private List<int> _screenInBlockSub (List<int> fp_ones, Block block, byte[] chunk, byte[] chunk2, SqlConnection conn, int? next_after_storate_id) { List<int> results = new List<int>(); if (next_after_storate_id.HasValue && block.maximum_index < next_after_storate_id.Value) return results; int min_storate_id_bound = -1; if (next_after_storate_id.HasValue) min_storate_id_bound = next_after_storate_id.Value; // Sort ones fp_ones.Sort( (i1, i2) => block.counters[i1].CompareTo(block.counters[i2])); if (fp_ones.Count == 0) throw new Exception("Internal error: ableToScreen wasn't checked"); BingoTimer timer = new BingoTimer("fingerprints.screening_sub"); BingoCore.lib.bingoProfIncCounter("fingerprints.bits_total", fp_ones.Count); List<int> fp_ones_used = new List<int>(); for (int i = 0; i < _fp_sub_bits_used; i++) { if (i >= fp_ones.Count) break; fp_ones_used.Add(fp_ones[i]); } int iteration = 0; foreach (BitChunk bit_chunk in bitChunksReaderGrouped(conn, block, fp_ones_used, _fp_sub_bits_used)) { if (iteration == 0) { bit_chunk.chunk.CopyTo(chunk, 0); iteration++; continue; } else bit_chunk.chunk.CopyTo(chunk2, 0); iteration++; bool has_nonzero = false; for (int i = 0; i < chunk.Length; i++) { chunk[i] &= chunk2[i]; if (chunk[i] != 0) has_nonzero = true; } if (!has_nonzero) break; } BingoCore.lib.bingoProfIncCounter("fingerprints.bits_used", iteration); int max_byte_index = (block.indices.Count + 7) / 8; for (int i = 0; i < max_byte_index; i++) { byte b = chunk[i]; if (b == 0) continue; for (int j = 0; j < 8; j++) if ((b & (1 << j)) != 0) { int id = block.indices[8 * i + j]; if (id <= min_storate_id_bound) continue; results.Add(id); } } timer.end(); return results; }
private void _getBitChunk (Block block, int bit_index, SqlConnection conn, ref byte[] chunk) { BingoTimer timer = new BingoTimer("fingerprints.read"); if (block.bits != null) Buffer.BlockCopy(block.bits[bit_index], 0, chunk, 0, chunk.Length); else { chunk = (byte[])BingoSqlUtils.ExecObjQuery(conn, "SELECT bits_chunk from {0} where part = {1} and bit = {2}", _index_data.fingerprintBitsTable, block.part, bit_index); } timer.end(); }
private void _flushBlock (Block block, SqlConnection conn) { BingoTimer timer = new BingoTimer("fingerprints.flush"); if (block.part == -1) { // Add new block int? max_id = BingoSqlUtils.ExecIntQuery(conn, "SELECT MAX(part) from {0}", _index_data.fingerprintsTable); if (max_id == null) max_id = 0; block.part = max_id.Value + 1; BingoSqlUtils.ExecNonQuery(conn, "INSERT INTO {0} values ({1}, 0, null, null)", _index_data.fingerprintsTable, block.part); } BingoLog.logMessage("Flushing fingerprints block {0}...", block.part); // Update used column and counters column string update_command_text = String.Format(@"UPDATE {0} SET used = @used, counters = @counters, mapping = @mapping where part = {1}", _index_data.fingerprintsTable, block.part); using (SqlCommand command = new SqlCommand(update_command_text, conn)) { command.CommandTimeout = 3600; command.Parameters.AddWithValue("@used", block.indices.Count); byte[] countes_bytes = new byte[8 * _fp_bytes * sizeof(int)]; Buffer.BlockCopy(block.counters, 0, countes_bytes, 0, countes_bytes.Length); SqlBinary countes = new SqlBinary(countes_bytes); command.Parameters.AddWithValue("@counters", countes); byte[] mapping_bytes = new byte[block.indices.Count * sizeof(int)]; Buffer.BlockCopy(block.indices.ToArray(), 0, mapping_bytes, 0, mapping_bytes.Length); SqlBinary mapping = new SqlBinary(mapping_bytes); command.Parameters.AddWithValue("@mapping", mapping); command.ExecuteNonQuery(); } // Update bit chunks BingoSqlUtils.ExecNonQuery(conn, "DELETE FROM {0} WHERE part = {1}", _index_data.fingerprintBitsTable, block.part); string update_bits_text = String.Format(@"INSERT INTO {0} VALUES ({1}, @bit, @bit_chunk)", _index_data.fingerprintBitsTable, block.part); using (SqlCommand command = new SqlCommand(update_bits_text, conn)) { command.CommandTimeout = 3600; command.Parameters.Add("@bit", SqlDbType.Int); command.Parameters.Add("@bit_chunk", SqlDbType.Binary); byte[] chunk = new byte[_chunk_bytes]; for (int i = 0; i < 8 * _fp_bytes; i++) { command.Parameters["@bit"].Value = i; Buffer.BlockCopy(block.bits[i], 0, chunk, 0, chunk.Length); SqlBinary sql_chunk = new SqlBinary(chunk); command.Parameters["@bit_chunk"].Value = sql_chunk; command.ExecuteNonQuery(); } } block.pending = false; block.bits = null; block.indices = null; BingoLog.logMessage(" Done."); timer.end(); }
private List <int> _screenInBlockSim(List <int> fp_ones, Block block, byte[] chunk, byte[] chunk2, SqlConnection conn, getBoundsDelegate boundsDelegate, int?next_after_storate_id) { List <int> passed_screening = new List <int>(); if (next_after_storate_id.HasValue && block.maximum_index < next_after_storate_id.Value) { return(passed_screening); } int min_storate_id_bound = -1; if (next_after_storate_id.HasValue) { min_storate_id_bound = next_after_storate_id.Value; } BingoTimer timer = new BingoTimer("fingerprints.screening_sim"); int[] max_common_ones = new int[block.indices.Count]; int[] min_common_ones = new int[block.indices.Count]; int[] one_counters = new int[block.indices.Count]; // Calculate max and min bounds BingoTimer timer2 = new BingoTimer("fingerprints.screening_bounds"); boundsDelegate(block.indices, ref min_common_ones, ref max_common_ones, conn); timer2.end(); List <int> passed_screening_tmp = new List <int>(); BingoCore.lib.bingoProfIncCounter("fingerprints.bits_total", fp_ones.Count); if (fp_ones.Count == 0) { for (int i = 0; i < block.indices.Count; i++) { if (block.indices[i] <= min_storate_id_bound) { continue; } if (min_common_ones[i] == 0) { passed_screening.Add(i); } } timer.end(); return(passed_screening); } int iteration = 0; foreach (BitChunk bit_chunk in bitChunksReaderGrouped(conn, block, fp_ones, _fp_sim_bits_group)) { chunk = bit_chunk.chunk; BingoTimer timer3 = new BingoTimer("fingerprints.screening_one_counters"); // Calculate ones count int max_byte_index = (block.indices.Count + 7) / 8; for (int i = 0; i < max_byte_index; i++) { byte b = chunk[i]; if (b == 0) { continue; } for (int j = 0; j < 8; j++) { if ((b & (1 << j)) != 0) { one_counters[8 * i + j]++; } } } timer3.end(); BingoTimer timer4 = new BingoTimer("fingerprints.screening_process"); if (iteration == 0) { for (int i = 0; i < block.indices.Count; i++) { if (block.indices[i] <= min_storate_id_bound) { continue; } int min_possible_ones = one_counters[i]; int max_possible_ones = one_counters[i] + fp_ones.Count; if (min_possible_ones <= max_common_ones[i] && max_possible_ones >= min_common_ones[i]) { passed_screening.Add(i); } } } else { passed_screening_tmp.Clear(); foreach (int i in passed_screening) { int min_possible_ones = one_counters[i]; int max_possible_ones = one_counters[i] + fp_ones.Count - iteration; if (min_possible_ones <= max_common_ones[i] && max_possible_ones >= min_common_ones[i]) { passed_screening_tmp.Add(i); } } // Swap then List <int> tmp = passed_screening; passed_screening = passed_screening_tmp; passed_screening_tmp = tmp; } timer4.end(); iteration++; if (passed_screening.Count < _sim_screening_pass_mark) { break; } } BingoCore.lib.bingoProfIncCounter("fingerprints.bits_used", iteration); for (int i = 0; i < passed_screening.Count; i++) { passed_screening[i] = block.indices[passed_screening[i]]; } timer.end(); return(passed_screening); }
private List <int> _screenInBlockSub(List <int> fp_ones, Block block, byte[] chunk, byte[] chunk2, SqlConnection conn, int?next_after_storate_id) { List <int> results = new List <int>(); if (next_after_storate_id.HasValue && block.maximum_index < next_after_storate_id.Value) { return(results); } int min_storate_id_bound = -1; if (next_after_storate_id.HasValue) { min_storate_id_bound = next_after_storate_id.Value; } // Sort ones fp_ones.Sort( (i1, i2) => block.counters[i1].CompareTo(block.counters[i2])); if (fp_ones.Count == 0) { throw new Exception("Internal error: ableToScreen wasn't checked"); } BingoTimer timer = new BingoTimer("fingerprints.screening_sub"); BingoCore.lib.bingoProfIncCounter("fingerprints.bits_total", fp_ones.Count); List <int> fp_ones_used = new List <int>(); for (int i = 0; i < _fp_sub_bits_used; i++) { if (i >= fp_ones.Count) { break; } fp_ones_used.Add(fp_ones[i]); } int iteration = 0; foreach (BitChunk bit_chunk in bitChunksReaderGrouped(conn, block, fp_ones_used, _fp_sub_bits_used)) { if (iteration == 0) { bit_chunk.chunk.CopyTo(chunk, 0); iteration++; continue; } else { bit_chunk.chunk.CopyTo(chunk2, 0); } iteration++; bool has_nonzero = false; for (int i = 0; i < chunk.Length; i++) { chunk[i] &= chunk2[i]; if (chunk[i] != 0) { has_nonzero = true; } } if (!has_nonzero) { break; } } BingoCore.lib.bingoProfIncCounter("fingerprints.bits_used", iteration); int max_byte_index = (block.indices.Count + 7) / 8; for (int i = 0; i < max_byte_index; i++) { byte b = chunk[i]; if (b == 0) { continue; } for (int j = 0; j < 8; j++) { if ((b & (1 << j)) != 0) { int id = block.indices[8 * i + j]; if (id <= min_storate_id_bound) { continue; } results.Add(id); } } } timer.end(); return(results); }
private static void _ImportData (string table_name, string data_column_name, string additional_parameters, bingoImportPopulateDataRow populateRowFunc) { using (SqlConnection ctx_conn = new SqlConnection("context connection=true")) { ctx_conn.Open(); List<string[]> parameters = new List<string[]>(); string[] params_array = additional_parameters.Split(new char[] {',', ';'}); foreach (string p in params_array) { if (p != "") { string[] name_and_column = p.Trim().Split(' '); parameters.Add(name_and_column); } } DataTable dt = new DataTable(); dt.Columns.Add(new DataColumn(data_column_name, Type.GetType("System.String"))); foreach (string[] p in parameters) dt.Columns.Add(new DataColumn(p[1], Type.GetType("System.String"))); using (BingoSession session = new BingoSession()) { BingoCore.lib.bingoProfilingReset(true); BingoTimer timer = new BingoTimer("total"); SqlConnection ext_conn = null; try { string conn_string = String.Format( "server={0};integrated security=true;database={1};enlist=false", getServername(ctx_conn), ctx_conn.Database); ext_conn = new SqlConnection(conn_string); ext_conn.Open(); int imported_count = 0; bool has_data = false; do { DataRow new_row = dt.NewRow(); has_data = populateRowFunc(new_row, parameters); if (has_data) { dt.Rows.Add(new_row); imported_count++; } if (dt.Rows.Count >= 10000 || !has_data) { // Flush data table via SqlBulkCopy BingoTimer timer_sql = new BingoTimer("import.sql_bulk_copy"); using (SqlTransaction transaction = ext_conn.BeginTransaction()) { using (SqlBulkCopy bulkCopy = new SqlBulkCopy(ext_conn, SqlBulkCopyOptions.FireTriggers, transaction)) { bulkCopy.DestinationTableName = table_name; bulkCopy.ColumnMappings.Add(data_column_name, data_column_name); foreach (string[] p in parameters) bulkCopy.ColumnMappings.Add(p[1], p[1]); bulkCopy.BatchSize = dt.Rows.Count; bulkCopy.BulkCopyTimeout = 3600; bulkCopy.WriteToServer(dt); } transaction.Commit(); } timer_sql.end(); BingoCore.lib.bingoProfIncCounter("import.sql_bulk_copy_size", dt.Rows.Count); dt.Rows.Clear(); BingoLog.logMessage(" {0} molecules imported", imported_count); BingoLog.logMessage("Intermediate statistics for import into table {0}:\n{1}\n", table_name, BingoCore.bingoProfilingGetStatistics(true)); BingoCore.lib.bingoProfilingReset(false); } } while (has_data); BingoLog.logMessage(" Done."); timer.end(); BingoLog.logMessage("Statistics for import into table {0}:\n{1}\n", table_name, BingoCore.bingoProfilingGetStatistics(true)); } catch (Exception ex) { BingoLog.logMessage("Exception {0} in {1}:\n{2}", ex.Message, ex.Source, ex.StackTrace); throw; } finally { if (ext_conn != null) ext_conn.Close(); // Close import BingoCore.lib.bingoSDFImportClose(); BingoCore.lib.bingoRDFImportClose(); BingoCore.lib.bingoSMILESImportClose(); } } } }
IEnumerable<BitChunk> bitChunksReaderGrouped (SqlConnection conn, Block block, List<int> bits, int size) { int offset = 0; while (offset < bits.Count) { int cur_size = size; if (offset + cur_size >= bits.Count) cur_size = bits.Count - offset; StringBuilder indices = new StringBuilder(); for (int i = offset; i < offset + cur_size; i++) { if (indices.Length != 0) indices.Append(", "); indices.AppendFormat("{0}", bits[i]); } string command_text = String.Format("select bit, bits_chunk from {0} where part={1} and bit in ({2})", _index_data.fingerprintBitsTable, block.part, indices.ToString()); using (SqlCommand command = new SqlCommand(command_text, conn)) { command.CommandTimeout = 3600; BingoTimer timer = new BingoTimer("fingerprints.read_grouped_exec"); using (SqlDataReader reader = command.ExecuteReader()) { timer.end(); while (true) { timer = new BingoTimer("fingerprints.read_grouped_read"); bool ret = reader.Read(); if (!ret) break; BitChunk bit_chunk = new BitChunk(); int bit = (int)reader[0]; bit_chunk.bit_index = bit; bit_chunk.chunk = (byte[])reader[1]; timer.end(); yield return bit_chunk; } } } offset += size; } }
public static void ImportRDF (SqlString table_name, SqlString react_column_name, SqlString file_name, SqlString additional_parameters, SqlString bingo_schema) { bool reader_opened = false; bingoImportPopulateDataRow populateRowFunc = (row, parameters) => { if (!reader_opened) { if (BingoCore.lib.bingoRDFImportOpen(file_name.Value) != 1) throw new Exception(BingoCore.lib.bingoGetError()); reader_opened = true; } if (BingoCore.lib.bingoRDFImportEOF() != 0) return false; BingoTimer timer_mol = new BingoTimer("import.read_mol"); row[react_column_name.Value] = BingoCore.bingoRDFImportGetNext(); timer_mol.end(); foreach (string[] p in parameters) row[p[1]] = BingoCore.bingoRDFImportGetParameter(p[0]); return true; }; BingoLog.logMessage("Importing into {0} from {1}", table_name, file_name); _ImportData(table_name.Value, react_column_name.Value, additional_parameters.Value, populateRowFunc); }
private void _validateBlock (short block_index, SqlConnection conn) { if (_blockLoaded(block_index)) return; lock (_sync_object) { // Double-checked locking if (_blockLoaded(block_index)) return; BingoTimer timer = new BingoTimer("storage.validate_block"); BingoLog.logMessage("Loading storage block {0} for table {1}...", block_index, _index_data.id.InformationName()); string text = "SELECT [data], [first_index], [offsets], [lengths], [count] from " + _index_data.storageTable + " where id = " + block_index; using (SqlCommand command = new SqlCommand(text, conn)) { command.CommandTimeout = 3600; using (SqlDataReader reader = command.ExecuteReader()) { if (!reader.Read()) throw new Exception("Block cannot be found"); while (_blocks.Count <= block_index) _blocks.Add(new _Block()); if (_blocks[block_index] == null) _blocks[block_index] = new _Block(); _Block block = _blocks[block_index]; block.block_index = block_index; block.data = (byte[])reader["data"]; block.first_index = (int)reader["first_index"]; int count = (int)reader["count"]; block.offsets = new int[count]; MemoryStream mem_stream = new MemoryStream((byte[])reader["offsets"]); BinaryReader bin_reader = new BinaryReader(mem_stream); for (int i = 0; i < count; i++) block.offsets[i] = bin_reader.ReadInt32(); block.lengths = new short[count]; mem_stream = new MemoryStream((byte[])reader["lengths"]); bin_reader = new BinaryReader(mem_stream); for (int i = 0; i < count; i++) block.lengths[i] = bin_reader.ReadInt16(); block.end_index = block.first_index + count; } } BingoLog.logMessage(" Done."); timer.end(); } }
// Parameters: // table: // Table name for selecting records // existing_cursor_name // Existing cursor name for selecting records // Remarks: // Only table or existing_cursor_name must be non null // private static bingoOperationDelegate getInsertRecordsDelegate (string table, bool insert_records, bool flush_and_create_index, bool create_cursor, ArrayList error_list) { UTF8Encoding encoding = new UTF8Encoding(); bingoOperationDelegate opWithIndex = (ctx_conn, conn, data) => { // Process each molecule // If ThreadAbortException occurs then Thread.ResetAbort() is // called and indexing is terminated String select_command = String.Format("SELECT {0}, {1} FROM {2}", data.id_column, data.data_column, table); using (SqlCommand command = new SqlCommand(select_command, ctx_conn)) { command.CommandTimeout = 3600 * 10; using (SqlDataReader cursor = command.ExecuteReader()) { Exception exception = null; int counter = 0; BingoCore.GetNextRecordHandler get_next_record = (IntPtr context) => { int? id = null; try { if (exception != null) return 0; while (cursor.Read()) { if (cursor[0] == DBNull.Value) { string message = String.Format("Record with id=null was skipped."); if (SqlContext.Pipe != null) SqlContext.Pipe.Send(message); BingoLog.logMessage(message); continue; } id = Convert.ToInt32(cursor[0]); if (cursor[1] == DBNull.Value) { string message = String.Format("Record with id={0} has null data. Skipped.", id); if (SqlContext.Pipe != null) SqlContext.Pipe.Send(message); BingoLog.logMessage(message); if (error_list != null) error_list.Add(new FetchedData(id.Value) { str = "null data" }); continue; } counter++; if (counter % 10000 == 0) { BingoLog.logMessage("Processing record #{0} with id = {1}", counter, id); } byte[] record_data; if (cursor[1].GetType() == typeof(byte[])) record_data = (byte[])cursor[1]; else record_data = encoding.GetBytes((string)cursor[1]); BingoCore.lib.bingoSetIndexRecordData(id.Value, record_data, record_data.Length); return 1; } // No records more return 0; } catch (Exception ex) { if ((Thread.CurrentThread.ThreadState & ThreadState.AbortRequested) != 0) Thread.ResetAbort(); if (id.HasValue) BingoLog.logMessage("Failed on id = {0}", id); else BingoLog.logMessage("Exception {0} in {1}:\n{2}", ex.Message, ex.Source, ex.StackTrace); BingoCore.lib.bingoIndexMarkTermintate(); exception = ex; return 0; } }; BingoCore.ProcessResultHandler process_result = (IntPtr context) => { try { if (exception != null) return; if (insert_records) { BingoTimer add_timer = new BingoTimer("index.add_to_index"); if (data.getIndexType() == BingoIndexData.IndexType.Molecule) _AddMoleculeToIndex(conn, data); else _AddReactionToIndex(conn, data); add_timer.end(); } } catch (Exception ex) { BingoLog.logMessage("Exception {0} in {1}:\n{2}", ex.Message, ex.Source, ex.StackTrace); BingoCore.lib.bingoIndexMarkTermintate(); if ((Thread.CurrentThread.ThreadState & ThreadState.AbortRequested) != 0) Thread.ResetAbort(); exception = ex; } }; BingoCore.ProcessErrorHandler process_error = (int id_with_error, IntPtr context) => { try { if (exception != null) return; string message = String.Format("Record with ID={0} wasn't added to the index: {1}", id_with_error, BingoCore.lib.bingoGetWarning()); if (SqlContext.Pipe != null) SqlContext.Pipe.Send(message); BingoLog.logMessage(message); if (error_list != null) error_list.Add( new FetchedData(id_with_error) { str = BingoCore.lib.bingoGetWarning() }); } catch (Exception ex) { BingoLog.logMessage("Exception {0} in {1}:\n{2}", ex.Message, ex.Source, ex.StackTrace); BingoCore.lib.bingoIndexMarkTermintate(); if ((Thread.CurrentThread.ThreadState & ThreadState.AbortRequested) != 0) Thread.ResetAbort(); exception = ex; } }; try { int ret = BingoCore.lib.bingoIndexProcess( data.getIndexType() == BingoIndexData.IndexType.Reaction, get_next_record, process_result, process_error, IntPtr.Zero); if (ret == -1) { string msg = BingoCore.lib.bingoGetError(); BingoLog.logMessage("Internal exception: {0}", BingoCore.lib.bingoGetError()); throw new Exception(msg); } } catch (Exception ex) { // Terminate parallel indexing because it causes unhandled exception if not terminated // Index termination should be here because function pointers must be valid BingoCore.lib.bingoIndexEnd(); throw ex; } if (exception != null) throw exception; } } if (flush_and_create_index) { data.flush(conn); BingoTimer indices_timer = new BingoTimer("index.create_indices"); data.createIndices(conn); indices_timer.end(); BingoTimer fp_indices_timer = new BingoTimer("index.create_fp_indices"); data.fingerprints.createIndices(conn); fp_indices_timer.end(); data.CreateTriggers(conn); } }; return opWithIndex; }
private void _flushBlock(Block block, SqlConnection conn) { BingoTimer timer = new BingoTimer("fingerprints.flush"); if (block.part == -1) { // Add new block int?max_id = BingoSqlUtils.ExecIntQuery(conn, "SELECT MAX(part) from {0}", _index_data.fingerprintsTable); if (max_id == null) { max_id = 0; } block.part = max_id.Value + 1; BingoSqlUtils.ExecNonQuery(conn, "INSERT INTO {0} values ({1}, 0, null, null)", _index_data.fingerprintsTable, block.part); } BingoLog.logMessage("Flushing fingerprints block {0}...", block.part); // Update used column and counters column string update_command_text = String.Format(@"UPDATE {0} SET used = @used, counters = @counters, mapping = @mapping where part = {1}", _index_data.fingerprintsTable, block.part); using (SqlCommand command = new SqlCommand(update_command_text, conn)) { command.CommandTimeout = 3600; command.Parameters.AddWithValue("@used", block.indices.Count); byte[] countes_bytes = new byte[8 * _fp_bytes * sizeof(int)]; Buffer.BlockCopy(block.counters, 0, countes_bytes, 0, countes_bytes.Length); SqlBinary countes = new SqlBinary(countes_bytes); command.Parameters.AddWithValue("@counters", countes); byte[] mapping_bytes = new byte[block.indices.Count * sizeof(int)]; Buffer.BlockCopy(block.indices.ToArray(), 0, mapping_bytes, 0, mapping_bytes.Length); SqlBinary mapping = new SqlBinary(mapping_bytes); command.Parameters.AddWithValue("@mapping", mapping); command.ExecuteNonQuery(); } // Update bit chunks BingoSqlUtils.ExecNonQuery(conn, "DELETE FROM {0} WHERE part = {1}", _index_data.fingerprintBitsTable, block.part); string update_bits_text = String.Format(@"INSERT INTO {0} VALUES ({1}, @bit, @bit_chunk)", _index_data.fingerprintBitsTable, block.part); using (SqlCommand command = new SqlCommand(update_bits_text, conn)) { command.CommandTimeout = 3600; command.Parameters.Add("@bit", SqlDbType.Int); command.Parameters.Add("@bit_chunk", SqlDbType.Binary); byte[] chunk = new byte[_chunk_bytes]; for (int i = 0; i < 8 * _fp_bytes; i++) { command.Parameters["@bit"].Value = i; Buffer.BlockCopy(block.bits[i], 0, chunk, 0, chunk.Length); SqlBinary sql_chunk = new SqlBinary(chunk); command.Parameters["@bit_chunk"].Value = sql_chunk; command.ExecuteNonQuery(); } } block.pending = false; block.bits = null; block.indices = null; BingoLog.logMessage(" Done."); timer.end(); }