private void simGetMinMaxBounds(IList <int> storage_ids, ref int[] min_common_ones,
ref int[] max_common_ones, SqlConnection conn)
{
BingoTimer timer2 = new BingoTimer("fingerprints.screening_bounds_ones");
int cache_index = 0;
int[] targets_ones = new int[storage_ids.Count];
for (int i = 0; i < storage_ids.Count; i++)
{
int storage_id = storage_ids[i];
targets_ones[i] = _index_data.storage.getShort(storage_id, 4, conn, ref cache_index);
}
timer2.end();
BingoTimer timer3 = new BingoTimer("fingerprints.screening_bounds_get");
IntPtr min_common_ones_ptr, max_common_ones_ptr;
BingoCore.lib.mangoSimilarityGetBitMinMaxBoundsArray(targets_ones.Length, targets_ones,
out min_common_ones_ptr, out max_common_ones_ptr);
if (min_common_ones.Length != storage_ids.Count ||
max_common_ones.Length != storage_ids.Count)
{
throw new Exception("Internal error in simGetMinMaxBounds: array lengths mismatch");
}
Marshal.Copy(min_common_ones_ptr, min_common_ones, 0, targets_ones.Length);
Marshal.Copy(max_common_ones_ptr, max_common_ones, 0, targets_ones.Length);
timer3.end();
}
private void simGetMinMaxBounds (IList<int> storage_ids, ref int[] min_common_ones,
ref int[] max_common_ones, SqlConnection conn)
{
BingoTimer timer2 = new BingoTimer("fingerprints.screening_bounds_ones");
int cache_index = 0;
int[] targets_ones = new int[storage_ids.Count];
for (int i = 0; i < storage_ids.Count; i++)
{
int storage_id = storage_ids[i];
targets_ones[i] = _index_data.storage.getShort(storage_id, 4, conn, ref cache_index);
}
timer2.end();
BingoTimer timer3 = new BingoTimer("fingerprints.screening_bounds_get");
IntPtr min_common_ones_ptr, max_common_ones_ptr;
BingoCore.lib.mangoSimilarityGetBitMinMaxBoundsArray(targets_ones.Length, targets_ones,
out min_common_ones_ptr, out max_common_ones_ptr);
if (min_common_ones.Length != storage_ids.Count ||
max_common_ones.Length != storage_ids.Count)
throw new Exception("Internal error in simGetMinMaxBounds: array lengths mismatch");
Marshal.Copy(min_common_ones_ptr, min_common_ones, 0, targets_ones.Length);
Marshal.Copy(max_common_ones_ptr, max_common_ones, 0, targets_ones.Length);
timer3.end();
}
private void _flushShadowTable(SqlConnection conn)
{
if (shadow_datatable == null || (shadow_datatable.Rows.Count == 0 && components_datatable.Rows.Count == 0))
{
return;
}
if (conn.ConnectionString == "context connection=true")
{
// SqlBulkInsert cannot be used in the context connection
_flushShadowTableInContext(conn);
return;
}
BingoTimer timer = new BingoTimer("shadow_table.flush");
using (SqlTransaction transaction =
conn.BeginTransaction())
{
// Copy shadow table
using (SqlBulkCopy bulkCopy = new SqlBulkCopy(conn,
SqlBulkCopyOptions.TableLock, transaction))
{
bulkCopy.DestinationTableName = shadowTable;
foreach (DataColumn dc in shadow_datatable.Columns)
{
bulkCopy.ColumnMappings.Add(dc.ColumnName, dc.ColumnName);
}
bulkCopy.BatchSize = shadow_datatable.Rows.Count;
bulkCopy.BulkCopyTimeout = 3600;
bulkCopy.WriteToServer(shadow_datatable);
}
shadow_datatable.Rows.Clear();
// Copy components table
using (SqlBulkCopy bulkCopy = new SqlBulkCopy(conn,
SqlBulkCopyOptions.TableLock, transaction))
{
bulkCopy.DestinationTableName = componentsTable;
foreach (DataColumn dc in components_datatable.Columns)
{
bulkCopy.ColumnMappings.Add(dc.ColumnName, dc.ColumnName);
}
bulkCopy.BatchSize = components_datatable.Rows.Count;
bulkCopy.BulkCopyTimeout = 3600;
bulkCopy.WriteToServer(components_datatable);
}
components_datatable.Rows.Clear();
transaction.Commit();
}
timer.end();
}
IEnumerable <BitChunk> bitChunksReaderGrouped(SqlConnection conn, Block block,
List <int> bits, int size)
{
int offset = 0;
while (offset < bits.Count)
{
int cur_size = size;
if (offset + cur_size >= bits.Count)
{
cur_size = bits.Count - offset;
}
StringBuilder indices = new StringBuilder();
for (int i = offset; i < offset + cur_size; i++)
{
if (indices.Length != 0)
{
indices.Append(", ");
}
indices.AppendFormat("{0}", bits[i]);
}
string command_text = String.Format("select bit, bits_chunk from {0} where part={1} and bit in ({2})",
_index_data.fingerprintBitsTable, block.part, indices.ToString());
using (SqlCommand command = new SqlCommand(command_text, conn))
{
command.CommandTimeout = 3600;
BingoTimer timer = new BingoTimer("fingerprints.read_grouped_exec");
using (SqlDataReader reader = command.ExecuteReader())
{
timer.end();
while (true)
{
timer = new BingoTimer("fingerprints.read_grouped_read");
bool ret = reader.Read();
if (!ret)
{
break;
}
BitChunk bit_chunk = new BitChunk();
int bit = (int)reader[0];
bit_chunk.bit_index = bit;
bit_chunk.chunk = (byte[])reader[1];
timer.end();
yield return(bit_chunk);
}
}
}
offset += size;
}
}
private void _getBitChunk(Block block, int bit_index, SqlConnection conn, ref byte[] chunk)
{
BingoTimer timer = new BingoTimer("fingerprints.read");
if (block.bits != null)
{
Buffer.BlockCopy(block.bits[bit_index], 0, chunk, 0, chunk.Length);
}
else
{
chunk = (byte[])BingoSqlUtils.ExecObjQuery(conn,
"SELECT bits_chunk from {0} where part = {1} and bit = {2}",
_index_data.fingerprintBitsTable, block.part, bit_index);
}
timer.end();
}
private void _flushShadowTable (SqlConnection conn)
{
if (shadow_datatable == null || (shadow_datatable.Rows.Count == 0 && components_datatable.Rows.Count == 0))
return;
if (conn.ConnectionString == "context connection=true")
{
// SqlBulkInsert cannot be used in the context connection
_flushShadowTableInContext(conn);
return;
}
BingoTimer timer = new BingoTimer("shadow_table.flush");
using (SqlTransaction transaction =
conn.BeginTransaction())
{
// Copy shadow table
using (SqlBulkCopy bulkCopy = new SqlBulkCopy(conn,
SqlBulkCopyOptions.TableLock, transaction))
{
bulkCopy.DestinationTableName = shadowTable;
foreach (DataColumn dc in shadow_datatable.Columns)
bulkCopy.ColumnMappings.Add(dc.ColumnName, dc.ColumnName);
bulkCopy.BatchSize = shadow_datatable.Rows.Count;
bulkCopy.BulkCopyTimeout = 3600;
bulkCopy.WriteToServer(shadow_datatable);
}
shadow_datatable.Rows.Clear();
// Copy components table
using (SqlBulkCopy bulkCopy = new SqlBulkCopy(conn,
SqlBulkCopyOptions.TableLock, transaction))
{
bulkCopy.DestinationTableName = componentsTable;
foreach (DataColumn dc in components_datatable.Columns)
bulkCopy.ColumnMappings.Add(dc.ColumnName, dc.ColumnName);
bulkCopy.BatchSize = components_datatable.Rows.Count;
bulkCopy.BulkCopyTimeout = 3600;
bulkCopy.WriteToServer(components_datatable);
}
components_datatable.Rows.Clear();
transaction.Commit();
}
timer.end();
}
private void _validateBlock(short block_index, SqlConnection conn)
{
if (_blockLoaded(block_index))
{
return;
}
lock (_sync_object)
{
// Double-checked locking
if (_blockLoaded(block_index))
{
return;
}
BingoTimer timer = new BingoTimer("storage.validate_block");
BingoLog.logMessage("Loading storage block {0} for table {1}...",
block_index, _index_data.id.InformationName());
string text = "SELECT [data], [first_index], [offsets], [lengths], [count] from " +
_index_data.storageTable + " where id = " + block_index;
using (SqlCommand command = new SqlCommand(text, conn))
{
command.CommandTimeout = 3600;
using (SqlDataReader reader = command.ExecuteReader())
{
if (!reader.Read())
{
throw new Exception("Block cannot be found");
}
while (_blocks.Count <= block_index)
{
_blocks.Add(new _Block());
}
if (_blocks[block_index] == null)
{
_blocks[block_index] = new _Block();
}
_Block block = _blocks[block_index];
block.block_index = block_index;
block.data = (byte[])reader["data"];
block.first_index = (int)reader["first_index"];
int count = (int)reader["count"];
block.offsets = new int[count];
MemoryStream mem_stream = new MemoryStream((byte[])reader["offsets"]);
BinaryReader bin_reader = new BinaryReader(mem_stream);
for (int i = 0; i < count; i++)
{
block.offsets[i] = bin_reader.ReadInt32();
}
block.lengths = new short[count];
mem_stream = new MemoryStream((byte[])reader["lengths"]);
bin_reader = new BinaryReader(mem_stream);
for (int i = 0; i < count; i++)
{
block.lengths[i] = bin_reader.ReadInt16();
}
block.end_index = block.first_index + count;
}
}
BingoLog.logMessage(" Done.");
timer.end();
}
}
// Method for executing abstract operation with BingoIndexData
private static void _ExecuteBingoOperation(SqlString bingo_schema,
bingoOperationDelegate operationDelegate,
bingoGetIndexDataDelegate getBingoDataDelegate, BingoOp op_flags)
{
string table_name = "<Undef>";
BingoIndexID id = null;
SqlConnection ext_conn = null;
try
{
using (SqlConnection ctx_conn = new SqlConnection("context connection=true"))
{
ctx_conn.Open();
SqlConnection conn = ctx_conn;
if ((op_flags & BingoOp.NON_CONTEXT_CONN) != 0)
{
string conn_string = String.Format(
"server={0};integrated security=true;database={1};enlist=false",
getServername(ctx_conn), ctx_conn.Database);
ext_conn = new SqlConnection(conn_string);
ext_conn.Open();
conn = ext_conn;
}
using (BingoSession session = new BingoSession())
{
BingoCore.lib.bingoProfilingReset(false);
BingoTimer timer = new BingoTimer("total");
BingoIndexData index_data = getBingoDataDelegate(ctx_conn, conn, bingo_schema);
table_name = index_data.id.FullTableName(ctx_conn);
id = index_data.id;
if (index_data.locked)
{
BingoLog.logMessage("Attempt to get locked index for the table {0}", table_name);
throw new Exception("Chemical index for the table '" + table_name + "' is locked");
}
if ((op_flags & BingoOp.LOCK_INDEX) != 0)
index_data.locked = true;
ContextFlags flags = ContextFlags.NTHREADS | ContextFlags.FINGERPRINTS;
if ((op_flags & BingoOp.LOAD_TAU_RULES) != 0)
flags |= ContextFlags.TAU_RULES;
if ((op_flags & BingoOp.LOAD_CMF) != 0)
flags |= ContextFlags.CMF;
prepareContext(ctx_conn, bingo_schema.Value, index_data.id.table_id, flags);
index_data.syncContextParameters(ctx_conn, bingo_schema.Value);
try
{
operationDelegate(ctx_conn, conn, index_data);
}
catch (Exception ex)
{
if ((Thread.CurrentThread.ThreadState & ThreadState.AbortRequested) != 0)
Thread.ResetAbort();
BingoLog.logMessage("Exception {0} in {1}:\n{2}", ex.Message, ex.Source, ex.StackTrace);
if ((op_flags & BingoOp.LOCK_INDEX) != 0)
index_data.locked = false;
if ((op_flags & BingoOp.DROP_ON_EXCEPTION) != 0)
BingoIndexData.DropIndexData(conn, bingo_schema.Value, id, false);
throw ex;
}
if ((op_flags & BingoOp.LOCK_INDEX) != 0)
index_data.locked = false;
timer.end();
if ((op_flags & BingoOp.NEED_STAT) != 0)
BingoLog.logMessage("Statistics for table {0}:\n{1}\n",
table_name, BingoCore.bingoProfilingGetStatistics(false));
}
}
}
finally
{
if ((Thread.CurrentThread.ThreadState & ThreadState.AbortRequested) != 0)
Thread.ResetAbort();
if (ext_conn != null)
ext_conn.Close();
}
}
private List<int> _screenInBlockSim (List<int> fp_ones, Block block,
byte[] chunk, byte[] chunk2, SqlConnection conn, getBoundsDelegate boundsDelegate,
int? next_after_storate_id)
{
List<int> passed_screening = new List<int>();
if (next_after_storate_id.HasValue && block.maximum_index < next_after_storate_id.Value)
return passed_screening;
int min_storate_id_bound = -1;
if (next_after_storate_id.HasValue)
min_storate_id_bound = next_after_storate_id.Value;
BingoTimer timer = new BingoTimer("fingerprints.screening_sim");
int[] max_common_ones = new int[block.indices.Count];
int[] min_common_ones = new int[block.indices.Count];
int[] one_counters = new int[block.indices.Count];
// Calculate max and min bounds
BingoTimer timer2 = new BingoTimer("fingerprints.screening_bounds");
boundsDelegate(block.indices, ref min_common_ones, ref max_common_ones, conn);
timer2.end();
List<int> passed_screening_tmp = new List<int>();
BingoCore.lib.bingoProfIncCounter("fingerprints.bits_total", fp_ones.Count);
if (fp_ones.Count == 0)
{
for (int i = 0; i < block.indices.Count; i++)
{
if (block.indices[i] <= min_storate_id_bound)
continue;
if (min_common_ones[i] == 0)
passed_screening.Add(i);
}
timer.end();
return passed_screening;
}
int iteration = 0;
foreach (BitChunk bit_chunk in bitChunksReaderGrouped(conn, block,
fp_ones, _fp_sim_bits_group))
{
chunk = bit_chunk.chunk;
BingoTimer timer3 = new BingoTimer("fingerprints.screening_one_counters");
// Calculate ones count
int max_byte_index = (block.indices.Count + 7) / 8;
for (int i = 0; i < max_byte_index; i++)
{
byte b = chunk[i];
if (b == 0)
continue;
for (int j = 0; j < 8; j++)
if ((b & (1 << j)) != 0)
one_counters[8 * i + j]++;
}
timer3.end();
BingoTimer timer4 = new BingoTimer("fingerprints.screening_process");
if (iteration == 0)
{
for (int i = 0; i < block.indices.Count; i++)
{
if (block.indices[i] <= min_storate_id_bound)
continue;
int min_possible_ones = one_counters[i];
int max_possible_ones = one_counters[i] + fp_ones.Count;
if (min_possible_ones <= max_common_ones[i] && max_possible_ones >= min_common_ones[i])
passed_screening.Add(i);
}
}
else
{
passed_screening_tmp.Clear();
foreach (int i in passed_screening)
{
int min_possible_ones = one_counters[i];
int max_possible_ones = one_counters[i] + fp_ones.Count - iteration;
if (min_possible_ones <= max_common_ones[i] && max_possible_ones >= min_common_ones[i])
passed_screening_tmp.Add(i);
}
// Swap then
List<int> tmp = passed_screening;
passed_screening = passed_screening_tmp;
passed_screening_tmp = tmp;
}
timer4.end();
iteration++;
if (passed_screening.Count < _sim_screening_pass_mark)
break;
}
BingoCore.lib.bingoProfIncCounter("fingerprints.bits_used", iteration);
for (int i = 0; i < passed_screening.Count; i++)
passed_screening[i] = block.indices[passed_screening[i]];
timer.end();
return passed_screening;
}
private List<int> _screenInBlockSub (List<int> fp_ones, Block block,
byte[] chunk, byte[] chunk2, SqlConnection conn, int? next_after_storate_id)
{
List<int> results = new List<int>();
if (next_after_storate_id.HasValue && block.maximum_index < next_after_storate_id.Value)
return results;
int min_storate_id_bound = -1;
if (next_after_storate_id.HasValue)
min_storate_id_bound = next_after_storate_id.Value;
// Sort ones
fp_ones.Sort(
(i1, i2) => block.counters[i1].CompareTo(block.counters[i2]));
if (fp_ones.Count == 0)
throw new Exception("Internal error: ableToScreen wasn't checked");
BingoTimer timer = new BingoTimer("fingerprints.screening_sub");
BingoCore.lib.bingoProfIncCounter("fingerprints.bits_total", fp_ones.Count);
List<int> fp_ones_used = new List<int>();
for (int i = 0; i < _fp_sub_bits_used; i++)
{
if (i >= fp_ones.Count)
break;
fp_ones_used.Add(fp_ones[i]);
}
int iteration = 0;
foreach (BitChunk bit_chunk in bitChunksReaderGrouped(conn, block,
fp_ones_used, _fp_sub_bits_used))
{
if (iteration == 0)
{
bit_chunk.chunk.CopyTo(chunk, 0);
iteration++;
continue;
}
else
bit_chunk.chunk.CopyTo(chunk2, 0);
iteration++;
bool has_nonzero = false;
for (int i = 0; i < chunk.Length; i++)
{
chunk[i] &= chunk2[i];
if (chunk[i] != 0)
has_nonzero = true;
}
if (!has_nonzero)
break;
}
BingoCore.lib.bingoProfIncCounter("fingerprints.bits_used", iteration);
int max_byte_index = (block.indices.Count + 7) / 8;
for (int i = 0; i < max_byte_index; i++)
{
byte b = chunk[i];
if (b == 0)
continue;
for (int j = 0; j < 8; j++)
if ((b & (1 << j)) != 0)
{
int id = block.indices[8 * i + j];
if (id <= min_storate_id_bound)
continue;
results.Add(id);
}
}
timer.end();
return results;
}
private void _getBitChunk (Block block, int bit_index, SqlConnection conn, ref byte[] chunk)
{
BingoTimer timer = new BingoTimer("fingerprints.read");
if (block.bits != null)
Buffer.BlockCopy(block.bits[bit_index], 0, chunk, 0, chunk.Length);
else
{
chunk = (byte[])BingoSqlUtils.ExecObjQuery(conn,
"SELECT bits_chunk from {0} where part = {1} and bit = {2}",
_index_data.fingerprintBitsTable, block.part, bit_index);
}
timer.end();
}
private void _flushBlock (Block block, SqlConnection conn)
{
BingoTimer timer = new BingoTimer("fingerprints.flush");
if (block.part == -1)
{
// Add new block
int? max_id = BingoSqlUtils.ExecIntQuery(conn,
"SELECT MAX(part) from {0}", _index_data.fingerprintsTable);
if (max_id == null)
max_id = 0;
block.part = max_id.Value + 1;
BingoSqlUtils.ExecNonQuery(conn, "INSERT INTO {0} values ({1}, 0, null, null)",
_index_data.fingerprintsTable, block.part);
}
BingoLog.logMessage("Flushing fingerprints block {0}...", block.part);
// Update used column and counters column
string update_command_text = String.Format(@"UPDATE {0} SET used = @used,
counters = @counters, mapping = @mapping
where part = {1}", _index_data.fingerprintsTable, block.part);
using (SqlCommand command = new SqlCommand(update_command_text, conn))
{
command.CommandTimeout = 3600;
command.Parameters.AddWithValue("@used", block.indices.Count);
byte[] countes_bytes = new byte[8 * _fp_bytes * sizeof(int)];
Buffer.BlockCopy(block.counters, 0, countes_bytes, 0, countes_bytes.Length);
SqlBinary countes = new SqlBinary(countes_bytes);
command.Parameters.AddWithValue("@counters", countes);
byte[] mapping_bytes = new byte[block.indices.Count * sizeof(int)];
Buffer.BlockCopy(block.indices.ToArray(), 0, mapping_bytes, 0, mapping_bytes.Length);
SqlBinary mapping = new SqlBinary(mapping_bytes);
command.Parameters.AddWithValue("@mapping", mapping);
command.ExecuteNonQuery();
}
// Update bit chunks
BingoSqlUtils.ExecNonQuery(conn, "DELETE FROM {0} WHERE part = {1}",
_index_data.fingerprintBitsTable, block.part);
string update_bits_text = String.Format(@"INSERT INTO {0} VALUES ({1}, @bit, @bit_chunk)",
_index_data.fingerprintBitsTable, block.part);
using (SqlCommand command = new SqlCommand(update_bits_text, conn))
{
command.CommandTimeout = 3600;
command.Parameters.Add("@bit", SqlDbType.Int);
command.Parameters.Add("@bit_chunk", SqlDbType.Binary);
byte[] chunk = new byte[_chunk_bytes];
for (int i = 0; i < 8 * _fp_bytes; i++)
{
command.Parameters["@bit"].Value = i;
Buffer.BlockCopy(block.bits[i], 0, chunk, 0, chunk.Length);
SqlBinary sql_chunk = new SqlBinary(chunk);
command.Parameters["@bit_chunk"].Value = sql_chunk;
command.ExecuteNonQuery();
}
}
block.pending = false;
block.bits = null;
block.indices = null;
BingoLog.logMessage(" Done.");
timer.end();
}
private List <int> _screenInBlockSim(List <int> fp_ones, Block block,
byte[] chunk, byte[] chunk2, SqlConnection conn, getBoundsDelegate boundsDelegate,
int?next_after_storate_id)
{
List <int> passed_screening = new List <int>();
if (next_after_storate_id.HasValue && block.maximum_index < next_after_storate_id.Value)
{
return(passed_screening);
}
int min_storate_id_bound = -1;
if (next_after_storate_id.HasValue)
{
min_storate_id_bound = next_after_storate_id.Value;
}
BingoTimer timer = new BingoTimer("fingerprints.screening_sim");
int[] max_common_ones = new int[block.indices.Count];
int[] min_common_ones = new int[block.indices.Count];
int[] one_counters = new int[block.indices.Count];
// Calculate max and min bounds
BingoTimer timer2 = new BingoTimer("fingerprints.screening_bounds");
boundsDelegate(block.indices, ref min_common_ones, ref max_common_ones, conn);
timer2.end();
List <int> passed_screening_tmp = new List <int>();
BingoCore.lib.bingoProfIncCounter("fingerprints.bits_total", fp_ones.Count);
if (fp_ones.Count == 0)
{
for (int i = 0; i < block.indices.Count; i++)
{
if (block.indices[i] <= min_storate_id_bound)
{
continue;
}
if (min_common_ones[i] == 0)
{
passed_screening.Add(i);
}
}
timer.end();
return(passed_screening);
}
int iteration = 0;
foreach (BitChunk bit_chunk in bitChunksReaderGrouped(conn, block,
fp_ones, _fp_sim_bits_group))
{
chunk = bit_chunk.chunk;
BingoTimer timer3 = new BingoTimer("fingerprints.screening_one_counters");
// Calculate ones count
int max_byte_index = (block.indices.Count + 7) / 8;
for (int i = 0; i < max_byte_index; i++)
{
byte b = chunk[i];
if (b == 0)
{
continue;
}
for (int j = 0; j < 8; j++)
{
if ((b & (1 << j)) != 0)
{
one_counters[8 * i + j]++;
}
}
}
timer3.end();
BingoTimer timer4 = new BingoTimer("fingerprints.screening_process");
if (iteration == 0)
{
for (int i = 0; i < block.indices.Count; i++)
{
if (block.indices[i] <= min_storate_id_bound)
{
continue;
}
int min_possible_ones = one_counters[i];
int max_possible_ones = one_counters[i] + fp_ones.Count;
if (min_possible_ones <= max_common_ones[i] && max_possible_ones >= min_common_ones[i])
{
passed_screening.Add(i);
}
}
}
else
{
passed_screening_tmp.Clear();
foreach (int i in passed_screening)
{
int min_possible_ones = one_counters[i];
int max_possible_ones = one_counters[i] + fp_ones.Count - iteration;
if (min_possible_ones <= max_common_ones[i] && max_possible_ones >= min_common_ones[i])
{
passed_screening_tmp.Add(i);
}
}
// Swap then
List <int> tmp = passed_screening;
passed_screening = passed_screening_tmp;
passed_screening_tmp = tmp;
}
timer4.end();
iteration++;
if (passed_screening.Count < _sim_screening_pass_mark)
{
break;
}
}
BingoCore.lib.bingoProfIncCounter("fingerprints.bits_used", iteration);
for (int i = 0; i < passed_screening.Count; i++)
{
passed_screening[i] = block.indices[passed_screening[i]];
}
timer.end();
return(passed_screening);
}
private List <int> _screenInBlockSub(List <int> fp_ones, Block block,
byte[] chunk, byte[] chunk2, SqlConnection conn, int?next_after_storate_id)
{
List <int> results = new List <int>();
if (next_after_storate_id.HasValue && block.maximum_index < next_after_storate_id.Value)
{
return(results);
}
int min_storate_id_bound = -1;
if (next_after_storate_id.HasValue)
{
min_storate_id_bound = next_after_storate_id.Value;
}
// Sort ones
fp_ones.Sort(
(i1, i2) => block.counters[i1].CompareTo(block.counters[i2]));
if (fp_ones.Count == 0)
{
throw new Exception("Internal error: ableToScreen wasn't checked");
}
BingoTimer timer = new BingoTimer("fingerprints.screening_sub");
BingoCore.lib.bingoProfIncCounter("fingerprints.bits_total", fp_ones.Count);
List <int> fp_ones_used = new List <int>();
for (int i = 0; i < _fp_sub_bits_used; i++)
{
if (i >= fp_ones.Count)
{
break;
}
fp_ones_used.Add(fp_ones[i]);
}
int iteration = 0;
foreach (BitChunk bit_chunk in bitChunksReaderGrouped(conn, block,
fp_ones_used, _fp_sub_bits_used))
{
if (iteration == 0)
{
bit_chunk.chunk.CopyTo(chunk, 0);
iteration++;
continue;
}
else
{
bit_chunk.chunk.CopyTo(chunk2, 0);
}
iteration++;
bool has_nonzero = false;
for (int i = 0; i < chunk.Length; i++)
{
chunk[i] &= chunk2[i];
if (chunk[i] != 0)
{
has_nonzero = true;
}
}
if (!has_nonzero)
{
break;
}
}
BingoCore.lib.bingoProfIncCounter("fingerprints.bits_used", iteration);
int max_byte_index = (block.indices.Count + 7) / 8;
for (int i = 0; i < max_byte_index; i++)
{
byte b = chunk[i];
if (b == 0)
{
continue;
}
for (int j = 0; j < 8; j++)
{
if ((b & (1 << j)) != 0)
{
int id = block.indices[8 * i + j];
if (id <= min_storate_id_bound)
{
continue;
}
results.Add(id);
}
}
}
timer.end();
return(results);
}
private static void _ImportData (string table_name, string data_column_name,
string additional_parameters, bingoImportPopulateDataRow populateRowFunc)
{
using (SqlConnection ctx_conn = new SqlConnection("context connection=true"))
{
ctx_conn.Open();
List<string[]> parameters = new List<string[]>();
string[] params_array = additional_parameters.Split(new char[] {',', ';'});
foreach (string p in params_array)
{
if (p != "")
{
string[] name_and_column = p.Trim().Split(' ');
parameters.Add(name_and_column);
}
}
DataTable dt = new DataTable();
dt.Columns.Add(new DataColumn(data_column_name,
Type.GetType("System.String")));
foreach (string[] p in parameters)
dt.Columns.Add(new DataColumn(p[1], Type.GetType("System.String")));
using (BingoSession session = new BingoSession())
{
BingoCore.lib.bingoProfilingReset(true);
BingoTimer timer = new BingoTimer("total");
SqlConnection ext_conn = null;
try
{
string conn_string = String.Format(
"server={0};integrated security=true;database={1};enlist=false",
getServername(ctx_conn), ctx_conn.Database);
ext_conn = new SqlConnection(conn_string);
ext_conn.Open();
int imported_count = 0;
bool has_data = false;
do
{
DataRow new_row = dt.NewRow();
has_data = populateRowFunc(new_row, parameters);
if (has_data)
{
dt.Rows.Add(new_row);
imported_count++;
}
if (dt.Rows.Count >= 10000 || !has_data)
{
// Flush data table via SqlBulkCopy
BingoTimer timer_sql = new BingoTimer("import.sql_bulk_copy");
using (SqlTransaction transaction =
ext_conn.BeginTransaction())
{
using (SqlBulkCopy bulkCopy = new SqlBulkCopy(ext_conn,
SqlBulkCopyOptions.FireTriggers, transaction))
{
bulkCopy.DestinationTableName = table_name;
bulkCopy.ColumnMappings.Add(data_column_name, data_column_name);
foreach (string[] p in parameters)
bulkCopy.ColumnMappings.Add(p[1], p[1]);
bulkCopy.BatchSize = dt.Rows.Count;
bulkCopy.BulkCopyTimeout = 3600;
bulkCopy.WriteToServer(dt);
}
transaction.Commit();
}
timer_sql.end();
BingoCore.lib.bingoProfIncCounter("import.sql_bulk_copy_size", dt.Rows.Count);
dt.Rows.Clear();
BingoLog.logMessage(" {0} molecules imported", imported_count);
BingoLog.logMessage("Intermediate statistics for import into table {0}:\n{1}\n",
table_name, BingoCore.bingoProfilingGetStatistics(true));
BingoCore.lib.bingoProfilingReset(false);
}
} while (has_data);
BingoLog.logMessage(" Done.");
timer.end();
BingoLog.logMessage("Statistics for import into table {0}:\n{1}\n",
table_name, BingoCore.bingoProfilingGetStatistics(true));
}
catch (Exception ex)
{
BingoLog.logMessage("Exception {0} in {1}:\n{2}", ex.Message, ex.Source, ex.StackTrace);
throw;
}
finally
{
if (ext_conn != null)
ext_conn.Close();
// Close import
BingoCore.lib.bingoSDFImportClose();
BingoCore.lib.bingoRDFImportClose();
BingoCore.lib.bingoSMILESImportClose();
}
}
}
}
IEnumerable<BitChunk> bitChunksReaderGrouped (SqlConnection conn, Block block,
List<int> bits, int size)
{
int offset = 0;
while (offset < bits.Count)
{
int cur_size = size;
if (offset + cur_size >= bits.Count)
cur_size = bits.Count - offset;
StringBuilder indices = new StringBuilder();
for (int i = offset; i < offset + cur_size; i++)
{
if (indices.Length != 0)
indices.Append(", ");
indices.AppendFormat("{0}", bits[i]);
}
string command_text = String.Format("select bit, bits_chunk from {0} where part={1} and bit in ({2})",
_index_data.fingerprintBitsTable, block.part, indices.ToString());
using (SqlCommand command = new SqlCommand(command_text, conn))
{
command.CommandTimeout = 3600;
BingoTimer timer = new BingoTimer("fingerprints.read_grouped_exec");
using (SqlDataReader reader = command.ExecuteReader())
{
timer.end();
while (true)
{
timer = new BingoTimer("fingerprints.read_grouped_read");
bool ret = reader.Read();
if (!ret)
break;
BitChunk bit_chunk = new BitChunk();
int bit = (int)reader[0];
bit_chunk.bit_index = bit;
bit_chunk.chunk = (byte[])reader[1];
timer.end();
yield return bit_chunk;
}
}
}
offset += size;
}
}
public static void ImportRDF (SqlString table_name, SqlString react_column_name,
SqlString file_name, SqlString additional_parameters, SqlString bingo_schema)
{
bool reader_opened = false;
bingoImportPopulateDataRow populateRowFunc =
(row, parameters) =>
{
if (!reader_opened)
{
if (BingoCore.lib.bingoRDFImportOpen(file_name.Value) != 1)
throw new Exception(BingoCore.lib.bingoGetError());
reader_opened = true;
}
if (BingoCore.lib.bingoRDFImportEOF() != 0)
return false;
BingoTimer timer_mol = new BingoTimer("import.read_mol");
row[react_column_name.Value] = BingoCore.bingoRDFImportGetNext();
timer_mol.end();
foreach (string[] p in parameters)
row[p[1]] = BingoCore.bingoRDFImportGetParameter(p[0]);
return true;
};
BingoLog.logMessage("Importing into {0} from {1}", table_name, file_name);
_ImportData(table_name.Value, react_column_name.Value,
additional_parameters.Value, populateRowFunc);
}
private void _validateBlock (short block_index, SqlConnection conn)
{
if (_blockLoaded(block_index))
return;
lock (_sync_object)
{
// Double-checked locking
if (_blockLoaded(block_index))
return;
BingoTimer timer = new BingoTimer("storage.validate_block");
BingoLog.logMessage("Loading storage block {0} for table {1}...",
block_index, _index_data.id.InformationName());
string text = "SELECT [data], [first_index], [offsets], [lengths], [count] from " +
_index_data.storageTable + " where id = " + block_index;
using (SqlCommand command = new SqlCommand(text, conn))
{
command.CommandTimeout = 3600;
using (SqlDataReader reader = command.ExecuteReader())
{
if (!reader.Read())
throw new Exception("Block cannot be found");
while (_blocks.Count <= block_index)
_blocks.Add(new _Block());
if (_blocks[block_index] == null)
_blocks[block_index] = new _Block();
_Block block = _blocks[block_index];
block.block_index = block_index;
block.data = (byte[])reader["data"];
block.first_index = (int)reader["first_index"];
int count = (int)reader["count"];
block.offsets = new int[count];
MemoryStream mem_stream = new MemoryStream((byte[])reader["offsets"]);
BinaryReader bin_reader = new BinaryReader(mem_stream);
for (int i = 0; i < count; i++)
block.offsets[i] = bin_reader.ReadInt32();
block.lengths = new short[count];
mem_stream = new MemoryStream((byte[])reader["lengths"]);
bin_reader = new BinaryReader(mem_stream);
for (int i = 0; i < count; i++)
block.lengths[i] = bin_reader.ReadInt16();
block.end_index = block.first_index + count;
}
}
BingoLog.logMessage(" Done.");
timer.end();
}
}
// Parameters:
// table:
// Table name for selecting records
// existing_cursor_name
// Existing cursor name for selecting records
// Remarks:
// Only table or existing_cursor_name must be non null
//
private static bingoOperationDelegate getInsertRecordsDelegate (string table,
bool insert_records, bool flush_and_create_index, bool create_cursor, ArrayList error_list)
{
UTF8Encoding encoding = new UTF8Encoding();
bingoOperationDelegate opWithIndex =
(ctx_conn, conn, data) =>
{
// Process each molecule
// If ThreadAbortException occurs then Thread.ResetAbort() is
// called and indexing is terminated
String select_command =
String.Format("SELECT {0}, {1} FROM {2}", data.id_column, data.data_column, table);
using (SqlCommand command = new SqlCommand(select_command, ctx_conn))
{
command.CommandTimeout = 3600 * 10;
using (SqlDataReader cursor = command.ExecuteReader())
{
Exception exception = null;
int counter = 0;
BingoCore.GetNextRecordHandler get_next_record =
(IntPtr context) =>
{
int? id = null;
try
{
if (exception != null)
return 0;
while (cursor.Read())
{
if (cursor[0] == DBNull.Value)
{
string message =
String.Format("Record with id=null was skipped.");
if (SqlContext.Pipe != null)
SqlContext.Pipe.Send(message);
BingoLog.logMessage(message);
continue;
}
id = Convert.ToInt32(cursor[0]);
if (cursor[1] == DBNull.Value)
{
string message =
String.Format("Record with id={0} has null data. Skipped.", id);
if (SqlContext.Pipe != null)
SqlContext.Pipe.Send(message);
BingoLog.logMessage(message);
if (error_list != null)
error_list.Add(new FetchedData(id.Value) { str = "null data" });
continue;
}
counter++;
if (counter % 10000 == 0)
{
BingoLog.logMessage("Processing record #{0} with id = {1}",
counter, id);
}
byte[] record_data;
if (cursor[1].GetType() == typeof(byte[]))
record_data = (byte[])cursor[1];
else
record_data = encoding.GetBytes((string)cursor[1]);
BingoCore.lib.bingoSetIndexRecordData(id.Value, record_data, record_data.Length);
return 1;
}
// No records more
return 0;
}
catch (Exception ex)
{
if ((Thread.CurrentThread.ThreadState & ThreadState.AbortRequested) != 0)
Thread.ResetAbort();
if (id.HasValue)
BingoLog.logMessage("Failed on id = {0}", id);
else
BingoLog.logMessage("Exception {0} in {1}:\n{2}",
ex.Message, ex.Source, ex.StackTrace);
BingoCore.lib.bingoIndexMarkTermintate();
exception = ex;
return 0;
}
};
BingoCore.ProcessResultHandler process_result =
(IntPtr context) =>
{
try
{
if (exception != null)
return;
if (insert_records)
{
BingoTimer add_timer = new BingoTimer("index.add_to_index");
if (data.getIndexType() == BingoIndexData.IndexType.Molecule)
_AddMoleculeToIndex(conn, data);
else
_AddReactionToIndex(conn, data);
add_timer.end();
}
}
catch (Exception ex)
{
BingoLog.logMessage("Exception {0} in {1}:\n{2}",
ex.Message, ex.Source, ex.StackTrace);
BingoCore.lib.bingoIndexMarkTermintate();
if ((Thread.CurrentThread.ThreadState & ThreadState.AbortRequested) != 0)
Thread.ResetAbort();
exception = ex;
}
};
BingoCore.ProcessErrorHandler process_error =
(int id_with_error, IntPtr context) =>
{
try
{
if (exception != null)
return;
string message =
String.Format("Record with ID={0} wasn't added to the index: {1}",
id_with_error, BingoCore.lib.bingoGetWarning());
if (SqlContext.Pipe != null)
SqlContext.Pipe.Send(message);
BingoLog.logMessage(message);
if (error_list != null)
error_list.Add(
new FetchedData(id_with_error) { str = BingoCore.lib.bingoGetWarning() });
}
catch (Exception ex)
{
BingoLog.logMessage("Exception {0} in {1}:\n{2}",
ex.Message, ex.Source, ex.StackTrace);
BingoCore.lib.bingoIndexMarkTermintate();
if ((Thread.CurrentThread.ThreadState & ThreadState.AbortRequested) != 0)
Thread.ResetAbort();
exception = ex;
}
};
try
{
int ret = BingoCore.lib.bingoIndexProcess(
data.getIndexType() == BingoIndexData.IndexType.Reaction,
get_next_record, process_result, process_error, IntPtr.Zero);
if (ret == -1)
{
string msg = BingoCore.lib.bingoGetError();
BingoLog.logMessage("Internal exception: {0}", BingoCore.lib.bingoGetError());
throw new Exception(msg);
}
}
catch (Exception ex)
{
// Terminate parallel indexing because it causes unhandled exception if not terminated
// Index termination should be here because function pointers must be valid
BingoCore.lib.bingoIndexEnd();
throw ex;
}
if (exception != null)
throw exception;
}
}
if (flush_and_create_index)
{
data.flush(conn);
BingoTimer indices_timer = new BingoTimer("index.create_indices");
data.createIndices(conn);
indices_timer.end();
BingoTimer fp_indices_timer = new BingoTimer("index.create_fp_indices");
data.fingerprints.createIndices(conn);
fp_indices_timer.end();
data.CreateTriggers(conn);
}
};
return opWithIndex;
}
private void _flushBlock(Block block, SqlConnection conn)
{
BingoTimer timer = new BingoTimer("fingerprints.flush");
if (block.part == -1)
{
// Add new block
int?max_id = BingoSqlUtils.ExecIntQuery(conn,
"SELECT MAX(part) from {0}", _index_data.fingerprintsTable);
if (max_id == null)
{
max_id = 0;
}
block.part = max_id.Value + 1;
BingoSqlUtils.ExecNonQuery(conn, "INSERT INTO {0} values ({1}, 0, null, null)",
_index_data.fingerprintsTable, block.part);
}
BingoLog.logMessage("Flushing fingerprints block {0}...", block.part);
// Update used column and counters column
string update_command_text = String.Format(@"UPDATE {0} SET used = @used,
counters = @counters, mapping = @mapping
where part = {1}", _index_data.fingerprintsTable, block.part);
using (SqlCommand command = new SqlCommand(update_command_text, conn))
{
command.CommandTimeout = 3600;
command.Parameters.AddWithValue("@used", block.indices.Count);
byte[] countes_bytes = new byte[8 * _fp_bytes * sizeof(int)];
Buffer.BlockCopy(block.counters, 0, countes_bytes, 0, countes_bytes.Length);
SqlBinary countes = new SqlBinary(countes_bytes);
command.Parameters.AddWithValue("@counters", countes);
byte[] mapping_bytes = new byte[block.indices.Count * sizeof(int)];
Buffer.BlockCopy(block.indices.ToArray(), 0, mapping_bytes, 0, mapping_bytes.Length);
SqlBinary mapping = new SqlBinary(mapping_bytes);
command.Parameters.AddWithValue("@mapping", mapping);
command.ExecuteNonQuery();
}
// Update bit chunks
BingoSqlUtils.ExecNonQuery(conn, "DELETE FROM {0} WHERE part = {1}",
_index_data.fingerprintBitsTable, block.part);
string update_bits_text = String.Format(@"INSERT INTO {0} VALUES ({1}, @bit, @bit_chunk)",
_index_data.fingerprintBitsTable, block.part);
using (SqlCommand command = new SqlCommand(update_bits_text, conn))
{
command.CommandTimeout = 3600;
command.Parameters.Add("@bit", SqlDbType.Int);
command.Parameters.Add("@bit_chunk", SqlDbType.Binary);
byte[] chunk = new byte[_chunk_bytes];
for (int i = 0; i < 8 * _fp_bytes; i++)
{
command.Parameters["@bit"].Value = i;
Buffer.BlockCopy(block.bits[i], 0, chunk, 0, chunk.Length);
SqlBinary sql_chunk = new SqlBinary(chunk);
command.Parameters["@bit_chunk"].Value = sql_chunk;
command.ExecuteNonQuery();
}
}
block.pending = false;
block.bits = null;
block.indices = null;
BingoLog.logMessage(" Done.");
timer.end();
}
