private static BingoIndexData GetIndexDataById(SqlConnection conn, string bingo_schema,
BingoIndexID id, int spid)
{
lock (index_data_list_lock)
{
foreach (BingoIndexDataRefs index_data_refs in index_data_list)
{
if (index_data_refs.index_data.id.Equals(id))
{
if (!index_data_refs.session_ids.Contains(spid))
{
BingoLog.logMessage("Existing BingoIndexData added for spid={0} table={1}",
spid, id.FullTableName(conn));
index_data_refs.session_ids.Add(spid);
}
return(index_data_refs.index_data);
}
}
BingoLog.logMessage("Extracting new BingoIndexData for spid={0} table={1}",
spid, id.FullTableName(conn));
BingoIndexData data = _extractIndexData(conn, bingo_schema, id, true);
_AddIndexDataToList(spid, data);
return(data);
}
}
public static BingoIndexData CreateIndexData(int spid, SqlConnection conn, string bingo_schema,
string table, string id_column, string data_column, bool reaction)
{
lock (index_data_list_lock)
{
BingoIndexID id = new BingoIndexID(conn, table);
_DropIndexData(conn, bingo_schema, id, false);
BingoIndexData data;
if (reaction)
{
data = new RingoIndexData(id, id_column, data_column, bingo_schema);
}
else
{
data = new MangoIndexData(id, id_column, data_column, bingo_schema);
}
BingoSqlUtils.ExecNonQuery(conn,
"INSERT INTO {0} VALUES({1}, {2}, '{3}', '{4}', '{5}', '{6}')",
_contextTable(bingo_schema), id.table_id, id.database_id, id.FullTableName(conn),
id_column, data_column,
reaction ? "reaction" : "molecule");
data.CreateTables(conn);
_AddIndexDataToList(spid, data);
BingoLog.logMessage("Bingo index for table {0} has been initialized", id.FullTableName(conn));
return(data);
}
}
private static void _DropIndexData(SqlConnection conn, string bingo_schema,
BingoIndexID id, bool throw_if_not_exists)
{
BingoIndexData data = _extractIndexData(conn, bingo_schema, id, throw_if_not_exists);
if (data != null)
{
data.DropTables(conn);
data.DropTriggers(conn);
}
BingoSqlUtils.ExecNonQueryNoThrow(conn, "DELETE FROM {0} WHERE obj_id = '{1}'",
_contextTable(bingo_schema), id.table_id);
for (int i = index_data_list.Count - 1; i >= 0; i--)
{
BingoIndexDataRefs refs = (BingoIndexDataRefs)index_data_list[i];
if (refs.index_data.id.Equals(id))
{
index_data_list.RemoveAt(i);
BingoLog.logMessage("Sessions for table {0} have been released", id.InformationName(conn));
}
}
if (data != null)
{
BingoLog.logMessage("Bingo index for table {0} has been dropped", id.InformationName(conn));
}
}
public void setKeepCache(SqlConnection conn, string bingo_schema, bool keep)
{
if (keep_cache != keep)
{
keep_cache = keep;
BingoConfig.setInt(conn, bingo_schema, "KEEP_CACHE", id.table_id, keep ? 1 : 0);
BingoLog.logMessage("SetKeepCache has changed for {0} table. New value is {1}",
id.FullTableName(conn), keep);
}
}
public static void ExecNonQueryNoThrow(SqlConnection conn, string command, params object[] args)
{
try
{
ExecNonQuery(conn, command, args);
}
catch (SqlException)
{
}
catch (Exception ex)
{
BingoLog.logMessage("Exception in ExecNonQueryNoThrow: {0} in {1}:\n{2}",
ex.Message, ex.Source, ex.StackTrace);
}
}
public static void OnSessionClose(int spid)
{
FlushInAllSessions(spid, true);
lock (index_data_list_lock)
{
for (int i = index_data_list.Count - 1; i >= 0; i--)
{
BingoIndexDataRefs refs = (BingoIndexDataRefs)index_data_list[i];
refs.session_ids.Remove(spid);
if (refs.session_ids.Count < 1 && !refs.index_data.keep_cache)
{
index_data_list.RemoveAt(i);
BingoLog.logMessage("Session for table {0} has been released",
refs.index_data.id.InformationName());
}
}
}
}
private void _readBlockBits(ref Block block, SqlConnection conn)
{
if (block.bits != null)
{
return; // Bits have already been loaded
}
BingoLog.logMessage("_readBlockBits: allocating block {1} buffer {0} size...", 8 * _fp_bytes * _chunk_bytes,
_all_blocks.Count);
block.bits = new List <byte[]>(8 * _fp_bytes);
for (int i = 0; i < 8 * _fp_bytes; i++)
{
block.bits.Add(new byte[_chunk_bytes]);
}
BingoLog.logMessage(" Done.");
string command_text =
String.Format("SELECT bit, bits_chunk from {0} where part = {1}",
_index_data.fingerprintBitsTable, block.part);
using (SqlCommand command = new SqlCommand(command_text, conn))
{
command.CommandTimeout = 3600;
using (SqlDataReader reader = command.ExecuteReader())
{
while (reader.Read())
{
int bit = (int)reader[0];
byte[] bit_chunk = (byte[])reader[1];
if (bit_chunk.Length != _chunk_bytes)
{
throw new Exception("Bits block length is incorrect");
}
bit_chunk.CopyTo(block.bits[bit], 0);
}
}
}
}
private void _validateBlockIndices(SqlConnection conn, Block block)
{
if (block.indices != null)
{
return;
}
lock (_sync_object)
{
if (block.indices != null)
{
return;
}
BingoLog.logMessage("validating fingerprint {0} block indices", block.part);
using (SqlCommand command =
new SqlCommand("SELECT [mapping], [used] from " +
_index_data.fingerprintsTable + " WHERE [part] = " + block.part, conn))
{
command.CommandTimeout = 3600;
using (SqlDataReader reader = command.ExecuteReader())
{
if (!reader.Read())
{
throw new Exception("Cannot read fingerprint " + block.part + " block");
}
byte[] mapping = (byte[])reader[0];
int used = Convert.ToInt32(reader[1]);
// Copy mapping
int[] data = new int[used];
Buffer.BlockCopy(mapping, 0, data, 0, mapping.Length);
block.indices = new List <int>();
block.indices.AddRange(data);
block.validateMinMax();
}
}
}
}
public void flush(SqlConnection conn)
{
lock (_sync_object)
{
for (int i = 0; i < _blocks.Count; i++)
{
_Block b = _blocks[i];
if (b != null)
{
if (b.dirty)
{
_flushBlock(conn, b);
BingoLog.logMessage("Disposing memory for block {0}...", b.block_index);
_blocks[i] = null;
BingoLog.logMessage(" Done");
}
}
}
}
}
private void _validateBlock(short block_index, SqlConnection conn)
{
if (_blockLoaded(block_index))
{
return;
}
lock (_sync_object)
{
// Double-checked locking
if (_blockLoaded(block_index))
{
return;
}
BingoTimer timer = new BingoTimer("storage.validate_block");
BingoLog.logMessage("Loading storage block {0} for table {1}...",
block_index, _index_data.id.InformationName());
string text = "SELECT [data], [first_index], [offsets], [lengths], [count] from " +
_index_data.storageTable + " where id = " + block_index;
using (SqlCommand command = new SqlCommand(text, conn))
{
command.CommandTimeout = 3600;
using (SqlDataReader reader = command.ExecuteReader())
{
if (!reader.Read())
{
throw new Exception("Block cannot be found");
}
while (_blocks.Count <= block_index)
{
_blocks.Add(new _Block());
}
if (_blocks[block_index] == null)
{
_blocks[block_index] = new _Block();
}
_Block block = _blocks[block_index];
block.block_index = block_index;
block.data = (byte[])reader["data"];
block.first_index = (int)reader["first_index"];
int count = (int)reader["count"];
block.offsets = new int[count];
MemoryStream mem_stream = new MemoryStream((byte[])reader["offsets"]);
BinaryReader bin_reader = new BinaryReader(mem_stream);
for (int i = 0; i < count; i++)
{
block.offsets[i] = bin_reader.ReadInt32();
}
block.lengths = new short[count];
mem_stream = new MemoryStream((byte[])reader["lengths"]);
bin_reader = new BinaryReader(mem_stream);
for (int i = 0; i < count; i++)
{
block.lengths[i] = bin_reader.ReadInt16();
}
block.end_index = block.first_index + count;
}
}
BingoLog.logMessage(" Done.");
timer.end();
}
}
private void _flushBlock(SqlConnection conn, _Block b)
{
BingoLog.logMessage("Flushing storage block {0}", b.block_index);
BingoSqlUtils.ExecNonQueryNoThrow(conn, "DELETE FROM {0} WHERE id={1}",
_index_data.storageTable, b.block_index);
string text = String.Format(@"INSERT INTO {0}
([id], [first_index], [count], [offsets], [lengths], [data])
VALUES ({1}, {2}, {3}, @offsets, @lengths, @data)",
_index_data.storageTable, b.block_index, b.first_index, b.end_index - b.first_index);
using (SqlCommand command = new SqlCommand(text, conn))
{
command.CommandTimeout = 3600;
byte[] data;
if (b.pending_data != null)
{
data = b.pending_data.ToArray();
}
else
{
data = b.data;
}
SqlBinary binary_data = new SqlBinary(data);
command.Parameters.AddWithValue("@data", binary_data);
ICollection <int> offsets;
if (b.pending_offsets != null)
{
offsets = b.pending_offsets;
}
else
{
offsets = b.offsets;
}
MemoryStream mem_stream = new MemoryStream(offsets.Count * 4);
BinaryWriter writer = new BinaryWriter(mem_stream);
foreach (int offset in offsets)
{
writer.Write(offset);
}
byte[] buffer = mem_stream.GetBuffer();
command.Parameters.AddWithValue("@offsets", buffer);
ICollection <short> lengths;
if (b.pending_lengths != null)
{
lengths = b.pending_lengths;
}
else
{
lengths = b.lengths;
}
mem_stream = new MemoryStream(offsets.Count * 2);
writer = new BinaryWriter(mem_stream);
foreach (short length in lengths)
{
writer.Write(length);
}
buffer = mem_stream.GetBuffer();
command.Parameters.AddWithValue("@lengths", buffer);
command.ExecuteNonQuery();
}
_convertPendingBlockToNormal(b);
}
private void _flushBlock(Block block, SqlConnection conn)
{
BingoTimer timer = new BingoTimer("fingerprints.flush");
if (block.part == -1)
{
// Add new block
int?max_id = BingoSqlUtils.ExecIntQuery(conn,
"SELECT MAX(part) from {0}", _index_data.fingerprintsTable);
if (max_id == null)
{
max_id = 0;
}
block.part = max_id.Value + 1;
BingoSqlUtils.ExecNonQuery(conn, "INSERT INTO {0} values ({1}, 0, null, null)",
_index_data.fingerprintsTable, block.part);
}
BingoLog.logMessage("Flushing fingerprints block {0}...", block.part);
// Update used column and counters column
string update_command_text = String.Format(@"UPDATE {0} SET used = @used,
counters = @counters, mapping = @mapping
where part = {1}", _index_data.fingerprintsTable, block.part);
using (SqlCommand command = new SqlCommand(update_command_text, conn))
{
command.CommandTimeout = 3600;
command.Parameters.AddWithValue("@used", block.indices.Count);
byte[] countes_bytes = new byte[8 * _fp_bytes * sizeof(int)];
Buffer.BlockCopy(block.counters, 0, countes_bytes, 0, countes_bytes.Length);
SqlBinary countes = new SqlBinary(countes_bytes);
command.Parameters.AddWithValue("@counters", countes);
byte[] mapping_bytes = new byte[block.indices.Count * sizeof(int)];
Buffer.BlockCopy(block.indices.ToArray(), 0, mapping_bytes, 0, mapping_bytes.Length);
SqlBinary mapping = new SqlBinary(mapping_bytes);
command.Parameters.AddWithValue("@mapping", mapping);
command.ExecuteNonQuery();
}
// Update bit chunks
BingoSqlUtils.ExecNonQuery(conn, "DELETE FROM {0} WHERE part = {1}",
_index_data.fingerprintBitsTable, block.part);
string update_bits_text = String.Format(@"INSERT INTO {0} VALUES ({1}, @bit, @bit_chunk)",
_index_data.fingerprintBitsTable, block.part);
using (SqlCommand command = new SqlCommand(update_bits_text, conn))
{
command.CommandTimeout = 3600;
command.Parameters.Add("@bit", SqlDbType.Int);
command.Parameters.Add("@bit_chunk", SqlDbType.Binary);
byte[] chunk = new byte[_chunk_bytes];
for (int i = 0; i < 8 * _fp_bytes; i++)
{
command.Parameters["@bit"].Value = i;
Buffer.BlockCopy(block.bits[i], 0, chunk, 0, chunk.Length);
SqlBinary sql_chunk = new SqlBinary(chunk);
command.Parameters["@bit_chunk"].Value = sql_chunk;
command.ExecuteNonQuery();
}
}
block.pending = false;
block.bits = null;
block.indices = null;
BingoLog.logMessage(" Done.");
timer.end();
}