Ejemplo n.º 1
0
        private static BingoIndexData GetIndexDataById(SqlConnection conn, string bingo_schema,
                                                       BingoIndexID id, int spid)
        {
            lock (index_data_list_lock)
            {
                foreach (BingoIndexDataRefs index_data_refs in index_data_list)
                {
                    if (index_data_refs.index_data.id.Equals(id))
                    {
                        if (!index_data_refs.session_ids.Contains(spid))
                        {
                            BingoLog.logMessage("Existing BingoIndexData added for spid={0} table={1}",
                                                spid, id.FullTableName(conn));
                            index_data_refs.session_ids.Add(spid);
                        }
                        return(index_data_refs.index_data);
                    }
                }

                BingoLog.logMessage("Extracting new BingoIndexData for spid={0} table={1}",
                                    spid, id.FullTableName(conn));

                BingoIndexData data = _extractIndexData(conn, bingo_schema, id, true);
                _AddIndexDataToList(spid, data);

                return(data);
            }
        }
Ejemplo n.º 2
0
        public static BingoIndexData CreateIndexData(int spid, SqlConnection conn, string bingo_schema,
                                                     string table, string id_column, string data_column, bool reaction)
        {
            lock (index_data_list_lock)
            {
                BingoIndexID id = new BingoIndexID(conn, table);
                _DropIndexData(conn, bingo_schema, id, false);

                BingoIndexData data;

                if (reaction)
                {
                    data = new RingoIndexData(id, id_column, data_column, bingo_schema);
                }
                else
                {
                    data = new MangoIndexData(id, id_column, data_column, bingo_schema);
                }

                BingoSqlUtils.ExecNonQuery(conn,
                                           "INSERT INTO {0} VALUES({1}, {2}, '{3}', '{4}', '{5}', '{6}')",
                                           _contextTable(bingo_schema), id.table_id, id.database_id, id.FullTableName(conn),
                                           id_column, data_column,
                                           reaction ? "reaction" : "molecule");

                data.CreateTables(conn);
                _AddIndexDataToList(spid, data);

                BingoLog.logMessage("Bingo index for table {0} has been initialized", id.FullTableName(conn));

                return(data);
            }
        }
Ejemplo n.º 3
0
        private static void _DropIndexData(SqlConnection conn, string bingo_schema,
                                           BingoIndexID id, bool throw_if_not_exists)
        {
            BingoIndexData data = _extractIndexData(conn, bingo_schema, id, throw_if_not_exists);

            if (data != null)
            {
                data.DropTables(conn);
                data.DropTriggers(conn);
            }

            BingoSqlUtils.ExecNonQueryNoThrow(conn, "DELETE FROM {0} WHERE obj_id = '{1}'",
                                              _contextTable(bingo_schema), id.table_id);

            for (int i = index_data_list.Count - 1; i >= 0; i--)
            {
                BingoIndexDataRefs refs = (BingoIndexDataRefs)index_data_list[i];
                if (refs.index_data.id.Equals(id))
                {
                    index_data_list.RemoveAt(i);
                    BingoLog.logMessage("Sessions for table {0} have been released", id.InformationName(conn));
                }
            }
            if (data != null)
            {
                BingoLog.logMessage("Bingo index for table {0} has been dropped", id.InformationName(conn));
            }
        }
Ejemplo n.º 4
0
        public void setKeepCache(SqlConnection conn, string bingo_schema, bool keep)
        {
            if (keep_cache != keep)
            {
                keep_cache = keep;
                BingoConfig.setInt(conn, bingo_schema, "KEEP_CACHE", id.table_id, keep ? 1 : 0);

                BingoLog.logMessage("SetKeepCache has changed for {0} table. New value is {1}",
                                    id.FullTableName(conn), keep);
            }
        }
Ejemplo n.º 5
0
 public static void ExecNonQueryNoThrow(SqlConnection conn, string command, params object[] args)
 {
     try
     {
         ExecNonQuery(conn, command, args);
     }
     catch (SqlException)
     {
     }
     catch (Exception ex)
     {
         BingoLog.logMessage("Exception in ExecNonQueryNoThrow: {0} in {1}:\n{2}",
                             ex.Message, ex.Source, ex.StackTrace);
     }
 }
Ejemplo n.º 6
0
        public static void OnSessionClose(int spid)
        {
            FlushInAllSessions(spid, true);
            lock (index_data_list_lock)
            {
                for (int i = index_data_list.Count - 1; i >= 0; i--)
                {
                    BingoIndexDataRefs refs = (BingoIndexDataRefs)index_data_list[i];

                    refs.session_ids.Remove(spid);
                    if (refs.session_ids.Count < 1 && !refs.index_data.keep_cache)
                    {
                        index_data_list.RemoveAt(i);
                        BingoLog.logMessage("Session for table {0} has been released",
                                            refs.index_data.id.InformationName());
                    }
                }
            }
        }
Ejemplo n.º 7
0
        private void _readBlockBits(ref Block block, SqlConnection conn)
        {
            if (block.bits != null)
            {
                return; // Bits have already been loaded
            }
            BingoLog.logMessage("_readBlockBits: allocating block {1} buffer {0} size...", 8 * _fp_bytes * _chunk_bytes,
                                _all_blocks.Count);

            block.bits = new List <byte[]>(8 * _fp_bytes);
            for (int i = 0; i < 8 * _fp_bytes; i++)
            {
                block.bits.Add(new byte[_chunk_bytes]);
            }

            BingoLog.logMessage("  Done.");

            string command_text =
                String.Format("SELECT bit, bits_chunk from {0} where part = {1}",
                              _index_data.fingerprintBitsTable, block.part);

            using (SqlCommand command = new SqlCommand(command_text, conn))
            {
                command.CommandTimeout = 3600;
                using (SqlDataReader reader = command.ExecuteReader())
                {
                    while (reader.Read())
                    {
                        int    bit       = (int)reader[0];
                        byte[] bit_chunk = (byte[])reader[1];

                        if (bit_chunk.Length != _chunk_bytes)
                        {
                            throw new Exception("Bits block length is incorrect");
                        }

                        bit_chunk.CopyTo(block.bits[bit], 0);
                    }
                }
            }
        }
Ejemplo n.º 8
0
        private void _validateBlockIndices(SqlConnection conn, Block block)
        {
            if (block.indices != null)
            {
                return;
            }

            lock (_sync_object)
            {
                if (block.indices != null)
                {
                    return;
                }

                BingoLog.logMessage("validating fingerprint {0} block indices", block.part);

                using (SqlCommand command =
                           new SqlCommand("SELECT [mapping], [used] from " +
                                          _index_data.fingerprintsTable + " WHERE [part] = " + block.part, conn))
                {
                    command.CommandTimeout = 3600;
                    using (SqlDataReader reader = command.ExecuteReader())
                    {
                        if (!reader.Read())
                        {
                            throw new Exception("Cannot read fingerprint " + block.part + " block");
                        }

                        byte[] mapping = (byte[])reader[0];
                        int    used    = Convert.ToInt32(reader[1]);

                        // Copy mapping
                        int[] data = new int[used];
                        Buffer.BlockCopy(mapping, 0, data, 0, mapping.Length);
                        block.indices = new List <int>();
                        block.indices.AddRange(data);
                        block.validateMinMax();
                    }
                }
            }
        }
Ejemplo n.º 9
0
        public void flush(SqlConnection conn)
        {
            lock (_sync_object)
            {
                for (int i = 0; i < _blocks.Count; i++)
                {
                    _Block b = _blocks[i];
                    if (b != null)
                    {
                        if (b.dirty)
                        {
                            _flushBlock(conn, b);

                            BingoLog.logMessage("Disposing memory for block {0}...", b.block_index);
                            _blocks[i] = null;
                            BingoLog.logMessage("   Done");
                        }
                    }
                }
            }
        }
Ejemplo n.º 10
0
        private void _validateBlock(short block_index, SqlConnection conn)
        {
            if (_blockLoaded(block_index))
            {
                return;
            }

            lock (_sync_object)
            {
                // Double-checked locking
                if (_blockLoaded(block_index))
                {
                    return;
                }

                BingoTimer timer = new BingoTimer("storage.validate_block");

                BingoLog.logMessage("Loading storage block {0} for table {1}...",
                                    block_index, _index_data.id.InformationName());

                string text = "SELECT [data], [first_index], [offsets], [lengths], [count]  from " +
                              _index_data.storageTable + " where id = " + block_index;
                using (SqlCommand command = new SqlCommand(text, conn))
                {
                    command.CommandTimeout = 3600;
                    using (SqlDataReader reader = command.ExecuteReader())
                    {
                        if (!reader.Read())
                        {
                            throw new Exception("Block cannot be found");
                        }

                        while (_blocks.Count <= block_index)
                        {
                            _blocks.Add(new _Block());
                        }

                        if (_blocks[block_index] == null)
                        {
                            _blocks[block_index] = new _Block();
                        }

                        _Block block = _blocks[block_index];
                        block.block_index = block_index;
                        block.data        = (byte[])reader["data"];
                        block.first_index = (int)reader["first_index"];

                        int count = (int)reader["count"];

                        block.offsets = new int[count];
                        MemoryStream mem_stream = new MemoryStream((byte[])reader["offsets"]);
                        BinaryReader bin_reader = new BinaryReader(mem_stream);
                        for (int i = 0; i < count; i++)
                        {
                            block.offsets[i] = bin_reader.ReadInt32();
                        }

                        block.lengths = new short[count];
                        mem_stream    = new MemoryStream((byte[])reader["lengths"]);
                        bin_reader    = new BinaryReader(mem_stream);
                        for (int i = 0; i < count; i++)
                        {
                            block.lengths[i] = bin_reader.ReadInt16();
                        }

                        block.end_index = block.first_index + count;
                    }
                }
                BingoLog.logMessage("   Done.");

                timer.end();
            }
        }
Ejemplo n.º 11
0
        private void _flushBlock(SqlConnection conn, _Block b)
        {
            BingoLog.logMessage("Flushing storage block {0}", b.block_index);

            BingoSqlUtils.ExecNonQueryNoThrow(conn, "DELETE FROM {0} WHERE id={1}",
                                              _index_data.storageTable, b.block_index);
            string text = String.Format(@"INSERT INTO {0} 
            ([id], [first_index], [count], [offsets], [lengths], [data]) 
            VALUES ({1}, {2}, {3}, @offsets, @lengths, @data)",
                                        _index_data.storageTable, b.block_index, b.first_index, b.end_index - b.first_index);

            using (SqlCommand command = new SqlCommand(text, conn))
            {
                command.CommandTimeout = 3600;

                byte[] data;
                if (b.pending_data != null)
                {
                    data = b.pending_data.ToArray();
                }
                else
                {
                    data = b.data;
                }
                SqlBinary binary_data = new SqlBinary(data);
                command.Parameters.AddWithValue("@data", binary_data);

                ICollection <int> offsets;
                if (b.pending_offsets != null)
                {
                    offsets = b.pending_offsets;
                }
                else
                {
                    offsets = b.offsets;
                }

                MemoryStream mem_stream = new MemoryStream(offsets.Count * 4);
                BinaryWriter writer     = new BinaryWriter(mem_stream);
                foreach (int offset in offsets)
                {
                    writer.Write(offset);
                }
                byte[] buffer = mem_stream.GetBuffer();
                command.Parameters.AddWithValue("@offsets", buffer);

                ICollection <short> lengths;
                if (b.pending_lengths != null)
                {
                    lengths = b.pending_lengths;
                }
                else
                {
                    lengths = b.lengths;
                }

                mem_stream = new MemoryStream(offsets.Count * 2);
                writer     = new BinaryWriter(mem_stream);
                foreach (short length in lengths)
                {
                    writer.Write(length);
                }
                buffer = mem_stream.GetBuffer();
                command.Parameters.AddWithValue("@lengths", buffer);

                command.ExecuteNonQuery();
            }
            _convertPendingBlockToNormal(b);
        }
Ejemplo n.º 12
0
        private void _flushBlock(Block block, SqlConnection conn)
        {
            BingoTimer timer = new BingoTimer("fingerprints.flush");

            if (block.part == -1)
            {
                // Add new block
                int?max_id = BingoSqlUtils.ExecIntQuery(conn,
                                                        "SELECT MAX(part) from {0}", _index_data.fingerprintsTable);
                if (max_id == null)
                {
                    max_id = 0;
                }

                block.part = max_id.Value + 1;

                BingoSqlUtils.ExecNonQuery(conn, "INSERT INTO {0} values ({1}, 0, null, null)",
                                           _index_data.fingerprintsTable, block.part);
            }

            BingoLog.logMessage("Flushing fingerprints block {0}...", block.part);

            // Update used column and counters column
            string update_command_text = String.Format(@"UPDATE {0} SET used = @used, 
                              counters = @counters, mapping = @mapping
                              where part = {1}", _index_data.fingerprintsTable, block.part);

            using (SqlCommand command = new SqlCommand(update_command_text, conn))
            {
                command.CommandTimeout = 3600;
                command.Parameters.AddWithValue("@used", block.indices.Count);

                byte[] countes_bytes = new byte[8 * _fp_bytes * sizeof(int)];
                Buffer.BlockCopy(block.counters, 0, countes_bytes, 0, countes_bytes.Length);

                SqlBinary countes = new SqlBinary(countes_bytes);
                command.Parameters.AddWithValue("@counters", countes);

                byte[] mapping_bytes = new byte[block.indices.Count * sizeof(int)];
                Buffer.BlockCopy(block.indices.ToArray(), 0, mapping_bytes, 0, mapping_bytes.Length);

                SqlBinary mapping = new SqlBinary(mapping_bytes);
                command.Parameters.AddWithValue("@mapping", mapping);

                command.ExecuteNonQuery();
            }

            // Update bit chunks
            BingoSqlUtils.ExecNonQuery(conn, "DELETE FROM {0} WHERE part = {1}",
                                       _index_data.fingerprintBitsTable, block.part);

            string update_bits_text = String.Format(@"INSERT INTO {0} VALUES ({1}, @bit, @bit_chunk)",
                                                    _index_data.fingerprintBitsTable, block.part);

            using (SqlCommand command = new SqlCommand(update_bits_text, conn))
            {
                command.CommandTimeout = 3600;
                command.Parameters.Add("@bit", SqlDbType.Int);
                command.Parameters.Add("@bit_chunk", SqlDbType.Binary);

                byte[] chunk = new byte[_chunk_bytes];

                for (int i = 0; i < 8 * _fp_bytes; i++)
                {
                    command.Parameters["@bit"].Value = i;

                    Buffer.BlockCopy(block.bits[i], 0, chunk, 0, chunk.Length);

                    SqlBinary sql_chunk = new SqlBinary(chunk);
                    command.Parameters["@bit_chunk"].Value = sql_chunk;

                    command.ExecuteNonQuery();
                }
            }

            block.pending = false;
            block.bits    = null;
            block.indices = null;

            BingoLog.logMessage("  Done.");
            timer.end();
        }