internal override void AddPartition(int partitionCount) { int prePartitionCount = this.PartitionCount; RedisConnectionPool clientPool = null; if (this.redisVersionDbMode == RedisVersionDbMode.Cluster) { clientPool = this.singletonConnPool; } Array.Resize(ref this.tableVisitors, partitionCount); Array.Resize(ref this.requestQueues, partitionCount); Array.Resize(ref this.flushQueues, partitionCount); Array.Resize(ref this.queueLatches, partitionCount); for (int pk = prePartitionCount; pk < partitionCount; pk++) { if (clientPool == null) { clientPool = this.RedisManager.GetClientPool( this.redisDbIndex, RedisVersionDb.GetRedisInstanceIndex(pk)); } this.tableVisitors[pk] = new RedisVersionTableVisitor( clientPool, this.LuaManager, this.responseVisitor, this.redisVersionDbMode); this.requestQueues[pk] = new Queue <VersionEntryRequest>(VersionDb.REQUEST_QUEUE_CAPACITY); this.flushQueues[pk] = new Queue <VersionEntryRequest>(VersionDb.REQUEST_QUEUE_CAPACITY); this.queueLatches[pk] = 0; } base.AddPartition(partitionCount); }
/// <summary> /// Start a new redis client pool, which will create an instance of RedisClientPool /// and start a new daemon thread to send requests and collect results /// </summary> /// <param name="redisDbIndex"></param> private void StartNewRedisClientPool(RedisConnectionKey key) { RedisConnectionPool pool = new RedisConnectionPool( this.readWriteHosts[key.RedisInstanceIndex], key.RedisDbIndex, this.redisLuaScriptManager); this.clientPools.Add(key, pool); // If the redis version db in pipeline mode, start a daemon thread // if (key.RedisDbIndex != 0) // { // Thread t = new Thread(new ThreadStart(pool.Monitor)); // t.Start(); // } }
public RedisVersionDbVisitor( RedisConnectionPool clientPool, RedisLuaScriptManager redisLuaScriptManager, RedisResponseVisitor redisResponseVisitor, RedisVersionDbMode mode) { this.clientPool = clientPool; this.redisClient = clientPool.GetRedisClient(); this.redisLuaScriptManager = redisLuaScriptManager; this.RedisResponseVisitor = redisResponseVisitor; this.redisVersionDbMode = mode; this.redisRequests = new List <RedisRequest>(); this.reqIndex = 0; }
private RedisVersionDb(int partitionCount, string[] readWriteHosts, RedisVersionDbMode mode = RedisVersionDbMode.Cluster) : base(partitionCount) { this.PartitionCount = partitionCount; if (readWriteHosts == null) { throw new ArgumentException("readWriteHosts must be a null array"); } this.readWriteHosts = readWriteHosts; this.Mode = mode; this.tableLock = new object(); this.responseVisitor = new RedisResponseVisitor(); this.Setup(); this.txEntryRequestQueues = new Queue <TxEntryRequest> [partitionCount]; this.flushQueues = new Queue <TxEntryRequest> [partitionCount]; this.queueLatches = new int[partitionCount]; for (int pid = 0; pid < this.PartitionCount; pid++) { RedisConnectionPool clientPool = null; if (this.Mode == RedisVersionDbMode.Cluster) { clientPool = SingletonConnPool; } else { clientPool = this.RedisManager.GetClientPool( RedisVersionDb.TX_DB_INDEX, RedisVersionDb.GetRedisInstanceIndex(pid)); } this.dbVisitors[pid] = new RedisVersionDbVisitor( clientPool, this.RedisLuaManager, this.responseVisitor, this.Mode); this.txEntryRequestQueues[pid] = new Queue <TxEntryRequest>(VersionDb.REQUEST_QUEUE_CAPACITY); this.flushQueues[pid] = new Queue <TxEntryRequest>(VersionDb.REQUEST_QUEUE_CAPACITY); this.queueLatches[pid] = 0; } }
public RedisVersionTable(VersionDb versionDb, string tableId, long redisDbIndex) : base(versionDb, tableId, versionDb.PartitionCount) { this.redisDbIndex = redisDbIndex; this.responseVisitor = new RedisResponseVisitor(); this.redisVersionDb = ((RedisVersionDb)this.VersionDb); this.RedisManager = redisVersionDb.RedisManager; this.LuaManager = redisVersionDb.RedisLuaManager; this.singletonConnPool = redisVersionDb.SingletonConnPool; this.redisVersionDbMode = redisVersionDb.Mode; this.requestQueues = new Queue <VersionEntryRequest> [this.PartitionCount]; this.flushQueues = new Queue <VersionEntryRequest> [this.PartitionCount]; this.queueLatches = new int[this.PartitionCount]; RedisConnectionPool clientPool = null; for (int pid = 0; pid < this.PartitionCount; pid++) { if (this.redisVersionDbMode == RedisVersionDbMode.Cluster) { clientPool = this.singletonConnPool; } else { clientPool = this.RedisManager.GetClientPool( this.redisDbIndex, RedisVersionDb.GetRedisInstanceIndex(pid)); } this.tableVisitors[pid] = new RedisVersionTableVisitor( clientPool, this.LuaManager, this.responseVisitor, this.redisVersionDbMode); this.requestQueues[pid] = new Queue <VersionEntryRequest>(VersionDb.REQUEST_QUEUE_CAPACITY); this.flushQueues[pid] = new Queue <VersionEntryRequest>(VersionDb.REQUEST_QUEUE_CAPACITY); this.queueLatches[pid] = 0; } }
/// <summary> /// Only for debug, the reason why we need such a method in YCSBBenchmarkTest/GetCurrentCommandCount /// </summary> /// <param name="redisDbIndex"></param> /// <param name="partition"></param> /// <returns></returns> internal RedisClient GetLastestClient(long redisDbIndex, int partition) { RedisConnectionPool pool = new RedisConnectionPool(this.readWriteHosts[partition], redisDbIndex, this.redisLuaScriptManager); return(pool.GetRedisClient()); }
/// <summary> /// Reshuffle records in the version db, which will recalculate the partition of record keys, /// and put them to right partitions under the new number of partition /// </summary> /// <param name="perPartitionCount">The number of partitions before adding partitions</param> /// <param name="partitionCount">The current number of partitions</param> private void ReshuffleRecords(int prePartitionCount, int partitionCount) { List <Tuple <byte[], byte[][]> >[] reshuffledRecords = new List <Tuple <byte[], byte[][]> > [partitionCount]; for (int npk = 0; npk < partitionCount; npk++) { reshuffledRecords[npk] = new List <Tuple <byte[], byte[][]> >(); } RedisConnectionPool connPool = null; if (this.redisVersionDbMode == RedisVersionDbMode.Cluster) { connPool = this.singletonConnPool; } for (int pk = 0; pk < prePartitionCount; pk++) { if (connPool == null) { connPool = this.RedisManager.GetClientPool(this.redisDbIndex, RedisVersionDb.GetRedisInstanceIndex(pk)); } using (RedisClient redisClient = connPool.GetRedisClient()) { byte[][] keys = redisClient.Keys("*"); foreach (byte[] key in keys) { byte[][] values = redisClient.HGetAll(Encoding.ASCII.GetString(key)); string recordKey = Encoding.ASCII.GetString(key); // TODO: Only For Benchmark Test as the recordKey is an integer int intRecordKey = int.Parse(recordKey); int npk = this.VersionDb.PhysicalPartitionByKey(intRecordKey); reshuffledRecords[npk].Add(Tuple.Create(key, values)); } redisClient.FlushDb(); } } // flush the redis db for (int pk = 0; pk < partitionCount; pk++) { using (RedisClient redisClient = this.RedisManager.GetClient( this.redisDbIndex, RedisVersionDb.GetRedisInstanceIndex(pk))) { redisClient.FlushDb(); } } Console.WriteLine("Reshuffled Records into Memory"); for (int pk = 0; pk < partitionCount; pk++) { Console.WriteLine("Reshuffled Partition {0}", pk); List <Tuple <byte[], byte[][]> > records = reshuffledRecords[pk]; using (RedisClient redisClient = this.RedisManager.GetClient( this.redisDbIndex, RedisVersionDb.GetRedisInstanceIndex(pk))) { foreach (Tuple <byte[], byte[][]> versions in records) { string hashId = Encoding.ASCII.GetString(versions.Item1); byte[][] keys = new byte[versions.Item2.Length / 2][]; byte[][] values = new byte[versions.Item2.Length / 2][]; for (int i = 0; 2 * i < versions.Item2.Length; i++) { keys[i] = versions.Item2[2 * i]; values[i] = versions.Item2[2 * i + 1]; } redisClient.HMSet(hashId, keys, values); } } } }
internal override void MockLoadData(int recordCount) { int pk = 0; RedisConnectionPool connPool = null; if (this.redisVersionDbMode == RedisVersionDbMode.Cluster) { connPool = this.singletonConnPool; } int loaded = 0; while (pk < this.PartitionCount) { Console.WriteLine("Loading Partition {0}", pk); if (connPool == null) { connPool = this.RedisManager.GetClientPool(this.redisDbIndex, RedisVersionDb.GetRedisInstanceIndex(pk)); } using (RedisClient redisClient = connPool.GetRedisClient()) { int batchSize = 100; int partitions = this.PartitionCount; for (int i = pk; i < recordCount; i += partitions * batchSize) { int upperBound = Math.Min(recordCount, i + partitions * batchSize); using (IRedisPipeline pipe = redisClient.CreatePipeline()) { for (int j = i; j < upperBound; j += partitions) { object recordKey = j; string hashId = recordKey.ToString(); if (this.redisVersionDbMode == RedisVersionDbMode.Cluster) { hashId = RedisVersionDb.PACK_KEY(RedisVersionDb.VER_KEY_PREFIX, hashId); } VersionEntry versionEntry = new VersionEntry(); VersionEntry.InitFirstVersionEntry(new String('a', 100), versionEntry); byte[] key = BitConverter.GetBytes(VersionEntry.VERSION_KEY_START_INDEX + 1); byte[] value = VersionEntry.Serialize(versionEntry); pipe.QueueCommand(r => ((RedisNativeClient)r).HSet(hashId, key, value)); pipe.QueueCommand(r => ((RedisNativeClient)r).HSet(hashId, RedisVersionDb.LATEST_VERSION_PTR_FIELD, key)); loaded++; } pipe.Flush(); } } } pk++; if (this.redisVersionDbMode != RedisVersionDbMode.Cluster) { connPool = null; } } Console.WriteLine("Loaded {0} records Successfully", loaded); }