Пример #1
0
 public DataLoadWorker(VersionDb db, int start, int end, YCSBAsyncBenchmarkTest test)
 {
     this.db    = db;
     this.start = start;
     this.end   = end;
     this.test  = test;
 }
Пример #2
0
        public void TestInit()
        {
            // RedisVersionDb.Instance.PipelineMode = true;
            //this.versionDb = RedisVersionDb.Instance;
            //this.clientManager = ((RedisVersionDb)this.versionDb).RedisManager;

            //int partition = versionDb.PhysicalPartitionByKey(DEFAULT_KEY);
            //using (RedisClient redisClient = (RedisClient)this.clientManager.GetClient(TEST_REDIS_DB, partition))
            //{
            //    // 1. flush the test db
            //    redisClient.ChangeDb(TEST_REDIS_DB);
            //    redisClient.FlushDb();

            //    // 2. create version table, if table is null, it means the table with the same tableId has existed
            //    VersionTable table = this.versionDb.CreateVersionTable(TABLE_ID, TEST_REDIS_DB);

            //    // 3. load data
            //    Transaction tx = new Transaction(null, this.versionDb);
            //    tx.ReadAndInitialize(TABLE_ID, DEFAULT_KEY);
            //    tx.Insert(TABLE_ID, DEFAULT_KEY, DEFAULT_VALUE);
            //    tx.Commit();
            //}
            this.versionDb = CassandraVersionDb.Instance(4);
            VersionTable table = this.versionDb.CreateVersionTable(TABLE_ID, TEST_REDIS_DB);
            Transaction  tx    = new Transaction(null, this.versionDb);

            tx.ReadAndInitialize(TABLE_ID, DEFAULT_KEY);
            tx.Insert(TABLE_ID, DEFAULT_KEY, DEFAULT_VALUE);
            tx.Commit();
        }
Пример #3
0
        public YCSBAsyncBenchmarkTest(
            int recordCount,
            int executorCount,
            int txCountPerExecutor,
            VersionDb versionDb,
            string[] flushTables       = null,
            BenchmarkTestConfig config = null)
        {
            this.versionDb          = versionDb;
            this.recordCount        = recordCount;
            this.executorCount      = executorCount;
            this.txCountPerExecutor = txCountPerExecutor;
            this.executorList       = new List <TransactionExecutor>();
            this.totalTasks         = 0;
            this.tables             = flushTables;

            if (config == null)
            {
                config = new BenchmarkTestConfig();
            }
            this.config = config;

            Console.WriteLine("Current Benchmark Test Config: {0}", this.config.ToString());
            Console.WriteLine("Wait for 5 seconds to confirm");
            Console.Out.Flush();
            //Thread.Sleep(5000);
        }
 static internal TransactionExecution MakeTxExec(
     this VersionDb versionDb, int workerId = 0)
 {
     return(new TransactionExecution(
                null, versionDb, null, new TxRange(workerId),
                workerId % versionDb.PartitionCount));
 }
Пример #5
0
        internal void load(VersionDb db, int threadCount, int totalRecord)
        {
            int           interval   = totalRecord / threadCount;
            List <Thread> threadList = new List <Thread>();

            int[] count = new int[threadCount];
            List <DataLoadWorker> loadList = new List <DataLoadWorker>();

            for (int i = 0; i < threadCount; i++)
            {
                DataLoadWorker load = new DataLoadWorker(db, interval * i,
                                                         Math.Min(interval * (i + 1), totalRecord), this);
                Thread t = new Thread(new ThreadStart(load.load));
                loadList.Add(load);
                threadList.Add(t);
                t.Start();
            }

            long lastTime = DateTime.Now.Ticks;
            long now;
            int  lastSum = 0;
            int  sum     = 0;
            int  thread  = 0;

            foreach (DataLoadWorker loader in loadList)
            {
                lastSum += loader.getCount();
            }

            bool isFinished = false;

            while (!isFinished)
            {
                Thread.Sleep(1000);
                sum        = 0;
                isFinished = true;
                thread     = 0;
                foreach (DataLoadWorker loader in loadList)
                {
                    sum += loader.getCount();
                    if (!loader.getIsFinished())
                    {
                        thread++;
                        isFinished = false;
                    }
                }
                now = DateTime.Now.Ticks;
                long runSeconds = (now - lastTime) / 10000000;
                Console.WriteLine("Time Inteval : {0}, Total Finish Count: {1}, Throughput: {2} txn/s, thread : {3}",
                                  runSeconds, sum, (sum - lastSum) / runSeconds, thread);
                lastSum  = sum;
                lastTime = now;
            }

            foreach (Thread t in threadList)
            {
                t.Join();
            }
        }
Пример #6
0
        TransactionExecution MakeSimpleExecution(
            VersionDb versionDb, int workerId)
        {
            int visitorId = workerId % versionDb.PartitionCount;

            return(new TransactionExecution(
                       null, versionDb, null, new TxRange(workerId), visitorId));
        }
Пример #7
0
        internal void loadWithOneThread(VersionDb db, int start, int end, ref int sum)
        {
            int batchSize = 1;

            for (int i = start; i < end; i += batchSize)
            {
                sum += batchInsert(db, i, Math.Min(end, i + batchSize));
            }
        }
        public YcsbBenchmarkEnv(
            VersionDb versionDb, Func <TransactionExecution, YcsbWorker> workerFactory)
        {
            var txExecs = SingletonYCSBTxExecHelper.MakeTxExecs(versionDb);

            this.workers = txExecs.AsParallel()
                           .Select(workerFactory)
                           .ToArray();
        }
        TransactionExecution[] MakeTxExecs(VersionDb versionDb)
        {
            var txExecs = new TransactionExecution[versionDb.PartitionCount];

            for (int i = 0; i < versionDb.PartitionCount; ++i)
            {
                txExecs[i] = versionDb.MakeTxExec(i);
            }
            return(txExecs);
        }
Пример #10
0
 static void SyncLoadTpccTablesInto(VersionDb versionDb, string dir)
 {
     // foreach (var t in TpccTable.allTypes)
     // {
     //     SyncLoadTpccTable(TpccTable.Instance(t), versionDb, dir);
     // }
     Parallel.ForEach(
         TpccTable.AllUsedTypes,
         t => SyncLoadTpccTable(TpccTable.Instance(t), versionDb, dir));
 }
Пример #11
0
        static void SyncLoadTpccTable(
            TpccTable table, VersionDb versionDb, string dir)
        {
            Console.WriteLine($"Start loading table: '{table.Type().Name()}'");
            var startTime = DateTime.UtcNow;

            int c = TPCCTableLoader.Load(dir, table, versionDb);

            var time = (DateTime.UtcNow - startTime).TotalSeconds;

            Console.WriteLine(
                $"{c} records in '{table.Name()}' loaded in {time:F3} sec");
        }
Пример #12
0
        static void LoadDataWithSyncForCassandra(string filename, int nThread, VersionDb vdb)
        {
            List <VersionDb> vdbList = new List <VersionDb>();

            for (int i = 0; i < nThread; i++)
            {
                vdbList.Add(vdb);
            }

            YCSBBenchmarkTest sync_test = new YCSBBenchmarkTest(0, 0, vdb);

            sync_test.LoadDataWithMultiThreads(filename, vdbList, nThread);
        }
Пример #13
0
        internal void RunTxOnly()
        {
            VersionDb vdb = RedisVersionDb.Instance();

            for (int i = 0; i < this.txTaskQueue.Length; i++)
            {
                long txId = vdb.InsertNewTx();
                //vdb.UpdateCommitLowerBound(txId, 50);
                //vdb.SetAndGetCommitTime(txId, 90);
                //vdb.UpdateTxStatus(txId, TxStatus.Committed);

                this.FinishedTxs++;
            }
        }
Пример #14
0
        /// <summary>
        /// For YCSB sync benchmark test
        /// </summary>
        static void YCSBTest()
        {
            const int    workerCount        = 4;       // 4;
            const int    taskCountPerWorker = 2000000; // 50000;
            const string dataFile           = "ycsb_data_lg_r.in";
            const string operationFile      = "ycsb_ops_lg_r.in";

            // REDIS VERSION DB
            // VersionDb versionDb = RedisVersionDb.Instance;
            // SINGLETON VERSION DB
            VersionDb versionDb = SingletonVersionDb.Instance();

            YCSBBenchmarkTest test = new YCSBBenchmarkTest(workerCount, taskCountPerWorker, versionDb);

            test.Setup(dataFile, operationFile);
            test.Run();
            test.Stats();
        }
Пример #15
0
            public override int SaveTo(VersionDb versionDb)
            {
                CreateTable(versionDb, TpccTable.Instance(TableType.CUSTOMER_INDEX));
                int           recordCount = 0;
                SyncExecution txExec      = new SyncExecution(versionDb);

                foreach (var kv in this.tempStore)
                {
                    CustomerLastNameIndexKey lastNameKey = kv.Key;
                    List <uint> cids = kv.Value;
                    txExec.Start();
                    txExec.Insert(
                        lastNameKey, CustomerLastNamePayloads.FromList(cids));
                    txExec.Commit();
                    ++recordCount;
                }
                return(recordCount);
            }
Пример #16
0
        internal int batchInsert(VersionDb db, int start, int end)
        {
            int         count     = 0;
            Transaction txn       = new Transaction(null, db);
            string      readValue = null;

            for (int i = start; i < end; i++)
            {
                readValue = (string)txn.ReadAndInitialize(TABLE_ID, i);
                if (readValue == null)
                {
                    txn.Insert(TABLE_ID, i, new String('a', 100));
                    count++;
                }
            }
            txn.Commit();
            return(count);
        }
Пример #17
0
        void batchInsert(int s, int e, VersionDb vdb, List <string> lines)
        {
            //Console.WriteLine("s={0}, e={1}", s, e);
            Transaction tx        = new Transaction(null, vdb);
            string      readValue = null;

            for (int i = s; i < e; i++)
            {
                //Console.WriteLine("i={0}, cnt={1}", i, lines.Count);
                string[] fields = this.ParseCommandFormat(lines[i]);
                //readValue = (string)tx.ReadAndInitialize(TABLE_ID, fields[2]);
                readValue = (string)tx.ReadAndInitialize(TABLE_ID, i);
                if (readValue == null)
                {
                    //tx.Insert(TABLE_ID, fields[2], fields[3]);
                    tx.Insert(TABLE_ID, i, fields[3]);
                }
            }
            tx.Commit();
        }
Пример #18
0
        public YCSBBenchmarkTest(int workerCount, int taskCountPerWorker, VersionDb versionDb = null)
        {
            this.workerCount        = workerCount;
            this.taskCountPerWorker = taskCountPerWorker;

            if (versionDb != null)
            {
                this.versionDb = versionDb;
            }

            this.workers = new List <Worker>();
            for (int i = 0; i < this.workerCount; i++)
            {
                this.workers.Add(new Worker(i + 1, taskCountPerWorker));
            }

            this.executors    = new List <TransactionExecutor>();
            this.transactions = new List <Transaction>();
            for (int i = 0; i < this.workerCount; i++)
            {
                this.executors.Add(new TransactionExecutor(this.versionDb, null, null, 0, i));
                this.transactions.Add(new Transaction(null, this.versionDb, 10));
            }
        }
Пример #19
0
        int Load(string dir, TpccTable table, VersionDb versionDb)
        {
            CreateTable(versionDb, table);
            int recordCount    = 0;
            var txExec         = new SyncExecution(versionDb, (int)table.Type());
            var auxIndexLoader = AuxIndexLoader.FromTableType(table.Type());

            foreach (var kv in LoadKvsFromDir(dir, table))
            {
                txExec.Start();
                if (txExec.Insert(kv.Item1, kv.Item2).IsAborted())
                {
                    continue;
                }
                if (txExec.Commit().IsAborted())
                {
                    continue;
                }
                auxIndexLoader.BuildAuxIndex(kv.Item1, kv.Item2);
                ++recordCount;
            }
            recordCount += auxIndexLoader.SaveTo(versionDb);
            return(recordCount);
        }
Пример #20
0
 public SyncExecutionBuilder(VersionDb versionDb)
 {
     this.VersionDb = versionDb;
 }
Пример #21
0
 static private void CreateTable(VersionDb versionDb, TpccTable table)
 {
     versionDb.CreateVersionTable(table.Name());
 }
Пример #22
0
 public virtual int SaveTo(VersionDb versionDb)
 {
     return(0);
 }
Пример #23
0
 public SyncExecution(VersionDb versionDb, int workerId = 0)
 {
     this.txExec = MakeSimpleExecution(versionDb, workerId);
 }