public DataLoadWorker(VersionDb db, int start, int end, YCSBAsyncBenchmarkTest test)
{
this.db = db;
this.start = start;
this.end = end;
this.test = test;
}
public void TestInit()
{
// RedisVersionDb.Instance.PipelineMode = true;
//this.versionDb = RedisVersionDb.Instance;
//this.clientManager = ((RedisVersionDb)this.versionDb).RedisManager;
//int partition = versionDb.PhysicalPartitionByKey(DEFAULT_KEY);
//using (RedisClient redisClient = (RedisClient)this.clientManager.GetClient(TEST_REDIS_DB, partition))
//{
// // 1. flush the test db
// redisClient.ChangeDb(TEST_REDIS_DB);
// redisClient.FlushDb();
// // 2. create version table, if table is null, it means the table with the same tableId has existed
// VersionTable table = this.versionDb.CreateVersionTable(TABLE_ID, TEST_REDIS_DB);
// // 3. load data
// Transaction tx = new Transaction(null, this.versionDb);
// tx.ReadAndInitialize(TABLE_ID, DEFAULT_KEY);
// tx.Insert(TABLE_ID, DEFAULT_KEY, DEFAULT_VALUE);
// tx.Commit();
//}
this.versionDb = CassandraVersionDb.Instance(4);
VersionTable table = this.versionDb.CreateVersionTable(TABLE_ID, TEST_REDIS_DB);
Transaction tx = new Transaction(null, this.versionDb);
tx.ReadAndInitialize(TABLE_ID, DEFAULT_KEY);
tx.Insert(TABLE_ID, DEFAULT_KEY, DEFAULT_VALUE);
tx.Commit();
}
public YCSBAsyncBenchmarkTest(
int recordCount,
int executorCount,
int txCountPerExecutor,
VersionDb versionDb,
string[] flushTables = null,
BenchmarkTestConfig config = null)
{
this.versionDb = versionDb;
this.recordCount = recordCount;
this.executorCount = executorCount;
this.txCountPerExecutor = txCountPerExecutor;
this.executorList = new List <TransactionExecutor>();
this.totalTasks = 0;
this.tables = flushTables;
if (config == null)
{
config = new BenchmarkTestConfig();
}
this.config = config;
Console.WriteLine("Current Benchmark Test Config: {0}", this.config.ToString());
Console.WriteLine("Wait for 5 seconds to confirm");
Console.Out.Flush();
//Thread.Sleep(5000);
}
static internal TransactionExecution MakeTxExec(
this VersionDb versionDb, int workerId = 0)
{
return(new TransactionExecution(
null, versionDb, null, new TxRange(workerId),
workerId % versionDb.PartitionCount));
}
internal void load(VersionDb db, int threadCount, int totalRecord)
{
int interval = totalRecord / threadCount;
List <Thread> threadList = new List <Thread>();
int[] count = new int[threadCount];
List <DataLoadWorker> loadList = new List <DataLoadWorker>();
for (int i = 0; i < threadCount; i++)
{
DataLoadWorker load = new DataLoadWorker(db, interval * i,
Math.Min(interval * (i + 1), totalRecord), this);
Thread t = new Thread(new ThreadStart(load.load));
loadList.Add(load);
threadList.Add(t);
t.Start();
}
long lastTime = DateTime.Now.Ticks;
long now;
int lastSum = 0;
int sum = 0;
int thread = 0;
foreach (DataLoadWorker loader in loadList)
{
lastSum += loader.getCount();
}
bool isFinished = false;
while (!isFinished)
{
Thread.Sleep(1000);
sum = 0;
isFinished = true;
thread = 0;
foreach (DataLoadWorker loader in loadList)
{
sum += loader.getCount();
if (!loader.getIsFinished())
{
thread++;
isFinished = false;
}
}
now = DateTime.Now.Ticks;
long runSeconds = (now - lastTime) / 10000000;
Console.WriteLine("Time Inteval : {0}, Total Finish Count: {1}, Throughput: {2} txn/s, thread : {3}",
runSeconds, sum, (sum - lastSum) / runSeconds, thread);
lastSum = sum;
lastTime = now;
}
foreach (Thread t in threadList)
{
t.Join();
}
}
TransactionExecution MakeSimpleExecution(
VersionDb versionDb, int workerId)
{
int visitorId = workerId % versionDb.PartitionCount;
return(new TransactionExecution(
null, versionDb, null, new TxRange(workerId), visitorId));
}
internal void loadWithOneThread(VersionDb db, int start, int end, ref int sum)
{
int batchSize = 1;
for (int i = start; i < end; i += batchSize)
{
sum += batchInsert(db, i, Math.Min(end, i + batchSize));
}
}
public YcsbBenchmarkEnv(
VersionDb versionDb, Func <TransactionExecution, YcsbWorker> workerFactory)
{
var txExecs = SingletonYCSBTxExecHelper.MakeTxExecs(versionDb);
this.workers = txExecs.AsParallel()
.Select(workerFactory)
.ToArray();
}
TransactionExecution[] MakeTxExecs(VersionDb versionDb)
{
var txExecs = new TransactionExecution[versionDb.PartitionCount];
for (int i = 0; i < versionDb.PartitionCount; ++i)
{
txExecs[i] = versionDb.MakeTxExec(i);
}
return(txExecs);
}
static void SyncLoadTpccTablesInto(VersionDb versionDb, string dir)
{
// foreach (var t in TpccTable.allTypes)
// {
// SyncLoadTpccTable(TpccTable.Instance(t), versionDb, dir);
// }
Parallel.ForEach(
TpccTable.AllUsedTypes,
t => SyncLoadTpccTable(TpccTable.Instance(t), versionDb, dir));
}
static void SyncLoadTpccTable(
TpccTable table, VersionDb versionDb, string dir)
{
Console.WriteLine($"Start loading table: '{table.Type().Name()}'");
var startTime = DateTime.UtcNow;
int c = TPCCTableLoader.Load(dir, table, versionDb);
var time = (DateTime.UtcNow - startTime).TotalSeconds;
Console.WriteLine(
$"{c} records in '{table.Name()}' loaded in {time:F3} sec");
}
static void LoadDataWithSyncForCassandra(string filename, int nThread, VersionDb vdb)
{
List <VersionDb> vdbList = new List <VersionDb>();
for (int i = 0; i < nThread; i++)
{
vdbList.Add(vdb);
}
YCSBBenchmarkTest sync_test = new YCSBBenchmarkTest(0, 0, vdb);
sync_test.LoadDataWithMultiThreads(filename, vdbList, nThread);
}
internal void RunTxOnly()
{
VersionDb vdb = RedisVersionDb.Instance();
for (int i = 0; i < this.txTaskQueue.Length; i++)
{
long txId = vdb.InsertNewTx();
//vdb.UpdateCommitLowerBound(txId, 50);
//vdb.SetAndGetCommitTime(txId, 90);
//vdb.UpdateTxStatus(txId, TxStatus.Committed);
this.FinishedTxs++;
}
}
/// <summary>
/// For YCSB sync benchmark test
/// </summary>
static void YCSBTest()
{
const int workerCount = 4; // 4;
const int taskCountPerWorker = 2000000; // 50000;
const string dataFile = "ycsb_data_lg_r.in";
const string operationFile = "ycsb_ops_lg_r.in";
// REDIS VERSION DB
// VersionDb versionDb = RedisVersionDb.Instance;
// SINGLETON VERSION DB
VersionDb versionDb = SingletonVersionDb.Instance();
YCSBBenchmarkTest test = new YCSBBenchmarkTest(workerCount, taskCountPerWorker, versionDb);
test.Setup(dataFile, operationFile);
test.Run();
test.Stats();
}
public override int SaveTo(VersionDb versionDb)
{
CreateTable(versionDb, TpccTable.Instance(TableType.CUSTOMER_INDEX));
int recordCount = 0;
SyncExecution txExec = new SyncExecution(versionDb);
foreach (var kv in this.tempStore)
{
CustomerLastNameIndexKey lastNameKey = kv.Key;
List <uint> cids = kv.Value;
txExec.Start();
txExec.Insert(
lastNameKey, CustomerLastNamePayloads.FromList(cids));
txExec.Commit();
++recordCount;
}
return(recordCount);
}
internal int batchInsert(VersionDb db, int start, int end)
{
int count = 0;
Transaction txn = new Transaction(null, db);
string readValue = null;
for (int i = start; i < end; i++)
{
readValue = (string)txn.ReadAndInitialize(TABLE_ID, i);
if (readValue == null)
{
txn.Insert(TABLE_ID, i, new String('a', 100));
count++;
}
}
txn.Commit();
return(count);
}
void batchInsert(int s, int e, VersionDb vdb, List <string> lines)
{
//Console.WriteLine("s={0}, e={1}", s, e);
Transaction tx = new Transaction(null, vdb);
string readValue = null;
for (int i = s; i < e; i++)
{
//Console.WriteLine("i={0}, cnt={1}", i, lines.Count);
string[] fields = this.ParseCommandFormat(lines[i]);
//readValue = (string)tx.ReadAndInitialize(TABLE_ID, fields[2]);
readValue = (string)tx.ReadAndInitialize(TABLE_ID, i);
if (readValue == null)
{
//tx.Insert(TABLE_ID, fields[2], fields[3]);
tx.Insert(TABLE_ID, i, fields[3]);
}
}
tx.Commit();
}
public YCSBBenchmarkTest(int workerCount, int taskCountPerWorker, VersionDb versionDb = null)
{
this.workerCount = workerCount;
this.taskCountPerWorker = taskCountPerWorker;
if (versionDb != null)
{
this.versionDb = versionDb;
}
this.workers = new List <Worker>();
for (int i = 0; i < this.workerCount; i++)
{
this.workers.Add(new Worker(i + 1, taskCountPerWorker));
}
this.executors = new List <TransactionExecutor>();
this.transactions = new List <Transaction>();
for (int i = 0; i < this.workerCount; i++)
{
this.executors.Add(new TransactionExecutor(this.versionDb, null, null, 0, i));
this.transactions.Add(new Transaction(null, this.versionDb, 10));
}
}
int Load(string dir, TpccTable table, VersionDb versionDb)
{
CreateTable(versionDb, table);
int recordCount = 0;
var txExec = new SyncExecution(versionDb, (int)table.Type());
var auxIndexLoader = AuxIndexLoader.FromTableType(table.Type());
foreach (var kv in LoadKvsFromDir(dir, table))
{
txExec.Start();
if (txExec.Insert(kv.Item1, kv.Item2).IsAborted())
{
continue;
}
if (txExec.Commit().IsAborted())
{
continue;
}
auxIndexLoader.BuildAuxIndex(kv.Item1, kv.Item2);
++recordCount;
}
recordCount += auxIndexLoader.SaveTo(versionDb);
return(recordCount);
}
public SyncExecutionBuilder(VersionDb versionDb)
{
this.VersionDb = versionDb;
}
static private void CreateTable(VersionDb versionDb, TpccTable table)
{
versionDb.CreateVersionTable(table.Name());
}
public virtual int SaveTo(VersionDb versionDb)
{
return(0);
}
public SyncExecution(VersionDb versionDb, int workerId = 0)
{
this.txExec = MakeSimpleExecution(versionDb, workerId);
}