public static void generateCustomerProduct(AerospikeClient client)
{
Random products = new Random (2727);
Random productsPerAccount = new Random (9898);
Random productQuantity = new Random (1919);
for (int i = 0; i < accountTotal; i++) {
int productsToAdd = productsPerAccount.Next (1, 150);
string keyString = i.ToString ();
Key cdtkey = new Key (ns, cdtSet, keyString);
Aerospike.Helper.Collection.LargeList clist = new Aerospike.Helper.Collection.LargeList (client, null, cdtkey, cdtBinName);
Key ldtkey = new Key (ns, ldtSet, keyString);
LargeList llist = client.GetLargeList (null, ldtkey, ldtBinName);
//for diagnositics
client.Put (null, cdtkey, new Bin (keyBinName, keyString), new Bin (accBinName, keyString));
client.Put (null, ldtkey, new Bin (keyBinName, keyString), new Bin (accBinName, keyString));
for (int j = 0; j < productsToAdd; j++) {
int product = products.Next (1, productTotal);
int productAmount = productQuantity.Next (1, 100);
Value value = makeValue (product, productAmount);
llist.Update (value);
clist.Update (value);
}
}
}
public static void Main(string[] args)
{
AerospikeClient client = null;
try {
Console.WriteLine ("Connecting to Aerospike cluster...");
Stopwatch stopwatch = new Stopwatch ();
//Thread[] array = new Thread[accountTotal];
// Establish connection
client = new AerospikeClient (asServerIP, asServerPort);
// Check to see if the cluster connection succeeded
if (client.Connected) {
Console.WriteLine ("Connection succeeded!\n");
int feature = 0;
while (feature != 99) {
// Present options
Console.WriteLine ("\n\nWhat would you like to do:");
Console.WriteLine ("1> Generate sequence data for last " + days + " days");
Console.WriteLine ("2> Reconsile sequence data");
Console.WriteLine ("3> Generate Position using CDT and LDT");
Console.WriteLine ("4> List CDT vs LDT - whole list");
Console.WriteLine ("5> List CDT vs LDT - one element");
Console.WriteLine ("99> Exit");
Console.Write ("\nSelect 1-5, 99 and hit enter:");
string input = Console.ReadLine ();
if (input.Trim().Length == 0)
feature = 99;
else
feature = int.Parse (input);
if (feature != 99) {
switch (feature) {
case 1:
Console.WriteLine ("Generating data...");
DateTime today = DateTime.Now;
today = today.Date;
// write yesterday to Aerospike
DateTime yesterday = today.AddDays (-1);
generateTimeSeries (yesterday.ToShortDateString (), client);
// write yesterday to file
generateTimeSeries (yesterday.ToShortDateString (), null);
// write n days
for (int dif = 2; dif < days; dif++) {
DateTime dayBefore = today.AddDays (-dif);
Console.WriteLine ("Writing: " + dayBefore.ToShortDateString ());
generateTimeSeries (dayBefore.ToShortDateString (), client);
}
Console.WriteLine ("Generating data completed");
break;
case 2:
Console.WriteLine ("Reconsiling yesterday's data");
#region setup
// use admin tools to do this in production
RegisterTask rTask = client.Register(null,
udfDir + "utility.lua",
"utility.lua", Language.LUA);
while(!rTask.QueryIfDone()){
Thread.Sleep(20);
}
// client.CreateIndex(null, ns, seqSet,
// "day-seq-index", dayBinName, IndexType.STRING);
#endregion
today = DateTime.Now;
today = today.Date;
yesterday = today.AddDays (-1);
string yesterdayString = yesterday.ToShortDateString ();
// purge first sequence
PurgeDay (yesterdayString, client);
/// reconciliation from file
ReconcileDay (yesterdayString, client);
Console.WriteLine ("Reconciliation complete");
break;
case 3:
Console.WriteLine ("Generating position data using CDT and LDT...");
generateCustomerProduct (client);
Console.WriteLine ("Generating data completed");
break;
case 4:
Console.WriteLine ("CDT vs LDT - Whole list...");
long cdtTotal = 0, ldtTotal = 0, cdtProdCount = 0, ldtProdCount = 0;
for (int acc = 0; acc < accountTotal; acc++) {
string accString = (acc + 1).ToString ();
Key cdtkey = new Key (ns, cdtSet, accString);
Key ldtkey = new Key (ns, ldtSet, accString);
Aerospike.Helper.Collection.LargeList clist = new Aerospike.Helper.Collection.LargeList (client, null, cdtkey, cdtBinName);
LargeList llist = client.GetLargeList (null, ldtkey, ldtBinName);
stopwatch.Start ();
IList cresult = clist.Scan ();
stopwatch.Stop ();
if (cresult != null)
cdtProdCount += cresult.Count;
cdtTotal += stopwatch.ElapsedMilliseconds;
stopwatch.Reset ();
stopwatch.Start ();
IList lresult = llist.Scan ();
stopwatch.Stop ();
if (lresult != null)
ldtProdCount += lresult.Count;
ldtTotal += stopwatch.ElapsedMilliseconds;
}
Console.WriteLine ("CDT avg latency: {0:F5} ms", (double)cdtTotal / cdtProdCount);
Console.WriteLine ("LDT avg latency: {0:F5} ms", (double)ldtTotal / ldtProdCount);
break;
case 5:
const int attempts = 50000;
Console.WriteLine ("CDT vs LDT - one element..., {0} customers, each product for a customer", attempts);
cdtTotal = 0;
ldtTotal = 0;
long prodCount = 0;
Random accRand = new Random (12121);
for (int i = 0; i < 50000; i++) {
string accString = accRand.Next (1, accountTotal).ToString ();
Key cdtkey = new Key (ns, cdtSet, accString);
Key ldtkey = new Key (ns, ldtSet, accString);
Aerospike.Helper.Collection.LargeList clist = new Aerospike.Helper.Collection.LargeList (client, null, cdtkey, cdtBinName);
LargeList llist = client.GetLargeList (null, ldtkey, ldtBinName);
IList prods = llist.Scan ();
if (prods != null) {
prodCount += prods.Count;
foreach (IDictionary product in prods) {
//Dictionary<string, Object> keyMap = makeKeyMap (product);
// CDT
stopwatch.Start ();
clist.Find (Value.Get (product));
stopwatch.Stop ();
cdtTotal += stopwatch.ElapsedMilliseconds;
stopwatch.Reset ();
// LDT
stopwatch.Start ();
llist.Find (Value.Get (product));
stopwatch.Stop ();
ldtTotal += stopwatch.ElapsedMilliseconds;
}
}
}
Console.WriteLine ("CDT avg latency: {0:F5} ms", (double)cdtTotal / prodCount);
Console.WriteLine ("LDT avg latency: {0:F5} ms", (double)ldtTotal / prodCount);
break;
default:
break;
}
}
}
}
} catch (AerospikeException e) {
Console.WriteLine ("AerospikeException - Message: " + e.Message);
Console.WriteLine ("AerospikeException - StackTrace: " + e.StackTrace);
} catch (Exception e) {
Console.WriteLine ("Exception - Message: " + e.Message);
Console.WriteLine ("Exception - StackTrace: " + e.StackTrace);
} finally {
if (client != null && client.Connected) {
// Close Aerospike server connection
client.Close ();
}
}
}
/// <summary>
/// Simple examples of large list functionality.
/// </summary>
private void RunSimpleExample(AerospikeClient client, Arguments args)
{
Key key = new Key(args.ns, args.set, "setkey");
string binName = args.GetBinName("ListBin");
// Delete record if it already exists.
client.Delete(args.writePolicy, key);
// Initialize large set operator.
Aerospike.Client.LargeList llist = client.GetLargeList(args.writePolicy, key, binName);
string orig1 = "llistValue1";
string orig2 = "llistValue2";
string orig3 = "llistValue3";
// Write values.
llist.Add(Value.Get(orig1));
llist.Add(Value.Get(orig2));
llist.Add(Value.Get(orig3));
IDictionary map = llist.GetConfig();
foreach (DictionaryEntry entry in map)
{
console.Info(entry.Key.ToString() + ',' + entry.Value);
}
IList rangeList = llist.Range(Value.Get(orig2), Value.Get(orig3));
if (rangeList == null)
{
throw new Exception("Range returned null.");
}
if (rangeList.Count != 2)
{
throw new Exception("Range Size mismatch. Expected 2 Received " + rangeList.Count);
}
string v2 = (string) rangeList[0];
string v3 = (string) rangeList[1];
if (v2.Equals(orig2) && v3.Equals(orig3))
{
console.Info("Range Query matched: v2=" + orig2 + " v3=" + orig3);
}
else
{
throw new Exception("Range Content mismatch. Expected (" + orig2 + ":" + orig3 +
") Received (" + v2 + ":" + v3 + ")");
}
// Remove last value.
llist.Remove(Value.Get(orig3));
int size = llist.Size();
if (size != 2)
{
throw new Exception("Size mismatch. Expected 2 Received " + size);
}
IList listReceived = llist.Find(Value.Get(orig2));
string expected = orig2;
if (listReceived == null)
{
console.Error("Data mismatch: Expected " + expected + " Received null");
return;
}
string stringReceived = (string) listReceived[0];
if (stringReceived != null && stringReceived.Equals(expected))
{
console.Info("Data matched: namespace=" + key.ns + " set=" + key.setName + " key=" + key.userKey +
" value=" + stringReceived);
}
else
{
console.Error("Data mismatch: Expected " + expected + " Received " + stringReceived);
}
}
/// <summary>
/// Use serialized bin for row in largelist bin.
/// </summary>
private void RunWithSerializedBin(AerospikeClient client, Arguments args)
{
Key key = new Key(args.ns, args.set, "accountId");
// Delete record if it already exists.
client.Delete(args.writePolicy, key);
// Initialize large list operator.
Aerospike.Client.LargeList list = client.GetLargeList(args.writePolicy, key, "trades");
// Write trades
Dictionary<string, Value> dict = new Dictionary<string, Value>();
MemoryStream ms = new MemoryStream(500);
DateTime timestamp1 = new DateTime(2014, 6, 25, 12, 18, 43);
dict["key"] = Value.Get(timestamp1.Ticks);
BinaryWriter writer = new BinaryWriter(ms);
writer.Write("IBM"); // ticker
writer.Write(100); // qty
writer.Write(181.82); // price
dict["value"] = Value.Get(ms.ToArray());
list.Add(Value.Get(dict));
DateTime timestamp2 = new DateTime(2014, 6, 26, 9, 33, 17);
dict["key"] = Value.Get(timestamp2.Ticks);
ms.SetLength(0);
writer = new BinaryWriter(ms);
writer.Write("GE"); // ticker
writer.Write(500); // qty
writer.Write(26.36); // price
dict["value"] = Value.Get(ms.ToArray());
list.Add(Value.Get(dict));
DateTime timestamp3 = new DateTime(2014, 6, 27, 14, 40, 19);
dict["key"] = Value.Get(timestamp3.Ticks);
ms.SetLength(0);
writer = new BinaryWriter(ms);
writer.Write("AAPL"); // ticker
writer.Write(75); // qty
writer.Write(91.85); // price
dict["value"] = Value.Get(ms.ToArray());
list.Add(Value.Get(dict));
// Verify list size
int size = list.Size();
if (size != 3)
{
throw new Exception("List size mismatch. Expected 3 Received " + size);
}
// Filter on range of timestamps
DateTime begin = new DateTime(2014, 6, 26);
DateTime end = new DateTime(2014, 6, 28);
IList results = list.Range(Value.Get(begin.Ticks), Value.Get(end.Ticks));
if (results.Count != 2)
{
throw new Exception("Query results size mismatch. Expected 2 Received " + results.Count);
}
// Verify data.
ValidateWithSerializedBin(results, 0, timestamp2, "GE", 500, 26.36);
ValidateWithSerializedBin(results, 1, timestamp3, "AAPL", 75, 91.85);
console.Info("Data matched.");
}
/// <summary>
/// Use distinct sub-bins for row in largelist bin.
/// </summary>
private void RunWithDistinctBins(AerospikeClient client, Arguments args)
{
Key key = new Key(args.ns, args.set, "accountId");
// Delete record if it already exists.
client.Delete(args.writePolicy, key);
// Initialize large list operator.
Aerospike.Client.LargeList list = client.GetLargeList(args.writePolicy, key, "trades");
// Write trades
Dictionary<string,Value> dict = new Dictionary<string,Value>();
DateTime timestamp1 = new DateTime(2014, 6, 25, 12, 18, 43);
dict["key"] = Value.Get(timestamp1.Ticks);
dict["ticker"] = Value.Get("IBM");
dict["qty"] = Value.Get(100);
dict["price"] = Value.Get(BitConverter.GetBytes(181.82));
list.Add(Value.Get(dict));
DateTime timestamp2 = new DateTime(2014, 6, 26, 9, 33, 17);
dict["key"] = Value.Get(timestamp2.Ticks);
dict["ticker"] = Value.Get("GE");
dict["qty"] = Value.Get(500);
dict["price"] = Value.Get(BitConverter.GetBytes(26.36));
list.Add(Value.Get(dict));
DateTime timestamp3 = new DateTime(2014, 6, 27, 14, 40, 19);
dict["key"] = Value.Get(timestamp3.Ticks);
dict["ticker"] = Value.Get("AAPL");
dict["qty"] = Value.Get(75);
dict["price"] = Value.Get(BitConverter.GetBytes(91.85));
list.Add(Value.Get(dict));
// Verify list size
int size = list.Size();
if (size != 3)
{
throw new Exception("List size mismatch. Expected 3 Received " + size);
}
// Filter on range of timestamps
DateTime begin = new DateTime(2014, 6, 26);
DateTime end = new DateTime(2014, 6, 28);
IList results = list.Range(Value.Get(begin.Ticks), Value.Get(end.Ticks));
if (results.Count != 2)
{
throw new Exception("Query results size mismatch. Expected 2 Received " + results.Count);
}
// Verify data.
ValidateWithDistinctBins(results, 0, timestamp2, "GE", 500, 26.36);
ValidateWithDistinctBins(results, 1, timestamp3, "AAPL", 75, 91.85);
console.Info("Data matched.");
console.Info("Run large list scan.");
IList rows = list.Scan();
foreach (IDictionary row in rows)
{
foreach (DictionaryEntry entry in row)
{
//console.Info(entry.Key.ToString());
//console.Info(entry.Value.ToString());
}
}
console.Info("Large list scan complete.");
}
/// <summary>
/// Use default serialized bin for row in largelist bin.
/// </summary>
private void RunWithDefaultSerializedBin(AerospikeClient client, Arguments args)
{
Key key = new Key(args.ns, args.set, "accountId");
object value1 = new CompoundObject("IBM", 100);
object value2 = new CompoundObject("GE", 500);
object value3 = new CompoundObject("AAPL", 75);
// Delete record if it already exists.
client.Delete(args.writePolicy, key);
// Initialize large list operator.
Aerospike.Client.LargeList list = client.GetLargeList(args.writePolicy, key, "trades");
// Write trades
Dictionary<string, Value> dict = new Dictionary<string, Value>();
DateTime timestamp1 = new DateTime(2014, 6, 25, 12, 18, 43);
dict["key"] = Value.Get(timestamp1.Ticks);
dict["value"] = Value.Get(value1);
list.Add(Value.Get(dict));
DateTime timestamp2 = new DateTime(2014, 6, 26, 9, 33, 17);
dict["key"] = Value.Get(timestamp2.Ticks);
dict["value"] = Value.Get(value2);
list.Add(Value.Get(dict));
DateTime timestamp3 = new DateTime(2014, 6, 27, 14, 40, 19);
dict["key"] = Value.Get(timestamp3.Ticks);
dict["value"] = Value.Get(value3);
list.Add(Value.Get(dict));
// Verify list size
int size = list.Size();
if (size != 3)
{
throw new Exception("List size mismatch. Expected 3 Received " + size);
}
// Filter on range of timestamps
DateTime begin = new DateTime(2014, 6, 26);
DateTime end = new DateTime(2014, 6, 28);
IList results = list.Range(Value.Get(begin.Ticks), Value.Get(end.Ticks));
if (results.Count != 2)
{
throw new Exception("Query results size mismatch. Expected 2 Received " + results.Count);
}
// Verify data.
ValidateDefault(results, 0, timestamp2, value2);
ValidateDefault(results, 1, timestamp3, value3);
console.Info("Data matched.");
}
public override void Init(int flowCount, long flowRecordCount)
{
string[] str = ConnectionString.Split(';');
string server = str[0].Split(':')[1];
int port = Int32.Parse(str[1].Split(':')[1]);
string ns = str[2].Split(':')[1];
string set = str[3].Split(':')[1];
client = new AerospikeClient(server, port);
indexes = new LargeList[flowCount];
Key listKey = new Key(ns, set, CollectionName);
client.Delete(null, listKey);
WritePolicy policy = new WritePolicy();
policy.recordExistsAction = RecordExistsAction.REPLACE;
for (int i = 0; i < flowCount; i++)
indexes[i] = client.GetLargeList(policy, listKey, CollectionName, null);
}