public static void query()
{
var transport = new TBufferedTransport(new TSocket("hserver", 10000));
var protocol = new TBinaryProtocol(transport);
var client = new ThriftHive.Client(protocol);
transport.Open();
//client.execute("CREATE TABLE r(a STRING, b INT, c DOUBLE)");
//client.execute("LOAD TABLE LOCAL INPATH '/path' INTO TABLE r");
client.execute("SELECT * FROM pokes where foo=180");
while (true)
{
var row = client.fetchOne();
if (string.IsNullOrEmpty(row))
break;
System.Console.WriteLine(row);
}
client.execute("SELECT * FROM pokes");
var all = client.fetchAll();
System.Console.WriteLine(all);
transport.Close();
}
public static void test()
{
var hadoop_socket = new TSocket("hserver", 59256);
hadoop_socket.Timeout = 10000;// Ten seconds
//$hadoop_socket -> setRecvTimeout(20000); // Twenty seconds
var hadoop_transport = new TBufferedTransport(hadoop_socket);
var hadoop_protocol = new TBinaryProtocol(hadoop_transport);
var hadoopClient = new ThriftHadoopFileSystem.Client(hadoop_protocol);
hadoop_transport.Open();
//if (hadoopClient.exists(new Pathname { pathname = "/user/root/input" }))
//{
// System.Console.WriteLine("exists");
//}
//else
//{
// Console.WriteLine("no exists");
//}
//hadoopClient.mkdirs(new Pathname { pathname = "/zbw911_user" });
//hadoopClient.rm(new Pathname { pathname = "/zbw911_user" }, true);
//var h = hadoopClient.createFile(new Pathname { pathname = "user_zbw911" }, 1, true, 1000, 1, 64000);
var list = hadoopClient.listStatus(new Pathname { pathname = "/user/root" });
foreach (var item in list)
{
System.Console.WriteLine(item);
//if (item.Isdir)
//{
// System.Console.WriteLine("文件夹：" + item.Path + "；权限：" + item.Permission);
//}
//else
//{
// System.Console.WriteLine("文件：" + item.Path + "；权限：" + item.Permission);
//}
}
//hadoopClient.close(h);
//hadoopClient.open(new Pathname { pathname = "/user/root" });
hadoop_transport.Close();
}
public static void t()
{
var transport = new TBufferedTransport(new TSocket("hserver", 10000));
var protocol = new TBinaryProtocol(transport);
var client = new ThriftHive.Client(protocol);
transport.Open();
//client.execute("CREATE TABLE r(a STRING, b INT, c DOUBLE)");
//client.execute("LOAD TABLE LOCAL INPATH '/path' INTO TABLE r");
var database = new Database();
database.Name = "helloworldDB";
database.Description = "测试用的第一个DB";
client.create_database(database);
transport.Close();
}
public static void t2()
{
var transport = new TBufferedTransport(new TSocket("hserver", 10000));
var protocol = new TBinaryProtocol(transport);
var client = new ThriftHive.Client(protocol);
transport.Open();
var database = new Database();
database.Name = "helloworldDB";
database.Description = "测试用的第一个DB";
var tables = client.get_all_tables("default");
//var tables = client.get_all_databases();
foreach (var table in tables)
{
System.Console.WriteLine(table);
}
transport.Close();
}
public static void UpdateData()
{
var transport = new TBufferedTransport(new TSocket("master", 9090));
transport.Open();
var protocol = new TBinaryProtocol(transport);
var client = new Apache.Hadoop.Hbase.Hbase.Client(protocol);
//client.createTable(Unitl.StrToBytes(tablename), new List<ColumnDescriptor> { new ColumnDescriptor { Name = Unitl.StrToBytes("zbw911"), InMemory = false } });
client.mutateRow(Unitl.StrToBytes("1"), Unitl.StrToBytes("key"), new List<Mutation> { new Mutation { Column = Unitl.StrToBytes("zbw911"), Value = Unitl.StrToBytes("zbw911-Modifyed") } });
transport.Close();
}
public static void ShowTables()
{
var transport = new TBufferedTransport(new TSocket("master", 9090));
transport.Open();
var protocol = new TBinaryProtocol(transport);
var client = new Apache.Hadoop.Hbase.Hbase.Client(protocol);
var tables = client.getTableNames();
foreach (var table in tables)
{
Console.WriteLine(Unitl.BytesToStr(table));
}
transport.Close();
}
public static void Scan()
{
var transport = new TBufferedTransport(new TSocket("master", 9090));
transport.Open();
var protocol = new TBinaryProtocol(transport);
var client = new Apache.Hadoop.Hbase.Hbase.Client(protocol);
var scanid = client.scannerOpenWithStop(Unitl.StrToBytes("t1"), Unitl.StrToBytes("rowno"), Unitl.StrToBytes("rowno"),
new List<byte[]> { Unitl.StrToBytes("f1:99913") });
var result = client.scannerGet(scanid);
var colum = client.getColumnDescriptors("t1".ToBytes());
var rows = client.getRowsWithColumns("t1".ToBytes()
, new List<byte[]> { "rowno".ToBytes() }
, new List<byte[]> { "f1:9".ToBytes(), "f1:9999".ToBytes() });
//scanid = client.scannerOpenWithPrefix("".ToBytes(), "".ToBytes(), new List<byte[]> { });
var v = client.getVer("table".ToBytes(), "row".ToBytes(), "col".ToBytes(), 1000);
foreach (var xx in v)
{
}
foreach (var r in result)
{
Console.WriteLine(r.Columns[Unitl.StrToBytes("f1:99913")].Value);
}
client.scannerClose(scanid);
transport.Close();
}
public static void GetData()
{
var transport = new TBufferedTransport(new TSocket("master", 9090));
transport.Open();
var protocol = new TBinaryProtocol(transport);
var client = new Apache.Hadoop.Hbase.Hbase.Client(protocol);
var rows = client.getRow(Unitl.StrToBytes("1"), Unitl.StrToBytes("row9916"));
foreach (var row in rows)
{
Console.WriteLine(row.Columns.ToString());
}
transport.Close();
}
public static void BatAddData()
{
var transport = new TBufferedTransport(new TSocket("master", 9090));
transport.Open();
var protocol = new TBinaryProtocol(transport);
var client = new Apache.Hadoop.Hbase.Hbase.Client(protocol);
//client.createTable(Unitl.StrToBytes(tablename), new List<ColumnDescriptor> { new ColumnDescriptor { Name = Unitl.StrToBytes("zbw911"), InMemory = false } });
var bytevalue = Unitl.StrToBytes(Longdata);
for (var j = 0; j < 100000; j++)
{
//var j = 0;
for (var i = 0; i < 1000; i++)
{
client.mutateRow(Unitl.StrToBytes("1"), Unitl.StrToBytes("" + i + "___" + j), new List<Mutation> { new Mutation { Column = Unitl.StrToBytes("zbw911"), Value = bytevalue } });
}
Console.WriteLine(j + "批=" + 1000 * j);
}
transport.Close();
}
public static void AddTable(string tablename)
{
var transport = new TBufferedTransport(new TSocket("master", 9090));
transport.Open();
var protocol = new TBinaryProtocol(transport);
var client = new Apache.Hadoop.Hbase.Hbase.Client(protocol);
client.createTable(Unitl.StrToBytes(tablename), new List<ColumnDescriptor> { new ColumnDescriptor { Name = Unitl.StrToBytes("zbw911"), InMemory = false } });
transport.Close();
}
public static void AddColumn()
{
var transport = new TBufferedTransport(new TSocket("master", 9090));
transport.Open();
var protocol = new TBinaryProtocol(transport);
var client = new Apache.Hadoop.Hbase.Hbase.Client(protocol);
//client.createTable(Unitl.StrToBytes("t"), new List<ColumnDescriptor> { new ColumnDescriptor { Name = Unitl.StrToBytes("default"), InMemory = false } });
//client.createTable(Unitl.StrToBytes("thetable"), new List<ColumnDescriptor> { new ColumnDescriptor { Name = Unitl.StrToBytes("default"), InMemory = false } });
for (var i = 0; i < 10; i++)
{
client.mutateRow(Unitl.StrToBytes("t"),
Unitl.StrToBytes("key"),
new List<Mutation> {
new Mutation { Column = Unitl.StrToBytes("default:"+i ), Value = Unitl.StrToBytes( System.DateTime.Now.ToString() ) }
});
}
transport.Close();
}
public static void CorrelateDFE(double[,] data, int sizeTimeseries,
int numTimeseries, double[] correlations)
{
// Calculates correlations on DFE.
Stopwatch sw = new Stopwatch();
sw.Start();
// Make socket
var socket = new TSocket("localhost", 9090);
// Buffering is critical. Raw sockets are very slow
var transport = new TBufferedTransport(socket);
// Wrap in a protocol
var protocol = new TBinaryProtocol(transport);
// Create a client to use the protocol encoder
var client = new correlationService.Client(protocol);
sw.Stop();
Console.WriteLine("Creating a client:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
try {
// Connect!
sw.Reset(); sw.Start();
transport.Open();
sw.Stop();
Console.WriteLine("Opening connection:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
int numTimesteps = sizeTimeseries;
double windowSize = (double)sizeTimeseries;
int numBursts = calcNumBursts(numTimeseries);
// Get loop length
sw.Reset(); sw.Start();
List<int> loopLength = new List<int>();
loopLength.Add(client.correlation_get_CorrelationKernel_loopLength());
sw.Stop();
Console.WriteLine("Getting Correlation Kernel loopLength:\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Prepare data for DFE
sw.Reset(); sw.Start();
double[] precalculations = new double [2 * numTimeseries * numTimesteps];
double[] dataPairs = new double [2 * numTimeseries * numTimesteps];
int burstSize = 384 / 2; // for anything other than ISCA this should be 384
List<int> inMemLoad = new List<int>();
for(int i = 0; i < numBursts * burstSize; i++) {
inMemLoad.Add(0);
}
prepareDataForDFE (data, sizeTimeseries, numTimeseries, numTimesteps, windowSize,
precalculations, dataPairs);
List<double> precalculationsVec = new List<double>();
List<double> dataPairsVec = new List<double>();
for(int i = 0; i < 2 * numTimeseries * numTimesteps; i++) {
precalculationsVec.Add(precalculations[i]);
dataPairsVec.Add(dataPairs[i]);
}
sw.Stop();
Console.WriteLine("Data reordering time:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Allocate and send input streams to server
sw.Reset(); sw.Start();
var loopLengthSize = 1;
var addressLoopLength = client.malloc_int32_t(loopLengthSize);
client.send_data_int32_t(addressLoopLength, loopLength);
sw.Stop();
var loopLengthTime = sw.Elapsed.TotalMilliseconds / 1000;
Console.WriteLine("\tSending LoopLength:\t\t(size = {0} bit)\t\t{1}s",
loopLengthSize * 32, loopLengthTime);
sw.Reset(); sw.Start();
var inMemLoadSize = numBursts * burstSize;
var addressInMemLoad = client.malloc_int32_t(inMemLoadSize);
client.send_data_int32_t(addressInMemLoad, inMemLoad);
sw.Stop();
var inMemLoadTime = sw.Elapsed.TotalMilliseconds / 1000;
Console.WriteLine("\tSending InMemLoad:\t\t(size = {0} bit)\t{1}s",
inMemLoadSize * 32, inMemLoadTime);
sw.Reset(); sw.Start();
var precalculationsSize = 2 * numTimeseries * numTimesteps;
var addressPrecalculations = client.malloc_double(precalculationsSize);
client.send_data_double(addressPrecalculations, precalculationsVec);
sw.Stop();
var precalculationsTime = sw.Elapsed.TotalMilliseconds / 1000;
Console.WriteLine("\tSending Precalculations:\t(size = {0} bit)\t{1}s",
precalculationsSize * 32, precalculationsTime);
sw.Reset(); sw.Start();
var dataPairsSize = 2 * numTimeseries * numTimesteps;
var addressDataPairs = client.malloc_double(dataPairsSize);
client.send_data_double(addressDataPairs, dataPairsVec);
sw.Stop();
var dataPairsTime = sw.Elapsed.TotalMilliseconds / 1000;
Console.WriteLine("\tSending DataPairs:\t\t(size = {0} bit)\t{1}s",
dataPairsSize * 64, dataPairsTime);
var time = loopLengthTime + inMemLoadTime + precalculationsTime + dataPairsTime;
var speed = (loopLengthSize * 32 + inMemLoadSize * 32 + precalculationsSize * 64 + dataPairsSize * 64) / time / 1000000;
Console.WriteLine("Sending input streams to server total time:\t{0}s\t(average speed = {1}Mb/s)", time, speed);
// Allocate memory for output stream on server
sw.Reset(); sw.Start();
var addressOutCorrelation = client.malloc_double(numTimesteps * loopLength[0] *
correlationNumTopScores * correlationNumPipes +
numBursts * 24);
var addressOutIndices = client.malloc_int32_t(2 * numTimesteps * loopLength[0] *
correlationNumTopScores * correlationNumPipes);
sw.Stop();
Console.WriteLine("Allocating memory for output stream on server:\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Initialize LMem
sw.Reset(); sw.Start();
client.correlation_loadLMem(numBursts, addressLoopLength, addressInMemLoad);
sw.Stop();
Console.WriteLine("LMem initialization:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
//Executing correlation action
sw.Reset(); sw.Start();
client.correlation(numBursts, // scalar input
numTimesteps, // scalar input
numTimeseries, // scalar input
1, // scalar input
windowSize, // scalar input
addressPrecalculations, // streaming reordered input
addressDataPairs, // streaming reordered input
addressOutCorrelation, // streaming unordered output
addressOutIndices); // streaming unordered output
sw.Stop();
Console.WriteLine("Correlation time:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Get output stream from server
sw.Reset(); sw.Start();
var outCorrelationSize = numTimesteps * loopLength[0] * correlationNumTopScores * correlationNumPipes + numBursts * 24;
List<double> outCorrelation = new List<double>();
outCorrelation = client.receive_data_double(addressOutCorrelation, outCorrelationSize);
sw.Stop();
var outCorrelationTime = sw.Elapsed.TotalMilliseconds / 1000;
Console.WriteLine("\tGet output stream Correlation:\t(size = {0} bit)\t{1}s",
outCorrelationSize * 64, outCorrelationTime);
sw.Reset(); sw.Start();
var outIndicesSzie = 2 * numTimesteps * loopLength[0] * correlationNumTopScores * correlationNumPipes;
List<int> outIndices = new List<int>();
outIndices = client.receive_data_int32_t(addressOutIndices, outIndicesSzie);
sw.Stop();
var outIndicesTime = sw.Elapsed.TotalMilliseconds / 1000;
Console.WriteLine("\tGet output stream outIndices:\t(size = {0} bit)\t{1}s",
outIndicesSzie * 32, outIndicesTime);
sw.Reset(); sw.Start();
loopLengthSize = 1;
loopLength = client.receive_data_int32_t(addressLoopLength, loopLengthSize);
sw.Stop();
loopLengthTime = sw.Elapsed.TotalMilliseconds / 1000;
Console.WriteLine("\tGet output stream loopLength:\t(size = {0} bit)\t\t{1}s",
loopLengthSize * 32, loopLengthTime);
time = loopLengthTime + outCorrelationTime + outIndicesTime;
speed = (loopLengthSize * 32 + outIndicesSzie * 32 + outCorrelationSize * 64) / time / 1000000;
Console.WriteLine("Getting output stream from server total time:\t{0}s\t(average speed = {1}Mb/s)", time, speed);
// Free allocated memory for streams on server
sw.Reset(); sw.Start();
client.free(addressLoopLength);
client.free(addressInMemLoad);
client.free(addressPrecalculations);
client.free(addressDataPairs);
client.free(addressOutCorrelation);
client.free(addressOutIndices);
sw.Stop();
Console.WriteLine("Freeing allocated memory for streams on server:\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Close!
sw.Reset(); sw.Start();
transport.Close();
sw.Stop();
Console.WriteLine("Closing connection:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Store data
sw.Reset(); sw.Start();
int position = 0;
int index = 0;
int start = (numTimesteps-1) * loopLength[0] * correlationNumTopScores * correlationNumPipes;
for (int i=0; i < numTimeseries; i++) {
for(int j = 0; j < i; j++) {
correlations[index + j] = outCorrelation[start + position + j];
}
index += i;
position += ((i/12)+1)*12;
}
sw.Stop();
Console.WriteLine("LMem initialization:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
} catch (SocketException e) {
Console.WriteLine("Could not connect to the server: {0}.", e.Message);
Environment.Exit(-1);
} catch (Exception e) {
Console.WriteLine("An error occured: {0}", e.Message);
Environment.Exit(-1);
}
}
public static List<double> PassThroughDFE(int size, List<double> dataIn)
{
List<double> dataOut = new List<double>();
Stopwatch sw = new Stopwatch();
sw.Start();
// Make socket
var socket = new TSocket("localhost", 9090);
// Buffering is critical. Raw sockets are very slow
var transport = new TBufferedTransport(socket);
// Wrap in a protocol
var protocol = new TBinaryProtocol(transport);
// Create a client to use the protocol encoder
var client = new PassThroughService.Client(protocol);
sw.Stop();
Console.WriteLine("Creating a client:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
try {
// Connect!
sw.Reset(); sw.Start();
transport.Open();
sw.Stop();
Console.WriteLine("Opening connection:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Initialize maxfile
sw.Reset(); sw.Start();
var maxfile = client.PassThrough_init();
sw.Stop();
Console.WriteLine("Initializing maxfile:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Load DFE
sw.Reset(); sw.Start();
var engine = client.max_load(maxfile, "*");
sw.Stop();
Console.WriteLine("Loading DFE:\t\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Allocate and send input streams to server
sw.Reset(); sw.Start();
var address_dataIn = client.malloc_float(size);
client.send_data_float(address_dataIn, dataIn);
sw.Stop();
Console.WriteLine("Sending input data:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Allocate memory for output stream on server
sw.Reset(); sw.Start();
var address_dataOut = client.malloc_float(size);
sw.Stop();
Console.WriteLine("Allocating memory for output stream on server:\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Action default
sw.Reset(); sw.Start();
var action = new PassThrough_actions_t_struct();
action.Param_N = size;
action.Instream_x = address_dataIn;
action.Outstream_y = address_dataOut;
var address_action = client.send_PassThrough_actions_t(action);
client.PassThrough_run(engine, address_action);
sw.Stop();
Console.WriteLine("Pass through time:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Unload DFE
sw.Reset(); sw.Start();
client.max_unload(engine);
sw.Stop();
Console.WriteLine("Unloading DFE:\t\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Get output stream from server
sw.Reset(); sw.Start();
dataOut = client.receive_data_float(address_dataOut, size);
sw.Stop();
Console.WriteLine("Getting output stream:\t(size = {0} bit)\t{1}s", size * 32, sw.Elapsed.TotalMilliseconds / 1000);
// Free allocated memory for streams on server
sw.Reset(); sw.Start();
client.free(address_dataIn);
client.free(address_dataOut);
sw.Stop();
Console.WriteLine("Freeing allocated memory for streams on server:\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Free allocated maxfile data
sw.Reset(); sw.Start();
client.PassThrough_free();
sw.Stop();
Console.WriteLine("Freeing allocated maxfile data:\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
sw.Reset(); sw.Start();
transport.Close();
sw.Stop();
Console.WriteLine("Closing connection:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
} catch (SocketException e) {
Console.WriteLine("Could not connect to the server: {0}.", e.Message);
Environment.Exit(-1);
} catch (Exception e) {
Console.WriteLine("An error occured: {0}", e.Message);
Environment.Exit(-1);
}
return dataOut;
}
public static void Main(string[] args)
{
Stopwatch sw = new Stopwatch();
Stopwatch swDFE = new Stopwatch();
sw.Start();
swDFE.Start();
// Make socket
var socket = new TSocket("localhost", 9090);
// Buffering is critical. Raw sockets are very slow
var transport = new TBufferedTransport(socket);
// Wrap in a protocol
var protocol = new TBinaryProtocol(transport);
// Create a client to use the protocol encoder
var client = new MovingAverageService.Client(protocol);
sw.Stop();
Console.WriteLine("Creating a client:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
try {
// Connect!
sw.Reset(); sw.Start();
transport.Open();
sw.Stop();
Console.WriteLine("Opening connection:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
sw.Reset(); sw.Start();
const int size = 384;
int sizeBytes = size * 4;
List<double> dataIn = new List<double>();
// Generate input data
Random random = new Random();
for(int i = 0; i<size; i++) {
dataIn.Add(random.Next(0, 100));
}
sw.Stop();
Console.WriteLine("Generating input data:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Initialize maxfile
sw.Reset(); sw.Start();
var maxfile = client.MovingAverage_init();
sw.Stop();
Console.WriteLine("Initializing maxfile:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Load DFE
sw.Reset(); sw.Start();
var engine = client.max_load(maxfile, "*");
sw.Stop();
Console.WriteLine("Loading DFE:\t\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Allocate and send input streams to server
sw.Reset(); sw.Start();
var address_dataIn = client.malloc_float(size);
client.send_data_float(address_dataIn, dataIn);
sw.Stop();
Console.WriteLine("Sending input data:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Allocate memory for output stream on server
sw.Reset(); sw.Start();
var address_dataOut = client.malloc_float(size);
sw.Stop();
Console.WriteLine("Allocating memory for output stream on server:\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Action default
sw.Reset(); sw.Start();
var actions = client.max_actions_init(maxfile, "default");
client.max_set_param_uint64t(actions, "N", size);
client.max_queue_input(actions, "x", address_dataIn, sizeBytes);
client.max_queue_output(actions, "y", address_dataOut, sizeBytes);
client.max_run(engine, actions);
sw.Stop();
Console.WriteLine("Moving average time:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Unload DFE
sw.Reset(); sw.Start();
client.max_unload (engine);
sw.Stop();
Console.WriteLine("Unloading DFE:\t\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Get output stream from server
sw.Reset(); sw.Start();
List<double> dataOut = client.receive_data_float(address_dataOut, size);
sw.Stop();
Console.WriteLine("Getting output stream:\t(size = {0} bit)\t{1}s", size * 32, sw.Elapsed.TotalMilliseconds / 1000);
// Free allocated memory for streams on server
sw.Reset(); sw.Start();
client.free(address_dataIn);
client.free(address_dataOut);
sw.Stop();
Console.WriteLine("Freeing allocated memory for streams on server:\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Free allocated maxfile data
sw.Reset(); sw.Start();
client.MovingAverage_free();
sw.Stop();
Console.WriteLine("Freeing allocated maxfile data:\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Checking results
sw.Reset(); sw.Start();
int status = check(dataIn, dataOut, size);
sw.Stop();
Console.WriteLine("Checking results:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
sw.Reset(); sw.Start();
transport.Close();
sw.Stop();
Console.WriteLine("Closing connection:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
swDFE.Stop();
Console.WriteLine("DFE moving average total time:\t\t\t{0}s", swDFE.Elapsed.TotalMilliseconds / 1000);
if (status==0) {
Console.WriteLine("Test successful!");
} else {
Console.WriteLine("Test failed {0} times!", status);
Environment.Exit(-1);
}
} catch (SocketException e) {
Console.WriteLine("Could not connect to the server: {0}.", e.Message);
Environment.Exit(-1);
} catch (Exception e) {
Console.WriteLine("An error occured: {0}", e.Message);
Environment.Exit(-1);
}
}
public IDisposable Query()
{
Contract.Requires(Observer != null);
Contract.Requires(StartDate != null);
Contract.Requires(EndDate != null);
// restrict to 1 concurrent active query
mBarrier.Wait();
mBarrier.Reset();
if (Observer != null) // won't ever be null, we're using code contracts to check this!
{
try
{
// create an observable to request the data and encapsulate all the processing logic
return Observable.Create<Data>(o =>
{
var tokenSource = new CancellationTokenSource();
var cancelToken = tokenSource.Token;
_cancellationSource = tokenSource;
var task = Task.Factory.StartNew(() =>
{
// create thrift client connection
var socket = new TSocket(mServer, 5049);
var transport = new TBufferedTransport(socket);
var protocol = new TBinaryProtocol(transport);
var client = new TCLIService.Client(protocol);
// open transport/establish server connection
transport.Open();
var req = new TOpenSessionReq()
{
Client_protocol = TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V1,
Username = mUser,
Password = mPassword
};
var openResp = client.OpenSession(req);
var sessHandle = openResp.SessionHandle;
//-----------------QUERY---------------------------------
#region Query
//Perform actual query -- by switching on option from subscribe
#region check UnionType
bool imbExist = false;
/* if ((Options & SubscriptionOptions.IMBALANCE) == SubscriptionOptions.IMBALANCE)
{
//Console.WriteLine("IMBALANCES");
Options = Options
| SubscriptionOptions.AMEX_IMBALANCE
| SubscriptionOptions.ARCA_IMBALANCE
| SubscriptionOptions.NYSE_IMBALANCE
| SubscriptionOptions.NASD_IMBALANCE;
}*/
if ((Options & SubscriptionOptions.TRADE) == SubscriptionOptions.TRADE)
{
//Console.WriteLine("T");
unionType = "T";
unionNum++;
}
if ((Options & SubscriptionOptions.QUOTE) == SubscriptionOptions.QUOTE)
{
//Console.WriteLine("Q");
unionType = unionType + "Q";
unionNum++;
}
if ((Options & SubscriptionOptions.AMEX_IMBALANCE) == SubscriptionOptions.AMEX_IMBALANCE)
{
//Console.WriteLine("AMEX");
unionType = unionType + "I";
imbExist = true;
unionNum++;
}
if ((Options & SubscriptionOptions.ARCA_IMBALANCE) == SubscriptionOptions.ARCA_IMBALANCE)
{
//Console.WriteLine("ARCA");
if (imbExist == false)
{
unionType = unionType + "I";
imbExist = true;
}
unionNum++;
}
if ((Options & SubscriptionOptions.NYSE_IMBALANCE) == SubscriptionOptions.NYSE_IMBALANCE)
{
//Console.WriteLine("NYSE");
if (imbExist == false)
{
unionType = unionType + "I";
imbExist = true;
}
unionNum++;
}
if ((Options & SubscriptionOptions.NASD_IMBALANCE) == SubscriptionOptions.NASD_IMBALANCE)
{
//Console.WriteLine("NASD");
if (imbExist == false)
{
unionType = unionType + "I";
imbExist = true;
}
unionNum++;
}
#endregion
if ( unionNum == 1 || unionNum==0)
{
#region benchmark test
/*Console.WriteLine("no block marketdata 1");
doFetchTradeNo(client, sessHandle,StartDate,EndDate);
Console.WriteLine("no block marketdata 2");
doFetchTrade2No(client, sessHandle, StartDate, EndDate);
Console.WriteLine("block marketdata 1");
#region single table
for (DateTime QueryDate = StartDate; QueryDate <= EndDate; QueryDate = QueryDate.AddDays(1))
{
if ((Options & SubscriptionOptions.TRADE) == SubscriptionOptions.TRADE)
{
doFetchTrade(client, sessHandle, QueryDate);
}
if ((Options & SubscriptionOptions.QUOTE) == SubscriptionOptions.QUOTE)
{
doFetchQuote(client, sessHandle, QueryDate);
}
if ((Options & SubscriptionOptions.AMEX_IMBALANCE) == SubscriptionOptions.AMEX_IMBALANCE)
{
doFetchAMEXImbalance(client, sessHandle, QueryDate);
}
if ((Options & SubscriptionOptions.ARCA_IMBALANCE) == SubscriptionOptions.ARCA_IMBALANCE)
{
doFetchARCAImbalance(client, sessHandle, QueryDate);
}
if ((Options & SubscriptionOptions.NYSE_IMBALANCE) == SubscriptionOptions.NYSE_IMBALANCE)
{
doFetchNYSEImbalance(client, sessHandle, QueryDate);
}
if ((Options & SubscriptionOptions.NASD_IMBALANCE) == SubscriptionOptions.NASD_IMBALANCE)
{
doFetchNASDImbalance(client, sessHandle, QueryDate);
}
if ((Options & SubscriptionOptions.DAILY) == SubscriptionOptions.DAILY)
{
doFetchDaily(client, sessHandle, QueryDate);
}
}
#endregion
Console.WriteLine("block marketdata 2");*/
#endregion
#region single table
for (DateTime QueryDate = StartDate; QueryDate <= EndDate; QueryDate = QueryDate.AddDays(1))
{
if ((Options & SubscriptionOptions.TRADE) == SubscriptionOptions.TRADE)
{
doFetchTrade2(client, sessHandle, QueryDate, QueryDate.Date==EndDate.Date);
}
if ((Options & SubscriptionOptions.QUOTE) == SubscriptionOptions.QUOTE)
{
doFetchQuote(client, sessHandle, QueryDate, QueryDate.Date == EndDate.Date);
}
if ((Options & SubscriptionOptions.AMEX_IMBALANCE) == SubscriptionOptions.AMEX_IMBALANCE)
{
doFetchAMEXImbalance(client, sessHandle, QueryDate);
}
if ((Options & SubscriptionOptions.ARCA_IMBALANCE) == SubscriptionOptions.ARCA_IMBALANCE)
{
doFetchARCAImbalance(client, sessHandle, QueryDate);
}
if ((Options & SubscriptionOptions.NYSE_IMBALANCE) == SubscriptionOptions.NYSE_IMBALANCE)
{
doFetchNYSEImbalance(client, sessHandle, QueryDate, QueryDate.Date==EndDate.Date);
}
if ((Options & SubscriptionOptions.NASD_IMBALANCE) == SubscriptionOptions.NASD_IMBALANCE)
{
doFetchNASDImbalance(client, sessHandle, QueryDate);
}
if ((Options & SubscriptionOptions.DAILY) == SubscriptionOptions.DAILY)
{
doFetchDaily(client, sessHandle, QueryDate, QueryDate.Date == EndDate.Date);
}
if ((Options & SubscriptionOptions.EXGPRINTS) == SubscriptionOptions.EXGPRINTS)
{
doFetchExgprints(client, sessHandle, QueryDate, QueryDate.Date == EndDate.Date);
}
if ((Options & SubscriptionOptions.NEWS) == SubscriptionOptions.NEWS)
{
doFetchNews(client, sessHandle, QueryDate, QueryDate.Date == EndDate.Date);
}
}
#endregion
}
else
{
#region Union fetchdata
for (DateTime QueryDate = StartDate; QueryDate <= EndDate; QueryDate=QueryDate.AddDays(1))
{
string query = "";
bool firstUnion = false;
if ((Options & SubscriptionOptions.TRADE) == SubscriptionOptions.TRADE)
{
if (firstUnion == true)
{
query = query + " union all " + createUnionTradeQuery(QueryDate);
}
else
{
query = createUnionTradeQuery(QueryDate);
}
firstUnion = true;
}
if ((Options & SubscriptionOptions.QUOTE) == SubscriptionOptions.QUOTE)
{
if (firstUnion == true)
{
query = query + " union all " + createUnionQuoteQuery(QueryDate);
}
else
{
query = createUnionQuoteQuery(QueryDate);
}
firstUnion = true;
}
if ((Options & SubscriptionOptions.AMEX_IMBALANCE) == SubscriptionOptions.AMEX_IMBALANCE)
{
if (firstUnion == true)
{
query = query + " union all " + createUnionAmexQuery(QueryDate);
}
else
{
query = createUnionAmexQuery(QueryDate);
}
firstUnion = true;
}
if ((Options & SubscriptionOptions.ARCA_IMBALANCE) == SubscriptionOptions.ARCA_IMBALANCE)
{
if (firstUnion == true)
{
query = query + " union all " + createUnionArcaQuery(QueryDate);
}
else
{
query = createUnionArcaQuery(QueryDate);
}
firstUnion = true;
}
if ((Options & SubscriptionOptions.NYSE_IMBALANCE) == SubscriptionOptions.NYSE_IMBALANCE)
{
if (firstUnion == true)
{
query = query + " union all " + createUnionNyseQuery(QueryDate);
}
else
{
query = createUnionNyseQuery(QueryDate);
}
firstUnion = true;
}
if ((Options & SubscriptionOptions.NASD_IMBALANCE) == SubscriptionOptions.NASD_IMBALANCE)
{
if (firstUnion == true)
{
query = query + " union all " + createUnionNasdQuery(QueryDate);
}
else
{
query = createUnionNasdQuery(QueryDate);
}
firstUnion = true;
}
if ((Options & SubscriptionOptions.DAILY) == SubscriptionOptions.DAILY)
{
doFetchDaily(client, sessHandle,QueryDate);
}
query = "select * from ( " + query + " ) log order by log.seqno limit " + Maxrows;
// query = "select * from ( " + query + " )";
//Console.WriteLine(query);
doFetchUnion(client, sessHandle, query);
}
#endregion
}
#endregion
//-------------------------------------------------------
// close the session/transport
var closeReq = new TCloseSessionReq
{
SessionHandle = sessHandle
};
client.CloseSession(closeReq);
transport.Close();
}, cancelToken);
return Disposable.Create(tokenSource.Cancel);
}).Subscribe(Observer);
}
catch (Exception ex)
{
Observer.OnError(ex);
}
finally
{
// reset barrier
mBarrier.Set();
}
}
return null;
}