//获取连接
public ThriftHadoopFileSystem.Client Connect(out TBufferedTransport tsport)
{
if (HostServer == null)
{
throw new ArgumentNullException("HostServer");
}
if (Port == 0)
{
throw new ArgumentNullException("Port");
}
TSocket hadoop_socket = new TSocket(HostServer, Port);
//hadoop_socket.Timeout = 10000;// Ten seconds
tsport = new TBufferedTransport(hadoop_socket);
TBinaryProtocol hadoop_protocol = new TBinaryProtocol(tsport, false, false);
ThriftHadoopFileSystem.Client client = new ThriftHadoopFileSystem.Client(hadoop_protocol);
try
{
tsport.Open();
return client;
}
catch (Exception ex)
{
//throw (new Exception("打开连接失败！", ex));
tsport = null;
return null;
}
}
public void GetHBaseImage(string rowKey = "20150822180801118")
{
var socket = new TSocket(hbaseThrif, hbaseThrifPort);
var transport = new TBufferedTransport(socket);
var protocol = new TBinaryProtocol(transport);
Hbase.Client hbaseClient = new Hbase.Client(protocol);
transport.Open();
//List<byte[]> tableNames = hc.getTableNames();
byte[] table = Encoding.UTF8.GetBytes("FiservImages");
byte[] row = Encoding.UTF8.GetBytes(rowKey);
byte[] column = Encoding.UTF8.GetBytes("ImageData:Image");
List<TRowResult> results = hbaseClient.getRow(table, row, null);
TCell t = new TCell();
//t.
foreach (TRowResult result in results)
{
//result.Columns[column]
//result.Columns;
Dictionary<byte[], TCell> konj;
konj = result.Columns;
//string s = Encoding.UTF8.GetString(result.Columns[column].Value);
foreach (KeyValuePair<byte[], TCell> singleRow in konj)
{
//File.WriteAllBytes(@"c:\test\output.docx", singleRow.Value.Value);
OpenFile(singleRow.Value.Value, "output.docx");
}
}
}
public static void query()
{
var transport = new TBufferedTransport(new TSocket("hserver", 10000));
var protocol = new TBinaryProtocol(transport);
var client = new ThriftHive.Client(protocol);
transport.Open();
//client.execute("CREATE TABLE r(a STRING, b INT, c DOUBLE)");
//client.execute("LOAD TABLE LOCAL INPATH '/path' INTO TABLE r");
client.execute("SELECT * FROM pokes where foo=180");
while (true)
{
var row = client.fetchOne();
if (string.IsNullOrEmpty(row))
break;
System.Console.WriteLine(row);
}
client.execute("SELECT * FROM pokes");
var all = client.fetchAll();
System.Console.WriteLine(all);
transport.Close();
}
internal ThriftMessageSender(Uri serviceUri)
{
socket = new TSocket(serviceUri.Host, serviceUri.Port);
transport = new TBufferedTransport(socket);
transport.Open();
client = new MessageService.Client(new TCompactProtocol(transport));
}
static void Main(string[] args)
{
TTransport trans = new TSocket("localhost", 8585);
trans = new TBufferedTransport(trans as TStreamTransport);
trans.Open();
TBinaryProtocol proto = new TBinaryProtocol(trans);
helloSvc.Client client = new helloSvc.Client(proto);
string result = client.getMessage("Ginger");
Console.WriteLine("Received from server: " + result);
}
public static void test()
{
var hadoop_socket = new TSocket("hserver", 59256);
hadoop_socket.Timeout = 10000;// Ten seconds
//$hadoop_socket -> setRecvTimeout(20000); // Twenty seconds
var hadoop_transport = new TBufferedTransport(hadoop_socket);
var hadoop_protocol = new TBinaryProtocol(hadoop_transport);
var hadoopClient = new ThriftHadoopFileSystem.Client(hadoop_protocol);
hadoop_transport.Open();
//if (hadoopClient.exists(new Pathname { pathname = "/user/root/input" }))
//{
// System.Console.WriteLine("exists");
//}
//else
//{
// Console.WriteLine("no exists");
//}
//hadoopClient.mkdirs(new Pathname { pathname = "/zbw911_user" });
//hadoopClient.rm(new Pathname { pathname = "/zbw911_user" }, true);
//var h = hadoopClient.createFile(new Pathname { pathname = "user_zbw911" }, 1, true, 1000, 1, 64000);
var list = hadoopClient.listStatus(new Pathname { pathname = "/user/root" });
foreach (var item in list)
{
System.Console.WriteLine(item);
//if (item.Isdir)
//{
// System.Console.WriteLine("文件夹：" + item.Path + "；权限：" + item.Permission);
//}
//else
//{
// System.Console.WriteLine("文件：" + item.Path + "；权限：" + item.Permission);
//}
}
//hadoopClient.close(h);
//hadoopClient.open(new Pathname { pathname = "/user/root" });
hadoop_transport.Close();
}
public static void t()
{
var transport = new TBufferedTransport(new TSocket("hserver", 10000));
var protocol = new TBinaryProtocol(transport);
var client = new ThriftHive.Client(protocol);
transport.Open();
//client.execute("CREATE TABLE r(a STRING, b INT, c DOUBLE)");
//client.execute("LOAD TABLE LOCAL INPATH '/path' INTO TABLE r");
var database = new Database();
database.Name = "helloworldDB";
database.Description = "测试用的第一个DB";
client.create_database(database);
transport.Close();
}
//获取连接
public ThriftHadoopFileSystem.Client Connect(out TBufferedTransport tsport)
{
TSocket hadoop_socket = new TSocket(HostIP, HostPort);
//hadoop_socket.Timeout = 10000;// Ten seconds
tsport = new TBufferedTransport(hadoop_socket);
TBinaryProtocol hadoop_protocol = new TBinaryProtocol(tsport, false, false);
ThriftHadoopFileSystem.Client client = new ThriftHadoopFileSystem.Client(hadoop_protocol);
try
{
tsport.Open();
return client;
}
catch (Exception ee)
{
logger.Error("打开连接失败！", ee);
tsport = null;
return null;
}
}
public Dictionary<byte[], TCell> GetHBaseImage()
{
var socket = new TSocket(_hbaseThrif, _hbaseThrifPort);
var transport = new TBufferedTransport(socket);
var protocol = new TBinaryProtocol(transport);
Hbase.Client hbaseClient = new Hbase.Client(protocol);
Dictionary<byte[], TCell> hbaseResult = new Dictionary<byte[], TCell>();
transport.Open();
//List<byte[]> tableNames = hc.getTableNames();
byte[] table = Encoding.UTF8.GetBytes("FiservImages");
byte[] row = Encoding.UTF8.GetBytes(_documentId);
byte[] column = Encoding.UTF8.GetBytes("ImageData:Image");
List<TRowResult> results = hbaseClient.getRow(table, row, null);
TCell t = new TCell();
//t.
foreach (TRowResult result in results)
{
//result.Columns[column]
//result.Columns;
hbaseResult = result.Columns;
//string s = Encoding.UTF8.GetString(result.Columns[column].Value);
//foreach (KeyValuePair<byte[], TCell> singleRow in konj)
//{
// //File.WriteAllBytes(@"c:\test\output.docx", singleRow.Value.Value);
// Main a = new Main();
// a.OpenFile(singleRow.Value.Value, "output.docx");
//}
}
return hbaseResult;
}
public static void t2()
{
var transport = new TBufferedTransport(new TSocket("hserver", 10000));
var protocol = new TBinaryProtocol(transport);
var client = new ThriftHive.Client(protocol);
transport.Open();
var database = new Database();
database.Name = "helloworldDB";
database.Description = "测试用的第一个DB";
var tables = client.get_all_tables("default");
//var tables = client.get_all_databases();
foreach (var table in tables)
{
System.Console.WriteLine(table);
}
transport.Close();
}
public static void ShowTables()
{
var transport = new TBufferedTransport(new TSocket("master", 9090));
transport.Open();
var protocol = new TBinaryProtocol(transport);
var client = new Apache.Hadoop.Hbase.Hbase.Client(protocol);
var tables = client.getTableNames();
foreach (var table in tables)
{
Console.WriteLine(Unitl.BytesToStr(table));
}
transport.Close();
}
public static void Scan()
{
var transport = new TBufferedTransport(new TSocket("master", 9090));
transport.Open();
var protocol = new TBinaryProtocol(transport);
var client = new Apache.Hadoop.Hbase.Hbase.Client(protocol);
var scanid = client.scannerOpenWithStop(Unitl.StrToBytes("t1"), Unitl.StrToBytes("rowno"), Unitl.StrToBytes("rowno"),
new List<byte[]> { Unitl.StrToBytes("f1:99913") });
var result = client.scannerGet(scanid);
var colum = client.getColumnDescriptors("t1".ToBytes());
var rows = client.getRowsWithColumns("t1".ToBytes()
, new List<byte[]> { "rowno".ToBytes() }
, new List<byte[]> { "f1:9".ToBytes(), "f1:9999".ToBytes() });
//scanid = client.scannerOpenWithPrefix("".ToBytes(), "".ToBytes(), new List<byte[]> { });
var v = client.getVer("table".ToBytes(), "row".ToBytes(), "col".ToBytes(), 1000);
foreach (var xx in v)
{
}
foreach (var r in result)
{
Console.WriteLine(r.Columns[Unitl.StrToBytes("f1:99913")].Value);
}
client.scannerClose(scanid);
transport.Close();
}
public static void GetData()
{
var transport = new TBufferedTransport(new TSocket("master", 9090));
transport.Open();
var protocol = new TBinaryProtocol(transport);
var client = new Apache.Hadoop.Hbase.Hbase.Client(protocol);
var rows = client.getRow(Unitl.StrToBytes("1"), Unitl.StrToBytes("row9916"));
foreach (var row in rows)
{
Console.WriteLine(row.Columns.ToString());
}
transport.Close();
}
public static void BatAddData()
{
var transport = new TBufferedTransport(new TSocket("master", 9090));
transport.Open();
var protocol = new TBinaryProtocol(transport);
var client = new Apache.Hadoop.Hbase.Hbase.Client(protocol);
//client.createTable(Unitl.StrToBytes(tablename), new List<ColumnDescriptor> { new ColumnDescriptor { Name = Unitl.StrToBytes("zbw911"), InMemory = false } });
var bytevalue = Unitl.StrToBytes(Longdata);
for (var j = 0; j < 100000; j++)
{
//var j = 0;
for (var i = 0; i < 1000; i++)
{
client.mutateRow(Unitl.StrToBytes("1"), Unitl.StrToBytes("" + i + "___" + j), new List<Mutation> { new Mutation { Column = Unitl.StrToBytes("zbw911"), Value = bytevalue } });
}
Console.WriteLine(j + "批=" + 1000 * j);
}
transport.Close();
}
public static void AddTable(string tablename)
{
var transport = new TBufferedTransport(new TSocket("master", 9090));
transport.Open();
var protocol = new TBinaryProtocol(transport);
var client = new Apache.Hadoop.Hbase.Hbase.Client(protocol);
client.createTable(Unitl.StrToBytes(tablename), new List<ColumnDescriptor> { new ColumnDescriptor { Name = Unitl.StrToBytes("zbw911"), InMemory = false } });
transport.Close();
}
public static void AddColumn()
{
var transport = new TBufferedTransport(new TSocket("master", 9090));
transport.Open();
var protocol = new TBinaryProtocol(transport);
var client = new Apache.Hadoop.Hbase.Hbase.Client(protocol);
//client.createTable(Unitl.StrToBytes("t"), new List<ColumnDescriptor> { new ColumnDescriptor { Name = Unitl.StrToBytes("default"), InMemory = false } });
//client.createTable(Unitl.StrToBytes("thetable"), new List<ColumnDescriptor> { new ColumnDescriptor { Name = Unitl.StrToBytes("default"), InMemory = false } });
for (var i = 0; i < 10; i++)
{
client.mutateRow(Unitl.StrToBytes("t"),
Unitl.StrToBytes("key"),
new List<Mutation> {
new Mutation { Column = Unitl.StrToBytes("default:"+i ), Value = Unitl.StrToBytes( System.DateTime.Now.ToString() ) }
});
}
transport.Close();
}
static void Main(string[] args)
{
Log.Use().SimpleConsoleColored();
Log.Information("Booting");
try
{
string host = ConfigurationManager.AppSettings["Interface.Host"];
int port = Int32.Parse(ConfigurationManager.AppSettings["Interface.Port"]);
var socket = new TSocket(host, port);
var transport = new TBufferedTransport(socket);
var protocol = new TBinaryProtocol(transport);
var client = new InterfaceNodeService.Client(protocol);
transport.Open();
#region Input loop
bool doContinueLoop = true;
string cmd = String.Empty;
string input = String.Empty;
Entry tmpEntry = null;
Result res = null;
while (doContinueLoop)
{
cmd = Prompt("What do you want to do? (insert, getreport, getset, quit)");
switch (cmd)
{
#region Quit
case "quit":
{
doContinueLoop = false;
break;
}
#endregion Quit
#region Insert
case "insert":
{
tmpEntry = new Entry();
tmpEntry.Data = new Dictionary<string, string>();
input = Prompt("Enter an Entry type");
tmpEntry.Type = input;
string tmpVarName = String.Empty;
string tmpVarValue = String.Empty;
while (true)
{
tmpVarName = Prompt("Enter name of var (or \"send\")");
if (tmpVarName.Equals("send"))
{
break;
}
tmpVarValue = Prompt("Enter the value of the var (or \"send\")");
if (tmpVarValue.Equals("send"))
{
break;
}
tmpEntry.Data.Add(tmpVarName, tmpVarValue);
}
Log.Trace("Sending Insert");
res = client.Insert(tmpEntry);
Log.Debug(res.DumpToString());
tmpEntry = null;
break;
}
#endregion Insert
#region Get report
case "getreport":
{
input = Prompt("Enter the name of the report you wish to get");
Log.Trace("Sending GetReport");
res = client.GetReport(input);
Log.Debug(res.DumpToString());
break;
}
#endregion Get report
#region Get set
case "getset":
{
input = Prompt("Enter the name of the set you wish to get");
Log.Trace("Sending GetSet");
res = client.GetSet(input);
Log.Debug(res.DumpToString());
break;
}
#endregion Get set
default:
{
Log.Error("Invalid command '{0}'", cmd);
break;
}
}
}
#endregion Input loop
}
catch(Exception ex)
{
Log.Error(ex.ToString());
}
Log.Information("Done... closing");
}
public static void CorrelateDFE(double[,] data, int sizeTimeseries,
int numTimeseries, double[] correlations)
{
// Calculates correlations on DFE.
Stopwatch sw = new Stopwatch();
sw.Start();
// Make socket
var socket = new TSocket("localhost", 9090);
// Buffering is critical. Raw sockets are very slow
var transport = new TBufferedTransport(socket);
// Wrap in a protocol
var protocol = new TBinaryProtocol(transport);
// Create a client to use the protocol encoder
var client = new correlationService.Client(protocol);
sw.Stop();
Console.WriteLine("Creating a client:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
try {
// Connect!
sw.Reset(); sw.Start();
transport.Open();
sw.Stop();
Console.WriteLine("Opening connection:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
int numTimesteps = sizeTimeseries;
double windowSize = (double)sizeTimeseries;
int numBursts = calcNumBursts(numTimeseries);
// Get loop length
sw.Reset(); sw.Start();
List<int> loopLength = new List<int>();
loopLength.Add(client.correlation_get_CorrelationKernel_loopLength());
sw.Stop();
Console.WriteLine("Getting Correlation Kernel loopLength:\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Prepare data for DFE
sw.Reset(); sw.Start();
double[] precalculations = new double [2 * numTimeseries * numTimesteps];
double[] dataPairs = new double [2 * numTimeseries * numTimesteps];
int burstSize = 384 / 2; // for anything other than ISCA this should be 384
List<int> inMemLoad = new List<int>();
for(int i = 0; i < numBursts * burstSize; i++) {
inMemLoad.Add(0);
}
prepareDataForDFE (data, sizeTimeseries, numTimeseries, numTimesteps, windowSize,
precalculations, dataPairs);
List<double> precalculationsVec = new List<double>();
List<double> dataPairsVec = new List<double>();
for(int i = 0; i < 2 * numTimeseries * numTimesteps; i++) {
precalculationsVec.Add(precalculations[i]);
dataPairsVec.Add(dataPairs[i]);
}
sw.Stop();
Console.WriteLine("Data reordering time:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Allocate and send input streams to server
sw.Reset(); sw.Start();
var loopLengthSize = 1;
var addressLoopLength = client.malloc_int32_t(loopLengthSize);
client.send_data_int32_t(addressLoopLength, loopLength);
sw.Stop();
var loopLengthTime = sw.Elapsed.TotalMilliseconds / 1000;
Console.WriteLine("\tSending LoopLength:\t\t(size = {0} bit)\t\t{1}s",
loopLengthSize * 32, loopLengthTime);
sw.Reset(); sw.Start();
var inMemLoadSize = numBursts * burstSize;
var addressInMemLoad = client.malloc_int32_t(inMemLoadSize);
client.send_data_int32_t(addressInMemLoad, inMemLoad);
sw.Stop();
var inMemLoadTime = sw.Elapsed.TotalMilliseconds / 1000;
Console.WriteLine("\tSending InMemLoad:\t\t(size = {0} bit)\t{1}s",
inMemLoadSize * 32, inMemLoadTime);
sw.Reset(); sw.Start();
var precalculationsSize = 2 * numTimeseries * numTimesteps;
var addressPrecalculations = client.malloc_double(precalculationsSize);
client.send_data_double(addressPrecalculations, precalculationsVec);
sw.Stop();
var precalculationsTime = sw.Elapsed.TotalMilliseconds / 1000;
Console.WriteLine("\tSending Precalculations:\t(size = {0} bit)\t{1}s",
precalculationsSize * 32, precalculationsTime);
sw.Reset(); sw.Start();
var dataPairsSize = 2 * numTimeseries * numTimesteps;
var addressDataPairs = client.malloc_double(dataPairsSize);
client.send_data_double(addressDataPairs, dataPairsVec);
sw.Stop();
var dataPairsTime = sw.Elapsed.TotalMilliseconds / 1000;
Console.WriteLine("\tSending DataPairs:\t\t(size = {0} bit)\t{1}s",
dataPairsSize * 64, dataPairsTime);
var time = loopLengthTime + inMemLoadTime + precalculationsTime + dataPairsTime;
var speed = (loopLengthSize * 32 + inMemLoadSize * 32 + precalculationsSize * 64 + dataPairsSize * 64) / time / 1000000;
Console.WriteLine("Sending input streams to server total time:\t{0}s\t(average speed = {1}Mb/s)", time, speed);
// Allocate memory for output stream on server
sw.Reset(); sw.Start();
var addressOutCorrelation = client.malloc_double(numTimesteps * loopLength[0] *
correlationNumTopScores * correlationNumPipes +
numBursts * 24);
var addressOutIndices = client.malloc_int32_t(2 * numTimesteps * loopLength[0] *
correlationNumTopScores * correlationNumPipes);
sw.Stop();
Console.WriteLine("Allocating memory for output stream on server:\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Initialize LMem
sw.Reset(); sw.Start();
client.correlation_loadLMem(numBursts, addressLoopLength, addressInMemLoad);
sw.Stop();
Console.WriteLine("LMem initialization:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
//Executing correlation action
sw.Reset(); sw.Start();
client.correlation(numBursts, // scalar input
numTimesteps, // scalar input
numTimeseries, // scalar input
1, // scalar input
windowSize, // scalar input
addressPrecalculations, // streaming reordered input
addressDataPairs, // streaming reordered input
addressOutCorrelation, // streaming unordered output
addressOutIndices); // streaming unordered output
sw.Stop();
Console.WriteLine("Correlation time:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Get output stream from server
sw.Reset(); sw.Start();
var outCorrelationSize = numTimesteps * loopLength[0] * correlationNumTopScores * correlationNumPipes + numBursts * 24;
List<double> outCorrelation = new List<double>();
outCorrelation = client.receive_data_double(addressOutCorrelation, outCorrelationSize);
sw.Stop();
var outCorrelationTime = sw.Elapsed.TotalMilliseconds / 1000;
Console.WriteLine("\tGet output stream Correlation:\t(size = {0} bit)\t{1}s",
outCorrelationSize * 64, outCorrelationTime);
sw.Reset(); sw.Start();
var outIndicesSzie = 2 * numTimesteps * loopLength[0] * correlationNumTopScores * correlationNumPipes;
List<int> outIndices = new List<int>();
outIndices = client.receive_data_int32_t(addressOutIndices, outIndicesSzie);
sw.Stop();
var outIndicesTime = sw.Elapsed.TotalMilliseconds / 1000;
Console.WriteLine("\tGet output stream outIndices:\t(size = {0} bit)\t{1}s",
outIndicesSzie * 32, outIndicesTime);
sw.Reset(); sw.Start();
loopLengthSize = 1;
loopLength = client.receive_data_int32_t(addressLoopLength, loopLengthSize);
sw.Stop();
loopLengthTime = sw.Elapsed.TotalMilliseconds / 1000;
Console.WriteLine("\tGet output stream loopLength:\t(size = {0} bit)\t\t{1}s",
loopLengthSize * 32, loopLengthTime);
time = loopLengthTime + outCorrelationTime + outIndicesTime;
speed = (loopLengthSize * 32 + outIndicesSzie * 32 + outCorrelationSize * 64) / time / 1000000;
Console.WriteLine("Getting output stream from server total time:\t{0}s\t(average speed = {1}Mb/s)", time, speed);
// Free allocated memory for streams on server
sw.Reset(); sw.Start();
client.free(addressLoopLength);
client.free(addressInMemLoad);
client.free(addressPrecalculations);
client.free(addressDataPairs);
client.free(addressOutCorrelation);
client.free(addressOutIndices);
sw.Stop();
Console.WriteLine("Freeing allocated memory for streams on server:\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Close!
sw.Reset(); sw.Start();
transport.Close();
sw.Stop();
Console.WriteLine("Closing connection:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Store data
sw.Reset(); sw.Start();
int position = 0;
int index = 0;
int start = (numTimesteps-1) * loopLength[0] * correlationNumTopScores * correlationNumPipes;
for (int i=0; i < numTimeseries; i++) {
for(int j = 0; j < i; j++) {
correlations[index + j] = outCorrelation[start + position + j];
}
index += i;
position += ((i/12)+1)*12;
}
sw.Stop();
Console.WriteLine("LMem initialization:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
} catch (SocketException e) {
Console.WriteLine("Could not connect to the server: {0}.", e.Message);
Environment.Exit(-1);
} catch (Exception e) {
Console.WriteLine("An error occured: {0}", e.Message);
Environment.Exit(-1);
}
}
protected void HBaseMutateImage(string rowKey, byte[] image)
{
var socket = new TSocket("172.24.2.144", 9090);
var transport = new TBufferedTransport(socket);
var protocol = new TBinaryProtocol(transport);
Hbase.Client hc = new Hbase.Client(protocol);
transport.Open();
//List<byte[]> tableNames = hc.getTableNames();
byte[] table = Encoding.UTF8.GetBytes("FiservImages");
byte[] row = Encoding.UTF8.GetBytes(rowKey);
byte[] column = Encoding.UTF8.GetBytes("ImageData:Image");
List<Mutation> mutation = new List<Mutation>();
Mutation m = new Mutation();
m.Column = column;
m.Value = image;
m.IsDelete = false;
m.WriteToWAL = true;
mutation.Add(m);
hc.mutateRow(table, row, mutation, null);
}
public static void UpdateData()
{
var transport = new TBufferedTransport(new TSocket("master", 9090));
transport.Open();
var protocol = new TBinaryProtocol(transport);
var client = new Apache.Hadoop.Hbase.Hbase.Client(protocol);
//client.createTable(Unitl.StrToBytes(tablename), new List<ColumnDescriptor> { new ColumnDescriptor { Name = Unitl.StrToBytes("zbw911"), InMemory = false } });
client.mutateRow(Unitl.StrToBytes("1"), Unitl.StrToBytes("key"), new List<Mutation> { new Mutation { Column = Unitl.StrToBytes("zbw911"), Value = Unitl.StrToBytes("zbw911-Modifyed") } });
transport.Close();
}
public static void Main(string[] args)
{
Stopwatch sw = new Stopwatch();
Stopwatch swDFE = new Stopwatch();
sw.Start();
swDFE.Start();
// Make socket
var socket = new TSocket("localhost", 9090);
// Buffering is critical. Raw sockets are very slow
var transport = new TBufferedTransport(socket);
// Wrap in a protocol
var protocol = new TBinaryProtocol(transport);
// Create a client to use the protocol encoder
var client = new MovingAverageService.Client(protocol);
sw.Stop();
Console.WriteLine("Creating a client:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
try {
// Connect!
sw.Reset(); sw.Start();
transport.Open();
sw.Stop();
Console.WriteLine("Opening connection:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
sw.Reset(); sw.Start();
const int size = 384;
int sizeBytes = size * 4;
List<double> dataIn = new List<double>();
// Generate input data
Random random = new Random();
for(int i = 0; i<size; i++) {
dataIn.Add(random.Next(0, 100));
}
sw.Stop();
Console.WriteLine("Generating input data:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Initialize maxfile
sw.Reset(); sw.Start();
var maxfile = client.MovingAverage_init();
sw.Stop();
Console.WriteLine("Initializing maxfile:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Load DFE
sw.Reset(); sw.Start();
var engine = client.max_load(maxfile, "*");
sw.Stop();
Console.WriteLine("Loading DFE:\t\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Allocate and send input streams to server
sw.Reset(); sw.Start();
var address_dataIn = client.malloc_float(size);
client.send_data_float(address_dataIn, dataIn);
sw.Stop();
Console.WriteLine("Sending input data:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Allocate memory for output stream on server
sw.Reset(); sw.Start();
var address_dataOut = client.malloc_float(size);
sw.Stop();
Console.WriteLine("Allocating memory for output stream on server:\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Action default
sw.Reset(); sw.Start();
var actions = client.max_actions_init(maxfile, "default");
client.max_set_param_uint64t(actions, "N", size);
client.max_queue_input(actions, "x", address_dataIn, sizeBytes);
client.max_queue_output(actions, "y", address_dataOut, sizeBytes);
client.max_run(engine, actions);
sw.Stop();
Console.WriteLine("Moving average time:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Unload DFE
sw.Reset(); sw.Start();
client.max_unload (engine);
sw.Stop();
Console.WriteLine("Unloading DFE:\t\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Get output stream from server
sw.Reset(); sw.Start();
List<double> dataOut = client.receive_data_float(address_dataOut, size);
sw.Stop();
Console.WriteLine("Getting output stream:\t(size = {0} bit)\t{1}s", size * 32, sw.Elapsed.TotalMilliseconds / 1000);
// Free allocated memory for streams on server
sw.Reset(); sw.Start();
client.free(address_dataIn);
client.free(address_dataOut);
sw.Stop();
Console.WriteLine("Freeing allocated memory for streams on server:\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Free allocated maxfile data
sw.Reset(); sw.Start();
client.MovingAverage_free();
sw.Stop();
Console.WriteLine("Freeing allocated maxfile data:\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Checking results
sw.Reset(); sw.Start();
int status = check(dataIn, dataOut, size);
sw.Stop();
Console.WriteLine("Checking results:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
sw.Reset(); sw.Start();
transport.Close();
sw.Stop();
Console.WriteLine("Closing connection:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
swDFE.Stop();
Console.WriteLine("DFE moving average total time:\t\t\t{0}s", swDFE.Elapsed.TotalMilliseconds / 1000);
if (status==0) {
Console.WriteLine("Test successful!");
} else {
Console.WriteLine("Test failed {0} times!", status);
Environment.Exit(-1);
}
} catch (SocketException e) {
Console.WriteLine("Could not connect to the server: {0}.", e.Message);
Environment.Exit(-1);
} catch (Exception e) {
Console.WriteLine("An error occured: {0}", e.Message);
Environment.Exit(-1);
}
}
public static List<double> PassThroughDFE(int size, List<double> dataIn)
{
List<double> dataOut = new List<double>();
Stopwatch sw = new Stopwatch();
sw.Start();
// Make socket
var socket = new TSocket("localhost", 9090);
// Buffering is critical. Raw sockets are very slow
var transport = new TBufferedTransport(socket);
// Wrap in a protocol
var protocol = new TBinaryProtocol(transport);
// Create a client to use the protocol encoder
var client = new PassThroughService.Client(protocol);
sw.Stop();
Console.WriteLine("Creating a client:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
try {
// Connect!
sw.Reset(); sw.Start();
transport.Open();
sw.Stop();
Console.WriteLine("Opening connection:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Initialize maxfile
sw.Reset(); sw.Start();
var maxfile = client.PassThrough_init();
sw.Stop();
Console.WriteLine("Initializing maxfile:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Load DFE
sw.Reset(); sw.Start();
var engine = client.max_load(maxfile, "*");
sw.Stop();
Console.WriteLine("Loading DFE:\t\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Allocate and send input streams to server
sw.Reset(); sw.Start();
var address_dataIn = client.malloc_float(size);
client.send_data_float(address_dataIn, dataIn);
sw.Stop();
Console.WriteLine("Sending input data:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Allocate memory for output stream on server
sw.Reset(); sw.Start();
var address_dataOut = client.malloc_float(size);
sw.Stop();
Console.WriteLine("Allocating memory for output stream on server:\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Action default
sw.Reset(); sw.Start();
var action = new PassThrough_actions_t_struct();
action.Param_N = size;
action.Instream_x = address_dataIn;
action.Outstream_y = address_dataOut;
var address_action = client.send_PassThrough_actions_t(action);
client.PassThrough_run(engine, address_action);
sw.Stop();
Console.WriteLine("Pass through time:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Unload DFE
sw.Reset(); sw.Start();
client.max_unload(engine);
sw.Stop();
Console.WriteLine("Unloading DFE:\t\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Get output stream from server
sw.Reset(); sw.Start();
dataOut = client.receive_data_float(address_dataOut, size);
sw.Stop();
Console.WriteLine("Getting output stream:\t(size = {0} bit)\t{1}s", size * 32, sw.Elapsed.TotalMilliseconds / 1000);
// Free allocated memory for streams on server
sw.Reset(); sw.Start();
client.free(address_dataIn);
client.free(address_dataOut);
sw.Stop();
Console.WriteLine("Freeing allocated memory for streams on server:\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Free allocated maxfile data
sw.Reset(); sw.Start();
client.PassThrough_free();
sw.Stop();
Console.WriteLine("Freeing allocated maxfile data:\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
sw.Reset(); sw.Start();
transport.Close();
sw.Stop();
Console.WriteLine("Closing connection:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
} catch (SocketException e) {
Console.WriteLine("Could not connect to the server: {0}.", e.Message);
Environment.Exit(-1);
} catch (Exception e) {
Console.WriteLine("An error occured: {0}", e.Message);
Environment.Exit(-1);
}
return dataOut;
}
public IDisposable Query()
{
Contract.Requires(Observer != null);
Contract.Requires(StartDate != null);
Contract.Requires(EndDate != null);
// restrict to 1 concurrent active query
mBarrier.Wait();
mBarrier.Reset();
if (Observer != null) // won't ever be null, we're using code contracts to check this!
{
try
{
// create an observable to request the data and encapsulate all the processing logic
return Observable.Create<Data>(o =>
{
var tokenSource = new CancellationTokenSource();
var cancelToken = tokenSource.Token;
_cancellationSource = tokenSource;
var task = Task.Factory.StartNew(() =>
{
// create thrift client connection
var socket = new TSocket(mServer, 5049);
var transport = new TBufferedTransport(socket);
var protocol = new TBinaryProtocol(transport);
var client = new TCLIService.Client(protocol);
// open transport/establish server connection
transport.Open();
var req = new TOpenSessionReq()
{
Client_protocol = TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V1,
Username = mUser,
Password = mPassword
};
var openResp = client.OpenSession(req);
var sessHandle = openResp.SessionHandle;
//-----------------QUERY---------------------------------
#region Query
//Perform actual query -- by switching on option from subscribe
#region check UnionType
bool imbExist = false;
/* if ((Options & SubscriptionOptions.IMBALANCE) == SubscriptionOptions.IMBALANCE)
{
//Console.WriteLine("IMBALANCES");
Options = Options
| SubscriptionOptions.AMEX_IMBALANCE
| SubscriptionOptions.ARCA_IMBALANCE
| SubscriptionOptions.NYSE_IMBALANCE
| SubscriptionOptions.NASD_IMBALANCE;
}*/
if ((Options & SubscriptionOptions.TRADE) == SubscriptionOptions.TRADE)
{
//Console.WriteLine("T");
unionType = "T";
unionNum++;
}
if ((Options & SubscriptionOptions.QUOTE) == SubscriptionOptions.QUOTE)
{
//Console.WriteLine("Q");
unionType = unionType + "Q";
unionNum++;
}
if ((Options & SubscriptionOptions.AMEX_IMBALANCE) == SubscriptionOptions.AMEX_IMBALANCE)
{
//Console.WriteLine("AMEX");
unionType = unionType + "I";
imbExist = true;
unionNum++;
}
if ((Options & SubscriptionOptions.ARCA_IMBALANCE) == SubscriptionOptions.ARCA_IMBALANCE)
{
//Console.WriteLine("ARCA");
if (imbExist == false)
{
unionType = unionType + "I";
imbExist = true;
}
unionNum++;
}
if ((Options & SubscriptionOptions.NYSE_IMBALANCE) == SubscriptionOptions.NYSE_IMBALANCE)
{
//Console.WriteLine("NYSE");
if (imbExist == false)
{
unionType = unionType + "I";
imbExist = true;
}
unionNum++;
}
if ((Options & SubscriptionOptions.NASD_IMBALANCE) == SubscriptionOptions.NASD_IMBALANCE)
{
//Console.WriteLine("NASD");
if (imbExist == false)
{
unionType = unionType + "I";
imbExist = true;
}
unionNum++;
}
#endregion
if ( unionNum == 1 || unionNum==0)
{
#region benchmark test
/*Console.WriteLine("no block marketdata 1");
doFetchTradeNo(client, sessHandle,StartDate,EndDate);
Console.WriteLine("no block marketdata 2");
doFetchTrade2No(client, sessHandle, StartDate, EndDate);
Console.WriteLine("block marketdata 1");
#region single table
for (DateTime QueryDate = StartDate; QueryDate <= EndDate; QueryDate = QueryDate.AddDays(1))
{
if ((Options & SubscriptionOptions.TRADE) == SubscriptionOptions.TRADE)
{
doFetchTrade(client, sessHandle, QueryDate);
}
if ((Options & SubscriptionOptions.QUOTE) == SubscriptionOptions.QUOTE)
{
doFetchQuote(client, sessHandle, QueryDate);
}
if ((Options & SubscriptionOptions.AMEX_IMBALANCE) == SubscriptionOptions.AMEX_IMBALANCE)
{
doFetchAMEXImbalance(client, sessHandle, QueryDate);
}
if ((Options & SubscriptionOptions.ARCA_IMBALANCE) == SubscriptionOptions.ARCA_IMBALANCE)
{
doFetchARCAImbalance(client, sessHandle, QueryDate);
}
if ((Options & SubscriptionOptions.NYSE_IMBALANCE) == SubscriptionOptions.NYSE_IMBALANCE)
{
doFetchNYSEImbalance(client, sessHandle, QueryDate);
}
if ((Options & SubscriptionOptions.NASD_IMBALANCE) == SubscriptionOptions.NASD_IMBALANCE)
{
doFetchNASDImbalance(client, sessHandle, QueryDate);
}
if ((Options & SubscriptionOptions.DAILY) == SubscriptionOptions.DAILY)
{
doFetchDaily(client, sessHandle, QueryDate);
}
}
#endregion
Console.WriteLine("block marketdata 2");*/
#endregion
#region single table
for (DateTime QueryDate = StartDate; QueryDate <= EndDate; QueryDate = QueryDate.AddDays(1))
{
if ((Options & SubscriptionOptions.TRADE) == SubscriptionOptions.TRADE)
{
doFetchTrade2(client, sessHandle, QueryDate, QueryDate.Date==EndDate.Date);
}
if ((Options & SubscriptionOptions.QUOTE) == SubscriptionOptions.QUOTE)
{
doFetchQuote(client, sessHandle, QueryDate, QueryDate.Date == EndDate.Date);
}
if ((Options & SubscriptionOptions.AMEX_IMBALANCE) == SubscriptionOptions.AMEX_IMBALANCE)
{
doFetchAMEXImbalance(client, sessHandle, QueryDate);
}
if ((Options & SubscriptionOptions.ARCA_IMBALANCE) == SubscriptionOptions.ARCA_IMBALANCE)
{
doFetchARCAImbalance(client, sessHandle, QueryDate);
}
if ((Options & SubscriptionOptions.NYSE_IMBALANCE) == SubscriptionOptions.NYSE_IMBALANCE)
{
doFetchNYSEImbalance(client, sessHandle, QueryDate, QueryDate.Date==EndDate.Date);
}
if ((Options & SubscriptionOptions.NASD_IMBALANCE) == SubscriptionOptions.NASD_IMBALANCE)
{
doFetchNASDImbalance(client, sessHandle, QueryDate);
}
if ((Options & SubscriptionOptions.DAILY) == SubscriptionOptions.DAILY)
{
doFetchDaily(client, sessHandle, QueryDate, QueryDate.Date == EndDate.Date);
}
if ((Options & SubscriptionOptions.EXGPRINTS) == SubscriptionOptions.EXGPRINTS)
{
doFetchExgprints(client, sessHandle, QueryDate, QueryDate.Date == EndDate.Date);
}
if ((Options & SubscriptionOptions.NEWS) == SubscriptionOptions.NEWS)
{
doFetchNews(client, sessHandle, QueryDate, QueryDate.Date == EndDate.Date);
}
}
#endregion
}
else
{
#region Union fetchdata
for (DateTime QueryDate = StartDate; QueryDate <= EndDate; QueryDate=QueryDate.AddDays(1))
{
string query = "";
bool firstUnion = false;
if ((Options & SubscriptionOptions.TRADE) == SubscriptionOptions.TRADE)
{
if (firstUnion == true)
{
query = query + " union all " + createUnionTradeQuery(QueryDate);
}
else
{
query = createUnionTradeQuery(QueryDate);
}
firstUnion = true;
}
if ((Options & SubscriptionOptions.QUOTE) == SubscriptionOptions.QUOTE)
{
if (firstUnion == true)
{
query = query + " union all " + createUnionQuoteQuery(QueryDate);
}
else
{
query = createUnionQuoteQuery(QueryDate);
}
firstUnion = true;
}
if ((Options & SubscriptionOptions.AMEX_IMBALANCE) == SubscriptionOptions.AMEX_IMBALANCE)
{
if (firstUnion == true)
{
query = query + " union all " + createUnionAmexQuery(QueryDate);
}
else
{
query = createUnionAmexQuery(QueryDate);
}
firstUnion = true;
}
if ((Options & SubscriptionOptions.ARCA_IMBALANCE) == SubscriptionOptions.ARCA_IMBALANCE)
{
if (firstUnion == true)
{
query = query + " union all " + createUnionArcaQuery(QueryDate);
}
else
{
query = createUnionArcaQuery(QueryDate);
}
firstUnion = true;
}
if ((Options & SubscriptionOptions.NYSE_IMBALANCE) == SubscriptionOptions.NYSE_IMBALANCE)
{
if (firstUnion == true)
{
query = query + " union all " + createUnionNyseQuery(QueryDate);
}
else
{
query = createUnionNyseQuery(QueryDate);
}
firstUnion = true;
}
if ((Options & SubscriptionOptions.NASD_IMBALANCE) == SubscriptionOptions.NASD_IMBALANCE)
{
if (firstUnion == true)
{
query = query + " union all " + createUnionNasdQuery(QueryDate);
}
else
{
query = createUnionNasdQuery(QueryDate);
}
firstUnion = true;
}
if ((Options & SubscriptionOptions.DAILY) == SubscriptionOptions.DAILY)
{
doFetchDaily(client, sessHandle,QueryDate);
}
query = "select * from ( " + query + " ) log order by log.seqno limit " + Maxrows;
// query = "select * from ( " + query + " )";
//Console.WriteLine(query);
doFetchUnion(client, sessHandle, query);
}
#endregion
}
#endregion
//-------------------------------------------------------
// close the session/transport
var closeReq = new TCloseSessionReq
{
SessionHandle = sessHandle
};
client.CloseSession(closeReq);
transport.Close();
}, cancelToken);
return Disposable.Create(tokenSource.Cancel);
}).Subscribe(Observer);
}
catch (Exception ex)
{
Observer.OnError(ex);
}
finally
{
// reset barrier
mBarrier.Set();
}
}
return null;
}