static void _Sets_hbase(string baseName, string columnFamily, DateTime transaction_date, List <global::BatchMutation> data)
{
var socket = null as TSocket;
var transport = null as TBufferedTransport;
try
{
// Сделаем вычисление апи хоста по базе данных
socket = new TSocket(getHost(baseName, columnFamily), Port);
transport = new TBufferedTransport(socket);
var proto = new TBinaryProtocol(transport);
var hbase = new Hbase.Client(proto);
var tableName = getBaseTableHbase(baseName, columnFamily);
transport.Open();
if (data.Count == 1)
{
hbase.mutateRowTs(tableName, data.First().Row, data.First().Mutations, transaction_date.Ticks, null);
}
else
{
hbase.mutateRowsTs(tableName, data, transaction_date.Ticks, null);
}
}
finally
{
if (transport != null)
{
transport.Close();
transport = null;
}
}
}
static void Main(string[] args)
{
// Connection
var socket = new TSocket("hdp1.localdomain", 9090);
var transport = new TBufferedTransport(socket);
var protocol = new TBinaryProtocol(transport);
Hbase.Client hba = new Hbase.Client(protocol);
transport.Open();
// Get table names
Console.WriteLine("<-GET LIST OF TABLES->");
var tableNames = hba.getTableNames();
foreach (var tableName in tableNames)
{
Console.WriteLine(Encoding.UTF8.GetString(tableName, 0, tableName.Length));
}
// Insert rows
Console.WriteLine("<-INSERT->");
Mutation _mutation = new Mutation();
_mutation.IsDelete = false;
_mutation.Column = Encoding.UTF8.GetBytes("image:bodyimage");
_mutation.Value = Encoding.UTF8.GetBytes("newnew image 2.jpg");
hba.mutateRow(Encoding.UTF8.GetBytes("blogposts"), Encoding.UTF8.GetBytes("post1"), new List <Mutation> {
_mutation
}, null);
// Finished
Console.WriteLine("<-FINISHED->");
Console.ReadKey();
}
public string OpenTransaction()
{
try
{
if (_transport == null)
{
return(string.Format("Cannot open Transaction to Client - IP: {0} / Port {1}", _serverIP, _serverPort));
}
if (_transport.IsOpen == false && _socket.IsOpen == false)
{
_transport.Open();
}
return("");
}
catch (Exception exc)
{
AppLogger.logError("{REQUEST-ERROR} Login", exc);
if (_disconnectCallBack != null)
{
_disconnectCallBack();
}
return(exc.Message);
}
}
internal ThriftMessageSender(Uri serviceUri)
{
socket = new TSocket(serviceUri.Host, serviceUri.Port);
transport = new TBufferedTransport(socket);
transport.Open();
client = new MessageService.Client(new TCompactProtocol(transport));
}
public static void test()
{
var hadoop_socket = new TSocket("hserver", 59256);
hadoop_socket.Timeout = 10000;// Ten seconds
//$hadoop_socket -> setRecvTimeout(20000); // Twenty seconds
var hadoop_transport = new TBufferedTransport(hadoop_socket);
var hadoop_protocol = new TBinaryProtocol(hadoop_transport);
var hadoopClient = new ThriftHadoopFileSystem.Client(hadoop_protocol);
hadoop_transport.Open();
//if (hadoopClient.exists(new Pathname { pathname = "/user/root/input" }))
//{
// System.Console.WriteLine("exists");
//}
//else
//{
// Console.WriteLine("no exists");
//}
//hadoopClient.mkdirs(new Pathname { pathname = "/zbw911_user" });
//hadoopClient.rm(new Pathname { pathname = "/zbw911_user" }, true);
//var h = hadoopClient.createFile(new Pathname { pathname = "user_zbw911" }, 1, true, 1000, 1, 64000);
var list = hadoopClient.listStatus(new Pathname {
pathname = "/user/root"
});
foreach (var item in list)
{
System.Console.WriteLine(item);
//if (item.Isdir)
//{
// System.Console.WriteLine("文件夹：" + item.Path + "；权限：" + item.Permission);
//}
//else
//{
// System.Console.WriteLine("文件：" + item.Path + "；权限：" + item.Permission);
//}
}
//hadoopClient.close(h);
//hadoopClient.open(new Pathname { pathname = "/user/root" });
hadoop_transport.Close();
}
public HiveService()
{
var socket = new TSocket(HiveConfigure.Instance.GetDefaultHost(), HiveConfigure.Instance.GetDefaultPort());
transport = new TBufferedTransport(socket);
var proto = new TBinaryProtocol(transport);
client = new ThriftHive.Client(proto);
transport.Open();
}
public MahoutService()
{
var socket = new TSocket(MahoutConfigure.Instance.GetDefaultHost(), MahoutConfigure.Instance.GetDefaultPort());
transport = new TBufferedTransport(socket);
var proto = new TBinaryProtocol(transport);
mahoutClient = new MahoutThriftClient(proto);
_daoInstance = new HiveService.HiveService();
transport.Open();
}
public static void addALotCol()
{
var transport = new TBufferedTransport(new TSocket("master", 9090));
transport.Open();
var protocol = new TBinaryProtocol(transport);
var client = new Apache.Hadoop.Hbase.Hbase.Client(protocol);
//client.createTable(Unitl.StrToBytes("t"), new List<ColumnDescriptor> { new ColumnDescriptor { Name = Unitl.StrToBytes("default"), InMemory = false } });
//client.createTable(Unitl.StrToBytes("thetable"), new List<ColumnDescriptor> { new ColumnDescriptor { Name = Unitl.StrToBytes("default"), InMemory = false } });
for (var i = 0; i < 1000000; i++)
{
client.mutateRow(Unitl.StrToBytes("t1"),
Unitl.StrToBytes("rowno"),
new List <Mutation> {
new Mutation {
Column = Unitl.StrToBytes("f1:" + i), Value = Unitl.StrToBytes(System.DateTime.Now.ToString())
}
});
}
for (var i = 0; i < 10; i++)
{
client.mutateRow(Unitl.StrToBytes("t1"),
Unitl.StrToBytes("rowno"),
new List <Mutation> {
new Mutation {
Column = Unitl.StrToBytes("f2:" + i), Value = Unitl.StrToBytes(System.DateTime.Now.ToString())
}
});
}
for (var i = 0; i < 10; i++)
{
client.mutateRow(Unitl.StrToBytes("t1"),
Unitl.StrToBytes("rowno"),
new List <Mutation> {
new Mutation {
Column = Unitl.StrToBytes("f3:" + i), Value = Unitl.StrToBytes(System.DateTime.Now.ToString())
}
});
}
transport.Close();
}
static void Main(string[] args)
{
TTransport trans = new TSocket("localhost", 8585);
trans = new TBufferedTransport(trans as TStreamTransport);
trans.Open();
TBinaryProtocol proto = new TBinaryProtocol(trans);
helloSvc.Client client = new helloSvc.Client(proto);
string result = client.getMessage("Ginger");
Console.WriteLine("Received from server: " + result);
}
public string OpenTransaction()
{
if (_transport == null)
{
return(string.Format("Cannot open Transaction to Client - IP: {0} / Port {1}", this.IPAddress, this.Port));
}
if (_transport.IsOpen == false)
{
_transport.Open();
}
return("");
}
public List <T> SelectList <T>(string sql) where T : new()
{
Dictionary <int, string> columns = GetColumnsBySql(sql);
List <string> result = null;
try
{
transport.Open();
client.execute(sql);
result = client.fetchAll();
transport.Close();
}
catch (Exception ex)
{
transport.Close();
Console.WriteLine(ex.Message);
LogService.LogService.Instance.Logger(ex.Message, "hive");
}
List <T> list = ClassReflection.GetList <T>(columns, result, ',');
return(list);
}
public static void Scan()
{
var transport = new TBufferedTransport(new TSocket("master", 9090));
transport.Open();
var protocol = new TBinaryProtocol(transport);
var client = new Apache.Hadoop.Hbase.Hbase.Client(protocol);
var scanid = client.scannerOpenWithStop(Unitl.StrToBytes("t1"), Unitl.StrToBytes("rowno"), Unitl.StrToBytes("rowno"),
new List <byte[]> {
Unitl.StrToBytes("f1:99913")
});
var result = client.scannerGet(scanid);
var colum = client.getColumnDescriptors("t1".ToBytes());
var rows = client.getRowsWithColumns("t1".ToBytes()
, new List <byte[]> {
"rowno".ToBytes()
}
, new List <byte[]> {
"f1:9".ToBytes(), "f1:9999".ToBytes()
});
//scanid = client.scannerOpenWithPrefix("".ToBytes(), "".ToBytes(), new List<byte[]> { });
var v = client.getVer("table".ToBytes(), "row".ToBytes(), "col".ToBytes(), 1000);
foreach (var xx in v)
{
}
foreach (var r in result)
{
Console.WriteLine(r.Columns[Unitl.StrToBytes("f1:99913")].Value);
}
client.scannerClose(scanid);
transport.Close();
}
public static void Insert(FuncionarioModel model)
{
try
{
var socket = new TSocket(host, port);
var transport = new TBufferedTransport(socket);
var proto = new TBinaryProtocol(transport);
_hbase = new Hbase.Client(proto);
transport.Open();
_hbase.mutateRows(table_name, new List <BatchMutation>()
{
new BatchMutation()
{
Row = Encoding.UTF8.GetBytes(model.GetRowKey()),
Mutations = new List <Mutation> {
new Mutation {
Column = Encoding.UTF8.GetBytes("fc:ano"), IsDelete = false, Value = Encoding.UTF8.GetBytes(model.Ano.ToString())
},
new Mutation {
Column = Encoding.UTF8.GetBytes("fc:mes"), IsDelete = false, Value = Encoding.UTF8.GetBytes(model.Mes.ToString())
},
new Mutation {
Column = Encoding.UTF8.GetBytes("fc:cpf"), IsDelete = false, Value = Encoding.UTF8.GetBytes(model.Cpf.ToString())
},
new Mutation {
Column = Encoding.UTF8.GetBytes("fc:nome"), IsDelete = false, Value = Encoding.UTF8.GetBytes(model.Nome.ToString())
},
new Mutation {
Column = Encoding.UTF8.GetBytes("fc:salario"), IsDelete = false, Value = Encoding.UTF8.GetBytes(model.Salario.ToString())
},
new Mutation {
Column = Encoding.UTF8.GetBytes("fc:jetons"), IsDelete = false, Value = Encoding.UTF8.GetBytes(model.Jetons.ToString())
}
}
}
});
transport.Close();
}
catch (Exception)
{
throw;
}
}
static void Main(string[] args)
{
int port = 10111;
var socket = new TSocket("ca1.newco.com", port);
var transport = new TBufferedTransport(socket);
var proto = new TBinaryProtocol(transport);
var client = new ThriftHive.Client(proto);
transport.Open();
client.execute("select userAgent from clicks limit 100");
var rows = client.fetchAll();
transport.Close();
rows.ForEach(Console.WriteLine);
}
internal ThriftClient(IHBaseConfiguration configuration)
{
Configuration = configuration;
_thriftSocket = new TSocket(Configuration.ThriftHost, Configuration.ThriftPort);
_thriftTransport = new TBufferedTransport(_thriftSocket);
_thriftProtocol = new TBinaryProtocol(_thriftTransport);
var thriftClient = new Hbase.Client(_thriftProtocol);
_thriftTransport.Open();
_connection = thriftClient;
_timestamp = DateTime.UtcNow.Subtract(new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc)).TotalSeconds +
(Configuration.ThriftTimeout ?? DEFAULT_TIMEOUT);
}
public static ListModel List()
{
try
{
var list = new ListModel();
list.Funcionarios = new List <FuncionarioModel>();
var socket = new TSocket(host, port);
var transport = new TBufferedTransport(socket);
var proto = new TBinaryProtocol(transport);
_hbase = new Hbase.Client(proto);
transport.Open();
//Conectado
var scanner = _hbase.scannerOpen(table_name, Guid.Empty.ToByteArray(), new List <byte[]>()
{
Family
});
for (var entry = _hbase.scannerGet(scanner); entry.Count > 0; entry = _hbase.scannerGet(scanner))
{
foreach (var rowResult in entry)
{
var funcionario = new FuncionarioModel();
funcionario.Key = Encoding.UTF8.GetString(rowResult.Row);
var res = rowResult.Columns.Select(c => BitConverter.ToInt32(c.Value.Value, 0));
//foreach (var cell in res)
//{
// Console.WriteLine("{0}", cell);
//}
list.Funcionarios.Add(funcionario);
}
}
transport.Close();
return(list);
}
catch (Exception)
{
throw;
}
}
public static void Search()
{
try
{
var socket = new TSocket(host, port);
var transport = new TBufferedTransport(socket);
var proto = new TBinaryProtocol(transport);
_hbase = new Hbase.Client(proto);
transport.Open();
//Conectado
transport.Close();
}
catch (Exception)
{
throw;
}
}
static void _CreateDataBase(string baseName, string columnFamily)
{
if (string.IsNullOrWhiteSpace(baseName))
{
return;
}
if (string.IsNullOrWhiteSpace(columnFamily))
{
return;
}
var socket = null as TSocket;
var transport = null as TBufferedTransport;
try
{
// Сделаем вычисление апи хоста по базе данных
socket = new TSocket(getHost(baseName, columnFamily), Port);
transport = new TBufferedTransport(socket);
var proto = new TBinaryProtocol(transport);
var hbase = new Hbase.Client(proto);
transport.Open();
hbase.createTable(
getBaseTableHbase(baseName, columnFamily),
new List <ColumnDescriptor>()
{
new ColumnDescriptor {
Name = getAttrNameHbase(columnFamily), BlockCacheEnabled = true, MaxVersions = 1, InMemory = true
}
});
}
finally
{
if (transport != null)
{
transport.Close();
transport = null;
}
}
}
public static void Run(int times)
{
Parallel.For(0, times, i =>
{
using (TSocket socket = new TSocket("localhost", 8005))
{
using (TTransport transport = new TBufferedTransport(socket))
{
using (TProtocol protocol = new TCompactProtocol(transport))
{
using (var client = new IUserService.Client(protocol))
{
transport.Open();
var list = client.GetAll();
}
}
}
}
});
}
public static void t()
{
var transport = new TBufferedTransport(new TSocket("hserver", 10000));
var protocol = new TBinaryProtocol(transport);
var client = new ThriftHive.Client(protocol);
transport.Open();
//client.execute("CREATE TABLE r(a STRING, b INT, c DOUBLE)");
//client.execute("LOAD TABLE LOCAL INPATH '/path' INTO TABLE r");
var database = new Database();
database.Name = "helloworldDB";
database.Description = "测试用的第一个DB";
client.create_database(database);
transport.Close();
}
static void _DropDataBase(string baseName, string columnFamily)
{
if (string.IsNullOrWhiteSpace(baseName))
{
return;
}
if (string.IsNullOrWhiteSpace(columnFamily))
{
return;
}
var socket = null as TSocket;
var transport = null as TBufferedTransport;
try
{
// Сделаем вычисление апи хоста по базе данных
socket = new TSocket(getHost(baseName, columnFamily), Port);
transport = new TBufferedTransport(socket);
var proto = new TBinaryProtocol(transport);
var hbase = new Hbase.Client(proto);
transport.Open();
var tableName = getBaseTableHbase(baseName, columnFamily);
if (hbase.isTableEnabled(tableName))
{
hbase.disableTable(tableName);
hbase.deleteTable(tableName);
}
}
finally
{
if (transport != null)
{
transport.Close();
transport = null;
}
}
}
public static void GetData()
{
var transport = new TBufferedTransport(new TSocket("master", 9090));
transport.Open();
var protocol = new TBinaryProtocol(transport);
var client = new Apache.Hadoop.Hbase.Hbase.Client(protocol);
var rows = client.getRow(Unitl.StrToBytes("1"), Unitl.StrToBytes("row9916"));
foreach (var row in rows)
{
Console.WriteLine(row.Columns.ToString());
}
transport.Close();
}
public static void ShowTables()
{
var transport = new TBufferedTransport(new TSocket("master", 9090));
transport.Open();
var protocol = new TBinaryProtocol(transport);
var client = new Apache.Hadoop.Hbase.Hbase.Client(protocol);
var tables = client.getTableNames();
foreach (var table in tables)
{
Console.WriteLine(Unitl.BytesToStr(table));
}
transport.Close();
}
public Dictionary <byte[], TCell> GetHBaseImage()
{
var socket = new TSocket(_hbaseThrif, _hbaseThrifPort);
var transport = new TBufferedTransport(socket);
var protocol = new TBinaryProtocol(transport);
Hbase.Client hbaseClient = new Hbase.Client(protocol);
Dictionary <byte[], TCell> hbaseResult = new Dictionary <byte[], TCell>();
transport.Open();
//List<byte[]> tableNames = hc.getTableNames();
byte[] table = Encoding.UTF8.GetBytes("FiservImages");
byte[] row = Encoding.UTF8.GetBytes(_documentId);
byte[] column = Encoding.UTF8.GetBytes("ImageData:Image");
List <TRowResult> results = hbaseClient.getRow(table, row, null);
TCell t = new TCell();
//t.
foreach (TRowResult result in results)
{
//result.Columns[column]
//result.Columns;
hbaseResult = result.Columns;
//string s = Encoding.UTF8.GetString(result.Columns[column].Value);
//foreach (KeyValuePair<byte[], TCell> singleRow in konj)
//{
// //File.WriteAllBytes(@"c:\test\output.docx", singleRow.Value.Value);
// Main a = new Main();
// a.OpenFile(singleRow.Value.Value, "output.docx");
//}
}
return(hbaseResult);
}
//获取连接
public ThriftHadoopFileSystem.Client Connect(out TBufferedTransport tsport)
{
TSocket hadoop_socket = new TSocket(HostIP, HostPort);
//hadoop_socket.Timeout = 10000;// Ten seconds
tsport = new TBufferedTransport(hadoop_socket);
TBinaryProtocol hadoop_protocol = new TBinaryProtocol(tsport, false, false);
ThriftHadoopFileSystem.Client client = new ThriftHadoopFileSystem.Client(hadoop_protocol);
try
{
tsport.Open();
return(client);
}
catch (Exception ee)
{
logger.Error("打开连接失败！", ee);
tsport = null;
return(null);
}
}
public static void AddData(string row)
{
var transport = new TBufferedTransport(new TSocket("master", 9090));
transport.Open();
var protocol = new TBinaryProtocol(transport);
var client = new Apache.Hadoop.Hbase.Hbase.Client(protocol);
//client.createTable(Unitl.StrToBytes(tablename), new List<ColumnDescriptor> { new ColumnDescriptor { Name = Unitl.StrToBytes("zbw911"), InMemory = false } });
client.mutateRow(Unitl.StrToBytes("1"), Unitl.StrToBytes(row), new List <Mutation> {
new Mutation {
Column = Unitl.StrToBytes("zbw911"), Value = Unitl.StrToBytes("zbw911")
}
});
transport.Close();
}
public void GetHBaseImage(string rowKey = "20150822180801118")
{
var socket = new TSocket(hbaseThrif, hbaseThrifPort);
var transport = new TBufferedTransport(socket);
var protocol = new TBinaryProtocol(transport);
Hbase.Client hbaseClient = new Hbase.Client(protocol);
transport.Open();
//List<byte[]> tableNames = hc.getTableNames();
byte[] table = Encoding.UTF8.GetBytes("FiservImages");
byte[] row = Encoding.UTF8.GetBytes(rowKey);
byte[] column = Encoding.UTF8.GetBytes("ImageData:Image");
List <TRowResult> results = hbaseClient.getRow(table, row, null);
TCell t = new TCell();
//t.
foreach (TRowResult result in results)
{
//result.Columns[column]
//result.Columns;
Dictionary <byte[], TCell> konj;
konj = result.Columns;
//string s = Encoding.UTF8.GetString(result.Columns[column].Value);
foreach (KeyValuePair <byte[], TCell> singleRow in konj)
{
//File.WriteAllBytes(@"c:\test\output.docx", singleRow.Value.Value);
OpenFile(singleRow.Value.Value, "output.docx");
}
}
}
static void Main(string[] args)
{
using (TSocket socket = new TSocket("localhost", 8800))
using (TTransport transport = new TBufferedTransport(socket))
using (TProtocol protocol = new TBinaryProtocol(transport))
using (var client = new MessageService.Client(protocol))
{
transport.Open();
Console.WriteLine(client.sendEmailMessage());
;
}
//using (TTransport transport = new TSocket("127.0.0.1", 8800))
//using (TProtocol protocol = new TBinaryProtocol(transport))
//using (var clientUser = new MessageService.Client(protocol))
//{
// transport.Open();
// Console.WriteLine(clientUser.sendEmailMessage());
//}
Console.ReadKey();
}
public static void CorrelateDFE(double[,] data, int sizeTimeseries,
int numTimeseries, double[] correlations)
{
// Calculates correlations on DFE.
Stopwatch sw = new Stopwatch();
sw.Start();
// Make socket
var socket = new TSocket("localhost", 9090);
// Buffering is critical. Raw sockets are very slow
var transport = new TBufferedTransport(socket);
// Wrap in a protocol
var protocol = new TBinaryProtocol(transport);
// Create a client to use the protocol encoder
var client = new correlationService.Client(protocol);
sw.Stop();
Console.WriteLine("Creating a client:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
try {
// Connect!
sw.Reset(); sw.Start();
transport.Open();
sw.Stop();
Console.WriteLine("Opening connection:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
int numTimesteps = sizeTimeseries;
double windowSize = (double)sizeTimeseries;
int numBursts = calcNumBursts(numTimeseries);
// Get loop length
sw.Reset(); sw.Start();
List <int> loopLength = new List <int>();
loopLength.Add(client.correlation_get_CorrelationKernel_loopLength());
sw.Stop();
Console.WriteLine("Getting Correlation Kernel loopLength:\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Prepare data for DFE
sw.Reset(); sw.Start();
double[] precalculations = new double [2 * numTimeseries * numTimesteps];
double[] dataPairs = new double [2 * numTimeseries * numTimesteps];
int burstSize = 384 / 2; // for anything other than ISCA this should be 384
List <int> inMemLoad = new List <int>();
for (int i = 0; i < numBursts * burstSize; i++)
{
inMemLoad.Add(0);
}
prepareDataForDFE(data, sizeTimeseries, numTimeseries, numTimesteps, windowSize,
precalculations, dataPairs);
List <double> precalculationsVec = new List <double>();
List <double> dataPairsVec = new List <double>();
for (int i = 0; i < 2 * numTimeseries * numTimesteps; i++)
{
precalculationsVec.Add(precalculations[i]);
dataPairsVec.Add(dataPairs[i]);
}
sw.Stop();
Console.WriteLine("Data reordering time:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Allocate and send input streams to server
sw.Reset(); sw.Start();
var loopLengthSize = 1;
var addressLoopLength = client.malloc_int32_t(loopLengthSize);
client.send_data_int32_t(addressLoopLength, loopLength);
sw.Stop();
var loopLengthTime = sw.Elapsed.TotalMilliseconds / 1000;
Console.WriteLine("\tSending LoopLength:\t\t(size = {0} bit)\t\t{1}s",
loopLengthSize * 32, loopLengthTime);
sw.Reset(); sw.Start();
var inMemLoadSize = numBursts * burstSize;
var addressInMemLoad = client.malloc_int32_t(inMemLoadSize);
client.send_data_int32_t(addressInMemLoad, inMemLoad);
sw.Stop();
var inMemLoadTime = sw.Elapsed.TotalMilliseconds / 1000;
Console.WriteLine("\tSending InMemLoad:\t\t(size = {0} bit)\t{1}s",
inMemLoadSize * 32, inMemLoadTime);
sw.Reset(); sw.Start();
var precalculationsSize = 2 * numTimeseries * numTimesteps;
var addressPrecalculations = client.malloc_double(precalculationsSize);
client.send_data_double(addressPrecalculations, precalculationsVec);
sw.Stop();
var precalculationsTime = sw.Elapsed.TotalMilliseconds / 1000;
Console.WriteLine("\tSending Precalculations:\t(size = {0} bit)\t{1}s",
precalculationsSize * 32, precalculationsTime);
sw.Reset(); sw.Start();
var dataPairsSize = 2 * numTimeseries * numTimesteps;
var addressDataPairs = client.malloc_double(dataPairsSize);
client.send_data_double(addressDataPairs, dataPairsVec);
sw.Stop();
var dataPairsTime = sw.Elapsed.TotalMilliseconds / 1000;
Console.WriteLine("\tSending DataPairs:\t\t(size = {0} bit)\t{1}s",
dataPairsSize * 64, dataPairsTime);
var time = loopLengthTime + inMemLoadTime + precalculationsTime + dataPairsTime;
var speed = (loopLengthSize * 32 + inMemLoadSize * 32 + precalculationsSize * 64 + dataPairsSize * 64) / time / 1000000;
Console.WriteLine("Sending input streams to server total time:\t{0}s\t(average speed = {1}Mb/s)", time, speed);
// Allocate memory for output stream on server
sw.Reset(); sw.Start();
var addressOutCorrelation = client.malloc_double(numTimesteps * loopLength[0] *
correlationNumTopScores * correlationNumPipes +
numBursts * 24);
var addressOutIndices = client.malloc_int32_t(2 * numTimesteps * loopLength[0] *
correlationNumTopScores * correlationNumPipes);
sw.Stop();
Console.WriteLine("Allocating memory for output stream on server:\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Initialize LMem
sw.Reset(); sw.Start();
client.correlation_loadLMem(numBursts, addressLoopLength, addressInMemLoad);
sw.Stop();
Console.WriteLine("LMem initialization:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
//Executing correlation action
sw.Reset(); sw.Start();
client.correlation(numBursts, // scalar input
numTimesteps, // scalar input
numTimeseries, // scalar input
1, // scalar input
windowSize, // scalar input
addressPrecalculations, // streaming reordered input
addressDataPairs, // streaming reordered input
addressOutCorrelation, // streaming unordered output
addressOutIndices); // streaming unordered output
sw.Stop();
Console.WriteLine("Correlation time:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Get output stream from server
sw.Reset(); sw.Start();
var outCorrelationSize = numTimesteps * loopLength[0] * correlationNumTopScores * correlationNumPipes + numBursts * 24;
List <double> outCorrelation = new List <double>();
outCorrelation = client.receive_data_double(addressOutCorrelation, outCorrelationSize);
sw.Stop();
var outCorrelationTime = sw.Elapsed.TotalMilliseconds / 1000;
Console.WriteLine("\tGet output stream Correlation:\t(size = {0} bit)\t{1}s",
outCorrelationSize * 64, outCorrelationTime);
sw.Reset(); sw.Start();
var outIndicesSzie = 2 * numTimesteps * loopLength[0] * correlationNumTopScores * correlationNumPipes;
List <int> outIndices = new List <int>();
outIndices = client.receive_data_int32_t(addressOutIndices, outIndicesSzie);
sw.Stop();
var outIndicesTime = sw.Elapsed.TotalMilliseconds / 1000;
Console.WriteLine("\tGet output stream outIndices:\t(size = {0} bit)\t{1}s",
outIndicesSzie * 32, outIndicesTime);
sw.Reset(); sw.Start();
loopLengthSize = 1;
loopLength = client.receive_data_int32_t(addressLoopLength, loopLengthSize);
sw.Stop();
loopLengthTime = sw.Elapsed.TotalMilliseconds / 1000;
Console.WriteLine("\tGet output stream loopLength:\t(size = {0} bit)\t\t{1}s",
loopLengthSize * 32, loopLengthTime);
time = loopLengthTime + outCorrelationTime + outIndicesTime;
speed = (loopLengthSize * 32 + outIndicesSzie * 32 + outCorrelationSize * 64) / time / 1000000;
Console.WriteLine("Getting output stream from server total time:\t{0}s\t(average speed = {1}Mb/s)", time, speed);
// Free allocated memory for streams on server
sw.Reset(); sw.Start();
client.free(addressLoopLength);
client.free(addressInMemLoad);
client.free(addressPrecalculations);
client.free(addressDataPairs);
client.free(addressOutCorrelation);
client.free(addressOutIndices);
sw.Stop();
Console.WriteLine("Freeing allocated memory for streams on server:\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Close!
sw.Reset(); sw.Start();
transport.Close();
sw.Stop();
Console.WriteLine("Closing connection:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Store data
sw.Reset(); sw.Start();
int position = 0;
int index = 0;
int start = (numTimesteps - 1) * loopLength[0] * correlationNumTopScores * correlationNumPipes;
for (int i = 0; i < numTimeseries; i++)
{
for (int j = 0; j < i; j++)
{
correlations[index + j] = outCorrelation[start + position + j];
}
index += i;
position += ((i / 12) + 1) * 12;
}
sw.Stop();
Console.WriteLine("LMem initialization:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
} catch (SocketException e) {
Console.WriteLine("Could not connect to the server: {0}.", e.Message);
Environment.Exit(-1);
} catch (Exception e) {
Console.WriteLine("An error occured: {0}", e.Message);
Environment.Exit(-1);
}
}
public static void Main(string[] args)
{
Stopwatch sw = new Stopwatch();
Stopwatch swDFE = new Stopwatch();
sw.Start();
swDFE.Start();
// Make socket
var socket = new TSocket("localhost", 9090);
// Buffering is critical. Raw sockets are very slow
var transport = new TBufferedTransport(socket);
// Wrap in a protocol
var protocol = new TBinaryProtocol(transport);
// Create a client to use the protocol encoder
var client = new MovingAverageService.Client(protocol);
sw.Stop();
Console.WriteLine("Creating a client:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
try {
// Connect!
sw.Reset(); sw.Start();
transport.Open();
sw.Stop();
Console.WriteLine("Opening connection:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
sw.Reset(); sw.Start();
const int size = 384;
List <double> dataIn = new List <double>();
// Generate input data
Random random = new Random();
for (int i = 0; i < size; i++)
{
dataIn.Add(random.Next(0, 100));
}
sw.Stop();
Console.WriteLine("Generating input data:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Initialize maxfile
sw.Reset(); sw.Start();
var maxfile = client.MovingAverage_init();
sw.Stop();
Console.WriteLine("Initializing maxfile:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Load DFE
sw.Reset(); sw.Start();
var engine = client.max_load(maxfile, "*");
sw.Stop();
Console.WriteLine("Loading DFE:\t\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Allocate and send input streams to server
sw.Reset(); sw.Start();
var address_dataIn = client.malloc_float(size);
client.send_data_float(address_dataIn, dataIn);
sw.Stop();
Console.WriteLine("Sending input data:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Allocate memory for output stream on server
sw.Reset(); sw.Start();
var address_dataOut = client.malloc_float(size);
sw.Stop();
Console.WriteLine("Allocating memory for output stream on server:\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Action default
sw.Reset(); sw.Start();
var action = new MovingAverage_actions_t_struct();
action.Param_N = size;
action.Instream_x = address_dataIn;
action.Outstream_y = address_dataOut;
var address_action = client.send_MovingAverage_actions_t(action);
client.MovingAverage_run(engine, address_action);
sw.Stop();
Console.WriteLine("Moving average time:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Unload DFE
sw.Reset(); sw.Start();
client.max_unload(engine);
sw.Stop();
Console.WriteLine("Unloading DFE:\t\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Get output stream from server
sw.Reset(); sw.Start();
List <double> dataOut = client.receive_data_float(address_dataOut, size);
sw.Stop();
Console.WriteLine("Getting output stream:\t(size = {0} bit)\t{1}s", size * 32, sw.Elapsed.TotalMilliseconds / 1000);
// Free allocated memory for streams on server
sw.Reset(); sw.Start();
client.free(address_dataIn);
client.free(address_dataOut);
sw.Stop();
Console.WriteLine("Freeing allocated memory for streams on server:\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Free allocated maxfile data
sw.Reset(); sw.Start();
client.MovingAverage_free();
sw.Stop();
Console.WriteLine("Freeing allocated maxfile data:\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Checking results
sw.Reset(); sw.Start();
int status = check(dataIn, dataOut, size);
sw.Stop();
Console.WriteLine("Checking results:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
sw.Reset(); sw.Start();
transport.Close();
sw.Stop();
Console.WriteLine("Closing connection:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
swDFE.Stop();
Console.WriteLine("DFE moving average total time:\t\t\t{0}s", swDFE.Elapsed.TotalMilliseconds / 1000);
if (status == 0)
{
Console.WriteLine("Test successful!");
}
else
{
Console.WriteLine("Test failed {0} times!", status);
Environment.Exit(-1);
}
} catch (SocketException e) {
Console.WriteLine("Could not connect to the server: {0}.", e.Message);
Environment.Exit(-1);
} catch (Exception e) {
Console.WriteLine("An error occured: {0}", e.Message);
Environment.Exit(-1);
}
}