public IClient Create(IEndpoint endpoint, IClientPool ownerPool)
{
TSocket socket = null;
TTransport transport = null;
if (endpoint.Timeout == 0)
{
socket = new TSocket(endpoint.Address, endpoint.Port);
}
else
{
socket = new TSocket(endpoint.Address, endpoint.Port, endpoint.Timeout);
}
TcpClient tcpClient = socket.TcpClient;
if (this.isBufferSizeSet)
{
transport = new TBufferedTransport(socket, this.bufferSize);
}
else
{
transport = new TBufferedTransport(socket);
}
TProtocol protocol = new TBinaryProtocol(transport);
CassandraClient cassandraClient = new CassandraClient(protocol);
IClient client = new DefaultClient() {
CassandraClient = cassandraClient,
Endpoint = endpoint,
OwnerPool = ownerPool,
TcpClient = tcpClient,
Created = DateTime.Now
};
return client;
}
//获取连接
public ThriftHadoopFileSystem.Client Connect(out TBufferedTransport tsport)
{
if (HostServer == null)
{
throw new ArgumentNullException("HostServer");
}
if (Port == 0)
{
throw new ArgumentNullException("Port");
}
TSocket hadoop_socket = new TSocket(HostServer, Port);
//hadoop_socket.Timeout = 10000;// Ten seconds
tsport = new TBufferedTransport(hadoop_socket);
TBinaryProtocol hadoop_protocol = new TBinaryProtocol(tsport, false, false);
ThriftHadoopFileSystem.Client client = new ThriftHadoopFileSystem.Client(hadoop_protocol);
try
{
tsport.Open();
return client;
}
catch (Exception ex)
{
//throw (new Exception("打开连接失败！", ex));
tsport = null;
return null;
}
}
public static void query()
{
var transport = new TBufferedTransport(new TSocket("hserver", 10000));
var protocol = new TBinaryProtocol(transport);
var client = new ThriftHive.Client(protocol);
transport.Open();
//client.execute("CREATE TABLE r(a STRING, b INT, c DOUBLE)");
//client.execute("LOAD TABLE LOCAL INPATH '/path' INTO TABLE r");
client.execute("SELECT * FROM pokes where foo=180");
while (true)
{
var row = client.fetchOne();
if (string.IsNullOrEmpty(row))
break;
System.Console.WriteLine(row);
}
client.execute("SELECT * FROM pokes");
var all = client.fetchAll();
System.Console.WriteLine(all);
transport.Close();
}
public void GetHBaseImage(string rowKey = "20150822180801118")
{
var socket = new TSocket(hbaseThrif, hbaseThrifPort);
var transport = new TBufferedTransport(socket);
var protocol = new TBinaryProtocol(transport);
Hbase.Client hbaseClient = new Hbase.Client(protocol);
transport.Open();
//List<byte[]> tableNames = hc.getTableNames();
byte[] table = Encoding.UTF8.GetBytes("FiservImages");
byte[] row = Encoding.UTF8.GetBytes(rowKey);
byte[] column = Encoding.UTF8.GetBytes("ImageData:Image");
List<TRowResult> results = hbaseClient.getRow(table, row, null);
TCell t = new TCell();
//t.
foreach (TRowResult result in results)
{
//result.Columns[column]
//result.Columns;
Dictionary<byte[], TCell> konj;
konj = result.Columns;
//string s = Encoding.UTF8.GetString(result.Columns[column].Value);
foreach (KeyValuePair<byte[], TCell> singleRow in konj)
{
//File.WriteAllBytes(@"c:\test\output.docx", singleRow.Value.Value);
OpenFile(singleRow.Value.Value, "output.docx");
}
}
}
/// <summary>
/// Callback for Accept Implementation
/// </summary>
/// <returns>
/// TTransport-object.
/// </returns>
protected override TTransport AcceptImpl()
{
if (this.server == null)
{
throw new TTransportException(TTransportException.ExceptionType.NotOpen, "No underlying server socket.");
}
try
{
TcpClient client = this.server.AcceptTcpClient();
client.SendTimeout = client.ReceiveTimeout = this.clientTimeout;
//wrap the client in an SSL Socket passing in the SSL cert
TTLSSocket socket = new TTLSSocket(client, this.serverCertificate, true, this.clientCertValidator);
socket.setupTLS();
if (useBufferedSockets)
{
TBufferedTransport trans = new TBufferedTransport(socket);
return(trans);
}
else
{
return(socket);
}
}
catch (Exception ex)
{
throw new TTransportException(ex.ToString());
}
}
protected override TTransport AcceptImpl()
{
if (server == null)
{
throw new TTransportException(TTransportException.ExceptionType.NotOpen, "No underlying server socket.");
}
try
{
TcpClient result = server.AcceptTcpClient();
TSocket result2 = new TSocket(result);
result2.Timeout = clientTimeout;
if (useBufferedSockets)
{
TBufferedTransport result3 = new TBufferedTransport(result2);
return(result3);
}
else
{
return(result2);
}
}
catch (Exception ex)
{
throw new TTransportException(ex.ToString());
}
}
public CassandraClient Create(CassandraEndpoint endpoint)
{
TSocket socket = null;
TTransport transport = null;
if (endpoint.Timeout == 0)
{
socket = new TSocket(endpoint.Address, endpoint.Port);
}
else
{
socket = new TSocket(endpoint.Address, endpoint.Port, endpoint.Timeout);
}
if (this.isBufferSizeSet)
{
transport = new TBufferedTransport(socket, this.bufferSize);
}
else
{
transport = new TBufferedTransport(socket);
}
TProtocol protocol = new TBinaryProtocol(transport);
Cassandra.Client cassandraClient = new Cassandra.Client(protocol);
CassandraClient client = new CassandraClient(cassandraClient, endpoint);
this.logger.Debug(logger.StringFormatInvariantCulture("Created a new connection using: '{0}'", endpoint.ToString()));
return client;
}
internal ThriftMessageSender(Uri serviceUri)
{
socket = new TSocket(serviceUri.Host, serviceUri.Port);
transport = new TBufferedTransport(socket);
transport.Open();
client = new MessageService.Client(new TCompactProtocol(transport));
}
static void Main(string[] args)
{
TTransport trans = new TSocket("localhost", 8585);
trans = new TBufferedTransport(trans as TStreamTransport);
trans.Open();
TBinaryProtocol proto = new TBinaryProtocol(trans);
helloSvc.Client client = new helloSvc.Client(proto);
string result = client.getMessage("Ginger");
Console.WriteLine("Received from server: " + result);
}
protected override TTransport AcceptImpl()
{
if (server == null)
{
throw new TTransportException(TTransportException.ExceptionType.NotOpen, "No underlying server socket.");
}
try
{
TSocket result2 = null;
#if NETSTANDARD1_4 || NETSTANDARD1_5
TcpClient result = server.AcceptTcpClientAsync().Result;
#else
TcpClient result = server.AcceptTcpClient();
#endif
try
{
result2 = new TSocket(result);
result2.Timeout = clientTimeout;
if (useBufferedSockets)
{
#if NETSTANDARD1_4
throw new NotSupportedException("Not supported with NetStandard 1.4, use NetStandard 1.5 instead.");
#else
TBufferedTransport result3 = new TBufferedTransport(result2);
return(result3);
#endif
}
else
{
return(result2);
}
}
catch (System.Exception)
{
// If a TSocket was successfully created, then let
// it do proper cleanup of the TcpClient object.
if (result2 != null)
{
result2.Dispose();
}
else // Otherwise, clean it up ourselves.
{
((IDisposable)result).Dispose();
}
throw;
}
}
catch (Exception ex)
{
throw new TTransportException(ex.ToString());
}
}
public static void test()
{
var hadoop_socket = new TSocket("hserver", 59256);
hadoop_socket.Timeout = 10000;// Ten seconds
//$hadoop_socket -> setRecvTimeout(20000); // Twenty seconds
var hadoop_transport = new TBufferedTransport(hadoop_socket);
var hadoop_protocol = new TBinaryProtocol(hadoop_transport);
var hadoopClient = new ThriftHadoopFileSystem.Client(hadoop_protocol);
hadoop_transport.Open();
//if (hadoopClient.exists(new Pathname { pathname = "/user/root/input" }))
//{
// System.Console.WriteLine("exists");
//}
//else
//{
// Console.WriteLine("no exists");
//}
//hadoopClient.mkdirs(new Pathname { pathname = "/zbw911_user" });
//hadoopClient.rm(new Pathname { pathname = "/zbw911_user" }, true);
//var h = hadoopClient.createFile(new Pathname { pathname = "user_zbw911" }, 1, true, 1000, 1, 64000);
var list = hadoopClient.listStatus(new Pathname { pathname = "/user/root" });
foreach (var item in list)
{
System.Console.WriteLine(item);
//if (item.Isdir)
//{
// System.Console.WriteLine("文件夹：" + item.Path + "；权限：" + item.Permission);
//}
//else
//{
// System.Console.WriteLine("文件：" + item.Path + "；权限：" + item.Permission);
//}
}
//hadoopClient.close(h);
//hadoopClient.open(new Pathname { pathname = "/user/root" });
hadoop_transport.Close();
}
protected override TTransport AcceptImpl()
{
TTransport tBufferedTransport;
if (this.server == null)
{
throw new TTransportException(TTransportException.ExceptionType.NotOpen, "No underlying server socket.");
}
try
{
TSocket tSocket = null;
var t = this.server.AcceptTcpClientAsync();
t.Wait();
TcpClient tcpClient = t.Result;
try
{
tSocket = new TSocket(tcpClient)
{
Timeout = this.clientTimeout
};
if (!this.useBufferedSockets)
{
tBufferedTransport = tSocket;
}
else
{
tBufferedTransport = new TBufferedTransport(tSocket);
}
}
catch (Exception exception)
{
if (tSocket == null)
{
((IDisposable)tcpClient).Dispose();
}
else
{
tSocket.Dispose();
}
throw;
}
}
catch (Exception exception1)
{
throw new TTransportException(exception1.ToString());
}
return(tBufferedTransport);
}
protected override TTransport AcceptImpl()
{
if (server == null)
{
throw new TTransportException(TTransportException.ExceptionType.NotOpen, "No underlying server socket.");
}
try
{
TSocket result2 = null;
TcpClient result = server.AcceptTcpClient();
try
{
result2 = new TSocket(result);
result2.Timeout = clientTimeout;
if (useBufferedSockets)
{
TBufferedTransport result3 = new TBufferedTransport(result2);
return(result3);
}
else
{
return(result2);
}
}
catch (System.Exception)
{
// If a TSocket was successfully created, then let
// it do proper cleanup of the TcpClient object.
if (result2 != null)
{
result2.Dispose();
}
else // Otherwise, clean it up ourselves.
{
((IDisposable)result).Dispose();
}
throw;
}
}
catch (SocketException ex) when(ex.SocketErrorCode.Equals(SocketError.Interrupted))
{
throw new TTransportException(TTransportException.ExceptionType.Interrupted, ex.ToString());
}
catch (Exception ex)
{
throw new TTransportException(ex.ToString());
}
}
public ThriftConnection(IConnectionSettings connectionSettings)
{
this._timeout = connectionSettings.Timeout;
this._poolSize = connectionSettings.MaximumAsyncConnections;
this._resourceLock = new Semaphore(_poolSize, _poolSize);
for (var i = 0; i <= connectionSettings.MaximumAsyncConnections; i++)
{
var tsocket = new TSocket(connectionSettings.Host, connectionSettings.Port);
var transport = new TBufferedTransport(tsocket, 1024);
var protocol = new TBinaryProtocol(transport);
var client = new Rest.Client(protocol);
_clients.Enqueue(client);
}
}
/// <summary>
/// Callback for Accept Implementation
/// </summary>
/// <returns>
/// TTransport-object.
/// </returns>
protected override TTransport AcceptImpl()
{
if (this.server == null)
{
throw new TTransportException(TTransportException.ExceptionType.NotOpen, "No underlying server socket.");
}
try
{
#if NETSTANDARD1_4 || NETSTANDARD1_5
TcpClient client = this.server.AcceptTcpClientAsync().Result;
#else
TcpClient client = this.server.AcceptTcpClient();
#endif
client.SendTimeout = client.ReceiveTimeout = this.clientTimeout;
//wrap the client in an SSL Socket passing in the SSL cert
TTLSSocket socket = new TTLSSocket(
client,
this.serverCertificate,
true,
this.clientCertValidator,
this.localCertificateSelectionCallback
);
socket.setupTLS();
if (useBufferedSockets)
{
#if NETSTANDARD1_4
throw new NotImplementedException("Not supported with NetStandard 1.4, use NetStandard 1.5 instead.");
#else
TBufferedTransport trans = new TBufferedTransport(socket);
return(trans);
#endif
}
else
{
return(socket);
}
}
catch (Exception ex)
{
throw new TTransportException(ex.ToString());
}
}
public TTransport CreateTransport()
{
if (url == null)
{
// endpoint transport
TTransport trans = null;
if (pipe != null)
trans = new TNamedPipeClientTransport(pipe);
else
{
if (encrypted)
{
string certPath = "../keys/client.p12";
X509Certificate cert = new X509Certificate2(certPath, "thrift");
trans = new TTLSSocket(host, port, 0, cert, (o, c, chain, errors) => true, null, SslProtocols.Tls);
}
else
{
trans = new TSocket(host, port);
}
}
// layered transport
if (buffered)
trans = new TBufferedTransport(trans);
if (framed)
trans = new TFramedTransport(trans);
if (_isFirstTransport)
{
//ensure proper open/close of transport
trans.Open();
trans.Close();
_isFirstTransport = false;
}
return trans;
}
else
{
return new THttpClient(new Uri(url));
}
}
public static void t()
{
var transport = new TBufferedTransport(new TSocket("hserver", 10000));
var protocol = new TBinaryProtocol(transport);
var client = new ThriftHive.Client(protocol);
transport.Open();
//client.execute("CREATE TABLE r(a STRING, b INT, c DOUBLE)");
//client.execute("LOAD TABLE LOCAL INPATH '/path' INTO TABLE r");
var database = new Database();
database.Name = "helloworldDB";
database.Description = "测试用的第一个DB";
client.create_database(database);
transport.Close();
}
/// <summary>
/// Callback for Accept Implementation
/// </summary>
/// <returns>
/// TTransport-object.
/// </returns>
protected override TTransport AcceptImpl()
{
if (server == null)
{
throw new TTransportException(TTransportException.ExceptionType.NotOpen, "No underlying server socket.");
}
try
{
var client = server.AcceptTcpClient();
client.SendTimeout = client.ReceiveTimeout = clientTimeout;
//wrap the client in an SSL Socket passing in the SSL cert
var socket = new TTLSSocket(
client,
serverCertificate,
true,
clientCertValidator,
localCertificateSelectionCallback,
sslProtocols);
socket.setupTLS();
if (useBufferedSockets)
{
var trans = new TBufferedTransport(socket);
return(trans);
}
else
{
return(socket);
}
}
catch (Exception ex)
{
throw new TTransportException(ex.ToString(), ex);
}
}
//获取连接
public ThriftHadoopFileSystem.Client Connect(out TBufferedTransport tsport)
{
TSocket hadoop_socket = new TSocket(HostIP, HostPort);
//hadoop_socket.Timeout = 10000;// Ten seconds
tsport = new TBufferedTransport(hadoop_socket);
TBinaryProtocol hadoop_protocol = new TBinaryProtocol(tsport, false, false);
ThriftHadoopFileSystem.Client client = new ThriftHadoopFileSystem.Client(hadoop_protocol);
try
{
tsport.Open();
return client;
}
catch (Exception ee)
{
logger.Error("打开连接失败！", ee);
tsport = null;
return null;
}
}
public Dictionary<byte[], TCell> GetHBaseImage()
{
var socket = new TSocket(_hbaseThrif, _hbaseThrifPort);
var transport = new TBufferedTransport(socket);
var protocol = new TBinaryProtocol(transport);
Hbase.Client hbaseClient = new Hbase.Client(protocol);
Dictionary<byte[], TCell> hbaseResult = new Dictionary<byte[], TCell>();
transport.Open();
//List<byte[]> tableNames = hc.getTableNames();
byte[] table = Encoding.UTF8.GetBytes("FiservImages");
byte[] row = Encoding.UTF8.GetBytes(_documentId);
byte[] column = Encoding.UTF8.GetBytes("ImageData:Image");
List<TRowResult> results = hbaseClient.getRow(table, row, null);
TCell t = new TCell();
//t.
foreach (TRowResult result in results)
{
//result.Columns[column]
//result.Columns;
hbaseResult = result.Columns;
//string s = Encoding.UTF8.GetString(result.Columns[column].Value);
//foreach (KeyValuePair<byte[], TCell> singleRow in konj)
//{
// //File.WriteAllBytes(@"c:\test\output.docx", singleRow.Value.Value);
// Main a = new Main();
// a.OpenFile(singleRow.Value.Value, "output.docx");
//}
}
return hbaseResult;
}
public static void t2()
{
var transport = new TBufferedTransport(new TSocket("hserver", 10000));
var protocol = new TBinaryProtocol(transport);
var client = new ThriftHive.Client(protocol);
transport.Open();
var database = new Database();
database.Name = "helloworldDB";
database.Description = "测试用的第一个DB";
var tables = client.get_all_tables("default");
//var tables = client.get_all_databases();
foreach (var table in tables)
{
System.Console.WriteLine(table);
}
transport.Close();
}
public static void BatAddData()
{
var transport = new TBufferedTransport(new TSocket("master", 9090));
transport.Open();
var protocol = new TBinaryProtocol(transport);
var client = new Apache.Hadoop.Hbase.Hbase.Client(protocol);
//client.createTable(Unitl.StrToBytes(tablename), new List<ColumnDescriptor> { new ColumnDescriptor { Name = Unitl.StrToBytes("zbw911"), InMemory = false } });
var bytevalue = Unitl.StrToBytes(Longdata);
for (var j = 0; j < 100000; j++)
{
//var j = 0;
for (var i = 0; i < 1000; i++)
{
client.mutateRow(Unitl.StrToBytes("1"), Unitl.StrToBytes("" + i + "___" + j), new List<Mutation> { new Mutation { Column = Unitl.StrToBytes("zbw911"), Value = bytevalue } });
}
Console.WriteLine(j + "批=" + 1000 * j);
}
transport.Close();
}
public static void AddColumn()
{
var transport = new TBufferedTransport(new TSocket("master", 9090));
transport.Open();
var protocol = new TBinaryProtocol(transport);
var client = new Apache.Hadoop.Hbase.Hbase.Client(protocol);
//client.createTable(Unitl.StrToBytes("t"), new List<ColumnDescriptor> { new ColumnDescriptor { Name = Unitl.StrToBytes("default"), InMemory = false } });
//client.createTable(Unitl.StrToBytes("thetable"), new List<ColumnDescriptor> { new ColumnDescriptor { Name = Unitl.StrToBytes("default"), InMemory = false } });
for (var i = 0; i < 10; i++)
{
client.mutateRow(Unitl.StrToBytes("t"),
Unitl.StrToBytes("key"),
new List<Mutation> {
new Mutation { Column = Unitl.StrToBytes("default:"+i ), Value = Unitl.StrToBytes( System.DateTime.Now.ToString() ) }
});
}
transport.Close();
}
public static void AddTable(string tablename)
{
var transport = new TBufferedTransport(new TSocket("master", 9090));
transport.Open();
var protocol = new TBinaryProtocol(transport);
var client = new Apache.Hadoop.Hbase.Hbase.Client(protocol);
client.createTable(Unitl.StrToBytes(tablename), new List<ColumnDescriptor> { new ColumnDescriptor { Name = Unitl.StrToBytes("zbw911"), InMemory = false } });
transport.Close();
}
protected override TTransport AcceptImpl()
{
if (server == null)
{
throw new TTransportException(TTransportException.ExceptionType.NotOpen, "No underlying server socket.");
}
try
{
TSocket result2 = null;
TcpClient result = server.AcceptTcpClient();
try
{
result2 = new TSocket(result);
result2.Timeout = clientTimeout;
if (useBufferedSockets)
{
TBufferedTransport result3 = new TBufferedTransport(result2);
return result3;
}
else
{
return result2;
}
}
catch (System.Exception)
{
// If a TSocket was successfully created, then let
// it do proper cleanup of the TcpClient object.
if (result2 != null)
result2.Dispose();
else // Otherwise, clean it up ourselves.
((IDisposable)result).Dispose();
throw;
}
}
catch (Exception ex)
{
throw new TTransportException(ex.ToString());
}
}
/// <summary>
/// Callback for Accept Implementation
/// </summary>
/// <returns>
/// TTransport-object.
/// </returns>
protected override TTransport AcceptImpl()
{
if (this.server == null)
{
throw new TTransportException(TTransportException.ExceptionType.NotOpen, "No underlying server socket.");
}
try
{
TcpClient client = this.server.AcceptTcpClient();
client.SendTimeout = client.ReceiveTimeout = this.clientTimeout;
//wrap the client in an SSL Socket passing in the SSL cert
TTLSSocket socket = new TTLSSocket(
client,
this.serverCertificate,
true,
this.clientCertValidator,
this.localCertificateSelectionCallback,
this.sslProtocols);
socket.setupTLS();
if (useBufferedSockets)
{
TBufferedTransport trans = new TBufferedTransport(socket);
return trans;
}
else
{
return socket;
}
}
catch (Exception ex)
{
throw new TTransportException(ex.ToString());
}
}
static void Main(string[] args)
{
Log.Use().SimpleConsoleColored();
Log.Information("Booting");
try
{
string host = ConfigurationManager.AppSettings["Interface.Host"];
int port = Int32.Parse(ConfigurationManager.AppSettings["Interface.Port"]);
var socket = new TSocket(host, port);
var transport = new TBufferedTransport(socket);
var protocol = new TBinaryProtocol(transport);
var client = new InterfaceNodeService.Client(protocol);
transport.Open();
#region Input loop
bool doContinueLoop = true;
string cmd = String.Empty;
string input = String.Empty;
Entry tmpEntry = null;
Result res = null;
while (doContinueLoop)
{
cmd = Prompt("What do you want to do? (insert, getreport, getset, quit)");
switch (cmd)
{
#region Quit
case "quit":
{
doContinueLoop = false;
break;
}
#endregion Quit
#region Insert
case "insert":
{
tmpEntry = new Entry();
tmpEntry.Data = new Dictionary<string, string>();
input = Prompt("Enter an Entry type");
tmpEntry.Type = input;
string tmpVarName = String.Empty;
string tmpVarValue = String.Empty;
while (true)
{
tmpVarName = Prompt("Enter name of var (or \"send\")");
if (tmpVarName.Equals("send"))
{
break;
}
tmpVarValue = Prompt("Enter the value of the var (or \"send\")");
if (tmpVarValue.Equals("send"))
{
break;
}
tmpEntry.Data.Add(tmpVarName, tmpVarValue);
}
Log.Trace("Sending Insert");
res = client.Insert(tmpEntry);
Log.Debug(res.DumpToString());
tmpEntry = null;
break;
}
#endregion Insert
#region Get report
case "getreport":
{
input = Prompt("Enter the name of the report you wish to get");
Log.Trace("Sending GetReport");
res = client.GetReport(input);
Log.Debug(res.DumpToString());
break;
}
#endregion Get report
#region Get set
case "getset":
{
input = Prompt("Enter the name of the set you wish to get");
Log.Trace("Sending GetSet");
res = client.GetSet(input);
Log.Debug(res.DumpToString());
break;
}
#endregion Get set
default:
{
Log.Error("Invalid command '{0}'", cmd);
break;
}
}
}
#endregion Input loop
}
catch(Exception ex)
{
Log.Error(ex.ToString());
}
Log.Information("Done... closing");
}
protected override TTransport AcceptImpl()
{
if (server == null)
{
throw new TTransportException(TTransportException.ExceptionType.NotOpen, "No underlying server socket.");
}
try
{
TcpClient result = server.AcceptTcpClient();
var result2 = new TSocket(result);
result2.Timeout = clientTimeout;
if (useBufferedSockets)
{
var result3 = new TBufferedTransport(result2);
return result3;
}
else
{
return result2;
}
}
catch (Exception ex)
{
throw new TTransportException(ex.ToString());
}
}
//批量上传
public void MutUpload(BackgroundWorker worker, List <string> localPath, string remoteRootPath, string localRootPath)
{
//相同操作
bool sameOp = false;
//是否覆盖
bool IsOver = false;
//准备创建连接
Thrift.Transport.TBufferedTransport btsport = null;
ThriftHadoopFileSystem.Client thriftClient = Connect(out btsport);
List <string> NoSuccessList = new List <string>();
List <string> skipList = new List <string>();
if (thriftClient != null)//连接成功
{
int totalCount = localPath.Count;
int currentCount = 0;
//开始上传
worker.ReportProgress(0, new ProgressState()
{
ListBoxMsg = "开始上传！", totalCount = totalCount, CurrentCount = 0
});
//循环
foreach (string localFilePath in localPath)
{
string fileName = Path.GetFileName(localFilePath);
if (!string.IsNullOrEmpty(localRootPath))//如果是文件夹，则包含原有路径
{
fileName = localFilePath.Replace(localRootPath, "");
}
string remoteFilePath = remoteRootPath + "/" + fileName;
currentCount++;
int pgPresent = (int)((double)currentCount / totalCount * 100);
//显示总进度
worker.ReportProgress(pgPresent, new ProgressState()
{
CurrentTitle = fileName, CurrentCount = currentCount
});
#region 是否存在
if (!sameOp)
{
bool exsitFile = thriftClient.exists(new Pathname()
{
pathname = remoteFilePath
});
if (exsitFile)
{
SureDialog myDialog = new SureDialog();
MyShowDialogResult myDR = new MyShowDialogResult();
myDialog.ShowDialog(fileName, myDR);
sameOp = myDR.IsCheck;
IsOver = myDR.Result;
if (!myDR.Result)
{
worker.ReportProgress(pgPresent, new ProgressState()
{
ListBoxMsg = fileName + " 跳过 "
});
skipList.Add(fileName);
continue;
}
}
}
else
{
bool exsitFile = thriftClient.exists(new Pathname()
{
pathname = remoteFilePath
});
if (exsitFile)
{
if (!IsOver)
{
worker.ReportProgress(pgPresent, new ProgressState()
{
ListBoxMsg = fileName + "跳过 "
});
skipList.Add(fileName);
continue;
}
}
}
#endregion
#region 单个文件
bool singleResult = false;
ThriftHandle th = null;
FileStream fs = null;
try
{
Pathname myNewFile = new Pathname()
{
pathname = remoteFilePath
};
//创建一个文件
th = thriftClient.createFile(myNewFile, 1, true, 1024 * 1024 * 10, ConfigHelper.HDFSREPLICATION, 1024 * 1024 * 512);
UTF8Encoding utf8 = new UTF8Encoding(false, true);
fs = new FileStream(localFilePath, FileMode.Open, FileAccess.Read);
byte[] fileBuffer = new byte[1024 * 1024 * 10]; // 每次传1MB
int bytesRead;
long bytesTotal = 0;
while ((bytesRead = fs.Read(fileBuffer, 0, fileBuffer.Length)) > 0)
{
bytesTotal += bytesRead;
byte[] realBuffer = new byte[bytesRead];
Array.Copy(fileBuffer, realBuffer, bytesRead);
//将utf8转为可存储编码
realBuffer = Encoding.Convert(Encoding.GetEncoding("iso-8859-1"), utf8, realBuffer);
//发送
thriftClient.write(th, realBuffer);
//清仓缓存
Array.Clear(fileBuffer, 0, fileBuffer.Length);
realBuffer = null;
//显示单个上传进度
int mypresent = (int)((double)bytesTotal / fs.Length * 100);
worker.ReportProgress(pgPresent, new ProgressState()
{
CurrentTitle = mypresent + "% " + fileName
});
}
singleResult = true;
}
catch (Exception ee)
{
//显示上传错误
worker.ReportProgress(pgPresent, new ProgressState()
{
ListBoxMsg = ee.Message
});
}
finally
{
if (th != null)
{
thriftClient.close(th);
}
if (fs != null)
{
fs.Close();
}
}
#endregion
//显示单个上传结果
string msg = string.Format("{0} 上传{1}", fileName, singleResult ? "成功" : "失败");
worker.ReportProgress(pgPresent, new ProgressState()
{
ListBoxMsg = msg
});
if (!singleResult)
{
NoSuccessList.Add(fileName);
}
}
}
//释放连接
if (btsport != null)
{
btsport.Close();
}
//输出没有上传成功的
if (NoSuccessList.Count > 0)
{
try
{
File.WriteAllText("c:/UploadNoSuccess.txt", string.Join("\r\n", NoSuccessList.ToArray()));
worker.ReportProgress(100, new ProgressState()
{
ListBoxMsg = NoSuccessList.Count + "个上传错误！请查看c:/UploadNoSuccess.txt"
});
}
catch (Exception ee)
{
worker.ReportProgress(100, new ProgressState()
{
ListBoxMsg = ee.Message
});
}
}
//输出跳过的
if (skipList.Count > 0)
{
try
{
File.WriteAllText("c:/UploadSkip.txt", string.Join("\r\n", skipList.ToArray()));
worker.ReportProgress(100, new ProgressState()
{
ListBoxMsg = "跳过" + skipList.Count + "个文件！请查看c:/UploadSkip.txt"
});
}
catch (Exception ee)
{
worker.ReportProgress(100, new ProgressState()
{
ListBoxMsg = ee.Message
});
}
}
}
public static void Execute(string[] args)
{
try
{
string host = "localhost";
int port = 9090;
string url = null, pipe = null;
int numThreads = 1;
bool buffered = false, framed = false;
try
{
for (int i = 0; i < args.Length; i++)
{
if (args[i] == "-h")
{
string[] hostport = args[++i].Split(':');
host = hostport[0];
if (hostport.Length > 1)
{
port = Convert.ToInt32(hostport[1]);
}
}
else if (args[i] == "-u")
{
url = args[++i];
}
else if (args[i] == "-n")
{
numIterations = Convert.ToInt32(args[++i]);
}
else if (args[i] == "-b" || args[i] == "-buffered")
{
buffered = true;
Console.WriteLine("Using buffered sockets");
}
else if (args[i] == "-f" || args[i] == "-framed")
{
framed = true;
Console.WriteLine("Using framed transport");
}
else if (args[i] == "-pipe") // -pipe <name>
{
pipe = args[++i];
Console.WriteLine("Using named pipes transport");
}
else if (args[i] == "-t")
{
numThreads = Convert.ToInt32(args[++i]);
}
}
}
catch (Exception e)
{
Console.WriteLine(e.StackTrace);
}
//issue tests on separate threads simultaneously
Thread[] threads = new Thread[numThreads];
DateTime start = DateTime.Now;
for (int test = 0; test < numThreads; test++)
{
Thread t = new Thread(new ParameterizedThreadStart(ClientThread));
threads[test] = t;
if (url == null)
{
// endpoint transport
TTransport trans = null;
if( pipe != null)
trans = new TNamedPipeClientTransport(pipe);
else
trans = new TSocket(host, port);
// layered transport
if (buffered)
trans = new TBufferedTransport(trans as TStreamTransport);
if (framed)
trans = new TFramedTransport(trans);
//ensure proper open/close of transport
trans.Open();
trans.Close();
t.Start(trans);
}
else
{
THttpClient http = new THttpClient(new Uri(url));
t.Start(http);
}
}
for (int test = 0; test < numThreads; test++)
{
threads[test].Join();
}
Console.Write("Total time: " + (DateTime.Now - start));
}
catch (Exception outerEx)
{
Console.WriteLine(outerEx.Message + " ST: " + outerEx.StackTrace);
}
Console.WriteLine();
Console.WriteLine();
}
public static List<double> PassThroughDFE(int size, List<double> dataIn)
{
List<double> dataOut = new List<double>();
Stopwatch sw = new Stopwatch();
sw.Start();
// Make socket
var socket = new TSocket("localhost", 9090);
// Buffering is critical. Raw sockets are very slow
var transport = new TBufferedTransport(socket);
// Wrap in a protocol
var protocol = new TBinaryProtocol(transport);
// Create a client to use the protocol encoder
var client = new PassThroughService.Client(protocol);
sw.Stop();
Console.WriteLine("Creating a client:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
try {
// Connect!
sw.Reset(); sw.Start();
transport.Open();
sw.Stop();
Console.WriteLine("Opening connection:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Initialize maxfile
sw.Reset(); sw.Start();
var maxfile = client.PassThrough_init();
sw.Stop();
Console.WriteLine("Initializing maxfile:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Load DFE
sw.Reset(); sw.Start();
var engine = client.max_load(maxfile, "*");
sw.Stop();
Console.WriteLine("Loading DFE:\t\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Allocate and send input streams to server
sw.Reset(); sw.Start();
var address_dataIn = client.malloc_float(size);
client.send_data_float(address_dataIn, dataIn);
sw.Stop();
Console.WriteLine("Sending input data:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Allocate memory for output stream on server
sw.Reset(); sw.Start();
var address_dataOut = client.malloc_float(size);
sw.Stop();
Console.WriteLine("Allocating memory for output stream on server:\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Action default
sw.Reset(); sw.Start();
var action = new PassThrough_actions_t_struct();
action.Param_N = size;
action.Instream_x = address_dataIn;
action.Outstream_y = address_dataOut;
var address_action = client.send_PassThrough_actions_t(action);
client.PassThrough_run(engine, address_action);
sw.Stop();
Console.WriteLine("Pass through time:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Unload DFE
sw.Reset(); sw.Start();
client.max_unload(engine);
sw.Stop();
Console.WriteLine("Unloading DFE:\t\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Get output stream from server
sw.Reset(); sw.Start();
dataOut = client.receive_data_float(address_dataOut, size);
sw.Stop();
Console.WriteLine("Getting output stream:\t(size = {0} bit)\t{1}s", size * 32, sw.Elapsed.TotalMilliseconds / 1000);
// Free allocated memory for streams on server
sw.Reset(); sw.Start();
client.free(address_dataIn);
client.free(address_dataOut);
sw.Stop();
Console.WriteLine("Freeing allocated memory for streams on server:\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Free allocated maxfile data
sw.Reset(); sw.Start();
client.PassThrough_free();
sw.Stop();
Console.WriteLine("Freeing allocated maxfile data:\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
sw.Reset(); sw.Start();
transport.Close();
sw.Stop();
Console.WriteLine("Closing connection:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
} catch (SocketException e) {
Console.WriteLine("Could not connect to the server: {0}.", e.Message);
Environment.Exit(-1);
} catch (Exception e) {
Console.WriteLine("An error occured: {0}", e.Message);
Environment.Exit(-1);
}
return dataOut;
}
public static void ShowTables()
{
var transport = new TBufferedTransport(new TSocket("master", 9090));
transport.Open();
var protocol = new TBinaryProtocol(transport);
var client = new Apache.Hadoop.Hbase.Hbase.Client(protocol);
var tables = client.getTableNames();
foreach (var table in tables)
{
Console.WriteLine(Unitl.BytesToStr(table));
}
transport.Close();
}
public static void GetData()
{
var transport = new TBufferedTransport(new TSocket("master", 9090));
transport.Open();
var protocol = new TBinaryProtocol(transport);
var client = new Apache.Hadoop.Hbase.Hbase.Client(protocol);
var rows = client.getRow(Unitl.StrToBytes("1"), Unitl.StrToBytes("row9916"));
foreach (var row in rows)
{
Console.WriteLine(row.Columns.ToString());
}
transport.Close();
}
public static void Scan()
{
var transport = new TBufferedTransport(new TSocket("master", 9090));
transport.Open();
var protocol = new TBinaryProtocol(transport);
var client = new Apache.Hadoop.Hbase.Hbase.Client(protocol);
var scanid = client.scannerOpenWithStop(Unitl.StrToBytes("t1"), Unitl.StrToBytes("rowno"), Unitl.StrToBytes("rowno"),
new List<byte[]> { Unitl.StrToBytes("f1:99913") });
var result = client.scannerGet(scanid);
var colum = client.getColumnDescriptors("t1".ToBytes());
var rows = client.getRowsWithColumns("t1".ToBytes()
, new List<byte[]> { "rowno".ToBytes() }
, new List<byte[]> { "f1:9".ToBytes(), "f1:9999".ToBytes() });
//scanid = client.scannerOpenWithPrefix("".ToBytes(), "".ToBytes(), new List<byte[]> { });
var v = client.getVer("table".ToBytes(), "row".ToBytes(), "col".ToBytes(), 1000);
foreach (var xx in v)
{
}
foreach (var r in result)
{
Console.WriteLine(r.Columns[Unitl.StrToBytes("f1:99913")].Value);
}
client.scannerClose(scanid);
transport.Close();
}
public static void CorrelateDFE(double[,] data, int sizeTimeseries,
int numTimeseries, double[] correlations)
{
// Calculates correlations on DFE.
Stopwatch sw = new Stopwatch();
sw.Start();
// Make socket
var socket = new TSocket("localhost", 9090);
// Buffering is critical. Raw sockets are very slow
var transport = new TBufferedTransport(socket);
// Wrap in a protocol
var protocol = new TBinaryProtocol(transport);
// Create a client to use the protocol encoder
var client = new correlationService.Client(protocol);
sw.Stop();
Console.WriteLine("Creating a client:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
try {
// Connect!
sw.Reset(); sw.Start();
transport.Open();
sw.Stop();
Console.WriteLine("Opening connection:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
int numTimesteps = sizeTimeseries;
double windowSize = (double)sizeTimeseries;
int numBursts = calcNumBursts(numTimeseries);
// Get loop length
sw.Reset(); sw.Start();
List<int> loopLength = new List<int>();
loopLength.Add(client.correlation_get_CorrelationKernel_loopLength());
sw.Stop();
Console.WriteLine("Getting Correlation Kernel loopLength:\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Prepare data for DFE
sw.Reset(); sw.Start();
double[] precalculations = new double [2 * numTimeseries * numTimesteps];
double[] dataPairs = new double [2 * numTimeseries * numTimesteps];
int burstSize = 384 / 2; // for anything other than ISCA this should be 384
List<int> inMemLoad = new List<int>();
for(int i = 0; i < numBursts * burstSize; i++) {
inMemLoad.Add(0);
}
prepareDataForDFE (data, sizeTimeseries, numTimeseries, numTimesteps, windowSize,
precalculations, dataPairs);
List<double> precalculationsVec = new List<double>();
List<double> dataPairsVec = new List<double>();
for(int i = 0; i < 2 * numTimeseries * numTimesteps; i++) {
precalculationsVec.Add(precalculations[i]);
dataPairsVec.Add(dataPairs[i]);
}
sw.Stop();
Console.WriteLine("Data reordering time:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Allocate and send input streams to server
sw.Reset(); sw.Start();
var loopLengthSize = 1;
var addressLoopLength = client.malloc_int32_t(loopLengthSize);
client.send_data_int32_t(addressLoopLength, loopLength);
sw.Stop();
var loopLengthTime = sw.Elapsed.TotalMilliseconds / 1000;
Console.WriteLine("\tSending LoopLength:\t\t(size = {0} bit)\t\t{1}s",
loopLengthSize * 32, loopLengthTime);
sw.Reset(); sw.Start();
var inMemLoadSize = numBursts * burstSize;
var addressInMemLoad = client.malloc_int32_t(inMemLoadSize);
client.send_data_int32_t(addressInMemLoad, inMemLoad);
sw.Stop();
var inMemLoadTime = sw.Elapsed.TotalMilliseconds / 1000;
Console.WriteLine("\tSending InMemLoad:\t\t(size = {0} bit)\t{1}s",
inMemLoadSize * 32, inMemLoadTime);
sw.Reset(); sw.Start();
var precalculationsSize = 2 * numTimeseries * numTimesteps;
var addressPrecalculations = client.malloc_double(precalculationsSize);
client.send_data_double(addressPrecalculations, precalculationsVec);
sw.Stop();
var precalculationsTime = sw.Elapsed.TotalMilliseconds / 1000;
Console.WriteLine("\tSending Precalculations:\t(size = {0} bit)\t{1}s",
precalculationsSize * 32, precalculationsTime);
sw.Reset(); sw.Start();
var dataPairsSize = 2 * numTimeseries * numTimesteps;
var addressDataPairs = client.malloc_double(dataPairsSize);
client.send_data_double(addressDataPairs, dataPairsVec);
sw.Stop();
var dataPairsTime = sw.Elapsed.TotalMilliseconds / 1000;
Console.WriteLine("\tSending DataPairs:\t\t(size = {0} bit)\t{1}s",
dataPairsSize * 64, dataPairsTime);
var time = loopLengthTime + inMemLoadTime + precalculationsTime + dataPairsTime;
var speed = (loopLengthSize * 32 + inMemLoadSize * 32 + precalculationsSize * 64 + dataPairsSize * 64) / time / 1000000;
Console.WriteLine("Sending input streams to server total time:\t{0}s\t(average speed = {1}Mb/s)", time, speed);
// Allocate memory for output stream on server
sw.Reset(); sw.Start();
var addressOutCorrelation = client.malloc_double(numTimesteps * loopLength[0] *
correlationNumTopScores * correlationNumPipes +
numBursts * 24);
var addressOutIndices = client.malloc_int32_t(2 * numTimesteps * loopLength[0] *
correlationNumTopScores * correlationNumPipes);
sw.Stop();
Console.WriteLine("Allocating memory for output stream on server:\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Initialize LMem
sw.Reset(); sw.Start();
client.correlation_loadLMem(numBursts, addressLoopLength, addressInMemLoad);
sw.Stop();
Console.WriteLine("LMem initialization:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
//Executing correlation action
sw.Reset(); sw.Start();
client.correlation(numBursts, // scalar input
numTimesteps, // scalar input
numTimeseries, // scalar input
1, // scalar input
windowSize, // scalar input
addressPrecalculations, // streaming reordered input
addressDataPairs, // streaming reordered input
addressOutCorrelation, // streaming unordered output
addressOutIndices); // streaming unordered output
sw.Stop();
Console.WriteLine("Correlation time:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Get output stream from server
sw.Reset(); sw.Start();
var outCorrelationSize = numTimesteps * loopLength[0] * correlationNumTopScores * correlationNumPipes + numBursts * 24;
List<double> outCorrelation = new List<double>();
outCorrelation = client.receive_data_double(addressOutCorrelation, outCorrelationSize);
sw.Stop();
var outCorrelationTime = sw.Elapsed.TotalMilliseconds / 1000;
Console.WriteLine("\tGet output stream Correlation:\t(size = {0} bit)\t{1}s",
outCorrelationSize * 64, outCorrelationTime);
sw.Reset(); sw.Start();
var outIndicesSzie = 2 * numTimesteps * loopLength[0] * correlationNumTopScores * correlationNumPipes;
List<int> outIndices = new List<int>();
outIndices = client.receive_data_int32_t(addressOutIndices, outIndicesSzie);
sw.Stop();
var outIndicesTime = sw.Elapsed.TotalMilliseconds / 1000;
Console.WriteLine("\tGet output stream outIndices:\t(size = {0} bit)\t{1}s",
outIndicesSzie * 32, outIndicesTime);
sw.Reset(); sw.Start();
loopLengthSize = 1;
loopLength = client.receive_data_int32_t(addressLoopLength, loopLengthSize);
sw.Stop();
loopLengthTime = sw.Elapsed.TotalMilliseconds / 1000;
Console.WriteLine("\tGet output stream loopLength:\t(size = {0} bit)\t\t{1}s",
loopLengthSize * 32, loopLengthTime);
time = loopLengthTime + outCorrelationTime + outIndicesTime;
speed = (loopLengthSize * 32 + outIndicesSzie * 32 + outCorrelationSize * 64) / time / 1000000;
Console.WriteLine("Getting output stream from server total time:\t{0}s\t(average speed = {1}Mb/s)", time, speed);
// Free allocated memory for streams on server
sw.Reset(); sw.Start();
client.free(addressLoopLength);
client.free(addressInMemLoad);
client.free(addressPrecalculations);
client.free(addressDataPairs);
client.free(addressOutCorrelation);
client.free(addressOutIndices);
sw.Stop();
Console.WriteLine("Freeing allocated memory for streams on server:\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Close!
sw.Reset(); sw.Start();
transport.Close();
sw.Stop();
Console.WriteLine("Closing connection:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Store data
sw.Reset(); sw.Start();
int position = 0;
int index = 0;
int start = (numTimesteps-1) * loopLength[0] * correlationNumTopScores * correlationNumPipes;
for (int i=0; i < numTimeseries; i++) {
for(int j = 0; j < i; j++) {
correlations[index + j] = outCorrelation[start + position + j];
}
index += i;
position += ((i/12)+1)*12;
}
sw.Stop();
Console.WriteLine("LMem initialization:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
} catch (SocketException e) {
Console.WriteLine("Could not connect to the server: {0}.", e.Message);
Environment.Exit(-1);
} catch (Exception e) {
Console.WriteLine("An error occured: {0}", e.Message);
Environment.Exit(-1);
}
}
public static void UpdateData()
{
var transport = new TBufferedTransport(new TSocket("master", 9090));
transport.Open();
var protocol = new TBinaryProtocol(transport);
var client = new Apache.Hadoop.Hbase.Hbase.Client(protocol);
//client.createTable(Unitl.StrToBytes(tablename), new List<ColumnDescriptor> { new ColumnDescriptor { Name = Unitl.StrToBytes("zbw911"), InMemory = false } });
client.mutateRow(Unitl.StrToBytes("1"), Unitl.StrToBytes("key"), new List<Mutation> { new Mutation { Column = Unitl.StrToBytes("zbw911"), Value = Unitl.StrToBytes("zbw911-Modifyed") } });
transport.Close();
}
//批量下载
public void MutDownLoad(BackgroundWorker worker, string localRootPath, List <FileStatus> fileList, int fileType)
{
//相同操作
bool sameOp = false;
//是否覆盖
bool IsOver = false;
//准备创建连接
Thrift.Transport.TBufferedTransport btsport = null;
ThriftHadoopFileSystem.Client thriftClient = Connect(out btsport);
if (thriftClient != null)//连接成功
{
int totalCount = fileList.Count;
int currentCount = 0;
//开始上传
worker.ReportProgress(0, new ProgressState()
{
ListBoxMsg = "开始下载！", totalCount = totalCount, CurrentCount = 0
});
//循环
foreach (FileStatus myfile in fileList)
{
currentCount++;
int pgPresent = (int)((double)currentCount / totalCount * 100);
string fileName = Path.GetFileName(myfile.Path);
if (myfile.Isdir == false)
{
string savePath = localRootPath + "/" + fileName;
if (fileType == 1 && myfile.FileName != null)
{
savePath = localRootPath + "/" + myfile.FileName;
}
if (fileType == 2 && myfile.FileName != null)
{
savePath = localRootPath + "/" + myfile.FileName + "/" + fileName;
}
//显示总进度
worker.ReportProgress(pgPresent, new ProgressState()
{
CurrentTitle = fileName, CurrentCount = currentCount
});
#region 是否存在
if (!sameOp)
{
bool exsitFile = File.Exists(savePath);
if (exsitFile)
{
SureDialog myDialog = new SureDialog();
MyShowDialogResult myDR = new MyShowDialogResult();
myDialog.ShowDialog(fileName, myDR);
sameOp = myDR.IsCheck;
IsOver = myDR.Result;
if (!myDR.Result)
{
worker.ReportProgress(pgPresent, new ProgressState()
{
ListBoxMsg = fileName + " 跳过 "
});
continue;
}
}
}
else
{
bool exsitFile = File.Exists(savePath);
if (exsitFile)
{
if (!IsOver)
{
worker.ReportProgress(pgPresent, new ProgressState()
{
ListBoxMsg = fileName + "跳过 "
});
continue;
}
}
}
#endregion
#region 单个下载
bool result = false;
ThriftHandle th = thriftClient.open(new Pathname()
{
pathname = myfile.Path
});
// 创建文件流
FileStream fs = new FileStream(savePath, FileMode.Create, FileAccess.Write);
long totalBytes = 0;
int readLength = 1024 * 1024;
try
{
UTF8Encoding utf8 = new UTF8Encoding(false, true);
while (true)
{
int needRead = readLength;
if (myfile.Length - totalBytes < readLength)
{
needRead = (int)(myfile.Length - totalBytes);
}
if (needRead <= 0)
{
break;
}
byte[] fileBuffer = thriftClient.read(th, totalBytes, needRead);
byte[] myfileBuffer = Encoding.Convert(utf8, Encoding.GetEncoding("iso-8859-1"), fileBuffer);
totalBytes += needRead;
fs.Write(myfileBuffer, 0, myfileBuffer.Length);
//显示单个上传进度
int mypresent = (int)((double)totalBytes / myfile.Length * 100);
worker.ReportProgress(pgPresent, new ProgressState()
{
CurrentTitle = mypresent + "% " + fileName
});
}
result = true;
}
catch (Exception ee)
{
throw ee;
}
finally
{
fs.Dispose();
if (thriftClient != null)
{
thriftClient.close(th);
}
}
#endregion
string msg = string.Format("{0} 下载{1}", Path.GetFileName(fileName), result ? "成功" : "失败");
worker.ReportProgress(pgPresent, new ProgressState()
{
ListBoxMsg = msg
});
}
else
{
worker.ReportProgress(pgPresent, new ProgressState()
{
ListBoxMsg = fileName + "不是文件！"
});
}
}
}
//释放连接
if (btsport != null)
{
btsport.Close();
}
}
public static void Execute(string[] args)
{
try
{
string host = "localhost";
int port = 9090;
string url = null;
int numThreads = 1;
bool buffered = false;
try
{
for (int i = 0; i < args.Length; i++)
{
if (args[i] == "-h")
{
string[] hostport = args[++i].Split(':');
host = hostport[0];
if (hostport.Length > 1)
{
port = Convert.ToInt32(hostport[1]);
}
}
else if (args[i] == "-u")
{
url = args[++i];
}
else if (args[i] == "-n")
{
numIterations = Convert.ToInt32(args[++i]);
}
else if (args[i] == "-b" || args[i] == "-buffered")
{
buffered = true;
Console.WriteLine("Using buffered sockets");
}
else if (args[i] == "-t")
{
numThreads = Convert.ToInt32(args[++i]);
}
}
}
catch (Exception e)
{
Console.WriteLine(e.StackTrace);
}
//issue tests on separate threads simultaneously
Thread[] threads = new Thread[numThreads];
DateTime start = DateTime.Now;
for (int test = 0; test < numThreads; test++)
{
Thread t = new Thread(new ParameterizedThreadStart(ClientThread));
threads[test] = t;
TSocket socket = new TSocket(host, port);
if (buffered)
{
TBufferedTransport buffer = new TBufferedTransport(socket);
t.Start(buffer);
}
else
{
t.Start(socket);
}
}
for (int test = 0; test < numThreads; test++)
{
threads[test].Join();
}
Console.Write("Total time: " + (DateTime.Now - start));
}
catch (Exception outerEx)
{
Console.WriteLine(outerEx.Message + " ST: " + outerEx.StackTrace);
}
Console.WriteLine();
Console.WriteLine();
}
public static void Main(string[] args)
{
Stopwatch sw = new Stopwatch();
Stopwatch swDFE = new Stopwatch();
sw.Start();
swDFE.Start();
// Make socket
var socket = new TSocket("localhost", 9090);
// Buffering is critical. Raw sockets are very slow
var transport = new TBufferedTransport(socket);
// Wrap in a protocol
var protocol = new TBinaryProtocol(transport);
// Create a client to use the protocol encoder
var client = new MovingAverageService.Client(protocol);
sw.Stop();
Console.WriteLine("Creating a client:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
try {
// Connect!
sw.Reset(); sw.Start();
transport.Open();
sw.Stop();
Console.WriteLine("Opening connection:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
sw.Reset(); sw.Start();
const int size = 384;
int sizeBytes = size * 4;
List<double> dataIn = new List<double>();
// Generate input data
Random random = new Random();
for(int i = 0; i<size; i++) {
dataIn.Add(random.Next(0, 100));
}
sw.Stop();
Console.WriteLine("Generating input data:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Initialize maxfile
sw.Reset(); sw.Start();
var maxfile = client.MovingAverage_init();
sw.Stop();
Console.WriteLine("Initializing maxfile:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Load DFE
sw.Reset(); sw.Start();
var engine = client.max_load(maxfile, "*");
sw.Stop();
Console.WriteLine("Loading DFE:\t\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Allocate and send input streams to server
sw.Reset(); sw.Start();
var address_dataIn = client.malloc_float(size);
client.send_data_float(address_dataIn, dataIn);
sw.Stop();
Console.WriteLine("Sending input data:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Allocate memory for output stream on server
sw.Reset(); sw.Start();
var address_dataOut = client.malloc_float(size);
sw.Stop();
Console.WriteLine("Allocating memory for output stream on server:\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Action default
sw.Reset(); sw.Start();
var actions = client.max_actions_init(maxfile, "default");
client.max_set_param_uint64t(actions, "N", size);
client.max_queue_input(actions, "x", address_dataIn, sizeBytes);
client.max_queue_output(actions, "y", address_dataOut, sizeBytes);
client.max_run(engine, actions);
sw.Stop();
Console.WriteLine("Moving average time:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Unload DFE
sw.Reset(); sw.Start();
client.max_unload (engine);
sw.Stop();
Console.WriteLine("Unloading DFE:\t\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Get output stream from server
sw.Reset(); sw.Start();
List<double> dataOut = client.receive_data_float(address_dataOut, size);
sw.Stop();
Console.WriteLine("Getting output stream:\t(size = {0} bit)\t{1}s", size * 32, sw.Elapsed.TotalMilliseconds / 1000);
// Free allocated memory for streams on server
sw.Reset(); sw.Start();
client.free(address_dataIn);
client.free(address_dataOut);
sw.Stop();
Console.WriteLine("Freeing allocated memory for streams on server:\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Free allocated maxfile data
sw.Reset(); sw.Start();
client.MovingAverage_free();
sw.Stop();
Console.WriteLine("Freeing allocated maxfile data:\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
// Checking results
sw.Reset(); sw.Start();
int status = check(dataIn, dataOut, size);
sw.Stop();
Console.WriteLine("Checking results:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
sw.Reset(); sw.Start();
transport.Close();
sw.Stop();
Console.WriteLine("Closing connection:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
swDFE.Stop();
Console.WriteLine("DFE moving average total time:\t\t\t{0}s", swDFE.Elapsed.TotalMilliseconds / 1000);
if (status==0) {
Console.WriteLine("Test successful!");
} else {
Console.WriteLine("Test failed {0} times!", status);
Environment.Exit(-1);
}
} catch (SocketException e) {
Console.WriteLine("Could not connect to the server: {0}.", e.Message);
Environment.Exit(-1);
} catch (Exception e) {
Console.WriteLine("An error occured: {0}", e.Message);
Environment.Exit(-1);
}
}