コード例 #1
0
        public IClient Create(IEndpoint endpoint, IClientPool ownerPool)
        {
            TSocket socket = null;
            TTransport transport = null;
            if (endpoint.Timeout == 0)
            {
                socket = new TSocket(endpoint.Address, endpoint.Port);
            }
            else
            {
                socket = new TSocket(endpoint.Address, endpoint.Port, endpoint.Timeout);
            }
            TcpClient tcpClient = socket.TcpClient;

            if (this.isBufferSizeSet)
            {
                transport = new TBufferedTransport(socket, this.bufferSize);
            }
            else
            {
                transport = new TBufferedTransport(socket);
            }

            TProtocol protocol = new TBinaryProtocol(transport);
            CassandraClient cassandraClient = new CassandraClient(protocol);
            IClient client = new DefaultClient() {
                CassandraClient = cassandraClient,
                Endpoint = endpoint,
                OwnerPool = ownerPool,
                TcpClient = tcpClient,
                Created = DateTime.Now
            };

            return client;
        }
コード例 #2
0
ファイル: HdfsClient.cs プロジェクト: cvs1989/hooyeswidget
        //获取连接
        public ThriftHadoopFileSystem.Client Connect(out TBufferedTransport tsport)
        {
            if (HostServer == null)
               {
               throw new ArgumentNullException("HostServer");
               }

               if (Port == 0)
               {
               throw new ArgumentNullException("Port");
               }

               TSocket hadoop_socket = new TSocket(HostServer, Port);

               //hadoop_socket.Timeout = 10000;// Ten seconds

               tsport = new TBufferedTransport(hadoop_socket);

               TBinaryProtocol hadoop_protocol = new TBinaryProtocol(tsport, false, false);

               ThriftHadoopFileSystem.Client client = new ThriftHadoopFileSystem.Client(hadoop_protocol);
               try
               {
               tsport.Open();
               return client;
               }
               catch (Exception ex)
               {
               //throw (new Exception("打开连接失败!", ex));
               tsport = null;
               return null;

               }
        }
コード例 #3
0
ファイル: Program.cs プロジェクト: zbw911/CS4Hadoop
        public static void query()
        {
            var transport = new TBufferedTransport(new TSocket("hserver", 10000));
            var protocol = new TBinaryProtocol(transport);
            var client = new ThriftHive.Client(protocol);

            transport.Open();

            //client.execute("CREATE TABLE r(a STRING, b INT, c DOUBLE)");
            //client.execute("LOAD TABLE LOCAL INPATH '/path' INTO TABLE r");
            client.execute("SELECT * FROM pokes where foo=180");
            while (true)
            {
                var row = client.fetchOne();
                if (string.IsNullOrEmpty(row))
                    break;
                System.Console.WriteLine(row);
            }
            client.execute("SELECT * FROM pokes");
            var all = client.fetchAll();

            System.Console.WriteLine(all);

            transport.Close();
        }
コード例 #4
0
ファイル: Main.aspx.cs プロジェクト: acesir/hdfs-daemon
        public void GetHBaseImage(string rowKey = "20150822180801118")
        {
            var socket = new TSocket(hbaseThrif, hbaseThrifPort);
            var transport = new TBufferedTransport(socket);
            var protocol = new TBinaryProtocol(transport);
            Hbase.Client hbaseClient = new Hbase.Client(protocol);
            transport.Open();

            //List<byte[]> tableNames = hc.getTableNames();

            byte[] table = Encoding.UTF8.GetBytes("FiservImages");
            byte[] row = Encoding.UTF8.GetBytes(rowKey);
            byte[] column = Encoding.UTF8.GetBytes("ImageData:Image");

            List<TRowResult> results = hbaseClient.getRow(table, row, null);
            TCell t = new TCell();
            //t.

            foreach (TRowResult result in results)
            {
                //result.Columns[column]
                //result.Columns;
                Dictionary<byte[], TCell> konj;
                konj = result.Columns;

                //string s = Encoding.UTF8.GetString(result.Columns[column].Value);

                foreach (KeyValuePair<byte[], TCell> singleRow in konj)
                {
                    //File.WriteAllBytes(@"c:\test\output.docx", singleRow.Value.Value);
                    OpenFile(singleRow.Value.Value, "output.docx");
                }
            }
        }
コード例 #5
0
ファイル: TTLSServerSocket.cs プロジェクト: runxue/thrift
        /// <summary>
        /// Callback for Accept Implementation
        /// </summary>
        /// <returns>
        /// TTransport-object.
        /// </returns>
        protected override TTransport AcceptImpl()
        {
            if (this.server == null)
            {
                throw new TTransportException(TTransportException.ExceptionType.NotOpen, "No underlying server socket.");
            }

            try
            {
                TcpClient client = this.server.AcceptTcpClient();
                client.SendTimeout = client.ReceiveTimeout = this.clientTimeout;

                //wrap the client in an SSL Socket passing in the SSL cert
                TTLSSocket socket = new TTLSSocket(client, this.serverCertificate, true, this.clientCertValidator);

                socket.setupTLS();

                if (useBufferedSockets)
                {
                    TBufferedTransport trans = new TBufferedTransport(socket);
                    return(trans);
                }
                else
                {
                    return(socket);
                }
            }
            catch (Exception ex)
            {
                throw new TTransportException(ex.ToString());
            }
        }
コード例 #6
0
ファイル: TServerSocket.cs プロジェクト: MingHuaL1/Test
 protected override TTransport AcceptImpl()
 {
     if (server == null)
     {
         throw new TTransportException(TTransportException.ExceptionType.NotOpen, "No underlying server socket.");
     }
     try
     {
         TcpClient result  = server.AcceptTcpClient();
         TSocket   result2 = new TSocket(result);
         result2.Timeout = clientTimeout;
         if (useBufferedSockets)
         {
             TBufferedTransport result3 = new TBufferedTransport(result2);
             return(result3);
         }
         else
         {
             return(result2);
         }
     }
     catch (Exception ex)
     {
         throw new TTransportException(ex.ToString());
     }
 }
コード例 #7
0
        public CassandraClient Create(CassandraEndpoint endpoint)
        {
            TSocket socket = null;
            TTransport transport = null;
            if (endpoint.Timeout == 0)
            {
                socket = new TSocket(endpoint.Address, endpoint.Port);
            }
            else
            {
                socket = new TSocket(endpoint.Address, endpoint.Port, endpoint.Timeout);
            }

            if (this.isBufferSizeSet)
            {
                transport = new TBufferedTransport(socket, this.bufferSize);
            }
            else
            {
                transport = new TBufferedTransport(socket);
            }

            TProtocol protocol = new TBinaryProtocol(transport);
            Cassandra.Client cassandraClient = new Cassandra.Client(protocol);
            CassandraClient client = new CassandraClient(cassandraClient, endpoint);

            this.logger.Debug(logger.StringFormatInvariantCulture("Created a new connection using: '{0}'", endpoint.ToString()));

            return client;
        }
コード例 #8
0
        internal ThriftMessageSender(Uri serviceUri)
        {
            socket = new TSocket(serviceUri.Host, serviceUri.Port);
            transport = new TBufferedTransport(socket);
            transport.Open();

            client = new MessageService.Client(new TCompactProtocol(transport));
        }
コード例 #9
0
ファイル: helloClient.cs プロジェクト: arjun2k15/ThriftBook
 static void Main(string[] args)
 {
     TTransport trans = new TSocket("localhost", 8585);
     trans = new TBufferedTransport(trans as TStreamTransport);
     trans.Open();
     TBinaryProtocol proto = new TBinaryProtocol(trans);
     helloSvc.Client client = new helloSvc.Client(proto);
     string result = client.getMessage("Ginger");
     Console.WriteLine("Received from server: " + result);
 }
コード例 #10
0
        protected override TTransport AcceptImpl()
        {
            if (server == null)
            {
                throw new TTransportException(TTransportException.ExceptionType.NotOpen, "No underlying server socket.");
            }
            try
            {
                TSocket result2 = null;
#if NETSTANDARD1_4 || NETSTANDARD1_5
                TcpClient result = server.AcceptTcpClientAsync().Result;
#else
                TcpClient result = server.AcceptTcpClient();
#endif
                try
                {
                    result2         = new TSocket(result);
                    result2.Timeout = clientTimeout;
                    if (useBufferedSockets)
                    {
#if NETSTANDARD1_4
                        throw new NotSupportedException("Not supported with NetStandard 1.4, use NetStandard 1.5 instead.");
#else
                        TBufferedTransport result3 = new TBufferedTransport(result2);
                        return(result3);
#endif
                    }
                    else
                    {
                        return(result2);
                    }
                }
                catch (System.Exception)
                {
                    // If a TSocket was successfully created, then let
                    // it do proper cleanup of the TcpClient object.
                    if (result2 != null)
                    {
                        result2.Dispose();
                    }
                    else //  Otherwise, clean it up ourselves.
                    {
                        ((IDisposable)result).Dispose();
                    }
                    throw;
                }
            }
            catch (Exception ex)
            {
                throw new TTransportException(ex.ToString());
            }
        }
コード例 #11
0
ファイル: Program.cs プロジェクト: zbw911/CS4Hadoop
        public static void test()
        {
            var hadoop_socket = new TSocket("hserver", 59256);
            hadoop_socket.Timeout = 10000;// Ten seconds

            //$hadoop_socket -> setRecvTimeout(20000); // Twenty seconds
            var hadoop_transport = new TBufferedTransport(hadoop_socket);
            var hadoop_protocol = new TBinaryProtocol(hadoop_transport);
            var hadoopClient = new ThriftHadoopFileSystem.Client(hadoop_protocol);
            hadoop_transport.Open();

            //if (hadoopClient.exists(new Pathname { pathname = "/user/root/input" }))
            //{
            //    System.Console.WriteLine("exists");
            //}
            //else
            //{
            //    Console.WriteLine("no exists");

            //}

            //hadoopClient.mkdirs(new Pathname { pathname = "/zbw911_user" });
            //hadoopClient.rm(new Pathname { pathname = "/zbw911_user" }, true);

            //var h = hadoopClient.createFile(new Pathname { pathname = "user_zbw911" }, 1, true, 1000, 1, 64000);

            var list = hadoopClient.listStatus(new Pathname { pathname = "/user/root" });

            foreach (var item in list)
            {
                System.Console.WriteLine(item);
                //if (item.Isdir)
                //{
                //    System.Console.WriteLine("文件夹:" + item.Path + ";权限:" + item.Permission);
                //}
                //else
                //{

                //    System.Console.WriteLine("文件:" + item.Path + ";权限:" + item.Permission);

                //}
            }

            //hadoopClient.close(h);

            //hadoopClient.open(new Pathname { pathname = "/user/root" });

            hadoop_transport.Close();
        }
コード例 #12
0
ファイル: TServerSocket.cs プロジェクト: noahvans/HiveThrift
        protected override TTransport AcceptImpl()
        {
            TTransport tBufferedTransport;

            if (this.server == null)
            {
                throw new TTransportException(TTransportException.ExceptionType.NotOpen, "No underlying server socket.");
            }
            try
            {
                TSocket tSocket = null;
                var     t       = this.server.AcceptTcpClientAsync();
                t.Wait();
                TcpClient tcpClient = t.Result;
                try
                {
                    tSocket = new TSocket(tcpClient)
                    {
                        Timeout = this.clientTimeout
                    };
                    if (!this.useBufferedSockets)
                    {
                        tBufferedTransport = tSocket;
                    }
                    else
                    {
                        tBufferedTransport = new TBufferedTransport(tSocket);
                    }
                }
                catch (Exception exception)
                {
                    if (tSocket == null)
                    {
                        ((IDisposable)tcpClient).Dispose();
                    }
                    else
                    {
                        tSocket.Dispose();
                    }
                    throw;
                }
            }
            catch (Exception exception1)
            {
                throw new TTransportException(exception1.ToString());
            }
            return(tBufferedTransport);
        }
コード例 #13
0
 protected override TTransport AcceptImpl()
 {
     if (server == null)
     {
         throw new TTransportException(TTransportException.ExceptionType.NotOpen, "No underlying server socket.");
     }
     try
     {
         TSocket   result2 = null;
         TcpClient result  = server.AcceptTcpClient();
         try
         {
             result2         = new TSocket(result);
             result2.Timeout = clientTimeout;
             if (useBufferedSockets)
             {
                 TBufferedTransport result3 = new TBufferedTransport(result2);
                 return(result3);
             }
             else
             {
                 return(result2);
             }
         }
         catch (System.Exception)
         {
             // If a TSocket was successfully created, then let
             // it do proper cleanup of the TcpClient object.
             if (result2 != null)
             {
                 result2.Dispose();
             }
             else //  Otherwise, clean it up ourselves.
             {
                 ((IDisposable)result).Dispose();
             }
             throw;
         }
     }
     catch (SocketException ex) when(ex.SocketErrorCode.Equals(SocketError.Interrupted))
     {
         throw new TTransportException(TTransportException.ExceptionType.Interrupted, ex.ToString());
     }
     catch (Exception ex)
     {
         throw new TTransportException(ex.ToString());
     }
 }
コード例 #14
0
ファイル: ThriftConnection.cs プロジェクト: rmarinho/NEST
        public ThriftConnection(IConnectionSettings connectionSettings)
        {
            this._timeout = connectionSettings.Timeout;
            this._poolSize = connectionSettings.MaximumAsyncConnections;

            this._resourceLock = new Semaphore(_poolSize, _poolSize);

            for (var i = 0; i <= connectionSettings.MaximumAsyncConnections; i++)
            {
                var tsocket = new TSocket(connectionSettings.Host, connectionSettings.Port);
                var transport = new TBufferedTransport(tsocket, 1024);
                var protocol = new TBinaryProtocol(transport);
                var client = new Rest.Client(protocol);
                _clients.Enqueue(client);
            }
        }
コード例 #15
0
        /// <summary>
        /// Callback for Accept Implementation
        /// </summary>
        /// <returns>
        /// TTransport-object.
        /// </returns>
        protected override TTransport AcceptImpl()
        {
            if (this.server == null)
            {
                throw new TTransportException(TTransportException.ExceptionType.NotOpen, "No underlying server socket.");
            }

            try
            {
#if NETSTANDARD1_4 || NETSTANDARD1_5
                TcpClient client = this.server.AcceptTcpClientAsync().Result;
#else
                TcpClient client = this.server.AcceptTcpClient();
#endif
                client.SendTimeout = client.ReceiveTimeout = this.clientTimeout;

                //wrap the client in an SSL Socket passing in the SSL cert
                TTLSSocket socket = new TTLSSocket(
                    client,
                    this.serverCertificate,
                    true,
                    this.clientCertValidator,
                    this.localCertificateSelectionCallback
                    );

                socket.setupTLS();

                if (useBufferedSockets)
                {
#if NETSTANDARD1_4
                    throw new NotImplementedException("Not supported with NetStandard 1.4, use NetStandard 1.5 instead.");
#else
                    TBufferedTransport trans = new TBufferedTransport(socket);
                    return(trans);
#endif
                }
                else
                {
                    return(socket);
                }
            }
            catch (Exception ex)
            {
                throw new TTransportException(ex.ToString());
            }
        }
コード例 #16
0
            public TTransport CreateTransport()
            {
                if (url == null)
                {
                    // endpoint transport
                    TTransport trans = null;
                    if (pipe != null)
                        trans = new TNamedPipeClientTransport(pipe);
                    else
                    {
                        if (encrypted)
                        {
                            string certPath = "../keys/client.p12";
                            X509Certificate cert = new X509Certificate2(certPath, "thrift");
                            trans = new TTLSSocket(host, port, 0, cert, (o, c, chain, errors) => true, null, SslProtocols.Tls);
                        }
                        else
                        {
                            trans = new TSocket(host, port);
                        }
                    }

                    // layered transport
                    if (buffered)
                        trans = new TBufferedTransport(trans);
                    if (framed)
                        trans = new TFramedTransport(trans);

                    if (_isFirstTransport)
                    {
                        //ensure proper open/close of transport
                        trans.Open();
                        trans.Close();
                        _isFirstTransport = false;
                    }
                    return trans;
                }
                else
                {
                    return new THttpClient(new Uri(url));
                }
            }
コード例 #17
0
ファイル: Program.cs プロジェクト: zbw911/CS4Hadoop
        public static void t()
        {
            var transport = new TBufferedTransport(new TSocket("hserver", 10000));
            var protocol = new TBinaryProtocol(transport);
            var client = new ThriftHive.Client(protocol);

            transport.Open();

            //client.execute("CREATE TABLE r(a STRING, b INT, c DOUBLE)");
            //client.execute("LOAD TABLE LOCAL INPATH '/path' INTO TABLE r");

            var database = new Database();

            database.Name = "helloworldDB";
            database.Description = "测试用的第一个DB";

            client.create_database(database);

            transport.Close();
        }
コード例 #18
0
ファイル: TTLSServerSocket.cs プロジェクト: zxbe/Anno.Core
        /// <summary>
        /// Callback for Accept Implementation
        /// </summary>
        /// <returns>
        /// TTransport-object.
        /// </returns>
        protected override TTransport AcceptImpl()
        {
            if (server == null)
            {
                throw new TTransportException(TTransportException.ExceptionType.NotOpen, "No underlying server socket.");
            }

            try
            {
                var client = server.AcceptTcpClient();
                client.SendTimeout = client.ReceiveTimeout = clientTimeout;

                //wrap the client in an SSL Socket passing in the SSL cert
                var socket = new TTLSSocket(
                    client,
                    serverCertificate,
                    true,
                    clientCertValidator,
                    localCertificateSelectionCallback,
                    sslProtocols);

                socket.setupTLS();

                if (useBufferedSockets)
                {
                    var trans = new TBufferedTransport(socket);
                    return(trans);
                }
                else
                {
                    return(socket);
                }
            }
            catch (Exception ex)
            {
                throw new TTransportException(ex.ToString(), ex);
            }
        }
コード例 #19
0
ファイル: FSClient.cs プロジェクト: gudao/NetHadoop
        //获取连接
        public ThriftHadoopFileSystem.Client Connect(out TBufferedTransport tsport)
        {
            TSocket hadoop_socket = new TSocket(HostIP, HostPort);

               //hadoop_socket.Timeout = 10000;// Ten seconds

               tsport = new TBufferedTransport(hadoop_socket);

               TBinaryProtocol hadoop_protocol = new TBinaryProtocol(tsport, false, false);

               ThriftHadoopFileSystem.Client client = new ThriftHadoopFileSystem.Client(hadoop_protocol);
               try
               {
               tsport.Open();
               return client;
               }
               catch (Exception ee)
               {
               logger.Error("打开连接失败!", ee);
               tsport = null;
               return null;
               }
        }
コード例 #20
0
ファイル: CallHBase.cs プロジェクト: acesir/hdfs-daemon
        public Dictionary<byte[], TCell> GetHBaseImage()
        {
            var socket = new TSocket(_hbaseThrif, _hbaseThrifPort);
            var transport = new TBufferedTransport(socket);
            var protocol = new TBinaryProtocol(transport);
            Hbase.Client hbaseClient = new Hbase.Client(protocol);
            Dictionary<byte[], TCell> hbaseResult = new Dictionary<byte[], TCell>();
            transport.Open();

            //List<byte[]> tableNames = hc.getTableNames();

            byte[] table = Encoding.UTF8.GetBytes("FiservImages");
            byte[] row = Encoding.UTF8.GetBytes(_documentId);
            byte[] column = Encoding.UTF8.GetBytes("ImageData:Image");

            List<TRowResult> results = hbaseClient.getRow(table, row, null);
            TCell t = new TCell();
            //t.

            foreach (TRowResult result in results)
            {
                //result.Columns[column]
                //result.Columns;

                hbaseResult = result.Columns;

                //string s = Encoding.UTF8.GetString(result.Columns[column].Value);

                //foreach (KeyValuePair<byte[], TCell> singleRow in konj)
                //{
                //    //File.WriteAllBytes(@"c:\test\output.docx", singleRow.Value.Value);
                //    Main a = new Main();
                //    a.OpenFile(singleRow.Value.Value, "output.docx");
                //}
            }
            return hbaseResult;
        }
コード例 #21
0
ファイル: Program.cs プロジェクト: zbw911/CS4Hadoop
        public static void t2()
        {
            var transport = new TBufferedTransport(new TSocket("hserver", 10000));
            var protocol = new TBinaryProtocol(transport);
            var client = new ThriftHive.Client(protocol);

            transport.Open();

            var database = new Database();

            database.Name = "helloworldDB";
            database.Description = "测试用的第一个DB";

            var tables = client.get_all_tables("default");

            //var tables = client.get_all_databases();

            foreach (var table in tables)
            {
                System.Console.WriteLine(table);
            }

            transport.Close();
        }
コード例 #22
0
ファイル: HClient.cs プロジェクト: zbw911/CS4Hadoop
        public static void BatAddData()
        {
            var transport = new TBufferedTransport(new TSocket("master", 9090));

            transport.Open();

            var protocol = new TBinaryProtocol(transport);

            var client = new Apache.Hadoop.Hbase.Hbase.Client(protocol);

            //client.createTable(Unitl.StrToBytes(tablename), new List<ColumnDescriptor> { new ColumnDescriptor { Name = Unitl.StrToBytes("zbw911"), InMemory = false } });

            var bytevalue = Unitl.StrToBytes(Longdata);

            for (var j = 0; j < 100000; j++)
            {
                //var j = 0;
                for (var i = 0; i < 1000; i++)
                {
                    client.mutateRow(Unitl.StrToBytes("1"), Unitl.StrToBytes("" + i + "___" + j), new List<Mutation> { new Mutation { Column = Unitl.StrToBytes("zbw911"), Value = bytevalue } });
                }

                Console.WriteLine(j + "批=" + 1000 * j);
            }

            transport.Close();
        }
コード例 #23
0
ファイル: HClient.cs プロジェクト: zbw911/CS4Hadoop
        public static void AddColumn()
        {
            var transport = new TBufferedTransport(new TSocket("master", 9090));

            transport.Open();

            var protocol = new TBinaryProtocol(transport);

            var client = new Apache.Hadoop.Hbase.Hbase.Client(protocol);

            //client.createTable(Unitl.StrToBytes("t"), new List<ColumnDescriptor> { new ColumnDescriptor { Name = Unitl.StrToBytes("default"), InMemory = false } });
            //client.createTable(Unitl.StrToBytes("thetable"), new List<ColumnDescriptor> { new ColumnDescriptor { Name = Unitl.StrToBytes("default"), InMemory = false } });

            for (var i = 0; i < 10; i++)
            {

                client.mutateRow(Unitl.StrToBytes("t"),

                    Unitl.StrToBytes("key"),
                    new List<Mutation> {
                        new Mutation { Column = Unitl.StrToBytes("default:"+i ), Value = Unitl.StrToBytes( System.DateTime.Now.ToString() ) }

                    });

            }
            transport.Close();
        }
コード例 #24
0
ファイル: HClient.cs プロジェクト: zbw911/CS4Hadoop
        public static void AddTable(string tablename)
        {
            var transport = new TBufferedTransport(new TSocket("master", 9090));

            transport.Open();

            var protocol = new TBinaryProtocol(transport);

            var client = new Apache.Hadoop.Hbase.Hbase.Client(protocol);

            client.createTable(Unitl.StrToBytes(tablename), new List<ColumnDescriptor> { new ColumnDescriptor { Name = Unitl.StrToBytes("zbw911"), InMemory = false } });

            transport.Close();
        }
コード例 #25
0
ファイル: TServerSocket.cs プロジェクト: GDGroup/thrift
 protected override TTransport AcceptImpl()
 {
     if (server == null)
     {
         throw new TTransportException(TTransportException.ExceptionType.NotOpen, "No underlying server socket.");
     }
     try
     {
         TSocket result2 = null;
         TcpClient result = server.AcceptTcpClient();
         try
         {
             result2 = new TSocket(result);
             result2.Timeout = clientTimeout;
             if (useBufferedSockets)
             {
                 TBufferedTransport result3 = new TBufferedTransport(result2);
                 return result3;
             }
             else
             {
                 return result2;
             }
         }
         catch (System.Exception)
         {
             // If a TSocket was successfully created, then let 
             // it do proper cleanup of the TcpClient object.
             if (result2 != null)
                 result2.Dispose();
             else //  Otherwise, clean it up ourselves.
                 ((IDisposable)result).Dispose();
             throw;
         }
     }
     catch (Exception ex)
     {
         throw new TTransportException(ex.ToString());
     }
 }
コード例 #26
0
ファイル: TTLSServerSocket.cs プロジェクト: jsirois/thrift
        /// <summary>
        /// Callback for Accept Implementation
        /// </summary>
        /// <returns>
        /// TTransport-object.
        /// </returns>
        protected override TTransport AcceptImpl()
        {
            if (this.server == null)
            {
                throw new TTransportException(TTransportException.ExceptionType.NotOpen, "No underlying server socket.");
            }

            try
            {
                TcpClient client = this.server.AcceptTcpClient();
                client.SendTimeout = client.ReceiveTimeout = this.clientTimeout;

                //wrap the client in an SSL Socket passing in the SSL cert
                TTLSSocket socket = new TTLSSocket(
                    client,
                    this.serverCertificate,
                    true,
                    this.clientCertValidator,
                    this.localCertificateSelectionCallback,
                    this.sslProtocols);

                socket.setupTLS();

                if (useBufferedSockets)
                {
                    TBufferedTransport trans = new TBufferedTransport(socket);
                    return trans;
                }
                else
                {
                    return socket;
                }

            }
            catch (Exception ex)
            {
                throw new TTransportException(ex.ToString());
            }
        }
コード例 #27
0
ファイル: Main.cs プロジェクト: pingvinen/calculondb
        static void Main(string[] args)
        {
            Log.Use().SimpleConsoleColored();

            Log.Information("Booting");

            try
            {
                string host = ConfigurationManager.AppSettings["Interface.Host"];
                int port = Int32.Parse(ConfigurationManager.AppSettings["Interface.Port"]);

                var socket = new TSocket(host, port);
                var transport = new TBufferedTransport(socket);
                var protocol = new TBinaryProtocol(transport);
                var client = new InterfaceNodeService.Client(protocol);
                transport.Open();

                #region Input loop
                bool doContinueLoop = true;
                string cmd = String.Empty;
                string input = String.Empty;

                Entry tmpEntry = null;
                Result res = null;

                while (doContinueLoop)
                {
                    cmd = Prompt("What do you want to do? (insert, getreport, getset, quit)");

                    switch (cmd)
                    {
                        #region Quit
                        case "quit":
                        {
                            doContinueLoop = false;
                            break;
                        }
                        #endregion Quit

                        #region Insert
                        case "insert":
                        {
                            tmpEntry = new Entry();
                            tmpEntry.Data = new Dictionary<string, string>();

                            input = Prompt("Enter an Entry type");

                            tmpEntry.Type = input;
                            string tmpVarName = String.Empty;
                            string tmpVarValue = String.Empty;

                            while (true)
                            {
                                tmpVarName = Prompt("Enter name of var (or \"send\")");

                                if (tmpVarName.Equals("send"))
                                {
                                    break;
                                }

                                tmpVarValue = Prompt("Enter the value of the var (or \"send\")");

                                if (tmpVarValue.Equals("send"))
                                {
                                    break;
                                }

                                tmpEntry.Data.Add(tmpVarName, tmpVarValue);
                            }

                            Log.Trace("Sending Insert");
                            res = client.Insert(tmpEntry);
                            Log.Debug(res.DumpToString());

                            tmpEntry = null;

                            break;
                        }
                        #endregion Insert

                        #region Get report
                        case "getreport":
                        {
                            input = Prompt("Enter the name of the report you wish to get");

                            Log.Trace("Sending GetReport");
                            res = client.GetReport(input);
                            Log.Debug(res.DumpToString());

                            break;
                        }
                        #endregion Get report

                        #region Get set
                        case "getset":
                        {
                            input = Prompt("Enter the name of the set you wish to get");

                            Log.Trace("Sending GetSet");
                            res = client.GetSet(input);
                            Log.Debug(res.DumpToString());

                            break;
                        }
                        #endregion Get set

                        default:
                        {
                            Log.Error("Invalid command '{0}'", cmd);
                            break;
                        }
                    }
                }
                #endregion Input loop
            }
            catch(Exception ex)
            {
                Log.Error(ex.ToString());
            }

            Log.Information("Done... closing");
        }
コード例 #28
0
ファイル: TServerSocket.cs プロジェクト: romankor/NEST
 protected override TTransport AcceptImpl()
 {
     if (server == null)
     {
         throw new TTransportException(TTransportException.ExceptionType.NotOpen, "No underlying server socket.");
     }
     try
     {
         TcpClient result = server.AcceptTcpClient();
         var result2 = new TSocket(result);
         result2.Timeout = clientTimeout;
         if (useBufferedSockets)
         {
             var result3 = new TBufferedTransport(result2);
             return result3;
         }
         else
         {
             return result2;
         }
     }
     catch (Exception ex)
     {
         throw new TTransportException(ex.ToString());
     }
 }
コード例 #29
0
ファイル: FSClient.cs プロジェクト: yaowenhua/NetHadoop
        //批量上传
        public void MutUpload(BackgroundWorker worker, List <string> localPath, string remoteRootPath, string localRootPath)
        {
            //相同操作
            bool sameOp = false;
            //是否覆盖
            bool IsOver = false;

            //准备创建连接
            Thrift.Transport.TBufferedTransport btsport      = null;
            ThriftHadoopFileSystem.Client       thriftClient = Connect(out btsport);
            List <string> NoSuccessList = new List <string>();
            List <string> skipList      = new List <string>();

            if (thriftClient != null)//连接成功
            {
                int totalCount   = localPath.Count;
                int currentCount = 0;
                //开始上传
                worker.ReportProgress(0, new ProgressState()
                {
                    ListBoxMsg = "开始上传!", totalCount = totalCount, CurrentCount = 0
                });
                //循环
                foreach (string localFilePath in localPath)
                {
                    string fileName = Path.GetFileName(localFilePath);
                    if (!string.IsNullOrEmpty(localRootPath))//如果是文件夹,则包含原有路径
                    {
                        fileName = localFilePath.Replace(localRootPath, "");
                    }

                    string remoteFilePath = remoteRootPath + "/" + fileName;
                    currentCount++;
                    int pgPresent = (int)((double)currentCount / totalCount * 100);
                    //显示总进度
                    worker.ReportProgress(pgPresent, new ProgressState()
                    {
                        CurrentTitle = fileName, CurrentCount = currentCount
                    });

                    #region 是否存在
                    if (!sameOp)
                    {
                        bool exsitFile = thriftClient.exists(new Pathname()
                        {
                            pathname = remoteFilePath
                        });
                        if (exsitFile)
                        {
                            SureDialog         myDialog = new SureDialog();
                            MyShowDialogResult myDR     = new MyShowDialogResult();
                            myDialog.ShowDialog(fileName, myDR);
                            sameOp = myDR.IsCheck;
                            IsOver = myDR.Result;
                            if (!myDR.Result)
                            {
                                worker.ReportProgress(pgPresent, new ProgressState()
                                {
                                    ListBoxMsg = fileName + " 跳过 "
                                });
                                skipList.Add(fileName);
                                continue;
                            }
                        }
                    }
                    else
                    {
                        bool exsitFile = thriftClient.exists(new Pathname()
                        {
                            pathname = remoteFilePath
                        });
                        if (exsitFile)
                        {
                            if (!IsOver)
                            {
                                worker.ReportProgress(pgPresent, new ProgressState()
                                {
                                    ListBoxMsg = fileName + "跳过 "
                                });
                                skipList.Add(fileName);
                                continue;
                            }
                        }
                    }
                    #endregion


                    #region   单个文件
                    bool         singleResult = false;
                    ThriftHandle th           = null;
                    FileStream   fs           = null;
                    try
                    {
                        Pathname myNewFile = new Pathname()
                        {
                            pathname = remoteFilePath
                        };

                        //创建一个文件
                        th = thriftClient.createFile(myNewFile, 1, true, 1024 * 1024 * 10, ConfigHelper.HDFSREPLICATION, 1024 * 1024 * 512);

                        UTF8Encoding utf8 = new UTF8Encoding(false, true);

                        fs = new FileStream(localFilePath, FileMode.Open, FileAccess.Read);

                        byte[] fileBuffer = new byte[1024 * 1024 * 10]; // 每次传1MB
                        int    bytesRead;
                        long   bytesTotal = 0;

                        while ((bytesRead = fs.Read(fileBuffer, 0, fileBuffer.Length)) > 0)
                        {
                            bytesTotal += bytesRead;
                            byte[] realBuffer = new byte[bytesRead];
                            Array.Copy(fileBuffer, realBuffer, bytesRead);
                            //将utf8转为可存储编码
                            realBuffer = Encoding.Convert(Encoding.GetEncoding("iso-8859-1"), utf8, realBuffer);
                            //发送
                            thriftClient.write(th, realBuffer);
                            //清仓缓存
                            Array.Clear(fileBuffer, 0, fileBuffer.Length);
                            realBuffer = null;
                            //显示单个上传进度
                            int mypresent = (int)((double)bytesTotal / fs.Length * 100);
                            worker.ReportProgress(pgPresent, new ProgressState()
                            {
                                CurrentTitle = mypresent + "% " + fileName
                            });
                        }
                        singleResult = true;
                    }
                    catch (Exception ee)
                    {
                        //显示上传错误
                        worker.ReportProgress(pgPresent, new ProgressState()
                        {
                            ListBoxMsg = ee.Message
                        });
                    }
                    finally
                    {
                        if (th != null)
                        {
                            thriftClient.close(th);
                        }
                        if (fs != null)
                        {
                            fs.Close();
                        }
                    }
                    #endregion
                    //显示单个上传结果
                    string msg = string.Format("{0} 上传{1}", fileName, singleResult ? "成功" : "失败");
                    worker.ReportProgress(pgPresent, new ProgressState()
                    {
                        ListBoxMsg = msg
                    });
                    if (!singleResult)
                    {
                        NoSuccessList.Add(fileName);
                    }
                }
            }

            //释放连接
            if (btsport != null)
            {
                btsport.Close();
            }
            //输出没有上传成功的
            if (NoSuccessList.Count > 0)
            {
                try
                {
                    File.WriteAllText("c:/UploadNoSuccess.txt", string.Join("\r\n", NoSuccessList.ToArray()));
                    worker.ReportProgress(100, new ProgressState()
                    {
                        ListBoxMsg = NoSuccessList.Count + "个上传错误!请查看c:/UploadNoSuccess.txt"
                    });
                }
                catch (Exception ee)
                {
                    worker.ReportProgress(100, new ProgressState()
                    {
                        ListBoxMsg = ee.Message
                    });
                }
            }
            //输出跳过的
            if (skipList.Count > 0)
            {
                try
                {
                    File.WriteAllText("c:/UploadSkip.txt", string.Join("\r\n", skipList.ToArray()));
                    worker.ReportProgress(100, new ProgressState()
                    {
                        ListBoxMsg = "跳过" + skipList.Count + "个文件!请查看c:/UploadSkip.txt"
                    });
                }
                catch (Exception ee)
                {
                    worker.ReportProgress(100, new ProgressState()
                    {
                        ListBoxMsg = ee.Message
                    });
                }
            }
        }
コード例 #30
0
ファイル: TestClient.cs プロジェクト: ably-forks/thrift
        public static void Execute(string[] args)
        {
            try
            {
                string host = "localhost";
                int port = 9090;
                string url = null, pipe = null;
                int numThreads = 1;
                bool buffered = false, framed = false;

                try
                {
                    for (int i = 0; i < args.Length; i++)
                    {
                        if (args[i] == "-h")
                        {
                            string[] hostport = args[++i].Split(':');
                            host = hostport[0];
                            if (hostport.Length > 1)
                            {
                                port = Convert.ToInt32(hostport[1]);
                            }
                        }
                        else if (args[i] == "-u")
                        {
                            url = args[++i];
                        }
                        else if (args[i] == "-n")
                        {
                            numIterations = Convert.ToInt32(args[++i]);
                        }
                        else if (args[i] == "-b" || args[i] == "-buffered")
                        {
                            buffered = true;
                            Console.WriteLine("Using buffered sockets");
                        }
                        else if (args[i] == "-f" || args[i] == "-framed")
                        {
                            framed = true;
                            Console.WriteLine("Using framed transport");
                        }
                        else if (args[i] == "-pipe")  // -pipe <name>
                        {
                            pipe = args[++i];
                            Console.WriteLine("Using named pipes transport");
                        }
                        else if (args[i] == "-t")
                        {
                            numThreads = Convert.ToInt32(args[++i]);
                        }
                    }
                }
                catch (Exception e)
                {
                    Console.WriteLine(e.StackTrace);
                }

                //issue tests on separate threads simultaneously
                Thread[] threads = new Thread[numThreads];
                DateTime start = DateTime.Now;
                for (int test = 0; test < numThreads; test++)
                {
                    Thread t = new Thread(new ParameterizedThreadStart(ClientThread));
                    threads[test] = t;
                    if (url == null)
                    {
                        // endpoint transport
                        TTransport trans = null;
                        if( pipe != null)
                            trans = new TNamedPipeClientTransport(pipe);
                        else
                            trans = new TSocket(host, port);

                        // layered transport
                        if (buffered)
                            trans = new TBufferedTransport(trans as TStreamTransport);
                        if (framed)
                            trans = new TFramedTransport(trans);

                        //ensure proper open/close of transport
                        trans.Open();
                        trans.Close();
                        t.Start(trans);
                    }
                    else
                    {
                        THttpClient http = new THttpClient(new Uri(url));
                        t.Start(http);
                    }
                }

                for (int test = 0; test < numThreads; test++)
                {
                    threads[test].Join();
                }
                Console.Write("Total time: " + (DateTime.Now - start));
            }
            catch (Exception outerEx)
            {
                Console.WriteLine(outerEx.Message + " ST: " + outerEx.StackTrace);
            }

            Console.WriteLine();
            Console.WriteLine();
        }
コード例 #31
0
ファイル: PassThroughClient.cs プロジェクト: maxeler/maxskins
    public static List<double> PassThroughDFE(int size, List<double> dataIn)
    {
        List<double> dataOut = new List<double>();
        Stopwatch sw = new Stopwatch();
        sw.Start();

        // Make socket
        var socket = new TSocket("localhost", 9090);

        // Buffering is critical. Raw sockets are very slow
        var transport = new TBufferedTransport(socket);

        // Wrap in a protocol
        var protocol = new TBinaryProtocol(transport);

        // Create a client to use the protocol encoder
        var client = new PassThroughService.Client(protocol);

        sw.Stop();
        Console.WriteLine("Creating a client:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

        try {
            // Connect!
            sw.Reset(); sw.Start();
            transport.Open();
            sw.Stop();
            Console.WriteLine("Opening connection:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

            // Initialize maxfile
            sw.Reset(); sw.Start();
            var maxfile = client.PassThrough_init();
            sw.Stop();
            Console.WriteLine("Initializing maxfile:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

            // Load DFE
            sw.Reset(); sw.Start();
            var engine = client.max_load(maxfile, "*");
            sw.Stop();
            Console.WriteLine("Loading DFE:\t\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

            // Allocate and send input streams to server
            sw.Reset(); sw.Start();
            var address_dataIn = client.malloc_float(size);
            client.send_data_float(address_dataIn, dataIn);
            sw.Stop();
            Console.WriteLine("Sending input data:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

            // Allocate memory for output stream on server
            sw.Reset(); sw.Start();
            var address_dataOut = client.malloc_float(size);
            sw.Stop();
            Console.WriteLine("Allocating memory for output stream on server:\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

            // Action default
            sw.Reset(); sw.Start();
            var action = new PassThrough_actions_t_struct();
            action.Param_N = size;
            action.Instream_x = address_dataIn;
            action.Outstream_y = address_dataOut;
            var address_action = client.send_PassThrough_actions_t(action);
            client.PassThrough_run(engine, address_action);
            sw.Stop();
            Console.WriteLine("Pass through time:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

            // Unload DFE
            sw.Reset(); sw.Start();
            client.max_unload(engine);
            sw.Stop();
            Console.WriteLine("Unloading DFE:\t\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

            // Get output stream from server
            sw.Reset(); sw.Start();
            dataOut = client.receive_data_float(address_dataOut, size);
            sw.Stop();
            Console.WriteLine("Getting output stream:\t(size = {0} bit)\t{1}s", size * 32, sw.Elapsed.TotalMilliseconds / 1000);

            // Free allocated memory for streams on server
            sw.Reset(); sw.Start();
            client.free(address_dataIn);
            client.free(address_dataOut);
            sw.Stop();
            Console.WriteLine("Freeing allocated memory for streams on server:\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

            // Free allocated maxfile data
            sw.Reset(); sw.Start();
            client.PassThrough_free();
            sw.Stop();
            Console.WriteLine("Freeing allocated maxfile data:\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

            sw.Reset(); sw.Start();
            transport.Close();
            sw.Stop();
            Console.WriteLine("Closing connection:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

        } catch (SocketException e) {
            Console.WriteLine("Could not connect to the server: {0}.", e.Message);
            Environment.Exit(-1);
        } catch (Exception e) {
            Console.WriteLine("An error occured: {0}", e.Message);
            Environment.Exit(-1);
        }

        return dataOut;
    }
コード例 #32
0
ファイル: HClient.cs プロジェクト: zbw911/CS4Hadoop
        public static void ShowTables()
        {
            var transport = new TBufferedTransport(new TSocket("master", 9090));

            transport.Open();

            var protocol = new TBinaryProtocol(transport);

            var client = new Apache.Hadoop.Hbase.Hbase.Client(protocol);

            var tables = client.getTableNames();

            foreach (var table in tables)
            {
                Console.WriteLine(Unitl.BytesToStr(table));
            }

            transport.Close();
        }
コード例 #33
0
ファイル: HClient.cs プロジェクト: zbw911/CS4Hadoop
        public static void GetData()
        {
            var transport = new TBufferedTransport(new TSocket("master", 9090));

            transport.Open();

            var protocol = new TBinaryProtocol(transport);

            var client = new Apache.Hadoop.Hbase.Hbase.Client(protocol);

            var rows = client.getRow(Unitl.StrToBytes("1"), Unitl.StrToBytes("row9916"));

            foreach (var row in rows)
            {
                Console.WriteLine(row.Columns.ToString());
            }

            transport.Close();
        }
コード例 #34
0
ファイル: HClient.cs プロジェクト: zbw911/CS4Hadoop
        public static void Scan()
        {
            var transport = new TBufferedTransport(new TSocket("master", 9090));

            transport.Open();

            var protocol = new TBinaryProtocol(transport);

            var client = new Apache.Hadoop.Hbase.Hbase.Client(protocol);

            var scanid = client.scannerOpenWithStop(Unitl.StrToBytes("t1"), Unitl.StrToBytes("rowno"), Unitl.StrToBytes("rowno"),
                new List<byte[]> { Unitl.StrToBytes("f1:99913") });

            var result = client.scannerGet(scanid);

            var colum = client.getColumnDescriptors("t1".ToBytes());

            var rows = client.getRowsWithColumns("t1".ToBytes()
                  , new List<byte[]> { "rowno".ToBytes() }
                  , new List<byte[]> { "f1:9".ToBytes(), "f1:9999".ToBytes() });

            //scanid = client.scannerOpenWithPrefix("".ToBytes(), "".ToBytes(), new List<byte[]> { });

            var v = client.getVer("table".ToBytes(), "row".ToBytes(), "col".ToBytes(), 1000);

            foreach (var xx in v)
            {

            }

            foreach (var r in result)
            {
                Console.WriteLine(r.Columns[Unitl.StrToBytes("f1:99913")].Value);
            }

            client.scannerClose(scanid);

            transport.Close();
        }
コード例 #35
0
ファイル: CorrelationClient.cs プロジェクト: maxeler/maxskins
    public static void CorrelateDFE(double[,] data, int sizeTimeseries,
                                        int numTimeseries, double[] correlations)
    {
        // Calculates correlations on DFE.
                Stopwatch sw = new Stopwatch();
                sw.Start();

                // Make socket
                var socket = new TSocket("localhost", 9090);

                // Buffering is critical. Raw sockets are very slow
                var transport = new TBufferedTransport(socket);

                // Wrap in a protocol
                var protocol = new TBinaryProtocol(transport);

                // Create a client to use the protocol encoder
                var client = new correlationService.Client(protocol);

                sw.Stop();
                Console.WriteLine("Creating a client:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

                try {
                        // Connect!
                        sw.Reset(); sw.Start();
                        transport.Open();
                        sw.Stop();
                        Console.WriteLine("Opening connection:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

                        int numTimesteps = sizeTimeseries;
                        double windowSize = (double)sizeTimeseries;
                        int numBursts = calcNumBursts(numTimeseries);

                        // Get loop length
                        sw.Reset(); sw.Start();
                        List<int> loopLength = new List<int>();
                        loopLength.Add(client.correlation_get_CorrelationKernel_loopLength());
                        sw.Stop();
                        Console.WriteLine("Getting Correlation Kernel loopLength:\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

                        // Prepare data for DFE
                        sw.Reset(); sw.Start();

                        double[] precalculations = new double [2 * numTimeseries * numTimesteps];
                        double[] dataPairs = new double [2 * numTimeseries * numTimesteps];

                        int burstSize = 384 / 2;  // for anything other than ISCA this should be 384
                        List<int> inMemLoad = new List<int>();
                        for(int i = 0; i < numBursts * burstSize; i++) {
                                inMemLoad.Add(0);
                        }

                        prepareDataForDFE (data, sizeTimeseries, numTimeseries, numTimesteps, windowSize,
                                           precalculations, dataPairs);

                        List<double> precalculationsVec = new List<double>();
                        List<double> dataPairsVec = new List<double>();
                        for(int i = 0; i < 2 * numTimeseries * numTimesteps; i++) {
                                precalculationsVec.Add(precalculations[i]);
                                dataPairsVec.Add(dataPairs[i]);
                        }

                        sw.Stop();
                        Console.WriteLine("Data reordering time:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

                        // Allocate and send input streams to server
                        sw.Reset(); sw.Start();
                        var loopLengthSize = 1;
                        var addressLoopLength = client.malloc_int32_t(loopLengthSize);
                        client.send_data_int32_t(addressLoopLength, loopLength);
                        sw.Stop();
                        var loopLengthTime = sw.Elapsed.TotalMilliseconds / 1000;
                        Console.WriteLine("\tSending LoopLength:\t\t(size = {0} bit)\t\t{1}s",
                                          loopLengthSize * 32, loopLengthTime);

                        sw.Reset(); sw.Start();
                        var inMemLoadSize = numBursts * burstSize;
                        var addressInMemLoad = client.malloc_int32_t(inMemLoadSize);
                        client.send_data_int32_t(addressInMemLoad, inMemLoad);
                        sw.Stop();
                        var inMemLoadTime = sw.Elapsed.TotalMilliseconds / 1000;
                        Console.WriteLine("\tSending InMemLoad:\t\t(size = {0} bit)\t{1}s",
                                          inMemLoadSize * 32, inMemLoadTime);

                        sw.Reset(); sw.Start();
                        var precalculationsSize = 2 * numTimeseries * numTimesteps;
                        var addressPrecalculations = client.malloc_double(precalculationsSize);
                        client.send_data_double(addressPrecalculations, precalculationsVec);
                        sw.Stop();
                        var precalculationsTime = sw.Elapsed.TotalMilliseconds / 1000;
                        Console.WriteLine("\tSending Precalculations:\t(size = {0} bit)\t{1}s",
                                          precalculationsSize * 32, precalculationsTime);

                        sw.Reset(); sw.Start();
                        var dataPairsSize = 2 * numTimeseries * numTimesteps;
                        var addressDataPairs = client.malloc_double(dataPairsSize);
                        client.send_data_double(addressDataPairs, dataPairsVec);
                        sw.Stop();
                        var dataPairsTime = sw.Elapsed.TotalMilliseconds / 1000;
                        Console.WriteLine("\tSending DataPairs:\t\t(size = {0} bit)\t{1}s",
                                          dataPairsSize * 64, dataPairsTime);

                        var time = loopLengthTime + inMemLoadTime + precalculationsTime + dataPairsTime;
                        var speed = (loopLengthSize * 32 + inMemLoadSize * 32 + precalculationsSize * 64 + dataPairsSize * 64) / time / 1000000;
                        Console.WriteLine("Sending input streams to server total time:\t{0}s\t(average speed = {1}Mb/s)", time, speed);

                        // Allocate memory for output stream on server
                        sw.Reset(); sw.Start();
                        var addressOutCorrelation = client.malloc_double(numTimesteps * loopLength[0] *
                                                                         correlationNumTopScores * correlationNumPipes +
                                                                         numBursts * 24);
                        var addressOutIndices = client.malloc_int32_t(2 * numTimesteps * loopLength[0] *
                                                                      correlationNumTopScores * correlationNumPipes);
                        sw.Stop();
                        Console.WriteLine("Allocating memory for output stream on server:\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

                        // Initialize LMem
                        sw.Reset(); sw.Start();

                        client.correlation_loadLMem(numBursts, addressLoopLength, addressInMemLoad);

                        sw.Stop();
                        Console.WriteLine("LMem initialization:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

                        //Executing correlation action
                        sw.Reset(); sw.Start();

                        client.correlation(numBursts,               // scalar input
                                           numTimesteps,            // scalar input
                                           numTimeseries,           // scalar input
                                           1,                       // scalar input
                                           windowSize,              // scalar input
                                           addressPrecalculations,  // streaming reordered input
                                           addressDataPairs,        // streaming reordered input
                                           addressOutCorrelation,   // streaming unordered output
                                           addressOutIndices);      // streaming unordered output

                        sw.Stop();
                        Console.WriteLine("Correlation time:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

                        // Get output stream from server
                        sw.Reset(); sw.Start();
                        var outCorrelationSize = numTimesteps * loopLength[0] * correlationNumTopScores * correlationNumPipes + numBursts * 24;
                        List<double> outCorrelation = new List<double>();
                        outCorrelation = client.receive_data_double(addressOutCorrelation, outCorrelationSize);
                        sw.Stop();
                        var outCorrelationTime = sw.Elapsed.TotalMilliseconds / 1000;
                        Console.WriteLine("\tGet output stream Correlation:\t(size = {0} bit)\t{1}s",
                                          outCorrelationSize * 64, outCorrelationTime);

                        sw.Reset(); sw.Start();
                        var outIndicesSzie = 2 * numTimesteps * loopLength[0] * correlationNumTopScores * correlationNumPipes;
                        List<int> outIndices = new List<int>();
                        outIndices = client.receive_data_int32_t(addressOutIndices, outIndicesSzie);
                        sw.Stop();
                        var outIndicesTime = sw.Elapsed.TotalMilliseconds / 1000;
                        Console.WriteLine("\tGet output stream outIndices:\t(size = {0} bit)\t{1}s",
                                          outIndicesSzie * 32, outIndicesTime);

                        sw.Reset(); sw.Start();
                        loopLengthSize = 1;
                        loopLength = client.receive_data_int32_t(addressLoopLength, loopLengthSize);
                        sw.Stop();
                        loopLengthTime = sw.Elapsed.TotalMilliseconds / 1000;
                        Console.WriteLine("\tGet output stream loopLength:\t(size = {0} bit)\t\t{1}s",
                                          loopLengthSize * 32, loopLengthTime);

                        time = loopLengthTime + outCorrelationTime + outIndicesTime;
                        speed = (loopLengthSize * 32 + outIndicesSzie * 32 + outCorrelationSize * 64) / time / 1000000;
                        Console.WriteLine("Getting output stream from server total time:\t{0}s\t(average speed = {1}Mb/s)", time, speed);

                        // Free allocated memory for streams on server
                        sw.Reset(); sw.Start();
                        client.free(addressLoopLength);
                        client.free(addressInMemLoad);
                        client.free(addressPrecalculations);
                        client.free(addressDataPairs);
                        client.free(addressOutCorrelation);
                        client.free(addressOutIndices);
                        sw.Stop();
                        Console.WriteLine("Freeing allocated memory for streams on server:\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

                        // Close!
                        sw.Reset(); sw.Start();
                        transport.Close();
                        sw.Stop();
                        Console.WriteLine("Closing connection:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

                        // Store data
                        sw.Reset(); sw.Start();

                        int position = 0;
                        int index = 0;
                        int start = (numTimesteps-1) * loopLength[0] * correlationNumTopScores * correlationNumPipes;

                        for (int i=0; i < numTimeseries; i++) {
                                for(int j = 0; j < i; j++) {
                                        correlations[index + j] = outCorrelation[start + position + j];
                                }
                                index += i;
                                position += ((i/12)+1)*12;
                        }

                        sw.Stop();
                        Console.WriteLine("LMem initialization:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
                } catch (SocketException e) {
                        Console.WriteLine("Could not connect to the server: {0}.", e.Message);
                        Environment.Exit(-1);
                } catch (Exception e) {
                        Console.WriteLine("An error occured: {0}", e.Message);
                        Environment.Exit(-1);
                }
    }
コード例 #36
0
ファイル: HClient.cs プロジェクト: zbw911/CS4Hadoop
        public static void UpdateData()
        {
            var transport = new TBufferedTransport(new TSocket("master", 9090));

            transport.Open();

            var protocol = new TBinaryProtocol(transport);

            var client = new Apache.Hadoop.Hbase.Hbase.Client(protocol);

            //client.createTable(Unitl.StrToBytes(tablename), new List<ColumnDescriptor> { new ColumnDescriptor { Name = Unitl.StrToBytes("zbw911"), InMemory = false } });

            client.mutateRow(Unitl.StrToBytes("1"), Unitl.StrToBytes("key"), new List<Mutation> { new Mutation { Column = Unitl.StrToBytes("zbw911"), Value = Unitl.StrToBytes("zbw911-Modifyed") } });

            transport.Close();
        }
コード例 #37
0
ファイル: FSClient.cs プロジェクト: yaowenhua/NetHadoop
        //批量下载
        public void MutDownLoad(BackgroundWorker worker, string localRootPath, List <FileStatus> fileList, int fileType)
        {
            //相同操作
            bool sameOp = false;
            //是否覆盖
            bool IsOver = false;

            //准备创建连接
            Thrift.Transport.TBufferedTransport btsport      = null;
            ThriftHadoopFileSystem.Client       thriftClient = Connect(out btsport);

            if (thriftClient != null)//连接成功
            {
                int totalCount   = fileList.Count;
                int currentCount = 0;
                //开始上传
                worker.ReportProgress(0, new ProgressState()
                {
                    ListBoxMsg = "开始下载!", totalCount = totalCount, CurrentCount = 0
                });
                //循环
                foreach (FileStatus myfile in fileList)
                {
                    currentCount++;
                    int pgPresent = (int)((double)currentCount / totalCount * 100);

                    string fileName = Path.GetFileName(myfile.Path);
                    if (myfile.Isdir == false)
                    {
                        string savePath = localRootPath + "/" + fileName;
                        if (fileType == 1 && myfile.FileName != null)
                        {
                            savePath = localRootPath + "/" + myfile.FileName;
                        }
                        if (fileType == 2 && myfile.FileName != null)
                        {
                            savePath = localRootPath + "/" + myfile.FileName + "/" + fileName;
                        }
                        //显示总进度
                        worker.ReportProgress(pgPresent, new ProgressState()
                        {
                            CurrentTitle = fileName, CurrentCount = currentCount
                        });

                        #region 是否存在
                        if (!sameOp)
                        {
                            bool exsitFile = File.Exists(savePath);
                            if (exsitFile)
                            {
                                SureDialog         myDialog = new SureDialog();
                                MyShowDialogResult myDR     = new MyShowDialogResult();
                                myDialog.ShowDialog(fileName, myDR);
                                sameOp = myDR.IsCheck;
                                IsOver = myDR.Result;
                                if (!myDR.Result)
                                {
                                    worker.ReportProgress(pgPresent, new ProgressState()
                                    {
                                        ListBoxMsg = fileName + " 跳过 "
                                    });
                                    continue;
                                }
                            }
                        }
                        else
                        {
                            bool exsitFile = File.Exists(savePath);
                            if (exsitFile)
                            {
                                if (!IsOver)
                                {
                                    worker.ReportProgress(pgPresent, new ProgressState()
                                    {
                                        ListBoxMsg = fileName + "跳过 "
                                    });
                                    continue;
                                }
                            }
                        }
                        #endregion

                        #region 单个下载
                        bool         result = false;
                        ThriftHandle th     = thriftClient.open(new Pathname()
                        {
                            pathname = myfile.Path
                        });
                        // 创建文件流
                        FileStream fs         = new FileStream(savePath, FileMode.Create, FileAccess.Write);
                        long       totalBytes = 0;
                        int        readLength = 1024 * 1024;
                        try
                        {
                            UTF8Encoding utf8 = new UTF8Encoding(false, true);
                            while (true)
                            {
                                int needRead = readLength;
                                if (myfile.Length - totalBytes < readLength)
                                {
                                    needRead = (int)(myfile.Length - totalBytes);
                                }
                                if (needRead <= 0)
                                {
                                    break;
                                }

                                byte[] fileBuffer   = thriftClient.read(th, totalBytes, needRead);
                                byte[] myfileBuffer = Encoding.Convert(utf8, Encoding.GetEncoding("iso-8859-1"), fileBuffer);
                                totalBytes += needRead;
                                fs.Write(myfileBuffer, 0, myfileBuffer.Length);

                                //显示单个上传进度
                                int mypresent = (int)((double)totalBytes / myfile.Length * 100);
                                worker.ReportProgress(pgPresent, new ProgressState()
                                {
                                    CurrentTitle = mypresent + "% " + fileName
                                });
                            }
                            result = true;
                        }
                        catch (Exception ee)
                        {
                            throw ee;
                        }
                        finally
                        {
                            fs.Dispose();
                            if (thriftClient != null)
                            {
                                thriftClient.close(th);
                            }
                        }

                        #endregion

                        string msg = string.Format("{0} 下载{1}", Path.GetFileName(fileName), result ? "成功" : "失败");
                        worker.ReportProgress(pgPresent, new ProgressState()
                        {
                            ListBoxMsg = msg
                        });
                    }
                    else
                    {
                        worker.ReportProgress(pgPresent, new ProgressState()
                        {
                            ListBoxMsg = fileName + "不是文件!"
                        });
                    }
                }
            }

            //释放连接
            if (btsport != null)
            {
                btsport.Close();
            }
        }
コード例 #38
0
ファイル: TestClient.cs プロジェクト: vicaya/thrift
        public static void Execute(string[] args)
        {
            try
            {
                string host = "localhost";
                int port = 9090;
                string url = null;
                int numThreads = 1;
                bool buffered = false;

                try
                {
                    for (int i = 0; i < args.Length; i++)
                    {
                        if (args[i] == "-h")
                        {
                            string[] hostport = args[++i].Split(':');
                            host = hostport[0];
                            if (hostport.Length > 1)
                            {
                                port = Convert.ToInt32(hostport[1]);
                            }
                        }
                        else if (args[i] == "-u")
                        {
                            url = args[++i];
                        }
                        else if (args[i] == "-n")
                        {
                            numIterations = Convert.ToInt32(args[++i]);
                        }
                        else if (args[i] == "-b" || args[i] == "-buffered")
                        {
                            buffered = true;
                            Console.WriteLine("Using buffered sockets");
                        }
                        else if (args[i] == "-t")
                        {
                            numThreads = Convert.ToInt32(args[++i]);
                        }
                    }
                }
                catch (Exception e)
                {
                    Console.WriteLine(e.StackTrace);
                }

                //issue tests on separate threads simultaneously
                Thread[] threads = new Thread[numThreads];
                DateTime start = DateTime.Now;
                for (int test = 0; test < numThreads; test++)
                {
                    Thread t = new Thread(new ParameterizedThreadStart(ClientThread));
                    threads[test] = t;
                    TSocket socket = new TSocket(host, port);
                    if (buffered)
                    {
                        TBufferedTransport buffer = new TBufferedTransport(socket);
                        t.Start(buffer);
                    }
                    else
                    {
                        t.Start(socket);
                    }
                }

                for (int test = 0; test < numThreads; test++)
                {
                    threads[test].Join();
                }
                Console.Write("Total time: " + (DateTime.Now - start));
            }
            catch (Exception outerEx)
            {
                Console.WriteLine(outerEx.Message + " ST: " + outerEx.StackTrace);
            }

            Console.WriteLine();
            Console.WriteLine();
        }
コード例 #39
0
    public static void Main(string[] args)
    {
        Stopwatch sw = new Stopwatch();
                Stopwatch swDFE = new Stopwatch();
                sw.Start();
                swDFE.Start();

                // Make socket
                var socket = new TSocket("localhost", 9090);

                // Buffering is critical. Raw sockets are very slow
                var transport = new TBufferedTransport(socket);

                // Wrap in a protocol
                var protocol = new TBinaryProtocol(transport);

                // Create a client to use the protocol encoder
                var client = new MovingAverageService.Client(protocol);

                sw.Stop();
                Console.WriteLine("Creating a client:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

                try {
                        // Connect!
                        sw.Reset(); sw.Start();
                        transport.Open();
                        sw.Stop();
                        Console.WriteLine("Opening connection:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

                        sw.Reset(); sw.Start();
                        const int size = 384;
                        int sizeBytes = size * 4;
                        List<double> dataIn = new List<double>();
                        // Generate input data
                        Random random = new Random();
                        for(int i = 0; i<size; i++) {
                                dataIn.Add(random.Next(0, 100));
                        }
                        sw.Stop();
                        Console.WriteLine("Generating input data:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

                        // Initialize maxfile
                        sw.Reset(); sw.Start();
                        var maxfile = client.MovingAverage_init();
                        sw.Stop();
                        Console.WriteLine("Initializing maxfile:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

                        // Load DFE
                        sw.Reset(); sw.Start();
                        var engine = client.max_load(maxfile, "*");
                        sw.Stop();
                        Console.WriteLine("Loading DFE:\t\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

                        // Allocate and send input streams to server
                        sw.Reset(); sw.Start();
                        var address_dataIn = client.malloc_float(size);
                        client.send_data_float(address_dataIn, dataIn);
                        sw.Stop();
                        Console.WriteLine("Sending input data:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

                        // Allocate memory for output stream on server
                        sw.Reset(); sw.Start();
                        var address_dataOut = client.malloc_float(size);
                        sw.Stop();
                        Console.WriteLine("Allocating memory for output stream on server:\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

                        // Action default
                        sw.Reset(); sw.Start();
                        var actions = client.max_actions_init(maxfile, "default");
                        client.max_set_param_uint64t(actions, "N", size);
                        client.max_queue_input(actions, "x", address_dataIn, sizeBytes);
                        client.max_queue_output(actions, "y", address_dataOut, sizeBytes);
                        client.max_run(engine, actions);
                        sw.Stop();
                        Console.WriteLine("Moving average time:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

                        // Unload DFE
                        sw.Reset(); sw.Start();
                        client.max_unload (engine);
                        sw.Stop();
                        Console.WriteLine("Unloading DFE:\t\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

                        // Get output stream from server
                        sw.Reset(); sw.Start();
                        List<double> dataOut = client.receive_data_float(address_dataOut, size);
                        sw.Stop();
                        Console.WriteLine("Getting output stream:\t(size = {0} bit)\t{1}s", size * 32, sw.Elapsed.TotalMilliseconds / 1000);

                        // Free allocated memory for streams on server
                        sw.Reset(); sw.Start();
                        client.free(address_dataIn);
                        client.free(address_dataOut);
                        sw.Stop();
                        Console.WriteLine("Freeing allocated memory for streams on server:\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

                        // Free allocated maxfile data
                        sw.Reset(); sw.Start();
                        client.MovingAverage_free();
                        sw.Stop();
                        Console.WriteLine("Freeing allocated maxfile data:\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

                        // Checking results
                        sw.Reset(); sw.Start();
                        int status = check(dataIn, dataOut, size);
                        sw.Stop();
                        Console.WriteLine("Checking results:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

                        sw.Reset(); sw.Start();
                        transport.Close();
                        sw.Stop();
                        Console.WriteLine("Closing connection:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

                        swDFE.Stop();
                        Console.WriteLine("DFE moving average total time:\t\t\t{0}s", swDFE.Elapsed.TotalMilliseconds / 1000);

                        if (status==0) {
                                Console.WriteLine("Test successful!");
                        } else {
                                Console.WriteLine("Test failed {0} times!", status);
                                Environment.Exit(-1);
                        }

                } catch (SocketException e) {
                        Console.WriteLine("Could not connect to the server: {0}.", e.Message);
                        Environment.Exit(-1);
                } catch (Exception e) {
                        Console.WriteLine("An error occured: {0}", e.Message);
                        Environment.Exit(-1);
                }
    }