Example #1
0
        //获取连接
        public ThriftHadoopFileSystem.Client Connect(out TBufferedTransport tsport)
        {
            if (HostServer == null)
               {
               throw new ArgumentNullException("HostServer");
               }

               if (Port == 0)
               {
               throw new ArgumentNullException("Port");
               }

               TSocket hadoop_socket = new TSocket(HostServer, Port);

               //hadoop_socket.Timeout = 10000;// Ten seconds

               tsport = new TBufferedTransport(hadoop_socket);

               TBinaryProtocol hadoop_protocol = new TBinaryProtocol(tsport, false, false);

               ThriftHadoopFileSystem.Client client = new ThriftHadoopFileSystem.Client(hadoop_protocol);
               try
               {
               tsport.Open();
               return client;
               }
               catch (Exception ex)
               {
               //throw (new Exception("打开连接失败!", ex));
               tsport = null;
               return null;

               }
        }
Example #2
0
        public void GetHBaseImage(string rowKey = "20150822180801118")
        {
            var socket = new TSocket(hbaseThrif, hbaseThrifPort);
            var transport = new TBufferedTransport(socket);
            var protocol = new TBinaryProtocol(transport);
            Hbase.Client hbaseClient = new Hbase.Client(protocol);
            transport.Open();

            //List<byte[]> tableNames = hc.getTableNames();

            byte[] table = Encoding.UTF8.GetBytes("FiservImages");
            byte[] row = Encoding.UTF8.GetBytes(rowKey);
            byte[] column = Encoding.UTF8.GetBytes("ImageData:Image");

            List<TRowResult> results = hbaseClient.getRow(table, row, null);
            TCell t = new TCell();
            //t.

            foreach (TRowResult result in results)
            {
                //result.Columns[column]
                //result.Columns;
                Dictionary<byte[], TCell> konj;
                konj = result.Columns;

                //string s = Encoding.UTF8.GetString(result.Columns[column].Value);

                foreach (KeyValuePair<byte[], TCell> singleRow in konj)
                {
                    //File.WriteAllBytes(@"c:\test\output.docx", singleRow.Value.Value);
                    OpenFile(singleRow.Value.Value, "output.docx");
                }
            }
        }
Example #3
0
        public static void query()
        {
            var transport = new TBufferedTransport(new TSocket("hserver", 10000));
            var protocol = new TBinaryProtocol(transport);
            var client = new ThriftHive.Client(protocol);

            transport.Open();

            //client.execute("CREATE TABLE r(a STRING, b INT, c DOUBLE)");
            //client.execute("LOAD TABLE LOCAL INPATH '/path' INTO TABLE r");
            client.execute("SELECT * FROM pokes where foo=180");
            while (true)
            {
                var row = client.fetchOne();
                if (string.IsNullOrEmpty(row))
                    break;
                System.Console.WriteLine(row);
            }
            client.execute("SELECT * FROM pokes");
            var all = client.fetchAll();

            System.Console.WriteLine(all);

            transport.Close();
        }
        internal ThriftMessageSender(Uri serviceUri)
        {
            socket = new TSocket(serviceUri.Host, serviceUri.Port);
            transport = new TBufferedTransport(socket);
            transport.Open();

            client = new MessageService.Client(new TCompactProtocol(transport));
        }
Example #5
0
 static void Main(string[] args)
 {
     TTransport trans = new TSocket("localhost", 8585);
     trans = new TBufferedTransport(trans as TStreamTransport);
     trans.Open();
     TBinaryProtocol proto = new TBinaryProtocol(trans);
     helloSvc.Client client = new helloSvc.Client(proto);
     string result = client.getMessage("Ginger");
     Console.WriteLine("Received from server: " + result);
 }
Example #6
0
        public static void test()
        {
            var hadoop_socket = new TSocket("hserver", 59256);
            hadoop_socket.Timeout = 10000;// Ten seconds

            //$hadoop_socket -> setRecvTimeout(20000); // Twenty seconds
            var hadoop_transport = new TBufferedTransport(hadoop_socket);
            var hadoop_protocol = new TBinaryProtocol(hadoop_transport);
            var hadoopClient = new ThriftHadoopFileSystem.Client(hadoop_protocol);
            hadoop_transport.Open();

            //if (hadoopClient.exists(new Pathname { pathname = "/user/root/input" }))
            //{
            //    System.Console.WriteLine("exists");
            //}
            //else
            //{
            //    Console.WriteLine("no exists");

            //}

            //hadoopClient.mkdirs(new Pathname { pathname = "/zbw911_user" });
            //hadoopClient.rm(new Pathname { pathname = "/zbw911_user" }, true);

            //var h = hadoopClient.createFile(new Pathname { pathname = "user_zbw911" }, 1, true, 1000, 1, 64000);

            var list = hadoopClient.listStatus(new Pathname { pathname = "/user/root" });

            foreach (var item in list)
            {
                System.Console.WriteLine(item);
                //if (item.Isdir)
                //{
                //    System.Console.WriteLine("文件夹:" + item.Path + ";权限:" + item.Permission);
                //}
                //else
                //{

                //    System.Console.WriteLine("文件:" + item.Path + ";权限:" + item.Permission);

                //}
            }

            //hadoopClient.close(h);

            //hadoopClient.open(new Pathname { pathname = "/user/root" });

            hadoop_transport.Close();
        }
Example #7
0
        public static void t()
        {
            var transport = new TBufferedTransport(new TSocket("hserver", 10000));
            var protocol = new TBinaryProtocol(transport);
            var client = new ThriftHive.Client(protocol);

            transport.Open();

            //client.execute("CREATE TABLE r(a STRING, b INT, c DOUBLE)");
            //client.execute("LOAD TABLE LOCAL INPATH '/path' INTO TABLE r");

            var database = new Database();

            database.Name = "helloworldDB";
            database.Description = "测试用的第一个DB";

            client.create_database(database);

            transport.Close();
        }
Example #8
0
        //获取连接
        public ThriftHadoopFileSystem.Client Connect(out TBufferedTransport tsport)
        {
            TSocket hadoop_socket = new TSocket(HostIP, HostPort);

               //hadoop_socket.Timeout = 10000;// Ten seconds

               tsport = new TBufferedTransport(hadoop_socket);

               TBinaryProtocol hadoop_protocol = new TBinaryProtocol(tsport, false, false);

               ThriftHadoopFileSystem.Client client = new ThriftHadoopFileSystem.Client(hadoop_protocol);
               try
               {
               tsport.Open();
               return client;
               }
               catch (Exception ee)
               {
               logger.Error("打开连接失败!", ee);
               tsport = null;
               return null;
               }
        }
Example #9
0
        public Dictionary<byte[], TCell> GetHBaseImage()
        {
            var socket = new TSocket(_hbaseThrif, _hbaseThrifPort);
            var transport = new TBufferedTransport(socket);
            var protocol = new TBinaryProtocol(transport);
            Hbase.Client hbaseClient = new Hbase.Client(protocol);
            Dictionary<byte[], TCell> hbaseResult = new Dictionary<byte[], TCell>();
            transport.Open();

            //List<byte[]> tableNames = hc.getTableNames();

            byte[] table = Encoding.UTF8.GetBytes("FiservImages");
            byte[] row = Encoding.UTF8.GetBytes(_documentId);
            byte[] column = Encoding.UTF8.GetBytes("ImageData:Image");

            List<TRowResult> results = hbaseClient.getRow(table, row, null);
            TCell t = new TCell();
            //t.

            foreach (TRowResult result in results)
            {
                //result.Columns[column]
                //result.Columns;

                hbaseResult = result.Columns;

                //string s = Encoding.UTF8.GetString(result.Columns[column].Value);

                //foreach (KeyValuePair<byte[], TCell> singleRow in konj)
                //{
                //    //File.WriteAllBytes(@"c:\test\output.docx", singleRow.Value.Value);
                //    Main a = new Main();
                //    a.OpenFile(singleRow.Value.Value, "output.docx");
                //}
            }
            return hbaseResult;
        }
Example #10
0
        public static void t2()
        {
            var transport = new TBufferedTransport(new TSocket("hserver", 10000));
            var protocol = new TBinaryProtocol(transport);
            var client = new ThriftHive.Client(protocol);

            transport.Open();

            var database = new Database();

            database.Name = "helloworldDB";
            database.Description = "测试用的第一个DB";

            var tables = client.get_all_tables("default");

            //var tables = client.get_all_databases();

            foreach (var table in tables)
            {
                System.Console.WriteLine(table);
            }

            transport.Close();
        }
Example #11
0
        public static void ShowTables()
        {
            var transport = new TBufferedTransport(new TSocket("master", 9090));

            transport.Open();

            var protocol = new TBinaryProtocol(transport);

            var client = new Apache.Hadoop.Hbase.Hbase.Client(protocol);

            var tables = client.getTableNames();

            foreach (var table in tables)
            {
                Console.WriteLine(Unitl.BytesToStr(table));
            }

            transport.Close();
        }
Example #12
0
        public static void Scan()
        {
            var transport = new TBufferedTransport(new TSocket("master", 9090));

            transport.Open();

            var protocol = new TBinaryProtocol(transport);

            var client = new Apache.Hadoop.Hbase.Hbase.Client(protocol);

            var scanid = client.scannerOpenWithStop(Unitl.StrToBytes("t1"), Unitl.StrToBytes("rowno"), Unitl.StrToBytes("rowno"),
                new List<byte[]> { Unitl.StrToBytes("f1:99913") });

            var result = client.scannerGet(scanid);

            var colum = client.getColumnDescriptors("t1".ToBytes());

            var rows = client.getRowsWithColumns("t1".ToBytes()
                  , new List<byte[]> { "rowno".ToBytes() }
                  , new List<byte[]> { "f1:9".ToBytes(), "f1:9999".ToBytes() });

            //scanid = client.scannerOpenWithPrefix("".ToBytes(), "".ToBytes(), new List<byte[]> { });

            var v = client.getVer("table".ToBytes(), "row".ToBytes(), "col".ToBytes(), 1000);

            foreach (var xx in v)
            {

            }

            foreach (var r in result)
            {
                Console.WriteLine(r.Columns[Unitl.StrToBytes("f1:99913")].Value);
            }

            client.scannerClose(scanid);

            transport.Close();
        }
Example #13
0
        public static void GetData()
        {
            var transport = new TBufferedTransport(new TSocket("master", 9090));

            transport.Open();

            var protocol = new TBinaryProtocol(transport);

            var client = new Apache.Hadoop.Hbase.Hbase.Client(protocol);

            var rows = client.getRow(Unitl.StrToBytes("1"), Unitl.StrToBytes("row9916"));

            foreach (var row in rows)
            {
                Console.WriteLine(row.Columns.ToString());
            }

            transport.Close();
        }
Example #14
0
        public static void BatAddData()
        {
            var transport = new TBufferedTransport(new TSocket("master", 9090));

            transport.Open();

            var protocol = new TBinaryProtocol(transport);

            var client = new Apache.Hadoop.Hbase.Hbase.Client(protocol);

            //client.createTable(Unitl.StrToBytes(tablename), new List<ColumnDescriptor> { new ColumnDescriptor { Name = Unitl.StrToBytes("zbw911"), InMemory = false } });

            var bytevalue = Unitl.StrToBytes(Longdata);

            for (var j = 0; j < 100000; j++)
            {
                //var j = 0;
                for (var i = 0; i < 1000; i++)
                {
                    client.mutateRow(Unitl.StrToBytes("1"), Unitl.StrToBytes("" + i + "___" + j), new List<Mutation> { new Mutation { Column = Unitl.StrToBytes("zbw911"), Value = bytevalue } });
                }

                Console.WriteLine(j + "批=" + 1000 * j);
            }

            transport.Close();
        }
Example #15
0
        public static void AddTable(string tablename)
        {
            var transport = new TBufferedTransport(new TSocket("master", 9090));

            transport.Open();

            var protocol = new TBinaryProtocol(transport);

            var client = new Apache.Hadoop.Hbase.Hbase.Client(protocol);

            client.createTable(Unitl.StrToBytes(tablename), new List<ColumnDescriptor> { new ColumnDescriptor { Name = Unitl.StrToBytes("zbw911"), InMemory = false } });

            transport.Close();
        }
Example #16
0
        public static void AddColumn()
        {
            var transport = new TBufferedTransport(new TSocket("master", 9090));

            transport.Open();

            var protocol = new TBinaryProtocol(transport);

            var client = new Apache.Hadoop.Hbase.Hbase.Client(protocol);

            //client.createTable(Unitl.StrToBytes("t"), new List<ColumnDescriptor> { new ColumnDescriptor { Name = Unitl.StrToBytes("default"), InMemory = false } });
            //client.createTable(Unitl.StrToBytes("thetable"), new List<ColumnDescriptor> { new ColumnDescriptor { Name = Unitl.StrToBytes("default"), InMemory = false } });

            for (var i = 0; i < 10; i++)
            {

                client.mutateRow(Unitl.StrToBytes("t"),

                    Unitl.StrToBytes("key"),
                    new List<Mutation> {
                        new Mutation { Column = Unitl.StrToBytes("default:"+i ), Value = Unitl.StrToBytes( System.DateTime.Now.ToString() ) }

                    });

            }
            transport.Close();
        }
Example #17
0
        static void Main(string[] args)
        {
            Log.Use().SimpleConsoleColored();

            Log.Information("Booting");

            try
            {
                string host = ConfigurationManager.AppSettings["Interface.Host"];
                int port = Int32.Parse(ConfigurationManager.AppSettings["Interface.Port"]);

                var socket = new TSocket(host, port);
                var transport = new TBufferedTransport(socket);
                var protocol = new TBinaryProtocol(transport);
                var client = new InterfaceNodeService.Client(protocol);
                transport.Open();

                #region Input loop
                bool doContinueLoop = true;
                string cmd = String.Empty;
                string input = String.Empty;

                Entry tmpEntry = null;
                Result res = null;

                while (doContinueLoop)
                {
                    cmd = Prompt("What do you want to do? (insert, getreport, getset, quit)");

                    switch (cmd)
                    {
                        #region Quit
                        case "quit":
                        {
                            doContinueLoop = false;
                            break;
                        }
                        #endregion Quit

                        #region Insert
                        case "insert":
                        {
                            tmpEntry = new Entry();
                            tmpEntry.Data = new Dictionary<string, string>();

                            input = Prompt("Enter an Entry type");

                            tmpEntry.Type = input;
                            string tmpVarName = String.Empty;
                            string tmpVarValue = String.Empty;

                            while (true)
                            {
                                tmpVarName = Prompt("Enter name of var (or \"send\")");

                                if (tmpVarName.Equals("send"))
                                {
                                    break;
                                }

                                tmpVarValue = Prompt("Enter the value of the var (or \"send\")");

                                if (tmpVarValue.Equals("send"))
                                {
                                    break;
                                }

                                tmpEntry.Data.Add(tmpVarName, tmpVarValue);
                            }

                            Log.Trace("Sending Insert");
                            res = client.Insert(tmpEntry);
                            Log.Debug(res.DumpToString());

                            tmpEntry = null;

                            break;
                        }
                        #endregion Insert

                        #region Get report
                        case "getreport":
                        {
                            input = Prompt("Enter the name of the report you wish to get");

                            Log.Trace("Sending GetReport");
                            res = client.GetReport(input);
                            Log.Debug(res.DumpToString());

                            break;
                        }
                        #endregion Get report

                        #region Get set
                        case "getset":
                        {
                            input = Prompt("Enter the name of the set you wish to get");

                            Log.Trace("Sending GetSet");
                            res = client.GetSet(input);
                            Log.Debug(res.DumpToString());

                            break;
                        }
                        #endregion Get set

                        default:
                        {
                            Log.Error("Invalid command '{0}'", cmd);
                            break;
                        }
                    }
                }
                #endregion Input loop
            }
            catch(Exception ex)
            {
                Log.Error(ex.ToString());
            }

            Log.Information("Done... closing");
        }
Example #18
0
    public static void CorrelateDFE(double[,] data, int sizeTimeseries,
                                        int numTimeseries, double[] correlations)
    {
        // Calculates correlations on DFE.
                Stopwatch sw = new Stopwatch();
                sw.Start();

                // Make socket
                var socket = new TSocket("localhost", 9090);

                // Buffering is critical. Raw sockets are very slow
                var transport = new TBufferedTransport(socket);

                // Wrap in a protocol
                var protocol = new TBinaryProtocol(transport);

                // Create a client to use the protocol encoder
                var client = new correlationService.Client(protocol);

                sw.Stop();
                Console.WriteLine("Creating a client:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

                try {
                        // Connect!
                        sw.Reset(); sw.Start();
                        transport.Open();
                        sw.Stop();
                        Console.WriteLine("Opening connection:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

                        int numTimesteps = sizeTimeseries;
                        double windowSize = (double)sizeTimeseries;
                        int numBursts = calcNumBursts(numTimeseries);

                        // Get loop length
                        sw.Reset(); sw.Start();
                        List<int> loopLength = new List<int>();
                        loopLength.Add(client.correlation_get_CorrelationKernel_loopLength());
                        sw.Stop();
                        Console.WriteLine("Getting Correlation Kernel loopLength:\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

                        // Prepare data for DFE
                        sw.Reset(); sw.Start();

                        double[] precalculations = new double [2 * numTimeseries * numTimesteps];
                        double[] dataPairs = new double [2 * numTimeseries * numTimesteps];

                        int burstSize = 384 / 2;  // for anything other than ISCA this should be 384
                        List<int> inMemLoad = new List<int>();
                        for(int i = 0; i < numBursts * burstSize; i++) {
                                inMemLoad.Add(0);
                        }

                        prepareDataForDFE (data, sizeTimeseries, numTimeseries, numTimesteps, windowSize,
                                           precalculations, dataPairs);

                        List<double> precalculationsVec = new List<double>();
                        List<double> dataPairsVec = new List<double>();
                        for(int i = 0; i < 2 * numTimeseries * numTimesteps; i++) {
                                precalculationsVec.Add(precalculations[i]);
                                dataPairsVec.Add(dataPairs[i]);
                        }

                        sw.Stop();
                        Console.WriteLine("Data reordering time:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

                        // Allocate and send input streams to server
                        sw.Reset(); sw.Start();
                        var loopLengthSize = 1;
                        var addressLoopLength = client.malloc_int32_t(loopLengthSize);
                        client.send_data_int32_t(addressLoopLength, loopLength);
                        sw.Stop();
                        var loopLengthTime = sw.Elapsed.TotalMilliseconds / 1000;
                        Console.WriteLine("\tSending LoopLength:\t\t(size = {0} bit)\t\t{1}s",
                                          loopLengthSize * 32, loopLengthTime);

                        sw.Reset(); sw.Start();
                        var inMemLoadSize = numBursts * burstSize;
                        var addressInMemLoad = client.malloc_int32_t(inMemLoadSize);
                        client.send_data_int32_t(addressInMemLoad, inMemLoad);
                        sw.Stop();
                        var inMemLoadTime = sw.Elapsed.TotalMilliseconds / 1000;
                        Console.WriteLine("\tSending InMemLoad:\t\t(size = {0} bit)\t{1}s",
                                          inMemLoadSize * 32, inMemLoadTime);

                        sw.Reset(); sw.Start();
                        var precalculationsSize = 2 * numTimeseries * numTimesteps;
                        var addressPrecalculations = client.malloc_double(precalculationsSize);
                        client.send_data_double(addressPrecalculations, precalculationsVec);
                        sw.Stop();
                        var precalculationsTime = sw.Elapsed.TotalMilliseconds / 1000;
                        Console.WriteLine("\tSending Precalculations:\t(size = {0} bit)\t{1}s",
                                          precalculationsSize * 32, precalculationsTime);

                        sw.Reset(); sw.Start();
                        var dataPairsSize = 2 * numTimeseries * numTimesteps;
                        var addressDataPairs = client.malloc_double(dataPairsSize);
                        client.send_data_double(addressDataPairs, dataPairsVec);
                        sw.Stop();
                        var dataPairsTime = sw.Elapsed.TotalMilliseconds / 1000;
                        Console.WriteLine("\tSending DataPairs:\t\t(size = {0} bit)\t{1}s",
                                          dataPairsSize * 64, dataPairsTime);

                        var time = loopLengthTime + inMemLoadTime + precalculationsTime + dataPairsTime;
                        var speed = (loopLengthSize * 32 + inMemLoadSize * 32 + precalculationsSize * 64 + dataPairsSize * 64) / time / 1000000;
                        Console.WriteLine("Sending input streams to server total time:\t{0}s\t(average speed = {1}Mb/s)", time, speed);

                        // Allocate memory for output stream on server
                        sw.Reset(); sw.Start();
                        var addressOutCorrelation = client.malloc_double(numTimesteps * loopLength[0] *
                                                                         correlationNumTopScores * correlationNumPipes +
                                                                         numBursts * 24);
                        var addressOutIndices = client.malloc_int32_t(2 * numTimesteps * loopLength[0] *
                                                                      correlationNumTopScores * correlationNumPipes);
                        sw.Stop();
                        Console.WriteLine("Allocating memory for output stream on server:\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

                        // Initialize LMem
                        sw.Reset(); sw.Start();

                        client.correlation_loadLMem(numBursts, addressLoopLength, addressInMemLoad);

                        sw.Stop();
                        Console.WriteLine("LMem initialization:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

                        //Executing correlation action
                        sw.Reset(); sw.Start();

                        client.correlation(numBursts,               // scalar input
                                           numTimesteps,            // scalar input
                                           numTimeseries,           // scalar input
                                           1,                       // scalar input
                                           windowSize,              // scalar input
                                           addressPrecalculations,  // streaming reordered input
                                           addressDataPairs,        // streaming reordered input
                                           addressOutCorrelation,   // streaming unordered output
                                           addressOutIndices);      // streaming unordered output

                        sw.Stop();
                        Console.WriteLine("Correlation time:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

                        // Get output stream from server
                        sw.Reset(); sw.Start();
                        var outCorrelationSize = numTimesteps * loopLength[0] * correlationNumTopScores * correlationNumPipes + numBursts * 24;
                        List<double> outCorrelation = new List<double>();
                        outCorrelation = client.receive_data_double(addressOutCorrelation, outCorrelationSize);
                        sw.Stop();
                        var outCorrelationTime = sw.Elapsed.TotalMilliseconds / 1000;
                        Console.WriteLine("\tGet output stream Correlation:\t(size = {0} bit)\t{1}s",
                                          outCorrelationSize * 64, outCorrelationTime);

                        sw.Reset(); sw.Start();
                        var outIndicesSzie = 2 * numTimesteps * loopLength[0] * correlationNumTopScores * correlationNumPipes;
                        List<int> outIndices = new List<int>();
                        outIndices = client.receive_data_int32_t(addressOutIndices, outIndicesSzie);
                        sw.Stop();
                        var outIndicesTime = sw.Elapsed.TotalMilliseconds / 1000;
                        Console.WriteLine("\tGet output stream outIndices:\t(size = {0} bit)\t{1}s",
                                          outIndicesSzie * 32, outIndicesTime);

                        sw.Reset(); sw.Start();
                        loopLengthSize = 1;
                        loopLength = client.receive_data_int32_t(addressLoopLength, loopLengthSize);
                        sw.Stop();
                        loopLengthTime = sw.Elapsed.TotalMilliseconds / 1000;
                        Console.WriteLine("\tGet output stream loopLength:\t(size = {0} bit)\t\t{1}s",
                                          loopLengthSize * 32, loopLengthTime);

                        time = loopLengthTime + outCorrelationTime + outIndicesTime;
                        speed = (loopLengthSize * 32 + outIndicesSzie * 32 + outCorrelationSize * 64) / time / 1000000;
                        Console.WriteLine("Getting output stream from server total time:\t{0}s\t(average speed = {1}Mb/s)", time, speed);

                        // Free allocated memory for streams on server
                        sw.Reset(); sw.Start();
                        client.free(addressLoopLength);
                        client.free(addressInMemLoad);
                        client.free(addressPrecalculations);
                        client.free(addressDataPairs);
                        client.free(addressOutCorrelation);
                        client.free(addressOutIndices);
                        sw.Stop();
                        Console.WriteLine("Freeing allocated memory for streams on server:\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

                        // Close!
                        sw.Reset(); sw.Start();
                        transport.Close();
                        sw.Stop();
                        Console.WriteLine("Closing connection:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

                        // Store data
                        sw.Reset(); sw.Start();

                        int position = 0;
                        int index = 0;
                        int start = (numTimesteps-1) * loopLength[0] * correlationNumTopScores * correlationNumPipes;

                        for (int i=0; i < numTimeseries; i++) {
                                for(int j = 0; j < i; j++) {
                                        correlations[index + j] = outCorrelation[start + position + j];
                                }
                                index += i;
                                position += ((i/12)+1)*12;
                        }

                        sw.Stop();
                        Console.WriteLine("LMem initialization:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);
                } catch (SocketException e) {
                        Console.WriteLine("Could not connect to the server: {0}.", e.Message);
                        Environment.Exit(-1);
                } catch (Exception e) {
                        Console.WriteLine("An error occured: {0}", e.Message);
                        Environment.Exit(-1);
                }
    }
Example #19
0
        protected void HBaseMutateImage(string rowKey, byte[] image)
        {
            var socket = new TSocket("172.24.2.144", 9090);
            var transport = new TBufferedTransport(socket);
            var protocol = new TBinaryProtocol(transport);
            Hbase.Client hc = new Hbase.Client(protocol);
            transport.Open();

            //List<byte[]> tableNames = hc.getTableNames();

            byte[] table = Encoding.UTF8.GetBytes("FiservImages");
            byte[] row = Encoding.UTF8.GetBytes(rowKey);
            byte[] column = Encoding.UTF8.GetBytes("ImageData:Image");

            List<Mutation> mutation = new List<Mutation>();
            Mutation m = new Mutation();
            m.Column = column;
            m.Value = image;
            m.IsDelete = false;
            m.WriteToWAL = true;
            mutation.Add(m);

            hc.mutateRow(table, row, mutation, null);
        }
Example #20
0
        public static void UpdateData()
        {
            var transport = new TBufferedTransport(new TSocket("master", 9090));

            transport.Open();

            var protocol = new TBinaryProtocol(transport);

            var client = new Apache.Hadoop.Hbase.Hbase.Client(protocol);

            //client.createTable(Unitl.StrToBytes(tablename), new List<ColumnDescriptor> { new ColumnDescriptor { Name = Unitl.StrToBytes("zbw911"), InMemory = false } });

            client.mutateRow(Unitl.StrToBytes("1"), Unitl.StrToBytes("key"), new List<Mutation> { new Mutation { Column = Unitl.StrToBytes("zbw911"), Value = Unitl.StrToBytes("zbw911-Modifyed") } });

            transport.Close();
        }
Example #21
0
    public static void Main(string[] args)
    {
        Stopwatch sw = new Stopwatch();
                Stopwatch swDFE = new Stopwatch();
                sw.Start();
                swDFE.Start();

                // Make socket
                var socket = new TSocket("localhost", 9090);

                // Buffering is critical. Raw sockets are very slow
                var transport = new TBufferedTransport(socket);

                // Wrap in a protocol
                var protocol = new TBinaryProtocol(transport);

                // Create a client to use the protocol encoder
                var client = new MovingAverageService.Client(protocol);

                sw.Stop();
                Console.WriteLine("Creating a client:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

                try {
                        // Connect!
                        sw.Reset(); sw.Start();
                        transport.Open();
                        sw.Stop();
                        Console.WriteLine("Opening connection:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

                        sw.Reset(); sw.Start();
                        const int size = 384;
                        int sizeBytes = size * 4;
                        List<double> dataIn = new List<double>();
                        // Generate input data
                        Random random = new Random();
                        for(int i = 0; i<size; i++) {
                                dataIn.Add(random.Next(0, 100));
                        }
                        sw.Stop();
                        Console.WriteLine("Generating input data:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

                        // Initialize maxfile
                        sw.Reset(); sw.Start();
                        var maxfile = client.MovingAverage_init();
                        sw.Stop();
                        Console.WriteLine("Initializing maxfile:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

                        // Load DFE
                        sw.Reset(); sw.Start();
                        var engine = client.max_load(maxfile, "*");
                        sw.Stop();
                        Console.WriteLine("Loading DFE:\t\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

                        // Allocate and send input streams to server
                        sw.Reset(); sw.Start();
                        var address_dataIn = client.malloc_float(size);
                        client.send_data_float(address_dataIn, dataIn);
                        sw.Stop();
                        Console.WriteLine("Sending input data:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

                        // Allocate memory for output stream on server
                        sw.Reset(); sw.Start();
                        var address_dataOut = client.malloc_float(size);
                        sw.Stop();
                        Console.WriteLine("Allocating memory for output stream on server:\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

                        // Action default
                        sw.Reset(); sw.Start();
                        var actions = client.max_actions_init(maxfile, "default");
                        client.max_set_param_uint64t(actions, "N", size);
                        client.max_queue_input(actions, "x", address_dataIn, sizeBytes);
                        client.max_queue_output(actions, "y", address_dataOut, sizeBytes);
                        client.max_run(engine, actions);
                        sw.Stop();
                        Console.WriteLine("Moving average time:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

                        // Unload DFE
                        sw.Reset(); sw.Start();
                        client.max_unload (engine);
                        sw.Stop();
                        Console.WriteLine("Unloading DFE:\t\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

                        // Get output stream from server
                        sw.Reset(); sw.Start();
                        List<double> dataOut = client.receive_data_float(address_dataOut, size);
                        sw.Stop();
                        Console.WriteLine("Getting output stream:\t(size = {0} bit)\t{1}s", size * 32, sw.Elapsed.TotalMilliseconds / 1000);

                        // Free allocated memory for streams on server
                        sw.Reset(); sw.Start();
                        client.free(address_dataIn);
                        client.free(address_dataOut);
                        sw.Stop();
                        Console.WriteLine("Freeing allocated memory for streams on server:\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

                        // Free allocated maxfile data
                        sw.Reset(); sw.Start();
                        client.MovingAverage_free();
                        sw.Stop();
                        Console.WriteLine("Freeing allocated maxfile data:\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

                        // Checking results
                        sw.Reset(); sw.Start();
                        int status = check(dataIn, dataOut, size);
                        sw.Stop();
                        Console.WriteLine("Checking results:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

                        sw.Reset(); sw.Start();
                        transport.Close();
                        sw.Stop();
                        Console.WriteLine("Closing connection:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

                        swDFE.Stop();
                        Console.WriteLine("DFE moving average total time:\t\t\t{0}s", swDFE.Elapsed.TotalMilliseconds / 1000);

                        if (status==0) {
                                Console.WriteLine("Test successful!");
                        } else {
                                Console.WriteLine("Test failed {0} times!", status);
                                Environment.Exit(-1);
                        }

                } catch (SocketException e) {
                        Console.WriteLine("Could not connect to the server: {0}.", e.Message);
                        Environment.Exit(-1);
                } catch (Exception e) {
                        Console.WriteLine("An error occured: {0}", e.Message);
                        Environment.Exit(-1);
                }
    }
Example #22
0
    public static List<double> PassThroughDFE(int size, List<double> dataIn)
    {
        List<double> dataOut = new List<double>();
        Stopwatch sw = new Stopwatch();
        sw.Start();

        // Make socket
        var socket = new TSocket("localhost", 9090);

        // Buffering is critical. Raw sockets are very slow
        var transport = new TBufferedTransport(socket);

        // Wrap in a protocol
        var protocol = new TBinaryProtocol(transport);

        // Create a client to use the protocol encoder
        var client = new PassThroughService.Client(protocol);

        sw.Stop();
        Console.WriteLine("Creating a client:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

        try {
            // Connect!
            sw.Reset(); sw.Start();
            transport.Open();
            sw.Stop();
            Console.WriteLine("Opening connection:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

            // Initialize maxfile
            sw.Reset(); sw.Start();
            var maxfile = client.PassThrough_init();
            sw.Stop();
            Console.WriteLine("Initializing maxfile:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

            // Load DFE
            sw.Reset(); sw.Start();
            var engine = client.max_load(maxfile, "*");
            sw.Stop();
            Console.WriteLine("Loading DFE:\t\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

            // Allocate and send input streams to server
            sw.Reset(); sw.Start();
            var address_dataIn = client.malloc_float(size);
            client.send_data_float(address_dataIn, dataIn);
            sw.Stop();
            Console.WriteLine("Sending input data:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

            // Allocate memory for output stream on server
            sw.Reset(); sw.Start();
            var address_dataOut = client.malloc_float(size);
            sw.Stop();
            Console.WriteLine("Allocating memory for output stream on server:\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

            // Action default
            sw.Reset(); sw.Start();
            var action = new PassThrough_actions_t_struct();
            action.Param_N = size;
            action.Instream_x = address_dataIn;
            action.Outstream_y = address_dataOut;
            var address_action = client.send_PassThrough_actions_t(action);
            client.PassThrough_run(engine, address_action);
            sw.Stop();
            Console.WriteLine("Pass through time:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

            // Unload DFE
            sw.Reset(); sw.Start();
            client.max_unload(engine);
            sw.Stop();
            Console.WriteLine("Unloading DFE:\t\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

            // Get output stream from server
            sw.Reset(); sw.Start();
            dataOut = client.receive_data_float(address_dataOut, size);
            sw.Stop();
            Console.WriteLine("Getting output stream:\t(size = {0} bit)\t{1}s", size * 32, sw.Elapsed.TotalMilliseconds / 1000);

            // Free allocated memory for streams on server
            sw.Reset(); sw.Start();
            client.free(address_dataIn);
            client.free(address_dataOut);
            sw.Stop();
            Console.WriteLine("Freeing allocated memory for streams on server:\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

            // Free allocated maxfile data
            sw.Reset(); sw.Start();
            client.PassThrough_free();
            sw.Stop();
            Console.WriteLine("Freeing allocated maxfile data:\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

            sw.Reset(); sw.Start();
            transport.Close();
            sw.Stop();
            Console.WriteLine("Closing connection:\t\t\t\t{0}s", sw.Elapsed.TotalMilliseconds / 1000);

        } catch (SocketException e) {
            Console.WriteLine("Could not connect to the server: {0}.", e.Message);
            Environment.Exit(-1);
        } catch (Exception e) {
            Console.WriteLine("An error occured: {0}", e.Message);
            Environment.Exit(-1);
        }

        return dataOut;
    }
Example #23
0
        public IDisposable Query()
        {
            Contract.Requires(Observer != null);
            Contract.Requires(StartDate != null);
            Contract.Requires(EndDate != null);

            // restrict to 1 concurrent active query
            mBarrier.Wait();
            mBarrier.Reset();

            if (Observer != null) // won't ever be null, we're using code contracts to check this!
            {
                try
                {
                    // create an observable to request the data and encapsulate all the processing logic
                    return Observable.Create<Data>(o =>
                    {
                        var tokenSource = new CancellationTokenSource();
                        var cancelToken = tokenSource.Token;
                        _cancellationSource = tokenSource;

                        var task = Task.Factory.StartNew(() =>
                        {
                            // create thrift client connection
                            var socket = new TSocket(mServer, 5049);
                            var transport = new TBufferedTransport(socket);
                            var protocol = new TBinaryProtocol(transport);
                            var client = new TCLIService.Client(protocol);

                            // open transport/establish server connection
                            transport.Open();
                            var req = new TOpenSessionReq()
                            {
                                Client_protocol = TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V1,
                                Username = mUser,
                                Password = mPassword
                            };
                            var openResp = client.OpenSession(req);
                            var sessHandle = openResp.SessionHandle;

                            //-----------------QUERY---------------------------------
                            #region Query
                            //Perform actual query -- by switching on option from subscribe

                            #region  check UnionType
                            bool imbExist = false;

                          /*  if ((Options & SubscriptionOptions.IMBALANCE) == SubscriptionOptions.IMBALANCE)
                            {
                                //Console.WriteLine("IMBALANCES");
                                Options = Options
                                    | SubscriptionOptions.AMEX_IMBALANCE
                                    | SubscriptionOptions.ARCA_IMBALANCE
                                    | SubscriptionOptions.NYSE_IMBALANCE
                                    | SubscriptionOptions.NASD_IMBALANCE;
                            }*/

                            if ((Options & SubscriptionOptions.TRADE) == SubscriptionOptions.TRADE)
                            {
                                //Console.WriteLine("T");
                                unionType = "T";
                                unionNum++;
                            }
                            if ((Options & SubscriptionOptions.QUOTE) == SubscriptionOptions.QUOTE)
                            {
                                //Console.WriteLine("Q");
                                unionType = unionType + "Q";
                                unionNum++;
                            }
                            if ((Options & SubscriptionOptions.AMEX_IMBALANCE) == SubscriptionOptions.AMEX_IMBALANCE)
                            {
                                //Console.WriteLine("AMEX");
                                unionType = unionType + "I";
                                imbExist = true;
                                unionNum++;
                            }
                            if ((Options & SubscriptionOptions.ARCA_IMBALANCE) == SubscriptionOptions.ARCA_IMBALANCE)
                            {
                                //Console.WriteLine("ARCA");
                                if (imbExist == false)
                                {
                                    unionType = unionType + "I";
                                    imbExist = true;
                                }
                                unionNum++;
                            }
                            if ((Options & SubscriptionOptions.NYSE_IMBALANCE) == SubscriptionOptions.NYSE_IMBALANCE)
                            {
                                //Console.WriteLine("NYSE");
                                if (imbExist == false)
                                {
                                    unionType = unionType + "I";
                                    imbExist = true;
                                }
                                unionNum++;
                            }
                            if ((Options & SubscriptionOptions.NASD_IMBALANCE) == SubscriptionOptions.NASD_IMBALANCE)
                            {
                                //Console.WriteLine("NASD");
                                if (imbExist == false)
                                {
                                    unionType = unionType + "I";
                                    imbExist = true;
                                }
                                unionNum++;
                            }
                            #endregion

                            if ( unionNum == 1 || unionNum==0)
                            {
                                #region benchmark test

                                /*Console.WriteLine("no block marketdata 1");
                                doFetchTradeNo(client, sessHandle,StartDate,EndDate);
                                Console.WriteLine("no block marketdata 2");
                                doFetchTrade2No(client, sessHandle, StartDate, EndDate);
                                Console.WriteLine("block marketdata 1");
                                #region single table
                                for (DateTime QueryDate = StartDate; QueryDate <= EndDate; QueryDate = QueryDate.AddDays(1))
                                {
                                    if ((Options & SubscriptionOptions.TRADE) == SubscriptionOptions.TRADE)
                                    {
                                        doFetchTrade(client, sessHandle, QueryDate);
                                    }
                                    if ((Options & SubscriptionOptions.QUOTE) == SubscriptionOptions.QUOTE)
                                    {
                                        doFetchQuote(client, sessHandle, QueryDate);
                                    }
                                    if ((Options & SubscriptionOptions.AMEX_IMBALANCE) == SubscriptionOptions.AMEX_IMBALANCE)
                                    {
                                        doFetchAMEXImbalance(client, sessHandle, QueryDate);
                                    }
                                    if ((Options & SubscriptionOptions.ARCA_IMBALANCE) == SubscriptionOptions.ARCA_IMBALANCE)
                                    {
                                        doFetchARCAImbalance(client, sessHandle, QueryDate);
                                    }
                                    if ((Options & SubscriptionOptions.NYSE_IMBALANCE) == SubscriptionOptions.NYSE_IMBALANCE)
                                    {
                                        doFetchNYSEImbalance(client, sessHandle, QueryDate);
                                    }
                                    if ((Options & SubscriptionOptions.NASD_IMBALANCE) == SubscriptionOptions.NASD_IMBALANCE)
                                    {
                                        doFetchNASDImbalance(client, sessHandle, QueryDate);
                                    }
                                    if ((Options & SubscriptionOptions.DAILY) == SubscriptionOptions.DAILY)
                                    {
                                        doFetchDaily(client, sessHandle, QueryDate);
                                    }

                                }
                                #endregion
                                Console.WriteLine("block marketdata 2");*/

                                #endregion

                                #region single table
                                for (DateTime QueryDate = StartDate; QueryDate <= EndDate; QueryDate = QueryDate.AddDays(1))
                                {
                                    if ((Options & SubscriptionOptions.TRADE) == SubscriptionOptions.TRADE)
                                    {
                                        doFetchTrade2(client, sessHandle, QueryDate, QueryDate.Date==EndDate.Date);
                                    }
                                    if ((Options & SubscriptionOptions.QUOTE) == SubscriptionOptions.QUOTE)
                                    {
                                        doFetchQuote(client, sessHandle, QueryDate, QueryDate.Date == EndDate.Date);
                                    }
                                    if ((Options & SubscriptionOptions.AMEX_IMBALANCE) == SubscriptionOptions.AMEX_IMBALANCE)
                                    {
                                        doFetchAMEXImbalance(client, sessHandle, QueryDate);
                                    }
                                    if ((Options & SubscriptionOptions.ARCA_IMBALANCE) == SubscriptionOptions.ARCA_IMBALANCE)
                                    {
                                        doFetchARCAImbalance(client, sessHandle, QueryDate);
                                    }
                                    if ((Options & SubscriptionOptions.NYSE_IMBALANCE) == SubscriptionOptions.NYSE_IMBALANCE)
                                    {
                                        doFetchNYSEImbalance(client, sessHandle, QueryDate, QueryDate.Date==EndDate.Date);
                                    }
                                    if ((Options & SubscriptionOptions.NASD_IMBALANCE) == SubscriptionOptions.NASD_IMBALANCE)
                                    {
                                        doFetchNASDImbalance(client, sessHandle, QueryDate);
                                    }
                                    if ((Options & SubscriptionOptions.DAILY) == SubscriptionOptions.DAILY)
                                    {
                                        doFetchDaily(client, sessHandle, QueryDate, QueryDate.Date == EndDate.Date);
                                    }
                                    if ((Options & SubscriptionOptions.EXGPRINTS) == SubscriptionOptions.EXGPRINTS)
                                    {
                                        doFetchExgprints(client, sessHandle, QueryDate, QueryDate.Date == EndDate.Date);
                                    }
                                    if ((Options & SubscriptionOptions.NEWS) == SubscriptionOptions.NEWS)
                                    {
                                        doFetchNews(client, sessHandle, QueryDate, QueryDate.Date == EndDate.Date);
                                    }

                                }
                                #endregion
                            }
                            else
                            {
                                #region Union fetchdata

                                for (DateTime QueryDate = StartDate; QueryDate <= EndDate; QueryDate=QueryDate.AddDays(1))
                                {
                                    string query = "";
                                    bool firstUnion = false;
                                    if ((Options & SubscriptionOptions.TRADE) == SubscriptionOptions.TRADE)
                                    {
                                        if (firstUnion == true)
                                        {
                                            query = query + " union all " + createUnionTradeQuery(QueryDate);
                                        }
                                        else
                                        {
                                            query = createUnionTradeQuery(QueryDate);
                                        }
                                        firstUnion = true;
                                    }
                                    if ((Options & SubscriptionOptions.QUOTE) == SubscriptionOptions.QUOTE)
                                    {
                                        if (firstUnion == true)
                                        {
                                            query = query + " union all " + createUnionQuoteQuery(QueryDate);
                                        }
                                        else
                                        {
                                            query = createUnionQuoteQuery(QueryDate);
                                        }
                                        firstUnion = true;

                                    }
                                    if ((Options & SubscriptionOptions.AMEX_IMBALANCE) == SubscriptionOptions.AMEX_IMBALANCE)
                                    {
                                        if (firstUnion == true)
                                        {
                                            query = query + " union all " + createUnionAmexQuery(QueryDate);
                                        }
                                        else
                                        {
                                            query = createUnionAmexQuery(QueryDate);
                                        }
                                        firstUnion = true;
                                    }
                                    if ((Options & SubscriptionOptions.ARCA_IMBALANCE) == SubscriptionOptions.ARCA_IMBALANCE)
                                    {

                                        if (firstUnion == true)
                                        {
                                            query = query + " union all " + createUnionArcaQuery(QueryDate);
                                        }
                                        else
                                        {
                                            query = createUnionArcaQuery(QueryDate);
                                        }
                                        firstUnion = true;
                                    }
                                    if ((Options & SubscriptionOptions.NYSE_IMBALANCE) == SubscriptionOptions.NYSE_IMBALANCE)
                                    {
                                        if (firstUnion == true)
                                        {
                                            query = query + " union all " + createUnionNyseQuery(QueryDate);
                                        }
                                        else
                                        {
                                            query = createUnionNyseQuery(QueryDate);
                                        }
                                        firstUnion = true;
                                    }
                                    if ((Options & SubscriptionOptions.NASD_IMBALANCE) == SubscriptionOptions.NASD_IMBALANCE)
                                    {
                                        if (firstUnion == true)
                                        {
                                            query = query + " union all " + createUnionNasdQuery(QueryDate);
                                        }
                                        else
                                        {
                                            query = createUnionNasdQuery(QueryDate);
                                        }
                                        firstUnion = true;

                                    }
                                    if ((Options & SubscriptionOptions.DAILY) == SubscriptionOptions.DAILY)
                                    {
                                        doFetchDaily(client, sessHandle,QueryDate);
                                    }

                                    query = "select * from ( " + query + " ) log order by log.seqno limit " + Maxrows;
                                  //  query = "select * from ( " + query + " )";
                                    //Console.WriteLine(query);
                                    doFetchUnion(client, sessHandle, query);
                                }
                                #endregion
                            }
                            #endregion
                            //-------------------------------------------------------

                            // close the session/transport
                            var closeReq = new TCloseSessionReq
                            {
                                SessionHandle = sessHandle
                            };
                            client.CloseSession(closeReq);
                            transport.Close();
                        }, cancelToken);

                        return Disposable.Create(tokenSource.Cancel);
                    }).Subscribe(Observer);

                }
                catch (Exception ex)
                {
                    Observer.OnError(ex);
                }
                finally
                {
                    // reset barrier
                    mBarrier.Set();
                }
            }
            return null;
        }