public void When_I_Scan_with_a_SingleColumnValueFilter_and_a_SubstringComparator_with_the_operator_equal_I_get_the_expected_results()
        {
            // grab a substring that is guaranteed to match at least one record.
            string ss = _allExpectedRecords.First().A.Substring(1, 2);
            //Debug.WriteLine("The substring value is: " + ss);

            List <FilterTestRecord> expectedRecords = (from r in _allExpectedRecords where r.A.Contains(ss) select r).ToList();

            var client  = new HBaseClient(_credentials);
            var scanner = new Scanner();

            var comparer = new SubstringComparator(ss);

            var filter = new SingleColumnValueFilter(
                Encoding.UTF8.GetBytes(ColumnFamilyName1),
                Encoding.UTF8.GetBytes(ColumnNameA),
                CompareFilter.CompareOp.Equal,
                comparer);

            scanner.filter = filter.ToEncodedString();

            ScannerInformation      scanInfo      = client.CreateScanner(_tableName, scanner);
            List <FilterTestRecord> actualRecords = RetrieveResults(scanInfo).ToList();

            actualRecords.ShouldContainOnly(expectedRecords);
        }
        public void When_I_Scan_with_a_FilterList_with_OR_logic_I_get_the_expected_results()
        {
            List <FilterTestRecord> expectedRecords = (from r in _allExpectedRecords where r.LineNumber <= 2 select r).ToList();

            var client  = new HBaseClient(_credentials);
            var scanner = new Scanner();

            Filter f0 = new SingleColumnValueFilter(
                Encoding.UTF8.GetBytes(ColumnFamilyName1),
                Encoding.UTF8.GetBytes(LineNumberColumnName),
                CompareFilter.CompareOp.Equal,
                BitConverter.GetBytes(1));

            Filter f1 = new SingleColumnValueFilter(
                Encoding.UTF8.GetBytes(ColumnFamilyName1),
                Encoding.UTF8.GetBytes(LineNumberColumnName),
                CompareFilter.CompareOp.LessThanOrEqualTo,
                BitConverter.GetBytes(2));

            var filter = new FilterList(FilterList.Operator.MustPassOne, f0, f1);

            scanner.filter = filter.ToEncodedString();

            ScannerInformation      scanInfo      = client.CreateScanner(_tableName, scanner);
            List <FilterTestRecord> actualRecords = RetrieveResults(scanInfo).ToList();

            actualRecords.ShouldContainOnly(expectedRecords);
        }
        public void SingleColumnValueFilter_Validates_OK(Comparator comparator, ComparisonOperator comparison, bool latest, bool filterIfMissing)
        {
            var value  = "test";
            var filter = new SingleColumnValueFilter <MockDomainObject>(_map);


            filter.ComparisonOperator(comparison);
            filter.Comparator(comparator);

            filter.Column(x => x.Name);
            filter.ColumnValue(value);
            filter.FilterIfColumnMissing(filterIfMissing);
            filter.LatestVersion(latest);

            var validation = filter.ToFilterBuild();

            var isSpecialCase = filter.ValidateOperationForComparator(comparator, comparison);

            if (!isSpecialCase)
            {
                Assert.Throws <FilterException>(() => validation.Validate());
            }
            else
            {
                validation.Validate();
            }
        }
Exemple #4
0
        public void When_I_Scan_with_a_SingleColumnValueFilter_and_a_BinaryComparator_with_the_operator_not_equal_I_get_the_expected_results()
        {
            var expectedRecords = (from r in _allExpectedRecords where r.LineNumber != 1 select r).ToList();

            var client  = new HBaseClient(RequestOptionsFactory.GetDefaultOptions());
            var scanner = new Scanner();
            var filter  = new SingleColumnValueFilter(
                Encoding.UTF8.GetBytes(ColumnFamilyName1),
                Encoding.UTF8.GetBytes(LineNumberColumnName),
                CompareFilter.CompareOp.NotEqual,
                BitConverter.GetBytes(1));

            scanner.Filter = filter.ToEncodedString();


            ScannerInformation scanInfo = null;

            try
            {
                scanInfo = client.CreateScannerAsync(_tableName, scanner).Result;
                var actualRecords = RetrieveResults(scanInfo).ToList();

                actualRecords.ShouldContainOnly(expectedRecords);
            }
            finally
            {
                if (scanInfo != null)
                {
                    client.DeleteScannerAsync(_tableName, scanInfo).Wait();
                }
            }
        }
        public IFilter ToFilter()
        {
            SingleColumnValueFilter singleColumnValueFilter = new SingleColumnValueFilter(ColumnName, Operator, ColumnValue)
            {
                LatestVersionsOnly = LatestVersionsOnly,
                PassIfMissing      = PassIfMissing
            };

            return(singleColumnValueFilter);
        }
        public void SingleColumnValueFilter_Column_Enumerable_Validation_Ok()
        {
            var filter = new SingleColumnValueFilter <MockDomainObject>(_map);

            filter.Column(x => x.RandomList, "bla");
            filter.ColumnValue("test");

            var validation = filter.ToFilterBuild();

            validation.Validate();
        }
        public void SingleColumnValueFilter_NoValue_Validation_Throws_FilterException()
        {
            var filter = new SingleColumnValueFilter <MockDomainObject>(_map);

            filter.Column(x => x.Name);


            var validation = filter.ToFilterBuild();

            Assert.Throws <FilterException>(() => validation.Validate());
        }
        public void When_I_Scan_with_a_SingleColumnValueFilter_and_a_BinaryComparator_with_the_operator_not_equal_I_get_the_expected_results()
        {
            List <FilterTestRecord> expectedRecords = (from r in _allExpectedRecords where r.LineNumber != 1 select r).ToList();

            var client  = new HBaseClient(_credentials);
            var scanner = new Scanner();
            var filter  = new SingleColumnValueFilter(
                Encoding.UTF8.GetBytes(ColumnFamilyName1),
                Encoding.UTF8.GetBytes(LineNumberColumnName),
                CompareFilter.CompareOp.NotEqual,
                BitConverter.GetBytes(1));

            scanner.filter = filter.ToEncodedString();

            ScannerInformation      scanInfo      = client.CreateScanner(_tableName, scanner);
            List <FilterTestRecord> actualRecords = RetrieveResults(scanInfo).ToList();

            actualRecords.ShouldContainOnly(expectedRecords);
        }
Exemple #9
0
        public void When_I_Scan_with_a_FilterList_with_OR_logic_I_get_the_expected_results()
        {
            var expectedRecords = (from r in _allExpectedRecords where r.LineNumber <= 2 select r).ToList();

            var client  = new HBaseClient(RequestOptionsFactory.GetDefaultOptions());
            var scanner = new Scanner();

            Filter f0 = new SingleColumnValueFilter(
                Encoding.UTF8.GetBytes(ColumnFamilyName1),
                Encoding.UTF8.GetBytes(LineNumberColumnName),
                CompareFilter.CompareOp.Equal,
                BitConverter.GetBytes(1));

            Filter f1 = new SingleColumnValueFilter(
                Encoding.UTF8.GetBytes(ColumnFamilyName1),
                Encoding.UTF8.GetBytes(LineNumberColumnName),
                CompareFilter.CompareOp.LessThanOrEqualTo,
                BitConverter.GetBytes(2));

            var filter = new FilterList(FilterList.Operator.MustPassOne, f0, f1);

            scanner.Filter = filter.ToEncodedString();


            ScannerInformation scanInfo = null;

            try
            {
                scanInfo = client.CreateScannerAsync(_tableName, scanner).Result;
                var actualRecords = RetrieveResults(scanInfo).ToList();

                actualRecords.ShouldContainOnly(expectedRecords);
            }
            finally
            {
                if (scanInfo != null)
                {
                    client.DeleteScannerAsync(_tableName, scanInfo).Wait();
                }
            }
        }
        public void SingleColumnValueFilter_Builds_OK([CombinatorialValues(1.0f, 2.0d, 3L, 4, "test")] object value, Comparator comparator, ComparisonOperator comparison, bool latest, bool filterIfMissing)
        {
            var expected = $"SingleColumnValueFilter ('{Maps.MockDomainObject.TestColumnFamily}','{Maps.MockDomainObject.NameColumn}',{comparison.GetComparisonOperatorString()}," +
                           $"'{comparator.GetComparatorString()}{value}',{filterIfMissing},{latest})";

            var filter = new SingleColumnValueFilter <MockDomainObject>(_map);

            filter.ComparisonOperator(comparison);
            filter.Comparator(comparator);

            filter.Column(x => x.Name);
            filter.ColumnValue(value);
            filter.FilterIfColumnMissing(filterIfMissing);
            filter.LatestVersion(latest);

            var validation = filter.ToFilterBuild();

            var build = validation.Build();

            Assert.Equal(expected, build);
        }
        public void SingleColumnValueFilter_NoCF_Validation_Throws_FilterException()
        {
            var map = new ClassMap()
            {
                Maps = new List <IMap>()
                {
                    new PropertyMap()
                    {
                        ColumnName = Maps.MockDomainObject.NameColumn,
                        Name       = nameof(MockDomainObject.Name),
                        Type       = typeof(MockDomainObject)
                    }
                }
            };

            var filter = new SingleColumnValueFilter <MockDomainObject>(map);

            filter.Column(x => x.Name);
            var validation = filter.ToFilterBuild();

            Assert.Throws <FilterException>(() => validation.Validate());
        }
Exemple #12
0
        public void When_I_Scan_with_a_SingleColumnValueFilter_and_a_SubstringComparator_with_the_operator_equal_I_get_the_expected_results()
        {
            // grab a substring that is guaranteed to match at least one record.
            var ss = _allExpectedRecords.First().A.Substring(1, 2);
            //Debug.WriteLine("The substring value is: " + ss);

            var expectedRecords = (from r in _allExpectedRecords where r.A.Contains(ss) select r).ToList();

            var client  = new HBaseClient(RequestOptionsFactory.GetDefaultOptions());
            var scanner = new Scanner();

            var comparer = new SubstringComparator(ss);

            var filter = new SingleColumnValueFilter(
                Encoding.UTF8.GetBytes(ColumnFamilyName1),
                Encoding.UTF8.GetBytes(ColumnNameA),
                CompareFilter.CompareOp.Equal,
                comparer);

            scanner.Filter = filter.ToEncodedString();


            ScannerInformation scanInfo = null;

            try
            {
                scanInfo = client.CreateScannerAsync(_tableName, scanner).Result;
                var actualRecords = RetrieveResults(scanInfo).ToList();

                actualRecords.ShouldContainOnly(expectedRecords);
            }
            finally
            {
                if (scanInfo != null)
                {
                    client.DeleteScannerAsync(_tableName, scanInfo).Wait();
                }
            }
        }
        /// <summary>
        /// Retrieve sessions within the given time range
        /// </summary>
        /// <param name="hbaseClient">The hbase client</param>
        /// <param name="eventType">The type of event to look for</param>
        /// <param name="start">Lower bound of the time range</param>
        /// <param name="end">Upper bound of the time range</param>
        static void GetSessionsByTime(HBaseClient hbaseClient, string eventType, DateTime start, DateTime end)
        {
            //Create filters list
            FilterList filters = new FilterList(FilterList.Operator.MustPassAll);

            //Filter to search for the event type value
            SingleColumnValueFilter valueFilter = new SingleColumnValueFilter(
                Encoding.UTF8.GetBytes("cf"),
                Encoding.UTF8.GetBytes("event"),
                CompareFilter.CompareOp.Equal,
                Encoding.UTF8.GetBytes(eventType));

            filters.AddFilter(valueFilter);

            //Create scanner, set maxVersions so we can get previous versions of rows
            //Since START events may not be the currently returned value
            var scannerSettings = new Scanner()
            {
                filter      = filters.ToEncodedString(),
                maxVersions = 5,
                startTime   = ToUnixTime(start),
                endTime     = ToUnixTime(end)
            };

            var scanner = hbaseClient.CreateScanner(Properties.Settings.Default.HBaseTableName, scannerSettings);
            //Read data from scanner
            CellSet readSet = null;

            //While reading cell sets
            while ((readSet = hbaseClient.ScannerGetNext(scanner)) != null)
            {
                //Iterate over the rows returned
                foreach (var row in readSet.rows)
                {
                    //Get the time stored for the START event
                    var endTime = row.values.Where(v => Encoding.UTF8.GetString(v.column) == "cf:time")
                                  .Select(v => BitConverter.ToInt64(v.data, 0)).ToArray()[0];

                    //Get the hbase timestamp of the row
                    var timestamp = row.values.Select(v => v.timestamp).ToArray()[0];
                    //If it's an end event type
                    if (eventType == "END")
                    {
                        //Get the duration stored between END and START events
                        var duration = new TimeSpan(
                            row.values.Where(v => Encoding.UTF8.GetString(v.column) == "cf:duration")
                            .Select(v => BitConverter.ToInt64(v.data, 0)).ToArray()[0]);
                        //Write out the session info, including duration
                        Console.WriteLine("Session {0} lasted {1} minutes, and ended at {2}",
                                          Encoding.UTF8.GetString(row.key),
                                          duration.Minutes,
                                          FromUnixTime(endTime));
                    }
                    else
                    {
                        //If start event type, just write out when it started and the hbase timestamp for the row
                        Console.WriteLine("Session {0} started at {1}. Timestamp = {2}",
                                          Encoding.UTF8.GetString(row.key),
                                          FromUnixTime(endTime),
                                          timestamp);
                    }
                }
            }
        }
        public void SingleColumnValueFilter_NullMap_Validation_Throws_Exception()
        {
            var filter = new SingleColumnValueFilter <MockDomainObject>(null);

            Assert.Throws <MappingException>(() => filter.Column(x => x.Name));
        }
Exemple #15
0
        /// <summary>
        /// Executes incoming tuples
        /// </summary>
        /// <param name="tuple">The first field is treated as rowkey and rest as column names</param>
        public void Execute(SCPTuple tuple)
        {
            //get the tuple info
            string sessionId        = tuple.GetString(0);
            string sessionEvent     = tuple.GetString(1);
            long   sessionEventTime = tuple.GetLong(2);


            //If it's a start event, assume there's nothing to find so just re-emit
            //NOTE: If messages may arrive out of order, you would need to add logic to
            //query HBase to see if the end event has previously arrived,
            //calculate the duration, etc.
            if (sessionEvent == "START")
            {
                //Just re-emit the incoming data, plus 0 for duration, since we declare we send a 0 duration
                //since we don't know the END event yet.
                Values emitValues = new Values(tuple.GetValue(0), tuple.GetValue(1), tuple.GetValue(2), 0L);

                //Is ack enabled?
                if (enableAck)
                {
                    //Emit the values, anchored to the incoming tuple
                    this.context.Emit(Constants.DEFAULT_STREAM_ID, new List <SCPTuple>()
                    {
                        tuple
                    }, emitValues);
                    //Ack the incoming tuple
                    this.context.Ack(tuple);
                }
                else
                {
                    //No ack enabled? Fire and forget.
                    this.context.Emit(Constants.DEFAULT_STREAM_ID, emitValues);
                }
            }
            if (sessionEvent == "END")
            {
                //Use filters
                FilterList filters = new FilterList(FilterList.Operator.MustPassAll);
                //Filter on the row by sessionID
                RowFilter rowFilter = new RowFilter(CompareFilter.CompareOp.Equal, new BinaryComparator(TypeHelper.ToBytes(sessionId)));
                filters.AddFilter(rowFilter);
                //Filter on the event column for the START event
                SingleColumnValueFilter valueFilter = new SingleColumnValueFilter(
                    Encoding.UTF8.GetBytes("cf"),
                    Encoding.UTF8.GetBytes("event"),
                    CompareFilter.CompareOp.Equal,
                    Encoding.UTF8.GetBytes("START"));
                filters.AddFilter(valueFilter);
                //Create scanner settings using the filters
                var scannerSettings = new Scanner()
                {
                    filter = filters.ToEncodedString()
                };
                //Get the scanner
                var scanner = HBaseClusterClient.CreateScanner(HBaseTableName, scannerSettings);

                CellSet readSet = null;
                while ((readSet = HBaseClusterClient.ScannerGetNext(scanner)) != null)
                {
                    //In theory we should only find one row
                    foreach (var row in readSet.rows)
                    {
                        //Pull back just the event column
                        var rowState = row.values.Where(v => Encoding.UTF8.GetString(v.column) == "cf:event")
                                       .Select(v => Encoding.UTF8.GetString(v.data)).ToArray()[0];
                        //Is it a START event as expected?
                        if (rowState == "START")
                        {
                            //Get the start time
                            var startTime = TypeHelper.FromUnixTime(
                                row.values.Where(v => Encoding.UTF8.GetString(v.column) == "cf:time")
                                .Select(v => BitConverter.ToInt64(v.data, 0)).ToArray()[0]);
                            //Get the difference between start and end
                            DateTime endTime  = TypeHelper.FromUnixTime(sessionEventTime);
                            TimeSpan duration = endTime.Subtract(startTime);
                            //Emit the tuple, with the duration between start/end.
                            Values emitValues = new Values(sessionId, sessionEvent, sessionEventTime, duration.Ticks);
                            //If ack is enabled
                            if (enableAck)
                            {
                                //Emit the values, anchored to the incoming tuple
                                this.context.Emit(Constants.DEFAULT_STREAM_ID, new List <SCPTuple>()
                                {
                                    tuple
                                }, emitValues);
                                //Ack the incoming tuple
                                this.context.Ack(tuple);
                            }
                            else
                            {
                                //No ack enabled? Fire and forget.
                                this.context.Emit(Constants.DEFAULT_STREAM_ID, emitValues);
                            }
                        }
                        else
                        {
                            //Since this is a simple example, do nothing.
                            //In a real solution, you'd have to figure out what to do
                            //when receiving an END before a START.
                        }
                    }
                }
            }
        }