public void When_I_Scan_with_a_SingleColumnValueFilter_and_a_SubstringComparator_with_the_operator_equal_I_get_the_expected_results() { // grab a substring that is guaranteed to match at least one record. string ss = _allExpectedRecords.First().A.Substring(1, 2); //Debug.WriteLine("The substring value is: " + ss); List <FilterTestRecord> expectedRecords = (from r in _allExpectedRecords where r.A.Contains(ss) select r).ToList(); var client = new HBaseClient(_credentials); var scanner = new Scanner(); var comparer = new SubstringComparator(ss); var filter = new SingleColumnValueFilter( Encoding.UTF8.GetBytes(ColumnFamilyName1), Encoding.UTF8.GetBytes(ColumnNameA), CompareFilter.CompareOp.Equal, comparer); scanner.filter = filter.ToEncodedString(); ScannerInformation scanInfo = client.CreateScanner(_tableName, scanner); List <FilterTestRecord> actualRecords = RetrieveResults(scanInfo).ToList(); actualRecords.ShouldContainOnly(expectedRecords); }
public void When_I_Scan_with_a_FilterList_with_OR_logic_I_get_the_expected_results() { List <FilterTestRecord> expectedRecords = (from r in _allExpectedRecords where r.LineNumber <= 2 select r).ToList(); var client = new HBaseClient(_credentials); var scanner = new Scanner(); Filter f0 = new SingleColumnValueFilter( Encoding.UTF8.GetBytes(ColumnFamilyName1), Encoding.UTF8.GetBytes(LineNumberColumnName), CompareFilter.CompareOp.Equal, BitConverter.GetBytes(1)); Filter f1 = new SingleColumnValueFilter( Encoding.UTF8.GetBytes(ColumnFamilyName1), Encoding.UTF8.GetBytes(LineNumberColumnName), CompareFilter.CompareOp.LessThanOrEqualTo, BitConverter.GetBytes(2)); var filter = new FilterList(FilterList.Operator.MustPassOne, f0, f1); scanner.filter = filter.ToEncodedString(); ScannerInformation scanInfo = client.CreateScanner(_tableName, scanner); List <FilterTestRecord> actualRecords = RetrieveResults(scanInfo).ToList(); actualRecords.ShouldContainOnly(expectedRecords); }
public void SingleColumnValueFilter_Validates_OK(Comparator comparator, ComparisonOperator comparison, bool latest, bool filterIfMissing) { var value = "test"; var filter = new SingleColumnValueFilter <MockDomainObject>(_map); filter.ComparisonOperator(comparison); filter.Comparator(comparator); filter.Column(x => x.Name); filter.ColumnValue(value); filter.FilterIfColumnMissing(filterIfMissing); filter.LatestVersion(latest); var validation = filter.ToFilterBuild(); var isSpecialCase = filter.ValidateOperationForComparator(comparator, comparison); if (!isSpecialCase) { Assert.Throws <FilterException>(() => validation.Validate()); } else { validation.Validate(); } }
public void When_I_Scan_with_a_SingleColumnValueFilter_and_a_BinaryComparator_with_the_operator_not_equal_I_get_the_expected_results() { var expectedRecords = (from r in _allExpectedRecords where r.LineNumber != 1 select r).ToList(); var client = new HBaseClient(RequestOptionsFactory.GetDefaultOptions()); var scanner = new Scanner(); var filter = new SingleColumnValueFilter( Encoding.UTF8.GetBytes(ColumnFamilyName1), Encoding.UTF8.GetBytes(LineNumberColumnName), CompareFilter.CompareOp.NotEqual, BitConverter.GetBytes(1)); scanner.Filter = filter.ToEncodedString(); ScannerInformation scanInfo = null; try { scanInfo = client.CreateScannerAsync(_tableName, scanner).Result; var actualRecords = RetrieveResults(scanInfo).ToList(); actualRecords.ShouldContainOnly(expectedRecords); } finally { if (scanInfo != null) { client.DeleteScannerAsync(_tableName, scanInfo).Wait(); } } }
public IFilter ToFilter() { SingleColumnValueFilter singleColumnValueFilter = new SingleColumnValueFilter(ColumnName, Operator, ColumnValue) { LatestVersionsOnly = LatestVersionsOnly, PassIfMissing = PassIfMissing }; return(singleColumnValueFilter); }
public void SingleColumnValueFilter_Column_Enumerable_Validation_Ok() { var filter = new SingleColumnValueFilter <MockDomainObject>(_map); filter.Column(x => x.RandomList, "bla"); filter.ColumnValue("test"); var validation = filter.ToFilterBuild(); validation.Validate(); }
public void SingleColumnValueFilter_NoValue_Validation_Throws_FilterException() { var filter = new SingleColumnValueFilter <MockDomainObject>(_map); filter.Column(x => x.Name); var validation = filter.ToFilterBuild(); Assert.Throws <FilterException>(() => validation.Validate()); }
public void When_I_Scan_with_a_SingleColumnValueFilter_and_a_BinaryComparator_with_the_operator_not_equal_I_get_the_expected_results() { List <FilterTestRecord> expectedRecords = (from r in _allExpectedRecords where r.LineNumber != 1 select r).ToList(); var client = new HBaseClient(_credentials); var scanner = new Scanner(); var filter = new SingleColumnValueFilter( Encoding.UTF8.GetBytes(ColumnFamilyName1), Encoding.UTF8.GetBytes(LineNumberColumnName), CompareFilter.CompareOp.NotEqual, BitConverter.GetBytes(1)); scanner.filter = filter.ToEncodedString(); ScannerInformation scanInfo = client.CreateScanner(_tableName, scanner); List <FilterTestRecord> actualRecords = RetrieveResults(scanInfo).ToList(); actualRecords.ShouldContainOnly(expectedRecords); }
public void When_I_Scan_with_a_FilterList_with_OR_logic_I_get_the_expected_results() { var expectedRecords = (from r in _allExpectedRecords where r.LineNumber <= 2 select r).ToList(); var client = new HBaseClient(RequestOptionsFactory.GetDefaultOptions()); var scanner = new Scanner(); Filter f0 = new SingleColumnValueFilter( Encoding.UTF8.GetBytes(ColumnFamilyName1), Encoding.UTF8.GetBytes(LineNumberColumnName), CompareFilter.CompareOp.Equal, BitConverter.GetBytes(1)); Filter f1 = new SingleColumnValueFilter( Encoding.UTF8.GetBytes(ColumnFamilyName1), Encoding.UTF8.GetBytes(LineNumberColumnName), CompareFilter.CompareOp.LessThanOrEqualTo, BitConverter.GetBytes(2)); var filter = new FilterList(FilterList.Operator.MustPassOne, f0, f1); scanner.Filter = filter.ToEncodedString(); ScannerInformation scanInfo = null; try { scanInfo = client.CreateScannerAsync(_tableName, scanner).Result; var actualRecords = RetrieveResults(scanInfo).ToList(); actualRecords.ShouldContainOnly(expectedRecords); } finally { if (scanInfo != null) { client.DeleteScannerAsync(_tableName, scanInfo).Wait(); } } }
public void SingleColumnValueFilter_Builds_OK([CombinatorialValues(1.0f, 2.0d, 3L, 4, "test")] object value, Comparator comparator, ComparisonOperator comparison, bool latest, bool filterIfMissing) { var expected = $"SingleColumnValueFilter ('{Maps.MockDomainObject.TestColumnFamily}','{Maps.MockDomainObject.NameColumn}',{comparison.GetComparisonOperatorString()}," + $"'{comparator.GetComparatorString()}{value}',{filterIfMissing},{latest})"; var filter = new SingleColumnValueFilter <MockDomainObject>(_map); filter.ComparisonOperator(comparison); filter.Comparator(comparator); filter.Column(x => x.Name); filter.ColumnValue(value); filter.FilterIfColumnMissing(filterIfMissing); filter.LatestVersion(latest); var validation = filter.ToFilterBuild(); var build = validation.Build(); Assert.Equal(expected, build); }
public void SingleColumnValueFilter_NoCF_Validation_Throws_FilterException() { var map = new ClassMap() { Maps = new List <IMap>() { new PropertyMap() { ColumnName = Maps.MockDomainObject.NameColumn, Name = nameof(MockDomainObject.Name), Type = typeof(MockDomainObject) } } }; var filter = new SingleColumnValueFilter <MockDomainObject>(map); filter.Column(x => x.Name); var validation = filter.ToFilterBuild(); Assert.Throws <FilterException>(() => validation.Validate()); }
public void When_I_Scan_with_a_SingleColumnValueFilter_and_a_SubstringComparator_with_the_operator_equal_I_get_the_expected_results() { // grab a substring that is guaranteed to match at least one record. var ss = _allExpectedRecords.First().A.Substring(1, 2); //Debug.WriteLine("The substring value is: " + ss); var expectedRecords = (from r in _allExpectedRecords where r.A.Contains(ss) select r).ToList(); var client = new HBaseClient(RequestOptionsFactory.GetDefaultOptions()); var scanner = new Scanner(); var comparer = new SubstringComparator(ss); var filter = new SingleColumnValueFilter( Encoding.UTF8.GetBytes(ColumnFamilyName1), Encoding.UTF8.GetBytes(ColumnNameA), CompareFilter.CompareOp.Equal, comparer); scanner.Filter = filter.ToEncodedString(); ScannerInformation scanInfo = null; try { scanInfo = client.CreateScannerAsync(_tableName, scanner).Result; var actualRecords = RetrieveResults(scanInfo).ToList(); actualRecords.ShouldContainOnly(expectedRecords); } finally { if (scanInfo != null) { client.DeleteScannerAsync(_tableName, scanInfo).Wait(); } } }
/// <summary> /// Retrieve sessions within the given time range /// </summary> /// <param name="hbaseClient">The hbase client</param> /// <param name="eventType">The type of event to look for</param> /// <param name="start">Lower bound of the time range</param> /// <param name="end">Upper bound of the time range</param> static void GetSessionsByTime(HBaseClient hbaseClient, string eventType, DateTime start, DateTime end) { //Create filters list FilterList filters = new FilterList(FilterList.Operator.MustPassAll); //Filter to search for the event type value SingleColumnValueFilter valueFilter = new SingleColumnValueFilter( Encoding.UTF8.GetBytes("cf"), Encoding.UTF8.GetBytes("event"), CompareFilter.CompareOp.Equal, Encoding.UTF8.GetBytes(eventType)); filters.AddFilter(valueFilter); //Create scanner, set maxVersions so we can get previous versions of rows //Since START events may not be the currently returned value var scannerSettings = new Scanner() { filter = filters.ToEncodedString(), maxVersions = 5, startTime = ToUnixTime(start), endTime = ToUnixTime(end) }; var scanner = hbaseClient.CreateScanner(Properties.Settings.Default.HBaseTableName, scannerSettings); //Read data from scanner CellSet readSet = null; //While reading cell sets while ((readSet = hbaseClient.ScannerGetNext(scanner)) != null) { //Iterate over the rows returned foreach (var row in readSet.rows) { //Get the time stored for the START event var endTime = row.values.Where(v => Encoding.UTF8.GetString(v.column) == "cf:time") .Select(v => BitConverter.ToInt64(v.data, 0)).ToArray()[0]; //Get the hbase timestamp of the row var timestamp = row.values.Select(v => v.timestamp).ToArray()[0]; //If it's an end event type if (eventType == "END") { //Get the duration stored between END and START events var duration = new TimeSpan( row.values.Where(v => Encoding.UTF8.GetString(v.column) == "cf:duration") .Select(v => BitConverter.ToInt64(v.data, 0)).ToArray()[0]); //Write out the session info, including duration Console.WriteLine("Session {0} lasted {1} minutes, and ended at {2}", Encoding.UTF8.GetString(row.key), duration.Minutes, FromUnixTime(endTime)); } else { //If start event type, just write out when it started and the hbase timestamp for the row Console.WriteLine("Session {0} started at {1}. Timestamp = {2}", Encoding.UTF8.GetString(row.key), FromUnixTime(endTime), timestamp); } } } }
public void SingleColumnValueFilter_NullMap_Validation_Throws_Exception() { var filter = new SingleColumnValueFilter <MockDomainObject>(null); Assert.Throws <MappingException>(() => filter.Column(x => x.Name)); }
/// <summary> /// Executes incoming tuples /// </summary> /// <param name="tuple">The first field is treated as rowkey and rest as column names</param> public void Execute(SCPTuple tuple) { //get the tuple info string sessionId = tuple.GetString(0); string sessionEvent = tuple.GetString(1); long sessionEventTime = tuple.GetLong(2); //If it's a start event, assume there's nothing to find so just re-emit //NOTE: If messages may arrive out of order, you would need to add logic to //query HBase to see if the end event has previously arrived, //calculate the duration, etc. if (sessionEvent == "START") { //Just re-emit the incoming data, plus 0 for duration, since we declare we send a 0 duration //since we don't know the END event yet. Values emitValues = new Values(tuple.GetValue(0), tuple.GetValue(1), tuple.GetValue(2), 0L); //Is ack enabled? if (enableAck) { //Emit the values, anchored to the incoming tuple this.context.Emit(Constants.DEFAULT_STREAM_ID, new List <SCPTuple>() { tuple }, emitValues); //Ack the incoming tuple this.context.Ack(tuple); } else { //No ack enabled? Fire and forget. this.context.Emit(Constants.DEFAULT_STREAM_ID, emitValues); } } if (sessionEvent == "END") { //Use filters FilterList filters = new FilterList(FilterList.Operator.MustPassAll); //Filter on the row by sessionID RowFilter rowFilter = new RowFilter(CompareFilter.CompareOp.Equal, new BinaryComparator(TypeHelper.ToBytes(sessionId))); filters.AddFilter(rowFilter); //Filter on the event column for the START event SingleColumnValueFilter valueFilter = new SingleColumnValueFilter( Encoding.UTF8.GetBytes("cf"), Encoding.UTF8.GetBytes("event"), CompareFilter.CompareOp.Equal, Encoding.UTF8.GetBytes("START")); filters.AddFilter(valueFilter); //Create scanner settings using the filters var scannerSettings = new Scanner() { filter = filters.ToEncodedString() }; //Get the scanner var scanner = HBaseClusterClient.CreateScanner(HBaseTableName, scannerSettings); CellSet readSet = null; while ((readSet = HBaseClusterClient.ScannerGetNext(scanner)) != null) { //In theory we should only find one row foreach (var row in readSet.rows) { //Pull back just the event column var rowState = row.values.Where(v => Encoding.UTF8.GetString(v.column) == "cf:event") .Select(v => Encoding.UTF8.GetString(v.data)).ToArray()[0]; //Is it a START event as expected? if (rowState == "START") { //Get the start time var startTime = TypeHelper.FromUnixTime( row.values.Where(v => Encoding.UTF8.GetString(v.column) == "cf:time") .Select(v => BitConverter.ToInt64(v.data, 0)).ToArray()[0]); //Get the difference between start and end DateTime endTime = TypeHelper.FromUnixTime(sessionEventTime); TimeSpan duration = endTime.Subtract(startTime); //Emit the tuple, with the duration between start/end. Values emitValues = new Values(sessionId, sessionEvent, sessionEventTime, duration.Ticks); //If ack is enabled if (enableAck) { //Emit the values, anchored to the incoming tuple this.context.Emit(Constants.DEFAULT_STREAM_ID, new List <SCPTuple>() { tuple }, emitValues); //Ack the incoming tuple this.context.Ack(tuple); } else { //No ack enabled? Fire and forget. this.context.Emit(Constants.DEFAULT_STREAM_ID, emitValues); } } else { //Since this is a simple example, do nothing. //In a real solution, you'd have to figure out what to do //when receiving an END before a START. } } } } }