public void When_I_Scan_with_a_SingleColumnValueFilter_and_a_SubstringComparator_with_the_operator_equal_I_get_the_expected_results()
{
// grab a substring that is guaranteed to match at least one record.
string ss = _allExpectedRecords.First().A.Substring(1, 2);
//Debug.WriteLine("The substring value is: " + ss);
List <FilterTestRecord> expectedRecords = (from r in _allExpectedRecords where r.A.Contains(ss) select r).ToList();
var client = new HBaseClient(_credentials);
var scanner = new Scanner();
var comparer = new SubstringComparator(ss);
var filter = new SingleColumnValueFilter(
Encoding.UTF8.GetBytes(ColumnFamilyName1),
Encoding.UTF8.GetBytes(ColumnNameA),
CompareFilter.CompareOp.Equal,
comparer);
scanner.filter = filter.ToEncodedString();
ScannerInformation scanInfo = client.CreateScanner(_tableName, scanner);
List <FilterTestRecord> actualRecords = RetrieveResults(scanInfo).ToList();
actualRecords.ShouldContainOnly(expectedRecords);
}
public void When_I_Scan_with_a_FilterList_with_OR_logic_I_get_the_expected_results()
{
List <FilterTestRecord> expectedRecords = (from r in _allExpectedRecords where r.LineNumber <= 2 select r).ToList();
var client = new HBaseClient(_credentials);
var scanner = new Scanner();
Filter f0 = new SingleColumnValueFilter(
Encoding.UTF8.GetBytes(ColumnFamilyName1),
Encoding.UTF8.GetBytes(LineNumberColumnName),
CompareFilter.CompareOp.Equal,
BitConverter.GetBytes(1));
Filter f1 = new SingleColumnValueFilter(
Encoding.UTF8.GetBytes(ColumnFamilyName1),
Encoding.UTF8.GetBytes(LineNumberColumnName),
CompareFilter.CompareOp.LessThanOrEqualTo,
BitConverter.GetBytes(2));
var filter = new FilterList(FilterList.Operator.MustPassOne, f0, f1);
scanner.filter = filter.ToEncodedString();
ScannerInformation scanInfo = client.CreateScanner(_tableName, scanner);
List <FilterTestRecord> actualRecords = RetrieveResults(scanInfo).ToList();
actualRecords.ShouldContainOnly(expectedRecords);
}
public void SingleColumnValueFilter_Validates_OK(Comparator comparator, ComparisonOperator comparison, bool latest, bool filterIfMissing)
{
var value = "test";
var filter = new SingleColumnValueFilter <MockDomainObject>(_map);
filter.ComparisonOperator(comparison);
filter.Comparator(comparator);
filter.Column(x => x.Name);
filter.ColumnValue(value);
filter.FilterIfColumnMissing(filterIfMissing);
filter.LatestVersion(latest);
var validation = filter.ToFilterBuild();
var isSpecialCase = filter.ValidateOperationForComparator(comparator, comparison);
if (!isSpecialCase)
{
Assert.Throws <FilterException>(() => validation.Validate());
}
else
{
validation.Validate();
}
}
public void When_I_Scan_with_a_SingleColumnValueFilter_and_a_BinaryComparator_with_the_operator_not_equal_I_get_the_expected_results()
{
var expectedRecords = (from r in _allExpectedRecords where r.LineNumber != 1 select r).ToList();
var client = new HBaseClient(RequestOptionsFactory.GetDefaultOptions());
var scanner = new Scanner();
var filter = new SingleColumnValueFilter(
Encoding.UTF8.GetBytes(ColumnFamilyName1),
Encoding.UTF8.GetBytes(LineNumberColumnName),
CompareFilter.CompareOp.NotEqual,
BitConverter.GetBytes(1));
scanner.Filter = filter.ToEncodedString();
ScannerInformation scanInfo = null;
try
{
scanInfo = client.CreateScannerAsync(_tableName, scanner).Result;
var actualRecords = RetrieveResults(scanInfo).ToList();
actualRecords.ShouldContainOnly(expectedRecords);
}
finally
{
if (scanInfo != null)
{
client.DeleteScannerAsync(_tableName, scanInfo).Wait();
}
}
}
public IFilter ToFilter()
{
SingleColumnValueFilter singleColumnValueFilter = new SingleColumnValueFilter(ColumnName, Operator, ColumnValue)
{
LatestVersionsOnly = LatestVersionsOnly,
PassIfMissing = PassIfMissing
};
return(singleColumnValueFilter);
}
public void SingleColumnValueFilter_Column_Enumerable_Validation_Ok()
{
var filter = new SingleColumnValueFilter <MockDomainObject>(_map);
filter.Column(x => x.RandomList, "bla");
filter.ColumnValue("test");
var validation = filter.ToFilterBuild();
validation.Validate();
}
public void SingleColumnValueFilter_NoValue_Validation_Throws_FilterException()
{
var filter = new SingleColumnValueFilter <MockDomainObject>(_map);
filter.Column(x => x.Name);
var validation = filter.ToFilterBuild();
Assert.Throws <FilterException>(() => validation.Validate());
}
public void When_I_Scan_with_a_SingleColumnValueFilter_and_a_BinaryComparator_with_the_operator_not_equal_I_get_the_expected_results()
{
List <FilterTestRecord> expectedRecords = (from r in _allExpectedRecords where r.LineNumber != 1 select r).ToList();
var client = new HBaseClient(_credentials);
var scanner = new Scanner();
var filter = new SingleColumnValueFilter(
Encoding.UTF8.GetBytes(ColumnFamilyName1),
Encoding.UTF8.GetBytes(LineNumberColumnName),
CompareFilter.CompareOp.NotEqual,
BitConverter.GetBytes(1));
scanner.filter = filter.ToEncodedString();
ScannerInformation scanInfo = client.CreateScanner(_tableName, scanner);
List <FilterTestRecord> actualRecords = RetrieveResults(scanInfo).ToList();
actualRecords.ShouldContainOnly(expectedRecords);
}
public void When_I_Scan_with_a_FilterList_with_OR_logic_I_get_the_expected_results()
{
var expectedRecords = (from r in _allExpectedRecords where r.LineNumber <= 2 select r).ToList();
var client = new HBaseClient(RequestOptionsFactory.GetDefaultOptions());
var scanner = new Scanner();
Filter f0 = new SingleColumnValueFilter(
Encoding.UTF8.GetBytes(ColumnFamilyName1),
Encoding.UTF8.GetBytes(LineNumberColumnName),
CompareFilter.CompareOp.Equal,
BitConverter.GetBytes(1));
Filter f1 = new SingleColumnValueFilter(
Encoding.UTF8.GetBytes(ColumnFamilyName1),
Encoding.UTF8.GetBytes(LineNumberColumnName),
CompareFilter.CompareOp.LessThanOrEqualTo,
BitConverter.GetBytes(2));
var filter = new FilterList(FilterList.Operator.MustPassOne, f0, f1);
scanner.Filter = filter.ToEncodedString();
ScannerInformation scanInfo = null;
try
{
scanInfo = client.CreateScannerAsync(_tableName, scanner).Result;
var actualRecords = RetrieveResults(scanInfo).ToList();
actualRecords.ShouldContainOnly(expectedRecords);
}
finally
{
if (scanInfo != null)
{
client.DeleteScannerAsync(_tableName, scanInfo).Wait();
}
}
}
public void SingleColumnValueFilter_Builds_OK([CombinatorialValues(1.0f, 2.0d, 3L, 4, "test")] object value, Comparator comparator, ComparisonOperator comparison, bool latest, bool filterIfMissing)
{
var expected = $"SingleColumnValueFilter ('{Maps.MockDomainObject.TestColumnFamily}','{Maps.MockDomainObject.NameColumn}',{comparison.GetComparisonOperatorString()}," +
$"'{comparator.GetComparatorString()}{value}',{filterIfMissing},{latest})";
var filter = new SingleColumnValueFilter <MockDomainObject>(_map);
filter.ComparisonOperator(comparison);
filter.Comparator(comparator);
filter.Column(x => x.Name);
filter.ColumnValue(value);
filter.FilterIfColumnMissing(filterIfMissing);
filter.LatestVersion(latest);
var validation = filter.ToFilterBuild();
var build = validation.Build();
Assert.Equal(expected, build);
}
public void SingleColumnValueFilter_NoCF_Validation_Throws_FilterException()
{
var map = new ClassMap()
{
Maps = new List <IMap>()
{
new PropertyMap()
{
ColumnName = Maps.MockDomainObject.NameColumn,
Name = nameof(MockDomainObject.Name),
Type = typeof(MockDomainObject)
}
}
};
var filter = new SingleColumnValueFilter <MockDomainObject>(map);
filter.Column(x => x.Name);
var validation = filter.ToFilterBuild();
Assert.Throws <FilterException>(() => validation.Validate());
}
public void When_I_Scan_with_a_SingleColumnValueFilter_and_a_SubstringComparator_with_the_operator_equal_I_get_the_expected_results()
{
// grab a substring that is guaranteed to match at least one record.
var ss = _allExpectedRecords.First().A.Substring(1, 2);
//Debug.WriteLine("The substring value is: " + ss);
var expectedRecords = (from r in _allExpectedRecords where r.A.Contains(ss) select r).ToList();
var client = new HBaseClient(RequestOptionsFactory.GetDefaultOptions());
var scanner = new Scanner();
var comparer = new SubstringComparator(ss);
var filter = new SingleColumnValueFilter(
Encoding.UTF8.GetBytes(ColumnFamilyName1),
Encoding.UTF8.GetBytes(ColumnNameA),
CompareFilter.CompareOp.Equal,
comparer);
scanner.Filter = filter.ToEncodedString();
ScannerInformation scanInfo = null;
try
{
scanInfo = client.CreateScannerAsync(_tableName, scanner).Result;
var actualRecords = RetrieveResults(scanInfo).ToList();
actualRecords.ShouldContainOnly(expectedRecords);
}
finally
{
if (scanInfo != null)
{
client.DeleteScannerAsync(_tableName, scanInfo).Wait();
}
}
}
/// <summary>
/// Retrieve sessions within the given time range
/// </summary>
/// <param name="hbaseClient">The hbase client</param>
/// <param name="eventType">The type of event to look for</param>
/// <param name="start">Lower bound of the time range</param>
/// <param name="end">Upper bound of the time range</param>
static void GetSessionsByTime(HBaseClient hbaseClient, string eventType, DateTime start, DateTime end)
{
//Create filters list
FilterList filters = new FilterList(FilterList.Operator.MustPassAll);
//Filter to search for the event type value
SingleColumnValueFilter valueFilter = new SingleColumnValueFilter(
Encoding.UTF8.GetBytes("cf"),
Encoding.UTF8.GetBytes("event"),
CompareFilter.CompareOp.Equal,
Encoding.UTF8.GetBytes(eventType));
filters.AddFilter(valueFilter);
//Create scanner, set maxVersions so we can get previous versions of rows
//Since START events may not be the currently returned value
var scannerSettings = new Scanner()
{
filter = filters.ToEncodedString(),
maxVersions = 5,
startTime = ToUnixTime(start),
endTime = ToUnixTime(end)
};
var scanner = hbaseClient.CreateScanner(Properties.Settings.Default.HBaseTableName, scannerSettings);
//Read data from scanner
CellSet readSet = null;
//While reading cell sets
while ((readSet = hbaseClient.ScannerGetNext(scanner)) != null)
{
//Iterate over the rows returned
foreach (var row in readSet.rows)
{
//Get the time stored for the START event
var endTime = row.values.Where(v => Encoding.UTF8.GetString(v.column) == "cf:time")
.Select(v => BitConverter.ToInt64(v.data, 0)).ToArray()[0];
//Get the hbase timestamp of the row
var timestamp = row.values.Select(v => v.timestamp).ToArray()[0];
//If it's an end event type
if (eventType == "END")
{
//Get the duration stored between END and START events
var duration = new TimeSpan(
row.values.Where(v => Encoding.UTF8.GetString(v.column) == "cf:duration")
.Select(v => BitConverter.ToInt64(v.data, 0)).ToArray()[0]);
//Write out the session info, including duration
Console.WriteLine("Session {0} lasted {1} minutes, and ended at {2}",
Encoding.UTF8.GetString(row.key),
duration.Minutes,
FromUnixTime(endTime));
}
else
{
//If start event type, just write out when it started and the hbase timestamp for the row
Console.WriteLine("Session {0} started at {1}. Timestamp = {2}",
Encoding.UTF8.GetString(row.key),
FromUnixTime(endTime),
timestamp);
}
}
}
}
public void SingleColumnValueFilter_NullMap_Validation_Throws_Exception()
{
var filter = new SingleColumnValueFilter <MockDomainObject>(null);
Assert.Throws <MappingException>(() => filter.Column(x => x.Name));
}
/// <summary>
/// Executes incoming tuples
/// </summary>
/// <param name="tuple">The first field is treated as rowkey and rest as column names</param>
public void Execute(SCPTuple tuple)
{
//get the tuple info
string sessionId = tuple.GetString(0);
string sessionEvent = tuple.GetString(1);
long sessionEventTime = tuple.GetLong(2);
//If it's a start event, assume there's nothing to find so just re-emit
//NOTE: If messages may arrive out of order, you would need to add logic to
//query HBase to see if the end event has previously arrived,
//calculate the duration, etc.
if (sessionEvent == "START")
{
//Just re-emit the incoming data, plus 0 for duration, since we declare we send a 0 duration
//since we don't know the END event yet.
Values emitValues = new Values(tuple.GetValue(0), tuple.GetValue(1), tuple.GetValue(2), 0L);
//Is ack enabled?
if (enableAck)
{
//Emit the values, anchored to the incoming tuple
this.context.Emit(Constants.DEFAULT_STREAM_ID, new List <SCPTuple>()
{
tuple
}, emitValues);
//Ack the incoming tuple
this.context.Ack(tuple);
}
else
{
//No ack enabled? Fire and forget.
this.context.Emit(Constants.DEFAULT_STREAM_ID, emitValues);
}
}
if (sessionEvent == "END")
{
//Use filters
FilterList filters = new FilterList(FilterList.Operator.MustPassAll);
//Filter on the row by sessionID
RowFilter rowFilter = new RowFilter(CompareFilter.CompareOp.Equal, new BinaryComparator(TypeHelper.ToBytes(sessionId)));
filters.AddFilter(rowFilter);
//Filter on the event column for the START event
SingleColumnValueFilter valueFilter = new SingleColumnValueFilter(
Encoding.UTF8.GetBytes("cf"),
Encoding.UTF8.GetBytes("event"),
CompareFilter.CompareOp.Equal,
Encoding.UTF8.GetBytes("START"));
filters.AddFilter(valueFilter);
//Create scanner settings using the filters
var scannerSettings = new Scanner()
{
filter = filters.ToEncodedString()
};
//Get the scanner
var scanner = HBaseClusterClient.CreateScanner(HBaseTableName, scannerSettings);
CellSet readSet = null;
while ((readSet = HBaseClusterClient.ScannerGetNext(scanner)) != null)
{
//In theory we should only find one row
foreach (var row in readSet.rows)
{
//Pull back just the event column
var rowState = row.values.Where(v => Encoding.UTF8.GetString(v.column) == "cf:event")
.Select(v => Encoding.UTF8.GetString(v.data)).ToArray()[0];
//Is it a START event as expected?
if (rowState == "START")
{
//Get the start time
var startTime = TypeHelper.FromUnixTime(
row.values.Where(v => Encoding.UTF8.GetString(v.column) == "cf:time")
.Select(v => BitConverter.ToInt64(v.data, 0)).ToArray()[0]);
//Get the difference between start and end
DateTime endTime = TypeHelper.FromUnixTime(sessionEventTime);
TimeSpan duration = endTime.Subtract(startTime);
//Emit the tuple, with the duration between start/end.
Values emitValues = new Values(sessionId, sessionEvent, sessionEventTime, duration.Ticks);
//If ack is enabled
if (enableAck)
{
//Emit the values, anchored to the incoming tuple
this.context.Emit(Constants.DEFAULT_STREAM_ID, new List <SCPTuple>()
{
tuple
}, emitValues);
//Ack the incoming tuple
this.context.Ack(tuple);
}
else
{
//No ack enabled? Fire and forget.
this.context.Emit(Constants.DEFAULT_STREAM_ID, emitValues);
}
}
else
{
//Since this is a simple example, do nothing.
//In a real solution, you'd have to figure out what to do
//when receiving an END before a START.
}
}
}
}
}
