public void TestMyProcessorWithCollection()
{
//Schema: "a:int, b:int"
USqlSchema schema = new USqlSchema(
new USqlColumn <int>("a"),
new USqlColumn <int>("b")
);
IUpdatableRow output = new USqlRow(schema, null).AsUpdatable();
//Generate Rowset with specified values
List <object[]> values = new List <object[]> {
new object[2] {
2, 3
},
new object[2] {
10, 20
}
};
IEnumerable <IRow> rows = UnitTestHelper.CreateRowsFromValues(schema, values);
IRowset rowset = UnitTestHelper.GetRowsetFromCollection(rows, output.AsReadOnly());
//Create UDO instance
MyProcessor processor = new MyProcessor(floor: 1, enforce: true);
foreach (IRow r in rowset.Rows)
{
IRow after = processor.Process(r, output);
//Verify result
Assert.IsTrue(after.Get <int>(0) == 2);
Assert.IsTrue(after.Get <int>(1) == 4);
break;
}
}
public void TestMyProcessor()
{
// Define the schema for processor input rowset
// Schema: "a:int, b:int"
//
USqlColumn <int> col1 = new USqlColumn <int>("col1");
USqlColumn <int> col2 = new USqlColumn <int>("col2");
List <IColumn> columns = new List <IColumn> {
col1, col2
};
USqlSchema schema = new USqlSchema(columns);
// Generate one row with specified column values as input rowset
//
object[] values = new object[2] {
0, 0
};
IRow input = new USqlRow(schema, values);
IUpdatableRow output = input.AsUpdatable();
// Create processor instance for testing and run the processor with fake input
//
MyProcessor processor = new MyProcessor();
IRow newOutput = processor.Process(input, output);
//Verify results for processor output
//
Assert.IsTrue(newOutput.Schema.Count == 2);
Assert.IsTrue(newOutput.Get <int>(0) == 1);
Assert.IsTrue(newOutput.Get <int>(1) == 5);
}
public void TestMyProcessor()
{
//Schema: "a:int, b:int"
USqlColumn <int> col1 = new USqlColumn <int>("a");
USqlColumn <int> col2 = new USqlColumn <int>("b");
List <IColumn> columns = new List <IColumn> {
col1, col2
};
USqlSchema schema = new USqlSchema(columns);
//Generate one row with specified column values
object[] values = new object[2] {
2, 3
};
IRow input = new USqlRow(schema, values);
IUpdatableRow output = input.AsUpdatable();
//Create UDO instance
MyProcessor processor = new MyProcessor(floor: 4);
IRow newOutput = processor.Process(input, output);
//Verify results
Assert.IsTrue(newOutput.Schema.Count == 2);
Assert.IsTrue(newOutput.Get <int>(0) == 2);
Assert.IsTrue(newOutput.Get <int>(1) == 4);
}
public void TestF()
{
var columnA = new USqlColumn <string>("A");
var columnB = new USqlColumn <string>("B");
var columns = new List <IColumn> {
columnA, columnB
};
var schema = new USqlSchema(columns);
var row1 = new USqlRow(schema, new object[] { "1111", new SqlArray <string> {
} });
var row2 = new USqlRow(schema, new object[] { "1111", null });
var row3 = new USqlRow(schema, new object[] { "2222", new SqlArray <string>(new[] { "a" }) });
using (var f = File.Create(@"C:\code\notebooks\data\avro\tmp-avrooutputter\avrooutputtertest1.avro"))
{
var sw = new USqlStreamWriter(f);
var outputter = new AvroOutputter(@"{""type"":""record"",""name"":""Microsoft.Streaming.Avro.GenericFromIRecord0"",""fields"":[{""name"":""A"",""type"":[""null"",""string""]},{""name"": ""B"", ""type"": [""null"", {""type"" : ""array"", ""items"" : ""string""}]}]}");
outputter.Output(row1, sw);
outputter.Output(row2, sw);
outputter.Output(row3, sw);
outputter.Close();
sw.Close();
}
}
private USqlUpdatableRow getUpdatableRow()
{
ISchema schema = new USqlSchema(getColumns());
IRow row = new USqlRow(schema, new object[schema.Count]);
USqlUpdatableRow updRow = new USqlUpdatableRow(row);
return updRow;
}
public void EmptyFile()
{
IExtractor extractor = new EventhubCaptureExtractor();
FileStream stream = File.Open(@"sampleData/empty.avro", FileMode.Open);
IUnstructuredReader input = new UnstructuredReaderTest(stream);
var body = new USqlColumn <byte>("Body");
var columns = new List <IColumn> {
body
};
var schema = new USqlSchema(columns);
var usqlrow = new USqlRow(schema, null);
var result = extractor.Extract(input, usqlrow.AsUpdatable());
Assert.AreEqual(0, result.Count());
}
public void TestMyReducer()
{
USqlColumn <string> col1 = new USqlColumn <string>("Query");
USqlColumn <string> col2 = new USqlColumn <string>("Market");
USqlColumn <int> col3 = new USqlColumn <int>("Latency");
List <IColumn> columns = new List <IColumn> {
col1, col2, col3
};
USqlSchema schema = new USqlSchema(columns);
IUpdatableRow output = new USqlRow(schema, null).AsUpdatable();
//Generate Rowset with specified values
List <object[]> values = new List <object[]> {
new object[3] {
"Query1", "Market1", 1
},
new object[3] {
"Query2", "Market2", 2
},
new object[3] {
"Query3", "Market3", 3
},
new object[3] {
"Query4", "Market4", 4
},
};
IEnumerable <IRow> rows = UnitTestHelper.CreateRowsFromValues(schema, values);
IRowset rowset = UnitTestHelper.GetRowsetFromCollection(rows, output.AsReadOnly());
//Create UDO instance. The reducer should get the largest latency
MyReducer reducer = new MyReducer();
foreach (IRow r in reducer.Reduce(rowset, output))
{
Assert.IsTrue(rowset.Schema.Count == 3);
}
//Make sure the reducer returns the row of the largest latency
Assert.IsTrue(output.Get <int>("Latency") == 4);
}
public void TestMyProcessorWithFile()
{
//Schema: "a:int, b:int"
USqlColumn <int> col1 = new USqlColumn <int>("a");
USqlColumn <int> col2 = new USqlColumn <int>("b");
List <IColumn> columns = new List <IColumn> {
col1, col2
};
USqlSchema schema = new USqlSchema(columns);
//Generate one row with default values
IUpdatableRow output = new USqlRow(schema, null).AsUpdatable();
//Get upstreams from file
IRowset rowset = UnitTestHelper.GetRowsetFromFile(@"processor.txt", schema, output.AsReadOnly(), discardAdditionalColumns: true, rowDelimiter: null, columnSeparator: '\t');
//Create UDO instance
MyProcessor processor = new MyProcessor(floor: 50, enforce: true);
foreach (IRow r in rowset.Rows)
{
processor.Process(r, output);
}
}
public void TestMyCombiner()
{
//Construct left columns
USqlColumn <int> col1 = new USqlColumn <int>("employee_id");
USqlColumn <string> col2 = new USqlColumn <string>("employee_name");
USqlColumn <string> col3 = new USqlColumn <string>("department_name");
//Construct left schema
List <IColumn> columns_left = new List <IColumn> {
col1, col3
};
USqlSchema schema_left = new USqlSchema(columns_left);
//Construct right schema
List <IColumn> columns_right = new List <IColumn> {
col1, col2
};
USqlSchema schema_right = new USqlSchema(columns_right);
//Construct result schema. expected schema: employee_id, employee_name, department_name
List <IColumn> columns_result = new List <IColumn> {
col1, col2, col3
};
USqlSchema schema_result = new USqlSchema(columns_result);
//Construct the output schema which will be needed later
IUpdatableRow output_left = new USqlRow(schema_left, null).AsUpdatable();
IUpdatableRow output_right = new USqlRow(schema_right, null).AsUpdatable();
IUpdatableRow output_result = new USqlRow(schema_result, null).AsUpdatable();
//Generate Rowset with specified values for left input
List <object[]> values_left = new List <object[]> {
new object[2] {
1, "HR"
},
new object[2] {
2, "R&D"
},
new object[2] {
4, "Operation"
},
};
//Generate Rowset with specified values for right input
List <object[]> values_right = new List <object[]> {
new object[2] {
1, "John"
},
new object[2] {
2, "Tom"
},
new object[2] {
3, "Melinda"
},
};
IEnumerable <IRow> rows_left = UnitTestHelper.CreateRowsFromValues(schema_left, values_left);
IEnumerable <IRow> rows_right = UnitTestHelper.CreateRowsFromValues(schema_right, values_right);
IRowset rowset_left = UnitTestHelper.GetRowsetFromCollection(rows_left, output_left.AsReadOnly());
IRowset rowset_right = UnitTestHelper.GetRowsetFromCollection(rows_right, output_right.AsReadOnly());
//Create UDO instance. The combiner will do something like inner join
MyCombiner combiner = new MyCombiner();
IEnumerable <IRow> after = combiner.Combine(rowset_left, rowset_right, output_result);
//Verify result
List <IRow> resultlist = after.ToList();
Assert.IsTrue(resultlist[0].Schema.Count == 3);
//Verify the values. expected to see:
//the first row: 1, "John", "HR"
//the second row: 2, "Tom", "R&D"
//here we only verify the first row
Assert.IsTrue(resultlist[0].Get <int>("employee_id") == 1);
Assert.IsTrue(resultlist[0].Get <string>(1) == "John");
Assert.IsTrue(resultlist[0].Get <string>(2) == "HR");
}