public void TestF()
{
var columnA = new USqlColumn <string>("A");
var columnB = new USqlColumn <string>("B");
var columns = new List <IColumn> {
columnA, columnB
};
var schema = new USqlSchema(columns);
var row1 = new USqlRow(schema, new object[] { "1111", new SqlArray <string> {
} });
var row2 = new USqlRow(schema, new object[] { "1111", null });
var row3 = new USqlRow(schema, new object[] { "2222", new SqlArray <string>(new[] { "a" }) });
using (var f = File.Create(@"C:\code\notebooks\data\avro\tmp-avrooutputter\avrooutputtertest1.avro"))
{
var sw = new USqlStreamWriter(f);
var outputter = new AvroOutputter(@"{""type"":""record"",""name"":""Microsoft.Streaming.Avro.GenericFromIRecord0"",""fields"":[{""name"":""A"",""type"":[""null"",""string""]},{""name"": ""B"", ""type"": [""null"", {""type"" : ""array"", ""items"" : ""string""}]}]}");
outputter.Output(row1, sw);
outputter.Output(row2, sw);
outputter.Output(row3, sw);
outputter.Close();
sw.Close();
}
}
public void TestMyProcessor()
{
// Define the schema for processor input rowset
// Schema: "a:int, b:int"
//
USqlColumn <int> col1 = new USqlColumn <int>("col1");
USqlColumn <int> col2 = new USqlColumn <int>("col2");
List <IColumn> columns = new List <IColumn> {
col1, col2
};
USqlSchema schema = new USqlSchema(columns);
// Generate one row with specified column values as input rowset
//
object[] values = new object[2] {
0, 0
};
IRow input = new USqlRow(schema, values);
IUpdatableRow output = input.AsUpdatable();
// Create processor instance for testing and run the processor with fake input
//
MyProcessor processor = new MyProcessor();
IRow newOutput = processor.Process(input, output);
//Verify results for processor output
//
Assert.IsTrue(newOutput.Schema.Count == 2);
Assert.IsTrue(newOutput.Get <int>(0) == 1);
Assert.IsTrue(newOutput.Get <int>(1) == 5);
}
public void TestMyProcessor()
{
//Schema: "a:int, b:int"
USqlColumn <int> col1 = new USqlColumn <int>("a");
USqlColumn <int> col2 = new USqlColumn <int>("b");
List <IColumn> columns = new List <IColumn> {
col1, col2
};
USqlSchema schema = new USqlSchema(columns);
//Generate one row with specified column values
object[] values = new object[2] {
2, 3
};
IRow input = new USqlRow(schema, values);
IUpdatableRow output = input.AsUpdatable();
//Create UDO instance
MyProcessor processor = new MyProcessor(floor: 4);
IRow newOutput = processor.Process(input, output);
//Verify results
Assert.IsTrue(newOutput.Schema.Count == 2);
Assert.IsTrue(newOutput.Get <int>(0) == 2);
Assert.IsTrue(newOutput.Get <int>(1) == 4);
}
//
// You can use the following additional attributes as you write your tests:
//
// Use ClassInitialize to run code before running the first test in the class
// [ClassInitialize()]
// public static void MyClassInitialize(TestContext testContext) { }
//
// Use ClassCleanup to run code after all tests in a class have run
// [ClassCleanup()]
// public static void MyClassCleanup() { }
//
// Use TestInitialize to run code before running each test
// [TestInitialize()]
// public void MyTestInitialize() { }
//
// Use TestCleanup to run code after each test has run
// [TestCleanup()]
// public void MyTestCleanup() { }
//
#endregion
public IRow SingleColumnRowGenerator <T>()
{
var foo = new USqlColumn <T>("Value");
var columns = new List <IColumn> {
foo
};
var schema = new USqlSchema(columns);
return(new USqlRow(schema, null));
}
public IRow DualColumnRowGenerator <T, T2>()
{
var foo = new USqlColumn <T>("Value");
var bar = new USqlColumn <T2>("Value2");
var columns = new List <IColumn> {
foo, bar
};
var schema = new USqlSchema(columns);
return(new USqlRow(schema, null));
}
public IRow TwoColumnRowGenerator <T>()
{
var col1 = new USqlColumn <T>("Value1");
var col2 = new USqlColumn <T>("Value2");
var columns = new List <IColumn> {
col1, col2
};
var schema = new USqlSchema(columns);
return(new USqlRow(schema, null));
}
public IRow RowGenerator()
{
//generate the schema
USqlColumn <string> col1 = new USqlColumn <string>("contexts");
USqlColumn <string> col2 = new USqlColumn <string>("contexts.data.author");
List <IColumn> columns = new List <IColumn> {
col1, col2
};
USqlSchema schema = new USqlSchema(columns);
return(new USqlRow(schema, null));
}
public IRow GetCommentRow()
{
USqlColumn<int> id = new USqlColumn<int>("Id");
USqlColumn<int> postId = new USqlColumn<int>("PostId");
USqlColumn<int> score = new USqlColumn<int>("Score");
USqlColumn<string> text = new USqlColumn<string>("Text");
USqlColumn<DateTime> creationDate = new USqlColumn<DateTime>("CreationDate");
USqlColumn<int> userId = new USqlColumn<int>("UserId");
List<IColumn> columns = new List<IColumn> { id, postId, score, text, creationDate, userId };
USqlSchema schema = new USqlSchema(columns);
return new USqlRow(schema, null);
}
//
// You can use the following additional attributes as you write your tests:
//
// Use ClassInitialize to run code before running the first test in the class
// [ClassInitialize()]
// public static void MyClassInitialize(TestContext testContext) { }
//
// Use ClassCleanup to run code after all tests in a class have run
// [ClassCleanup()]
// public static void MyClassCleanup() { }
//
// Use TestInitialize to run code before running each test
// [TestInitialize()]
// public void MyTestInitialize() { }
//
// Use TestCleanup to run code after each test has run
// [TestCleanup()]
// public void MyTestCleanup() { }
//
#endregion
public IRow RowGenerator()
{
//generate the schema
USqlColumn <string> col1 = new USqlColumn <string>("Market");
USqlColumn <string> col2 = new USqlColumn <string>("Query");
USqlColumn <int> col3 = new USqlColumn <int>("Latency");
List <IColumn> columns = new List <IColumn> {
col1, col2, col3
};
USqlSchema schema = new USqlSchema(columns);
return(new USqlRow(schema, null));
}
public IRow RowGenerator()
{
USqlColumn <string> col1 = new USqlColumn <string>("col1");
USqlColumn <int> col2 = new USqlColumn <int>("id");
USqlColumn <int> col3 = new USqlColumn <int>("rank");
List <IColumn> columns = new List <IColumn> {
col1, col2, col3
};
USqlSchema schema = new USqlSchema(columns);
object[] values = new object[3] {
"name", 2, 100
};
return(new USqlRow(schema, values));
}
public IRow GetCommentRow()
{
USqlColumn <int> id = new USqlColumn <int>("Id");
USqlColumn <int> postId = new USqlColumn <int>("PostId");
USqlColumn <int> score = new USqlColumn <int>("Score");
USqlColumn <string> text = new USqlColumn <string>("Text");
USqlColumn <DateTime> creationDate = new USqlColumn <DateTime>("CreationDate");
USqlColumn <int> userId = new USqlColumn <int>("UserId");
List <IColumn> columns = new List <IColumn> {
id, postId, score, text, creationDate, userId
};
USqlSchema schema = new USqlSchema(columns);
return(new USqlRow(schema, null));
}
public void EmptyFile()
{
IExtractor extractor = new EventhubCaptureExtractor();
FileStream stream = File.Open(@"sampleData/empty.avro", FileMode.Open);
IUnstructuredReader input = new UnstructuredReaderTest(stream);
var body = new USqlColumn <byte>("Body");
var columns = new List <IColumn> {
body
};
var schema = new USqlSchema(columns);
var usqlrow = new USqlRow(schema, null);
var result = extractor.Extract(input, usqlrow.AsUpdatable());
Assert.AreEqual(0, result.Count());
}
public void TestMyReducer()
{
USqlColumn <string> col1 = new USqlColumn <string>("Query");
USqlColumn <string> col2 = new USqlColumn <string>("Market");
USqlColumn <int> col3 = new USqlColumn <int>("Latency");
List <IColumn> columns = new List <IColumn> {
col1, col2, col3
};
USqlSchema schema = new USqlSchema(columns);
IUpdatableRow output = new USqlRow(schema, null).AsUpdatable();
//Generate Rowset with specified values
List <object[]> values = new List <object[]> {
new object[3] {
"Query1", "Market1", 1
},
new object[3] {
"Query2", "Market2", 2
},
new object[3] {
"Query3", "Market3", 3
},
new object[3] {
"Query4", "Market4", 4
},
};
IEnumerable <IRow> rows = UnitTestHelper.CreateRowsFromValues(schema, values);
IRowset rowset = UnitTestHelper.GetRowsetFromCollection(rows, output.AsReadOnly());
//Create UDO instance. The reducer should get the largest latency
MyReducer reducer = new MyReducer();
foreach (IRow r in reducer.Reduce(rowset, output))
{
Assert.IsTrue(rowset.Schema.Count == 3);
}
//Make sure the reducer returns the row of the largest latency
Assert.IsTrue(output.Get <int>("Latency") == 4);
}
public IRow GetUserRow()
{
USqlColumn <int> id = new USqlColumn <int>("Id");
USqlColumn <int> reputation = new USqlColumn <int>("Reputation");
USqlColumn <DateTime> creationDate = new USqlColumn <DateTime>("CreationDate");
USqlColumn <string> displayName = new USqlColumn <string>("DisplayName");
USqlColumn <DateTime> lastAccessDate = new USqlColumn <DateTime>("LastAccessDate");
USqlColumn <string> location = new USqlColumn <string>("Location");
USqlColumn <string> aboutMe = new USqlColumn <string>("AboutMe");
USqlColumn <int> upVotes = new USqlColumn <int>("UpVotes");
USqlColumn <int> downVotes = new USqlColumn <int>("DownVotes");
USqlColumn <int> age = new USqlColumn <int>("Age");
USqlColumn <int> accountId = new USqlColumn <int>("AccountId");
USqlColumn <string> profileImageUrl = new USqlColumn <string>("ProfileImageUrl");
USqlColumn <string> websiteUrl = new USqlColumn <string>("WebsiteUrl");
List <IColumn> columns = new List <IColumn> {
id, reputation, creationDate, displayName, lastAccessDate, location, aboutMe, upVotes, downVotes, age, accountId, profileImageUrl, websiteUrl
};
USqlSchema schema = new USqlSchema(columns);
return(new USqlRow(schema, null));
}
public void TestMyProcessorWithFile()
{
//Schema: "a:int, b:int"
USqlColumn <int> col1 = new USqlColumn <int>("a");
USqlColumn <int> col2 = new USqlColumn <int>("b");
List <IColumn> columns = new List <IColumn> {
col1, col2
};
USqlSchema schema = new USqlSchema(columns);
//Generate one row with default values
IUpdatableRow output = new USqlRow(schema, null).AsUpdatable();
//Get upstreams from file
IRowset rowset = UnitTestHelper.GetRowsetFromFile(@"processor.txt", schema, output.AsReadOnly(), discardAdditionalColumns: true, rowDelimiter: null, columnSeparator: '\t');
//Create UDO instance
MyProcessor processor = new MyProcessor(floor: 50, enforce: true);
foreach (IRow r in rowset.Rows)
{
processor.Process(r, output);
}
}
public IRow GetUserRow()
{
USqlColumn<int> id = new USqlColumn<int>("Id");
USqlColumn<int> reputation = new USqlColumn<int>("Reputation");
USqlColumn<DateTime> creationDate = new USqlColumn<DateTime>("CreationDate");
USqlColumn<string> displayName = new USqlColumn<string>("DisplayName");
USqlColumn<DateTime> lastAccessDate = new USqlColumn<DateTime>("LastAccessDate");
USqlColumn<string> location = new USqlColumn<string>("Location");
USqlColumn<string> aboutMe = new USqlColumn<string>("AboutMe");
USqlColumn<int> upVotes = new USqlColumn<int>("UpVotes");
USqlColumn<int> downVotes = new USqlColumn<int>("DownVotes");
USqlColumn<int> age = new USqlColumn<int>("Age");
USqlColumn<int> accountId = new USqlColumn<int>("AccountId");
USqlColumn<string> profileImageUrl = new USqlColumn<string>("ProfileImageUrl");
USqlColumn<string> websiteUrl = new USqlColumn<string>("WebsiteUrl");
List<IColumn> columns = new List<IColumn> { id, reputation, creationDate, displayName, lastAccessDate, location, aboutMe, upVotes, downVotes, age, accountId, profileImageUrl, websiteUrl };
USqlSchema schema = new USqlSchema(columns);
return new USqlRow(schema, null);
}
public void TestMyCombiner()
{
//Construct left columns
USqlColumn <int> col1 = new USqlColumn <int>("employee_id");
USqlColumn <string> col2 = new USqlColumn <string>("employee_name");
USqlColumn <string> col3 = new USqlColumn <string>("department_name");
//Construct left schema
List <IColumn> columns_left = new List <IColumn> {
col1, col3
};
USqlSchema schema_left = new USqlSchema(columns_left);
//Construct right schema
List <IColumn> columns_right = new List <IColumn> {
col1, col2
};
USqlSchema schema_right = new USqlSchema(columns_right);
//Construct result schema. expected schema: employee_id, employee_name, department_name
List <IColumn> columns_result = new List <IColumn> {
col1, col2, col3
};
USqlSchema schema_result = new USqlSchema(columns_result);
//Construct the output schema which will be needed later
IUpdatableRow output_left = new USqlRow(schema_left, null).AsUpdatable();
IUpdatableRow output_right = new USqlRow(schema_right, null).AsUpdatable();
IUpdatableRow output_result = new USqlRow(schema_result, null).AsUpdatable();
//Generate Rowset with specified values for left input
List <object[]> values_left = new List <object[]> {
new object[2] {
1, "HR"
},
new object[2] {
2, "R&D"
},
new object[2] {
4, "Operation"
},
};
//Generate Rowset with specified values for right input
List <object[]> values_right = new List <object[]> {
new object[2] {
1, "John"
},
new object[2] {
2, "Tom"
},
new object[2] {
3, "Melinda"
},
};
IEnumerable <IRow> rows_left = UnitTestHelper.CreateRowsFromValues(schema_left, values_left);
IEnumerable <IRow> rows_right = UnitTestHelper.CreateRowsFromValues(schema_right, values_right);
IRowset rowset_left = UnitTestHelper.GetRowsetFromCollection(rows_left, output_left.AsReadOnly());
IRowset rowset_right = UnitTestHelper.GetRowsetFromCollection(rows_right, output_right.AsReadOnly());
//Create UDO instance. The combiner will do something like inner join
MyCombiner combiner = new MyCombiner();
IEnumerable <IRow> after = combiner.Combine(rowset_left, rowset_right, output_result);
//Verify result
List <IRow> resultlist = after.ToList();
Assert.IsTrue(resultlist[0].Schema.Count == 3);
//Verify the values. expected to see:
//the first row: 1, "John", "HR"
//the second row: 2, "Tom", "R&D"
//here we only verify the first row
Assert.IsTrue(resultlist[0].Get <int>("employee_id") == 1);
Assert.IsTrue(resultlist[0].Get <string>(1) == "John");
Assert.IsTrue(resultlist[0].Get <string>(2) == "HR");
}
