public void BadLinesHandlingTest()
{
var junk = @"blablabla";
var danglingReward = @"{""RewardValue"":846.7236,""ActionTaken"":false,""EnqueuedTimeUtc"":""2018-12-13T03:27:57.000Z"",""EventId"":""id"",""Observations"":[{""v"":846.7236,""ActionTaken"":false,""EventId"":""id"",""ActionId"":null}]}";
var extractor = new HeaderOnly();
var output = new USqlUpdatableRow(CreateDefaultRow(CreateErrorHandlingSchema()));
using (var stream = new MemoryStream(Encoding.UTF8.GetBytes($"{junk}\n{danglingReward}")))
{
var input = new USqlStreamReader(stream);
int counter = 0;
foreach (var outputRow in extractor.Extract(input, output))
{
if (counter++ == 0)
{
Assert.IsFalse(string.IsNullOrWhiteSpace(output.Get <string>("ParseError")));
}
else
{
Assert.AreEqual("id", output.Get <string>("EventId"));
Assert.IsTrue(string.IsNullOrWhiteSpace(output.Get <string>("ParseError")));
}
}
}
}
public void SkipLearnTest()
{
var skipLearnFalse = @"{""_label_cost"":-1,""_label_probability"":0.866666973,""_label_Action"":3,""_labelIndex"":2,""_skipLearn"":false,""o"":[{""EventId"":""924da681-ea4f-4434-bc52-e9879393a052"",""v"":1.000000}],""Timestamp"":""2018-10-19T15:09:15.1100000Z"",""Version"":""1"",""EventId"":""924da681-ea4f-4434-bc52-e9879393a052"",""DeferredAction"":true,""a"":[3,2,1],""c"":{""User"":{""id"":""mk"",""major"":""psychology"",""hobby"":""kids"",""favorite_character"":""7of9""},""_multi"":[{""a"":{""topic"":""HerbGarden""}},{""a"":{""topic"":""MachineLearning""}},{""a"":{""topic"":""Football""}}]},""p"":[0.866667,0.066667,0.066667],""VWState"":{""m"":""model_id""}}";
var skipLearnTrue = @"{""_label_cost"":-1,""_label_probability"":0.866666973,""_label_Action"":3,""_labelIndex"":2,""_skipLearn"":true,""o"":[{""EventId"":""924da681-ea4f-4434-bc52-e9879393a052"",""v"":1.000000}],""Timestamp"":""2018-10-19T15:09:15.1100000Z"",""Version"":""1"",""EventId"":""924da681-ea4f-4434-bc52-e9879393a052"",""DeferredAction"":true,""a"":[3,2,1],""c"":{""User"":{""id"":""mk"",""major"":""psychology"",""hobby"":""kids"",""favorite_character"":""7of9""},""_multi"":[{""a"":{""topic"":""HerbGarden""}},{""a"":{""topic"":""MachineLearning""}},{""a"":{""topic"":""Football""}}]},""p"":[0.866667,0.066667,0.066667],""VWState"":{""m"":""model_id""}}";
var noSkipLearn = @"{""_label_cost"":-1,""_label_probability"":0.866666973,""_label_Action"":3,""_labelIndex"":2,""o"":[{""EventId"":""924da681-ea4f-4434-bc52-e9879393a052"",""v"":1.000000}],""Timestamp"":""2018-10-19T15:09:15.1100000Z"",""Version"":""1"",""EventId"":""924da681-ea4f-4434-bc52-e9879393a052"",""DeferredAction"":true,""a"":[3,2,1],""c"":{""User"":{""id"":""mk"",""major"":""psychology"",""hobby"":""kids"",""favorite_character"":""7of9""},""_multi"":[{""a"":{""topic"":""HerbGarden""}},{""a"":{""topic"":""MachineLearning""}},{""a"":{""topic"":""Football""}}]},""p"":[0.866667,0.066667,0.066667],""VWState"":{""m"":""model_id""}}";
var extractor = new HeaderOnly();
var output = new USqlUpdatableRow(CreateDefaultRow(CreateSkipLearnSchema()));
using (var stream = new MemoryStream(Encoding.UTF8.GetBytes($"{skipLearnFalse}\n{skipLearnTrue}\n{noSkipLearn}")))
{
var input = new USqlStreamReader(stream);
int counter = 0;
foreach (var outputRow in extractor.Extract(input, output))
{
if (counter == 0)
{
Assert.IsFalse(output.Get <bool>("SkipLearn"));
}
else if (counter == 1)
{
Assert.IsTrue(output.Get <bool>("SkipLearn"));
}
else
{
Assert.IsFalse(output.Get <bool>("SkipLearn"));
}
counter++;
}
}
}
public void AvroExtractor_IgnoreColumnMismatch_Ignore()
{
var schema = @"{""type"":""record"",""name"":""SingleColumnPoco"",""namespace"":""Microsoft.Analytics.Samples.Formats.Tests"",""fields"":[{""name"":""Value"",""type"":[""null"",""string""]}]}";
var data = new List <SingleColumnPoco <string> >
{
new SingleColumnPoco <string>()
{
Value = "asdf"
},
new SingleColumnPoco <string>()
{
Value = null
}
};
var output = SingleColumnRowGenerator <int>().AsUpdatable();
using (var dataStream = new MemoryStream())
{
serializeAvro(dataStream, data, schema);
var reader = new USqlStreamReader(dataStream);
var extractor = new AvroExtractor(schema, ignoreColumnMismatches: true);
var result = extractor.Extract(reader, output).ToList();
Assert.AreEqual(0, result[0].Get <int>("Value"));
Assert.AreEqual(0, result[1].Get <int>("Value"));
}
}
public void MixedParsingTest()
{
var interaction = @"{""_label_cost"":-57,""_label_probability"":0.201250032,""_label_Action"":80,""_labelIndex"":79,""Timestamp"":""2019-03-20T23:21:29.000Z"",""Version"":""1"",""EventId"":""id1"",""a"":[0,1],""c"":{""GUser"":{""f1"":1},""_multi"":[{""a1"":{""af1"":1}},{""a80"":{""af1"":1}}]},""p"":[0.4,0.6],""VWState"":{""m"":""state_id""}}";
var danglingReward = @"{""RewardValue"":846.7236,""ActionTaken"":false,""EnqueuedTimeUtc"":""2018-12-13T03:27:57.000Z"",""EventId"":""id2"",""Observations"":[{""v"":846.7236,""ActionTaken"":false,""EventId"":""id"",""ActionId"":null}]}";
var extractor = new HeaderOnly();
var output = new USqlUpdatableRow(CreateDefaultRow(CreateMixedSchema()));
using (var stream = new MemoryStream(Encoding.UTF8.GetBytes($"{interaction}\n{danglingReward}")))
{
var input = new USqlStreamReader(stream);
int counter = 0;
foreach (var outputRow in extractor.Extract(input, output))
{
if (counter++ == 0)
{
Assert.AreEqual("id1", output.Get <string>("EventId"));
Assert.AreEqual(new DateTime(2019, 03, 20, 23, 21, 29), output.Get <DateTime>("Timestamp"));
Assert.AreEqual(false, output.Get <bool>("IsDangling"));
}
else
{
Assert.AreEqual("id2", output.Get <string>("EventId"));
Assert.AreEqual(new DateTime(2018, 12, 13, 3, 27, 57), output.Get <DateTime>("EnqueuedTimeUtc"));
Assert.AreEqual(true, output.Get <bool>("IsDangling"));
}
}
}
}
public void BadLinesHandlingTest()
{
var junk = @"blablabla";
var ccbLine = @"{ ""Timestamp"": ""2019-08-27T12:45:53.6300000Z"", ""Version"": ""1"", ""c"": { ""GUser"": { ""shared_feature"": ""feature"" }, ""_multi"": [ { ""TAction"": { ""feature1"": 3.0, ""feature2"": ""name1"" } }, { ""TAction"": { ""feature1"": 3.0, ""feature2"": ""name1"" } }, { ""TAction"": { ""feature1"": 3.0, ""feature2"": ""name1"" } } ], ""_slots"": [ { ""size"": ""small"", ""_inc"": [0, 2] }, { ""size"": ""large"" } ] }, ""_outcomes"": [ { ""_id"": ""62ddd79e-4d75-4c64-94f1-a5e13a75c2e4"", ""_label_cost"": 0, ""_a"": [2, 0], ""_p"": [0.9, 0.1], ""_o"": [] }, { ""_id"": ""042661c4-d433-4b05-83d6-d51a2d1c68be"", ""_label_cost"": 0, ""_a"": [1, 0], ""_p"": [0.1, 0.9], ""_o"": [-1.0, 0.0] } ], ""VWState"": { ""m"": ""da63c529-018b-44b1-ad0f-c2b13056832c/195fc8ed-224f-471a-90c4-d3e60b336f8f"" } }";
var extractor = new CcbExtractor();
var output = new USqlUpdatableRow(CreateDefaultRow(CreateErrorHandlingSchema()));
using (var stream = new MemoryStream(Encoding.UTF8.GetBytes($"{junk}\n{ccbLine}")))
{
var input = new USqlStreamReader(stream);
int counter = 0;
foreach (var outputRow in extractor.Extract(input, output))
{
if (counter++ == 0)
{
Assert.IsFalse(string.IsNullOrWhiteSpace(output.Get <string>("ParseError")));
}
else
{
Assert.IsNotNull(output.Get <string>("SessionId"));
Assert.IsTrue(string.IsNullOrWhiteSpace(output.Get <string>("ParseError")));
}
}
}
}
private IList <IRow> ExecuteExtract(string xml, string rowPath, SqlMap <string, string> columnMappings, USqlSchema schema, SqlMap <string, string> namespaces = null)
{
using (var dataStream = new MemoryStream(Encoding.UTF8.GetBytes(xml)))
{
var reader = new USqlStreamReader(dataStream);
var extractor = new XmlExtractor(rowPath, columnMappings, namespaces);
return(extractor.Extract(reader, new USqlRow(schema, null).AsUpdatable()).ToList());
}
}
private IList <IRow> ExecuteExtract <T>(List <SingleColumnPoco <T> > data, string schema)
{
var output = SingleColumnRowGenerator <T>().AsUpdatable();
using (var dataStream = new MemoryStream())
{
serializeAvro(dataStream, data, schema);
var reader = new USqlStreamReader(dataStream);
var extractor = new AvroExtractor(schema);
return(extractor.Extract(reader, output).ToList());
}
}
public void Extract_ValidCommentsInputFromCoffee_Succeeds()
{
IUpdatableRow output = GetCommentRow().AsUpdatable();
IEnumerable <IRow> result;
using (FileStream fileReader = new FileStream("Input/Coffee/Comments.xml", FileMode.Open, FileAccess.Read))
{
USqlStreamReader streamReader = new USqlStreamReader(fileReader);
XmlExtractor extractor = new XmlExtractor();
result = extractor.Extract(streamReader, output);
Assert.Equal(1279, result.Count());
}
}
public void Extract_ValidUsersInputFromBoardgames_Succeeds()
{
IUpdatableRow output = GetUserRow().AsUpdatable();
IEnumerable <IRow> result;
using (FileStream fileReader = new FileStream("Input/Board-Games/Users.xml", FileMode.Open, FileAccess.Read))
{
USqlStreamReader streamReader = new USqlStreamReader(fileReader);
XmlExtractor extractor = new XmlExtractor();
result = extractor.Extract(streamReader, output);
Assert.Equal(9352, result.Count());
}
}
public void Extract_ValidUsersInputFromCoffee_Succeeds()
{
IUpdatableRow output = GetUserRow().AsUpdatable();
IEnumerable<IRow> result;
using (FileStream fileReader = new FileStream("Input/Coffee/Users.xml", FileMode.Open, FileAccess.Read))
{
USqlStreamReader streamReader = new USqlStreamReader(fileReader);
XmlExtractor extractor = new XmlExtractor();
result = extractor.Extract(streamReader, output);
Assert.Equal(1162, result.Count());
}
}
public void Extract_ValidCommentsInputFromBoardgames_Succeeds()
{
IUpdatableRow output = GetCommentRow().AsUpdatable();
IEnumerable<IRow> result;
using (FileStream fileReader = new FileStream("Input/Board-Games/Comments.xml", FileMode.Open, FileAccess.Read))
{
USqlStreamReader streamReader = new USqlStreamReader(fileReader);
XmlExtractor extractor = new XmlExtractor();
result = extractor.Extract(streamReader, output);
Assert.Equal(23343, result.Count());
}
}
public void CcbTest()
{
var ccbLine = @"{ ""Timestamp"": ""2019-08-27T12:45:53.0000000Z"", ""Version"": ""1"", ""c"": { ""GUser"": { ""shared_feature"": ""feature"" }, ""_multi"": [ { ""TAction"": { ""feature1"": 3.0, ""feature2"": ""name1"" } }, { ""TAction"": { ""feature1"": 3.0, ""feature2"": ""name1"" } }, { ""TAction"": { ""feature1"": 3.0, ""feature2"": ""name1"" } } ], ""_slots"": [ { ""size"": ""small"", ""_inc"": [0, 2] }, { ""size"": ""large"" } ] }, ""_outcomes"": [ { ""_label_cost"": 0, ""_id"": ""62ddd79e-4d75-4c64-94f1-a5e13a75c2e4"", ""_a"": [2, 0], ""_p"": [0.9, 0.1], ""_o"": [] }, { ""_label_cost"": -1.0, ""_id"": ""042661c4-d433-4b05-83d6-d51a2d1c68be"", ""_a"": [1, 0], ""_p"": [0.1, 0.9], ""_o"": [{""v"": 1.0}] } ], ""VWState"": { ""m"": ""da63c529-018b-44b1-ad0f-c2b13056832c/195fc8ed-224f-471a-90c4-d3e60b336f8f"" } }";
var extractor = new CcbExtractor();
var output = new USqlUpdatableRow(CreateDefaultRow(CreateCcbBasicSchema()));
using (var stream = new MemoryStream(Encoding.UTF8.GetBytes($"{ccbLine}")))
{
var input = new USqlStreamReader(stream);
int counter = 0;
foreach (var outputRow in extractor.Extract(input, output))
{
if (counter == 0)
{
Assert.IsNotNull(output.Get <string>("SessionId"));
Assert.AreEqual(0, output.Get <int>("SlotIdx"));
Assert.AreEqual("62ddd79e-4d75-4c64-94f1-a5e13a75c2e4", output.Get <string>("EventId"));
Assert.AreEqual(new DateTime(2019, 8, 27, 12, 45, 53, 0), output.Get <DateTime>("Timestamp"));
Assert.AreEqual(0.0, output.Get <float>("Cost"), 1e-6);
Assert.AreEqual(0.9, output.Get <float>("Prob"), 1e-6);
Assert.AreEqual(2, output.Get <int>("NumActionsPerSlot"));
Assert.AreEqual(3, output.Get <int>("NumActions"));
Assert.AreEqual(2, output.Get <int>("Action"));
Assert.AreEqual(1, output.Get <int>("CbAction"));
Assert.AreEqual(0, output.Get <int>("HasObservations"));
Assert.AreEqual(0, output.Get <float>("pdrop"));
Assert.AreEqual(false, output.Get <bool>("IsDangling"));
}
else if (counter == 1)
{
Assert.IsNotNull(output.Get <string>("SessionId"));
Assert.AreEqual(1, output.Get <int>("SlotIdx"));
Assert.AreEqual("042661c4-d433-4b05-83d6-d51a2d1c68be", output.Get <string>("EventId"));
Assert.AreEqual(new DateTime(2019, 8, 27, 12, 45, 53, 0), output.Get <DateTime>("Timestamp"));
Assert.AreEqual(-1.0, output.Get <float>("Cost"), 1e-6);
Assert.AreEqual(0.1, output.Get <float>("Prob"), 1e-6);
Assert.AreEqual(2, output.Get <int>("NumActionsPerSlot"));
Assert.AreEqual(3, output.Get <int>("NumActions"));
Assert.AreEqual(1, output.Get <int>("Action"));
Assert.AreEqual(1, output.Get <int>("CbAction"));
Assert.AreEqual(1, output.Get <int>("HasObservations"));
Assert.AreEqual(0, output.Get <float>("pdrop"));
Assert.AreEqual(false, output.Get <bool>("IsDangling"));
}
counter++;
}
}
}
public void CcbDanglingRewardTest()
{
var danglingReward = @"{""RewardValue"":846.7236,""ActionTaken"":false,""EnqueuedTimeUtc"":""2018-12-13T03:27:57.000Z"",""EventId"":""id"",""Observations"":[{""v"":846.7236,""ActionTaken"":false,""EventId"":""id"",""ActionId"":null}]}";
var extractor = new CcbExtractor();
var output = new USqlUpdatableRow(CreateDefaultRow(CreateDanglingRewardSchema()));
using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(danglingReward)))
{
var input = new USqlStreamReader(stream);
foreach (var outputRow in extractor.Extract(input, output))
{
Assert.AreEqual("id", output.Get <string>("EventId"));
Assert.AreEqual(new DateTime(2018, 12, 13, 3, 27, 57), output.Get <DateTime>("EnqueuedTimeUtc"));
}
}
}
public void ParseNonExistingPdropTest()
{
var exampleWithPdrop = @"{""_label_cost"":-57,""_label_probability"":0.201250032,""_label_Action"":80,""_labelIndex"":79,""Timestamp"":""2019-03-20T23:21:29.000Z"",""Version"":""1"",""EventId"":""id"",""a"":[0,1],""c"":{""GUser"":{""f1"":1},""_multi"":[{""a1"":{""af1"":1}},{""a80"":{""af1"":1}}]},""p"":[0.4,0.6],""VWState"":{""m"":""state_id""}}";
var extractor = new HeaderOnly();
var output = new USqlUpdatableRow(CreateDefaultRow(CreateTestSchemaWithPdrop()));
using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(exampleWithPdrop)))
{
var input = new USqlStreamReader(stream);
foreach (var outputRow in extractor.Extract(input, output))
{
Assert.AreEqual("id", output.Get <string>("EventId"));
Assert.AreEqual(0, output.Get <float>("pdrop"));
}
}
}
public void TestMyExtractor()
{
IUpdatableRow output = RowGenerator().AsUpdatable();
using (FileStream stream = new FileStream(@"searchlog.txt", FileMode.Open))
{
//Read input file
USqlStreamReader reader = new USqlStreamReader(stream);
//Run the UDO
MyExtractor extractor = new MyExtractor();
List <IRow> result = extractor.Extract(reader, output).ToList();
//Verify the schema
Assert.IsTrue(result[0].Schema.Count == 3);
//Verify the result
Assert.IsTrue(result[0].Get <string>("Market") == "en-gb");
}
}
public void ActionCountTest()
{
var example = @"{""_label_cost"":-2,""_label_probability"":0.911111116,""_label_Action"":8,""_labelIndex"":7,""o"":[{""EventId"":""6c33a7c07aae4868b4b14f944e3320c0"",""v"":1},{""EventId"":""6c33a7c07aae4868b4b14f944e3320c0"",""v"":0},{""EventId"":""6c33a7c07aae4868b4b14f944e3320c0"",""v"":2}],""Timestamp"":""2018-07-19T23:30:57.6700000Z"",""Version"":""1"",""EventId"":""6c33a7c07aae4868b4b14f944e3320c0"",""a"":[8,1,6,2,10,11,9,5,3,7,4],""c"":{""_synthetic"":false,""User"":{""_age"":0},""Geo"":{""country"":""United States"",""_countrycf"":""8"",""state"":""Nevada"",""city"":""Las Vegas"",""_citycf"":""8"",""dma"":""839""},""MRefer"":{""referer"":""https://www.complex.com/""},""OUserAgent"":{""_ua"":""Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/603.3.8 (KHTML, like Gecko) Version/10.1.2 Safari/603.3.8"",""_DeviceBrand"":"""",""_DeviceFamily"":""Other"",""_DeviceIsSpider"":false,""_DeviceModel"":"""",""_OSFamily"":""Mac OS X"",""_OSMajor"":""10"",""_OSPatch"":""5"",""DeviceType"":""Desktop""},""_multi"":[{""_tag"":""cmplx$http://www.complex.com/music/2018/07/four-men-indicted-for-fatally-shooting-xxxtentacion"",""i"":{""constant"":1,""id"":""cmplx$http://www.complex.com/music/2018/07/four-men-indicted-for-fatally-shooting-xxxtentacion""},""j"":[{""_title"":""Four Men Indicted by Grand Jury for Fatally Shooting XXXTentacion""},null,{""Emotion0"":{""anger"":0.08193103,""contempt"":0.0142859118,""disgust"":0.0046643177,""fear"":0.000101052363,""happiness"":5.75684571E-05,""neutral"":0.8652449,""sadness"":0.0334045,""surprise"":0.000310689618},""Emotion1"":{""anger"":0.0361110829,""contempt"":0.008100673,""disgust"":0.000575012062,""fear"":6.130771E-05,""happiness"":5.01265458E-05,""neutral"":0.945477247,""sadness"":0.008256853,""surprise"":0.00136769784},""Emotion2"":{""anger"":0.0251881052,""contempt"":0.0003558025,""disgust"":0.00133104972,""fear"":0.00114253594,""happiness"":0.947571039,""neutral"":0.013514352,""sadness"":0.00408442831,""surprise"":0.006812664},""_expires"":""2018-07-22T22:24:33.6560076Z""},null,{""XSentiment"":0.5,""_expires"":""2018-07-22T22:24:33.6390641Z""},null,null]},{""_tag"":""cmplx$http://www.complex.com/music/2018/07/2-chainz-on-his-pink-trap-house-destruction-i-dont-think-they-would-known-down-the-statue-of-liberty"",""i"":{""constant"":1,""id"":""cmplx$http://www.complex.com/music/2018/07/2-chainz-on-his-pink-trap-house-destruction-i-dont-think-they-would-known-down-the-statue-of-liberty""},""j"":[{""_title"":""2 Chainz on His Pink Trap House's Destruction: 'I Don’t Think They Would Knock Down the Statue of Li""},null,{""_expires"":""2018-07-22T18:39:06.435326Z""},null,{""XSentiment"":0.5,""_expires"":""2018-07-22T18:39:06.437968Z""},null,null]},{""_tag"":""cmplx$http://www.complex.com/sports/2018/07/watch-chloe-kim-rap-cardi-b-part-in-no-limit-with-g-eazy"",""i"":{""constant"":1,""id"":""cmplx$http://www.complex.com/sports/2018/07/watch-chloe-kim-rap-cardi-b-part-in-no-limit-with-g-eazy""},""j"":[{""_title"":""Watch Chloe Kim Rap Cardi B’s Part in “No Limit” With G-Eazy""},null,{""Emotion0"":{""anger"":5.271395E-10,""contempt"":1.18699287E-11,""disgust"":1.64511382E-09,""fear"":1.06379437E-11,""happiness"":1,""neutral"":7.029596E-10,""sadness"":7.483194E-10,""surprise"":1.54087343E-09},""_expires"":""2018-07-22T21:38:36.3179955Z""},null,{""XSentiment"":0.5,""_expires"":""2018-07-22T21:38:36.1764128Z""},null,null]},{""_tag"":""cmplx$http://www.complex.com/pop-culture/2018/07/the-best-will-smith-movies"",""i"":{""constant"":1,""id"":""cmplx$http://www.complex.com/pop-culture/2018/07/the-best-will-smith-movies""},""j"":[{""_title"":""The Best Will Smith Movies""},null,{""Emotion0"":{""anger"":0.00134275958,""contempt"":0.01697544,""disgust"":0.0023304366,""fear"":9.310076E-05,""happiness"":0.5494229,""neutral"":0.421826631,""sadness"":0.00700154155,""surprise"":0.0010071845},""_expires"":""2018-07-22T21:09:36.1175959Z""},null,{""XSentiment"":0.9716126,""_expires"":""2018-07-22T21:09:35.8308744Z""},null,null]},{""_tag"":""cmplx$https://www.complex.com/sports/2018/07/2018-espys-red-carpet"",""i"":{""constant"":1,""id"":""cmplx$https://www.complex.com/sports/2018/07/2018-espys-red-carpet""},""j"":[{""_title"":""Professional Athletes Keep It Real About 'Loyalty' at the 2018 ESPYs""},null,{""Emotion0"":{""anger"":3.38298159E-05,""contempt"":2.0690366E-05,""disgust"":5.13375526E-05,""fear"":4.964087E-06,""happiness"":0.9976058,""neutral"":0.00200294587,""sadness"":0.000260378147,""surprise"":2.00380746E-05},""_expires"":""2018-07-22T22:41:52.3964008Z""},null,{""XSentiment"":0.5,""_expires"":""2018-07-22T22:41:52.0057354Z""},null,null]},{""_tag"":""cmplx$https://www.complex.com/sports/2018/07/jimmy-garoppolo-spotted-with-adult-film-star-kiara-mia"",""i"":{""constant"":1,""id"":""cmplx$https://www.complex.com/sports/2018/07/jimmy-garoppolo-spotted-with-adult-film-star-kiara-mia""},""j"":[{""_title"":""Jimmy Garoppolo Spotted in Beverly Hills With Adult Film Star Kiara Mia""},null,{""_expires"":""2018-07-22T21:34:08.4969709Z""},{""_expires"":""2018-07-19T23:09:18.243512Z""},{""XSentiment"":0.5,""_expires"":""2018-07-22T21:34:08.5229152Z""},null,null]},{""_tag"":""cmplx$https://www.complex.com/pop-culture/2018/07/joji-and-rich-brian-play-the-newlywed-game-while-eating-spicy-wings-hot-ones"",""i"":{""constant"":1,""id"":""cmplx$https://www.complex.com/pop-culture/2018/07/joji-and-rich-brian-play-the-newlywed-game-while-eating-spicy-wings-hot-ones""},""j"":[{""_title"":""Joji and Rich Brian Play the Newlywed Game While Eating Spicy Wings | Hot Ones""},null,{""Emotion0"":{""anger"":0.000149607382,""contempt"":0.00592999766,""disgust"":0.000307883281,""fear"":6.14888359E-06,""happiness"":0.05696309,""neutral"":0.9336905,""sadness"":0.00251298165,""surprise"":0.000439826224},""Emotion1"":{""anger"":1.47376359E-05,""contempt"":0.000332679017,""disgust"":3.11678159E-05,""fear"":6.127358E-08,""happiness"":0.9778741,""neutral"":0.0217296854,""sadness"":4.953524E-06,""surprise"":1.26064178E-05},""_expires"":""2018-07-22T15:08:54.8660979Z""},null,{""XSentiment"":0.5,""_expires"":""2018-07-22T15:08:54.3421013Z""},null,null]},{""_tag"":""cmplx$https://www.complex.com/music/2018/07/kid-cudi-says-he-and-kanye-west-are-planning-to-do-more-kids-see-ghosts-albums"",""i"":{""constant"":1,""id"":""cmplx$https://www.complex.com/music/2018/07/kid-cudi-says-he-and-kanye-west-are-planning-to-do-more-kids-see-ghosts-albums""},""j"":[{""_title"":""Kid Cudi Says He and Kanye West Are Planning to Do More 'Kids See Ghosts' Albums""},null,{""Emotion0"":{""anger"":0.004779356,""contempt"":0.0396654122,""disgust"":0.000898613245,""fear"":6.362816E-05,""happiness"":0.0329472534,""neutral"":0.9089038,""sadness"":0.01218673,""surprise"":0.000555203645},""Emotion1"":{""anger"":0.0016001676,""contempt"":0.00258500734,""disgust"":0.0004716456,""fear"":2.961242E-05,""happiness"":0.000576888851,""neutral"":0.9908969,""sadness"":0.0006469234,""surprise"":0.00319288624},""_expires"":""2018-07-22T19:13:10.3669958Z""},null,{""XSentiment"":0.5,""_expires"":""2018-07-22T19:13:10.2018036Z""},null,null]},{""_tag"":""cmplx$https://www.complex.com/pop-culture/2018/07/chance-drops-some-heat-chief-keef-hologram-tour-genius-or-lazy-everyday-struggle"",""i"":{""constant"":1,""id"":""cmplx$https://www.complex.com/pop-culture/2018/07/chance-drops-some-heat-chief-keef-hologram-tour-genius-or-lazy-everyday-struggle""},""j"":[{""_title"":""Chance Drops Some Heat, Chief Keef Hologram Tour Genius or Lazy? | Everyday Struggle ""},null,{""Emotion0"":{""anger"":0.008510953,""contempt"":0.004868189,""disgust"":0.00184451311,""fear"":0.0329660736,""happiness"":0.138749734,""neutral"":0.743005753,""sadness"":0.0405789427,""surprise"":0.0294758677},""_expires"":""2018-07-22T15:02:40.6014988Z""},null,{""XSentiment"":0.5,""_expires"":""2018-07-22T15:02:40.5104894Z""},null,null]}]},""p"":[0.9111111,0.0111111114,0.0111111114,0.0111111114,0.0111111114,0.0111111114,0.0111111114,0.0111111114,0.0111111114,0.0111111114,0.0111111114],""VWState"":{""m"":""5744c0fe554e462c8e300670c9029102-1Nhzo/59b51c004d714814b2d81313dc189440-1NhEl""}}";
var extractor = new HeaderOnly();
var output = new USqlUpdatableRow(CreateDefaultRow(CreateTestSchema()));
using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(example)))
{
var input = new USqlStreamReader(stream);
foreach (var outputRow in extractor.Extract(input, output))
{
Assert.AreEqual("6c33a7c07aae4868b4b14f944e3320c0", output.Get <string>("EventId"));
Assert.AreEqual(-2.0, output.Get <float>("Cost"), 1e-6);
Assert.AreEqual(0.911111116, output.Get <float>("Prob"), 1e-6);
Assert.AreEqual(11, output.Get <int>("NumActions"));
Assert.AreEqual(8, output.Get <int>("Action"));
Assert.AreEqual(1, output.Get <int>("HasObservations"));
}
}
}
public void ActionCountTest()
{
var example = @"{""_label_cost"":-57,""_label_probability"":0.6,""_label_Action"":2,""_labelIndex"":1,""o"":[{""EventId"":""id"",""v"":1}],""Timestamp"":""2019-03-20T23:21:29.5020000Z"",""Version"":""1"",""EventId"":""id"",""a"":[0,1],""c"":{""GUser"":{""f1"":1},""_multi"":[{""a1"":{""af1"":1}},{""a80"":{""af1"":1}}]},""p"":[0.4,0.6],""VWState"":{""m"":""state_id""},""pdrop"":0.5}";
var extractor = new HeaderOnly();
var output = new USqlUpdatableRow(CreateDefaultRow(CreateTestSchema()));
using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(example)))
{
var input = new USqlStreamReader(stream);
foreach (var outputRow in extractor.Extract(input, output))
{
Assert.AreEqual("id", output.Get <string>("EventId"));
Assert.AreEqual(-57.0, output.Get <float>("Cost"), 1e-6);
Assert.AreEqual(0.6, output.Get <float>("Prob"), 1e-6);
Assert.AreEqual(2, output.Get <int>("NumActions"));
Assert.AreEqual(2, output.Get <int>("Action"));
Assert.AreEqual(1, output.Get <int>("HasObservations"));
}
}
}
public void TestMyExtractor()
{
IUpdatableRow output = RowGenerator().AsUpdatable();
var testDataPath = "\"" + Directory.GetCurrentDirectory() + @"\test.txt" + "\"";
using (FileStream stream = new FileStream(@"test.txt", FileMode.Open))
{
//Read input file
USqlStreamReader reader = new USqlStreamReader(stream);
//Run the UDO
EventExtractor extractor = new EventExtractor();
List <IRow> result = extractor.Extract(reader, output).ToList();
//Verify the schema
//Assert.IsTrue(result[0].Schema.Count == 2);
//Verify the result
//Console.WriteLine("Result: "+result[0].Get<string>("contexts"));
//Assert.IsTrue(result[0].Get<string>("contexts") == "yolo");
//Assert.IsTrue(result[0].Get<string>("platform") == "web");
//var contexts = result[0].Get<string>("contexts");
//var schema = JsonFunctions.JsonTuple(contexts, "data[*]");
//schema.Values.ToList().ForEach(item => {
// var innerData = JsonFunctions.JsonTuple(item, "data");
//Console.WriteLine("innerdata length: "+ innerData.Count());
//innerData.Values.ToList().ForEach(innerItem => {
// var genre = JsonFunctions.JsonTuple(innerItem, "navigationStart");
// genre.Values.ToList().ForEach( g => Console.WriteLine(g));
//}
// );
//});
var sqlMap = result[0].Get <SqlMap <string, string> >("contexts.data.author");
var keys = sqlMap.Keys;
keys.ToList().ForEach(key => Console.WriteLine($"{key}: {sqlMap[key]}"));
//var data = schema.Values.ElementAt(0);
//var innerData = JsonFunctions.JsonTuple(data, "data");
}
}
public void LoadExtractorTest( )
{
string file = "TestFiles\\TestFile1.xml";
using (var st = new StreamReader(file))
{
USqlStreamReader reader = new USqlStreamReader(st.BaseStream);
USqlUpdatableRow updRow = getUpdatableRow();
XmlDomExtractor extractor = new XmlDomExtractor("Locations/Location");
var result = extractor.Extract(reader, updRow);
int cnt = 0;
foreach(var item in result)
{
cnt++;
Debug.WriteLine(item.Get<long>("Id"));
}
}
}
