public EntitiesResponse Post([FromBody] EntitiesRequest request)
{
EntitiesResponse result = new EntitiesResponse();
LucyEngine engine = null;
if (!_cache.TryGetValue <LucyEngine>(request.yaml, out engine))
{
try
{
engine = new LucyEngine(request.yaml, useAllBuiltIns: true);
_cache.Set(request.yaml, engine);
}
catch (Exception err)
{
result.message = err.Message;
return(result);
}
}
if (engine.Warnings.Any())
{
result.message = String.Join("\n", engine.Warnings);
}
Stopwatch sw = new Stopwatch();
sw.Start();
result.entities = engine.MatchEntities(request.text).ToArray();
sw.Stop();
result.elapsed = sw.ElapsedMilliseconds;
return(result);
}
public void TestLoadYaml()
{
LucyDocument lucyModel;
using (var streamReader = new StreamReader(File.OpenRead(Path.Combine(@"..", "..", "..", "lucy.yaml"))))
{
lucyModel = yamlDeserializer.Deserialize <LucyDocument>(streamReader);
}
var engine = new LucyEngine(lucyModel);
string text = "the box is 9 inches by 7.";
var results = engine.MatchEntities(text);
Trace.TraceInformation("\n" + LucyEngine.VisualEntities(text, results));
var entities = results.Where(e => e.Type == "boxSize").ToList();
Assert.AreEqual(1, entities.Count);
var entity = entities.Single().Children.First(entity => entity.Type == "twoDimensional");
Assert.AreEqual("twoDimensional", entity.Type);
Assert.AreEqual(2, entity.Children.Count);
Assert.AreEqual(1, entity.Children.Where(e => e.Type == "number").Count());
Assert.AreEqual(1, entity.Children.Where(e => e.Type == "dimension").Count());
}
public void TestLoadMacroTest()
{
LucyDocument lucyModel;
using (var streamReader = new StreamReader(File.OpenRead(Path.Combine(@"..", "..", "..", "lucy.yaml"))))
{
lucyModel = yamlDeserializer.Deserialize <LucyDocument>(streamReader);
}
var engine = new LucyEngine(lucyModel);
string text = "flight from seattle to chicago.";
var results = engine.MatchEntities(text, null);
Trace.TraceInformation("\n" + LucyEngine.VisualEntities(text, results));
var trip = results.Where(e => e.Type == "trip").Single();
var departure = trip.Children.Where(e => e.Type == "departure").Single();
var departurePlace = departure.Children.Where(e => e.Type == "placeAndTime").Single();
var departureCity = departurePlace.Children.Where(e => e.Type == "city").Single();
Assert.AreEqual("seattle", departureCity.Resolution);
var destination = trip.Children.Where(e => e.Type == "destination").Single();
var desinationPlace = destination.Children.Where(e => e.Type == "placeAndTime").Single();
var destinationCity = desinationPlace.Children.Where(e => e.Type == "city").Single();
Assert.AreEqual("chicago", destinationCity.Resolution);
}
public void WildcardSinglePrefixTest()
{
var engine = new LucyEngine(new LucyDocument()
{
Entities = new List <EntityDefinition>()
{
new EntityDefinition()
{
Name = "@beer", Patterns = new List <Pattern>()
{
"(value:___) beer"
}
},
}
});
string text = "like coors beer";
var results = engine.MatchEntities(text, null);
Trace.TraceInformation("\n" + LucyEngine.VisualEntities(text, results));
var entities = results.Where(e => e.Type == "beer").ToList();
Assert.AreEqual(1, entities.Count);
Assert.AreEqual("beer", entities[0].Type);
Assert.AreEqual("coors", entities[0].Children.First().Resolution);
Assert.AreEqual("coors beer", text.Substring(entities[0].Start, entities[0].End - entities[0].Start));
}
public void PatternParser_ZeroOrMoreModifierQuantityTests()
{
var engine = new LucyEngine(new LucyDocument()
{
Entities = new List <EntityDefinition>()
{
new EntityDefinition()
{
Name = "@test",
Patterns = new List <Pattern>()
{
"a (dog|cat|test)3*"
}
}
}
});
var text = "this is a dog dog dog test test test";
var results = engine.MatchEntities(text, null);
var entities = results.Where(e => e.Type == "test").ToList();
Assert.AreEqual(1, entities.Count);
Assert.AreEqual("test", entities[0].Type);
Assert.AreEqual("a dog dog dog", text.Substring(entities[0].Start, entities[0].End - entities[0].Start));
}
public void NestedPatternOrdinalityTests2()
{
var engine = new LucyEngine(new LucyDocument()
{
Entities = new List <EntityDefinition>()
{
new EntityDefinition()
{
Name = "@test", Patterns = new List <Pattern>()
{
"((x) (y))+"
}
},
}
});
string text = "x y x y";
var results = engine.MatchEntities(text, null);
Trace.TraceInformation("\n" + LucyEngine.VisualEntities(text, results));
var entities = results.Where(e => e.Type == "test").ToList();
Assert.AreEqual(1, entities.Count);
text = "x x z y y";
results = engine.MatchEntities(text, null);
Trace.TraceInformation("\n" + LucyEngine.VisualEntities(text, results));
entities = results.Where(e => e.Type == "test").ToList();
Assert.AreEqual(0, entities.Count);
}
public void WildcardSingleTest()
{
var engine = new LucyEngine(new LucyDocument()
{
Entities = new List <EntityDefinition>()
{
new EntityDefinition()
{
Name = "@name", Patterns = new List <Pattern>()
{
"name is (value:___)"
}
},
}
});
string text = "my name is joe smith and I am cool";
var results = engine.MatchEntities(text, null);
Trace.TraceInformation("\n" + LucyEngine.VisualEntities(text, results));
var entities = results.Where(e => e.Type == "name").ToList();
Assert.AreEqual(1, entities.Count);
Assert.AreEqual("name", entities[0].Type);
Assert.AreEqual("joe", entities[0].Children.First().Resolution);
Assert.AreEqual("name is joe", text.Substring(entities[0].Start, entities[0].End - entities[0].Start));
}
public void RegexPatternMatcher()
{
var engine = new LucyEngine(new LucyDocument()
{
Entities = new List <EntityDefinition>()
{
new EntityDefinition()
{
Name = "@test", Patterns = new List <Pattern>()
{
"/N[\\dA-Z]{5}/"
}
}
}
});
string text = "I fly N185LM and used to fly N87405.";
var results = engine.MatchEntities(text);
Trace.TraceInformation("\n" + LucyEngine.VisualEntities(text, results));
var entities = results.Where(e => e.Type == "test").ToList();
Assert.AreEqual(2, entities.Count);
Assert.AreEqual("test", entities[0].Type);
Assert.AreEqual("N185LM", text.Substring(entities[0].Start, entities[0].End - entities[0].Start));
Assert.AreEqual("N87405", text.Substring(entities[1].Start, entities[1].End - entities[1].Start));
}
public void PatternParser_MergeContigious()
{
var engine = new LucyEngine(new LucyDocument()
{
Entities = new List <EntityDefinition>()
{
new EntityDefinition()
{
Name = "@cold", Patterns = new List <Pattern>()
{
new string[] { "cold", "ice", "freezing", "frigid" }
}
},
new EntityDefinition()
{
Name = "@test", Patterns = new List <Pattern>()
{
"I want (@cold)* beer"
}
}
}
});
string text = "I want ice cold beer";
var results = engine.MatchEntities(text);
Trace.TraceInformation("\n" + LucyEngine.VisualEntities(text, results));
var entities = results.Where(e => e.Type == "test").ToList();
Assert.AreEqual(1, entities.Count);
Assert.AreEqual("test", entities[0].Type);
Assert.AreEqual("I want ice cold beer", text.Substring(entities[0].Start, entities[0].End - entities[0].Start));
Assert.AreEqual("ice cold", entities[0].Children.First().Text);
}
public void IgnoreTokenTests()
{
var engine = new LucyEngine(new LucyDocument()
{
Locale = "en",
Entities = new List <EntityDefinition>()
{
new EntityDefinition()
{
Name = "@test",
Ignore = new List <string>()
{
"is"
},
Patterns = new List <Pattern>()
{
new Pattern("name (value:___)")
}
}
}
});
string text = "my name is joe";
var results = engine.MatchEntities(text, null);
Trace.TraceInformation("\n" + LucyEngine.VisualEntities(text, results));
var entities = results.Where(e => e.Type == "test").ToList();
Assert.AreEqual(1, entities.Count);
Assert.AreEqual("test", entities[0].Type);
Assert.AreEqual("name is joe", text.Substring(entities[0].Start, entities[0].End - entities[0].Start));
Assert.AreEqual("joe", entities[0].Children.First().Resolution);
}
public void FuzzyTokenPatternMatcherTests()
{
var engine = new LucyEngine(new LucyDocument()
{
Entities = new List <EntityDefinition>()
{
new EntityDefinition()
{
Name = "@test",
FuzzyMatch = true,
Patterns = new List <Pattern>()
{
"test"
}
}
}
});
string text = "this is a tesst";
var results = engine.MatchEntities(text, null);
Trace.TraceInformation("\n" + LucyEngine.VisualEntities(text, results));
var entities = results.Where(e => e.Type == "test").ToList();
Assert.AreEqual(1, entities.Count);
Assert.AreEqual("test", entities[0].Type);
Assert.AreEqual("tesst", text.Substring(entities[0].Start, entities[0].End - entities[0].Start));
}
private void LoadModel()
{
// Trace.TraceInformation("Loading model");
var model = yamlDeserializer.Deserialize <LucyDocument>(new StringReader(this.editor.Document.Text));
engine = new LucyEngine(model, useAllBuiltIns: true);
//recognizer = new LucyRecognizer()
//{
// Model = model,
//};
StringBuilder sb = new StringBuilder();
for (int i = 0; i < 100; i++)
{
sb.AppendLine(engine.GenerateExample("desireIntent"));
}
this.examplesBox.Text = sb.ToString();
if (engine.Warnings.Any())
{
this.error.Content = String.Join("\n", engine.Warnings);
this.error.Visibility = Visibility.Visible;
}
else
{
this.error.Visibility = Visibility.Collapsed;
}
lucyModel = this.editor.Document.Text;
}
public void PatternParser_OneOrMoreModifierQuanittyTests()
{
var engine = new LucyEngine(new LucyDocument()
{
Entities = new List <EntityDefinition>()
{
new EntityDefinition()
{
Name = "@test",
Patterns = new List <Pattern>()
{
"a (dog|cat|test)+1"
}
}
}
});
string text = "this is a test dog frog";
var results = engine.MatchEntities(text, null);
Trace.TraceInformation("\n" + LucyEngine.VisualEntities(text, results));
var entities = results.Where(e => e.Type == "test").ToList();
Assert.AreEqual(1, entities.Count);
Assert.AreEqual("test", entities[0].Type);
Assert.AreEqual("a test", text.Substring(entities[0].Start, entities[0].End - entities[0].Start));
}
public override IEnumerable <string> GenerateExamples(LucyEngine engine)
{
StringBuilder sb = new StringBuilder();
for (int i = rnd.Next(7) + 3; i > 0; i--)
{
sb.Append((char)('a' + rnd.Next(26)));
}
yield return($"{sb}".Trim());
}
public override IEnumerable <string> GenerateExamples(LucyEngine engine)
{
foreach (var entityPattern in engine.EntityPatterns.Where(ep => ep.Name == EntityType))
{
foreach (var example in entityPattern.PatternMatcher.GenerateExamples(engine))
{
yield return(example.Trim());
}
}
}
public override string GenerateExample(LucyEngine engine)
{
StringBuilder sb = new StringBuilder();
foreach (var patternMatcher in this.PatternMatchers)
{
sb.Append($" {patternMatcher.GenerateExample(engine)}");
}
return(sb.ToString().Trim());
}
public void CascadingPatternTests()
{
var engine = new LucyEngine(new LucyDocument()
{
Entities = new List <EntityDefinition>()
{
new EntityDefinition()
{
Name = "@boxsize", Patterns = new List <Pattern>()
{
"box is @twoDimensional"
}
},
new EntityDefinition()
{
Name = "@height", Patterns = new List <Pattern>()
{
"(@dimension|@number) (height|tall)"
}
},
new EntityDefinition()
{
Name = "@width", Patterns = new List <Pattern>()
{
"(@dimension|@number) (width|wide)"
}
},
new EntityDefinition()
{
Name = "@twoDimensional",
Patterns = new List <Pattern>()
{
"(@width|@dimension|@number) (x|by)? (@height|@dimension|@number)",
"(@height|@dimension|@number) (x|by)? (@width|@dimension|@number)",
}
},
}
});
string text = "the box is 9 inches by 7.";
var results = engine.MatchEntities(text, null);
Trace.TraceInformation("\n" + LucyEngine.VisualEntities(text, results));
var entities = results.Where(e => e.Type == "boxsize").ToList();
Assert.AreEqual(1, entities.Count);
var entity = entities.Single().Children.Single();
Assert.AreEqual("twoDimensional", entity.Type);
Assert.AreEqual(2, entity.Children.Count);
Assert.AreEqual(1, entity.Children.Where(e => e.Type == "number").Count());
Assert.AreEqual(1, entity.Children.Where(e => e.Type == "dimension").Count());
}
private void ShowResults()
{
try
{
if (lucyModel != this.editor.Document.Text)
{
LoadModel();
}
var text = this.query?.Text?.Trim() ?? string.Empty;
if (text.Length > 0)
{
var sw = new System.Diagnostics.Stopwatch();
sw.Start();
var results = engine.MatchEntities(text, includeInternal: this.showInternal.IsChecked.Value);
sw.Stop();
this.tabs.SelectedIndex = 2;
this.labelBox.Text = $"{sw.ElapsedMilliseconds} ms\n" + String.Join("\n", results.Select(s => LucyEngine.VisualizeEntity(text, s, showSpans: true, showHierarchy: false)));
this.entitiesBox.Text = String.Join("\n", results.Select(s => LucyEngine.VisualizeEntity(text, s, showSpans: false, showHierarchy: true)));
//var activity = new Activity(ActivityTypes.Message) { Text = text };
//var tc = new TurnContext(new TestAdapter(), activity);
//var dc = new DialogContext(new DialogSet(), tc, new DialogState());
//var recognizerResult = recognizer.RecognizeAsync(dc, activity).Result;
//// this.recognizerBox.Text = JsonConvert.SerializeObject(recognizerResult, new JsonSerializerSettings() { Formatting = Formatting.Indented, NullValueHandling = NullValueHandling.Ignore });
//this.recognizerBox.Text = new Serializer().Serialize(JObject.FromObject(recognizerResult).ToObject<ExpandoObject>());
}
}
catch (SemanticErrorException err)
{
this.error.Content = err.Message;
this.error.Visibility = Visibility.Visible;
this.editor.ScrollToLine(err.Start.Line);
var line = this.editor.Document.GetLineByNumber(err.Start.Line - 1);
this.editor.Select(line.Offset, line.Length);
}
catch (SyntaxErrorException err)
{
this.error.Content = err.Message;
this.error.Visibility = Visibility.Visible;
this.editor.ScrollToLine(err.Start.Line);
var line = this.editor.Document.GetLineByNumber(err.Start.Line - 1);
this.editor.Select(line.Offset, line.Length);
}
catch (Exception err)
{
this.error.Content = err.Message;
this.error.Visibility = Visibility.Visible;
}
}
public void TestNoBuiltinsReferenced()
{
var engine = new LucyEngine(new LucyDocument()
{
Entities = new List <EntityDefinition>()
{
}
}, useAllBuiltIns: false);
string text = "this is a http://foo.com?x=13&y=132123 cheese";
var results = engine.MatchEntities(text, null);
Trace.TraceInformation("\n" + LucyEngine.VisualEntities(text, results));
Assert.AreEqual(0, results.Count());
}
public void WildcardDoubleInlineOrdinalTest()
{
var engine = new LucyEngine(new LucyDocument()
{
Entities = new List <EntityDefinition>()
{
new EntityDefinition()
{
Name = "@name", Patterns = new List <Pattern>()
{
"name is (value:___)+"
}
},
new EntityDefinition()
{
Name = "@entity", Patterns = new List <Pattern>()
{
"end"
}
},
}
});
// stop on token
string text = "my name is joe smith and stuff";
var results = engine.MatchEntities(text, null);
Trace.TraceInformation("\n" + LucyEngine.VisualEntities(text, results));
var entities = results.Where(e => e.Type == "name").ToList();
Assert.AreEqual(1, entities.Count);
Assert.AreEqual("name", entities[0].Type);
Assert.AreEqual("joe smith and stuff", entities[0].Children.First().Resolution);
Assert.AreEqual("name is joe smith and stuff", text.Substring(entities[0].Start, entities[0].End - entities[0].Start));
// stop on entity
text = "my name is joe smith end token";
results = engine.MatchEntities(text, null);
Trace.TraceInformation("\n" + LucyEngine.VisualEntities(text, results));
entities = results.Where(e => e.Type == "name").ToList();
Assert.AreEqual(1, entities.Count);
Assert.AreEqual("name", entities[0].Type);
Assert.AreEqual("joe smith", entities[0].Children.First().Resolution);
Assert.AreEqual("name is joe smith", text.Substring(entities[0].Start, entities[0].End - entities[0].Start));
}
public void CreatesTextTokens()
{
var engine = new LucyEngine(new LucyDocument());
string text = "this is a test";
var results = engine.MatchEntities(text, includeInternal: true);
Trace.TraceInformation("\n" + LucyEngine.VisualEntities(text, results));
var entities = results.Where(e => e.Type == TokenPatternMatcher.ENTITYTYPE).ToList();
Assert.AreEqual(4, entities.Count);
Assert.AreEqual("this", entities[0].Text);
Assert.AreEqual("is", entities[1].Text);
Assert.AreEqual("a", entities[2].Text);
Assert.AreEqual("test", entities[3].Text);
}
public void TestAllBuiltIns()
{
var engine = new LucyEngine(new LucyDocument()
{
Entities = new List <EntityDefinition>()
{
// new EntityDefinition() { Name = "@test", Patterns = new List<Pattern>(){ "@url" } },
}
}, useAllBuiltIns: true);
string text = "this is a http://foo.com?x=13&y=132123 cheese";
var results = engine.MatchEntities(text, null);
Trace.TraceInformation("\n" + LucyEngine.VisualEntities(text, results));
Assert.AreEqual(1, results.Count());
Assert.AreEqual("http://foo.com?x=13&y=132123", results[0].Text);
}
private void LoadModel(string yaml)
{
// Trace.TraceInformation("Loading model");
engine = new LucyEngine(yaml, useAllBuiltIns: true);
// this.examplesBox.Text = sb.ToString();
if (engine.Warnings.Any())
{
this.Error = String.Join("\n", engine.Warnings);
this.alertBox.Show();
}
else
{
this.Error = string.Empty;
this.alertBox.Hide();
}
}
public void TestExternalEntities()
{
var engine = new LucyEngine(new LucyDocument()
{
ExternalEntities = new List <string>()
{
"url"
},
Entities = new List <EntityDefinition>()
{
}
}, useAllBuiltIns: false);
string text = "this is a http://foo.com?x=13&y=132123 cheese";
var results = engine.MatchEntities(text, null);
Assert.AreEqual(1, results.Count());
Assert.AreEqual("http://foo.com?x=13&y=132123", results[0].Text);
}
public void MergeNestedChildren()
{
var engine = new LucyEngine(new LucyDocument()
{
Entities = new List <EntityDefinition>()
{
new EntityDefinition()
{
Name = "@PizzaOrder", Patterns = new List <Pattern>()
{
"(@Topping)* (@AddToppings)* "
}
},
new EntityDefinition()
{
Name = "@AddToppings", Patterns = new List <Pattern>()
{
"(@ToppingQuantifier)* (@Topping)+"
}
},
new EntityDefinition()
{
Name = "@ToppingQuantifier", Patterns = new List <Pattern>()
{
"extra"
}
},
new EntityDefinition()
{
Name = "@Topping", Patterns = new List <Pattern>()
{
"Cheese"
}
},
}
});
string text = "extra cheese";
var results = engine.MatchEntities(text, null);
Trace.TraceInformation("\n" + LucyEngine.VisualEntities(text, results));
Assert.AreEqual(1, results[0].Children.Count());
}
public void WildcardMatcherWithOptionalTokenTests()
{
var engine = new LucyEngine(new LucyDocument()
{
Entities = new List <EntityDefinition>()
{
new EntityDefinition()
{
Name = "@test1", Patterns = new List <Pattern>()
{
"test1"
}
},
new EntityDefinition()
{
Name = "@test2", Patterns = new List <Pattern>()
{
"test2"
}
},
new EntityDefinition()
{
Name = "@test3", Patterns = new List <Pattern>()
{
"start (@test1|@test2|label:___)* end (label:___)* "
}
},
}
});
string text = "start test1 cool end more stuff";
var results = engine.MatchEntities(text, null);
Trace.TraceInformation("\n" + LucyEngine.VisualEntities(text, results));
var entities = results.Where(e => e.Type == "test3").ToList();
Assert.AreEqual(1, entities.Count);
Assert.AreEqual("test1", entities.First().Children.First().Text);
Assert.AreEqual("cool", entities.First().Children.Skip(1).First().Text);
Assert.AreEqual("more stuff", entities.First().Children.Skip(2).First().Text);
}
public void WildcardNamedTests()
{
var engine = new LucyEngine(new LucyDocument()
{
Macros = new Dictionary <string, string>()
{
{ "$is", "(is|equals)" },
},
Entities = new List <EntityDefinition>()
{
new EntityDefinition()
{
Name = "@size", Patterns = new List <Pattern>()
{
"small"
}
},
new EntityDefinition()
{
Name = "@drink",
Patterns = new List <Pattern>()
{
"a (@size)? (label:___)* cocktail"
}
},
}
});
string text = "a clyde mills cocktail.";
var results = engine.MatchEntities(text, null);
Trace.TraceInformation("\n" + LucyEngine.VisualEntities(text, results));
var entities = results.Where(e => e.Type == "drink").ToList();
Assert.AreEqual(1, entities.Count);
var entity = entities.Single().Children.Single();
Assert.AreEqual("label", entity.Type);
Assert.AreEqual("clyde mills", entity.Resolution);
}
public void TokenPatternMatcherWithEntityTests()
{
var engine = new LucyEngine(new LucyDocument()
{
Entities = new List <EntityDefinition>()
{
new EntityDefinition()
{
Name = "@test1", Patterns = new List <Pattern>()
{
"test1"
}
},
new EntityDefinition()
{
Name = "@test2", Patterns = new List <Pattern>()
{
"test2"
}
},
new EntityDefinition()
{
Name = "@test3", Patterns = new List <Pattern>()
{
"@test1 @test2"
}
},
}
});
string text = "nomatch test1 test2 nomatch";
var results = engine.MatchEntities(text, null);
Trace.TraceInformation("\n" + LucyEngine.VisualEntities(text, results));
var entities = results.Where(e => e.Type == "test3").ToList();
Assert.AreEqual(1, entities.Count);
Assert.AreEqual("test1 test2", text.Substring(entities[0].Start, entities[0].End - entities[0].Start));
}
public override IEnumerable <string> GenerateExamples(LucyEngine engine)
{
List <string> examples = new List <string>()
{
String.Empty
};
foreach (var pm in PatternMatchers)
{
List <string> newExamples = new List <string>();
foreach (var example in pm.GenerateExamples(engine))
{
foreach (var previousExample in examples)
{
newExamples.Add($"{previousExample} {example}".Trim());
}
}
examples = newExamples.Distinct().ToList();
}
return(examples);
}
public void CanonicalValuesTest()
{
var engine = new LucyEngine(new LucyDocument()
{
Entities = new List <EntityDefinition>()
{
new EntityDefinition()
{
Name = "@test",
Patterns = new List <Pattern>()
{
new string[] { "LAX", "los angeles" },
new string[] { "DSM", "(des moines)~" },
new string[] { "SEA", "seattle" },
new string[] { "OHR", "o'hare", "ohare" },
new string[] { "MID", "midway" },
}
}
}
});
string text = "flight from seattle to dez moiynes";
var results = engine.MatchEntities(text, null);
Trace.TraceInformation("\n" + LucyEngine.VisualEntities(text, results));
var entities = results.Where(e => e.Type == "test").ToList();
Assert.AreEqual(2, entities.Count);
Assert.AreEqual("test", entities[0].Type);
Assert.AreEqual("DSM", entities[0].Resolution);
Assert.AreEqual("dez moiynes", text.Substring(entities[0].Start, entities[0].End - entities[0].Start));
Assert.AreEqual("test", entities[1].Type);
Assert.AreEqual("SEA", entities[1].Resolution);
Assert.AreEqual("seattle", text.Substring(entities[1].Start, entities[1].End - entities[1].Start));
}