public void TestCFG_Overlaps()
{
var input =
@"S -> \( S \) | [abx] [bxd] [xde]
";
ContextFreeGrammar cfg = GrammarParser <BDD> .Parse(MkAutomaton, input);
Assert.IsNotNull(cfg.BuiltinTerminalAlgebra);
Assert.AreEqual("S", cfg.StartSymbol.Name);
Assert.AreEqual(1, cfg.NonterminalCount);
Assert.AreEqual(2, cfg.ProductionCount);
Assert.IsFalse(cfg.IsInGNF());
var aut = MkAutomaton(@"\((xx)+\)");
BDD[] witness;
var no = cfg.Overlaps <BDD>(aut, out witness);
Assert.IsFalse(no);
var aut2 = MkAutomaton(@"\(x+\)");
BDD[] witness2 = null;
var yes = cfg.Overlaps <BDD>(aut2, out witness2);
Assert.IsTrue(yes);
string concrete_witness = new string(Array.ConvertAll(witness2, GetChar));
Assert.AreEqual <string>("(xxx)", concrete_witness);
}
public CorrectSpellingsRescueMatch(TryToRescueMatch fallback, SpellingBeeWordComparer comparer, GrammarParser parser, double weight)
{
this.fallback = fallback;
this.comparer = comparer;
this.parser = parser;
this.weight = weight;
}
public static void LoadVariables(Context context, PluginEnvironment plugenv, double basesal, POSTagger tagger)
{
GrammarParser parser = new GrammarParser(plugenv);
context.Map.Add("%AtLocation", new ConceptRelation(basesal, plugenv, "AtLocation", tagger, parser));
context.Map.Add("%DefinedAs", new ConceptRelation(basesal, plugenv, "DefinedAs", tagger, parser));
}
public void Initialize(string config)
{
plugenv = new PluginEnvironment(this);
if (!File.Exists(config))
{
Console.WriteLine("Cannot find configuration file at " + config);
return;
}
plugenv.Initialize(config, new NameValueCollection());
tagger = new POSTagger(plugenv);
parser = new GrammarParser(plugenv);
coderack = new ZippyCoderack(this);
memory = new Memory();
basectx = new Context(coderack);
GrammarVariables.LoadVariables(basectx, 100.0, memory, plugenv);
OutputVariables.LoadVariables(basectx, 100.0, plugenv);
ProgramVariables.LoadVariables(basectx, 100.0, plugenv);
basectx.Map.Add("$Compare", new WordComparer());
initialized = true;
}
public static string ContentsCode(Context context, POSTagger tagger, GrammarParser parser)
{
StringBuilder result = new StringBuilder();
foreach (IContent content in context.Contents)
{
string name = content.Name;
if (content is Variable)
{
IParsedPhrase phrase = ((Variable)content).Produce(context, tagger, parser);
if (phrase != null)
{
name = phrase.Text;
}
}
if (name[0] == ' ')
{
result.Append(name.Substring(1));
}
else
{
if (result.Length > 0)
{
result.Append(" ");
}
result.Append(name);
}
}
return(result.ToString());
}
public void TestEbnfGrammarParsing()
{
var gp = new GrammarParser();
var grammar = gp.ReadEbnfGrammar(
@"letter = ""A"" | ""B"" | ""C"" | ""D"" | ""E"" | ""F"" | ""G""
| ""H"" | ""I"" | ""J"" | ""K"" | ""L"" | ""M"" | ""N""
| ""O"" | ""P"" | ""Q"" | ""R"" | ""S"" | ""T"" | ""U""
| ""V"" | ""W"" | ""X"" | ""Y"" | ""Z"" | ""a"" | ""b""
| ""c"" | ""d"" | ""e"" | ""f"" | ""g"" | ""h"" | ""i""
| ""j"" | ""k"" | ""l"" | ""m"" | ""n"" | ""o"" | ""p""
| ""q"" | ""r"" | ""s"" | ""t"" | ""u"" | ""v"" | ""w""
| ""x"" | ""y"" | ""z"" ;
digit = ""0"" | ""1"" | ""2"" | ""3"" | ""4"" | ""5"" | ""6"" | ""7"" | ""8"" | ""9"";
symbol = ""["" | ""]"" | ""{"" | ""}"" | ""("" | "")"" | ""<"" | "">""
| ""'"" | '""' | ""="" | ""|"" | ""."" | "","" | "";"";
character = letter | digit | symbol | ""_"";
identifier = letter , { letter | digit | ""_"" };
terminal = ""'"" , character , { character } , ""'""
| '""' , character , { character } , '""';
lhs = identifier;
rhs = identifier
| terminal
| ""["" , rhs , ""]""
| ""{"" , rhs , ""}""
| ""("" , rhs , "")""
| rhs , ""|"" , rhs
| rhs , "","" , rhs;
rule = lhs , ""="" , rhs , "";"";
grammar = { rule };");
}
public static void CorrectParse_WeakPrecedenceGrammar()
{
var rules = new[]
{
"S aASb",
"S d",
"A Ac",
"A c",
};
var grammar = GrammarParser.Parse(rules);
var expectedSRules = new []
{
new Rule(new NonTerminal('S'), new Terminal('a'), new NonTerminal('A'), new NonTerminal('S'), new Terminal('b')),
new Rule(new NonTerminal('S'), new Terminal('d')),
};
var expectedARules = new[]
{
new Rule(new NonTerminal('A'), new NonTerminal('A'), new Terminal('c')),
new Rule(new NonTerminal('A'), new Terminal('c')),
};
grammar[new NonTerminal('S')].Should().BeEquivalentTo(expectedSRules);
grammar[new NonTerminal('A')].Should().BeEquivalentTo(expectedARules);
}
public MatchProduceAgent(ArgumentMode argmode, double salience, int space, int time, POSTagger tagger, GrammarParser parser)
: base(argmode, salience, space, time)
{
this.tagger = tagger;
this.parser = parser;
this.breakpointCall = false;
}
public int Count()
{
if (Expression == "")
{
var ex = new Exception();
ex.Data.Add("Type", "reference to empty cell");
throw ex;
}
if (IsReevaluated)
{
return((int)Value);
}
try
{
var inputStream = new AntlrInputStream(Expression);
var lexer = new GrammarLexer(inputStream);
var commonTokenStream = new CommonTokenStream(lexer);
var parser = new GrammarParser(commonTokenStream);
parser.RemoveErrorListeners();
parser.AddErrorListener(new ParsingErrorListener());
var expr = parser.rule();
int val = (new ParsingVisitor(this)).Visit(expr);
IsReevaluated = true;
return(val);
}
catch
{
throw;
}
}
public IParsedPhrase ToPhrase(POSTagger tagger, GrammarParser parser)
{
if (name.Contains(" "))
{
List <IParsedPhrase> phrases = new List <IParsedPhrase>();
List <string> words = StringUtilities.SplitWords(name, true);
foreach (string word in words)
{
phrases.Add(new WordPhrase(word, "??"));
}
List <KeyValuePair <string, string> > tokens = tagger.ResolveUnknowns(phrases);
return(parser.Parse(tokens));
}
else
{
if (kind == Kind.Event)
{
return(new WordPhrase(name, "VB"));
}
if (kind == Kind.Attribute)
{
return(new WordPhrase(name, "JJ"));
}
// entity
return(new WordPhrase(name, "NN"));
}
}
public ParseTable(GrammarParser grammar)
{
first = grammar.FIFO.first;
follow = grammar.FIFO.follow;
this.grammar = grammar;
this.filler();
}
public static void LoadVariables(Context context, double salience, Memory memory, ConceptTranslator produceTranslator, Verbs verbs, Nouns nouns, PluginEnvironment plugenv)
{
POSTagger tagger = new POSTagger(plugenv);
GrammarParser parser = new GrammarParser(plugenv);
context.Map.Add("@know", new KnowRule(salience, memory, tagger, parser));
context.Map.Add("@event", new EventRule(salience, memory, tagger, parser));
context.Map.Add("@Subject", new SimpleDatumRule(salience, Relations.Relation.Subject, memory, produceTranslator, tagger, parser));
context.Map.Add("@Object", new SimpleDatumRule(salience, Relations.Relation.Object, memory, produceTranslator, tagger, parser));
context.Map.Add("@Indirect", new SimpleDatumRule(salience, Relations.Relation.Indirect, memory, produceTranslator, tagger, parser));
context.Map.Add("@IsA", new SimpleDatumRule(salience, Relations.Relation.IsA, memory, produceTranslator, tagger, parser));
context.Map.Add("@HasProperty", new SimpleDatumRule(salience, Relations.Relation.HasProperty, memory, produceTranslator, tagger, parser));
context.Map.Add("@Means", new SimpleDatumRule(salience, Relations.Relation.Means, memory, produceTranslator, tagger, parser));
context.Map.Add("@Condition", new SimpleDatumRule(salience, Relations.Relation.Condition, memory, produceTranslator, tagger, parser));
context.Map.Add("@MotivatedBy", new SimpleDatumRule(salience, Relations.Relation.MotivatedByGoal, memory, produceTranslator, tagger, parser));
context.Map.Add("@Exists", new SimpleDatumRule(salience, Relations.Relation.Exists, memory, produceTranslator, tagger, parser));
context.Map.Add("@UsedFor", new SimpleDatumRule(salience, Relations.Relation.UsedFor, memory, produceTranslator, tagger, parser));
context.Map.Add("@AtTime", new AtTimeRule(salience, memory, tagger, parser));
context.Map.Add("@InLocation", new InLocationRule(salience, memory, verbs, tagger, parser));
//context.Map.Add("@ActiveObjects", new AllObjectsRule(salience, true, memory, produceTranslator, tagger, parser));
//context.Map.Add("@PassiveObjects", new AllObjectsRule(salience, false, memory, produceTranslator, tagger, parser));
context.Map.Add("@EntityPrep", new EntityPrepRule(salience, memory, verbs, tagger, parser));
context.Map.Add("@SubjectTense", new TenseRule(salience, Relations.Relation.Subject, memory, verbs, nouns, tagger, parser));
context.Map.Add("%unknown", new UnknownConceptVariable("%unknown", null));
context.Map.Add("%unevent", new UnknownConceptVariable("%unevent", Concept.Kind.Event));
context.Map.Add("%unthing", new UnknownConceptVariable("%unthing", Concept.Kind.Entity));
context.Map.Add("%unquality", new UnknownConceptVariable("%unquality", Concept.Kind.Attribute));
}
public static void Main(string[] args)
{
PluginEnvironment plugenv = new PluginEnvironment(new MainClass());
string plugbase = "/Users/jrising/projects/virsona/github";
plugenv.Initialize(plugbase + "/config.xml", new NameValueCollection());
// Test 1: POS Tagging
POSTagger tagger = new POSTagger(plugenv);
List <KeyValuePair <string, string> > tagged = tagger.TagList(StringUtilities.SplitWords("This is a test.", false));
foreach (KeyValuePair <string, string> kvp in tagged)
{
Console.WriteLine(kvp.Key + ": " + kvp.Value);
}
// Test 2: Grammar parsing
GrammarParser parser = new GrammarParser(plugenv);
IParsedPhrase before = parser.Parse("This is a rug and a keyboard.");
Console.WriteLine(before.ToString());
// Test 5: Pluralize nouns and conjugate verbs
Nouns nouns = new Nouns(plugenv);
Console.WriteLine("person becomes " + nouns.Pluralize("person"));
Verbs verbs = new Verbs(plugenv);
Console.WriteLine("goes becomes " + verbs.ComposePast("goes"));
Console.WriteLine("eats becomes " + verbs.ComposePrespart(verbs.InputToBase("eats")));
// Test 3: Paraphrasing
Random randgen = new Random();
try {
IParsedPhrase after = parser.Paraphrase(before, null, null, randgen.NextDouble());
Console.WriteLine(after.Text);
} catch (Exception ex) {
Console.WriteLine("Error: " + ex.Message);
}
// Test 4: Look up some indices
WordNetAccess wordnet = new WordNetAccess(plugenv);
List <string> synonyms = null;
try {
synonyms = wordnet.GetExactSynonyms("rug", WordNetAccess.PartOfSpeech.Noun);
} catch (Exception ex) {
Console.WriteLine("Error: " + ex.Message);
}
if (synonyms == null)
{
Console.WriteLine("Could not find a synonym for 'rug'. Is Memcached installed?");
}
else
{
Console.WriteLine("Synonyms: " + string.Join(", ", synonyms.ToArray()));
}
}
/// <summary>
/// Prepare the parser
/// </summary>
private static void PrepareParser()
{
CommonTokenStream tokenStream = new CommonTokenStream(lexer);
parser = new GrammarParser(tokenStream);
parser.RemoveErrorListeners();
parser.AddErrorListener(new GrammarErrorListener(handler));
}
public void InGrammarParser_30xyz02za0bc_return_xyxy()
{
string expected = "xyzxyzxyzzazabc";
string actual = GrammarParser.Parse("3[xyz]2[za]bc");
Assert.AreEqual(expected, actual);
}
public void InGrammarParser_xyz_return_xzy()
{
string expected = "xyz";
string actual = GrammarParser.Parse("xyz");
Assert.AreEqual(expected, actual);
}
public void InGrammarParser_20x02y00_return_xyxy()
{
string expected = "xyyxyy";
string actual = GrammarParser.Parse("2[x2[y]]");
Assert.AreEqual(expected, actual);
}
public void GrammarGradingTest1() // balanced parenthesis (wordsInGrammar) !!!could change if grading scale is changed!!!
{
String sg1 = "S -> S S|(S)|";
ContextFreeGrammar g = GrammarParser <char> .Parse(f1, sg1);
var res = GrammarGrading.gradeWordsInGrammar(g, new[] { "()", "())()()(", "()((" }, new[] { "()(", "())()(", "xyz" }, 10);
Assert.IsTrue(res.Item1 == 5);
}
public static IEnumerable <string> GenerateFile(string file)
{
var name = Path.GetFileNameWithoutExtension(file);
var directory = Path.GetDirectoryName(file);
File representation;
using (var stream = System.IO.File.OpenRead(file))
{
var lexer = new GrammarLexer(new AntlrInputStream(stream));
var tokenStream = new CommonTokenStream(lexer);
var context = new GrammarParser(tokenStream).program();
var dsl = new Dsl(name);
dsl.Visit(context);
representation = dsl.File;
}
var outputs = representation.GenerateSource();
if (outputs.HasMessageContent)
{
if (outputs.HasMarkers)
{
var path = Path.Combine(directory, $"{representation.Name}MarkerInterfaces.cs");
System.IO.File.WriteAllText(path, outputs.Markers);
yield return(path);
}
if (outputs.HasMessages)
{
var path = Path.Combine(directory, $"{representation.Name}Messages.cs");
System.IO.File.WriteAllText(path, outputs.Messages);
yield return(path);
}
if (outputs.HasEnumerations)
{
var path = Path.Combine(directory, $"{representation.Name}Enums.cs");
System.IO.File.WriteAllText(path, outputs.Enumerations);
yield return(path);
}
}
if (outputs.HasStateDefinitions)
{
var path = Path.Combine(directory, $"{representation.Name}StateSubscriptions.cs");
System.IO.File.WriteAllText(path, outputs.StateDefinitions);
yield return(path);
}
if (outputs.HasEntities)
{
var path = Path.Combine(directory, $"{representation.Name}Entities.cs");
System.IO.File.WriteAllText(path, outputs.Entities);
yield return(path);
}
}
public static char[] ReturnCorrectLastSymbols_When(char nonTerminal, params string[] rules)
{
var grammar = GrammarParser.Parse(rules);
return(grammar.GetAllLastSymbols(new NonTerminal(nonTerminal))
.Select(c => c.Value)
.OrderBy(c => c)
.ToArray());
}
public static IEnumerable<string> GenerateFile(string file)
{
var name = Path.GetFileNameWithoutExtension(file);
var directory = Path.GetDirectoryName(file);
File representation;
using (var stream = System.IO.File.OpenRead(file))
{
var lexer = new GrammarLexer(new AntlrInputStream(stream));
var tokenStream = new CommonTokenStream(lexer);
var context = new GrammarParser(tokenStream).program();
var dsl = new Dsl(name);
dsl.Visit(context);
representation = dsl.File;
}
var outputs = representation.GenerateSource();
if (outputs.HasMessageContent)
{
if (outputs.HasMarkers)
{
var path = Path.Combine(directory, $"{representation.Name}MarkerInterfaces.cs");
System.IO.File.WriteAllText(path, outputs.Markers);
yield return path;
}
if (outputs.HasMessages)
{
var path = Path.Combine(directory, $"{representation.Name}Messages.cs");
System.IO.File.WriteAllText(path, outputs.Messages);
yield return path;
}
if (outputs.HasEnumerations)
{
var path = Path.Combine(directory, $"{representation.Name}Enums.cs");
System.IO.File.WriteAllText(path, outputs.Enumerations);
yield return path;
}
}
if (outputs.HasStateDefinitions)
{
var path = Path.Combine(directory, $"{representation.Name}StateSubscriptions.cs");
System.IO.File.WriteAllText(path, outputs.StateDefinitions);
yield return path;
}
if (outputs.HasEntities)
{
var path = Path.Combine(directory, $"{representation.Name}Entities.cs");
System.IO.File.WriteAllText(path, outputs.Entities);
yield return path;
}
}
/// <summary>
/// 解析指令
/// </summary>
/// <returns></returns>
public Result Parse()
{
parseResult = Result.FAILED;
if (grammarParser == null)
{
grammarParser = new GrammarParser(this);
}
parseResult = grammarParser.Parse();
return(parseResult);
}
public static bool PrintContents(Context context, IContinuation succ, IFailure fail, params object[] args)
{
PluginEnvironment plugenv = (PluginEnvironment)args[0];
POSTagger tagger = new POSTagger(plugenv);
GrammarParser parser = new GrammarParser(plugenv);
Console.WriteLine(StarUtilities.ProducedCode(context, tagger, parser));
succ.Continue(new Context(context, new List <IContent>()), fail);
return(true);
}
static void Main(string[] args)
{
//Adding translating tools for bools
var charStream = CharStreams.fromPath("firstProgram.txt");
var lexer = new GrammarLexer(charStream);
var tokenStream = new CommonTokenStream(lexer);
var parser = new GrammarParser(tokenStream)
{
BuildParseTree = true
};
var tree = parser.program();
var listener = new ScopeListener();
var ParseTreeWalker = new ParseTreeWalker();
ParseTreeWalker.Walk(listener, tree);
MainScopeClass.Scopes = listener.Scopes;
var visitor = new PrintVisitor();
visitor.VisitProgram(tree);
Console.WriteLine("Basic visitor done!");
var typeVisitor = new TypeCheckerVisitor();
typeVisitor.VisitProgram(tree);
Console.WriteLine("Typechecker visitor done!");
Console.WriteLine("Checking scopes:");
foreach (var Scope in listener.Scopes)
{
Console.WriteLine("Scope start:");
foreach (var element in Scope.SymbolTable)
{
Console.WriteLine(element);
}
}
Console.ReadLine();
}
private static List <String> getProductions(String src, GrammarParser grammar)
{
for (int i = 0; i < grammar.grammer.Count; i++)
{
if (grammar.grammer[i].source == src)
{
return(grammar.grammer[i].production);
}
}
return(null);
}
public static string ProducedCode(Context context, POSTagger tagger, GrammarParser parser)
{
List <IContent> produced = Produced(context, tagger, parser);
if (produced == null)
{
return(null);
}
return(ContentsCode(new Context(context, produced), tagger, parser));
}
public ConceptRelation(double salience, PluginEnvironment plugenv, string relation, POSTagger tagger, GrammarParser parser)
: base(ArgumentMode.ManyArguments, salience, 4, 100)
{
this.relation = relation;
this.tagger = tagger;
this.parser = parser;
// Look up the datasources for ConceptNet
principleSource = plugenv.GetDataSource <string, Notion>(ConceptNetUtilities.PrincipleSourceName);
assertionSource = plugenv.GetDataSource <KeyValuePair <Notion, string>, List <Assertion> >(ConceptNetUtilities.AssertionSourceName);
}
public void EqualityTest4() // (a|b|c|d|e|f|g|h|i|j)*
{
String sg1 = "S->|a|b|c|d|e|f|g|h|i|j|S S";
String sg2 = "S->X|X S| X->a|b|c|d|e|f|g|h|i|j";
ContextFreeGrammar g1 = GrammarParser <char> .Parse(f1, sg1);
ContextFreeGrammar g2 = GrammarParser <char> .Parse(f1, sg2);
var res = GrammarUtilities.findDifferenceWithTimelimit(g1, g2, true, 50);
Assert.IsTrue(res.Item2.Count == 0 && res.Item3.Count == 0);
}
public void TestCFG_Parser_WithDummyAutomaton()
{
var input = @"
START -> AS BS
AS -> AS (a) (a) | (a) (a) | (@)
BS -> (b) (b) (b) BS | (b) (b) (b)";
TestCFGParser_validate(input);
var input2 = GrammarParser <string> .Parse(MapTerminalToDummyAutomaton, input).ToString();
TestCFGParser_validate(input2);
}
public void EqualityTest6() // balanced parenthesis
{
String sg1 = "S -> S S|(S)|";
String sg2 = "X -> | X X | (L L->X) | X X)";
ContextFreeGrammar g1 = GrammarParser <char> .Parse(f1, sg1);
ContextFreeGrammar g2 = GrammarParser <char> .Parse(f1, sg2);
var res = GrammarUtilities.findDifferenceWithTimelimit(g1, g2, true, 100);
Assert.IsTrue(res.Item2.Count == 0 && res.Item3.Count == 0);
}
public void EqualityTest5() // (a|b)^n
{
String sg1 = "S -> a|b|aa|aS|bS|bbbbS";
String sg2 = "S->X|X S X->a|b";
ContextFreeGrammar g1 = GrammarParser <char> .Parse(f1, sg1);
ContextFreeGrammar g2 = GrammarParser <char> .Parse(f1, sg2);
var res = GrammarUtilities.findDifferenceWithTimelimit(g1, g2, true, 100);
Assert.IsTrue(res.Item2.Count == 0 && res.Item3.Count == 0);
}
