public IExtractor CreateExtractor()
{
var tokenizer = Tokenizer;
var loggerFactory = _loggerFactory;
var gramRep = GrammarRepository;
var gramCompiler = new GrammarCompiler(tokenizer, _morph, _extensions);
var startTerminalsCreator = new StartTerminalsCreator(_settings);
var srcGrams = new List <(string key, string src)>();
if (!string.IsNullOrWhiteSpace(_settings.MainGrammar))
{
srcGrams.Add((key: "main", src: _settings.MainGrammar));
}
if (!string.IsNullOrWhiteSpace(_settings.GrammarsDirPath))
{
srcGrams.AddRange(gramRep.GetAll());
}
var grams = _getGrammars(loggerFactory, srcGrams, _settings.Language);
var rules = gramCompiler.Compile(grams, _settings.Variables);
var startRules = startTerminalsCreator.Create(rules);
var parserLogger = loggerFactory?.CreateLogger <IChartParser>();
var parser = new EarleyParser(startRules, parserLogger);
var extractorLogger = loggerFactory?.CreateLogger <IExtractor>();
var extractor = new Extractor(tokenizer, _morph, parser, _settings, extractorLogger);
return(extractor);
}
public void SimpleParenTest()
{
var grammar = new Grammar <ParenAlphabet>();
var ruleParen =
new Rule <ParenAlphabet>
{
Name = "paren",
Lhs = ParenAlphabet.Expr,
Rhs = Concat <ParenAlphabet>(
ParenAlphabet.L.ToRegex(),
ParenAlphabet.Expr.ToRegex(),
ParenAlphabet.R.ToRegex())
};
var ruleX =
new Rule <ParenAlphabet> {
Name = "X", Lhs = ParenAlphabet.Expr, Rhs = ParenAlphabet.X.ToRegex()
};
grammar.Rules = new List <Rule <ParenAlphabet> >()
{
ruleParen,
ruleX
};
grammar.StartSymbol = ParenAlphabet.Expr;
var compiledGrammar = GrammarCompiler <ParenAlphabet> .CompileGrammar(grammar);
var follow = FollowHelper <ParenAlphabet> .GetFollowSymbols(compiledGrammar, null, null, ParenAlphabet.EOF);
foreach (var c in follow)
{
Console.WriteLine("state: " + c.Key.Dfa.Label(c.Key.State) + " follow: " + string.Join(",", c.Value));
}
}
public void BuildGrammar()
{
try
{
Parser = GrammarCompiler.CreateParser(Grammar);
}
catch (Exception ex)
{
LastException = ex;
throw ex;
}
}
public void TestInterpreterOnFolder()
{
Interpreter interpreter = new Interpreter();
interpreter.AddFilesFromDirectory(RESOURCE_TEST_FOLDER);
interpreter.Compile();
// Test modules.
Assert.AreEqual(3, interpreter.ModuleMap.Count);
Assert.IsTrue(interpreter.ModuleMap.ContainsKey("outer"));
Assert.IsTrue(interpreter.ModuleMap.ContainsKey("package.test"));
Assert.IsTrue(interpreter.ModuleMap.ContainsKey("package.inner.inner"));
// Test imports.
Module module;
ReadOnlyDictionary <string, Tuple <string, bool> > moduleImportDict;
module = interpreter.ModuleMap["outer"];
moduleImportDict = module.LoadedImports;
Assert.AreEqual(1, moduleImportDict.Count);
Assert.IsTrue(moduleImportDict.ContainsKey("package"));
Assert.AreEqual("", moduleImportDict["package"].Item1);
Assert.IsTrue(moduleImportDict["package"].Item2);
module = interpreter.ModuleMap["package.test"];
moduleImportDict = module.LoadedImports;
Assert.AreEqual(1, moduleImportDict.Count);
Assert.IsTrue(moduleImportDict.ContainsKey("package.inner"));
Assert.AreEqual("yes", moduleImportDict["package.inner"].Item1);
Assert.IsFalse(moduleImportDict["package.inner"].Item2);
// Test compilation.
module = interpreter.ModuleMap["package.test"]; // Redundant, but futureproofing.
Assert.AreEqual("a* (b{3} c+ | (d | e{2,9}) [f]) ((g)) h{3,}", module.LoadedGrammarNodes["test"].ToString());
GrammarBuilder gb = GrammarCompiler.CompileToGrammarBuilder(module.LoadedGrammarNodes["test"]);
Assert.AreEqual("‘a’ [‘b’ ‘c’,[‘d’,‘e’] [‘f’]] [[‘g’]] ‘h’", gb.DebugShowPhrases);
}
public void HarderParenTest()
{
var grammar = new Grammar <ParenAlphabet>
{
Rules = new List <Rule <ParenAlphabet> >()
{
new Rule <ParenAlphabet>
{
Name = "paren",
Lhs = ParenAlphabet.Expr,
Rhs = Concat(
ParenAlphabet.L.ToRegex(),
ParenAlphabet.Expr.ToRegex(),
ParenAlphabet.S.ToRegex(),
ParenAlphabet.R.ToRegex())
},
new Rule <ParenAlphabet> {
Name = "X", Lhs = ParenAlphabet.Expr, Rhs = ParenAlphabet.X.ToRegex()
},
new Rule <ParenAlphabet>
{
Name = "SY", Lhs = ParenAlphabet.S, Rhs = ParenAlphabet.Y.ToRegex().Optional()
},
},
StartSymbol = ParenAlphabet.Expr
};
var compiledGrammar = GrammarCompiler <ParenAlphabet> .CompileGrammar(grammar);
var follow = FollowHelper <ParenAlphabet> .GetFollowSymbols(compiledGrammar, null, null, ParenAlphabet.EOF);
foreach (var c in follow)
{
Console.WriteLine("state: " + c.Key.Dfa.Label(c.Key.State) + " follow: " + string.Join(",", c.Value));
}
}
