protected internal virtual Collection compileGrammar()
{
this.initialGrammarState = this.grammar.getInitialNode();
this.nodeStateMap = new HashMap();
this.arcPool = new Cache();
ArrayList arrayList = new ArrayList();
TimerPool.getTimer(this, "Compile").start();
TimerPool.getTimer(this, "Create States").start();
Iterator iterator = this.grammar.getGrammarNodes().iterator();
while (iterator.hasNext())
{
GrammarNode grammarNode = (GrammarNode)iterator.next();
FlatLinguist.GState gstate = this.createGState(grammarNode);
arrayList.add(gstate);
}
TimerPool.getTimer(this, "Create States").stop();
this.addStartingPath();
TimerPool.getTimer(this, "Collect Contexts").start();
iterator = arrayList.iterator();
while (iterator.hasNext())
{
FlatLinguist.GState gstate2 = (FlatLinguist.GState)iterator.next();
gstate2.collectContexts();
}
TimerPool.getTimer(this, "Collect Contexts").stop();
TimerPool.getTimer(this, "Expand States").start();
iterator = arrayList.iterator();
while (iterator.hasNext())
{
FlatLinguist.GState gstate2 = (FlatLinguist.GState)iterator.next();
gstate2.expand();
}
TimerPool.getTimer(this, "Expand States").stop();
TimerPool.getTimer(this, "Connect Nodes").start();
iterator = arrayList.iterator();
while (iterator.hasNext())
{
FlatLinguist.GState gstate2 = (FlatLinguist.GState)iterator.next();
gstate2.connect();
}
TimerPool.getTimer(this, "Connect Nodes").stop();
SentenceHMMState sentenceHMMState = this.findStartingState();
if (this.addOutOfGrammarBranch)
{
CIPhoneLoop ciphoneLoop = new CIPhoneLoop(this.phoneLoopAcousticModel, this.logPhoneInsertionProbability);
SentenceHMMState nextState = (SentenceHMMState)ciphoneLoop.getSearchGraph().getInitialState();
sentenceHMMState.connect(this.getArc(nextState, 0f, this.logOutOfGrammarBranchProbability));
}
this.searchGraph = new FlatLinguist.FlatSearchGraph(this, sentenceHMMState);
TimerPool.getTimer(this, "Compile").stop();
if (this.dumpGStates)
{
Iterator iterator2 = this.grammar.getGrammarNodes().iterator();
while (iterator2.hasNext())
{
GrammarNode node = (GrammarNode)iterator2.next();
FlatLinguist.GState gstate3 = this.getGState(node);
gstate3.dumpInfo();
}
}
this.nodeStateMap = null;
this.arcPool = null;
return(SentenceHMMState.collectStates(sentenceHMMState));
}
public EBNF()
{
LanguageName = "EBNF";
ScannerNames = new List<string>();
ScannerProductions = new Dictionary<string, GrammarNode>();
GrammarNode name = new GrammarNode(GrammarNodeType.SEQUENCE, "", MatchType.RECURSE);
GrammarNode name_initial = new GrammarNode(GrammarNodeType.TERMINAL, "[a-z]", MatchType.REGEX);
GrammarNode name_subsequent = new GrammarNode(GrammarNodeType.TERMINAL, "[a-z0-9_]", MatchType.REGEX, GrammarNodeTypeQuantifier.ZERO_OR_MORE);
name.Add(name_initial);
name.Add(name_subsequent);
// TODO: Reserved words require lookahead for robust performance.
//GrammarNode environment_name = new GrammarNode(GrammarNodeType.ALTERNATION, "", MatchType.RECURSE);
//GrammarNode scanner_name = Language.ToSequence("scanner", GrammarNodeTypeQuantifier.ONE);
//GrammarNode parser_name = Language.ToSequence("parser", GrammarNodeTypeQuantifier.ONE);
//environment_name.Add(scanner_name);
//environment_name.Add(parser_name);
// On hold pending decision on and implementation of lookahead.
//GrammarNode ignore_production_name = ToSequence("ignore", GrammarNodeTypeQuantifier.ONE);
GrammarNode assignment = new GrammarNode(GrammarNodeType.SEQUENCE, "", MatchType.RECURSE);
GrammarNode colon = new GrammarNode(GrammarNodeType.TERMINAL, ":", MatchType.VALUE);
GrammarNode equal = new GrammarNode(GrammarNodeType.TERMINAL, "=", MatchType.VALUE);
assignment.Add(colon);
assignment.Add(equal);
GrammarNode whitespace = new GrammarNode(GrammarNodeType.TERMINAL, @"\s", MatchType.REGEX, GrammarNodeTypeQuantifier.ONE_OR_MORE);
// regex:= /\/([^\/]|(?<\\)\/)+\//;
GrammarNode regex = new GrammarNode(GrammarNodeType.SEQUENCE, "", MatchType.RECURSE);
GrammarNode regex_initial = new GrammarNode(GrammarNodeType.TERMINAL, "/", MatchType.VALUE);
GrammarNode regex_terminal = new GrammarNode(GrammarNodeType.TERMINAL, "/", MatchType.VALUE);
GrammarNode regex_piece = new GrammarNode(GrammarNodeType.ALTERNATION, "", MatchType.RECURSE, GrammarNodeTypeQuantifier.ONE_OR_MORE);
GrammarNode not_slashes = new GrammarNode(GrammarNodeType.TERMINAL, @"[^\\/]", MatchType.REGEX, GrammarNodeTypeQuantifier.ONE);
GrammarNode escape = new GrammarNode(GrammarNodeType.SEQUENCE, "", MatchType.RECURSE, GrammarNodeTypeQuantifier.ONE);
GrammarNode regex_backslash = new GrammarNode(GrammarNodeType.TERMINAL, @"\", MatchType.VALUE, GrammarNodeTypeQuantifier.ONE);
GrammarNode regex_thing_escaped = new GrammarNode(GrammarNodeType.TERMINAL, @"\S", MatchType.REGEX, GrammarNodeTypeQuantifier.ONE);
// assemble production 'regex'
regex.Add(regex_initial);
regex.Add(regex_piece);
regex.Add(regex_terminal);
// assemble label 'regex_piece'
regex_piece.Add(escape);
regex_piece.Add(not_slashes);
// assemble label 'escape'
escape.Add(regex_backslash); // escape sequences start with a backslash...
escape.Add(regex_thing_escaped); // ...and are followed by the thing escaped.
GrammarNode terminator = new GrammarNode(GrammarNodeType.TERMINAL, ";", MatchType.VALUE);
GrammarNode ebnf_string = new GrammarNode(GrammarNodeType.SEQUENCE, "", MatchType.RECURSE);
GrammarNode quote = new GrammarNode(GrammarNodeType.TERMINAL, "\"", MatchType.VALUE);
//string_element:= /[^\\"]/ | /\\\"/| /\\\\/;
GrammarNode string_element = new GrammarNode(GrammarNodeType.ALTERNATION, "", MatchType.RECURSE, GrammarNodeTypeQuantifier.ZERO_OR_MORE);
GrammarNode not_slash_or_quote = new GrammarNode(GrammarNodeType.TERMINAL, @"[^\\" + "\"]", MatchType.REGEX);
string_element.Add(not_slash_or_quote);
string_element.Add(escape);
ebnf_string.Add(quote);
ebnf_string.Add(string_element);
ebnf_string.Add(quote);
GrammarNode quantifier = new GrammarNode(GrammarNodeType.ALTERNATION, "", MatchType.RECURSE);
GrammarNode zero_or_one = new GrammarNode(GrammarNodeType.TERMINAL, "?", MatchType.VALUE);
GrammarNode zero_or_more = new GrammarNode(GrammarNodeType.TERMINAL, "*", MatchType.VALUE);
GrammarNode one_or_more = new GrammarNode(GrammarNodeType.TERMINAL, "+", MatchType.VALUE);
quantifier.Add(zero_or_more);
quantifier.Add(zero_or_one);
quantifier.Add(one_or_more);
GrammarNode comment = new GrammarNode(GrammarNodeType.SEQUENCE, "", MatchType.RECURSE);
GrammarNode slash = new GrammarNode(GrammarNodeType.TERMINAL, "/", MatchType.VALUE);
comment.Add(slash);
comment.Add(slash);
GrammarNode not_newline = new GrammarNode(GrammarNodeType.TERMINAL, @"[^\n\r]", MatchType.REGEX, GrammarNodeTypeQuantifier.ZERO_OR_MORE);
comment.Add(not_newline);
GrammarNode pipe = new GrammarNode(GrammarNodeType.TERMINAL, "|", MatchType.VALUE);
GrammarNode language_name = new GrammarNode(GrammarNodeType.SEQUENCE, "", MatchType.RECURSE, GrammarNodeTypeQuantifier.ONE);
GrammarNode language_sigil = new GrammarNode(GrammarNodeType.TERMINAL, "%", MatchType.VALUE, GrammarNodeTypeQuantifier.ONE);
GrammarNode Zor1whitespace = new GrammarNode(GrammarNodeType.ALTERNATION, "", MatchType.RECURSE, GrammarNodeTypeQuantifier.ZERO_OR_ONE);
Zor1whitespace.Add(whitespace);
GrammarNode mixed_name = new GrammarNode(GrammarNodeType.SEQUENCE, "", MatchType.RECURSE);
GrammarNode mixed_initial = new GrammarNode(GrammarNodeType.TERMINAL, "[a-zA-Z]", MatchType.REGEX);
GrammarNode mixed_subsequent = new GrammarNode(GrammarNodeType.TERMINAL, "[a-zA-Z0-9_]", MatchType.REGEX, GrammarNodeTypeQuantifier.ZERO_OR_MORE);
// assemble label 'mixed_name'
mixed_name.Add(mixed_initial);
mixed_name.Add(mixed_subsequent);
// assemble production 'language_name'
language_name.Add(language_sigil);
language_name.Add(language_sigil);
language_name.Add(Zor1whitespace);
language_name.Add(mixed_name);
GrammarNode environment_name = GrammarNode.SequenceNode();
environment_name.Add(language_sigil);
environment_name.Add(Zor1whitespace);
environment_name.Add(name);
GrammarNode eof = new GrammarNode(GrammarNodeType.TERMINAL, "EOF", MatchType.EOF);
AddProduction("scanner", "environment_name", environment_name);
AddProduction("scanner", "name", name);
AddProduction("scanner", "assignment", assignment);
AddProduction("scanner", "whitespace", whitespace);
AddProduction("scanner", "regex", regex);
AddProduction("scanner", "terminator", terminator);
AddProduction("scanner", "string", ebnf_string);
AddProduction("scanner", "quantifier", quantifier);
AddProduction("scanner", "comment", comment);
AddProduction("scanner", "pipe", pipe);
AddProduction("scanner", "language_name", language_name);
AddProduction("scanner", "EOF", eof);
// The scanner won't send these off to the parser.
this.Ignore = new Dictionary<string, string>();
Ignore.Add("whitespace", "whitespace");
Ignore.Add("comment", "comment");
ParserNames = new List<string>();
ParserProductions = new Dictionary<string, GrammarNode>();
//language_specifier:= language_name;
GrammarNode P_language_spec = new GrammarNode(GrammarNodeType.TERMINAL, "language_name", MatchType.TYPE);
GrammarNode P_environment_name = new GrammarNode(GrammarNodeType.TERMINAL, "environment_name", MatchType.TYPE);
//production:= production_name assignment sequence alternation_subsequent* terminator;
GrammarNode production = new GrammarNode(GrammarNodeType.SEQUENCE, "", MatchType.RECURSE);
GrammarNode P_name = GrammarNode.TerminalNodeByType("name");
GrammarNode P_assignment = GrammarNode.TerminalNodeByType("assignment");
GrammarNode P_sequence = new GrammarNode(GrammarNodeType.SEQUENCE, "", MatchType.RECURSE);
GrammarNode P_alternation_subsequent = new GrammarNode(GrammarNodeType.SEQUENCE, "", MatchType.RECURSE, GrammarNodeTypeQuantifier.ZERO_OR_MORE);
GrammarNode P_terminator = new GrammarNode(GrammarNodeType.TERMINAL, "terminator", MatchType.TYPE);
// assemble production 'production'
production.Add(P_name);
production.Add(P_assignment);
production.Add(P_sequence);
production.Add(P_alternation_subsequent);
production.Add(P_terminator);
//sequence:= sequence_atom+;
GrammarNode P_sequence_atom = new GrammarNode(GrammarNodeType.ALTERNATION, "", MatchType.RECURSE, GrammarNodeTypeQuantifier.ONE_OR_MORE);
P_sequence.Add(P_sequence_atom);
//sequence_atom:= quantified_name | quantified_string | regex;
GrammarNode P_quantified_name = GrammarNode.SequenceNode();
GrammarNode P_quantified_string = new GrammarNode(GrammarNodeType.SEQUENCE, "", MatchType.RECURSE);
GrammarNode P_regex = GrammarNode.TerminalNodeByType("regex");
P_sequence_atom.Add(P_quantified_name);
P_sequence_atom.Add(P_quantified_string);
P_sequence_atom.Add(P_regex);
//quantified_name:= name quantifier?;
GrammarNode P_quantifier = GrammarNode.TerminalNodeByType("quantifier");
P_quantifier.GNTQuantifier = GrammarNodeTypeQuantifier.ZERO_OR_ONE;
P_quantified_name.Add(P_name);
P_quantified_name.Add(P_quantifier);
//quantified_string:= string quantifier?;
GrammarNode P_string = GrammarNode.TerminalNodeByType("string");
P_quantified_string.Add(P_string);
P_quantified_string.Add(P_quantifier);
//alternation_subsequent:= pipe sequence;
GrammarNode P_pipe = GrammarNode.TerminalNodeByType("pipe");
P_alternation_subsequent.Add(P_pipe);
P_alternation_subsequent.Add(P_sequence);
AddProduction("parser", "production", production);
AddProduction("parser", "language_specifier", P_language_spec);
AddProduction("parser", "environment_specifier", P_environment_name);
AddProduction("parser", "EOF", eof);
}
internal GrammarState(AFlatLinguist aflatLinguist, GrammarNode grammarNode, float num, int num2) : this(aflatLinguist, grammarNode, num, num2, 0)
{
}
public void AddParent(GrammarNode _NewParent)
{
Parents.Add(_NewParent);
}
/// <inheritdoc />
protected override GrammarNode <Char>?VisitAlternation(Alternation <Char> alternation, OptimizeArgs argument)
{
var nodes = alternation.GrammarNodes.Select(node => this.Visit(node, argument))
.Where(node => node is not null)
.ToList();
var matchedCharTerminals = new HashSet <Char>();
var matchedStringTerminals = new HashSet <String>();
var matchedCharRanges = new HashSet <Range <Char> >();
var matchedUnicodeCategories = new HashSet <UnicodeCategory>();
var matchedNodes = new HashSet <GrammarNode <Char> >(GrammarTreeStructuralComparer.Instance);
for (var nodeIdx = 0; nodeIdx < nodes.Count; nodeIdx++)
{
loopStart:
GrammarNode <Char> currentNode = nodes[nodeIdx] !;
if (matchedNodes.Contains(currentNode))
{
nodes.RemoveAt(nodeIdx);
goto loopStart;
}
switch (currentNode.Kind)
{
case GrammarNodeKind.CharacterStringTerminal:
{
var strTerminal = (StringTerminal)currentNode;
if (matchedStringTerminals.Contains(strTerminal.Value))
{
nodes.RemoveAt(nodeIdx);
goto loopStart;
}
matchedStringTerminals.Add(strTerminal.Value);
break;
}
case GrammarNodeKind.CharacterTerminal:
{
var ch = ((CharacterTerminal)currentNode).Value;
if (isCharMatched(ch))
{
nodes.RemoveAt(nodeIdx);
goto loopStart;
}
matchedCharTerminals.Add(ch);
break;
}
case GrammarNodeKind.CharacterRange:
{
var characterRange = (CharacterRange)currentNode;
var newStart = characterRange.Range.Start;
while (isCharMatched(newStart) &&
newStart <= characterRange.Range.End)
{
newStart++;
}
var newEnd = characterRange.Range.End;
while (isCharMatched(newEnd) &&
newEnd >= newStart)
{
newEnd--;
}
// The new start will be greater than the new end if all characters in the
// range are already being matched.
if (newStart > newEnd)
{
nodes.RemoveAt(nodeIdx);
goto loopStart;
}
else if (newStart == newEnd)
{
nodes[nodeIdx] = new CharacterTerminal(newStart);
goto loopStart;
}
else if (characterRange.Range.Start != newStart || characterRange.Range.End != newEnd)
{
nodes[nodeIdx] =
currentNode =
characterRange = new CharacterRange(newStart, newEnd);
}
break;
}
}
matchedNodes.Add(currentNode);
}
if (nodes.Count == 0)
{
return(null);
}
else if (nodes.Count == 1)
{
return(nodes[0]);
}
else if (nodes.SequenceEqual(alternation.GrammarNodes))
{
return(alternation);
}
else
{
return(new Alternation <Char>(nodes !));
}
Boolean isCharMatched(Char ch) =>
matchedCharTerminals !.Contains(ch) ||
matchedCharRanges !.Any(range => CharUtils.IsInRange(range.Start, ch, range.End)) ||
matchedUnicodeCategories !.Contains(Char.GetUnicodeCategory(ch));
}
// private GrammarArc[] successors = null;
/// <summary>
/// Creates a GState for a grammar node
/// </summary>
/// <param name="node">the grammar node</param>
/// <param name="parent"></param>
public GState(GrammarNode node, FlatLinguist parent)
{
_parent = parent;
this._node = node;
_parent.NodeStateMap.Put(node, this);
}
/// <summary>
/// Returns a new GState for the given GrammarNode.
/// </summary>
/// <param name="grammarNode"></param>
/// <returns>a new GState for the given GrammarNode</returns>
protected GState CreateGState(GrammarNode grammarNode)
{
return(new GState(grammarNode, this));
}
/// <inheritdoc />
protected override GrammarNode <Char>?VisitSequence(Sequence <Char> sequence, OptimizeArgs argument)
{
var nodes = sequence.GrammarNodes.Select(node => this.Visit(node, argument))
.Where(node => node is not null)
.ToList();
for (var nodeIdx = 0; nodeIdx < nodes.Count; nodeIdx++)
{
loopStart:
GrammarNode <Char> currentNode = nodes[nodeIdx] !;
if (nodeIdx < nodes.Count - 1)
{
GrammarNode <Char> nextNode = nodes[nodeIdx + 1] !;
if (currentNode is CharacterTerminal currentCharacterTerminal)
{
if (nextNode is CharacterTerminal nextCharacterTerminal)
{
nodes.RemoveAt(nodeIdx);
nodes[nodeIdx] = new StringTerminal(new String(new[] { currentCharacterTerminal.Value, nextCharacterTerminal.Value }));
goto loopStart;
}
else if (nextNode is StringTerminal nextStringTerminal)
{
nodes.RemoveAt(nodeIdx);
nodes[nodeIdx] = new StringTerminal(currentCharacterTerminal.Value + nextStringTerminal.Value);
goto loopStart;
}
}
else if (currentNode is StringTerminal currentStringTerminal)
{
if (nextNode is CharacterTerminal nextCharacterTerminal)
{
nodes.RemoveAt(nodeIdx);
nodes[nodeIdx] = new StringTerminal(currentStringTerminal.Value + nextCharacterTerminal.Value);
goto loopStart;
}
else if (nextNode is StringTerminal nextStringTerminal)
{
nodes.RemoveAt(nodeIdx);
nodes[nodeIdx] = new StringTerminal(currentStringTerminal.Value + nextStringTerminal.Value);
goto loopStart;
}
}
}
}
if (nodes.Count == 0)
{
return(null);
}
else if (nodes.Count == 1)
{
return(nodes[0]);
}
else if (nodes.SequenceEqual(sequence.GrammarNodes))
{
return(sequence);
}
else
{
return(new Sequence <Char>(nodes !));
}
}
public void ProcessRule()
{
// If there is no rule, then why process one?
if (Terminal || Rule == null)
{
return;
}
GrammarNode[] LocalNodes = new GrammarNode[Rule.ReplacementNodes.Length];
GrammarNode AttachParentTo = null;
GrammarNode AttachChildrenTo = null;
// Create the nodes to replace this one
for (int i = 0; i < Rule.ReplacementNodes.Length; ++i)
{
GrammarRule.NodeInfo NodeInfo = Rule.ReplacementNodes[i];
GameObject tempObject = new GameObject();
tempObject.AddComponent <GrammarNode>();
tempObject.GetComponent <GrammarNode>().Init();
tempObject.GetComponent <GrammarNode>().RelativePosition = NodeInfo.Offset;
tempObject.GetComponent <GrammarNode>().Symbol = NodeInfo.Symbol;
tempObject.name = tempObject.GetComponent <GrammarNode>().Symbol.ToString();
//Add Cellular Automata
tempObject.AddComponent <CellularAutomata>();
tempObject.GetComponent <CellularAutomata>().Floor = MasterDungeon.FloorTile;
if (Parents.Count == 0)
{
tempObject.transform.position = tempObject.GetComponent <GrammarNode>().RelativePosition;
}
else
{
tempObject.transform.position = Parents[0].transform.position + tempObject.GetComponent <GrammarNode>().RelativePosition;
}
if (NodeInfo.AttachParent && AttachParentTo == null)
{
AttachParentTo = tempObject.GetComponent <GrammarNode>();
}
if (NodeInfo.AttachChild && AttachChildrenTo == null)
{
AttachChildrenTo = tempObject.GetComponent <GrammarNode>();
}
LocalNodes[i] = tempObject.GetComponent <GrammarNode>();
}
// Connect those nodes together
for (int i = 0; i < Rule.Connections.Length; ++i)
{
GrammarDungeon.NodeConnection NewConnect;
NewConnect.Node1 = LocalNodes[Rule.Connections[i].p1];
NewConnect.Node2 = LocalNodes[Rule.Connections[i].p2];
NewConnect.p1 = MasterDungeon.GetNodeID() + Rule.Connections[i].p1 + 1;
NewConnect.p2 = MasterDungeon.GetNodeID() + Rule.Connections[i].p2 + 1;
MasterDungeon.AddConnection(NewConnect);
LocalNodes[Rule.Connections[i].p1].AddChild(LocalNodes[Rule.Connections[i].p2]);
LocalNodes[Rule.Connections[i].p2].AddParent(LocalNodes[Rule.Connections[i].p1]);
}
// Add Nodes to Dungeon List
for (int i = 0; i < LocalNodes.Length; ++i)
{
MasterDungeon.AddNode(LocalNodes[i]);
}
// Hook up the parent nodes
foreach (GrammarNode _Parent in Parents)
{
AttachParentTo.AddParent(_Parent);
MasterDungeon.AddConnection(_Parent, AttachParentTo);
_Parent.RemoveChild(this);
_Parent.AddChild(AttachParentTo);
}
// Hook up the children nodes
foreach (GrammarNode _Child in Children)
{
AttachChildrenTo.AddChild(_Child);
MasterDungeon.AddConnection(AttachChildrenTo, _Child);
_Child.RemoveParent(this);
_Child.AddParent(AttachChildrenTo);
}
//Get rid of the original node
AttachParentTo.GetComponent <GrammarNode>().RelativePosition = this.RelativePosition;
MasterDungeon.RemoveConnection(this);
MasterDungeon.RemoveNode(this);
Destroy(this.gameObject);
}
/// <summary>
/// Compiles the grammar into a sentence HMM. A GrammarJob is created for the
/// initial grammar node and added to the GrammarJob queue. While there are
/// jobs left on the grammar job queue, a job is removed from the queue and
/// the associated grammar node is expanded and attached to the tails.
/// GrammarJobs for the successors are added to the grammar job queue.
/// </summary>
/// <returns></returns>
protected HashSet <SentenceHMMState> CompileGrammar()
{
InitialGrammarState = Grammar.InitialNode;
NodeStateMap = new HashMap <GrammarNode, GState>();
// create in declaration section (22.12.2014)
ArcPool = new Cache <SentenceHMMStateArc>();
var gstateList = new List <GState>();
TimerPool.GetTimer(this, "Compile").Start();
// get the nodes from the grammar and create states
// for them. Add the non-empty gstates to the gstate list.
TimerPool.GetTimer(this, "Create States").Start();
foreach (var grammarNode in Grammar.GrammarNodes)
{
var gstate = CreateGState(grammarNode);
gstateList.Add(gstate);
}
TimerPool.GetTimer(this, "Create States").Stop();
AddStartingPath();
// ensures an initial path to the start state
// Prep all the gstates, by gathering all of the contexts up
// this allows each gstate to know about its surrounding contexts
TimerPool.GetTimer(this, "Collect Contexts").Start();
foreach (var gstate in gstateList)
{
gstate.CollectContexts();
}
TimerPool.GetTimer(this, "Collect Contexts").Stop();
// now all gstates know all about their contexts, we can expand them fully
TimerPool.GetTimer(this, "Expand States").Start();
foreach (var gstate in gstateList)
{
gstate.Expand();
}
TimerPool.GetTimer(this, "Expand States").Stop();
// now that all states are expanded fully, we can connect all the states up
TimerPool.GetTimer(this, "Connect Nodes").Start();
foreach (var gstate in gstateList)
{
gstate.Connect();
}
TimerPool.GetTimer(this, "Connect Nodes").Stop();
var initialState = FindStartingState();
// add an out-of-grammar branch if configured to do so
if (AddOutOfGrammarBranch)
{
var phoneLoop = new CIPhoneLoop(PhoneLoopAcousticModel, LogPhoneInsertionProbability);
var firstBranchState = (SentenceHMMState)phoneLoop.GetSearchGraph().InitialState;
initialState.Connect(GetArc(firstBranchState, LogOne, LogOutOfGrammarBranchProbability));
}
_searchGraph = new FlatSearchGraph(initialState);
TimerPool.GetTimer(this, "Compile").Stop();
// Now that we are all done, dump out some interesting
// information about the process
if (_dumpGStates)
{
foreach (var grammarNode in Grammar.GrammarNodes)
{
var gstate = GetGState(grammarNode);
gstate.DumpInfo();
}
}
NodeStateMap = null;
ArcPool = null;
return(SentenceHMMState.CollectStates(initialState));
}
public void RemoveChild(GrammarNode _ChildToRemove)
{
Children.Remove(_ChildToRemove);
}
public void AddChild(GrammarNode _NewChild)
{
Children.Add(_NewChild);
}
public void RemoveParent(GrammarNode _ParentToRemove)
{
Parents.Remove(_ParentToRemove);
}
protected bool AddProduction(string EnvironmentName, string ProductionName, GrammarNode Production)
{
Dictionary<string, GrammarNode> ProductionDictToFill;
List<string> NameListToFill;
switch (EnvironmentName)
{
case "scanner":
ProductionDictToFill = ScannerProductions;
NameListToFill = ScannerNames;
break;
case "parser":
ProductionDictToFill = ParserProductions;
NameListToFill = ParserNames;
break;
default:
throw new Exception("Unknown environment name: " + EnvironmentName);
}
if (NameListToFill.Contains(ProductionName))
{
return false;
}
NameListToFill.Add(ProductionName);
ProductionDictToFill.Add(ProductionName, Production);
return true;
}
/**
* /// Given a grammar node, retrieve the grammar state
*
* /// @param node the grammar node
* /// @return the grammar state associated with the node
*/
protected GState GetGState(GrammarNode node)
{
return(NodeStateMap[node]);
}
public TestLanguage()
{
this.LanguageName = "Lispish";
// The scanner will understand tokens in the sentence "(one foo baz )", for example.
// Incidentally, it will also understand them in "())( zab (((()", but that's not the scanner's problem.
ScannerProductions = new Dictionary<string, GrammarNode>();
ScannerNames = new List<string>();
GrammarNode SPName = new GrammarNode(GrammarNodeType.TERMINAL, "[a-z]", MatchType.REGEX);
SPName.GNTQuantifier = GrammarNodeTypeQuantifier.ONE_OR_MORE;
AddProduction("scanner", "name", SPName);
GrammarNode SPLParen = new GrammarNode(GrammarNodeType.TERMINAL, "(", MatchType.VALUE);
SPLParen.GNTQuantifier = GrammarNodeTypeQuantifier.ONE;
AddProduction("scanner", "lparen", SPLParen);
GrammarNode SPRParen = new GrammarNode(GrammarNodeType.TERMINAL, ")", MatchType.VALUE);
SPLParen.GNTQuantifier = GrammarNodeTypeQuantifier.ONE;
AddProduction("scanner", "rparen", SPRParen);
GrammarNode SPWhiteSpace = new GrammarNode(GrammarNodeType.ALTERNATION, "", MatchType.RECURSE);
SPWhiteSpace.GNTQuantifier = GrammarNodeTypeQuantifier.ONE_OR_MORE;
GrammarNode SPSpace = new GrammarNode(GrammarNodeType.TERMINAL, " ", MatchType.VALUE);
SPSpace.GNTQuantifier = GrammarNodeTypeQuantifier.ONE;
GrammarNode SPReturn = new GrammarNode(GrammarNodeType.TERMINAL, "\r", MatchType.REGEX);
SPReturn.GNTQuantifier = GrammarNodeTypeQuantifier.ONE;
GrammarNode SPNewline = new GrammarNode(GrammarNodeType.TERMINAL, "\n", MatchType.REGEX);
SPNewline.GNTQuantifier = GrammarNodeTypeQuantifier.ONE;
SPWhiteSpace.Add(SPSpace);
SPWhiteSpace.Add(SPReturn);
SPWhiteSpace.Add(SPNewline);
AddProduction("scanner", "whitespace", SPWhiteSpace);
GrammarNode SPEOF = new GrammarNode(GrammarNodeType.TERMINAL, "EOF", MatchType.TYPE);
SPEOF.GNTQuantifier = GrammarNodeTypeQuantifier.ONE;
AddProduction("scanner", "EOF", SPEOF);
// The parser will understand sentences of the form "(foo baz bar one )", for example.
// It will also understand "(foo (baz bar) one )". That is, recursive lists of simple alphabetic string atoms.
ParserProductions = new Dictionary<string, GrammarNode>();
ParserNames = new List<string>();
//GrammarNode PWhitespace = new GrammarNode(GrammarNodeType.TERMINAL, "whitespace", MatchType.TYPE);
//PWhitespace.GNTQuantifier = GrammarNodeTypeQuantifier.ONE;
//GrammarNode PWSZeroOrOne = new GrammarNode(GrammarNodeType.TERMINAL, "whitespace", MatchType.TYPE);
//PWSZeroOrOne.GNTQuantifier = GrammarNodeTypeQuantifier.ZERO_OR_ONE;
GrammarNode PLBegin = new GrammarNode(GrammarNodeType.SEQUENCE, "", MatchType.RECURSE);
PLBegin.GNTQuantifier = GrammarNodeTypeQuantifier.ONE;
GrammarNode PLParen = new GrammarNode(GrammarNodeType.TERMINAL, "lparen", MatchType.TYPE);
PLParen.GNTQuantifier = GrammarNodeTypeQuantifier.ONE;
PLBegin.Add(PLParen);
//PLBegin.Add(PWSZeroOrOne);
GrammarNode PName = new GrammarNode(GrammarNodeType.TERMINAL, "name", MatchType.TYPE);
PName.GNTQuantifier = GrammarNodeTypeQuantifier.ONE;
GrammarNode PList = new GrammarNode(GrammarNodeType.SEQUENCE, "", MatchType.RECURSE);
PList.GNTQuantifier = GrammarNodeTypeQuantifier.ONE;
GrammarNode PListElement = new GrammarNode(GrammarNodeType.ALTERNATION, "", MatchType.RECURSE);
PListElement.GNTQuantifier = GrammarNodeTypeQuantifier.ZERO_OR_MORE;
PListElement.Add(PName);
PListElement.Add(PList);
//GrammarNode PConsequent = new GrammarNode(GrammarNodeType.SEQUENCE, "", MatchType.RECURSE);
//PConsequent.GNTQuantifier = GrammarNodeTypeQuantifier.ZERO_OR_MORE;
//PConsequent.Add(PWhitespace);
//PConsequent.Add(PInitial);
GrammarNode PLEnd = new GrammarNode(GrammarNodeType.SEQUENCE, "", MatchType.RECURSE);
PLEnd.GNTQuantifier = GrammarNodeTypeQuantifier.ONE;
GrammarNode PRParen = new GrammarNode(GrammarNodeType.TERMINAL, "rparen", MatchType.TYPE);
PRParen.GNTQuantifier = GrammarNodeTypeQuantifier.ONE;
//PLEnd.Add(PWSZeroOrOne);
PLEnd.Add(PRParen);
PList.Add(PLBegin);
PList.Add(PListElement);
//PList.Add(PConsequent);
PList.Add(PLEnd);
GrammarNode EOF = new GrammarNode(GrammarNodeType.TERMINAL, "EOF", MatchType.EOF);
//AddProduction("parser","whitespace", PWhitespace);
AddProduction("parser", "list", PList);
AddProduction("parser", "EOF", EOF);
// The scanner will suppress whitespace, which will simplify parsing.
Ignore = new Dictionary<string, string>();
Ignore.Add("whitespace", "whitespace"); // It's only the key that matters.
}
/**
* Creates a GrammarNode with the word in the given RuleToken.
*
* @param ruleToken
* the RuleToken that contains the word
* @return a GrammarNode with the word in the given RuleToken
*/
private GrammarGraph ProcessRuleToken(JSGFRuleToken ruleToken)
{
GrammarNode node = CreateGrammarNode(ruleToken.Text);
return(new GrammarGraph(node, node, this));
}
protected override GrammarNode <Char>?VisitNegatedSet(NegatedSet negatedSet, OptimizeArgs argument)
{
var characters = negatedSet.Characters.ToList();
var ranges = negatedSet.Ranges.ToList();
var categories = negatedSet.UnicodeCategories.ToList();
// We don't want inner sets to be optimized since we'll flatten them.
OptimizeArgs childrenArgument = argument.WithIsParentASet(true);
List <GrammarNode <Char> > nodes =
negatedSet.Nodes.Select(node => this.Visit(node, childrenArgument))
.Where(node => node is not null)
.ToList() !;
var nodeIdx = 0;
while (true)
{
loopStart:
if (nodeIdx >= nodes.Count)
{
break;
}
GrammarNode <Char> node = nodes[nodeIdx] !;
switch (node.Kind)
{
case GrammarNodeKind.CharacterTerminal:
{
var characterTerminal = (CharacterTerminal)node;
nodes.RemoveAt(nodeIdx);
characters.Add(characterTerminal.Value);
goto loopStart;
}
case GrammarNodeKind.CharacterRange:
{
var characterRange = (CharacterRange)node;
nodes.RemoveAt(nodeIdx);
ranges.Add(characterRange.Range);
goto loopStart;
}
case GrammarNodeKind.CharacterUnicodeCategoryTerminal:
{
var unicodeCategoryTerminal = (UnicodeCategoryTerminal)node;
nodes.RemoveAt(nodeIdx);
categories.Add(unicodeCategoryTerminal.Category);
goto loopStart;
}
case GrammarNodeKind.CharacterSet:
{
var subSet = (Set)node;
nodes.RemoveAt(nodeIdx);
characters.AddRange(subSet.Characters);
ranges.AddRange(subSet.Ranges);
categories.AddRange(subSet.UnicodeCategories);
nodes.AddRange(subSet.Nodes);
goto loopStart;
}
}
nodeIdx++;
}
characters.Sort();
OptimizationAlgorithms.ExpandRanges(characters, ranges, true);
OptimizationAlgorithms.RangifyCharacters(characters, ranges, true);
OptimizationAlgorithms.MergeRanges(ranges);
ImmutableArray <Char> flattenedRanges = CharUtils.FlattenRanges(ranges);
var categoriesFlagSet = CharUtils.CreateCategoryFlagSet(categories);
OptimizationAlgorithms.RemoveMatchedCharacters(characters, flattenedRanges, categoriesFlagSet);
// Characters are still sorted at this point
CharacterBitVector?characterBitVector = null;
if (characters.Any())
{
var charactersDistance = characters[characters.Count - 1] - characters[0];
if (charactersDistance is > 1 and <= 256)
{
characterBitVector = new CharacterBitVector(characters);
}
}
return(new OptimizedNegatedSet(
characters.ToImmutableHashSet(),
flattenedRanges,
categoriesFlagSet,
nodes.ToImmutableArray(),
characterBitVector));
}
protected internal virtual FlatLinguist.GState createGState(GrammarNode grammarNode)
{
return(new FlatLinguist.GState(this, grammarNode));
}
/// <summary>
/// Creates a grammar graph with the given nodes.
/// </summary>
/// <param name="startNode">The staring node of the graph.</param>
/// <param name="endNode">The ending node of the graph.</param>
/// <param name="parent">The parent.</param>
internal GrammarGraph(GrammarNode startNode, GrammarNode endNode, JSGFGrammar parent)
{
_parent = parent;
StartNode = startNode;
EndNode = endNode;
}
protected internal virtual FlatLinguist.GState getGState(GrammarNode node)
{
return((FlatLinguist.GState) this.nodeStateMap.get(node));
}
internal GrammarState(AFlatLinguist aflatLinguist, GrammarNode grammarNode) : this(aflatLinguist, grammarNode, 0f, UnitManager.__SILENCE.getBaseID())
{
}
protected internal virtual void addStartingPath(GrammarNode initialNode)
{
FlatLinguist.GState gstate = this.getGState(initialNode);
FlatLinguist.GState.access_000(gstate, UnitContext.SILENCE);
}
private bool hasEntryContext(GrammarNode grammarNode, int unitID)
{
Set set = (Set)AFlatLinguist.access_900(this.this_0).get(grammarNode);
return(set.contains(AFlatLinguist.access_800(this.this_0).getUnit(unitID)));
}
public void AddChild(GrammarNode node) => Children.Add(node);