private static void AdjustSpan(TerminalNode comment, TerminalNode nodeToAdjust)
{
var min = Math.Min(nodeToAdjust.Span.Start, comment.Span.Start);
var max = Math.Max(nodeToAdjust.Span.End, comment.Span.End);
nodeToAdjust.Span = new CharacterSpan(min, max);
}
public ExtraIdOrAssignmentRule(TerminalNode comma, IdOrAssignmentRule idOrAssignment,
ExtraIdOrAssignmentRule extraIdOrAssign)
{
Comma = Guard.OneOf(() => comma, Token.Comma);
IdOrAssignment = Guard.NonNull(() => idOrAssignment);
ExtraIdOrAssign = extraIdOrAssign;
}
public ExtraConditionRule(TerminalNode booleanOperator, ConditionRule condition,
ExtraConditionRule extraCondition)
{
BooleanOperator = Guard.OneOf(() => booleanOperator, TokensGroups.BooleanOperators);
Condition = Guard.NonNull(() => condition);
ExtraCondition = extraCondition;
}
FusionCore.DSASM.Operand buildOperand(TerminalNode node)
{
FusionCore.DSASM.Operand op = new FusionCore.DSASM.Operand();
op.optype = getOpType((FusionCore.PrimitiveType)node.type);
if (FusionCore.DSASM.AddressType.VarIndex == op.optype)
{
//FusionCore.DSASM.SymbolNode
int size = symbols.symbolList.Count;
for (int n = 0; n < size; ++n)
{
// TODO jun: Hash the string
if (functionindex == symbols.symbolList[n].functionIndex && symbols.symbolList[n].name == node.Value)
{
op.opdata = n; // symbols.symbolList[n].index;
break;
}
}
}
else if (FusionCore.DSASM.AddressType.Int == op.optype)
{
op.opdata = System.Convert.ToInt32(node.Value);
}
else if (FusionCore.DSASM.AddressType.Register == op.optype)
{
op.opdata = (int)FusionCore.DSASM.Registers.RX;
}
else
{
Debug.Assert(false);
}
return(op);
}
private static ContainerOrTerminalNode GetNodeToAdjust(Container parent, TerminalNode comment, int index, int first, int last)
{
var containers = parent.Children;
if (index == first && index == last)
{
return(parent);
}
if (index == first)
{
return(containers[index + 1]);
}
if (index == last)
{
return(containers[index - 1]);
}
var next = containers[index + 1];
var before = containers[index - 1];
var trailing = comment.LocationSpan.Start.LineNumber == before.LocationSpan.End.LineNumber;
return(trailing ? before : next);
}
public ExtraEquationRule(TerminalNode arithmeticOperator, EquationRule equation,
ExtraEquationRule extraEquation)
{
ArithmeticOperator = Guard.OneOf(() => arithmeticOperator, TokensGroups.ArithmeticOperators);
Equation = Guard.NonNull(() => equation);
ExtraEquation = extraEquation;
}
public FunctionBodyRule(TerminalNode leftCurlyBracket, ICollection <StatementRule> statements,
ReturnStatementRule returnStatement, TerminalNode rightCurlyBracket)
{
LeftCurlyBracket = Guard.OneOf(() => leftCurlyBracket, Token.CurlyBracketLeft);
Statements = statements;
ReturnStatement = Guard.NonNull(() => returnStatement);
RightCurlyBracket = Guard.OneOf(() => rightCurlyBracket, Token.CurlyBracketRight);
}
public AssignmentStatementRule(TerminalNode identifier, TerminalNode assignmentOperator,
ExpressionRule expression, TerminalNode semicolon)
{
Identifier = Guard.OneOf(() => identifier, Token.Identifier);
AssignmentOperator = Guard.OneOf(() => assignmentOperator, Token.Assignment);
Expression = expression;
Semicolon = Guard.OneOf(() => semicolon, Token.Semicolon);
}
public FunctionCallRule(TerminalNode identifier, TerminalNode parenthesisLeft,
ArgumentsRule arguments, TerminalNode parenthesisRight)
{
Identifier = Guard.OneOf(() => identifier, Token.Identifier);
ParenthesisLeft = Guard.OneOf(() => parenthesisLeft, Token.ParenthesisLeft);
Arguments = arguments;
ParenthesisRight = Guard.OneOf(() => parenthesisRight, Token.ParenthesisRight);
}
public EquationRule(TerminalNode parenthesisLeft, EquationRule equation,
TerminalNode parenthesisRight, ExtraEquationRule extraEquation)
{
ParenthesisLeft = Guard.OneOf(() => parenthesisLeft, Token.ParenthesisLeft);
Equation = Guard.NonNull(() => equation);
ParenthesisRight = Guard.OneOf(() => parenthesisRight, Token.ParenthesisRight);
ExtraEquation = extraEquation;
}
public DeclarationStatementRule(TerminalNode dataType, IdOrAssignmentRule idOrAssignment,
ExtraIdOrAssignmentRule extraIdOrAssignment, TerminalNode semicolon)
{
DataType = Guard.OneOf(() => dataType, TokensGroups.DataTypes);
IdOrAssignment = Guard.NonNull(() => idOrAssignment);
ExtraIdOrAssignment = extraIdOrAssignment;
Semicolon = Guard.OneOf(() => semicolon, Token.Semicolon);
}
public RepeatStatementRule(TerminalNode repeat, ICollection <StatementRule> statements,
TerminalNode until, ConditionStatementRule conditionStatement)
{
Repeat = Guard.OneOf(() => repeat, Token.Repeat);
Statements = statements;
Until = Guard.OneOf(() => until, Token.Until);
ConditionStatement = Guard.NonNull(() => conditionStatement);
}
private static void AdjustRoot(Container root, TerminalNode node)
{
var rootStart = node.LocationSpan.Start;
// adjust positions
root.LocationSpan = new LocationSpan(rootStart, root.LocationSpan.End);
root.HeaderSpan = new CharacterSpan(node.Span.Start, root.HeaderSpan.End);
}
public ElseIfStatementRule(TerminalNode elseIf, ConditionStatementRule conditionStatement, TerminalNode then,
ICollection <StatementRule> statements, ExtraElseIfRule extraElseIf)
{
ElseIf = Guard.OneOf(() => elseIf, Token.ElseIf);
ConditionStatement = Guard.NonNull(() => conditionStatement);
Then = Guard.OneOf(() => then, Token.Then);
Statements = statements;
ExtraElseIf = Guard.NonNull(() => extraElseIf);
}
protected internal override void VisitTerminalNode(SnapshotBuilder builder, TerminalNode node)
{
if (builder.IsVisited(node))
{
return;
}
builder.AddNode(node, NodeInfo.Create);
builder.AddLink(node, node.RuleNode);
base.VisitTerminalNode(builder, node);
}
public FunctionDeclarationRule(TerminalNode dataType, TerminalNode functionName,
TerminalNode leftParenthesis, ParametersRule parameters,
TerminalNode rightParenthesis)
{
DataType = Guard.OneOf(() => dataType, TokensGroups.DataTypes);
FunctionName = Guard.OneOf(() => functionName, Token.Identifier);
LeftParenthesis = Guard.OneOf(() => leftParenthesis, Token.ParenthesisLeft);
Parameters = parameters;
RightParenthesis = Guard.OneOf(() => rightParenthesis, Token.ParenthesisRight);
}
public MainFunctionRule(TerminalNode dataType, TerminalNode main,
TerminalNode leftParenthesis, TerminalNode rightParenthesis,
FunctionBodyRule functionBody)
{
DataType = Guard.OneOf(() => dataType, TokensGroups.DataTypes);
Main = Guard.OneOf(() => main, Token.Main);
LeftParenthesis = Guard.OneOf(() => leftParenthesis, Token.ParenthesisLeft);
RightParenthesis = Guard.OneOf(() => rightParenthesis, Token.ParenthesisRight);
FunctionBody = Guard.NonNull(() => functionBody);
}
public void EnterEveryRule(ParserRuleContext ctx)
{
string ruleName = ruleNames[ctx.RuleIndex];
IList <TerminalNode> tnodes = getDirectTerminalChildren(ctx);
// Find all ordered unique pairs of literals;
// no (a,a) pairs and only literals like '{', 'begin', '}', ...
// Add a for (a,a) into ruleToRepeatedTokensSet for later filtering
RuleAltKey ruleAltKey = new RuleAltKey(ruleName, ctx.getAltNumber());
for (int i = 0; i < tnodes.Count; i++)
{
for (int j = i + 1; j < tnodes.Count; j++)
{
TerminalNode a = tnodes[i];
TerminalNode b = tnodes[j];
int atype = a.Symbol.Type;
int btype = b.Symbol.Type;
// KED: THIS CODE DOES NOT WORK WITH GRAMMARS CONTAINING FRAGMENTS.
// IN ANTLRV4LEXER.G4 THAT IS IN THIS DIRECTORY, THE GRAMMAR DOES NOT USE
// FRAGMENT FOR COLON. BUT THE G4 GRAMMAR IN THE ANTLR GRAMMARS-G4 EXAMPLES,
// IT DOES. CONSEQUENTLY, GETLITERALNAME RETURNS NULL!
// FRAGMENTS AREN'T PART OF THE VOCABULARY, SO THIS CODE DOES NOT WORK!!
// only include literals like '{' and ':' not IDENTIFIER etc...
if (vocab.GetLiteralName(atype) == null || vocab.GetLiteralName(btype) == null)
{
continue;
}
if (atype == btype)
{
ISet <int> repeatedTokensSet = null;
if (!ruleToRepeatedTokensSet.TryGetValue(ruleAltKey, out repeatedTokensSet))
{
repeatedTokensSet = new HashSet <int>();
ruleToRepeatedTokensSet[ruleAltKey] = repeatedTokensSet;
}
repeatedTokensSet.Add(atype);
}
else
{
org.antlr.codebuff.misc.Pair <int, int> pair = new org.antlr.codebuff.misc.Pair <int, int>(atype, btype);
ISet <org.antlr.codebuff.misc.Pair <int, int> > pairsBag = null;
if (!ruleToPairsBag.TryGetValue(ruleAltKey, out pairsBag))
{
pairsBag = new HashSet <org.antlr.codebuff.misc.Pair <int, int> >();
ruleToPairsBag[ruleAltKey] = pairsBag;
}
pairsBag.Add(pair);
}
}
}
}
public void TerminalNode_WithString_ReturnsCorrectString()
{
// Arranged:
var node = new TerminalNode(new Token(TokenKind.String, "a"), _tracer);
// Act:
var actual = node.ToString();
// Assert:
Assert.That(actual, Is.EqualTo("\"a\""));
}
public FinalNode(TerminalNode data)
{
instructionBytes = data.Text.Bytes;
if (data.Text != null)
{
dataString_Renamed = data.Text;
}
else if (data.Symbol.Text != null)
{
dataString_Renamed = data.Symbol.Text;
}
}
public void SaveTerminalData()
{
CreateTerminalNode();
for (int i = 0; i < terminals.Length; i++)
{
TerminalNode terminal = new TerminalNode(GetXmlNode(terminals[i]));
TerminalInformation terminalInformation = SingletonManager.MainTerminalController.GetTerminalInformation(i);
terminal.SetActivated(terminalInformation.isActivated);
terminal.SetCollected(terminalInformation.isCollected);
}
}
public void initializeProject()
{
this.Model.Clear();
this.Nodes.Clear();
BaseNode.Controller = this;
ConnectorNode.Controller = this;
this.terminalS = new TerminalNode(TerminalNode.TerminalType.Start);
this.terminalE = new TerminalNode(TerminalNode.TerminalType.End);
this.terminalS.attachNode(terminalE);
this.terminalE.ParentNode = terminalS;
this.terminalS.addToModel();
this.terminalE.addToModel();
}
private static void AdjustLocationSpan(TerminalNode comment, ContainerOrTerminalNode nodeToAdjust)
{
var commentStart = comment.LocationSpan.Start;
var commentEnd = comment.LocationSpan.End;
var nodeStart = nodeToAdjust.LocationSpan.Start;
var nodeEnd = nodeToAdjust.LocationSpan.End;
var start = nodeStart < commentStart ? nodeStart : commentStart;
var end = nodeEnd < commentEnd ? commentEnd : nodeEnd;
nodeToAdjust.LocationSpan = new LocationSpan(start, end);
}
private static void Adjust(TerminalNode comment, ContainerOrTerminalNode nodeToAdjust)
{
AdjustLocationSpan(comment, nodeToAdjust);
if (nodeToAdjust is TerminalNode t)
{
AdjustSpan(comment, t);
}
else if (nodeToAdjust is Container c)
{
AdjustSpan(comment, c);
}
}
/// <summary>
/// Creates an expression node and adds it to the
/// give tree.
/// </summary>
/// <param name="root">Root node of expression tree</param>
/// <param name="term">Term for this node</param>
/// <param name="termForm">Indicates form of this term</param>
/// <param name="termExclude">Indicates if this is an excluded term</param>
/// <param name="conjunction">Conjunction used to join with other nodes</param>
/// <returns>The new root node</returns>
protected INode AddNode(INode root, string term, TermForms termForm, bool termExclude, ConjunctionTypes conjunction)
{
if (term.Length > 0)
{
INode node = new TerminalNode
{
Term = term,
TermForm = termForm,
Exclude = termExclude
};
root = AddNode(root, node, conjunction);
}
return(root);
}
//Parses the name of a variable, or throws an exception is token is not valid ID
private Node ParseVarName()
{
var valid = CurrentToken.tokenType == TokenTypes.identity;
if (!valid)
{
throw new ParserException("Line " + CurrentToken.lineNumber + ": Expected identifier; found " + CurrentToken.token);
}
else
{
var node = new TerminalNode(CurrentToken);
Consume();
return(node);
}
}
//Parses boolean, or throws an exception
private Node ParseBool()
{
var valid = CurrentToken.tokenType == TokenTypes.boolean;
if (!valid)
{
throw new ParserException("Line " + CurrentToken.lineNumber + ": Expected boolean; found " + CurrentToken.token);
}
else
{
var node = new TerminalNode(CurrentToken);
Consume();
return(node);
}
}
private static TerminalNode AddTerminalNode(Container parent, string type, string name, string content, LocationSpan locationSpan, CharacterSpan span)
{
var child = new TerminalNode
{
Type = type,
Name = name,
Content = content,
LocationSpan = locationSpan,
Span = span,
};
parent.Children.Add(child);
return(child);
}
public Project_Saver(Controller controller, TerminalNode start, TerminalNode end, String path)
{
Controller = controller;
stringBuilder.Append(begin);
stringBuilder.Append(getBlockXML(start, null));
stringBuilder.Append(endStr);
XmlString = stringBuilder.ToString();
try
{
File.WriteAllText(path, xmlString);
}
catch (Exception ex) {
Controller.showErrorMsg("File Not Found");
}
}
public virtual TokenPositionAnalysis getTokenAnalysis(int[] features, int[] featuresForAlign, int tokenIndexInStream, int injectNL_WS, int alignOrIndent, bool collectAnalysis)
{
CommonToken curToken = (CommonToken)originalDoc.tokens.Get(tokenIndexInStream);
TerminalNode nodeWithOriginalToken = originalTokenToNodeMap[curToken];
int actualWS = Trainer.getInjectWSCategory(originalTokens, tokenIndexInStream);
string actualWSNL = getWSCategoryStr(actualWS);
actualWSNL = !string.ReferenceEquals(actualWSNL, null) ? actualWSNL : string.Format("{0,8}", "none");
string wsDisplay = getWSCategoryStr(injectNL_WS);
if (string.ReferenceEquals(wsDisplay, null))
{
wsDisplay = string.Format("{0,8}", "none");
}
string alignDisplay = getHPosCategoryStr(alignOrIndent);
if (string.ReferenceEquals(alignDisplay, null))
{
alignDisplay = string.Format("{0,8}", "none");
}
string newlinePredictionString = string.Format("### line {0:D}: predicted {1} actual {2}", curToken.Line, wsDisplay, actualWSNL);
int actualAlignCategory = Trainer.getAlignmentCategory(originalDoc, nodeWithOriginalToken, indentSize);
string actualAlignDisplay = getHPosCategoryStr(actualAlignCategory);
actualAlignDisplay = !string.ReferenceEquals(actualAlignDisplay, null) ? actualAlignDisplay : string.Format("{0,8}", "none");
string alignPredictionString = string.Format("### line {0:D}: predicted {1} actual {2}", curToken.Line, alignDisplay, actualAlignDisplay);
string newlineAnalysis = "";
string alignAnalysis = "";
if (collectAnalysis)
{ // this can be slow
newlineAnalysis = newlinePredictionString + "\n" + wsClassifier.getPredictionAnalysis(testDoc, k, features, corpus.injectWhitespace, Trainer.MAX_WS_CONTEXT_DIFF_THRESHOLD);
if ((injectNL_WS & 0xFF) == Trainer.CAT_INJECT_NL)
{
alignAnalysis = alignPredictionString + "\n" + hposClassifier.getPredictionAnalysis(testDoc, k, featuresForAlign, corpus.hpos, Trainer.MAX_ALIGN_CONTEXT_DIFF_THRESHOLD);
}
}
TokenPositionAnalysis a = new TokenPositionAnalysis(curToken, injectNL_WS, newlineAnalysis, alignOrIndent, alignAnalysis);
a.actualWS = Trainer.getInjectWSCategory(originalTokens, tokenIndexInStream);
a.actualAlign = actualAlignCategory;
return(a);
}
/// <summary>
/// Creates an expression node and adds it to the
/// give tree.
/// </summary>
/// <param name="root">Root node of expression tree</param>
/// <param name="term">Term for this node</param>
/// <param name="termForm">Indicates form of this term</param>
/// <param name="termExclude">Indicates if this is an excluded term</param>
/// <param name="conjunction">Conjunction used to join with other nodes</param>
/// <returns>The new root node</returns>
protected INode AddNode(INode root, string term, TermForms termForm, bool termExclude, ConjunctionTypes conjunction)
{
if (term.Length > 0)
{
INode node = new TerminalNode
{
Term = term,
TermForm = termForm,
Exclude = termExclude
};
root = AddNode(root, node, conjunction);
}
return root;
}
FusionCore.DSASM.Operand buildOperand(TerminalNode node)
{
FusionCore.DSASM.Operand op = new FusionCore.DSASM.Operand();
op.optype = getOpType((FusionCore.PrimitiveType)node.type);
if (FusionCore.DSASM.AddressType.VarIndex == op.optype)
{
//FusionCore.DSASM.SymbolNode
int size = symbols.symbolList.Count;
for (int n = 0; n < size; ++n)
{
// TODO jun: Hash the string
if (functionindex == symbols.symbolList[n].functionIndex && symbols.symbolList[n].name == node.Value)
{
op.opdata = n; // symbols.symbolList[n].index;
break;
}
}
}
else if (FusionCore.DSASM.AddressType.Int == op.optype)
{
op.opdata = System.Convert.ToInt32(node.Value);
}
else if (FusionCore.DSASM.AddressType.Register == op.optype)
{
op.opdata = (int)FusionCore.DSASM.Registers.RX;
}
else
{
Debug.Assert(false);
}
return op;
}
private TerminalNode NewTerminalNode(XmlQualifiedName qname) {
TerminalNode t = new TerminalNode(qname);
t.Pos = terminalNodes.Count;
terminalNodes.Add(t);
if (!qname.IsEmpty && symbolTable[qname] == null) {
symbolTable.Add(qname, symbols.Count);
symbols.Add(qname);
}
return t;
}
internal CompiledContentModel( SchemaNames names ) {
schemaNames = names;
endNode = null;
}
internal void Finish(ValidationEventHandler eventHandler, bool compile) {
stack = null;
IsCompiled = !abnormalContent && compile;
if (contentNode == null)
return;
#if DEBUG
StringBuilder bb = new StringBuilder();
contentNode.Dump(bb);
Debug.WriteLineIf(CompModSwitches.XmlSchema.TraceVerbose, "\t\t\tContent: (" + bb.ToString() + ")");
#endif
// add end node
endNode = NewTerminalNode(XmlQualifiedName.Empty);
contentNode = new InternalNode(contentNode, endNode, ContentNode.Type.Sequence);
((InternalNode)contentNode).LeftNode.ParentNode = contentNode;
endNode.ParentNode = contentNode;
if (!IsCompiled) {
CheckXsdDeterministic(eventHandler);
return;
}
if (nodeTable == null)
nodeTable = new Hashtable();
// calculate followpos
int terminals = terminalNodes.Count;
BitSet[] followpos = new BitSet[terminals];
for (int i = 0; i < terminals; i++) {
followpos[i] = new BitSet(terminals);
}
contentNode.CalcFollowpos(followpos);
// state table
ArrayList Dstates = new ArrayList(16);
// transition table
dtrans = new ArrayList(16);
// lists unmarked states
ArrayList unmarked = new ArrayList(16);
// state lookup table
Hashtable statetable = new Hashtable();
BitSet empty = new BitSet(terminals);
statetable.Add(empty, -1);
// start with firstpos at the root
BitSet set = contentNode.Firstpos(terminals);
statetable.Add(set, Dstates.Count);
unmarked.Add(set);
Dstates.Add(set);
int[] a = new int[symbols.Count + 1];
dtrans.Add(a);
if (set.Get(endNode.Pos)) {
a[symbols.Count] = 1; // accepting
}
// current state processed
int state = 0;
// check all unmarked states
while (unmarked.Count > 0) {
int[] t = (int[])dtrans[state];
set = (BitSet)unmarked[0];
CheckDeterministic(set, terminals, eventHandler);
unmarked.RemoveAt(0);
// check all input symbols
for (int sym = 0; sym < symbols.Count; sym++) {
XmlQualifiedName n = (XmlQualifiedName)symbols[sym];
BitSet newset = new BitSet(terminals);
// if symbol is in the set add followpos to new set
for (int i = 0; i < terminals; i++) {
if (set.Get(i) && n.Equals(((TerminalNode)terminalNodes[i]).Name)) {
newset.Or(followpos[i]);
}
}
Object lookup = statetable[newset];
// this state will transition to
int transitionTo;
// if new set is not in states add it
if (lookup == null) {
transitionTo = Dstates.Count;
statetable.Add(newset, transitionTo);
unmarked.Add(newset);
Dstates.Add(newset);
a = new int[symbols.Count + 1];
dtrans.Add(a);
if (newset.Get(endNode.Pos)) {
a[symbols.Count] = 1; // accepting
}
}
else {
transitionTo = (int)lookup;
}
// set the transition for the symbol
t[sym] = transitionTo;
}
state++;
}
nodeTable = null;
}
internal static NodeInfo Create(TerminalNode node)
{
return new NodeInfo(NodeType.Terminal, string.Empty);
}
