// ---
public static HighlightingColor GetTokenStyle(Token token)
{
if (token.TokenFamily == TokenFamily.Comments)
{
return CommentStyle;
}
else if (token.TokenFamily == TokenFamily.StatementStartingKeyword || token.TokenFamily == TokenFamily.StatementEndingKeyword)
{
return CobolVerbStyle;
}
else if (token.TokenFamily == TokenFamily.SyntaxKeyword)
{
return CobolKeywordStyle;
}
else if (token.TokenFamily == TokenFamily.AlphanumericLiteral)
{
return CobolAlphanumericLiteralStyle;
}
else if (token.TokenFamily == TokenFamily.NumericLiteral)
{
return CobolNumericLiteralStyle;
}
else if (token.TokenFamily == TokenFamily.Symbol)
{
return CobolUserDefinedWordStyle;
}
return null;
}
internal ContinuationToken(Token virtualConcatenatedToken, int startIndex, int stopIndex, ITokensLine tokensLine, bool isContinuationFromPreviousLine, bool isContinuedOnNextLine)
: base(virtualConcatenatedToken.TokenType, startIndex, stopIndex, virtualConcatenatedToken.UsesVirtualSpaceAtEndOfLine, tokensLine)
{
// Store the concatenated source text
MultilineContinuationText = virtualConcatenatedToken.Text;
IsContinuationFromPreviousLine = isContinuationFromPreviousLine;
IsContinuedOnNextLine = isContinuedOnNextLine;
// Copy the delimiter properties
if (virtualConcatenatedToken.UsesDelimiters)
{
UsesDelimiters = virtualConcatenatedToken.UsesDelimiters;
HasOpeningDelimiter = virtualConcatenatedToken.HasOpeningDelimiter;
HasClosingDelimiter = virtualConcatenatedToken.HasClosingDelimiter;
ExpectedClosingDelimiter = virtualConcatenatedToken.ExpectedClosingDelimiter;
}
// Copy the literal value
LiteralValue = virtualConcatenatedToken.LiteralValue;
// Set specific Channel to enable filtering of all tokens participating in the continuation, except the first one
if (IsContinuationFromPreviousLine)
{
Channel = CHANNEL_ContinuationTokens;
}
}
public ImportedToken(Token originalToken, CopyDirective copyDirective)
: base(originalToken.TokenType, originalToken.StartIndex, originalToken.StopIndex, originalToken.UsesVirtualSpaceAtEndOfLine, originalToken.TokensLine,
originalToken.HasOpeningDelimiter, originalToken.HasClosingDelimiter, originalToken.ExpectedClosingDelimiter)
{
this.OriginalToken = originalToken;
this.CopyDirective = copyDirective;
HasError = originalToken.HasError;
UsesDelimiters = originalToken.UsesDelimiters;
LiteralValue = originalToken.LiteralValue;
}
private int IndexOf(IList<Token> tokens, out Token found, string text, int index = 0)
{
for(int c=index; c<tokens.Count; c++) {
if (text.Equals(tokens[c].Text)) {
found = tokens[c];
return c;
}
}
found = null;
throw new System.ArgumentException("No token with text=\""+text+"\" found among "+tokens.Count+" tokens (starting from "+index+")");
}
/// <summary>
/// Create a token placeholder
/// </summary>
public ReplacedToken(Token replacementToken, Token originalToken)
: base(replacementToken.TokenType, originalToken.StartIndex, originalToken.StopIndex, originalToken.UsesVirtualSpaceAtEndOfLine, originalToken.TokensLine)
{
ReplacementToken = replacementToken;
OriginalToken = originalToken;
HasError = replacementToken.HasError;
UsesDelimiters = replacementToken.UsesDelimiters;
HasOpeningDelimiter = replacementToken.HasOpeningDelimiter;
HasClosingDelimiter = replacementToken.HasClosingDelimiter;
ExpectedClosingDelimiter = replacementToken.ExpectedClosingDelimiter;
LiteralValue = replacementToken.LiteralValue;
}
private void GetSpan(Token first, Token last, out int begin, out int end)
{
// if SymbolReference is unqualified, last is nil
if (last == null) {
begin = first.StartIndex;
end = first.StopIndex+1;
} else
// as we can have either head::tail (TypeCobol) OR tail OF|IN head (COBOL),
// we can have first before last OR first _after_ last in tokens in ConsumedTokens
if (first.StartIndex < last.StartIndex) {
begin = first.StartIndex;
end = last.StopIndex+1;
} else {
begin = last.StartIndex;
end = first.StopIndex+1;
}
}
public void OnToken(Token token)
{
TokenType tokenType = token.TokenType;
TokenFamily tokenFamily = TokenUtils.GetTokenFamilyFromTokenType(tokenType);
if (LanguageModel.IsSignificantWord(tokenType, tokenFamily))
{
RegisterToken(token);
if (LanguageModel.IsKeywordToken(tokenType, tokenFamily))
{
lastKeywordToken = tokenType;
}
if (LanguageModel.IsElementStartingWord(tokenType, tokenFamily, lastWord))
{
if(tokenType == TokenType.ID || tokenType == TokenType.IDENTIFICATION)
{
elementStartingWordIndexInProgram = 0;
}
OnBeginElement(tokenType);
}
lastWord = tokenType;
wordIndexInElement++;
}
}
public ConditionOperand(Token selfObjectIdentifier)
: base(ExpressionNodeType.ConditionOperand)
{
SelfObjectIdentifier = selfObjectIdentifier;
}
public MultipleTokensReplaceOperation(Token firstComparisonToken, Token[] followingComparisonTokens, Token[] replacementTokens)
{
Type = ReplaceOperationType.MultipleTokens;
ComparisonToken = firstComparisonToken;
FollowingComparisonTokens = followingComparisonTokens;
ReplacementTokens = replacementTokens;
}
public SingleTokenReplaceOperation(Token comparisonToken, Token replacementToken)
{
Type = ReplaceOperationType.SingleToken;
ComparisonToken = comparisonToken;
ReplacementToken = replacementToken;
}
public SubscriptExpression(Token allToken)
{
ALL = new SyntaxProperty<bool>(true, allToken);
}
public FilePropertySpecialRegister(Token specialRegisterName, SymbolReference fileNameReference)
: base(StorageAreaKind.FilePropertySpecialRegister)
{
SpecialRegisterName = specialRegisterName;
FileNameReference = fileNameReference;
// This is both a storage area definition and a reference to the same storage area
DataDescriptionEntry = new SpecialRegisterDescriptionEntry(specialRegisterName, fileNameReference.ToString());
SymbolReference = new SymbolReference(DataDescriptionEntry.DataName);
}
/// <summary>
/// Compute the next scanner state after recognizing a new token
/// </summary>
public void AdvanceToNextState(Token newToken)
{
// Register the last token (except if it is a comment)
if (newToken.TokenFamily != TokenFamily.Comments)
{
LastToken = newToken;
// Register the last Keyword or Symbol token
if (newToken.TokenFamily >= TokenFamily.Symbol)
{
LastKeywordOrSymbolToken = newToken;
}
}
// Ignore whitespace separators and comments to advance keywords state
if (newToken.TokenFamily == TokenFamily.Whitespace ||
newToken.TokenFamily == TokenFamily.Comments)
{
return;
}
// Register the occurence of a DECIMAL-POINT IS COMMA clause
// (The configuration section can be specified only in the ENVIRONMENT
// DIVISION of the outermost program of a COBOL source program)
if(newToken.TokenType == TokenType.DECIMAL_POINT)
{
DecimalPointIsComma = true;
}
// Keywords state machine
switch (KeywordsState)
{
// Start state -> Token -> End state. Action
// 0 -> PICTURE -> 1.
// 0 -> PIC -> 1.
// 0 -> AUTHOR -> 3.
// 0 -> INSTALLATION -> 3.
// 0 -> DATE_WRITTEN -> 3.
// 0 -> DATE_COMPILED -> 3.
// 0 -> SECURITY -> 3.
// 0 -> FUNCTION -> 6.
// 0 -> EXEC -> 7.
// 0 -> EXECUTE -> 7.
// 0 -> DELETE -> 10.
// 0 -> DATA -> 11.
// 0 -> PROCEDURE -> 12.
// 0 -> TO -> 13.
// 0 -> SAME -> 14.
// 0 -> SYMBOLIC -> 15.
// 0 -> PseudoTextDelimiter -> 17.
// 0 -> DEBUGGING -> 18.
case KeywordsSequenceState.Default:
switch (newToken.TokenType)
{
case TokenType.PICTURE:
case TokenType.PIC:
KeywordsState = KeywordsSequenceState.After_PIC_orPICTURE;
break;
case TokenType.AUTHOR:
case TokenType.INSTALLATION:
case TokenType.DATE_WRITTEN:
case TokenType.DATE_COMPILED:
case TokenType.SECURITY:
KeywordsState = KeywordsSequenceState.After_AUTHOR_orINSTALLATION_orDATE_WRITTEN_orDATE_COMPILED_orSECURITY;
break;
// case TokenType.FUNCTION:
// KeywordsState = KeywordsSequenceState.After_FUNCTION;
// break;
case TokenType.EXEC:
case TokenType.EXECUTE:
KeywordsState = KeywordsSequenceState.After_EXEC_orEXECUTE;
break;
case TokenType.DELETE:
KeywordsState = KeywordsSequenceState.After_DELETE;
break;
case TokenType.DATA:
KeywordsState = KeywordsSequenceState.After_DATA;
break;
case TokenType.PROCEDURE:
KeywordsState = KeywordsSequenceState.After_PROCEDURE;
break;
case TokenType.TO:
KeywordsState = KeywordsSequenceState.After_TO;
break;
case TokenType.SAME:
KeywordsState = KeywordsSequenceState.After_SAME;
break;
case TokenType.SYMBOLIC:
KeywordsState = KeywordsSequenceState.After_SYMBOLIC;
break;
case TokenType.PseudoTextDelimiter:
KeywordsState = KeywordsSequenceState.InsidePseudoText;
break;
case TokenType.DEBUGGING:
KeywordsState = KeywordsSequenceState.After_DEBUGGING;
break;
}
break;
// 1 -> SYMBOL -> 0.
// 1 -> IS -> 2.
// 1 -> pictureCharacterString -> 0.
case KeywordsSequenceState.After_PIC_orPICTURE:
if (newToken.TokenType == TokenType.IS)
{
KeywordsState = KeywordsSequenceState.After_PIC_orPICTURE_IS;
}
else
{
KeywordsState = KeywordsSequenceState.Default;
}
break;
// 2 -> pictureCharacterString -> 0.
case KeywordsSequenceState.After_PIC_orPICTURE_IS:
KeywordsState = KeywordsSequenceState.Default;
break;
// 3 -> commentEntry -> 5.
// 3 -> PeriodSeparator -> 4.
case KeywordsSequenceState.After_AUTHOR_orINSTALLATION_orDATE_WRITTEN_orDATE_COMPILED_orSECURITY:
if (newToken.TokenType == TokenType.PeriodSeparator)
{
KeywordsState = KeywordsSequenceState.After_AUTHOR_orINSTALLATION_orDATE_WRITTEN_orDATE_COMPILED_orSECURITY_PeriodSeparator;
}
else if (newToken.TokenType == TokenType.CommentEntry)
{
KeywordsState = KeywordsSequenceState.After_CommentEntry;
}
else
{
KeywordsState = KeywordsSequenceState.Default;
}
break;
// 4 -> commentEntry -> 5.
// 4 -> chars in area A -> 0.
case KeywordsSequenceState.After_AUTHOR_orINSTALLATION_orDATE_WRITTEN_orDATE_COMPILED_orSECURITY_PeriodSeparator:
if (newToken.TokenType == TokenType.CommentEntry)
{
KeywordsState = KeywordsSequenceState.After_CommentEntry;
}
else
{
KeywordsState = KeywordsSequenceState.Default;
}
break;
// 5 -> commentEntry -> 5.
// 5 -> chars in area A -> 0.
case KeywordsSequenceState.After_CommentEntry:
if (newToken.TokenType != TokenType.CommentEntry)
{
KeywordsState = KeywordsSequenceState.Default;
}
break;
// 6 -> FunctionName -> 0.
// case KeywordsSequenceState.After_FUNCTION:
// KeywordsState = KeywordsSequenceState.Default;
// break;
// 7 -> execTranslatorName -> 8.
case KeywordsSequenceState.After_EXEC_orEXECUTE:
KeywordsState = KeywordsSequenceState.After_EXEC_orEXECUTE_ExecTranslatorName;
break;
// 8 -> execStatementText -> 9.
// 8 -> END_EXEC -> 0.
// 8 -> EXEC_SQL_INCLUDE -> 0.
case KeywordsSequenceState.After_EXEC_orEXECUTE_ExecTranslatorName:
if (newToken.TokenType == TokenType.ExecStatementText)
{
KeywordsState = KeywordsSequenceState.After_ExecStatementText;
}
else
{
KeywordsState = KeywordsSequenceState.Default;
}
break;
// 9 -> execStatementText -> 9.
// 8 -> END_EXEC -> 9.
case KeywordsSequenceState.After_ExecStatementText:
if (newToken.TokenType != TokenType.ExecStatementText)
{
KeywordsState = KeywordsSequenceState.Default;
}
break;
// 10 -> UserDefinedWord -> 0.
// 10 -> IntegerLiteral -> 0. =rw=> DELETE_CD
case KeywordsSequenceState.After_DELETE:
KeywordsState = KeywordsSequenceState.Default;
break;
// 11 -> DIVISION -> 0. inside data division = true
case KeywordsSequenceState.After_DATA:
if(newToken.TokenType == TokenType.DIVISION)
{
InsideDataDivision = true;
}
KeywordsState = KeywordsSequenceState.Default;
break;
// 12 -> DIVISION -> 0. inside data division = false
case KeywordsSequenceState.After_PROCEDURE:
if(newToken.TokenType == TokenType.DIVISION)
{
InsideDataDivision = false;
}
KeywordsState = KeywordsSequenceState.Default;
break;
// 13 -> ENTRY -> 0. =rw=> ENTRY_ARG
case KeywordsSequenceState.After_TO:
KeywordsState = KeywordsSequenceState.Default;
break;
// 14 -> SORT -> 0. =rw=> SORT_ARG
case KeywordsSequenceState.After_SAME:
KeywordsState = KeywordsSequenceState.Default;
break;
// 15 -> CHARACTERS -> 15
// 15 -> SymbolicCharacter -> 15
// 15 -> any other -> 16
case KeywordsSequenceState.After_SYMBOLIC:
if (newToken.TokenType != TokenType.CHARACTER &&
newToken.TokenType != TokenType.SymbolicCharacter)
{
KeywordsState = KeywordsSequenceState.After_SYMBOLIC_SymbolicCharacters;
}
break;
// 16 -> IntegerLiteral -> 16
// 16 -> SYMBOLIC | SymbolicCharacter -> 15
// 16 -> any other -> 0
case KeywordsSequenceState.After_SYMBOLIC_SymbolicCharacters:
if (newToken.TokenType == TokenType.SymbolicCharacter || newToken.TokenType == TokenType.SYMBOLIC)
{
KeywordsState = KeywordsSequenceState.After_SYMBOLIC;
}
else if(newToken.TokenType != TokenType.IntegerLiteral)
{
KeywordsState = KeywordsSequenceState.Default;
}
break;
// 17 -> PseudoTextDelimiter -> 0.
case KeywordsSequenceState.InsidePseudoText:
if(newToken.TokenType == TokenType.PseudoTextDelimiter || newToken.TokenType == TokenType.COPY)
{
KeywordsState = KeywordsSequenceState.Default;
}
break;
// 18 -> MODE : WithDebuggingMode = true -> 0.
// 1/ -> any other -> 0.
case KeywordsSequenceState.After_DEBUGGING:
if (newToken.TokenType == TokenType.MODE)
{
WithDebuggingMode = true;
}
KeywordsState = KeywordsSequenceState.Default;
break;
}
}
public StorageAreaPropertySpecialRegister(Token specialRegisterName, StorageArea storageAreaReference)
: base(StorageAreaKind.StorageAreaPropertySpecialRegister)
{
SpecialRegisterName = specialRegisterName;
StorageAreaReference = storageAreaReference;
}
public FilePropertySpecialRegister(Token specialRegisterName, SymbolReference fileNameReference)
: base(StorageAreaKind.FilePropertySpecialRegister)
{
SpecialRegisterName = specialRegisterName;
SymbolReference = fileNameReference;
}
public SymbolicCharacter(Token symbolicCharacter) :
base(symbolicCharacter, SymbolType.SymbolicCharacter)
{ }
private void RegisterToken(Token token)
{
totalWordCount++;
SymbolInformation symbolInfo = null;
symbolInformationForTokens.TryGetValue(token, out symbolInfo);
var wordProbabilities = WordProbabilitiesAfterElementStartingWord[lastElementStartingWord];
wordProbabilities.OnWords(lastWord, token.TokenType, symbolInfo, lastKeywordToken, wordIndexInElement);
}
/// <summary>
/// Attach a symbol type to a source token in a declaration statement
/// </summary>
/// <param name="nameToken">Token representing the name of the symbol (TokenFamily.Symbol or TokenFamily.AlphanumericLiteral)</param>
/// <param name="type">Type of symbol</param>
public Symbol(Token nameToken, SymbolType type)
{
NameToken = nameToken;
Type = type;
}
public ParagraphName(Token userDefinedWord) :
base(userDefinedWord, SymbolType.ParagraphName)
{ }
private string ToJSON(int a, Token t)
{
var b = new StringBuilder("{");
newline(b, a+1, "");
kv(b, "Type", "\"" + t.TokenType + "\"");
newline(b, a+1, ",");
kv(b, "Text", "\"" + escape(t.Text) + "\"");
newline(b, a+1, ",");
kv(b, "Line", ""+t.Line);
newline(b, a+1, ",");
kv(b, "Column", ""+t.Column);
newline(b, a+1, ",");
kv(b, "EndColumn", ""+t.EndColumn);
newline(b, a+1, ",");
kv(b, "UsesVirtualSpaceAtEndOfLine", ""+t.UsesVirtualSpaceAtEndOfLine.ToString().ToLower());
newline(b, a, "");
return b.Append("}").ToString();
}
public StorageAreaPropertySpecialRegister(Token specialRegisterName, StorageArea storageAreaReference)
: base(StorageAreaKind.StorageAreaPropertySpecialRegister)
{
SpecialRegisterName = specialRegisterName;
OtherStorageAreaReference = storageAreaReference;
// This is both a storage area definition and a reference to the same storage area
var storageAreaName = storageAreaReference != null ? storageAreaReference.ToString() : "null";
DataDescriptionEntry = new SpecialRegisterDescriptionEntry(specialRegisterName, storageAreaName);
SymbolReference = new SymbolReference(DataDescriptionEntry.DataName);
}
public SyntaxNumber(Token token)
{
Token = token;
}
protected static string NoQuotes(Token token)
{
if (token == null) return "?";
return token.SourceText.Trim('\"').Trim('\'');
}
public IToken Create(int type, string text)
{
if (text == null) text = String.Empty;
Token token = new Token((TokenType)type, 0, text.Length - 1, TokensLine.CreateVirtualLineForInsertedToken(-1, text));
token.SetAntlrSource(null);
SetTokenLiteralValue(token, text);
return token;
}
public SingleToMultipleTokensReplaceOperation(Token comparisonToken, Token[] replacementTokens)
{
Type = ReplaceOperationType.SingleToMultipleTokens;
ComparisonToken = comparisonToken;
ReplacementTokens = replacementTokens;
}
public IToken Create(Tuple<ITokenSource, ICharStream> source, int type, string text, int channel, int start, int stop, int line, int charPositionInLine)
{
if (text == null) text = String.Empty;
Token token = new Token((TokenType)type, 0, text.Length - 1, TokensLine.CreateVirtualLineForInsertedToken(line -1, text));
token.SetAntlrSource(source.Item1);
token.Channel = channel;
SetTokenLiteralValue(token, text);
return token;
}
public PartialWordReplaceOperation(Token comparisonToken, Token replacementToken)
{
Type = ReplaceOperationType.PartialWord;
ComparisonToken = comparisonToken;
PartialReplacementToken = replacementToken;
}
private void SetTokenLiteralValue(Token token, string text)
{
if(token.TokenFamily == TokenFamily.AlphanumericLiteral)
{
token.LiteralValue = new AlphanumericLiteralTokenValue(text);
}
else if(token.TokenType == TokenType.IntegerLiteral)
{
string sign = null;
if(text[0] == '+' || text[0] == '-')
{
sign = text.Substring(0,1);
text = text.Substring(1);
}
token.LiteralValue = new IntegerLiteralTokenValue(sign, text);
}
else if (token.TokenType == TokenType.DecimalLiteral || token.TokenType == TokenType.FloatingPointLiteral)
{
throw new NotImplementedException();
}
}
public SpecialRegisterDescriptionEntry(Token specialRegisterName, string storageAreaNameOrFileName)
: base()
{
// Generate a unique symbol name for this special register
var generatedSymbolName = new GeneratedSymbolName(specialRegisterName, specialRegisterName.Text + "-" + storageAreaNameOrFileName);
DataName = new SymbolDefinition(generatedSymbolName, SymbolType.DataName);
SpecialRegisterName = specialRegisterName;
}
// --- Token comparison for REPLACE directive ---
/// <summary>
/// Compare two tokens to implement the REPLACE directive
/// </summary>
public bool CompareForReplace(Token comparisonToken)
{
// 1. First compare the token type
if(this.TokenType != comparisonToken.TokenType)
{
return false;
}
// 2. For partial Cobol words, chech if the comparison token text (":TAG:")
// is contained in the current token text (":TAG:-AMOUNT")
else if(TokenType == TokenType.PartialCobolWord)
{
return Text.IndexOf(comparisonToken.Text, StringComparison.OrdinalIgnoreCase) >= 0;
}
// 3. For token families
// - AlphanumericLiteral
// - NumericLiteral
// - SyntaxLiteral
// - Symbol
// => compare Token text
else if (TokenFamily == TokenFamily.AlphanumericLiteral || TokenFamily == TokenFamily.NumericLiteral ||
TokenFamily == TokenFamily.Symbol || TokenFamily == TokenFamily.SyntaxLiteral)
{
return Text.Equals(comparisonToken.Text, StringComparison.OrdinalIgnoreCase);
}
// 4. In all other cases, token type comparison was enough
{
return true;
}
}