/// <summary>
/// Scan an isolated token in the following "default" context :
/// - insideDataDivision = true
/// - decimalPointIsComma = false
/// - withDebuggingMode = false
/// - encodingForAlphanumericLiterals = IBM 1147
/// - default compiler options
/// </summary>
public static Token ScanIsolatedTokenInDefaultContext(string tokenText, out Diagnostic error)
{
TokensLine tempTokensLine = TokensLine.CreateVirtualLineForInsertedToken(0, tokenText);
tempTokensLine.InitializeScanState(new MultilineScanState(true, false, false, IBMCodePages.GetDotNetEncodingFromIBMCCSID(1147)));
Scanner tempScanner = new Scanner(tokenText, 0, tokenText.Length - 1, tempTokensLine, new TypeCobolOptions());
Token candidateToken = tempScanner.GetNextToken();
if(tempTokensLine.ScannerDiagnostics.Count > 0)
{
error = tempTokensLine.ScannerDiagnostics[0];
}
else
{
error = null;
}
return candidateToken;
}
/// <summary>
/// Scan a line of a document
/// </summary>
public static void ScanTokensLine(TokensLine tokensLine, MultilineScanState initialScanState, TypeCobolOptions compilerOptions)
{
// Updates are forbidden after a snapshot of a specific version of a line
if(!tokensLine.CanStillBeUpdatedBy(CompilationStep.Scanner))
{
throw new InvalidOperationException("Can not update this TokensLine because it was already frozen by compilation step : " + tokensLine.CompilationStep.ToString());
}
// Set the initial scan state for the line
tokensLine.InitializeScanState(initialScanState);
// Alias to refer to Cobol text line properties
ICobolTextLine textLine = tokensLine;
// The source section of this line of text must be split into tokens
string line = tokensLine.Text;
int startIndex = textLine.Source.StartIndex;
int lastIndex = textLine.Source.EndIndex;
#if EUROINFO_LEGACY_REPLACING_SYNTAX
if (IsInsideRemarks(textLine.Type, tokensLine.SourceText)) tokensLine.ScanState.InsideRemarksDirective = true;
// Try to scan REMARKS compiler directive parameters inside the comment or non-comment line
if (tokensLine.ScanState.InsideRemarksDirective) {
string remarksLine = textLine.SourceText;
int startIndexForSignificantPart = GetStartIndexOfSignificantPart(remarksLine, tokensLine.ScanState);
int firstPeriodIndex = remarksLine.IndexOf('.', startIndexForSignificantPart);
int endIndexForSignificantPart = GetEndIndexOfSignificantPart(remarksLine, tokensLine.ScanState, firstPeriodIndex);
string significantPart = remarksLine.Substring(startIndexForSignificantPart, endIndexForSignificantPart - startIndexForSignificantPart + 1).Trim();
if (firstPeriodIndex >= 0 || (!tokensLine.ScanState.InsideRemarksParentheses && !remarksLine.Contains("COPY"))) {
tokensLine.ScanState.InsideRemarksDirective = false; // indicates the end of the REMARKS compiler directive
}
RemarksDirective remarksDirective = CreateRemarksDirective(significantPart, tokensLine.ScanState);
if (remarksDirective != null && remarksDirective.CopyTextNamesVariations.Count > 0) {
// A non empty remarks directive will replace the comment line
tokensLine.AddToken(CreateCompilerDirectiveToken(remarksDirective, tokensLine, startIndex, lastIndex));
return;
}
}
#endif
// Comment line => return only one token with type CommentLine
// Debug line => treated as a comment line if debugging mode was not activated
if (textLine.Type == CobolTextLineType.Comment ||
(textLine.Type == CobolTextLineType.Debug && !tokensLine.InitialScanState.WithDebuggingMode))
{
Token commentToken = new Token(TokenType.CommentLine, startIndex, lastIndex, tokensLine);
tokensLine.AddToken(commentToken);
return;
}
// Invalid indicator, the line type is unknown => the whole line text is handled as a single invalid token
else if (textLine.Type == CobolTextLineType.Invalid)
{
// Invalid indicator => register an error
tokensLine.AddDiagnostic(MessageCode.InvalidIndicatorCharacter, textLine.Indicator.StartIndex, textLine.Indicator.EndIndex, textLine.Indicator);
Token invalidToken = new Token(TokenType.InvalidToken, startIndex, lastIndex, tokensLine);
tokensLine.AddToken(invalidToken);
return;
}
// Empty line => return immediately an empty list of tokens
// Blank line => return only one token with type SpaceSeparator
if(textLine.Type == CobolTextLineType.Blank)
{
if(!String.IsNullOrEmpty(line))
{
Token whitespaceToken = new Token(TokenType.SpaceSeparator, startIndex, lastIndex, tokensLine);
tokensLine.AddToken(whitespaceToken);
}
return;
}
// Create a stateful line scanner, and iterate over the tokens
Scanner scanner = new Scanner(line, startIndex, lastIndex, tokensLine, compilerOptions);
Token nextToken = null;
while((nextToken = scanner.GetNextToken()) != null)
{
// Resolve DELETE ambiguity : DELETE + InterLiteral => DELETE_CD
// Warning : DELETE and the sequence-number-field must be on the same line
if(nextToken.TokenType == TokenType.IntegerLiteral && tokensLine.ScanState.KeywordsState == KeywordsSequenceState.After_DELETE)
{
tokensLine.ScanState.LastKeywordOrSymbolToken.CorrectType(TokenType.DELETE_CD);
}
tokensLine.AddToken(nextToken);
}
}