private Scanner(string line, int startIndex, int lastIndex, TokensLine tokensLine, TypeCobolOptions compilerOptions)
{
this.tokensLine = tokensLine;
this.line = line;
this.currentIndex = startIndex;
this.lastIndex = lastIndex;
this.compilerOptions = compilerOptions;
}
public static string ScanLines(string[] testLines)
{
TokensLine[] tokensLines = new TokensLine[testLines.Length];
for (int i = 0; i < testLines.Length; i++)
{
TokensLine tokensLine = new TestTokensLine(testLines[i]);
tokensLines[i] = tokensLine;
}
return(ScanLines(tokensLines));
}
public static string ScanLines(string[] testLines)
{
TokensLine[] tokensLines = new TokensLine[testLines.Length];
for(int i = 0; i < testLines.Length; i++)
{
TokensLine tokensLine = new TestTokensLine(testLines[i]);
tokensLines[i] = tokensLine;
}
return ScanLines(tokensLines);
}
private static IList <Tuple <string, bool> > Split(string line, int max, int min)
{
var lines = new List <Tuple <string, bool> >();
int nLine = (line.Length / max) + ((line.Length % max) != 0 ? 1 : 0);
if (nLine == 1)
{
lines.Add(new Tuple <string, bool>(line, false));
return(lines);
}
if (line.Length < 1)
{
lines.Add(new Tuple <string, bool>(line, false));
}
else
{
for (int i = 0; i < line.Length; i += (i == 0 ? max : min))
{
lines.Add(new Tuple <string, bool>(line.Substring(i, Math.Min((i == 0 ? max : min), line.Length - i)), true));
}
}
TokensLine tempTokensLine = TokensLine.CreateVirtualLineForInsertedToken(0, line);
tempTokensLine.InitializeScanState(new MultilineScanState(false, false, false, IBMCodePages.GetDotNetEncodingFromIBMCCSID(1147)));
Scanner.Scanner scanner = new Scanner.Scanner(line, 0, line.Length - 1, tempTokensLine, null, false);
Token t = null;
int nCurLength = 0;
int nSpan = max;
int index = 0;
bool bNextNoIndicator = false;
while ((t = scanner.GetNextToken()) != null)
{
nCurLength += t.Length;
if (nCurLength >= nSpan)
{
if (t.TokenFamily == TokenFamily.Whitespace || (nCurLength == nSpan))
{
bNextNoIndicator = true;
}
nSpan += min;
index++;
}
else if (bNextNoIndicator)
{
lines[index] = new Tuple <string, bool>(lines[index].Item1, false);
bNextNoIndicator = false;
}
}
return(lines);
}
public static string ScanLines(TokensLine[] tokensLines)
{
ImmutableList<TokensLine>.Builder tokensLinesList = ImmutableList<TokensLine>.Empty.ToBuilder();
tokensLinesList.AddRange(tokensLines);
ScannerStep.ScanDocument(TextSourceInfo, tokensLinesList, CompilerOptions);
StringBuilder sbResult = new StringBuilder();
for (int i = 0; i < tokensLines.Length; i++)
{
sbResult.AppendLine("-- Line " + (i + 1) + " --");
sbResult.AppendLine(BuildResultString(tokensLines[i]));
}
return sbResult.ToString();
}
private ReplaceStatus TryAndReplace(Token nextToken, ReplaceOperation replaceOperation)
{
ReplaceStatus status = new ReplaceStatus();
IList <Token> originalMatchingTokens;
#if EUROINFO_RULES
if (CompilerOptions.UseEuroInformationLegacyReplacingSyntax)
{
// Support for legacy replacing syntax semantics :
// Insert Suffix before the first '-' in all user defined words found in the COPY text
// before copying it into the main program
if (CopyReplacingDirective != null && CopyReplacingDirective.InsertSuffixChar && nextToken.TokenType == TokenType.UserDefinedWord)
{
string originalText = nextToken.Text;
if (originalText.Contains(CopyReplacingDirective.PreSuffix))
{
string replacedText = originalText.Replace(CopyReplacingDirective.PreSuffix, CopyReplacingDirective.PreSuffix.Insert(3, CopyReplacingDirective.Suffix));
TokensLine virtualTokensLine = TokensLine.CreateVirtualLineForInsertedToken(0, replacedText);
Token replacementToken = new Token(TokenType.UserDefinedWord, 0, replacedText.Length - 1,
virtualTokensLine);
status.replacedToken = new ReplacedToken(replacementToken, nextToken);
currentPosition.CurrentToken = status.replacedToken;
}
}
}
#endif
if (replaceOperation != null && TryMatchReplaceOperation(nextToken, replaceOperation, out originalMatchingTokens))
{
status.replacedToken = CreateReplacedTokens(nextToken, replaceOperation, originalMatchingTokens);
if (status.replacedToken != null)
{
// REPLACE pattern matched => return the first replaced token
currentPosition.CurrentToken = status.replacedToken;
}
else
{
// If the replacement token set is empty (REPLACE == ... = BY == ==), get next token and try again
status.tryAgain = true;
}
}
return(status);
}
private static Nodes.TypeDefinition CreateCurrency()
{
var entry = new DataTypeDescriptionEntry();
entry.LevelNumber = new GeneratedIntegerValue(1);
entry.DataName = new SymbolDefinition(new GeneratedAlphanumericValue("CURRENCY"), SymbolType.DataName);
entry.Picture = new GeneratedAlphanumericValue(string.Format("{0}({1})", 'X', 3));
entry.DataType = DataType.Currency;
var tokenLine = TokensLine.CreateVirtualLineForInsertedToken(entry.Line, "01 CURRENCY TYPEDEF STRICT PUBLIC PIC X(03).");
entry.ConsumedTokens.Add(new Token(TokenType.LevelNumber, 0, 1, tokenLine));
entry.ConsumedTokens.Add(new Token(TokenType.UserDefinedWord, 3, 10, tokenLine));
entry.ConsumedTokens.Add(new Token(TokenType.TYPEDEF, 12, 18, tokenLine));
entry.ConsumedTokens.Add(new Token(TokenType.STRICT, 20, 25, tokenLine));
entry.ConsumedTokens.Add(new Token(TokenType.PUBLIC, 27, 32, tokenLine));
entry.ConsumedTokens.Add(new Token(TokenType.PIC, 34, 36, tokenLine));
entry.ConsumedTokens.Add(new Token(TokenType.PictureCharacterString, 38, 42, tokenLine));
entry.ConsumedTokens.Add(new Token(TokenType.PeriodSeparator, 43, 43, tokenLine));
return(new Nodes.TypeDefinition(entry));
}
public static string ScanTextLine(TokensLine tokensLine)
{
TypeCobol.Compiler.Scanner.Scanner.ScanFirstLine(tokensLine, false, false, false, TextSourceInfo.EncodingForAlphanumericLiterals, CompilerOptions, CopyTextNameVariations);
return(BuildResultString(tokensLine));
}
public static string ScanTextLine(TokensLine tokensLine)
{
TypeCobol.Compiler.Scanner.Scanner.ScanFirstLine(tokensLine, false, false, false, TextSourceInfo.EncodingForAlphanumericLiterals, CompilerOptions);
return BuildResultString(tokensLine);
}
private static void ScanTokensLineWithMultilineScanState(int lineToScanIndex, TokensLine lineToScan, TextSourceInfo textSourceInfo, ISearchableReadOnlyList<TokensLine> documentLines, PrepareDocumentLineForUpdate prepareDocumentLineForUpdate, TypeCobolOptions compilerOptions, IList<DocumentChange<ITokensLine>> tokensLinesChanges, out int nextLineToScanIndex, out TokensLine nextLineToScan)
{
// Scan the current line (or continuation lines group)
MultilineScanState scanState = ScanTokensLineWithContinuations(lineToScanIndex, lineToScan, textSourceInfo, documentLines, prepareDocumentLineForUpdate, compilerOptions, tokensLinesChanges, out nextLineToScanIndex, out nextLineToScan);
// Scan the following lines until we find that the scan state at the beginning of the next line has been updated
while(nextLineToScan != null && nextLineToScan.InitialScanState != null && !nextLineToScan.InitialScanState.Equals(scanState))
{
scanState = ScanTokensLineWithContinuations(nextLineToScanIndex, nextLineToScan, textSourceInfo, documentLines, prepareDocumentLineForUpdate, compilerOptions, tokensLinesChanges, out nextLineToScanIndex, out nextLineToScan);
}
}
private static MultilineScanState ScanTokensLineWithContinuations(int lineToScanIndex, TokensLine lineToScan, TextSourceInfo textSourceInfo, ISearchableReadOnlyList<TokensLine> documentLines, PrepareDocumentLineForUpdate prepareDocumentLineForUpdate, TypeCobolOptions compilerOptions, IList<DocumentChange<ITokensLine>> tokensLinesChanges, out int nextLineToScanIndex, out TokensLine nextLineToScan)
{
// Initialize out parameters
if (lineToScanIndex < (documentLines.Count - 1))
{
nextLineToScanIndex = lineToScanIndex + 1;
nextLineToScan = documentLines[nextLineToScanIndex];
}
else
{
nextLineToScanIndex = -1;
nextLineToScan = null;
}
// Check if the line to scan participates in a multiline continuation
// - because it is itself a continuation line
bool partOfMultilineContinuation = lineToScan.Type == CobolTextLineType.Continuation;
// - or because the following line is a continuation line
if (!partOfMultilineContinuation && lineToScanIndex < (documentLines.Count - 1))
{
nextLineToScanIndex = lineToScanIndex + 1;
nextLineToScan = documentLines[nextLineToScanIndex];
partOfMultilineContinuation = nextLineToScan.Type == CobolTextLineType.Continuation;
}
// ^-- LIMITATION : we don't support the case where one or more comment lines follow the current line before a continuation line
// Case 1 : the line is not part of a multiline continuation => we can parse it as as a standalone line
if (!partOfMultilineContinuation)
{
// Create a new copy of the line before the update if necessary
lineToScan = (TokensLine)prepareDocumentLineForUpdate(lineToScanIndex, lineToScan, CompilationStep.Scanner);
if (lineToScanIndex == 0)
{
// Scan the first line of the document
Scanner.ScanFirstLine(lineToScan, false, false, false, textSourceInfo.EncodingForAlphanumericLiterals, compilerOptions);
}
else
{
// Get the scan state at the end of the previous line
TokensLine previousLine = documentLines[lineToScanIndex - 1];
// Scan the current line with this initial scan state
Scanner.ScanTokensLine(lineToScan, previousLine.ScanState, compilerOptions);
}
tokensLinesChanges.Add(new DocumentChange<ITokensLine>(DocumentChangeType.LineUpdated, lineToScanIndex, lineToScan));
return lineToScan.ScanState;
}
// Case 2 : the line is part of a multiline continuation => we must parse all continuation lines as a group
else
{
// Build a list of the lines we will have to scan as a group :
IList<TokensLine> continuationLinesGroup = new List<TokensLine>();
int firstLineIndex = lineToScanIndex;
continuationLinesGroup.Insert(0, (TokensLine)prepareDocumentLineForUpdate(lineToScanIndex, lineToScan, CompilationStep.Scanner));
// Navigate backwards to the start of the multiline continuation
if (lineToScan.Type == CobolTextLineType.Continuation && lineToScanIndex > 0)
{
int revLineToScanIndex = lineToScanIndex;
IEnumerator<TokensLine> reversedEnumerator = documentLines.GetEnumerator(lineToScanIndex - 1, -1, true);
while(reversedEnumerator.MoveNext())
{
// Get the previous line until a non continuation and non comment line is encountered
revLineToScanIndex--;
lineToScan = reversedEnumerator.Current;
if(lineToScan.Type != CobolTextLineType.Continuation /*&& <-- LIMITATION : this compiler does not support comment or blank lines between two continuation line
lineToScan.Type != CobolTextLineType.Comment && lineToScan.Type != CobolTextLineType.Blank*/) // see p54 : for continuation, blank lines are treated like comment lines
{
firstLineIndex = revLineToScanIndex;
continuationLinesGroup.Insert(0, (TokensLine)prepareDocumentLineForUpdate(lineToScanIndex, lineToScan, CompilationStep.Scanner));
}
else
{
break;
}
}
}
// Reuse our knowledge of the next line if it is available and if it is a continuation
if (nextLineToScan != null && nextLineToScan.Type == CobolTextLineType.Continuation)
{
lineToScanIndex++;
lineToScan = nextLineToScan;
continuationLinesGroup.Add((TokensLine)prepareDocumentLineForUpdate(lineToScanIndex, lineToScan, CompilationStep.Scanner));
nextLineToScanIndex = -1;
nextLineToScan = null;
}
// Navigate forwards to the end of the multiline continuation
if (lineToScanIndex < (documentLines.Count - 1))
{
IEnumerator<TokensLine> enumerator = documentLines.GetEnumerator(lineToScanIndex + 1, -1, false);
while (enumerator.MoveNext())
{
// Get the next line until a non continuation and non comment line is encountered
lineToScanIndex++;
lineToScan = enumerator.Current;
if (lineToScan.Type == CobolTextLineType.Continuation /*|| <-- LIMITATION : this compiler does not support comment or blank lines between two continuation line
lineToScan.Type == CobolTextLineType.Comment || lineToScan.Type == CobolTextLineType.Blank*/) // see p54 : for continuation, blank lines are treated like comment lines
{
// Add this line at the end of the list of continuation lines
continuationLinesGroup.Add((TokensLine)prepareDocumentLineForUpdate(lineToScanIndex, lineToScan, CompilationStep.Scanner));
}
else
{
// Save the knowledge of the next line and exit the loop
nextLineToScanIndex = lineToScanIndex;
nextLineToScan = lineToScan;
break;
}
}
}
// Scan the group of continuation lines
if (firstLineIndex == 0)
{
// Scan the first line group of the document
Scanner.ScanFirstLineContinuationGroup(continuationLinesGroup, false, false, false, textSourceInfo.EncodingForAlphanumericLiterals, compilerOptions);
}
else
{
// Get the scan state at the end of the previous line
TokensLine previousLine = documentLines[firstLineIndex - 1];
// Scan the current line group with this initial scan state
Scanner.ScanTokensLineContinuationGroup(continuationLinesGroup, previousLine.ScanState, compilerOptions);
}
int updatedLineIndex = firstLineIndex;
foreach(TokensLine updatedLine in continuationLinesGroup)
{
tokensLinesChanges.Add(new DocumentChange<ITokensLine>(DocumentChangeType.LineUpdated, updatedLineIndex, updatedLine));
updatedLineIndex++;
}
return continuationLinesGroup[continuationLinesGroup.Count - 1].ScanState;
}
}
/// <summary>
/// Scan a line of a document
/// </summary>
public static void ScanTokensLine(TokensLine tokensLine, MultilineScanState initialScanState, TypeCobolOptions compilerOptions)
{
// Updates are forbidden after a snapshot of a specific version of a line
if(!tokensLine.CanStillBeUpdatedBy(CompilationStep.Scanner))
{
throw new InvalidOperationException("Can not update this TokensLine because it was already frozen by compilation step : " + tokensLine.CompilationStep.ToString());
}
// Set the initial scan state for the line
tokensLine.InitializeScanState(initialScanState);
// Alias to refer to Cobol text line properties
ICobolTextLine textLine = tokensLine;
// The source section of this line of text must be split into tokens
string line = tokensLine.Text;
int startIndex = textLine.Source.StartIndex;
int lastIndex = textLine.Source.EndIndex;
#if EUROINFO_LEGACY_REPLACING_SYNTAX
if (IsInsideRemarks(textLine.Type, tokensLine.SourceText)) tokensLine.ScanState.InsideRemarksDirective = true;
// Try to scan REMARKS compiler directive parameters inside the comment or non-comment line
if (tokensLine.ScanState.InsideRemarksDirective) {
string remarksLine = textLine.SourceText;
int startIndexForSignificantPart = GetStartIndexOfSignificantPart(remarksLine, tokensLine.ScanState);
int firstPeriodIndex = remarksLine.IndexOf('.', startIndexForSignificantPart);
int endIndexForSignificantPart = GetEndIndexOfSignificantPart(remarksLine, tokensLine.ScanState, firstPeriodIndex);
string significantPart = remarksLine.Substring(startIndexForSignificantPart, endIndexForSignificantPart - startIndexForSignificantPart + 1).Trim();
if (firstPeriodIndex >= 0 || (!tokensLine.ScanState.InsideRemarksParentheses && !remarksLine.Contains("COPY"))) {
tokensLine.ScanState.InsideRemarksDirective = false; // indicates the end of the REMARKS compiler directive
}
RemarksDirective remarksDirective = CreateRemarksDirective(significantPart, tokensLine.ScanState);
if (remarksDirective != null && remarksDirective.CopyTextNamesVariations.Count > 0) {
// A non empty remarks directive will replace the comment line
tokensLine.AddToken(CreateCompilerDirectiveToken(remarksDirective, tokensLine, startIndex, lastIndex));
return;
}
}
#endif
// Comment line => return only one token with type CommentLine
// Debug line => treated as a comment line if debugging mode was not activated
if (textLine.Type == CobolTextLineType.Comment ||
(textLine.Type == CobolTextLineType.Debug && !tokensLine.InitialScanState.WithDebuggingMode))
{
Token commentToken = new Token(TokenType.CommentLine, startIndex, lastIndex, tokensLine);
tokensLine.AddToken(commentToken);
return;
}
// Invalid indicator, the line type is unknown => the whole line text is handled as a single invalid token
else if (textLine.Type == CobolTextLineType.Invalid)
{
// Invalid indicator => register an error
tokensLine.AddDiagnostic(MessageCode.InvalidIndicatorCharacter, textLine.Indicator.StartIndex, textLine.Indicator.EndIndex, textLine.Indicator);
Token invalidToken = new Token(TokenType.InvalidToken, startIndex, lastIndex, tokensLine);
tokensLine.AddToken(invalidToken);
return;
}
// Empty line => return immediately an empty list of tokens
// Blank line => return only one token with type SpaceSeparator
if(textLine.Type == CobolTextLineType.Blank)
{
if(!String.IsNullOrEmpty(line))
{
Token whitespaceToken = new Token(TokenType.SpaceSeparator, startIndex, lastIndex, tokensLine);
tokensLine.AddToken(whitespaceToken);
}
return;
}
// Create a stateful line scanner, and iterate over the tokens
Scanner scanner = new Scanner(line, startIndex, lastIndex, tokensLine, compilerOptions);
Token nextToken = null;
while((nextToken = scanner.GetNextToken()) != null)
{
// Resolve DELETE ambiguity : DELETE + InterLiteral => DELETE_CD
// Warning : DELETE and the sequence-number-field must be on the same line
if(nextToken.TokenType == TokenType.IntegerLiteral && tokensLine.ScanState.KeywordsState == KeywordsSequenceState.After_DELETE)
{
tokensLine.ScanState.LastKeywordOrSymbolToken.CorrectType(TokenType.DELETE_CD);
}
tokensLine.AddToken(nextToken);
}
}
/// <summary>
/// Scan a line of a document when no previous scan state object is available
/// </summary>
public static void ScanFirstLine(TokensLine tokensLine, bool insideDataDivision, bool decimalPointIsComma, bool withDebuggingMode, Encoding encodingForAlphanumericLiterals, TypeCobolOptions compilerOptions)
{
MultilineScanState initialScanState = new MultilineScanState(insideDataDivision, decimalPointIsComma, withDebuggingMode, encodingForAlphanumericLiterals);
ScanTokensLine(tokensLine, initialScanState, compilerOptions);
}
private static Token CreateCompilerDirectiveToken(RemarksDirective remarksDirective, TokensLine tokensLine, int start, int end)
{
tokensLine.ScanState.AddCopyTextNamesVariations(remarksDirective.CopyTextNamesVariations);
IList<Token> originalTokens = new List<Token>(1);
originalTokens.Add(new Token(TokenType.CommentLine, start,end, tokensLine));
return new CompilerDirectiveToken(remarksDirective, originalTokens, false);
}
