public void OnToken(Token token)
{
TokenType tokenType = token.TokenType;
TokenFamily tokenFamily = TokenUtils.GetTokenFamilyFromTokenType(tokenType);
if (LanguageModel.IsSignificantWord(tokenType, tokenFamily))
{
RegisterToken(token);
if (LanguageModel.IsKeywordToken(tokenType, tokenFamily))
{
lastKeywordToken = tokenType;
}
if (LanguageModel.IsElementStartingWord(tokenType, tokenFamily, lastWord))
{
if (tokenType == TokenType.ID || tokenType == TokenType.IDENTIFICATION)
{
elementStartingWordIndexInProgram = 0;
}
OnBeginElement(tokenType);
}
lastWord = tokenType;
wordIndexInElement++;
}
}
internal static bool IsKeywordToken(TokenType tokenType, TokenFamily tokenFamily)
{
return(tokenFamily >= TokenFamily.CompilerDirectiveStartingKeyword &&
tokenFamily != TokenFamily.SpecialRegisterKeyword &&
tokenFamily != TokenFamily.FigurativeConstantKeyword &&
tokenFamily != TokenFamily.TypeCobolOperators);
}
private IClassificationType GetClassificationType(TokenFamily family)
{
string type;
if (!mapTokenFamilyToClassificationType.TryGetValue(family, out type))
throw new ArgumentException("Unknown token family \""+family.ToString()+"\"");
return registry.GetClassificationType(type);
}
public static string GetDisplayNameForTokenFamily(TokenFamily tokenFamily)
{
switch (tokenFamily)
{
case TokenFamily.Whitespace:
return("whitespace");
case TokenFamily.Comments:
return("comments");
case TokenFamily.SyntaxSeparator:
return("separator");
case TokenFamily.ArithmeticOperator:
return("arithmetic operator");
case TokenFamily.RelationalOperator:
return("relational operator");
case TokenFamily.AlphanumericLiteral:
return("alphanumeric literal");
case TokenFamily.NumericLiteral:
return("numeric literal");
case TokenFamily.SyntaxLiteral:
return("character string");
case TokenFamily.Symbol:
return("symbol");
case TokenFamily.CompilerDirectiveStartingKeyword:
return("compiler directive starting keyword");
case TokenFamily.CodeElementStartingKeyword:
return("statement starting keyword");
case TokenFamily.SpecialRegisterKeyword:
return("special register");
case TokenFamily.FigurativeConstantKeyword:
return("figurative constant");
case TokenFamily.SpecialObjetIdentifierKeyword:
return("special object identifier");
case TokenFamily.SyntaxKeyword:
return("keyword");
case TokenFamily.Cobol2002Keyword:
return("cobol 2002 keyword");
case TokenFamily.TypeCobolKeyword:
return("TypeCobol keyword");
default:
return("...");
}
}
private IClassificationType GetClassificationType(TokenFamily family)
{
string type;
if (!mapTokenFamilyToClassificationType.TryGetValue(family, out type))
{
throw new ArgumentException("Unknown token family \"" + family.ToString() + "\"");
}
return(registry.GetClassificationType(type));
}
static TokenUtils()
{
// Map token types to token families
var types = (TokenType[])Enum.GetValues(typeof(TokenType));
var families = (TokenFamily[])Enum.GetValues(typeof(TokenFamily));
tokenFamilyFromTokenType = new TokenFamily[types.Length];
int family = 0;
for (int tokenType = 0; tokenType < types.Length - 1; tokenType++)
{
if (family < (families.Length - 1) && tokenType == (int)families[family + 1])
{
family++;
}
tokenFamilyFromTokenType[tokenType] = families[family];
}
// Register the token strings corresponding to each token type (for keywords only)
int keywordBegin = (int)TokenType.UserDefinedWord + 1;
int keywordEnd = (int)TokenType.CompilerDirective - 1;
tokenStringFromTokenType = new string[types.Length];
for (int c = keywordBegin; c < types.Length; c++)
{
var current = types[c];
if ((int)current > keywordEnd)
{
break;
}
tokenStringFromTokenType[(int)current] = current.ToString().Replace('_', '-');
}
tokenStringFromTokenType[(int)TokenType.ASTERISK_CBL] = "*CBL";
tokenStringFromTokenType[(int)TokenType.ASTERISK_CONTROL] = "*CONTROL";
tokenStringFromTokenType[(int)TokenType.DELETE_CD] = "DELETE";
tokenStringFromTokenType[(int)TokenType.SERVICE_CD] = "SERVICE";
tokenStringFromTokenType[(int)TokenType.EXEC_SQL_INCLUDE] = "INCLUDE";
// Map token string to token type
tokenTypeFromTokenString = new Dictionary <string, TokenType>(types.Length - 1, StringComparer.OrdinalIgnoreCase);
for (int tokenType = 0; tokenType < types.Length; tokenType++)
{
string tokenString = tokenStringFromTokenType[tokenType];
if (!String.IsNullOrEmpty(tokenString) && !tokenTypeFromTokenString.ContainsKey(tokenString))
{
tokenTypeFromTokenString[tokenString] = (TokenType)tokenType;
}
}
// Token type DELETE is much more frequent than DELETE_CD, it should have priority
tokenTypeFromTokenString["DELETE"] = TokenType.DELETE;
// Token type SERVICE is much more frequent than SERVICE_CD, it should have priority
tokenTypeFromTokenString["SERVICE"] = TokenType.SERVICE;
}
/// <summary>
/// Call this method each time you encounter a new Token while iterating
/// on the program lines.
/// It is not necessary to start iterating at the beginning of the program,
/// but you must start iterating before the last element starting
/// keyword preceding the current token.
/// </summary>
public void OnToken(Token token)
{
TokenType tokenType = token.TokenType;
TokenFamily tokenFamily = TokenUtils.GetTokenFamilyFromTokenType(tokenType);
if (IsSignificantWord(tokenType, tokenFamily))
{
if (IsKeywordToken(tokenType, tokenFamily))
{
lastKeywordToken = lastWord;
}
if (IsElementStartingWord(tokenType, tokenFamily, lastWord))
{
lastElementStartingWord = tokenType;
}
lastWord = tokenType;
}
}
public async Task <CreateTokenResponse> Handle(CreateTokenRequest request, CancellationToken cancellationToken)
{
var authenticationResult =
await _authenticationService.AuthenticateByEmail(request.Email, request.Password);
if (!authenticationResult)
{
return new CreateTokenResponse {
Errors = new List <string> {
"Invalid authentication"
}
}
}
;
var user = await _userService.GetUserByEmail(request.Email);
if (user == null)
{
return new CreateTokenResponse {
Errors = new List <string> {
"Invalid user"
}
}
}
;
var authenticationToken = _tokenService.CreateToken(TokenType.Authentication, user);
var refreshToken = request.CreateRefreshToken ? _tokenService.CreateToken(TokenType.Refresh, user) : null;
var tokenFamily = new TokenFamily
{
AuthenticationToken = authenticationToken,
RefreshToken = refreshToken
};
return(new CreateTokenResponse {
WorkProduct = tokenFamily
});
}
}
}
internal static bool IsSymbolOrLiteral(TokenType tokenType, TokenFamily tokenFamily)
{
return(tokenType == TokenType.UserDefinedWord || tokenType == TokenType.PartialCobolWord ||
tokenFamily == TokenFamily.NumericLiteral || tokenFamily == TokenFamily.AlphanumericLiteral);
}
internal static bool IsSignificantWord(TokenType tokenType, TokenFamily tokenFamily)
{
return tokenFamily != TokenFamily.Whitespace && tokenFamily != TokenFamily.Comments && tokenType != TokenType.CompilerDirective &&
tokenType != TokenType.EJECT && tokenType != TokenType.SKIP1 && tokenType != TokenType.SKIP2 && tokenType != TokenType.SKIP3;
}
internal static bool IsKeywordToken(TokenType tokenType, TokenFamily tokenFamily)
{
return tokenFamily >= TokenFamily.CompilerDirectiveStartingKeyword &&
tokenFamily != TokenFamily.SpecialRegisterKeyword &&
tokenFamily != TokenFamily.FigurativeConstantKeyword;
}
public static bool CppStyleEvaluator(IProcessorState processor, ref int bufferLength, ref int currentBufferPosition)
{
TokenTrie trie = new TokenTrie();
//Logic
trie.AddToken(processor.Encoding.GetBytes("&&"), 0);
trie.AddToken(processor.Encoding.GetBytes("||"), 1);
trie.AddToken(processor.Encoding.GetBytes("^"), 2);
trie.AddToken(processor.Encoding.GetBytes("!"), 3);
trie.AddToken(processor.Encoding.GetBytes(">"), 4);
trie.AddToken(processor.Encoding.GetBytes(">="), 5);
trie.AddToken(processor.Encoding.GetBytes("<"), 6);
trie.AddToken(processor.Encoding.GetBytes("<="), 7);
trie.AddToken(processor.Encoding.GetBytes("=="), 8);
trie.AddToken(processor.Encoding.GetBytes("="), 9);
trie.AddToken(processor.Encoding.GetBytes("!="), 10);
//Bitwise
trie.AddToken(processor.Encoding.GetBytes("&"), 11);
trie.AddToken(processor.Encoding.GetBytes("|"), 12);
trie.AddToken(processor.Encoding.GetBytes("<<"), 13);
trie.AddToken(processor.Encoding.GetBytes(">>"), 14);
//Braces
trie.AddToken(processor.Encoding.GetBytes("("), 15);
trie.AddToken(processor.Encoding.GetBytes(")"), 16);
//Whitespace
trie.AddToken(processor.Encoding.GetBytes(" "), 17);
trie.AddToken(processor.Encoding.GetBytes("\t"), 18);
//EOLs
trie.AddToken(processor.Encoding.GetBytes("\r\n"), 19);
trie.AddToken(processor.Encoding.GetBytes("\n"), 20);
trie.AddToken(processor.Encoding.GetBytes("\r"), 21);
//Tokens
trie.Append(processor.EncodingConfig.Variables);
//Run forward to EOL and collect args
TokenFamily currentTokenFamily;
List <byte> currentTokenBytes = new List <byte>();
List <TokenRef> tokens = new List <TokenRef>();
int token;
if (!trie.GetOperation(processor.CurrentBuffer, bufferLength, ref currentBufferPosition, out token))
{
currentTokenFamily = TokenFamily.Literal;
currentTokenBytes.Add(processor.CurrentBuffer[currentBufferPosition++]);
}
else if (token > ReservedTokenMaxIndex)
{
currentTokenFamily = TokenFamily.Reference | (TokenFamily)token;
tokens.Add(new TokenRef
{
Family = currentTokenFamily
});
}
else
{
currentTokenFamily = (TokenFamily)token;
if (currentTokenFamily != TokenFamily.WindowsEOL && currentTokenFamily != TokenFamily.LegacyMacEOL && currentTokenFamily != TokenFamily.UnixEOL)
{
tokens.Add(new TokenRef
{
Family = currentTokenFamily
});
}
else
{
return(EvaluateCondition(tokens, processor.EncodingConfig.VariableValues));
}
}
int braceDepth = 0;
if (tokens[0].Family == TokenFamily.OpenBrace)
{
++braceDepth;
}
bool first = true;
while ((first || braceDepth > 0) && bufferLength > 0)
{
int targetLen = Math.Min(bufferLength, trie.MaxLength);
for (; currentBufferPosition < bufferLength - targetLen + 1;)
{
int oldBufferPos = currentBufferPosition;
if (trie.GetOperation(processor.CurrentBuffer, bufferLength, ref currentBufferPosition, out token))
{
if (braceDepth == 0)
{
switch (tokens[tokens.Count - 1].Family)
{
case TokenFamily.Whitespace:
case TokenFamily.Tab:
case TokenFamily.CloseBrace:
case TokenFamily.WindowsEOL:
case TokenFamily.UnixEOL:
case TokenFamily.LegacyMacEOL:
TokenFamily thisFamily = (TokenFamily)token;
if (thisFamily == TokenFamily.WindowsEOL || thisFamily == TokenFamily.UnixEOL || thisFamily == TokenFamily.LegacyMacEOL)
{
currentBufferPosition = oldBufferPos;
}
break;
default:
currentBufferPosition = oldBufferPos;
first = false;
break;
}
if (!first)
{
break;
}
}
//We matched an item, so whatever this is, it's not a literal, end the current literal if that's
// what we currently have
if (currentTokenFamily == TokenFamily.Literal)
{
string literal = processor.Encoding.GetString(currentTokenBytes.ToArray());
tokens.Add(new TokenRef
{
Family = TokenFamily.Literal,
Literal = literal
});
currentTokenBytes.Clear();
}
//If we have a token from the args...
if (token > ReservedTokenMaxIndex)
{
if (currentTokenFamily == TokenFamily.Literal)
{
TokenRef previous = tokens[tokens.Count - 1];
previous.Literal += processor.Encoding.GetString(currentTokenBytes.ToArray());
currentTokenBytes = processor.Encoding.GetBytes(previous.Literal).ToList();
tokens.RemoveAt(tokens.Count - 1);
}
else
{
currentTokenFamily = TokenFamily.Reference | (TokenFamily)token;
tokens.Add(new TokenRef
{
Family = currentTokenFamily
});
}
}
//If we have a normal token...
else
{
currentTokenFamily = (TokenFamily)token;
if (currentTokenFamily != TokenFamily.WindowsEOL && currentTokenFamily != TokenFamily.LegacyMacEOL && currentTokenFamily != TokenFamily.UnixEOL)
{
if (currentTokenFamily == TokenFamily.OpenBrace)
{
++braceDepth;
}
else if (currentTokenFamily == TokenFamily.CloseBrace)
{
--braceDepth;
}
tokens.Add(new TokenRef
{
Family = currentTokenFamily
});
}
else
{
return(EvaluateCondition(tokens, processor.EncodingConfig.VariableValues));
}
}
}
else if (braceDepth > 0)
{
currentTokenFamily = TokenFamily.Literal;
currentTokenBytes.Add(processor.CurrentBuffer[currentBufferPosition++]);
}
else
{
first = false;
break;
}
}
processor.AdvanceBuffer(currentBufferPosition);
currentBufferPosition = processor.CurrentBufferPosition;
bufferLength = processor.CurrentBufferLength;
}
return(EvaluateCondition(tokens, processor.EncodingConfig.VariableValues));
}
public static bool Evaluate(IProcessorState processor, ref int bufferLength, ref int currentBufferPosition, out bool faulted)
{
faulted = false;
TokenTrie trie = new TokenTrie();
//Logic
trie.AddToken(processor.Encoding.GetBytes("&&"), 0);
trie.AddToken(processor.Encoding.GetBytes("||"), 1);
trie.AddToken(processor.Encoding.GetBytes("^"), 2);
trie.AddToken(processor.Encoding.GetBytes("!"), 3);
trie.AddToken(processor.Encoding.GetBytes(">"), 4);
trie.AddToken(processor.Encoding.GetBytes(">="), 5);
trie.AddToken(processor.Encoding.GetBytes("<"), 6);
trie.AddToken(processor.Encoding.GetBytes("<="), 7);
trie.AddToken(processor.Encoding.GetBytes("=="), 8);
trie.AddToken(processor.Encoding.GetBytes("="), 9);
trie.AddToken(processor.Encoding.GetBytes("!="), 10);
//Bitwise
trie.AddToken(processor.Encoding.GetBytes("&"), 11);
trie.AddToken(processor.Encoding.GetBytes("|"), 12);
trie.AddToken(processor.Encoding.GetBytes("<<"), 13);
trie.AddToken(processor.Encoding.GetBytes(">>"), 14);
//Braces
trie.AddToken(processor.Encoding.GetBytes("("), 15);
trie.AddToken(processor.Encoding.GetBytes(")"), 16);
//Whitespace
trie.AddToken(processor.Encoding.GetBytes(" "), 17);
trie.AddToken(processor.Encoding.GetBytes("\t"), 18);
//EOLs
trie.AddToken(processor.Encoding.GetBytes("\r\n"), 19);
trie.AddToken(processor.Encoding.GetBytes("\n"), 20);
trie.AddToken(processor.Encoding.GetBytes("\r"), 21);
// quotes
trie.AddToken(processor.Encoding.GetBytes("\""), 22);
trie.AddToken(processor.Encoding.GetBytes("'"), 23);
//Tokens
trie.Append(processor.EncodingConfig.Variables);
//Run forward to EOL and collect args
TokenFamily currentTokenFamily;
List <byte> currentTokenBytes = new List <byte>();
List <TokenRef> tokens = new List <TokenRef>();
if (!trie.GetOperation(processor.CurrentBuffer, bufferLength, ref currentBufferPosition, out int token))
{
currentTokenFamily = TokenFamily.Literal;
currentTokenBytes.Add(processor.CurrentBuffer[currentBufferPosition++]);
}
else if (token > ReservedTokenMaxIndex)
{
currentTokenFamily = TokenFamily.Reference | (TokenFamily)token;
tokens.Add(new TokenRef
{
Family = currentTokenFamily
});
}
else
{
currentTokenFamily = (TokenFamily)token;
if (currentTokenFamily != TokenFamily.WindowsEOL && currentTokenFamily != TokenFamily.LegacyMacEOL && currentTokenFamily != TokenFamily.UnixEOL)
{
tokens.Add(new TokenRef
{
Family = currentTokenFamily
});
}
else
{
return(EvaluateCondition(tokens, processor.EncodingConfig.VariableValues));
}
}
int braceDepth = 0;
if (tokens[0].Family == TokenFamily.OpenBrace)
{
++braceDepth;
}
bool first = true;
QuotedRegionKind inQuoteType = QuotedRegionKind.None;
while ((first || braceDepth > 0) && bufferLength > 0)
{
int targetLen = Math.Min(bufferLength, trie.MaxLength);
for (; currentBufferPosition < bufferLength - targetLen + 1;)
{
int oldBufferPos = currentBufferPosition;
if (trie.GetOperation(processor.CurrentBuffer, bufferLength, ref currentBufferPosition, out token))
{
if (braceDepth == 0)
{
switch (tokens[tokens.Count - 1].Family)
{
case TokenFamily.Whitespace:
case TokenFamily.Tab:
case TokenFamily.CloseBrace:
case TokenFamily.WindowsEOL:
case TokenFamily.UnixEOL:
case TokenFamily.LegacyMacEOL:
TokenFamily thisFamily = (TokenFamily)token;
if (thisFamily == TokenFamily.WindowsEOL || thisFamily == TokenFamily.UnixEOL || thisFamily == TokenFamily.LegacyMacEOL)
{
currentBufferPosition = oldBufferPos;
}
break;
default:
currentBufferPosition = oldBufferPos;
first = false;
break;
}
if (!first)
{
break;
}
}
// We matched an item, so whatever this is, it's not a literal.
// if the current token is a literal, end it.
if (currentTokenFamily == TokenFamily.Literal)
{
string literal = processor.Encoding.GetString(currentTokenBytes.ToArray());
tokens.Add(new TokenRef
{
Family = TokenFamily.Literal,
Literal = literal
});
currentTokenBytes.Clear();
}
TokenFamily foundTokenFamily = (TokenFamily)token;
if (foundTokenFamily == TokenFamily.QuotedLiteral || foundTokenFamily == TokenFamily.SingleQuotedLiteral)
{
QuotedRegionKind incomingQuoteKind;
switch (foundTokenFamily)
{
case TokenFamily.QuotedLiteral:
incomingQuoteKind = QuotedRegionKind.DoubleQuoteRegion;
break;
case TokenFamily.SingleQuotedLiteral:
incomingQuoteKind = QuotedRegionKind.SingleQuoteRegion;
break;
default:
incomingQuoteKind = QuotedRegionKind.None;
break;
}
if (inQuoteType == QuotedRegionKind.None)
{
// starting quote found
currentTokenBytes.AddRange(trie.Tokens[token].Value);
inQuoteType = incomingQuoteKind;
}
else if (incomingQuoteKind == inQuoteType)
{
// end quote found
currentTokenBytes.AddRange(trie.Tokens[token].Value);
tokens.Add(new TokenRef
{
Family = TokenFamily.Literal,
Literal = processor.Encoding.GetString(currentTokenBytes.ToArray())
});
currentTokenBytes.Clear();
inQuoteType = QuotedRegionKind.None;
}
else
{
// this is a different quote type. Treat it like a non-match, just add the token to the currentTokenBytes
currentTokenBytes.AddRange(trie.Tokens[token].Value);
}
}
else if (inQuoteType != QuotedRegionKind.None)
{
// we're inside a quoted literal, the token found by the trie should not be processed, just included with the literal
currentTokenBytes.AddRange(trie.Tokens[token].Value);
}
else if (token > ReservedTokenMaxIndex)
{
currentTokenFamily = TokenFamily.Reference | (TokenFamily)token;
tokens.Add(new TokenRef
{
Family = currentTokenFamily
});
}
else
{
//If we have a normal token...
currentTokenFamily = (TokenFamily)token;
if (currentTokenFamily != TokenFamily.WindowsEOL && currentTokenFamily != TokenFamily.LegacyMacEOL && currentTokenFamily != TokenFamily.UnixEOL)
{
switch (currentTokenFamily)
{
case TokenFamily.OpenBrace:
++braceDepth;
break;
case TokenFamily.CloseBrace:
--braceDepth;
break;
}
tokens.Add(new TokenRef
{
Family = currentTokenFamily
});
}
else
{
return(EvaluateCondition(tokens, processor.EncodingConfig.VariableValues));
}
}
}
else if (inQuoteType != QuotedRegionKind.None)
{
// we're in a quoted literal but did not match a token at the current position.
// so just add the current byte to the currentTokenBytes
currentTokenBytes.Add(processor.CurrentBuffer[currentBufferPosition++]);
}
else if (braceDepth > 0)
{
currentTokenFamily = TokenFamily.Literal;
currentTokenBytes.Add(processor.CurrentBuffer[currentBufferPosition++]);
}
else
{
first = false;
break;
}
}
processor.AdvanceBuffer(currentBufferPosition);
currentBufferPosition = processor.CurrentBufferPosition;
bufferLength = processor.CurrentBufferLength;
}
#if DEBUG
Debug.Assert(
inQuoteType == QuotedRegionKind.None,
$"Malformed predicate due to unmatched quotes. InitialBuffer = {processor.Encoding.GetString(processor.CurrentBuffer)} currentTokenFamily = {currentTokenFamily} | TokenFamily.QuotedLiteral = {TokenFamily.QuotedLiteral} | TokenFamily.SingleQuotedLiteral = {TokenFamily.SingleQuotedLiteral}");
#endif
return(EvaluateCondition(tokens, processor.EncodingConfig.VariableValues));
}
internal static bool IsSymbolOrLiteral(TokenType tokenType, TokenFamily tokenFamily)
{
return tokenType == TokenType.UserDefinedWord || tokenType == TokenType.PartialCobolWord ||
tokenFamily == TokenFamily.NumericLiteral || tokenFamily == TokenFamily.AlphanumericLiteral;
}
protected string GetListOfExpectedTokens(IntervalSet intervalSet, IVocabulary vocabulary)
{
StringBuilder buf = new StringBuilder();
if (intervalSet == null || intervalSet.Count == 0)
{
return("{}");
}
if (intervalSet.Count > 1)
{
buf.Append("{");
}
bool first = true;
// Never display more than 10 expected tokens
if (intervalSet.Count <= 10)
{
foreach (Interval I in intervalSet.GetIntervals())
{
if (!first)
{
buf.Append(", ");
}
first = false;
int a = I.a;
int b = I.b;
if (a == b)
{
buf.Append(TokenUtils.GetDisplayNameForTokenType((TokenType)a));
}
else
{
for (int i = a; i <= b; i++)
{
if (i > a)
{
buf.Append(", ");
}
buf.Append(TokenUtils.GetDisplayNameForTokenType((TokenType)i));
}
}
}
}
// If more than 10 expected tokens, display list of token families
else
{
HashSet <TokenFamily> familySet = new HashSet <TokenFamily>();
foreach (Interval I in intervalSet.GetIntervals())
{
for (int i = I.a; i <= I.b; i++)
{
TokenFamily tokenFamily = TokenUtils.GetTokenFamilyFromTokenType((TokenType)i);
if (familySet.Add(tokenFamily))
{
if (!first)
{
buf.Append(", ");
}
first = false;
buf.Append(TokenUtils.GetDisplayNameForTokenFamily(tokenFamily));
}
}
}
}
if (intervalSet.Count > 1)
{
buf.Append("}");
}
return(buf.ToString());
}
internal static bool IsElementStartingWord(TokenType tokenType, TokenFamily tokenFamily, TokenType lastWord)
{
return(tokenFamily == TokenFamily.CompilerDirectiveStartingKeyword || tokenFamily == TokenFamily.CodeElementStartingKeyword ||
tokenType == TokenType.LevelNumber || tokenType == TokenType.SectionParagraphName);
}
internal static bool IsElementStartingWord(TokenType tokenType, TokenFamily tokenFamily, TokenType lastWord)
{
return tokenFamily == TokenFamily.CompilerDirectiveStartingKeyword || tokenFamily == TokenFamily.CodeElementStartingKeyword ||
tokenFamily == TokenFamily.StatementStartingKeyword || tokenFamily == TokenFamily.StatementEndingKeyword ||
((lastWord == TokenType.InvalidToken || lastWord == TokenType.PeriodSeparator) &&
(tokenType == TokenType.IntegerLiteral || tokenType == TokenType.UserDefinedWord));
}
internal static bool IsSignificantWord(TokenType tokenType, TokenFamily tokenFamily)
{
return(tokenFamily != TokenFamily.Whitespace && tokenFamily != TokenFamily.Comments && tokenType != TokenType.CompilerDirective &&
tokenType != TokenType.EJECT && tokenType != TokenType.SKIP1 && tokenType != TokenType.SKIP2 && tokenType != TokenType.SKIP3);
}
