public void CopyToPrevious(RecognizedFlags target)
{
target.Character = Character;
target.IsSpace = IsSpace;
target.IsLineTerminator = IsLineTerminator;
target.IsDecimalCandidate = IsDecimalCandidate;
target.LineCommentCandidate = LineCommentCandidate;
target.BlockCommentBeginCandidate = BlockCommentBeginCandidate;
target.BlockCommentEndCandidate = BlockCommentEndCandidate;
target.ConcatenationCandidate = ConcatenationCandidate;
target.OptionalParameterCandidate = OptionalParameterCandidate;
target.RelationalOperatorCandidate = RelationalOperatorCandidate;
target.QuotedStringCandidate = QuotedStringCandidate;
target.UnicodeCandidate = UnicodeCandidate;
target.StringEndCandidate = StringEndCandidate;
target.ExponentCandidate = ExponentCandidate;
target.AssignmentOperatorCandidate = AssignmentOperatorCandidate;
target.LabelMarkerCandidate = LabelMarkerCandidate;
target.IsDigit = IsDigit;
target.IsDataTypePostFix = IsDataTypePostFix;
}
private static void FindFlags(SpecialMode specialMode, char character, RecognizedFlags flags)
{
flags.Reset();
flags.Character = character;
var specialModeFlags = specialMode.Flags;
var inBlockComment = (specialModeFlags & SpecialModeFlags.InBlockComment) != 0;
var notInComment = !inBlockComment && (specialModeFlags & SpecialModeFlags.InLineComment) == 0;
var inString = (specialModeFlags & SpecialModeFlags.InString) != 0;
var notInSpecialMode = specialModeFlags == 0;
switch (character)
{
case '/':
flags.BlockCommentBeginCandidate = notInComment;
break;
case AsteriskCharacter:
flags.BlockCommentEndCandidate = inBlockComment;
break;
case '-':
flags.LineCommentCandidate = notInComment;
break;
case '|':
flags.ConcatenationCandidate = notInSpecialMode;
break;
case '=':
flags.OptionalParameterCandidate = notInSpecialMode;
break;
case ':':
flags.AssignmentOperatorCandidate = notInSpecialMode;
break;
case '>':
case '<':
flags.LabelMarkerCandidate = notInSpecialMode;
flags.RelationalOperatorCandidate = notInSpecialMode;
break;
case '^':
case '!':
flags.RelationalOperatorCandidate = notInSpecialMode;
break;
case 'e':
case 'E':
flags.ExponentCandidate = specialMode.InNumber && notInSpecialMode;
break;
case 'd':
case 'D':
case 'f':
case 'F':
flags.IsDataTypePostFix = specialMode.InNumber && notInSpecialMode;
break;
case 'n':
case 'N':
flags.UnicodeCandidate = !inString;
break;
case 'q':
case 'Q':
flags.QuotedStringCandidate = !inString;
break;
case SingleQuoteCharacter:
flags.StringEndCandidate = inString;
break;
case '.':
flags.IsDecimalCandidate = notInSpecialMode;
break;
case ' ':
case '\u00A0':
case '\t':
flags.IsSpace = true;
break;
case '\r':
case '\n':
flags.IsLineTerminator = true;
break;
default:
flags.IsDigit = character >= 48 && character <= 57;
break;
}
}
public IEnumerable <OracleToken> GetTokens(bool includeCommentBlocks = false)
{
var builder = new StringBuilder();
var index = 0;
var inQuotedString = false;
var sqlPlusTerminatorCandidate = false;
var candidateCharacterCode = EmptyCharacterCode;
char?quotingInitializer = null;
var quotingInitializerIndex = 0;
var specialMode = new SpecialMode();
var flags = new RecognizedFlags();
var previousFlags = new RecognizedFlags();
int characterCode;
while ((characterCode = _sqlReader.Read()) != EmptyCharacterCode)
{
index++;
var character = (char)characterCode;
flags.CopyToPrevious(previousFlags);
FindFlags(specialMode, character, flags);
var isBlank = flags.IsSpace || flags.IsLineTerminator;
var isSingleCharacterTerminal = character == ',' || character == '(' || character == ')' || character == '+' || character == ';' || character == '@' || character == '[' || character == ']' || character == '{' || character == '}' || character == '%' || character == '?';
var characterYielded = false;
if (flags.BlockCommentBeginCandidate && previousFlags.IsLineTerminator)
{
sqlPlusTerminatorCandidate = true;
}
else
{
if (sqlPlusTerminatorCandidate && flags.IsLineTerminator)
{
builder.Append("\n/\n");
candidateCharacterCode = EmptyCharacterCode;
yield return(BuildToken(builder, index));
previousFlags.BlockCommentBeginCandidate = false;
}
sqlPlusTerminatorCandidate = false;
}
if ((flags.QuotedStringCandidate || flags.UnicodeCandidate) &&
builder.Length > 0 && character != 'q' && character != 'Q' && previousFlags.Character != 'n' && previousFlags.Character != 'N')
{
flags.QuotedStringCandidate = false;
flags.UnicodeCandidate = false;
}
if (inQuotedString && quotingInitializer == null)
{
quotingInitializer = character;
quotingInitializerIndex = index;
}
if (previousFlags.OptionalParameterCandidate)
{
if (builder.Length > 0)
{
yield return(BuildToken(builder, index - 2));
}
builder.Append(previousFlags.Character);
var indexOffset = 1;
if (character == '>')
{
builder.Append(character);
indexOffset = 0;
characterYielded = true;
flags.RelationalOperatorCandidate = false;
}
yield return(BuildToken(builder, index - indexOffset));
candidateCharacterCode = EmptyCharacterCode;
}
if ((specialMode.Flags & SpecialModeFlags.InString) != 0)
{
var isSimpleStringTerminator = !inQuotedString && previousFlags.StringEndCandidate && character != SingleQuoteCharacter;
var isQuotedStringTerminator = inQuotedString && character == SingleQuoteCharacter && index > quotingInitializerIndex + 1 && IsQuotedStringClosingCharacter(quotingInitializer.Value, previousFlags.Character);
if (isSimpleStringTerminator || isQuotedStringTerminator)
{
AppendCandidateCharacter(builder, ref candidateCharacterCode);
var indexOffset = isQuotedStringTerminator ? 0 : 1;
if (isQuotedStringTerminator)
{
builder.Append(character);
}
yield return(BuildToken(builder, index - indexOffset));
specialMode.Flags &= ~SpecialModeFlags.InString;
inQuotedString = false;
quotingInitializer = null;
characterYielded = isQuotedStringTerminator;
FindFlags(specialMode, character, flags);
}
else if (previousFlags.StringEndCandidate && flags.StringEndCandidate && character == SingleQuoteCharacter)
{
flags.StringEndCandidate = false;
}
}
else if (specialMode.Flags == 0 && character == SingleQuoteCharacter)
{
AppendCandidateCharacter(builder, ref candidateCharacterCode);
if (builder.Length > 0 && !previousFlags.QuotedStringCandidate && !previousFlags.UnicodeCandidate)
{
yield return(BuildToken(builder, index - 1));
}
inQuotedString = previousFlags.QuotedStringCandidate;
specialMode.Flags |= SpecialModeFlags.InString;
previousFlags.AssignmentOperatorCandidate = false;
}
if (previousFlags.BlockCommentBeginCandidate && specialMode.Flags == 0)
{
if (builder.Length > 0)
{
yield return(BuildToken(builder, index - 2));
}
if (character == AsteriskCharacter)
{
specialMode.Flags |= SpecialModeFlags.InBlockComment;
AppendCandidateCharacter(builder, ref candidateCharacterCode);
}
else
{
yield return(BuildToken((char)candidateCharacterCode, index - 1));
}
candidateCharacterCode = EmptyCharacterCode;
flags.BlockCommentEndCandidate = false;
}
if (previousFlags.LineCommentCandidate && specialMode.Flags == 0)
{
if (builder.Length > 0)
{
yield return(BuildToken(builder, index - 2));
}
AppendCandidateCharacter(builder, ref candidateCharacterCode);
if (character == '-')
{
specialMode.Flags |= SpecialModeFlags.InLineComment;
}
else
{
yield return(BuildToken(builder, index - 1));
}
}
if (flags.IsDecimalCandidate)
{
if (previousFlags.IsDecimalCandidate)
{
if (builder.Length > 0)
{
yield return(BuildToken(builder, index - 2));
}
AppendCandidateCharacter(builder, ref candidateCharacterCode);
builder.Append(character);
yield return(BuildToken(builder, index));
specialMode.InNumber = false;
flags.IsDecimalCandidate = false;
characterYielded = true;
}
else if (specialMode.InDecimalNumber || (!specialMode.InNumber && builder.Length > 0))
{
AppendCandidateCharacter(builder, ref candidateCharacterCode);
if (builder.Length > 0)
{
yield return(BuildToken(builder, index - 1));
}
specialMode.InNumber = false;
}
}
if (flags.IsDigit && builder.Length == 0)
{
specialMode.InNumber = true;
}
else if (!flags.IsDecimalCandidate && specialMode.InNumber)
{
if (!specialMode.InExponent && previousFlags.ExponentCandidate)
{
var isSign = character == '+' || character == '-';
if (isSign || flags.IsDigit)
{
specialMode.InExponent = true;
isSingleCharacterTerminal = false;
flags.LineCommentCandidate = false;
}
else
{
if (builder.Length > 0)
{
yield return(BuildToken(builder, index - 2));
}
specialMode.InNumber = false;
}
AppendCandidateCharacter(builder, ref candidateCharacterCode);
}
else if (!flags.ExponentCandidate && !flags.IsDigit)
{
if (specialMode.InPostfixedNumber || !flags.IsDataTypePostFix)
{
AppendCandidateCharacter(builder, ref candidateCharacterCode);
if (builder.Length > 0)
{
yield return(BuildToken(builder, index - 1));
}
specialMode.InNumber = false;
flags.ExponentCandidate = false;
}
else if (!specialMode.InPostfixedNumber && flags.IsDataTypePostFix)
{
specialMode.InPostfixedNumber = true;
}
}
}
if (previousFlags.IsDecimalCandidate)
{
if (specialMode.InNumber && !specialMode.InDecimalNumber)
{
AppendCandidateCharacter(builder, ref candidateCharacterCode);
specialMode.InDecimalNumber = true;
}
else if (candidateCharacterCode > EmptyCharacterCode)
{
yield return(BuildToken((char)candidateCharacterCode, index - 1));
specialMode.InNumber = false;
candidateCharacterCode = EmptyCharacterCode;
}
}
if (flags.AssignmentOperatorCandidate && builder.Length > 0)
{
yield return(BuildToken(builder, index - 1));
}
if (previousFlags.AssignmentOperatorCandidate)
{
if (character == '=')
{
AppendCandidateCharacter(builder, ref candidateCharacterCode);
if (builder.Length > 0)
{
builder.Append(character);
yield return(BuildToken(builder, index));
}
flags.OptionalParameterCandidate = false;
characterYielded = true;
}
else
{
if (builder.Length > 0)
{
yield return(BuildToken(builder, index - 2));
}
yield return(BuildToken(previousFlags.Character, index - 1));
candidateCharacterCode = EmptyCharacterCode;
}
}
if ((specialMode.Flags & SpecialModeFlags.InQuotedIdentifier) != 0 && character == '"')
{
AppendCandidateCharacter(builder, ref candidateCharacterCode);
if (builder.Length > 0)
{
builder.Append(character);
yield return(BuildToken(builder, index));
}
characterYielded = true;
specialMode.Flags &= ~SpecialModeFlags.InQuotedIdentifier;
}
else if (specialMode.Flags == 0 && character == '"')
{
AppendCandidateCharacter(builder, ref candidateCharacterCode);
if (builder.Length > 0)
{
yield return(BuildToken(builder, index - 1));
}
specialMode.Flags |= SpecialModeFlags.InQuotedIdentifier;
}
if (previousFlags.RelationalOperatorCandidate)
{
if (builder.Length > 0)
{
yield return(BuildToken(builder, index - 2));
}
if (character == '=' || (character == '>' && previousFlags.Character == '<') || (previousFlags.LabelMarkerCandidate && character == previousFlags.Character))
{
builder.Append(previousFlags.Character);
builder.Append(character);
yield return(BuildToken(builder, index));
characterYielded = true;
}
else
{
yield return(BuildToken(previousFlags.Character, index - 1));
}
candidateCharacterCode = EmptyCharacterCode;
flags.RelationalOperatorCandidate = false;
flags.LabelMarkerCandidate = false;
flags.OptionalParameterCandidate = false;
}
if (previousFlags.ConcatenationCandidate)
{
if (builder.Length > 0)
{
yield return(BuildToken(builder, index - 2));
}
builder.Append(previousFlags.Character);
var indexOffset = 1;
if (character == '|')
{
builder.Append(character);
indexOffset = 0;
characterYielded = true;
flags.ConcatenationCandidate = false;
}
yield return(BuildToken(builder, index - indexOffset));
candidateCharacterCode = EmptyCharacterCode;
}
if (specialMode.Flags == 0 && (isBlank || isSingleCharacterTerminal || character == AsteriskCharacter))
{
specialMode.InNumber = false;
AppendCandidateCharacter(builder, ref candidateCharacterCode);
if (builder.Length > 0)
{
yield return(BuildToken(builder, index - 1));
}
if (!isBlank)
{
characterYielded = true;
yield return(BuildToken(character, index));
}
}
if (flags.IsLineTerminator && (specialMode.Flags & SpecialModeFlags.InLineComment) != 0)
{
if (includeCommentBlocks)
{
builder.Append(character);
yield return(BuildToken(builder, index, CommentType.Line));
}
else
{
builder.Clear();
}
characterYielded = true;
specialMode.Flags &= ~(SpecialModeFlags.InBlockComment | SpecialModeFlags.InLineComment);
candidateCharacterCode = EmptyCharacterCode;
}
if (previousFlags.BlockCommentEndCandidate && (specialMode.Flags & SpecialModeFlags.InBlockComment) != 0)
{
if (character == '/')
{
specialMode.Flags &= ~(SpecialModeFlags.InBlockComment | SpecialModeFlags.InLineComment);
if (includeCommentBlocks)
{
builder.Append(character);
yield return(BuildToken(builder, index, CommentType.Block));
}
else
{
builder.Clear();
}
characterYielded = true;
}
candidateCharacterCode = EmptyCharacterCode;
flags.BlockCommentBeginCandidate = false;
}
if (!characterYielded)
{
var candidateMode = flags.ConcatenationCandidate || flags.OptionalParameterCandidate || flags.LineCommentCandidate || flags.BlockCommentBeginCandidate ||
flags.RelationalOperatorCandidate || flags.IsDecimalCandidate || flags.ExponentCandidate || flags.AssignmentOperatorCandidate || flags.LabelMarkerCandidate;
if (candidateMode && specialMode.Flags == 0)
{
candidateCharacterCode = characterCode;
}
else if (!isBlank || specialMode.Flags > 0)
{
builder.Append(character);
}
}
}
var commentType = (CommentType)(specialMode.Flags & (SpecialModeFlags.InLineComment | SpecialModeFlags.InBlockComment));
if (commentType == CommentType.None || includeCommentBlocks)
{
var indexOffset = candidateCharacterCode == EmptyCharacterCode ? 0 : 1;
if (builder.Length > 0)
{
yield return(new OracleToken(builder.ToString(), index - builder.Length - indexOffset, commentType));
}
if (candidateCharacterCode != EmptyCharacterCode)
{
yield return(new OracleToken(new String((char)candidateCharacterCode, 1), index - indexOffset, commentType));
}
}
}