private void MergeWithPreviousTwoTokens( SymToken aNewToken, SymToken.TClass aNewClassType )
{
System.Diagnostics.Debug.Assert( iCache.Count > 0 );
SymToken previousToken = iCache.PopTail();
// Combine it with the new token...
previousToken.Combine( aNewToken );
previousToken.Class = aNewClassType;
// And combine any previous previous token
MergeWithPreviousToken( previousToken );
}
private void MergeWithPreviousToken( SymToken aNewToken )
{
if ( iCache.Count > 0 )
{
if ( CheckIfStateChangeRequiredForEnqueuedToken( aNewToken ) == false )
{
SymToken previousOutputToken = PreviousOutputToken;
previousOutputToken.Combine( aNewToken );
}
}
else
{
EnqueueNewOutputToken( aNewToken );
}
}
public void MergeAllTokensWithinRange(int aStartIndex, int aEndIndex, bool aMergeInContinuations, bool aForceMerge)
{
int count = Count;
//
System.Diagnostics.Debug.Assert(count > aStartIndex);
System.Diagnostics.Debug.Assert(aEndIndex < count);
// Have to do this in two passes to ensure token
// text remains from left to right.
SymToken startingToken = this[aStartIndex++];
if (aForceMerge == false)
{
// Not force-merging, so need to find a valid combinable starting element
while (startingToken.CombiningAllowed == false && aStartIndex < aEndIndex)
{
startingToken = this[++aStartIndex];
}
}
// First pass - join tokens
for (int i = aStartIndex; i <= aEndIndex; i++)
{
SymToken thisToken = this[i];
// Ignore continuations during merging
if (thisToken.Class != SymToken.TClass.EClassContinuation || aMergeInContinuations)
{
if (aForceMerge == false)
{
startingToken.Combine(thisToken);
}
else
{
startingToken.ForceCombine(thisToken);
}
}
}
// Second pass - discard merged tokens.
for (int i = aEndIndex - 1; i >= aStartIndex; i--)
{
Remove(i);
}
//System.Diagnostics.Debug.WriteLine( "Merged: " + startingToken.Value );
}
private bool CheckIfStateChangeRequiredForEnqueuedToken( SymToken aToken )
{
// NB. This method is called before aToken has been enqueued
// or in the case of combining, before the token has been combined
// with any prior token.
bool tokenProcessed = false;
if ( InQuotation )
{
}
else if ( InComment )
{
}
else if ( InPreProcessorDirective )
{
}
else
{
if ( aToken.Class == SymToken.TClass.EClassQuotation )
{
#region Handle start of quotation
if ( iCache.Count > 0 )
{
// Check whether the previous symbol was a backslash. If it was
// then this must be an escaped " or ' character, in which case
// we don't change state.
SymToken previousToken = PreviousOutputToken;
if ( previousToken.Class == SymToken.TClass.EClassSymbol && previousToken.Value == @"\" )
{
// Last character was an escape marker. Combine it
// with the quotation
previousToken.Combine( aToken );
// Already handled the token
tokenProcessed = true;
}
else
{
// Really are starting a quotation.
FlushCache();
InQuotation = true;
}
}
#endregion
}
else if ( aToken.Class == SymToken.TClass.EClassSymbol )
{
if ( aToken.Value == "*" )
{
#region Handle Opening comment block [ /* ]
if ( iCache.Count > 0 )
{
SymToken previousToken = PreviousOutputToken;
//
if ( previousToken.Class == SymToken.TClass.EClassSymbol && previousToken.Value == "/" )
{
// "/*" case
//
// In this scenario, in order to ensure that we do not
// flush the first character of our comment marker, we must
// dequeue the tail item, then flush, then enqueue.
SymToken tailToken = iCache.PopTail(); // -> this is the initial "/" that we pop...
FlushCache();
// Forward slash and asterisk are combined
tailToken.Combine( aToken );
// Mark the token as a full line comment
tailToken.Class = SymToken.TClass.EClassComment;
tailToken.Type = SymToken.TType.ETypeCommentBlock;
// ...and re-added to the cache
iCache.Append( tailToken );
// aToken was already combined so we don't want the caller
// to add it twice.
tokenProcessed = true;
// We're now in a full line comment.
InComment = true;
}
}
#endregion
}
else if ( aToken.Value == "/" )
{
#region Handle Full-Line comment [ // ]
if ( iCache.Count > 0 )
{
SymToken previousToken = PreviousOutputToken;
//
if ( previousToken.Value == aToken.Value )
{
// "//" case
//
// In this scenario, in order to ensure that we do not
// flush the first character of our comment marker, we must
// dequeue the tail item, then flush, then enqueue.
SymToken tailToken = iCache.PopTail(); // -> this is the initial "/" that we pop...
FlushCache();
// Two forward slashes are combined into one.
tailToken.Combine( aToken );
// Mark the token as a full line comment
tailToken.Class = SymToken.TClass.EClassComment;
tailToken.Type = SymToken.TType.ETypeCommentFullLine;
// ...and re-added to the cache
iCache.Append( tailToken );
// aToken was already combined so we don't want the caller
// to add it twice.
tokenProcessed = true;
// We're now in a full line comment.
InComment = true;
}
}
#endregion
}
}
else if ( aToken.Class == SymToken.TClass.EClassPreProcessor )
{
#region Handle start of preprocessor directive
// Preprocessor directives must only appear on a line
// after whitespace. If there was any non-whitespace
// characters before the preprocessor directive, then its illegal.
bool tokensAreAllWhiteSpace = iCache.CheckTokensAreOfEitherClass( SymToken.TClass.EClassWhiteSpace, SymToken.TClass.EClassNewLine );
if ( aToken.Value == "#" && tokensAreAllWhiteSpace )
{
// Starting a preprocess directive
FlushCache();
InPreProcessorDirective = true;
}
#endregion
}
}
return tokenProcessed;
}
private void ProcessTokenDuringComment( SymToken aToken )
{
#region Comment examples
// // this is a comment
// /* this is also a comment */
// // "This is another comment"
// // This is a comment with a continuation \
// and here's the rest.
#endregion
System.Diagnostics.Debug.Assert( iCache.Count > 0 );
if ( aToken.Class == SymToken.TClass.EClassSymbol && aToken.Value == "*" )
{
#region Ensure asterisk is not merged with other comments
// The asterisk character is separated from
// the rest of the comment in order that we can
// ascertain when the end of a block comment has
// been reached.
EnqueueNewOutputToken( aToken );
#endregion
}
else if ( aToken.Class == SymToken.TClass.EClassNewLine )
{
#region New line during comment...
// Checking for continuations...
SymToken previousToken = PreviousOutputToken;
if ( previousToken.Value == @"\" )
{
// Discard new line
previousToken.Class = SymToken.TClass.EClassContinuation;
}
else
{
// If we're in a block comment, then we don't flush when we
// see a new line token.
SymToken firstToken = iCache.PeekHead;
EnqueueNewOutputToken( aToken );
//
if ( firstToken.Type == SymToken.TType.ETypeCommentFullLine )
{
// Flushing the cache resets the flags...
FlushCache();
}
else if ( firstToken.Type == SymToken.TType.ETypeCommentBlock )
{
// Don't end the comment until we see the closing block token.
}
}
#endregion
}
else if ( aToken.Class == SymToken.TClass.EClassSymbol && aToken.Value == "/" )
{
#region Handle Closing Comment Block [ */ ]
// For ending a comment region, we must have at least one token
// already in the cache.
SymToken previousToken = PreviousOutputToken;
// Check whether previous token was a "*" - we might be closing a block comment
if ( previousToken.Class == SymToken.TClass.EClassSymbol && previousToken.Value == "*" )
{
// Check whether first token was an opening block
SymToken firstToken = iCache.PeekHead;
if ( firstToken.Type == SymToken.TType.ETypeCommentBlock && firstToken.Value == "/*" )
{
// End of a block reached. Combine the closing "/" with the asterisk we already
// have in order to form a closing "*/" block token.
previousToken.Combine( aToken );
previousToken.Class = SymToken.TClass.EClassComment;
previousToken.Type = SymToken.TType.ETypeCommentBlock;
// No longer in a comment
InComment = false;
}
}
#endregion
}
else if ( aToken.Class == SymToken.TClass.EClassSymbol && aToken.Value == @"\" )
{
#region Handle possible continuation during comment
// We treat the possible continuation character as a comment.
// If the next character that arrives is really a new line, then we change
// the class to continuation and handle the situation accordingly...
aToken.Class = SymToken.TClass.EClassComment;
EnqueueNewOutputToken( aToken );
#endregion
}
else
{
aToken.Class = SymToken.TClass.EClassComment;
if ( PreviousOutputToken.Class == SymToken.TClass.EClassContinuation )
{
// In this scenario, we don't want to try to merge the specified token with the previous
// new line character, since new lines must be left intact. Just enque it, ensuring
// that the token class is suitably updated.
EnqueueNewOutputToken( aToken );
}
else if ( iCache.Count == 1 )
{
// We don't want to merge this token with the first token in the
// cache, or else we won't be able to successfully identify closing
// block comments
EnqueueNewOutputToken( aToken );
}
else
{
System.Diagnostics.Debug.Assert( PreviousOutputToken.CombiningAllowed );
ForceMergeWithPreviousToken( aToken );
}
}
}