예제 #1
0
		private void MergeWithPreviousTwoTokens( SymToken aNewToken, SymToken.TClass aNewClassType )
		{
			System.Diagnostics.Debug.Assert( iCache.Count > 0 );

			SymToken previousToken = iCache.PopTail();

			// Combine it with the new token...
			previousToken.Combine( aNewToken );
			previousToken.Class = aNewClassType;

			// And combine any previous previous token
			MergeWithPreviousToken( previousToken );
		}
예제 #2
0
		private void MergeWithPreviousToken( SymToken aNewToken )
		{
			if	( iCache.Count > 0 )
			{
				if	( CheckIfStateChangeRequiredForEnqueuedToken( aNewToken ) == false )
				{
					SymToken previousOutputToken = PreviousOutputToken;
					previousOutputToken.Combine( aNewToken );
				}
			}
			else
			{
				EnqueueNewOutputToken( aNewToken );
			}
		}
예제 #3
0
        public void MergeAllTokensWithinRange(int aStartIndex, int aEndIndex, bool aMergeInContinuations, bool aForceMerge)
        {
            int count = Count;

            //
            System.Diagnostics.Debug.Assert(count > aStartIndex);
            System.Diagnostics.Debug.Assert(aEndIndex < count);

            // Have to do this in two passes to ensure token
            // text remains from left to right.
            SymToken startingToken = this[aStartIndex++];

            if (aForceMerge == false)
            {
                // Not force-merging, so need to find a valid combinable starting element
                while (startingToken.CombiningAllowed == false && aStartIndex < aEndIndex)
                {
                    startingToken = this[++aStartIndex];
                }
            }

            // First pass - join tokens
            for (int i = aStartIndex; i <= aEndIndex; i++)
            {
                SymToken thisToken = this[i];

                // Ignore continuations during merging
                if (thisToken.Class != SymToken.TClass.EClassContinuation || aMergeInContinuations)
                {
                    if (aForceMerge == false)
                    {
                        startingToken.Combine(thisToken);
                    }
                    else
                    {
                        startingToken.ForceCombine(thisToken);
                    }
                }
            }

            // Second pass - discard merged tokens.
            for (int i = aEndIndex - 1; i >= aStartIndex; i--)
            {
                Remove(i);
            }

            //System.Diagnostics.Debug.WriteLine( "Merged: " + startingToken.Value );
        }
예제 #4
0
		private bool CheckIfStateChangeRequiredForEnqueuedToken( SymToken aToken )
		{
			// NB. This method is called before aToken has been enqueued
			// or in the case of combining, before the token has been combined
			// with any prior token.
			bool tokenProcessed = false;

			if	( InQuotation )
			{
			}
			else if ( InComment )
			{
			}
			else if ( InPreProcessorDirective )
			{
			}
			else
			{
				if	( aToken.Class == SymToken.TClass.EClassQuotation )
				{
					#region Handle start of quotation
					if	( iCache.Count > 0 )
					{
						// Check whether the previous symbol was a backslash. If it was
						// then this must be an escaped " or ' character, in which case
						// we don't change state.
						SymToken previousToken = PreviousOutputToken;

						if	( previousToken.Class == SymToken.TClass.EClassSymbol && previousToken.Value == @"\" )
						{
							// Last character was an escape marker. Combine it
							// with the quotation
							previousToken.Combine( aToken );

							// Already handled the token
							tokenProcessed = true;
						}
						else
						{
							// Really are starting a quotation.
							FlushCache();
							InQuotation = true;
						}
					}
					#endregion
				}
				else if ( aToken.Class == SymToken.TClass.EClassSymbol )
				{
					if ( aToken.Value == "*" ) 
					{
						#region Handle Opening comment block [ /* ]
						if	( iCache.Count > 0 )
						{
							SymToken previousToken = PreviousOutputToken;
							//
							if	( previousToken.Class == SymToken.TClass.EClassSymbol && previousToken.Value == "/" ) 
							{
								// "/*" case
								//
								// In this scenario, in order to ensure that we do not
								// flush the first character of our comment marker, we must
								// dequeue the tail item, then flush, then enqueue. 
								SymToken tailToken = iCache.PopTail(); // -> this is the initial "/" that we pop...
								FlushCache();

								// Forward slash and asterisk are combined
								tailToken.Combine( aToken );

								// Mark the token as a full line comment
								tailToken.Class = SymToken.TClass.EClassComment;
								tailToken.Type = SymToken.TType.ETypeCommentBlock;

								// ...and re-added to the cache
								iCache.Append( tailToken );

								// aToken was already combined so we don't want the caller
								// to add it twice.
								tokenProcessed = true;

								// We're now in a full line comment. 
								InComment = true;
							}
						}
						#endregion
					}
					else if ( aToken.Value == "/" )
					{
						#region Handle Full-Line comment [ // ]
						if	( iCache.Count > 0 )
						{
							SymToken previousToken = PreviousOutputToken;
							//
							if	( previousToken.Value == aToken.Value ) 
							{
								// "//" case
								//
								// In this scenario, in order to ensure that we do not
								// flush the first character of our comment marker, we must
								// dequeue the tail item, then flush, then enqueue. 
								SymToken tailToken = iCache.PopTail(); // -> this is the initial "/" that we pop...
								FlushCache();

								// Two forward slashes are combined into one.
								tailToken.Combine( aToken );

								// Mark the token as a full line comment
								tailToken.Class = SymToken.TClass.EClassComment;
								tailToken.Type = SymToken.TType.ETypeCommentFullLine;

								// ...and re-added to the cache
								iCache.Append( tailToken );

								// aToken was already combined so we don't want the caller
								// to add it twice.
								tokenProcessed = true;

								// We're now in a full line comment. 
								InComment = true;
							}
						}
						#endregion
					}
				}
				else if ( aToken.Class == SymToken.TClass.EClassPreProcessor )
				{
					#region Handle start of preprocessor directive
					// Preprocessor directives must only appear on a line
					// after whitespace. If there was any non-whitespace
					// characters before the preprocessor directive, then its illegal.
					bool tokensAreAllWhiteSpace = iCache.CheckTokensAreOfEitherClass( SymToken.TClass.EClassWhiteSpace, SymToken.TClass.EClassNewLine );
					if	( aToken.Value == "#" && tokensAreAllWhiteSpace )
					{
						// Starting a preprocess directive
						FlushCache();
						InPreProcessorDirective = true;
					}
					#endregion
				}
			}

			return tokenProcessed;
		}
예제 #5
0
		private void ProcessTokenDuringComment( SymToken aToken )
		{
			#region Comment examples
			//		// this is a comment
			//		/* this is also a comment */
			//		// "This is another comment"
			//		// This is a comment with a continuation \
			//		   and here's the rest.
			#endregion

			System.Diagnostics.Debug.Assert( iCache.Count > 0 );

			if	( aToken.Class == SymToken.TClass.EClassSymbol && aToken.Value == "*" )
			{
				#region Ensure asterisk is not merged with other comments
				// The asterisk character is separated from 
				// the rest of the comment in order that we can
				// ascertain when the end of a block comment has
				// been reached.
				EnqueueNewOutputToken( aToken );
				#endregion
			}
			else if ( aToken.Class == SymToken.TClass.EClassNewLine )
			{
				#region New line during comment...

				// Checking for continuations...
				SymToken previousToken = PreviousOutputToken;
				if	( previousToken.Value == @"\" )
				{
					// Discard new line
					previousToken.Class = SymToken.TClass.EClassContinuation;
				}
				else
				{
					// If we're in a block comment, then we don't flush when we
					// see a new line token.
					SymToken firstToken = iCache.PeekHead;
					EnqueueNewOutputToken( aToken );
					//
					if	( firstToken.Type == SymToken.TType.ETypeCommentFullLine )
					{
						// Flushing the cache resets the flags...
						FlushCache();
					}
					else if ( firstToken.Type == SymToken.TType.ETypeCommentBlock )
					{
						// Don't end the comment until we see the closing block token.
					}
				}
				#endregion
			}
			else if ( aToken.Class == SymToken.TClass.EClassSymbol && aToken.Value == "/" )
			{
				#region Handle Closing Comment Block [ */ ]
				// For ending a comment region, we must have at least one token
				// already in the cache.
				SymToken previousToken = PreviousOutputToken;

				// Check whether previous token was a "*" - we might be closing a block comment
				if	( previousToken.Class == SymToken.TClass.EClassSymbol && previousToken.Value == "*" )
				{
					// Check whether first token was an opening block
					SymToken firstToken = iCache.PeekHead;
					if	( firstToken.Type == SymToken.TType.ETypeCommentBlock && firstToken.Value == "/*" )
					{
						// End of a block reached. Combine the closing "/" with the asterisk we already
						// have in order to form a closing "*/" block token. 
						previousToken.Combine( aToken );
						previousToken.Class = SymToken.TClass.EClassComment;
						previousToken.Type = SymToken.TType.ETypeCommentBlock;

						// No longer in a comment
						InComment = false;
					}
				}
				#endregion
			}
			else if ( aToken.Class == SymToken.TClass.EClassSymbol && aToken.Value == @"\" )
			{
				#region Handle possible continuation during comment
				// We treat the possible continuation character as a comment.
				// If the next character that arrives is really a new line, then we change
				// the class to continuation and handle the situation accordingly...
				aToken.Class = SymToken.TClass.EClassComment;
				EnqueueNewOutputToken( aToken );
				#endregion
			}
			else
			{
				aToken.Class = SymToken.TClass.EClassComment;

				if	( PreviousOutputToken.Class == SymToken.TClass.EClassContinuation )
				{
					// In this scenario, we don't want to try to merge the specified token with the previous
					// new line character, since new lines must be left intact. Just enque it, ensuring
					// that the token class is suitably updated.
					EnqueueNewOutputToken( aToken );
				}
				else if ( iCache.Count == 1 )
				{
					// We don't want to merge this token with the first token in the
					// cache, or else we won't be able to successfully identify closing
					// block comments
					EnqueueNewOutputToken( aToken );
				}
				else
				{
					System.Diagnostics.Debug.Assert( PreviousOutputToken.CombiningAllowed );
					ForceMergeWithPreviousToken( aToken );
				}
			}
		}