/// <summary> /// Match is a static method that determines whether /// or not the token starting at /// <paramref name="begin">begin</paramref> /// is an C# identifier. /// </summary> /// <param name="begin">begin</param> /// <param name="end">end</param> internal static bool Match(Position begin, Position end) { if (begin == end) { return false; } char read = begin.Get(); return Ctype.IsLetter(read) || read == '_'; }
private void SplitIntoLines() { Position begin = new Position(contents, 0); Position end = new Position(contents, contents.Length); Position pos; bool inComment = false; while ((pos = Utility.Find(begin, end, '\r')) != end) { lines.Add(new Line(begin, pos, ref inComment)); begin = ++pos; if (begin != end && begin.Get() == '\n') { ++begin; } } // last line might not have a terminating newline if (begin != end) { lines.Add(new Line(begin, pos, ref inComment)); } // remove trailing blank lines while (lines.Count > 0 && ((Line)lines[lines.Count-1]).IsBlank()) { lines.RemoveAt(lines.Count - 1); } }
internal Line(Position begin, Position end, ref bool inComment) { while (begin != end) { Position pos; if (inComment || MultiLineComment.Match(begin, end)) { pos = MultiLineComment.Eat(begin, end); if (pos != end) { ++pos; ++pos; inComment = false; } else { inComment = true; } tokens.Add(MultiLineComment.MakeToken(begin, pos)); } else if (OneLineComment.Match(begin, end)) { pos = OneLineComment.Eat(begin, end); tokens.Add(OneLineComment.MakeToken(begin, pos)); } else if (WhiteSpace.Match(begin, end)) { pos = WhiteSpace.Eat(begin, end); tokens.Add(WhiteSpace.MakeToken(begin, pos)); } else if (Identifier.Match(begin, end)) { pos = Identifier.Eat(begin, end); if (Keyword.Match(begin, pos)) { tokens.Add(Keyword.MakeToken(begin, pos)); } else { tokens.Add(Identifier.MakeToken(begin, pos)); } } else if (Directive.Match(begin, end)) { pos = Directive.Eat(begin, end); tokens.Add(Directive.MakeToken(begin, pos)); } else { pos = end; tokens.Add(Other.MakeToken(begin, end)); } begin = pos; } }
internal WhiteSpaceToken(Position begin, Position end) : base(begin, end) { }
internal static bool Match(Position begin, Position end) { return begin != end && Ctype.IsWhiteSpace(begin.Get()); }
internal static IWhiteSpaceToken MakeToken(Position begin, Position end) { return new WhiteSpaceToken(begin, end); }
internal static Position Eat(Position begin, Position end) { return Utility.FindIf(begin, end, notWhiteSpace); }
internal IdentifierToken(Position begin, Position end) : base(begin, end) { }
/// <summary> /// MakeToken is a static method that creates and /// returns a token representing an identifier that /// starts at /// <paramref name="begin">begin</paramref> /// and finishes just before /// <paramref name="end">end</paramref>. /// </summary> /// <param name="begin">begin</param> /// <param name="end">end</param> internal static IIdentifierToken MakeToken(Position begin, Position end) { return new IdentifierToken(begin, end); }
/// <summary> /// Eat is a static method that eats the identifier /// token assumed to start at /// <paramref name="begin">begin</paramref>. /// </summary> /// <param name="begin">begin</param> /// <param name="end">end</param> /// <returns> /// A Position representing the first /// character not in the identifier token. The /// returned Position is useful as the second /// parameter to the MakeToken method. /// </returns> internal static Position Eat(Position begin, Position end) { return Utility.FindIf(begin, end, notIdentifier); }