public Scanner() { Regex regex; Patterns = new Dictionary <TokenType, Regex>(); Tokens = new List <TokenType>(); LookAheadToken = null; Skipped = new List <Token>(); SkipList = new List <TokenType> { TokenType.BlockComment, TokenType.Comment, TokenType.Whitespace, TokenType.LinePragma }; FileAndLine = TokenType.LinePragma; regex = new Regex(@"/\*([^*]|\*[^/])*\*/", RegexOptions.Compiled); Patterns.Add(TokenType.BlockComment, regex); Tokens.Add(TokenType.BlockComment); regex = new Regex(@"//[^\n\r]*", RegexOptions.Compiled); Patterns.Add(TokenType.Comment, regex); Tokens.Add(TokenType.Comment); regex = new Regex(@"[ \t\n\r]+", RegexOptions.Compiled); Patterns.Add(TokenType.Whitespace, regex); Tokens.Add(TokenType.Whitespace); regex = new Regex(@"^[ \t]*#line[ \t]*(?<Line>\d*)[ \t]*(\""(?<File>[^\""\\]*(?:\\.[^\""\\]*)*)\"")?\n", RegexOptions.Compiled); Patterns.Add(TokenType.LinePragma, regex); Tokens.Add(TokenType.LinePragma); regex = new Regex(@"pass", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.Pass, regex); Tokens.Add(TokenType.Pass); regex = new Regex(@"technique", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.Technique, regex); Tokens.Add(TokenType.Technique); regex = new Regex(@"sampler1D|sampler2D|sampler3D|samplerCUBE|SamplerState|sampler", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.Sampler, regex); Tokens.Add(TokenType.Sampler); regex = new Regex(@"sampler_state", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.SamplerState, regex); Tokens.Add(TokenType.SamplerState); regex = new Regex(@"VertexShader", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.VertexShader, regex); Tokens.Add(TokenType.VertexShader); regex = new Regex(@"PixelShader", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.PixelShader, regex); Tokens.Add(TokenType.PixelShader); regex = new Regex(@"register", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.Register, regex); Tokens.Add(TokenType.Register); regex = new Regex(@"true|false|0|1", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.Boolean, regex); Tokens.Add(TokenType.Boolean); regex = new Regex(@"[+-]? ?[0-9]?\.?[0-9]+[fF]?", RegexOptions.Compiled); Patterns.Add(TokenType.Number, regex); Tokens.Add(TokenType.Number); regex = new Regex(@"0x[0-9a-f]{6}([0-9a-f][0-9a-f])?", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.HexColor, regex); Tokens.Add(TokenType.HexColor); regex = new Regex(@"[A-Za-z_][A-Za-z0-9_]*", RegexOptions.Compiled); Patterns.Add(TokenType.Identifier, regex); Tokens.Add(TokenType.Identifier); regex = new Regex(@"{", RegexOptions.Compiled); Patterns.Add(TokenType.OpenBracket, regex); Tokens.Add(TokenType.OpenBracket); regex = new Regex(@"}", RegexOptions.Compiled); Patterns.Add(TokenType.CloseBracket, regex); Tokens.Add(TokenType.CloseBracket); regex = new Regex(@"=", RegexOptions.Compiled); Patterns.Add(TokenType.Equals, regex); Tokens.Add(TokenType.Equals); regex = new Regex(@":", RegexOptions.Compiled); Patterns.Add(TokenType.Colon, regex); Tokens.Add(TokenType.Colon); regex = new Regex(@",", RegexOptions.Compiled); Patterns.Add(TokenType.Comma, regex); Tokens.Add(TokenType.Comma); regex = new Regex(@";", RegexOptions.Compiled); Patterns.Add(TokenType.Semicolon, regex); Tokens.Add(TokenType.Semicolon); regex = new Regex(@"\|", RegexOptions.Compiled); Patterns.Add(TokenType.Or, regex); Tokens.Add(TokenType.Or); regex = new Regex(@"\(", RegexOptions.Compiled); Patterns.Add(TokenType.OpenParenthesis, regex); Tokens.Add(TokenType.OpenParenthesis); regex = new Regex(@"\)", RegexOptions.Compiled); Patterns.Add(TokenType.CloseParenthesis, regex); Tokens.Add(TokenType.CloseParenthesis); regex = new Regex(@"\[", RegexOptions.Compiled); Patterns.Add(TokenType.OpenSquareBracket, regex); Tokens.Add(TokenType.OpenSquareBracket); regex = new Regex(@"\]", RegexOptions.Compiled); Patterns.Add(TokenType.CloseSquareBracket, regex); Tokens.Add(TokenType.CloseSquareBracket); regex = new Regex(@"<", RegexOptions.Compiled); Patterns.Add(TokenType.LessThan, regex); Tokens.Add(TokenType.LessThan); regex = new Regex(@">", RegexOptions.Compiled); Patterns.Add(TokenType.GreaterThan, regex); Tokens.Add(TokenType.GreaterThan); regex = new Regex(@"compile", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.Compile, regex); Tokens.Add(TokenType.Compile); regex = new Regex(@"[A-Za-z_][A-Za-z0-9_]*", RegexOptions.Compiled); Patterns.Add(TokenType.ShaderModel, regex); Tokens.Add(TokenType.ShaderModel); regex = new Regex(@"[\S]+", RegexOptions.Compiled); Patterns.Add(TokenType.Code, regex); Tokens.Add(TokenType.Code); regex = new Regex(@"^$", RegexOptions.Compiled); Patterns.Add(TokenType.EndOfFile, regex); Tokens.Add(TokenType.EndOfFile); regex = new Regex(@"MinFilter", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.MinFilter, regex); Tokens.Add(TokenType.MinFilter); regex = new Regex(@"MagFilter", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.MagFilter, regex); Tokens.Add(TokenType.MagFilter); regex = new Regex(@"MipFilter", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.MipFilter, regex); Tokens.Add(TokenType.MipFilter); regex = new Regex(@"Filter", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.Filter, regex); Tokens.Add(TokenType.Filter); regex = new Regex(@"Texture", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.Texture, regex); Tokens.Add(TokenType.Texture); regex = new Regex(@"AddressU", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.AddressU, regex); Tokens.Add(TokenType.AddressU); regex = new Regex(@"AddressV", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.AddressV, regex); Tokens.Add(TokenType.AddressV); regex = new Regex(@"AddressW", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.AddressW, regex); Tokens.Add(TokenType.AddressW); regex = new Regex(@"BorderColor", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.BorderColor, regex); Tokens.Add(TokenType.BorderColor); regex = new Regex(@"MaxAnisotropy", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.MaxAnisotropy, regex); Tokens.Add(TokenType.MaxAnisotropy); regex = new Regex(@"MaxMipLevel|MaxLod", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.MaxMipLevel, regex); Tokens.Add(TokenType.MaxMipLevel); regex = new Regex(@"MipLodBias", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.MipLodBias, regex); Tokens.Add(TokenType.MipLodBias); regex = new Regex(@"Clamp", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.Clamp, regex); Tokens.Add(TokenType.Clamp); regex = new Regex(@"Wrap", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.Wrap, regex); Tokens.Add(TokenType.Wrap); regex = new Regex(@"Mirror", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.Mirror, regex); Tokens.Add(TokenType.Mirror); regex = new Regex(@"Border", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.Border, regex); Tokens.Add(TokenType.Border); regex = new Regex(@"None", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.None, regex); Tokens.Add(TokenType.None); regex = new Regex(@"Linear", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.Linear, regex); Tokens.Add(TokenType.Linear); regex = new Regex(@"Point", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.Point, regex); Tokens.Add(TokenType.Point); regex = new Regex(@"Anisotropic", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.Anisotropic, regex); Tokens.Add(TokenType.Anisotropic); regex = new Regex(@"AlphaBlendEnable", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.AlphaBlendEnable, regex); Tokens.Add(TokenType.AlphaBlendEnable); regex = new Regex(@"SrcBlend", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.SrcBlend, regex); Tokens.Add(TokenType.SrcBlend); regex = new Regex(@"DestBlend", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.DestBlend, regex); Tokens.Add(TokenType.DestBlend); regex = new Regex(@"BlendOp", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.BlendOp, regex); Tokens.Add(TokenType.BlendOp); regex = new Regex(@"ColorWriteEnable", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.ColorWriteEnable, regex); Tokens.Add(TokenType.ColorWriteEnable); regex = new Regex(@"ZEnable", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.ZEnable, regex); Tokens.Add(TokenType.ZEnable); regex = new Regex(@"ZWriteEnable", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.ZWriteEnable, regex); Tokens.Add(TokenType.ZWriteEnable); regex = new Regex(@"ZFunc", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.ZFunc, regex); Tokens.Add(TokenType.ZFunc); regex = new Regex(@"DepthBias", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.DepthBias, regex); Tokens.Add(TokenType.DepthBias); regex = new Regex(@"CullMode", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.CullMode, regex); Tokens.Add(TokenType.CullMode); regex = new Regex(@"FillMode", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.FillMode, regex); Tokens.Add(TokenType.FillMode); regex = new Regex(@"MultiSampleAntiAlias", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.MultiSampleAntiAlias, regex); Tokens.Add(TokenType.MultiSampleAntiAlias); regex = new Regex(@"ScissorTestEnable", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.ScissorTestEnable, regex); Tokens.Add(TokenType.ScissorTestEnable); regex = new Regex(@"SlopeScaleDepthBias", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.SlopeScaleDepthBias, regex); Tokens.Add(TokenType.SlopeScaleDepthBias); regex = new Regex(@"StencilEnable", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.StencilEnable, regex); Tokens.Add(TokenType.StencilEnable); regex = new Regex(@"StencilFail", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.StencilFail, regex); Tokens.Add(TokenType.StencilFail); regex = new Regex(@"StencilFunc", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.StencilFunc, regex); Tokens.Add(TokenType.StencilFunc); regex = new Regex(@"StencilMask", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.StencilMask, regex); Tokens.Add(TokenType.StencilMask); regex = new Regex(@"StencilPass", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.StencilPass, regex); Tokens.Add(TokenType.StencilPass); regex = new Regex(@"StencilRef", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.StencilRef, regex); Tokens.Add(TokenType.StencilRef); regex = new Regex(@"StencilWriteMask", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.StencilWriteMask, regex); Tokens.Add(TokenType.StencilWriteMask); regex = new Regex(@"StencilZFail", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.StencilZFail, regex); Tokens.Add(TokenType.StencilZFail); regex = new Regex(@"Never", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.Never, regex); Tokens.Add(TokenType.Never); regex = new Regex(@"Less", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.Less, regex); Tokens.Add(TokenType.Less); regex = new Regex(@"Equal", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.Equal, regex); Tokens.Add(TokenType.Equal); regex = new Regex(@"LessEqual", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.LessEqual, regex); Tokens.Add(TokenType.LessEqual); regex = new Regex(@"Greater", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.Greater, regex); Tokens.Add(TokenType.Greater); regex = new Regex(@"NotEqual", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.NotEqual, regex); Tokens.Add(TokenType.NotEqual); regex = new Regex(@"GreaterEqual", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.GreaterEqual, regex); Tokens.Add(TokenType.GreaterEqual); regex = new Regex(@"Always", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.Always, regex); Tokens.Add(TokenType.Always); regex = new Regex(@"Keep", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.Keep, regex); Tokens.Add(TokenType.Keep); regex = new Regex(@"Zero", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.Zero, regex); Tokens.Add(TokenType.Zero); regex = new Regex(@"Replace", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.Replace, regex); Tokens.Add(TokenType.Replace); regex = new Regex(@"IncrSat", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.IncrSat, regex); Tokens.Add(TokenType.IncrSat); regex = new Regex(@"DecrSat", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.DecrSat, regex); Tokens.Add(TokenType.DecrSat); regex = new Regex(@"Invert", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.Invert, regex); Tokens.Add(TokenType.Invert); regex = new Regex(@"Incr", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.Incr, regex); Tokens.Add(TokenType.Incr); regex = new Regex(@"Decr", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.Decr, regex); Tokens.Add(TokenType.Decr); regex = new Regex(@"Red", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.Red, regex); Tokens.Add(TokenType.Red); regex = new Regex(@"Green", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.Green, regex); Tokens.Add(TokenType.Green); regex = new Regex(@"Blue", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.Blue, regex); Tokens.Add(TokenType.Blue); regex = new Regex(@"Alpha", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.Alpha, regex); Tokens.Add(TokenType.Alpha); regex = new Regex(@"All", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.All, regex); Tokens.Add(TokenType.All); regex = new Regex(@"Cw", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.Cw, regex); Tokens.Add(TokenType.Cw); regex = new Regex(@"Ccw", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.Ccw, regex); Tokens.Add(TokenType.Ccw); regex = new Regex(@"Solid", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.Solid, regex); Tokens.Add(TokenType.Solid); regex = new Regex(@"WireFrame", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.WireFrame, regex); Tokens.Add(TokenType.WireFrame); regex = new Regex(@"Add", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.Add, regex); Tokens.Add(TokenType.Add); regex = new Regex(@"Subtract", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.Subtract, regex); Tokens.Add(TokenType.Subtract); regex = new Regex(@"RevSubtract", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.RevSubtract, regex); Tokens.Add(TokenType.RevSubtract); regex = new Regex(@"Min", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.Min, regex); Tokens.Add(TokenType.Min); regex = new Regex(@"Max", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.Max, regex); Tokens.Add(TokenType.Max); regex = new Regex(@"One", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.One, regex); Tokens.Add(TokenType.One); regex = new Regex(@"SrcColor", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.SrcColor, regex); Tokens.Add(TokenType.SrcColor); regex = new Regex(@"InvSrcColor", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.InvSrcColor, regex); Tokens.Add(TokenType.InvSrcColor); regex = new Regex(@"SrcAlpha", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.SrcAlpha, regex); Tokens.Add(TokenType.SrcAlpha); regex = new Regex(@"InvSrcAlpha", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.InvSrcAlpha, regex); Tokens.Add(TokenType.InvSrcAlpha); regex = new Regex(@"DestAlpha", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.DestAlpha, regex); Tokens.Add(TokenType.DestAlpha); regex = new Regex(@"InvDestAlpha", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.InvDestAlpha, regex); Tokens.Add(TokenType.InvDestAlpha); regex = new Regex(@"DestColor", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.DestColor, regex); Tokens.Add(TokenType.DestColor); regex = new Regex(@"InvDestColor", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.InvDestColor, regex); Tokens.Add(TokenType.InvDestColor); regex = new Regex(@"SrcAlphaSat", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.SrcAlphaSat, regex); Tokens.Add(TokenType.SrcAlphaSat); regex = new Regex(@"BlendFactor", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.BlendFactor, regex); Tokens.Add(TokenType.BlendFactor); regex = new Regex(@"InvBlendFactor", RegexOptions.Compiled | RegexOptions.IgnoreCase); Patterns.Add(TokenType.InvBlendFactor, regex); Tokens.Add(TokenType.InvBlendFactor); }
/// <summary> /// returns token with longest best match /// </summary> /// <returns></returns> public Token LookAhead(params TokenType[] expectedtokens) { int i; int startpos = StartPos; int endpos = EndPos; int currentline = CurrentLine; string currentFile = CurrentFile; Token tok = null; List <TokenType> scantokens; // this prevents double scanning and matching // increased performance if (LookAheadToken != null && LookAheadToken.Type != TokenType._UNDETERMINED_ && LookAheadToken.Type != TokenType._NONE_) { return(LookAheadToken); } // if no scantokens specified, then scan for all of them (= backward compatible) if (expectedtokens.Length == 0) { scantokens = Tokens; } else { scantokens = new List <TokenType>(expectedtokens); scantokens.AddRange(SkipList); } do { int len = -1; TokenType index = (TokenType)int.MaxValue; string input = Input.Substring(startpos); tok = new Token(startpos, endpos); for (i = 0; i < scantokens.Count; i++) { Regex r = Patterns[scantokens[i]]; Match m = r.Match(input); if (m.Success && m.Index == 0 && ((m.Length > len) || (scantokens[i] < index && m.Length == len))) { len = m.Length; index = scantokens[i]; } } if (index >= 0 && len >= 0) { tok.EndPos = startpos + len; tok.Text = Input.Substring(tok.StartPos, len); tok.Type = index; } else if (tok.StartPos == tok.EndPos) { if (tok.StartPos < Input.Length) { tok.Text = Input.Substring(tok.StartPos, 1); } else { tok.Text = "EOF"; } } // Update the line and column count for error reporting. tok.File = currentFile; tok.Line = currentline; if (tok.StartPos < Input.Length) { tok.Column = tok.StartPos - Input.LastIndexOf('\n', tok.StartPos); } if (SkipList.Contains(tok.Type)) { startpos = tok.EndPos; endpos = tok.EndPos; currentline = tok.Line + (tok.Text.Length - tok.Text.Replace("\n", "").Length); currentFile = tok.File; Skipped.Add(tok); } else { // only assign to non-skipped tokens tok.Skipped = Skipped; // assign prior skips to this token Skipped = new List <Token>(); //reset skips } // Check to see if the parsed token wants to // alter the file and line number. if (tok.Type == FileAndLine) { var match = Patterns[tok.Type].Match(tok.Text); var fileMatch = match.Groups["File"]; if (fileMatch.Success) { currentFile = fileMatch.Value.Replace("\\\\", "\\"); } var lineMatch = match.Groups["Line"]; if (lineMatch.Success) { currentline = int.Parse(lineMatch.Value, NumberStyles.Integer, CultureInfo.InvariantCulture); } } }while (SkipList.Contains(tok.Type)); LookAheadToken = tok; return(tok); }