public Scanner()
{
Regex regex;
Patterns = new Dictionary <TokenType, Regex>();
Tokens = new List <TokenType>();
LookAheadToken = null;
Skipped = new List <Token>();
SkipList = new List <TokenType>
{
TokenType.BlockComment,
TokenType.Comment,
TokenType.Whitespace,
TokenType.LinePragma
};
FileAndLine = TokenType.LinePragma;
regex = new Regex(@"/\*([^*]|\*[^/])*\*/", RegexOptions.Compiled);
Patterns.Add(TokenType.BlockComment, regex);
Tokens.Add(TokenType.BlockComment);
regex = new Regex(@"//[^\n\r]*", RegexOptions.Compiled);
Patterns.Add(TokenType.Comment, regex);
Tokens.Add(TokenType.Comment);
regex = new Regex(@"[ \t\n\r]+", RegexOptions.Compiled);
Patterns.Add(TokenType.Whitespace, regex);
Tokens.Add(TokenType.Whitespace);
regex = new Regex(@"^[ \t]*#line[ \t]*(?<Line>\d*)[ \t]*(\""(?<File>[^\""\\]*(?:\\.[^\""\\]*)*)\"")?\n", RegexOptions.Compiled);
Patterns.Add(TokenType.LinePragma, regex);
Tokens.Add(TokenType.LinePragma);
regex = new Regex(@"pass", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.Pass, regex);
Tokens.Add(TokenType.Pass);
regex = new Regex(@"technique", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.Technique, regex);
Tokens.Add(TokenType.Technique);
regex = new Regex(@"sampler1D|sampler2D|sampler3D|samplerCUBE|SamplerState|sampler", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.Sampler, regex);
Tokens.Add(TokenType.Sampler);
regex = new Regex(@"sampler_state", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.SamplerState, regex);
Tokens.Add(TokenType.SamplerState);
regex = new Regex(@"VertexShader", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.VertexShader, regex);
Tokens.Add(TokenType.VertexShader);
regex = new Regex(@"PixelShader", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.PixelShader, regex);
Tokens.Add(TokenType.PixelShader);
regex = new Regex(@"register", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.Register, regex);
Tokens.Add(TokenType.Register);
regex = new Regex(@"true|false|0|1", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.Boolean, regex);
Tokens.Add(TokenType.Boolean);
regex = new Regex(@"[+-]? ?[0-9]?\.?[0-9]+[fF]?", RegexOptions.Compiled);
Patterns.Add(TokenType.Number, regex);
Tokens.Add(TokenType.Number);
regex = new Regex(@"0x[0-9a-f]{6}([0-9a-f][0-9a-f])?", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.HexColor, regex);
Tokens.Add(TokenType.HexColor);
regex = new Regex(@"[A-Za-z_][A-Za-z0-9_]*", RegexOptions.Compiled);
Patterns.Add(TokenType.Identifier, regex);
Tokens.Add(TokenType.Identifier);
regex = new Regex(@"{", RegexOptions.Compiled);
Patterns.Add(TokenType.OpenBracket, regex);
Tokens.Add(TokenType.OpenBracket);
regex = new Regex(@"}", RegexOptions.Compiled);
Patterns.Add(TokenType.CloseBracket, regex);
Tokens.Add(TokenType.CloseBracket);
regex = new Regex(@"=", RegexOptions.Compiled);
Patterns.Add(TokenType.Equals, regex);
Tokens.Add(TokenType.Equals);
regex = new Regex(@":", RegexOptions.Compiled);
Patterns.Add(TokenType.Colon, regex);
Tokens.Add(TokenType.Colon);
regex = new Regex(@",", RegexOptions.Compiled);
Patterns.Add(TokenType.Comma, regex);
Tokens.Add(TokenType.Comma);
regex = new Regex(@";", RegexOptions.Compiled);
Patterns.Add(TokenType.Semicolon, regex);
Tokens.Add(TokenType.Semicolon);
regex = new Regex(@"\|", RegexOptions.Compiled);
Patterns.Add(TokenType.Or, regex);
Tokens.Add(TokenType.Or);
regex = new Regex(@"\(", RegexOptions.Compiled);
Patterns.Add(TokenType.OpenParenthesis, regex);
Tokens.Add(TokenType.OpenParenthesis);
regex = new Regex(@"\)", RegexOptions.Compiled);
Patterns.Add(TokenType.CloseParenthesis, regex);
Tokens.Add(TokenType.CloseParenthesis);
regex = new Regex(@"\[", RegexOptions.Compiled);
Patterns.Add(TokenType.OpenSquareBracket, regex);
Tokens.Add(TokenType.OpenSquareBracket);
regex = new Regex(@"\]", RegexOptions.Compiled);
Patterns.Add(TokenType.CloseSquareBracket, regex);
Tokens.Add(TokenType.CloseSquareBracket);
regex = new Regex(@"<", RegexOptions.Compiled);
Patterns.Add(TokenType.LessThan, regex);
Tokens.Add(TokenType.LessThan);
regex = new Regex(@">", RegexOptions.Compiled);
Patterns.Add(TokenType.GreaterThan, regex);
Tokens.Add(TokenType.GreaterThan);
regex = new Regex(@"compile", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.Compile, regex);
Tokens.Add(TokenType.Compile);
regex = new Regex(@"[A-Za-z_][A-Za-z0-9_]*", RegexOptions.Compiled);
Patterns.Add(TokenType.ShaderModel, regex);
Tokens.Add(TokenType.ShaderModel);
regex = new Regex(@"[\S]+", RegexOptions.Compiled);
Patterns.Add(TokenType.Code, regex);
Tokens.Add(TokenType.Code);
regex = new Regex(@"^$", RegexOptions.Compiled);
Patterns.Add(TokenType.EndOfFile, regex);
Tokens.Add(TokenType.EndOfFile);
regex = new Regex(@"MinFilter", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.MinFilter, regex);
Tokens.Add(TokenType.MinFilter);
regex = new Regex(@"MagFilter", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.MagFilter, regex);
Tokens.Add(TokenType.MagFilter);
regex = new Regex(@"MipFilter", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.MipFilter, regex);
Tokens.Add(TokenType.MipFilter);
regex = new Regex(@"Filter", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.Filter, regex);
Tokens.Add(TokenType.Filter);
regex = new Regex(@"Texture", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.Texture, regex);
Tokens.Add(TokenType.Texture);
regex = new Regex(@"AddressU", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.AddressU, regex);
Tokens.Add(TokenType.AddressU);
regex = new Regex(@"AddressV", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.AddressV, regex);
Tokens.Add(TokenType.AddressV);
regex = new Regex(@"AddressW", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.AddressW, regex);
Tokens.Add(TokenType.AddressW);
regex = new Regex(@"BorderColor", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.BorderColor, regex);
Tokens.Add(TokenType.BorderColor);
regex = new Regex(@"MaxAnisotropy", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.MaxAnisotropy, regex);
Tokens.Add(TokenType.MaxAnisotropy);
regex = new Regex(@"MaxMipLevel|MaxLod", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.MaxMipLevel, regex);
Tokens.Add(TokenType.MaxMipLevel);
regex = new Regex(@"MipLodBias", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.MipLodBias, regex);
Tokens.Add(TokenType.MipLodBias);
regex = new Regex(@"Clamp", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.Clamp, regex);
Tokens.Add(TokenType.Clamp);
regex = new Regex(@"Wrap", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.Wrap, regex);
Tokens.Add(TokenType.Wrap);
regex = new Regex(@"Mirror", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.Mirror, regex);
Tokens.Add(TokenType.Mirror);
regex = new Regex(@"Border", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.Border, regex);
Tokens.Add(TokenType.Border);
regex = new Regex(@"None", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.None, regex);
Tokens.Add(TokenType.None);
regex = new Regex(@"Linear", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.Linear, regex);
Tokens.Add(TokenType.Linear);
regex = new Regex(@"Point", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.Point, regex);
Tokens.Add(TokenType.Point);
regex = new Regex(@"Anisotropic", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.Anisotropic, regex);
Tokens.Add(TokenType.Anisotropic);
regex = new Regex(@"AlphaBlendEnable", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.AlphaBlendEnable, regex);
Tokens.Add(TokenType.AlphaBlendEnable);
regex = new Regex(@"SrcBlend", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.SrcBlend, regex);
Tokens.Add(TokenType.SrcBlend);
regex = new Regex(@"DestBlend", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.DestBlend, regex);
Tokens.Add(TokenType.DestBlend);
regex = new Regex(@"BlendOp", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.BlendOp, regex);
Tokens.Add(TokenType.BlendOp);
regex = new Regex(@"ColorWriteEnable", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.ColorWriteEnable, regex);
Tokens.Add(TokenType.ColorWriteEnable);
regex = new Regex(@"ZEnable", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.ZEnable, regex);
Tokens.Add(TokenType.ZEnable);
regex = new Regex(@"ZWriteEnable", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.ZWriteEnable, regex);
Tokens.Add(TokenType.ZWriteEnable);
regex = new Regex(@"ZFunc", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.ZFunc, regex);
Tokens.Add(TokenType.ZFunc);
regex = new Regex(@"DepthBias", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.DepthBias, regex);
Tokens.Add(TokenType.DepthBias);
regex = new Regex(@"CullMode", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.CullMode, regex);
Tokens.Add(TokenType.CullMode);
regex = new Regex(@"FillMode", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.FillMode, regex);
Tokens.Add(TokenType.FillMode);
regex = new Regex(@"MultiSampleAntiAlias", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.MultiSampleAntiAlias, regex);
Tokens.Add(TokenType.MultiSampleAntiAlias);
regex = new Regex(@"ScissorTestEnable", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.ScissorTestEnable, regex);
Tokens.Add(TokenType.ScissorTestEnable);
regex = new Regex(@"SlopeScaleDepthBias", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.SlopeScaleDepthBias, regex);
Tokens.Add(TokenType.SlopeScaleDepthBias);
regex = new Regex(@"StencilEnable", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.StencilEnable, regex);
Tokens.Add(TokenType.StencilEnable);
regex = new Regex(@"StencilFail", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.StencilFail, regex);
Tokens.Add(TokenType.StencilFail);
regex = new Regex(@"StencilFunc", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.StencilFunc, regex);
Tokens.Add(TokenType.StencilFunc);
regex = new Regex(@"StencilMask", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.StencilMask, regex);
Tokens.Add(TokenType.StencilMask);
regex = new Regex(@"StencilPass", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.StencilPass, regex);
Tokens.Add(TokenType.StencilPass);
regex = new Regex(@"StencilRef", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.StencilRef, regex);
Tokens.Add(TokenType.StencilRef);
regex = new Regex(@"StencilWriteMask", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.StencilWriteMask, regex);
Tokens.Add(TokenType.StencilWriteMask);
regex = new Regex(@"StencilZFail", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.StencilZFail, regex);
Tokens.Add(TokenType.StencilZFail);
regex = new Regex(@"Never", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.Never, regex);
Tokens.Add(TokenType.Never);
regex = new Regex(@"Less", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.Less, regex);
Tokens.Add(TokenType.Less);
regex = new Regex(@"Equal", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.Equal, regex);
Tokens.Add(TokenType.Equal);
regex = new Regex(@"LessEqual", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.LessEqual, regex);
Tokens.Add(TokenType.LessEqual);
regex = new Regex(@"Greater", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.Greater, regex);
Tokens.Add(TokenType.Greater);
regex = new Regex(@"NotEqual", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.NotEqual, regex);
Tokens.Add(TokenType.NotEqual);
regex = new Regex(@"GreaterEqual", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.GreaterEqual, regex);
Tokens.Add(TokenType.GreaterEqual);
regex = new Regex(@"Always", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.Always, regex);
Tokens.Add(TokenType.Always);
regex = new Regex(@"Keep", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.Keep, regex);
Tokens.Add(TokenType.Keep);
regex = new Regex(@"Zero", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.Zero, regex);
Tokens.Add(TokenType.Zero);
regex = new Regex(@"Replace", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.Replace, regex);
Tokens.Add(TokenType.Replace);
regex = new Regex(@"IncrSat", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.IncrSat, regex);
Tokens.Add(TokenType.IncrSat);
regex = new Regex(@"DecrSat", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.DecrSat, regex);
Tokens.Add(TokenType.DecrSat);
regex = new Regex(@"Invert", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.Invert, regex);
Tokens.Add(TokenType.Invert);
regex = new Regex(@"Incr", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.Incr, regex);
Tokens.Add(TokenType.Incr);
regex = new Regex(@"Decr", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.Decr, regex);
Tokens.Add(TokenType.Decr);
regex = new Regex(@"Red", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.Red, regex);
Tokens.Add(TokenType.Red);
regex = new Regex(@"Green", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.Green, regex);
Tokens.Add(TokenType.Green);
regex = new Regex(@"Blue", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.Blue, regex);
Tokens.Add(TokenType.Blue);
regex = new Regex(@"Alpha", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.Alpha, regex);
Tokens.Add(TokenType.Alpha);
regex = new Regex(@"All", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.All, regex);
Tokens.Add(TokenType.All);
regex = new Regex(@"Cw", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.Cw, regex);
Tokens.Add(TokenType.Cw);
regex = new Regex(@"Ccw", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.Ccw, regex);
Tokens.Add(TokenType.Ccw);
regex = new Regex(@"Solid", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.Solid, regex);
Tokens.Add(TokenType.Solid);
regex = new Regex(@"WireFrame", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.WireFrame, regex);
Tokens.Add(TokenType.WireFrame);
regex = new Regex(@"Add", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.Add, regex);
Tokens.Add(TokenType.Add);
regex = new Regex(@"Subtract", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.Subtract, regex);
Tokens.Add(TokenType.Subtract);
regex = new Regex(@"RevSubtract", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.RevSubtract, regex);
Tokens.Add(TokenType.RevSubtract);
regex = new Regex(@"Min", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.Min, regex);
Tokens.Add(TokenType.Min);
regex = new Regex(@"Max", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.Max, regex);
Tokens.Add(TokenType.Max);
regex = new Regex(@"One", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.One, regex);
Tokens.Add(TokenType.One);
regex = new Regex(@"SrcColor", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.SrcColor, regex);
Tokens.Add(TokenType.SrcColor);
regex = new Regex(@"InvSrcColor", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.InvSrcColor, regex);
Tokens.Add(TokenType.InvSrcColor);
regex = new Regex(@"SrcAlpha", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.SrcAlpha, regex);
Tokens.Add(TokenType.SrcAlpha);
regex = new Regex(@"InvSrcAlpha", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.InvSrcAlpha, regex);
Tokens.Add(TokenType.InvSrcAlpha);
regex = new Regex(@"DestAlpha", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.DestAlpha, regex);
Tokens.Add(TokenType.DestAlpha);
regex = new Regex(@"InvDestAlpha", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.InvDestAlpha, regex);
Tokens.Add(TokenType.InvDestAlpha);
regex = new Regex(@"DestColor", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.DestColor, regex);
Tokens.Add(TokenType.DestColor);
regex = new Regex(@"InvDestColor", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.InvDestColor, regex);
Tokens.Add(TokenType.InvDestColor);
regex = new Regex(@"SrcAlphaSat", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.SrcAlphaSat, regex);
Tokens.Add(TokenType.SrcAlphaSat);
regex = new Regex(@"BlendFactor", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.BlendFactor, regex);
Tokens.Add(TokenType.BlendFactor);
regex = new Regex(@"InvBlendFactor", RegexOptions.Compiled | RegexOptions.IgnoreCase);
Patterns.Add(TokenType.InvBlendFactor, regex);
Tokens.Add(TokenType.InvBlendFactor);
}
/// <summary>
/// returns token with longest best match
/// </summary>
/// <returns></returns>
public Token LookAhead(params TokenType[] expectedtokens)
{
int i;
int startpos = StartPos;
int endpos = EndPos;
int currentline = CurrentLine;
string currentFile = CurrentFile;
Token tok = null;
List <TokenType> scantokens;
// this prevents double scanning and matching
// increased performance
if (LookAheadToken != null &&
LookAheadToken.Type != TokenType._UNDETERMINED_ &&
LookAheadToken.Type != TokenType._NONE_)
{
return(LookAheadToken);
}
// if no scantokens specified, then scan for all of them (= backward compatible)
if (expectedtokens.Length == 0)
{
scantokens = Tokens;
}
else
{
scantokens = new List <TokenType>(expectedtokens);
scantokens.AddRange(SkipList);
}
do
{
int len = -1;
TokenType index = (TokenType)int.MaxValue;
string input = Input.Substring(startpos);
tok = new Token(startpos, endpos);
for (i = 0; i < scantokens.Count; i++)
{
Regex r = Patterns[scantokens[i]];
Match m = r.Match(input);
if (m.Success && m.Index == 0 && ((m.Length > len) || (scantokens[i] < index && m.Length == len)))
{
len = m.Length;
index = scantokens[i];
}
}
if (index >= 0 && len >= 0)
{
tok.EndPos = startpos + len;
tok.Text = Input.Substring(tok.StartPos, len);
tok.Type = index;
}
else if (tok.StartPos == tok.EndPos)
{
if (tok.StartPos < Input.Length)
{
tok.Text = Input.Substring(tok.StartPos, 1);
}
else
{
tok.Text = "EOF";
}
}
// Update the line and column count for error reporting.
tok.File = currentFile;
tok.Line = currentline;
if (tok.StartPos < Input.Length)
{
tok.Column = tok.StartPos - Input.LastIndexOf('\n', tok.StartPos);
}
if (SkipList.Contains(tok.Type))
{
startpos = tok.EndPos;
endpos = tok.EndPos;
currentline = tok.Line + (tok.Text.Length - tok.Text.Replace("\n", "").Length);
currentFile = tok.File;
Skipped.Add(tok);
}
else
{
// only assign to non-skipped tokens
tok.Skipped = Skipped; // assign prior skips to this token
Skipped = new List <Token>(); //reset skips
}
// Check to see if the parsed token wants to
// alter the file and line number.
if (tok.Type == FileAndLine)
{
var match = Patterns[tok.Type].Match(tok.Text);
var fileMatch = match.Groups["File"];
if (fileMatch.Success)
{
currentFile = fileMatch.Value.Replace("\\\\", "\\");
}
var lineMatch = match.Groups["Line"];
if (lineMatch.Success)
{
currentline = int.Parse(lineMatch.Value, NumberStyles.Integer, CultureInfo.InvariantCulture);
}
}
}while (SkipList.Contains(tok.Type));
LookAheadToken = tok;
return(tok);
}
