Beispiel #1
0
        /// <summary>
        /// returns token with longest best match
        /// </summary>
        /// <returns></returns>
        public Token LookAhead(params TokenType[] expectedtokens)
        {
            int i;
            int startpos = StartPos;
            int endpos = EndPos;
            int currentline = CurrentLine;
            string currentFile = CurrentFile;
            Token tok = null;
            List<TokenType> scantokens;


            // this prevents double scanning and matching
            // increased performance
            if (LookAheadToken != null 
                && LookAheadToken.Type != TokenType._UNDETERMINED_ 
                && LookAheadToken.Type != TokenType._NONE_) return LookAheadToken;

            // if no scantokens specified, then scan for all of them (= backward compatible)
            if (expectedtokens.Length == 0)
                scantokens = Tokens;
            else
            {
                scantokens = new List<TokenType>(expectedtokens);
                scantokens.AddRange(SkipList);
            }

            do
            {

                int len = -1;
                TokenType index = (TokenType)int.MaxValue;
                string input = Input.Substring(startpos);

                tok = new Token(startpos, endpos);

                for (i = 0; i < scantokens.Count; i++)
                {
                    Regex r = Patterns[scantokens[i]];
                    Match m = r.Match(input);
                    if (m.Success && m.Index == 0 && ((m.Length > len) || (scantokens[i] < index && m.Length == len )))
                    {
                        len = m.Length;
                        index = scantokens[i];  
                    }
                }

                if (index >= 0 && len >= 0)
                {
                    tok.EndPos = startpos + len;
                    tok.Text = Input.Substring(tok.StartPos, len);
                    tok.Type = index;
                }
                else if (tok.StartPos == tok.EndPos)
                {
                    if (tok.StartPos < Input.Length)
                        tok.Text = Input.Substring(tok.StartPos, 1);
                    else
                        tok.Text = "EOF";
                }

                // Update the line and column count for error reporting.
                tok.File = currentFile;
                tok.Line = currentline;
                if (tok.StartPos < Input.Length)
                    tok.Column = tok.StartPos - Input.LastIndexOf('\n', tok.StartPos);

                if (SkipList.Contains(tok.Type))
                {
                    startpos = tok.EndPos;
                    endpos = tok.EndPos;
                    currentline = tok.Line + (tok.Text.Length - tok.Text.Replace("\n", "").Length);
                    currentFile = tok.File;
                    Skipped.Add(tok);
                }
                else
                {
                    // only assign to non-skipped tokens
                    tok.Skipped = Skipped; // assign prior skips to this token
                    Skipped = new List<Token>(); //reset skips
                }

                // Check to see if the parsed token wants to 
                // alter the file and line number.
                if (tok.Type == FileAndLine)
                {
                    var match = Patterns[tok.Type].Match(tok.Text);
                    var fileMatch = match.Groups["File"];
                    if (fileMatch.Success)
                        currentFile = fileMatch.Value.Replace("\\\\", "\\");
                    var lineMatch = match.Groups["Line"];
                    if (lineMatch.Success)
                        currentline = int.Parse(lineMatch.Value, NumberStyles.Integer, CultureInfo.InvariantCulture);
                }
            }
            while (SkipList.Contains(tok.Type));

            LookAheadToken = tok;
            return tok;
        }
Beispiel #2
0
 public void UpdateRange(Token token)
 {
     if (token.StartPos < startpos) startpos = token.StartPos;
     if (token.EndPos > endpos) endpos = token.EndPos;
 }
Beispiel #3
0
  /// <summary>
 /// executes a lookahead of the next token
 /// and will advance the scan on the input string
 /// </summary>
 /// <returns></returns>
 public Token Scan(params TokenType[] expectedtokens)
 {
     Token tok = LookAhead(expectedtokens); // temporarely retrieve the lookahead
     LookAheadToken = null; // reset lookahead token, so scanning will continue
     StartPos = tok.EndPos;
     EndPos = tok.EndPos; // set the tokenizer to the new scan position
     CurrentLine = tok.Line + (tok.Text.Length - tok.Text.Replace("\n", "").Length);
     CurrentFile = tok.File;
     return tok;
 }
Beispiel #4
0
 public Token GetToken(TokenType type)
 {
     Token t = new Token(this.StartPos, this.EndPos);
     t.Type = type;
     return t;
 }
Beispiel #5
0
 public void Init(string input, string fileName)
 {
     this.Input = input;
     StartPos = 0;
     EndPos = 0;
     CurrentFile = fileName;
     CurrentLine = 1;
     CurrentColumn = 1;
     CurrentPosition = 0;
     LookAheadToken = null;
 }
Beispiel #6
0
        public Scanner()
        {
            Regex regex;
            Patterns = new Dictionary<TokenType, Regex>();
            Tokens = new List<TokenType>();
            LookAheadToken = null;
            Skipped = new List<Token>();

            SkipList = new List<TokenType>();
            SkipList.Add(TokenType.BlockComment);
            SkipList.Add(TokenType.Comment);
            SkipList.Add(TokenType.Whitespace);
            SkipList.Add(TokenType.LinePragma);
            FileAndLine = TokenType.LinePragma;

            regex = new Regex(@"/\*([^*]|\*[^/])*\*/", RegexOptions.Compiled);
            Patterns.Add(TokenType.BlockComment, regex);
            Tokens.Add(TokenType.BlockComment);

            regex = new Regex(@"//[^\n\r]*", RegexOptions.Compiled);
            Patterns.Add(TokenType.Comment, regex);
            Tokens.Add(TokenType.Comment);

            regex = new Regex(@"[ \t\n\r]+", RegexOptions.Compiled);
            Patterns.Add(TokenType.Whitespace, regex);
            Tokens.Add(TokenType.Whitespace);

            regex = new Regex(@"^[ \t]*#line[ \t]*(?<Line>\d*)[ \t]*(\""(?<File>[^\""\\]*(?:\\.[^\""\\]*)*)\"")?\n", RegexOptions.Compiled);
            Patterns.Add(TokenType.LinePragma, regex);
            Tokens.Add(TokenType.LinePragma);

            regex = new Regex(@"pass", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.Pass, regex);
            Tokens.Add(TokenType.Pass);

            regex = new Regex(@"technique", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.Technique, regex);
            Tokens.Add(TokenType.Technique);

            regex = new Regex(@"sampler1D|sampler2D|sampler3D|samplerCUBE|SamplerState|sampler", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.Sampler, regex);
            Tokens.Add(TokenType.Sampler);

            regex = new Regex(@"sampler_state", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.SamplerState, regex);
            Tokens.Add(TokenType.SamplerState);

            regex = new Regex(@"VertexShader", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.VertexShader, regex);
            Tokens.Add(TokenType.VertexShader);

            regex = new Regex(@"PixelShader", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.PixelShader, regex);
            Tokens.Add(TokenType.PixelShader);

            regex = new Regex(@"register", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.Register, regex);
            Tokens.Add(TokenType.Register);

            regex = new Regex(@"true|false|0|1", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.Boolean, regex);
            Tokens.Add(TokenType.Boolean);

            regex = new Regex(@"[+-]? ?[0-9]?\.?[0-9]+[fF]?", RegexOptions.Compiled);
            Patterns.Add(TokenType.Number, regex);
            Tokens.Add(TokenType.Number);

            regex = new Regex(@"0x[0-9a-f]{6}([0-9a-f][0-9a-f])?", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.HexColor, regex);
            Tokens.Add(TokenType.HexColor);

            regex = new Regex(@"[A-Za-z_][A-Za-z0-9_]*", RegexOptions.Compiled);
            Patterns.Add(TokenType.Identifier, regex);
            Tokens.Add(TokenType.Identifier);

            regex = new Regex(@"{", RegexOptions.Compiled);
            Patterns.Add(TokenType.OpenBracket, regex);
            Tokens.Add(TokenType.OpenBracket);

            regex = new Regex(@"}", RegexOptions.Compiled);
            Patterns.Add(TokenType.CloseBracket, regex);
            Tokens.Add(TokenType.CloseBracket);

            regex = new Regex(@"=", RegexOptions.Compiled);
            Patterns.Add(TokenType.Equals, regex);
            Tokens.Add(TokenType.Equals);

            regex = new Regex(@":", RegexOptions.Compiled);
            Patterns.Add(TokenType.Colon, regex);
            Tokens.Add(TokenType.Colon);

            regex = new Regex(@",", RegexOptions.Compiled);
            Patterns.Add(TokenType.Comma, regex);
            Tokens.Add(TokenType.Comma);

            regex = new Regex(@";", RegexOptions.Compiled);
            Patterns.Add(TokenType.Semicolon, regex);
            Tokens.Add(TokenType.Semicolon);

            regex = new Regex(@"\|", RegexOptions.Compiled);
            Patterns.Add(TokenType.Or, regex);
            Tokens.Add(TokenType.Or);

            regex = new Regex(@"\(", RegexOptions.Compiled);
            Patterns.Add(TokenType.OpenParenthesis, regex);
            Tokens.Add(TokenType.OpenParenthesis);

            regex = new Regex(@"\)", RegexOptions.Compiled);
            Patterns.Add(TokenType.CloseParenthesis, regex);
            Tokens.Add(TokenType.CloseParenthesis);

            regex = new Regex(@"\[", RegexOptions.Compiled);
            Patterns.Add(TokenType.OpenSquareBracket, regex);
            Tokens.Add(TokenType.OpenSquareBracket);

            regex = new Regex(@"\]", RegexOptions.Compiled);
            Patterns.Add(TokenType.CloseSquareBracket, regex);
            Tokens.Add(TokenType.CloseSquareBracket);

            regex = new Regex(@"<", RegexOptions.Compiled);
            Patterns.Add(TokenType.LessThan, regex);
            Tokens.Add(TokenType.LessThan);

            regex = new Regex(@">", RegexOptions.Compiled);
            Patterns.Add(TokenType.GreaterThan, regex);
            Tokens.Add(TokenType.GreaterThan);

            regex = new Regex(@"compile", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.Compile, regex);
            Tokens.Add(TokenType.Compile);

            regex = new Regex(@"[A-Za-z_][A-Za-z0-9_]*", RegexOptions.Compiled);
            Patterns.Add(TokenType.ShaderModel, regex);
            Tokens.Add(TokenType.ShaderModel);

            regex = new Regex(@"[\S]+", RegexOptions.Compiled);
            Patterns.Add(TokenType.Code, regex);
            Tokens.Add(TokenType.Code);

            regex = new Regex(@"^$", RegexOptions.Compiled);
            Patterns.Add(TokenType.EndOfFile, regex);
            Tokens.Add(TokenType.EndOfFile);

            regex = new Regex(@"MinFilter", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.MinFilter, regex);
            Tokens.Add(TokenType.MinFilter);

            regex = new Regex(@"MagFilter", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.MagFilter, regex);
            Tokens.Add(TokenType.MagFilter);

            regex = new Regex(@"MipFilter", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.MipFilter, regex);
            Tokens.Add(TokenType.MipFilter);

            regex = new Regex(@"Filter", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.Filter, regex);
            Tokens.Add(TokenType.Filter);

            regex = new Regex(@"Texture", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.Texture, regex);
            Tokens.Add(TokenType.Texture);

            regex = new Regex(@"AddressU", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.AddressU, regex);
            Tokens.Add(TokenType.AddressU);

            regex = new Regex(@"AddressV", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.AddressV, regex);
            Tokens.Add(TokenType.AddressV);

            regex = new Regex(@"AddressW", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.AddressW, regex);
            Tokens.Add(TokenType.AddressW);

            regex = new Regex(@"BorderColor", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.BorderColor, regex);
            Tokens.Add(TokenType.BorderColor);

            regex = new Regex(@"MaxAnisotropy", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.MaxAnisotropy, regex);
            Tokens.Add(TokenType.MaxAnisotropy);

            regex = new Regex(@"MaxMipLevel|MaxLod", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.MaxMipLevel, regex);
            Tokens.Add(TokenType.MaxMipLevel);

            regex = new Regex(@"MipLodBias", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.MipLodBias, regex);
            Tokens.Add(TokenType.MipLodBias);

            regex = new Regex(@"Clamp", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.Clamp, regex);
            Tokens.Add(TokenType.Clamp);

            regex = new Regex(@"Wrap", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.Wrap, regex);
            Tokens.Add(TokenType.Wrap);

            regex = new Regex(@"Mirror", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.Mirror, regex);
            Tokens.Add(TokenType.Mirror);

            regex = new Regex(@"Border", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.Border, regex);
            Tokens.Add(TokenType.Border);

            regex = new Regex(@"None", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.None, regex);
            Tokens.Add(TokenType.None);

            regex = new Regex(@"Linear", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.Linear, regex);
            Tokens.Add(TokenType.Linear);

            regex = new Regex(@"Point", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.Point, regex);
            Tokens.Add(TokenType.Point);

            regex = new Regex(@"Anisotropic", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.Anisotropic, regex);
            Tokens.Add(TokenType.Anisotropic);

            regex = new Regex(@"AlphaBlendEnable", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.AlphaBlendEnable, regex);
            Tokens.Add(TokenType.AlphaBlendEnable);

            regex = new Regex(@"SrcBlend", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.SrcBlend, regex);
            Tokens.Add(TokenType.SrcBlend);

            regex = new Regex(@"DestBlend", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.DestBlend, regex);
            Tokens.Add(TokenType.DestBlend);

            regex = new Regex(@"BlendOp", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.BlendOp, regex);
            Tokens.Add(TokenType.BlendOp);

            regex = new Regex(@"ColorWriteEnable", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.ColorWriteEnable, regex);
            Tokens.Add(TokenType.ColorWriteEnable);

            regex = new Regex(@"ZEnable", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.ZEnable, regex);
            Tokens.Add(TokenType.ZEnable);

            regex = new Regex(@"ZWriteEnable", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.ZWriteEnable, regex);
            Tokens.Add(TokenType.ZWriteEnable);

            regex = new Regex(@"ZFunc", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.ZFunc, regex);
            Tokens.Add(TokenType.ZFunc);

            regex = new Regex(@"DepthBias", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.DepthBias, regex);
            Tokens.Add(TokenType.DepthBias);

            regex = new Regex(@"CullMode", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.CullMode, regex);
            Tokens.Add(TokenType.CullMode);

            regex = new Regex(@"FillMode", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.FillMode, regex);
            Tokens.Add(TokenType.FillMode);

            regex = new Regex(@"MultiSampleAntiAlias", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.MultiSampleAntiAlias, regex);
            Tokens.Add(TokenType.MultiSampleAntiAlias);

            regex = new Regex(@"ScissorTestEnable", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.ScissorTestEnable, regex);
            Tokens.Add(TokenType.ScissorTestEnable);

            regex = new Regex(@"SlopeScaleDepthBias", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.SlopeScaleDepthBias, regex);
            Tokens.Add(TokenType.SlopeScaleDepthBias);

            regex = new Regex(@"StencilEnable", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.StencilEnable, regex);
            Tokens.Add(TokenType.StencilEnable);

            regex = new Regex(@"StencilFail", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.StencilFail, regex);
            Tokens.Add(TokenType.StencilFail);

            regex = new Regex(@"StencilFunc", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.StencilFunc, regex);
            Tokens.Add(TokenType.StencilFunc);

            regex = new Regex(@"StencilMask", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.StencilMask, regex);
            Tokens.Add(TokenType.StencilMask);

            regex = new Regex(@"StencilPass", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.StencilPass, regex);
            Tokens.Add(TokenType.StencilPass);

            regex = new Regex(@"StencilRef", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.StencilRef, regex);
            Tokens.Add(TokenType.StencilRef);

            regex = new Regex(@"StencilWriteMask", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.StencilWriteMask, regex);
            Tokens.Add(TokenType.StencilWriteMask);

            regex = new Regex(@"StencilZFail", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.StencilZFail, regex);
            Tokens.Add(TokenType.StencilZFail);

            regex = new Regex(@"Never", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.Never, regex);
            Tokens.Add(TokenType.Never);

            regex = new Regex(@"Less", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.Less, regex);
            Tokens.Add(TokenType.Less);

            regex = new Regex(@"Equal", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.Equal, regex);
            Tokens.Add(TokenType.Equal);

            regex = new Regex(@"LessEqual", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.LessEqual, regex);
            Tokens.Add(TokenType.LessEqual);

            regex = new Regex(@"Greater", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.Greater, regex);
            Tokens.Add(TokenType.Greater);

            regex = new Regex(@"NotEqual", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.NotEqual, regex);
            Tokens.Add(TokenType.NotEqual);

            regex = new Regex(@"GreaterEqual", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.GreaterEqual, regex);
            Tokens.Add(TokenType.GreaterEqual);

            regex = new Regex(@"Always", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.Always, regex);
            Tokens.Add(TokenType.Always);

            regex = new Regex(@"Keep", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.Keep, regex);
            Tokens.Add(TokenType.Keep);

            regex = new Regex(@"Zero", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.Zero, regex);
            Tokens.Add(TokenType.Zero);

            regex = new Regex(@"Replace", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.Replace, regex);
            Tokens.Add(TokenType.Replace);

            regex = new Regex(@"IncrSat", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.IncrSat, regex);
            Tokens.Add(TokenType.IncrSat);

            regex = new Regex(@"DecrSat", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.DecrSat, regex);
            Tokens.Add(TokenType.DecrSat);

            regex = new Regex(@"Invert", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.Invert, regex);
            Tokens.Add(TokenType.Invert);

            regex = new Regex(@"Incr", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.Incr, regex);
            Tokens.Add(TokenType.Incr);

            regex = new Regex(@"Decr", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.Decr, regex);
            Tokens.Add(TokenType.Decr);

            regex = new Regex(@"Red", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.Red, regex);
            Tokens.Add(TokenType.Red);

            regex = new Regex(@"Green", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.Green, regex);
            Tokens.Add(TokenType.Green);

            regex = new Regex(@"Blue", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.Blue, regex);
            Tokens.Add(TokenType.Blue);

            regex = new Regex(@"Alpha", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.Alpha, regex);
            Tokens.Add(TokenType.Alpha);

            regex = new Regex(@"All", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.All, regex);
            Tokens.Add(TokenType.All);

            regex = new Regex(@"Cw", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.Cw, regex);
            Tokens.Add(TokenType.Cw);

            regex = new Regex(@"Ccw", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.Ccw, regex);
            Tokens.Add(TokenType.Ccw);

            regex = new Regex(@"Solid", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.Solid, regex);
            Tokens.Add(TokenType.Solid);

            regex = new Regex(@"WireFrame", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.WireFrame, regex);
            Tokens.Add(TokenType.WireFrame);

            regex = new Regex(@"Add", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.Add, regex);
            Tokens.Add(TokenType.Add);

            regex = new Regex(@"Subtract", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.Subtract, regex);
            Tokens.Add(TokenType.Subtract);

            regex = new Regex(@"RevSubtract", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.RevSubtract, regex);
            Tokens.Add(TokenType.RevSubtract);

            regex = new Regex(@"Min", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.Min, regex);
            Tokens.Add(TokenType.Min);

            regex = new Regex(@"Max", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.Max, regex);
            Tokens.Add(TokenType.Max);

            regex = new Regex(@"One", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.One, regex);
            Tokens.Add(TokenType.One);

            regex = new Regex(@"SrcColor", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.SrcColor, regex);
            Tokens.Add(TokenType.SrcColor);

            regex = new Regex(@"InvSrcColor", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.InvSrcColor, regex);
            Tokens.Add(TokenType.InvSrcColor);

            regex = new Regex(@"SrcAlpha", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.SrcAlpha, regex);
            Tokens.Add(TokenType.SrcAlpha);

            regex = new Regex(@"InvSrcAlpha", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.InvSrcAlpha, regex);
            Tokens.Add(TokenType.InvSrcAlpha);

            regex = new Regex(@"DestAlpha", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.DestAlpha, regex);
            Tokens.Add(TokenType.DestAlpha);

            regex = new Regex(@"InvDestAlpha", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.InvDestAlpha, regex);
            Tokens.Add(TokenType.InvDestAlpha);

            regex = new Regex(@"DestColor", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.DestColor, regex);
            Tokens.Add(TokenType.DestColor);

            regex = new Regex(@"InvDestColor", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.InvDestColor, regex);
            Tokens.Add(TokenType.InvDestColor);

            regex = new Regex(@"SrcAlphaSat", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.SrcAlphaSat, regex);
            Tokens.Add(TokenType.SrcAlphaSat);

            regex = new Regex(@"BlendFactor", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.BlendFactor, regex);
            Tokens.Add(TokenType.BlendFactor);

            regex = new Regex(@"InvBlendFactor", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.InvBlendFactor, regex);
            Tokens.Add(TokenType.InvBlendFactor);


        }
Beispiel #7
0
        public Scanner()
        {
            Regex regex;
            Patterns = new Dictionary<TokenType, Regex>();
            Tokens = new List<TokenType>();
            LookAheadToken = null;
            Skipped = new List<Token>();

            SkipList = new List<TokenType>();
            SkipList.Add(TokenType.BlockComment);
            SkipList.Add(TokenType.Comment);
            SkipList.Add(TokenType.Whitespace);
            SkipList.Add(TokenType.LinePragma);
            FileAndLine = TokenType.LinePragma;

            regex = new Regex(@"/\*([^*]|\*[^/])*\*/", RegexOptions.Compiled);
            Patterns.Add(TokenType.BlockComment, regex);
            Tokens.Add(TokenType.BlockComment);

            regex = new Regex(@"//[^\n\r]*", RegexOptions.Compiled);
            Patterns.Add(TokenType.Comment, regex);
            Tokens.Add(TokenType.Comment);

            regex = new Regex(@"[ \t\n\r]+", RegexOptions.Compiled);
            Patterns.Add(TokenType.Whitespace, regex);
            Tokens.Add(TokenType.Whitespace);

            regex = new Regex(@"^[ \t]*#line[ \t]*(?<Line>\d*)[ \t]*(\""(?<File>[^\""\\]*(?:\\.[^\""\\]*)*)\"")?\n", RegexOptions.Compiled);
            Patterns.Add(TokenType.LinePragma, regex);
            Tokens.Add(TokenType.LinePragma);

            regex = new Regex(@"pass", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.Pass, regex);
            Tokens.Add(TokenType.Pass);

            regex = new Regex(@"technique", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.Technique, regex);
            Tokens.Add(TokenType.Technique);

            regex = new Regex(@"sampler1D|sampler2D|sampler3D|samplerCUBE|sampler", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.Sampler, regex);
            Tokens.Add(TokenType.Sampler);

            regex = new Regex(@"sampler_state", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.SamplerState, regex);
            Tokens.Add(TokenType.SamplerState);

            regex = new Regex(@"VertexShader", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.VertexShader, regex);
            Tokens.Add(TokenType.VertexShader);

            regex = new Regex(@"PixelShader", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.PixelShader, regex);
            Tokens.Add(TokenType.PixelShader);

            regex = new Regex(@"register", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.Register, regex);
            Tokens.Add(TokenType.Register);

            regex = new Regex(@"[0-9]?\.?[0-9]+", RegexOptions.Compiled);
            Patterns.Add(TokenType.Number, regex);
            Tokens.Add(TokenType.Number);

            regex = new Regex(@"-|\+", RegexOptions.Compiled);
            Patterns.Add(TokenType.Sign, regex);
            Tokens.Add(TokenType.Sign);

            regex = new Regex(@"[A-Za-z_][A-Za-z0-9_]*", RegexOptions.Compiled);
            Patterns.Add(TokenType.Identifier, regex);
            Tokens.Add(TokenType.Identifier);

            regex = new Regex(@"{", RegexOptions.Compiled);
            Patterns.Add(TokenType.OpenBracket, regex);
            Tokens.Add(TokenType.OpenBracket);

            regex = new Regex(@"}", RegexOptions.Compiled);
            Patterns.Add(TokenType.CloseBracket, regex);
            Tokens.Add(TokenType.CloseBracket);

            regex = new Regex(@"=", RegexOptions.Compiled);
            Patterns.Add(TokenType.Equals, regex);
            Tokens.Add(TokenType.Equals);

            regex = new Regex(@":", RegexOptions.Compiled);
            Patterns.Add(TokenType.Colon, regex);
            Tokens.Add(TokenType.Colon);

            regex = new Regex(@",", RegexOptions.Compiled);
            Patterns.Add(TokenType.Comma, regex);
            Tokens.Add(TokenType.Comma);

            regex = new Regex(@";", RegexOptions.Compiled);
            Patterns.Add(TokenType.Semicolon, regex);
            Tokens.Add(TokenType.Semicolon);

            regex = new Regex(@"\(", RegexOptions.Compiled);
            Patterns.Add(TokenType.OpenParenthesis, regex);
            Tokens.Add(TokenType.OpenParenthesis);

            regex = new Regex(@"\)", RegexOptions.Compiled);
            Patterns.Add(TokenType.CloseParenthesis, regex);
            Tokens.Add(TokenType.CloseParenthesis);

            regex = new Regex(@"\[", RegexOptions.Compiled);
            Patterns.Add(TokenType.OpenSquareBracket, regex);
            Tokens.Add(TokenType.OpenSquareBracket);

            regex = new Regex(@"\]", RegexOptions.Compiled);
            Patterns.Add(TokenType.CloseSquareBracket, regex);
            Tokens.Add(TokenType.CloseSquareBracket);

            regex = new Regex(@"<", RegexOptions.Compiled);
            Patterns.Add(TokenType.LessThan, regex);
            Tokens.Add(TokenType.LessThan);

            regex = new Regex(@">", RegexOptions.Compiled);
            Patterns.Add(TokenType.GreaterThan, regex);
            Tokens.Add(TokenType.GreaterThan);

            regex = new Regex(@"compile", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.Compile, regex);
            Tokens.Add(TokenType.Compile);

            regex = new Regex(@"(vs_|ps_)(2_0|3_0|4_0|5_0)((_level_)(9_1|9_2|9_3))?", RegexOptions.Compiled | RegexOptions.IgnoreCase);
            Patterns.Add(TokenType.ShaderModel, regex);
            Tokens.Add(TokenType.ShaderModel);

            regex = new Regex(@"[\S]+", RegexOptions.Compiled);
            Patterns.Add(TokenType.Code, regex);
            Tokens.Add(TokenType.Code);

            regex = new Regex(@"^$", RegexOptions.Compiled);
            Patterns.Add(TokenType.EndOfFile, regex);
            Tokens.Add(TokenType.EndOfFile);


        }