예제 #1
0
 public void TestTokenize4()
 {
     var CTokenizer = new CTokenizer("test\n  #include", TokenizeSpaces: false);
     var Tokens = CTokenizer.Tokenize().ToArray();
     Assert.AreEqual("Position:0, Row:0, Column:0, ColumnNoSpaces:0", Tokens[0].Position.ToString());
     Assert.AreEqual("Position:7, Row:1, Column:2, ColumnNoSpaces:0", Tokens[1].Position.ToString());
     Assert.AreEqual("Position:8, Row:1, Column:3, ColumnNoSpaces:1", Tokens[2].Position.ToString());
 }
예제 #2
0
 public void TestTokenizeDouble()
 {
     var CTokenizer = new CTokenizer("1.0, .0f");
     var Tokens = CTokenizer.Tokenize().ToArray();
     CollectionAssert.AreEqual(
         new[] { "1.0", ",", ".0f", "" },
         Tokens.Select(Item => Item.Raw).ToArray()
     );
 }
예제 #3
0
 public void TestTokenize3()
 {
     var CTokenizer = new CTokenizer("1, 2, 0x100");
     var Tokens = CTokenizer.Tokenize().ToArray();
     CollectionAssert.AreEqual(
         new[] { "1", ",", "2", ",", "0x100", "" },
         Tokens.Select(Item => Item.Raw).ToArray()
     );
 }
예제 #4
0
 public void TestTokenize()
 {
     var CTokenizer = new CTokenizer(" 'a' && 'b' test + 2 * test3");
     var Tokens = CTokenizer.Tokenize().ToArray();
     CollectionAssert.AreEqual(
         new[] { "'a'", "&&", "'b'", "test", "+", "2", "*", "test3", "" },
         Tokens.Select(Item => Item.Raw).ToArray()
     );
 }
예제 #5
0
 public static string RemoveComments(string Input)
 {
     var CTokenizer = new CTokenizer(Input, TokenizeSpaces: true);
     var Tokens = CTokenizer.Tokenize().GetEnumerator();
     string Output = "";
     while (Tokens.MoveNext())
     {
         switch (Tokens.Current.Raw)
         {
             case "//":
                 Output += new String(' ', CTokenizer.SkipUntilSequence("\n") - 1) + "\n";
                 break;
             case "/*":
                 Output += ReplaceNonSpaceWithSpaces(CTokenizer.ReadUntilSequence("*/"));
                 break;
             default:
                 Output += Tokens.Current.Raw;
                 break;
         }
     }
     return Output;
 }
예제 #6
0
        /// <summary>
        /// 
        /// </summary>
        /// <param name="FileName"></param>
        /// <param name="Text"></param>
        /// <param name="Context"></param>
        public CPreprocessorInternal(string FileName, string Text, CPreprocessorContext Context)
        {
            // Remove comments.

            try { FileName = Path.GetFullPath(FileName); }
            catch { }

            if (Context.DebugPreprocessor)
            {
                Console.WriteLine("CPreprocessorInternal(FileName={0})", FileName);
            }

            Text = CPreprocessor.RemoveComments(Text.Replace("\r\n", "\n").Replace("\r", "\n"));

            this.Text = Text;
            this.CurrentFileName = FileName;
            this.CTokenizer = new CTokenizer(Text, TokenizeSpaces: true);
            this.Context = Context;
            this.Tokens = new CTokenReader(CTokenizer.Tokenize());
            this.Tokens.MoveNextSpace();

            OutputLine();
            //Console.WriteLine(Tokens.GetString());
        }
예제 #7
0
 public void TestTokenizeStringFormat1()
 {
     var CTokenizer = new CTokenizer(@" ""\03a"" ");
     var Tokens = CTokenizer.Tokenize().ToArray();
     Console.WriteLine(Tokens[0].Raw);
     var Str = Tokens[0].GetStringValue();
     Assert.AreEqual(3, Str[0]);
     Assert.AreEqual('a', Str[1]);
 }
예제 #8
0
 public void TestTokenizeStringCat()
 {
     var CTokenizer = new CTokenizer(@" ""a"" ""b"" ");
     var Tokens = CTokenizer.Tokenize().ToArray();
     Console.WriteLine(String.Join("\n", Tokens.Select(Token => Token.Raw)));
     //Assert.AreEqual(CTokenType.Integer, Tokens[0].Type);
 }
예제 #9
0
 public void TestTokenizeLineFeeds()
 {
     var CTokenizer = new CTokenizer("\n\na");
     var Tokens = CTokenizer.Tokenize().ToArray();
     Assert.AreEqual(2, Tokens[0].Position.Row);
 }
예제 #10
0
 public void TestTokenize2()
 {
     var CTokenizer = new CTokenizer("/* comment's */", TokenizeSpaces: true);
     var Tokens = CTokenizer.Tokenize().GetEnumerator();
     Tokens.MoveNext();
 }
예제 #11
0
 public void TestTokenize0xff()
 {
     var CTokenizer = new CTokenizer("0xff");
     var Tokens = CTokenizer.Tokenize().ToArray();
     Assert.AreEqual(CTokenType.Integer, Tokens[0].Type);
 }
예제 #12
0
        public string GetStringValue()
        {
            if (Type != CTokenType.String && Type != CTokenType.Char)
            {
                throw (new Exception("Trying to get the string value from a token that is not a string"));
            }
            if (Raw.Length < 2)
            {
                throw(new Exception("Invalid string token"));
            }
            string Result = "";

            for (int n = 1; n < Raw.Length - 1; n++)
            {
                if (Raw[n] == '\\')
                {
                    var NextScape = Raw[n + 1];
                    switch (NextScape)
                    {
                    case 'n': Result += '\n'; n++; break;

                    case 'r': Result += '\r'; n++; break;

                    case 't': Result += '\t'; n++; break;

                    case '\\': Result += '\\'; n++; break;

                    case '"': Result += '\"'; n++; break;

                    case '\'': Result += '\''; n++; break;

                    case 'x':
                        Result += (char)Convert.ToUInt16(Raw.Substring(n + 2, 2), 16);
                        n      += 2;
                        break;

                    default:
                        if (CTokenizer.IsNumber(NextScape))
                        {
                            string StrNum = "";
                            while (CTokenizer.IsNumber(Raw[n + 1]))
                            {
                                StrNum += Raw[n + 1] - '0';
                                n++;
                            }
                            Result += (char)int.Parse(StrNum);
                            break;
                        }
                        else
                        {
                            throw (new NotImplementedException(String.Format("Unimplemented '{0}'", NextScape)));
                        }
                    }
                }
                else
                {
                    Result += Raw[n];
                }
            }
            return(Result);
        }