Пример #1
0
        private Token MakeTokenFromString(string InString)
        {
            try
            {
                Token  Tk       = new Token(MForm);
                string LevelStr = "";
                int    Position = 0;
                int    Last     = InString.Length;
                for (int Count = 0; Count < Last; Count++)
                {
                    char TestChar = InString[Count];
                    if (Markers.IsMarker(TestChar))
                    {
                        Tk.TokenType = TestChar;
                        Position     = Count + 1;
                        break;
                    }

                    LevelStr += Char.ToString(TestChar);
                }

                // ShowStatus( "LevelStr: " + LevelStr );
                Tk.Level = Int32.Parse(LevelStr);
                // ShowStatus( "Level: " + Tk.Level.ToString());

                for (int Count = Position; Count < Last; Count++)
                {
                    char TestChar = InString[Count];
                    if (Markers.IsMarker(TestChar))
                    {
                        break;
                    }

                    Tk.Text += Char.ToString(TestChar);
                }

                // ShowStatus( "Text: " + Tk.Text );
                return(Tk);
            }
            catch (Exception Except)
            {
                ShowStatus("Exception in Token.MakeTokenFromString():");
                ShowStatus(Except.Message);
                return(null);
            }
        }
Пример #2
0
        private static string GetCleanUnicodeString(string InString)
        {
            if (InString == null)
            {
                return("");
            }

            StringBuilder SBuilder = new StringBuilder();
            int           Last     = InString.Length;

            for (int Count = 0; Count < Last; Count++)
            {
                char ToCheck = InString[Count];

                // Get rid of control characters.
                // This also replaces a tab with a space.
                if (ToCheck < ' ')
                {
                    ToCheck = ' ';
                }

                //  Don't go higher than D800 (Surrogates).
                if (ToCheck >= 0xD800)
                {
                    ToCheck = ' ';
                }

                // This has already been converted from UTF8
                // to 16 bit characters.
                if ((ToCheck >= 127) && (ToCheck <= 255))
                {
                    ToCheck = ' ';
                }

                // Don't exclude any characters in the Basic
                // Multilingual Plane except markers that
                // shouldn't be in this.  See the Markers.cs
                // file.
                if (Markers.IsMarker(ToCheck))
                {
                    ToCheck = ' ';
                }

                // Basic Multilingual Plane
                // C0 Controls and Basic Latin (Basic Latin) (0000–007F)
                // C1 Controls and Latin-1 Supplement (0080–00FF)
                // Latin Extended-A (0100–017F)
                // Latin Extended-B (0180–024F)
                // IPA Extensions (0250–02AF)
                // Spacing Modifier Letters (02B0–02FF)
                // Combining Diacritical Marks (0300–036F)
                // General Punctuation (2000–206F)
                // Superscripts and Subscripts (2070–209F)
                // Currency Symbols (20A0–20CF)
                // Combining Diacritical Marks for Symbols (20D0–20FF)
                // Letterlike Symbols (2100–214F)
                // Number Forms (2150–218F)
                // Arrows (2190–21FF)
                // Mathematical Operators (2200–22FF)
                // Box Drawing (2500–257F)
                // Geometric Shapes (25A0–25FF)
                // Miscellaneous Symbols (2600–26FF)
                // Dingbats (2700–27BF)
                // Miscellaneous Symbols and Arrows (2B00–2BFF)

                SBuilder.Append(Char.ToString(ToCheck));
            }

            return(SBuilder.ToString());
        }
Пример #3
0
        internal static string MakeIdentifierObjects(MainForm MForm, string InString)
        {
            StringBuilder SBuilder = new StringBuilder();

            char PreviousChar   = '\n';
            bool IsInsideID     = false;
            bool IsInsideObject = false;
            int  Position       = 0;
            int  Last           = InString.Length;

            for (int Count = 0; Count < Last; Count++)
            {
                if (IsInsideID && IsInsideObject)
                {
                    MForm.ShowStatus("This should never happen.");
                    MForm.ShowStatus("IsInsideID && IsInsideObject");
                    SBuilder.Append(Char.ToString(Markers.ErrorPoint));
                    return(SBuilder.ToString());
                }

                char TestChar = InString[Count];

                if (Count > 0)
                {
                    PreviousChar = InString[Count - 1];
                }

                if (IsInsideID)
                {
                    if (Markers.IsMarker(TestChar))
                    {
                        IsInsideID = false;
                        SBuilder.Append(Char.ToString(
                                            Markers.End));
                        SBuilder.Append(Char.ToString(TestChar));
                        continue;
                    }
                }

                if (IsInsideObject)
                {
                    if (TestChar == Markers.End)
                    {
                        IsInsideObject = false;
                    }

                    SBuilder.Append(Char.ToString(TestChar));
                    continue;
                }



                if (TestChar == Markers.Begin)
                {
                    IsInsideObject = true;
                    SBuilder.Append(Char.ToString(TestChar));
                    continue;
                }

                if (!IsInsideID)
                {
                    Position = 0;
                    if (IsIdentifierCharacter(TestChar,
                                              PreviousChar,
                                              Position))
                    {
                        IsInsideID = true;
                        SBuilder.Append(Char.ToString(
                                            Markers.Begin));
                        SBuilder.Append(Char.ToString(
                                            Markers.TypeIdentifier));
                        SBuilder.Append(Char.ToString(TestChar));
                        continue;
                    }
                }
                else
                {
                    // It is inside.
                    Position++;
                    if (!IsIdentifierCharacter(TestChar,
                                               PreviousChar,
                                               Position))
                    {
                        IsInsideID = false;
                        SBuilder.Append(Char.ToString(
                                            Markers.End));
                        SBuilder.Append(Char.ToString(TestChar));
                        continue;
                    }
                }

                SBuilder.Append(Char.ToString(TestChar));
            }

            string Result = SBuilder.ToString();

            return(Result);
        }