private Token MakeTokenFromString(string InString)
{
try
{
Token Tk = new Token(MForm);
string LevelStr = "";
int Position = 0;
int Last = InString.Length;
for (int Count = 0; Count < Last; Count++)
{
char TestChar = InString[Count];
if (Markers.IsMarker(TestChar))
{
Tk.TokenType = TestChar;
Position = Count + 1;
break;
}
LevelStr += Char.ToString(TestChar);
}
// ShowStatus( "LevelStr: " + LevelStr );
Tk.Level = Int32.Parse(LevelStr);
// ShowStatus( "Level: " + Tk.Level.ToString());
for (int Count = Position; Count < Last; Count++)
{
char TestChar = InString[Count];
if (Markers.IsMarker(TestChar))
{
break;
}
Tk.Text += Char.ToString(TestChar);
}
// ShowStatus( "Text: " + Tk.Text );
return(Tk);
}
catch (Exception Except)
{
ShowStatus("Exception in Token.MakeTokenFromString():");
ShowStatus(Except.Message);
return(null);
}
}
private static string GetCleanUnicodeString(string InString)
{
if (InString == null)
{
return("");
}
StringBuilder SBuilder = new StringBuilder();
int Last = InString.Length;
for (int Count = 0; Count < Last; Count++)
{
char ToCheck = InString[Count];
// Get rid of control characters.
// This also replaces a tab with a space.
if (ToCheck < ' ')
{
ToCheck = ' ';
}
// Don't go higher than D800 (Surrogates).
if (ToCheck >= 0xD800)
{
ToCheck = ' ';
}
// This has already been converted from UTF8
// to 16 bit characters.
if ((ToCheck >= 127) && (ToCheck <= 255))
{
ToCheck = ' ';
}
// Don't exclude any characters in the Basic
// Multilingual Plane except markers that
// shouldn't be in this. See the Markers.cs
// file.
if (Markers.IsMarker(ToCheck))
{
ToCheck = ' ';
}
// Basic Multilingual Plane
// C0 Controls and Basic Latin (Basic Latin) (0000–007F)
// C1 Controls and Latin-1 Supplement (0080–00FF)
// Latin Extended-A (0100–017F)
// Latin Extended-B (0180–024F)
// IPA Extensions (0250–02AF)
// Spacing Modifier Letters (02B0–02FF)
// Combining Diacritical Marks (0300–036F)
// General Punctuation (2000–206F)
// Superscripts and Subscripts (2070–209F)
// Currency Symbols (20A0–20CF)
// Combining Diacritical Marks for Symbols (20D0–20FF)
// Letterlike Symbols (2100–214F)
// Number Forms (2150–218F)
// Arrows (2190–21FF)
// Mathematical Operators (2200–22FF)
// Box Drawing (2500–257F)
// Geometric Shapes (25A0–25FF)
// Miscellaneous Symbols (2600–26FF)
// Dingbats (2700–27BF)
// Miscellaneous Symbols and Arrows (2B00–2BFF)
SBuilder.Append(Char.ToString(ToCheck));
}
return(SBuilder.ToString());
}
internal static string MakeIdentifierObjects(MainForm MForm, string InString)
{
StringBuilder SBuilder = new StringBuilder();
char PreviousChar = '\n';
bool IsInsideID = false;
bool IsInsideObject = false;
int Position = 0;
int Last = InString.Length;
for (int Count = 0; Count < Last; Count++)
{
if (IsInsideID && IsInsideObject)
{
MForm.ShowStatus("This should never happen.");
MForm.ShowStatus("IsInsideID && IsInsideObject");
SBuilder.Append(Char.ToString(Markers.ErrorPoint));
return(SBuilder.ToString());
}
char TestChar = InString[Count];
if (Count > 0)
{
PreviousChar = InString[Count - 1];
}
if (IsInsideID)
{
if (Markers.IsMarker(TestChar))
{
IsInsideID = false;
SBuilder.Append(Char.ToString(
Markers.End));
SBuilder.Append(Char.ToString(TestChar));
continue;
}
}
if (IsInsideObject)
{
if (TestChar == Markers.End)
{
IsInsideObject = false;
}
SBuilder.Append(Char.ToString(TestChar));
continue;
}
if (TestChar == Markers.Begin)
{
IsInsideObject = true;
SBuilder.Append(Char.ToString(TestChar));
continue;
}
if (!IsInsideID)
{
Position = 0;
if (IsIdentifierCharacter(TestChar,
PreviousChar,
Position))
{
IsInsideID = true;
SBuilder.Append(Char.ToString(
Markers.Begin));
SBuilder.Append(Char.ToString(
Markers.TypeIdentifier));
SBuilder.Append(Char.ToString(TestChar));
continue;
}
}
else
{
// It is inside.
Position++;
if (!IsIdentifierCharacter(TestChar,
PreviousChar,
Position))
{
IsInsideID = false;
SBuilder.Append(Char.ToString(
Markers.End));
SBuilder.Append(Char.ToString(TestChar));
continue;
}
}
SBuilder.Append(Char.ToString(TestChar));
}
string Result = SBuilder.ToString();
return(Result);
}
