private void TokenizeRow()
{
if (string.IsNullOrEmpty(RowText))
{
throw new TvcBasicException(@" A basic sor üres sor!");
}
byte highByte = (byte)(RowNumber >> 8);
byte lowByte = (byte)(RowNumber & 0xFF);
List <byte> tokBytes = new List <byte> {
0x00, lowByte, highByte
};
TokenizingState charState = TokenizingState.CharMustTokenized;
for (int currentCharIndex = SkipRowNumberAndFirstSpaces(); currentCharIndex < RowText.Length;)
{
char currentChar = RowText[currentCharIndex];
// The numers and spaces must not be tokenised. They appear with their ASCII code in tokenised Tvc basic Row
if (charState == TokenizingState.CharMustTokenized && currentChar != ' ' && !char.IsNumber(currentChar))
{
KeyValuePair <string, byte>?foundToken = null;
//Detects if the current character can be part of a basic token
foreach (KeyValuePair <string, byte> pair in TvcBasicTokenLibrary.BasicTokens
.Where(kvp => RowText.Length - currentCharIndex >= kvp.Key.Length)
.Select(kvp => kvp))
{
string tokenStringForSearch = RowText.Substring(currentCharIndex, pair.Key.Length).ToUpper();
if (tokenStringForSearch == pair.Key)
{
foundToken = pair;
break;
}
}
if (foundToken.HasValue)
{
if (foundToken.Value.Value == 0xFC ||
foundToken.Value.Value == 0xFE ||
foundToken.Value.Value == 0xFB)
{
// If the found token is 'DATA', '!', or 'REM', the following characters must not be tokenised
charState |= TokenizingState.CharInDataOrCommentRow;
}
tokBytes.Add(foundToken.Value.Value);
currentCharIndex += foundToken.Value.Key.Length;
}
else
{
// if the current character is not part of a basic token
if (currentChar == '"')
{
charState ^= TokenizingState.CharInLiteral;
}
// In literal the character must not be converted into upper case
currentChar = charState.HasFlag(TokenizingState.CharInLiteral) ? currentChar : char.ToUpper(currentChar);
tokBytes.Add(currentChar.ToTvcBasicAscii());
currentCharIndex++;
}
}
else
{
if (currentChar == '"')
{
charState ^= TokenizingState.CharInLiteral;
//inLiteral = !inLiteral;
//charMustTokenize = true;
}
if (currentChar == ':' && !charState.HasFlag(TokenizingState.CharInLiteral))
{
tokBytes.Add(0xFD);
charState = TokenizingState.CharMustTokenized;
currentCharIndex++;
}
else
{
currentChar = charState.HasFlag(TokenizingState.CharInLiteral) ? currentChar : char.ToUpper(currentChar);
tokBytes.Add(currentChar.ToTvcBasicAscii());
currentCharIndex++;
}
}
}
tokBytes.Add(0xFF);
tokBytes[0] = (byte)tokBytes.Count;
TokenizedBytes = tokBytes.ToArray();
}
private void TokenizeRow()
{
if (string.IsNullOrEmpty(RowText))
{
throw new TvcBasicException(@" A basic sor üres sor!");
}
byte highByte = (byte)(RowNumber >> 8);
byte lowByte = (byte)(RowNumber & 0xFF);
List <byte> tokBytes = new List <byte> {
0x00, lowByte, highByte
};
TokenizingState charState = TokenizingState.CharMustTokenized;
for (int currentCharIndex = SkipRowNumberAndFirstSpaces(); currentCharIndex < RowText.Length;)
{
char currentChar = RowText[currentCharIndex];
// The numbers and spaces must not be tokenised. They appear with their ASCII code in tokenised Tvc basic Row
if (charState == TokenizingState.CharMustTokenized && currentChar != ' ' && !char.IsNumber(currentChar))
{
FindToken(currentCharIndex, out int lengthOfFoundToken, out byte tokenByte);
if (lengthOfFoundToken > 0)
{
if (tokenByte == 0xFC ||
tokenByte == 0xFE ||
tokenByte == 0xFB)
{
// If the found token is 'DATA', '!', or 'REM', the following characters must not be tokenised
charState |= TokenizingState.CharInDataOrCommentRow;
}
tokBytes.Add(tokenByte);
currentCharIndex += lengthOfFoundToken;
}
else
{
// if the current character is not part of a basic token
if (currentChar == '"')
{
charState ^= TokenizingState.CharInLiteral;
}
// In literal the character must not be converted into upper case
currentChar = charState.HasFlag(TokenizingState.CharInLiteral) ? currentChar : char.ToUpper(currentChar);
tokBytes.Add(currentChar.ToTvcBasicAscii());
currentCharIndex++;
}
}
else
{
if (currentChar == '"')
{
charState ^= TokenizingState.CharInLiteral;
}
if (currentChar == ':' && !charState.HasFlag(TokenizingState.CharInLiteral))
{
tokBytes.Add(0xFD);
charState = TokenizingState.CharMustTokenized;
currentCharIndex++;
}
else
{
currentChar = charState.HasFlag(TokenizingState.CharInLiteral) ? currentChar : char.ToUpper(currentChar);
tokBytes.Add(currentChar.ToTvcBasicAscii());
currentCharIndex++;
}
}
}
tokBytes.Add(0xFF);
tokBytes[0] = (byte)tokBytes.Count;
TokenizedBytes = tokBytes.ToArray();
}
