private void FindToken(int currentCharIndex, out int lengthOfFoundToken, out byte tokenByte)
{
bool endOfSearch = false;
int lengthofSearch = 1;
lengthOfFoundToken = -1;
tokenByte = 0x00;
Dictionary <string, byte> filteredTokenDic = TvcBasicTokenLibrary.BasicTokens;
while (!endOfSearch && (currentCharIndex + lengthofSearch) <= RowText.Length)
{
string tokenStringForSearch = RowText.Substring(currentCharIndex, lengthofSearch++).ToUpper();
var tokens = filteredTokenDic
.Where(kvp => kvp.Key.StartsWith(tokenStringForSearch, StringComparison.InvariantCultureIgnoreCase))
.Select(kvp => kvp);
if (tokens.Count() == 1)
{
KeyValuePair <string, byte> pair = tokens.First();
if (pair.Key.Equals(tokenStringForSearch))
{
endOfSearch = true;
lengthOfFoundToken = pair.Key.Length;
tokenByte = pair.Value;
}
}
else if (tokens.Count() > 1)
{
KeyValuePair <string, byte> pair = tokens.FirstOrDefault(kvp => kvp.Key == tokenStringForSearch);
if (pair.Key != null)
{
lengthOfFoundToken = pair.Key.Length;
tokenByte = pair.Value;
}
filteredTokenDic = tokens.ToDictionary(kvp => kvp.Key, KeyValuePair => KeyValuePair.Value);
}
else
{
endOfSearch = true;
}
}
}
private void TokenizeRow()
{
if (string.IsNullOrEmpty(RowText))
{
throw new TvcBasicException(@" A basic sor üres sor!");
}
byte highByte = (byte)(RowNumber >> 8);
byte lowByte = (byte)(RowNumber & 0xFF);
List <byte> tokBytes = new List <byte> {
0x00, lowByte, highByte
};
TokenizingState charState = TokenizingState.CharMustTokenized;
for (int currentCharIndex = SkipRowNumberAndFirstSpaces(); currentCharIndex < RowText.Length;)
{
char currentChar = RowText[currentCharIndex];
// The numers and spaces must not be tokenised. They appear with their ASCII code in tokenised Tvc basic Row
if (charState == TokenizingState.CharMustTokenized && currentChar != ' ' && !char.IsNumber(currentChar))
{
KeyValuePair <string, byte>?foundToken = null;
//Detects if the current character can be part of a basic token
foreach (KeyValuePair <string, byte> pair in TvcBasicTokenLibrary.BasicTokens
.Where(kvp => RowText.Length - currentCharIndex >= kvp.Key.Length)
.Select(kvp => kvp))
{
string tokenStringForSearch = RowText.Substring(currentCharIndex, pair.Key.Length).ToUpper();
if (tokenStringForSearch == pair.Key)
{
foundToken = pair;
break;
}
}
if (foundToken.HasValue)
{
if (foundToken.Value.Value == 0xFC ||
foundToken.Value.Value == 0xFE ||
foundToken.Value.Value == 0xFB)
{
// If the found token is 'DATA', '!', or 'REM', the following characters must not be tokenised
charState |= TokenizingState.CharInDataOrCommentRow;
}
tokBytes.Add(foundToken.Value.Value);
currentCharIndex += foundToken.Value.Key.Length;
}
else
{
// if the current character is not part of a basic token
if (currentChar == '"')
{
charState ^= TokenizingState.CharInLiteral;
}
// In literal the character must not be converted into upper case
currentChar = charState.HasFlag(TokenizingState.CharInLiteral) ? currentChar : char.ToUpper(currentChar);
tokBytes.Add(currentChar.ToTvcBasicAscii());
currentCharIndex++;
}
}
else
{
if (currentChar == '"')
{
charState ^= TokenizingState.CharInLiteral;
//inLiteral = !inLiteral;
//charMustTokenize = true;
}
if (currentChar == ':' && !charState.HasFlag(TokenizingState.CharInLiteral))
{
tokBytes.Add(0xFD);
charState = TokenizingState.CharMustTokenized;
currentCharIndex++;
}
else
{
currentChar = charState.HasFlag(TokenizingState.CharInLiteral) ? currentChar : char.ToUpper(currentChar);
tokBytes.Add(currentChar.ToTvcBasicAscii());
currentCharIndex++;
}
}
}
tokBytes.Add(0xFF);
tokBytes[0] = (byte)tokBytes.Count;
TokenizedBytes = tokBytes.ToArray();
}