private void FindToken(int currentCharIndex, out int lengthOfFoundToken, out byte tokenByte) { bool endOfSearch = false; int lengthofSearch = 1; lengthOfFoundToken = -1; tokenByte = 0x00; Dictionary <string, byte> filteredTokenDic = TvcBasicTokenLibrary.BasicTokens; while (!endOfSearch && (currentCharIndex + lengthofSearch) <= RowText.Length) { string tokenStringForSearch = RowText.Substring(currentCharIndex, lengthofSearch++).ToUpper(); var tokens = filteredTokenDic .Where(kvp => kvp.Key.StartsWith(tokenStringForSearch, StringComparison.InvariantCultureIgnoreCase)) .Select(kvp => kvp); if (tokens.Count() == 1) { KeyValuePair <string, byte> pair = tokens.First(); if (pair.Key.Equals(tokenStringForSearch)) { endOfSearch = true; lengthOfFoundToken = pair.Key.Length; tokenByte = pair.Value; } } else if (tokens.Count() > 1) { KeyValuePair <string, byte> pair = tokens.FirstOrDefault(kvp => kvp.Key == tokenStringForSearch); if (pair.Key != null) { lengthOfFoundToken = pair.Key.Length; tokenByte = pair.Value; } filteredTokenDic = tokens.ToDictionary(kvp => kvp.Key, KeyValuePair => KeyValuePair.Value); } else { endOfSearch = true; } } }
private void TokenizeRow() { if (string.IsNullOrEmpty(RowText)) { throw new TvcBasicException(@" A basic sor üres sor!"); } byte highByte = (byte)(RowNumber >> 8); byte lowByte = (byte)(RowNumber & 0xFF); List <byte> tokBytes = new List <byte> { 0x00, lowByte, highByte }; TokenizingState charState = TokenizingState.CharMustTokenized; for (int currentCharIndex = SkipRowNumberAndFirstSpaces(); currentCharIndex < RowText.Length;) { char currentChar = RowText[currentCharIndex]; // The numers and spaces must not be tokenised. They appear with their ASCII code in tokenised Tvc basic Row if (charState == TokenizingState.CharMustTokenized && currentChar != ' ' && !char.IsNumber(currentChar)) { KeyValuePair <string, byte>?foundToken = null; //Detects if the current character can be part of a basic token foreach (KeyValuePair <string, byte> pair in TvcBasicTokenLibrary.BasicTokens .Where(kvp => RowText.Length - currentCharIndex >= kvp.Key.Length) .Select(kvp => kvp)) { string tokenStringForSearch = RowText.Substring(currentCharIndex, pair.Key.Length).ToUpper(); if (tokenStringForSearch == pair.Key) { foundToken = pair; break; } } if (foundToken.HasValue) { if (foundToken.Value.Value == 0xFC || foundToken.Value.Value == 0xFE || foundToken.Value.Value == 0xFB) { // If the found token is 'DATA', '!', or 'REM', the following characters must not be tokenised charState |= TokenizingState.CharInDataOrCommentRow; } tokBytes.Add(foundToken.Value.Value); currentCharIndex += foundToken.Value.Key.Length; } else { // if the current character is not part of a basic token if (currentChar == '"') { charState ^= TokenizingState.CharInLiteral; } // In literal the character must not be converted into upper case currentChar = charState.HasFlag(TokenizingState.CharInLiteral) ? currentChar : char.ToUpper(currentChar); tokBytes.Add(currentChar.ToTvcBasicAscii()); currentCharIndex++; } } else { if (currentChar == '"') { charState ^= TokenizingState.CharInLiteral; //inLiteral = !inLiteral; //charMustTokenize = true; } if (currentChar == ':' && !charState.HasFlag(TokenizingState.CharInLiteral)) { tokBytes.Add(0xFD); charState = TokenizingState.CharMustTokenized; currentCharIndex++; } else { currentChar = charState.HasFlag(TokenizingState.CharInLiteral) ? currentChar : char.ToUpper(currentChar); tokBytes.Add(currentChar.ToTvcBasicAscii()); currentCharIndex++; } } } tokBytes.Add(0xFF); tokBytes[0] = (byte)tokBytes.Count; TokenizedBytes = tokBytes.ToArray(); }