SimisToken ReadTokenAsBinary() { SimisToken rv = new SimisToken(); var validStates = (BnfState == null ? new string[] { } : BnfState.ValidStates); //if (validStates.Length == 0) throw new ReaderException(BinaryReader, true, PinReaderChanged(), "SimisReader found no non-meta states available.", new BNFStateException(BNFState, "")); // If we have any valid data types, we read that instead of a block start. They should all be the same data type, too. var dataType = validStates.FirstOrDefault(s => DataTypes.Contains(s)); if (dataType != null) { if (!validStates.All(s => s == dataType || !DataTypes.Contains(s))) { throw new ReaderException(Reader, true, PinReaderChanged(), "SimisReader found inconsistent data types available.", new BnfStateException(BnfState, "")); } rv.Type = dataType; switch (rv.Type) { case "uint": rv.IntegerUnsigned = Reader.ReadUInt32(); rv.Kind = SimisTokenKind.IntegerUnsigned; break; case "sint": rv.IntegerSigned = Reader.ReadInt32(); rv.Kind = SimisTokenKind.IntegerSigned; break; case "dword": rv.IntegerDWord = Reader.ReadUInt32(); rv.Kind = SimisTokenKind.IntegerDWord; break; case "word": rv.IntegerDWord = Reader.ReadUInt16(); rv.Kind = SimisTokenKind.IntegerWord; break; case "byte": rv.IntegerDWord = Reader.ReadByte(); rv.Kind = SimisTokenKind.IntegerByte; break; case "float": rv.Float = Reader.ReadSingle(); rv.Kind = SimisTokenKind.Float; break; case "string": var stringLength = Reader.ReadUInt16(); if (stringLength > 10000) { throw new ReaderException(Reader, true, PinReaderChanged(), "SimisReader found a string longer than 10,000 characters.", new BnfStateException(BnfState, "")); } if (Reader.BaseStream.Position + stringLength * 2 > Reader.BaseStream.Length) { throw new ReaderException(Reader, true, PinReaderChanged(), "SimisReader found a string extending beyond the end of the file.", new BnfStateException(BnfState, "")); } for (var i = 0; i < stringLength; i++) { rv.String += (char)Reader.ReadUInt16(); } rv.Kind = SimisTokenKind.String; break; case "buffer": var bufferLength = BlockEndOffsets.Peek() - Reader.BaseStream.Position; rv.String = String.Join("", Reader.ReadBytes((int)bufferLength).Select(b => b.ToString("X2", CultureInfo.InvariantCulture)).ToArray()); rv.Kind = SimisTokenKind.String; break; default: throw new ReaderException(Reader, true, PinReaderChanged(), "SimisReader found unexpected data type '" + rv.Type + "'.", new BnfStateException(BnfState, "")); } } else { var tokenID = Reader.ReadUInt16(); var tokenType = Reader.ReadUInt16(); var token = ((uint)tokenType << 16) + tokenID; if (!SimisProvider.TokenNames.ContainsKey(token)) { throw new ReaderException(Reader, true, PinReaderChanged(), String.Format(CultureInfo.CurrentCulture, "SimisReader got invalid block: id={0:X4}, type={1:X4}.", tokenID, tokenType), new BnfStateException(BnfState, "")); } if ((tokenType != 0x0000) && (tokenType != 0x0004)) { throw new ReaderException(Reader, true, PinReaderChanged(), String.Format(CultureInfo.CurrentCulture, "SimisReader got invalid block: id={0:X4}, type={1:X4}, name={2}.", tokenID, tokenType, SimisProvider.TokenNames[token]), new BnfStateException(BnfState, "")); } rv.Type = SimisProvider.TokenNames[token]; rv.Kind = SimisTokenKind.Block; if (BnfState == null) { try { BnfState = new BnfState(JinxStreamFormat.Bnf); } catch (UnknownSimisFormatException e) { throw new ReaderException(Reader, true, PinReaderChanged(), "", e); } } var contentsLength = Reader.ReadUInt32(); if (contentsLength == 0) { throw new ReaderException(Reader, true, PinReaderChanged(), "SimisReader got block with length of 0.", new BnfStateException(BnfState, "")); } if (Reader.BaseStream.Position + contentsLength > StreamLength) { throw new ReaderException(Reader, true, PinReaderChanged(), "SimisReader got block longer than stream.", new BnfStateException(BnfState, "")); } BlockEndOffsets.Push((uint)Reader.BaseStream.Position + contentsLength); PendingTokens.Enqueue(new SimisToken() { Kind = SimisTokenKind.BlockBegin, String = BlockEndOffsets.Peek().ToString("X8", CultureInfo.CurrentCulture) }); var nameLength = Reader.Read(); if (nameLength > 0) { if (Reader.BaseStream.Position + nameLength * 2 > StreamLength) { throw new ReaderException(Reader, true, PinReaderChanged(), "SimisReader got block with name longer than stream.", new BnfStateException(BnfState, "")); } for (var i = 0; i < nameLength; i++) { rv.Name += (char)Reader.ReadUInt16(); } } } return(rv); }
public SimisToken ReadToken() { // Any pending tokens go first. if (PendingTokens.Count > 0) { try { if (PendingTokens.Peek().Kind == SimisTokenKind.BlockBegin) { BnfState.EnterBlock(); } if (PendingTokens.Peek().Kind == SimisTokenKind.BlockEnd) { BnfState.LeaveBlock(); } } catch (BnfStateException e) { throw new ReaderException(Reader, JinxStreamIsBinary, 0, "", e); } // If we've run out of stream and have no pending tokens, we're done. if ((Reader.BaseStream.Position >= Reader.BaseStream.Length) && (PendingTokens.Count == 1)) { try { BnfState.LeaveBlock(); } catch (BnfStateException e) { throw new ReaderException(Reader, JinxStreamIsBinary, 0, "", e); } if (!BnfState.IsCompleted) { throw new ReaderException(Reader, JinxStreamIsBinary, 0, "Unexpected end of stream."); } EndOfStream = true; } return(PendingTokens.Dequeue()); } var rv = new SimisToken(); if (JinxStreamIsBinary) { PinReader(); rv = ReadTokenAsBinary(); try { if ((rv.Kind != SimisTokenKind.BlockBegin) && (rv.Kind != SimisTokenKind.BlockEnd)) { BnfState.MoveTo(rv.Type); if (BnfState.State.Operator is NamedReferenceOperator) { rv.Name = ((NamedReferenceOperator)BnfState.State.Operator).Name; } } else { throw new ReaderException(Reader, JinxStreamIsBinary, PinReaderChanged(), "ReadTokenAsBinary returned invalid token type: " + rv.Type); } } catch (BnfStateException e) { throw new ReaderException(Reader, JinxStreamIsBinary, PinReaderChanged(), "", e); } } else { PinReader(); rv = ReadTokenAsText(); try { if (rv.Kind == SimisTokenKind.BlockBegin) { BnfState.EnterBlock(); } else if (rv.Kind == SimisTokenKind.BlockEnd) { BnfState.LeaveBlock(); } else if (rv.Kind != SimisTokenKind.None) { if (rv.Type.Length > 0) { BnfState.MoveTo(rv.Type); if (BnfState.State.Operator is NamedReferenceOperator) { rv.Name = ((NamedReferenceOperator)BnfState.State.Operator).Name; } } } else { throw new ReaderException(Reader, JinxStreamIsBinary, PinReaderChanged(), "ReadTokenAsText returned invalid token type: " + rv.Kind); } } catch (BnfStateException e) { throw new ReaderException(Reader, JinxStreamIsBinary, PinReaderChanged(), "", e); } // Consume all whitespace now that we've got a token. while ((Reader.BaseStream.Position < Reader.BaseStream.Length) && WhitespaceChars.Contains((char)Reader.PeekChar())) { Reader.ReadChar(); } } // Any blocks that should have ended at or before this point, are now ended. while ((BlockEndOffsets.Count > 0) && (Reader.BaseStream.Position >= BlockEndOffsets.Peek())) { if (Reader.BaseStream.Position > BlockEndOffsets.Peek()) { throw new ReaderException(Reader, JinxStreamIsBinary, (int)(Reader.BaseStream.Position - BlockEndOffsets.Peek()), "SimisReader stream positioned at 0x" + Reader.BaseStream.Position.ToString("X8", CultureInfo.CurrentCulture) + " but a block ended at 0x" + BlockEndOffsets.Peek().ToString("X8", CultureInfo.CurrentCulture) + "; overshot by " + (Reader.BaseStream.Position - BlockEndOffsets.Peek()) + " bytes."); } PendingTokens.Enqueue(new SimisToken() { Kind = SimisTokenKind.BlockEnd }); BlockEndOffsets.Pop(); } // If we've run out of stream and have no pending tokens, we're done. if ((Reader.BaseStream.Position >= Reader.BaseStream.Length) && (PendingTokens.Count == 0)) { try { BnfState.LeaveBlock(); } catch (BnfStateException e) { throw new ReaderException(Reader, JinxStreamIsBinary, 0, "", e); } if (!BnfState.IsCompleted) { throw new ReaderException(Reader, JinxStreamIsBinary, 0, "Unexpected end of stream."); } EndOfStream = true; } return(rv); }
SimisToken ReadTokenAsText() { SimisToken rv = new SimisToken(); if ('(' == Reader.PeekChar()) { Reader.ReadChar(); rv.Kind = SimisTokenKind.BlockBegin; return(rv); } if (')' == Reader.PeekChar()) { Reader.ReadChar(); rv.Kind = SimisTokenKind.BlockEnd; return(rv); } if (':' == Reader.PeekChar()) { Reader.ReadChar(); return(ReadTokenAsText()); } string token = ReadTokenOrString(); if (BnfState == null) { try { BnfState = new BnfState(JinxStreamFormat.Bnf); } catch (UnknownSimisFormatException e) { throw new ReaderException(Reader, false, PinReaderChanged(), "", e); } } if (BnfState.IsEnterBlockTime) { // We should only end up here when called recursively by the // if (validStates.Contains(token)) code below. rv.Name = token; rv.Kind = token.Length > 0 ? SimisTokenKind.Block : SimisTokenKind.None; return(rv); } if (token.StartsWith("_", StringComparison.InvariantCulture) || (token.ToUpperInvariant() == "COMMENT") || (token.ToUpperInvariant() == "INFO") || (token.ToUpperInvariant() == "SKIP")) { var oldPosition = Reader.BaseStream.Position; while ((Reader.BaseStream.Position < Reader.BaseStream.Length) && WhitespaceChars.Contains((char)Reader.PeekChar())) { Reader.ReadChar(); } var nextToken = Reader.PeekChar(); Reader.BaseStream.Position = oldPosition; if (nextToken == (int)'(') { var blockCount = 0; while ((Reader.BaseStream.Position < Reader.BaseStream.Length) && ((')' != Reader.PeekChar()) || (blockCount > 1))) { if (Reader.PeekChar() == '(') { blockCount++; } if (Reader.PeekChar() == ')') { blockCount--; } token += Reader.ReadChar(); } if (Reader.BaseStream.Position >= Reader.BaseStream.Length) { throw new ReaderException(Reader, false, 0, "SimisReader expected ')'; got EOF."); } token += Reader.ReadChar(); rv.String = token; rv.Name = "Comment"; rv.Kind = SimisTokenKind.String; return(rv); } } if (token.StartsWith("//", StringComparison.InvariantCulture)) { var blockCount = 0; while ((Reader.BaseStream.Position < Reader.BaseStream.Length) && ('\n' != Reader.PeekChar()) && ((')' != Reader.PeekChar()) || (blockCount > 0))) { if (Reader.PeekChar() == '(') { blockCount++; } if (Reader.PeekChar() == ')') { blockCount--; } token += Reader.ReadChar(); } rv.String = token; rv.Name = "Comment"; rv.Kind = SimisTokenKind.String; return(rv); } var validStates = BnfState.ValidStates; if (validStates.Contains(token, StringComparer.InvariantCultureIgnoreCase)) { // Token exactly matches a valid state transition, so let's use it. rv.Type = validStates.First(s => s.Equals(token, StringComparison.InvariantCultureIgnoreCase)); rv.Kind = SimisTokenKind.Block; // Do lookahead for block name. PinReader(); while ((Reader.BaseStream.Position < Reader.BaseStream.Length) && WhitespaceChars.Contains((char)Reader.PeekChar())) { Reader.ReadChar(); } string name = ReadTokenOrString(); if (name.Length > 0) { rv.Name = name; } else { PinReaderReset(); } return(rv); } var validDataTypeStates = validStates.Where(s => { switch (s) { case "uint": return(token.ToCharArray().All(c => DecDigits.Contains(c))); case "sint": if (token.StartsWith("-", StringComparison.Ordinal)) { return(token.Substring(1).ToCharArray().All(c => DecDigits.Contains(c))); } return(token.ToCharArray().All(c => DecDigits.Contains(c))); case "dword": return((token.Length == 8) && (token.ToCharArray().All(c => HexDigits.Contains(c)))); case "float": if (token.StartsWith("-", StringComparison.Ordinal)) { return(token.Substring(1).ToCharArray().All(c => DecFloatDigits.Contains(c))); } return(token.ToCharArray().All(c => DecFloatDigits.Contains(c))); case "string": return(true); case "buffer": default: return(false); } }).ToArray(); if (validDataTypeStates.Length == 0) { try { // This is *expected* to throw! We're doing this so that we get a proper BNF exception from the failed state. BnfState.MoveTo(token); } catch (BnfStateException ex) { throw new ReaderException(Reader, false, PinReaderChanged(), "", ex); } } rv.Type = validDataTypeStates[0]; switch (rv.Type) { case "uint": if (token.EndsWith(",", StringComparison.Ordinal)) { token = token.Substring(0, token.Length - 1); } try { rv.IntegerUnsigned = UInt32.Parse(token, CultureInfo.InvariantCulture); if (token.Length == 8) { throw new ReaderException(Reader, false, PinReaderChanged(), "SimisReader expected decimal number; got possible hex '" + token + "'."); } } catch (FormatException ex) { throw new ReaderException(Reader, false, PinReaderChanged(), "SimisReader failed to parse '" + token + "' as '" + rv.Type + "'.", ex); } catch (OverflowException ex) { throw new ReaderException(Reader, false, PinReaderChanged(), "SimisReader failed to parse '" + token + "' as '" + rv.Type + "'.", ex); } rv.Kind = SimisTokenKind.IntegerUnsigned; break; case "sint": if (token.EndsWith(",", StringComparison.Ordinal)) { token = token.Substring(0, token.Length - 1); } try { // Special-case -1 witten as an unsigned integer instead of signed. if (token == "4294967295") { rv.IntegerSigned = -1; } else { rv.IntegerSigned = Int32.Parse(token, CultureInfo.InvariantCulture); } } catch (FormatException ex) { throw new ReaderException(Reader, false, PinReaderChanged(), "SimisReader failed to parse '" + token + "' as '" + rv.Type + "'.", ex); } catch (OverflowException ex) { throw new ReaderException(Reader, false, PinReaderChanged(), "SimisReader failed to parse '" + token + "' as '" + rv.Type + "'.", ex); } rv.Kind = SimisTokenKind.IntegerSigned; break; case "dword": if (token.EndsWith(",", StringComparison.Ordinal)) { token = token.Substring(0, token.Length - 1); } if (token.Length != 8) { throw new ReaderException(Reader, false, PinReaderChanged(), "SimisReader expected 8-digit hex number; got '" + token + "'."); } try { rv.IntegerDWord = UInt32.Parse(token, NumberStyles.HexNumber, CultureInfo.InvariantCulture); } catch (FormatException ex) { throw new ReaderException(Reader, false, PinReaderChanged(), "SimisReader failed to parse '" + token + "' as '" + rv.Type + "'.", ex); } catch (OverflowException ex) { throw new ReaderException(Reader, false, PinReaderChanged(), "SimisReader failed to parse '" + token + "' as '" + rv.Type + "'.", ex); } rv.Kind = SimisTokenKind.IntegerDWord; break; case "word": if (token.EndsWith(",", StringComparison.Ordinal)) { token = token.Substring(0, token.Length - 1); } if (token.Length != 4) { throw new ReaderException(Reader, false, PinReaderChanged(), "SimisReader expected 4-digit hex number; got '" + token + "'."); } try { rv.IntegerDWord = UInt16.Parse(token, NumberStyles.HexNumber, CultureInfo.InvariantCulture); } catch (FormatException ex) { throw new ReaderException(Reader, false, PinReaderChanged(), "SimisReader failed to parse '" + token + "' as '" + rv.Type + "'.", ex); } catch (OverflowException ex) { throw new ReaderException(Reader, false, PinReaderChanged(), "SimisReader failed to parse '" + token + "' as '" + rv.Type + "'.", ex); } rv.Kind = SimisTokenKind.IntegerDWord; break; case "byte": if (token.EndsWith(",", StringComparison.Ordinal)) { token = token.Substring(0, token.Length - 1); } if (token.Length != 2) { throw new ReaderException(Reader, false, PinReaderChanged(), "SimisReader expected 2-digit hex number; got '" + token + "'."); } try { rv.IntegerDWord = Byte.Parse(token, NumberStyles.HexNumber, CultureInfo.InvariantCulture); } catch (FormatException ex) { throw new ReaderException(Reader, false, PinReaderChanged(), "SimisReader failed to parse '" + token + "' as '" + rv.Type + "'.", ex); } catch (OverflowException ex) { throw new ReaderException(Reader, false, PinReaderChanged(), "SimisReader failed to parse '" + token + "' as '" + rv.Type + "'.", ex); } rv.Kind = SimisTokenKind.IntegerDWord; break; case "float": if (token.EndsWith(",", StringComparison.Ordinal)) { token = token.Substring(0, token.Length - 1); } try { rv.Float = float.Parse(token, CultureInfo.InvariantCulture); } catch (FormatException ex) { throw new ReaderException(Reader, false, PinReaderChanged(), "SimisReader failed to parse '" + token + "' as '" + rv.Type + "'.", ex); } catch (OverflowException ex) { throw new ReaderException(Reader, false, PinReaderChanged(), "SimisReader failed to parse '" + token + "' as '" + rv.Type + "'.", ex); } rv.Kind = SimisTokenKind.Float; break; case "string": rv.String = token; rv.Kind = SimisTokenKind.String; break; default: throw new ReaderException(Reader, false, PinReaderChanged(), "SimisReader found unexpected data type '" + rv.Type + "'.", new BnfStateException(BnfState, "")); } return(rv); }
public void WriteToken(SimisToken token) { if (JinxStreamIsBinary) { switch (token.Kind) { case SimisTokenKind.Block: var id = SimisProvider.TokenIds[token.Type]; Writer.Write((uint)id); Writer.Write((uint)0x7F8F7F8F); // Length, set to sentinel value for now. BlockStarts.Push(Writer.BaseStream.Position); Writer.Write((byte)token.Name.Length); foreach (var ch in token.Name.ToCharArray()) { Writer.Write((ushort)ch); } break; case SimisTokenKind.BlockBegin: break; case SimisTokenKind.BlockEnd: var start = BlockStarts.Pop(); var length = (uint)(Writer.BaseStream.Position - start); Writer.BaseStream.Seek(start - 4, SeekOrigin.Begin); Writer.Write(length); Writer.BaseStream.Seek(0, SeekOrigin.End); break; case SimisTokenKind.IntegerUnsigned: Writer.Write(token.IntegerUnsigned); break; case SimisTokenKind.IntegerSigned: Writer.Write(token.IntegerSigned); break; case SimisTokenKind.IntegerDWord: Writer.Write(token.IntegerDWord); break; case SimisTokenKind.IntegerWord: Writer.Write((ushort)token.IntegerDWord); break; case SimisTokenKind.IntegerByte: Writer.Write((byte)token.IntegerDWord); break; case SimisTokenKind.Float: Writer.Write((float)token.Float); break; case SimisTokenKind.String: Writer.Write((ushort)token.String.Length); foreach (var ch in token.String.ToCharArray()) { Writer.Write((ushort)ch); } break; default: throw new InvalidDataException("SimisToken.Kind is invalid: " + token.Kind); } } else { switch (token.Kind) { case SimisTokenKind.Block: if (BnfState == null) { BnfState = new BnfState(JinxStreamFormat.Bnf); } if (token.Type.Length > 0) { BnfState.MoveTo(token.Type); } if (!TextBlocked) { Writer.Write("\r\n".ToCharArray()); } for (var i = 0; i < TextIndent; i++) { Writer.Write('\t'); } Writer.Write(token.Type.ToCharArray()); if (token.Name.Length > 0) { Writer.Write((" " + token.Name).ToCharArray()); } TextBlocked = true; break; case SimisTokenKind.BlockBegin: BnfState.EnterBlock(); Writer.Write(" (".ToCharArray()); TextIndent++; TextBlocked = false; TextBlockEmpty = true; break; case SimisTokenKind.BlockEnd: BnfState.LeaveBlock(); if (BnfState.IsCompleted) { BnfState = null; } TextIndent--; if (TextBlocked) { for (var i = 0; i < TextIndent; i++) { Writer.Write('\t'); } } else if (!TextBlockEmpty) { Writer.Write(' '); } Writer.Write(")\r\n".ToCharArray()); TextBlocked = true; break; case SimisTokenKind.IntegerUnsigned: BnfState.MoveTo(token.Type); if (TextBlocked) { for (var i = 0; i < TextIndent; i++) { Writer.Write('\t'); } } else { Writer.Write(' '); } Writer.Write(token.IntegerUnsigned.ToString(CultureInfo.InvariantCulture).ToCharArray()); if (TextBlocked) { Writer.Write("\r\n".ToCharArray()); } TextBlockEmpty = false; break; case SimisTokenKind.IntegerSigned: BnfState.MoveTo(token.Type); if (TextBlocked) { for (var i = 0; i < TextIndent; i++) { Writer.Write('\t'); } } else { Writer.Write(' '); } Writer.Write(token.IntegerSigned.ToString(CultureInfo.InvariantCulture).ToCharArray()); if (TextBlocked) { Writer.Write("\r\n".ToCharArray()); } TextBlockEmpty = false; break; case SimisTokenKind.IntegerDWord: case SimisTokenKind.IntegerWord: case SimisTokenKind.IntegerByte: BnfState.MoveTo(token.Type); if (TextBlocked) { for (var i = 0; i < TextIndent; i++) { Writer.Write('\t'); } } else { Writer.Write(' '); } Writer.Write(token.IntegerDWord.ToString(token.Kind == SimisTokenKind.IntegerDWord ? "X8" : token.Kind == SimisTokenKind.IntegerWord ? "X4" : "X2", CultureInfo.InvariantCulture).ToCharArray()); if (TextBlocked) { Writer.Write("\r\n".ToCharArray()); } TextBlockEmpty = false; break; case SimisTokenKind.Float: BnfState.MoveTo(token.Type); if (TextBlocked) { for (var i = 0; i < TextIndent; i++) { Writer.Write('\t'); } } else { Writer.Write(' '); } if (token.Float.ToString("G6", CultureInfo.InvariantCulture).IndexOf("E", StringComparison.OrdinalIgnoreCase) >= 0) { Writer.Write(token.Float.ToString("0.#####e000", CultureInfo.InvariantCulture).ToCharArray()); } else { Writer.Write(token.Float.ToString("G6", CultureInfo.InvariantCulture).ToCharArray()); } if (TextBlocked) { Writer.Write("\r\n".ToCharArray()); } TextBlockEmpty = false; break; case SimisTokenKind.String: // If the token has no type, it was a specially-skipped input (comment, SKIP(...) block etc.). if (token.Type.Length == 0) { if (!TextBlocked) { Writer.Write("\r\n".ToCharArray()); } for (var i = 0; i < TextIndent; i++) { Writer.Write('\t'); } Writer.Write(token.String.ToCharArray()); Writer.Write("\r\n".ToCharArray()); TextBlocked = true; TextBlockEmpty = true; } else { BnfState.MoveTo(token.Type); if (TextBlocked) { for (var i = 0; i < TextIndent; i++) { Writer.Write('\t'); } } else { Writer.Write(' '); } if ((token.String.Length > 0) && token.String.ToCharArray().All(c => SafeTokenCharacters.Contains(c))) { Writer.Write(token.String.ToCharArray()); } else { var wrap = "\"+\r\n"; for (var i = 0; i < TextIndent; i++) { wrap += '\t'; } wrap += " \""; Writer.Write(('"' + token.String.Replace("\\", "\\\\").Replace("\"", "\\\"").Replace("\t", "\\t").Replace("\n\n", "\\n\n").Replace("\n", "\\n" + wrap) + '"').Replace(wrap + '"', "\"" + wrap.Substring(2, wrap.Length - 4)).ToCharArray()); } if (TextBlocked) { Writer.Write("\r\n".ToCharArray()); } TextBlockEmpty = false; } break; default: throw new InvalidDataException("SimisToken.Kind is invalid: " + token.Kind); } } }