SimisToken ReadTokenAsText()
{
SimisToken rv = new SimisToken();
if ('(' == Reader.PeekChar())
{
Reader.ReadChar();
rv.Kind = SimisTokenKind.BlockBegin;
return(rv);
}
if (')' == Reader.PeekChar())
{
Reader.ReadChar();
rv.Kind = SimisTokenKind.BlockEnd;
return(rv);
}
if (':' == Reader.PeekChar())
{
Reader.ReadChar();
return(ReadTokenAsText());
}
string token = ReadTokenOrString();
if (BnfState == null)
{
try {
BnfState = new BnfState(JinxStreamFormat.Bnf);
} catch (UnknownSimisFormatException e) {
throw new ReaderException(Reader, false, PinReaderChanged(), "", e);
}
}
if (BnfState.IsEnterBlockTime)
{
// We should only end up here when called recursively by the
// if (validStates.Contains(token)) code below.
rv.Name = token;
rv.Kind = token.Length > 0 ? SimisTokenKind.Block : SimisTokenKind.None;
return(rv);
}
if (token.StartsWith("_", StringComparison.InvariantCulture) || (token.ToUpperInvariant() == "COMMENT") || (token.ToUpperInvariant() == "INFO") || (token.ToUpperInvariant() == "SKIP"))
{
var oldPosition = Reader.BaseStream.Position;
while ((Reader.BaseStream.Position < Reader.BaseStream.Length) && WhitespaceChars.Contains((char)Reader.PeekChar()))
{
Reader.ReadChar();
}
var nextToken = Reader.PeekChar();
Reader.BaseStream.Position = oldPosition;
if (nextToken == (int)'(')
{
var blockCount = 0;
while ((Reader.BaseStream.Position < Reader.BaseStream.Length) && ((')' != Reader.PeekChar()) || (blockCount > 1)))
{
if (Reader.PeekChar() == '(')
{
blockCount++;
}
if (Reader.PeekChar() == ')')
{
blockCount--;
}
token += Reader.ReadChar();
}
if (Reader.BaseStream.Position >= Reader.BaseStream.Length)
{
throw new ReaderException(Reader, false, 0, "SimisReader expected ')'; got EOF.");
}
token += Reader.ReadChar();
rv.String = token;
rv.Name = "Comment";
rv.Kind = SimisTokenKind.String;
return(rv);
}
}
if (token.StartsWith("//", StringComparison.InvariantCulture))
{
var blockCount = 0;
while ((Reader.BaseStream.Position < Reader.BaseStream.Length) && ('\n' != Reader.PeekChar()) && ((')' != Reader.PeekChar()) || (blockCount > 0)))
{
if (Reader.PeekChar() == '(')
{
blockCount++;
}
if (Reader.PeekChar() == ')')
{
blockCount--;
}
token += Reader.ReadChar();
}
rv.String = token;
rv.Name = "Comment";
rv.Kind = SimisTokenKind.String;
return(rv);
}
var validStates = BnfState.ValidStates;
if (validStates.Contains(token, StringComparer.InvariantCultureIgnoreCase))
{
// Token exactly matches a valid state transition, so let's use it.
rv.Type = validStates.First(s => s.Equals(token, StringComparison.InvariantCultureIgnoreCase));
rv.Kind = SimisTokenKind.Block;
// Do lookahead for block name.
PinReader();
while ((Reader.BaseStream.Position < Reader.BaseStream.Length) && WhitespaceChars.Contains((char)Reader.PeekChar()))
{
Reader.ReadChar();
}
string name = ReadTokenOrString();
if (name.Length > 0)
{
rv.Name = name;
}
else
{
PinReaderReset();
}
return(rv);
}
var validDataTypeStates = validStates.Where(s => {
switch (s)
{
case "uint":
return(token.ToCharArray().All(c => DecDigits.Contains(c)));
case "sint":
if (token.StartsWith("-", StringComparison.Ordinal))
{
return(token.Substring(1).ToCharArray().All(c => DecDigits.Contains(c)));
}
return(token.ToCharArray().All(c => DecDigits.Contains(c)));
case "dword":
return((token.Length == 8) && (token.ToCharArray().All(c => HexDigits.Contains(c))));
case "float":
if (token.StartsWith("-", StringComparison.Ordinal))
{
return(token.Substring(1).ToCharArray().All(c => DecFloatDigits.Contains(c)));
}
return(token.ToCharArray().All(c => DecFloatDigits.Contains(c)));
case "string":
return(true);
case "buffer":
default:
return(false);
}
}).ToArray();
if (validDataTypeStates.Length == 0)
{
try {
// This is *expected* to throw! We're doing this so that we get a proper BNF exception from the failed state.
BnfState.MoveTo(token);
} catch (BnfStateException ex) {
throw new ReaderException(Reader, false, PinReaderChanged(), "", ex);
}
}
rv.Type = validDataTypeStates[0];
switch (rv.Type)
{
case "uint":
if (token.EndsWith(",", StringComparison.Ordinal))
{
token = token.Substring(0, token.Length - 1);
}
try {
rv.IntegerUnsigned = UInt32.Parse(token, CultureInfo.InvariantCulture);
if (token.Length == 8)
{
throw new ReaderException(Reader, false, PinReaderChanged(), "SimisReader expected decimal number; got possible hex '" + token + "'.");
}
} catch (FormatException ex) {
throw new ReaderException(Reader, false, PinReaderChanged(), "SimisReader failed to parse '" + token + "' as '" + rv.Type + "'.", ex);
} catch (OverflowException ex) {
throw new ReaderException(Reader, false, PinReaderChanged(), "SimisReader failed to parse '" + token + "' as '" + rv.Type + "'.", ex);
}
rv.Kind = SimisTokenKind.IntegerUnsigned;
break;
case "sint":
if (token.EndsWith(",", StringComparison.Ordinal))
{
token = token.Substring(0, token.Length - 1);
}
try {
// Special-case -1 witten as an unsigned integer instead of signed.
if (token == "4294967295")
{
rv.IntegerSigned = -1;
}
else
{
rv.IntegerSigned = Int32.Parse(token, CultureInfo.InvariantCulture);
}
} catch (FormatException ex) {
throw new ReaderException(Reader, false, PinReaderChanged(), "SimisReader failed to parse '" + token + "' as '" + rv.Type + "'.", ex);
} catch (OverflowException ex) {
throw new ReaderException(Reader, false, PinReaderChanged(), "SimisReader failed to parse '" + token + "' as '" + rv.Type + "'.", ex);
}
rv.Kind = SimisTokenKind.IntegerSigned;
break;
case "dword":
if (token.EndsWith(",", StringComparison.Ordinal))
{
token = token.Substring(0, token.Length - 1);
}
if (token.Length != 8)
{
throw new ReaderException(Reader, false, PinReaderChanged(), "SimisReader expected 8-digit hex number; got '" + token + "'.");
}
try {
rv.IntegerDWord = UInt32.Parse(token, NumberStyles.HexNumber, CultureInfo.InvariantCulture);
} catch (FormatException ex) {
throw new ReaderException(Reader, false, PinReaderChanged(), "SimisReader failed to parse '" + token + "' as '" + rv.Type + "'.", ex);
} catch (OverflowException ex) {
throw new ReaderException(Reader, false, PinReaderChanged(), "SimisReader failed to parse '" + token + "' as '" + rv.Type + "'.", ex);
}
rv.Kind = SimisTokenKind.IntegerDWord;
break;
case "word":
if (token.EndsWith(",", StringComparison.Ordinal))
{
token = token.Substring(0, token.Length - 1);
}
if (token.Length != 4)
{
throw new ReaderException(Reader, false, PinReaderChanged(), "SimisReader expected 4-digit hex number; got '" + token + "'.");
}
try {
rv.IntegerDWord = UInt16.Parse(token, NumberStyles.HexNumber, CultureInfo.InvariantCulture);
} catch (FormatException ex) {
throw new ReaderException(Reader, false, PinReaderChanged(), "SimisReader failed to parse '" + token + "' as '" + rv.Type + "'.", ex);
} catch (OverflowException ex) {
throw new ReaderException(Reader, false, PinReaderChanged(), "SimisReader failed to parse '" + token + "' as '" + rv.Type + "'.", ex);
}
rv.Kind = SimisTokenKind.IntegerDWord;
break;
case "byte":
if (token.EndsWith(",", StringComparison.Ordinal))
{
token = token.Substring(0, token.Length - 1);
}
if (token.Length != 2)
{
throw new ReaderException(Reader, false, PinReaderChanged(), "SimisReader expected 2-digit hex number; got '" + token + "'.");
}
try {
rv.IntegerDWord = Byte.Parse(token, NumberStyles.HexNumber, CultureInfo.InvariantCulture);
} catch (FormatException ex) {
throw new ReaderException(Reader, false, PinReaderChanged(), "SimisReader failed to parse '" + token + "' as '" + rv.Type + "'.", ex);
} catch (OverflowException ex) {
throw new ReaderException(Reader, false, PinReaderChanged(), "SimisReader failed to parse '" + token + "' as '" + rv.Type + "'.", ex);
}
rv.Kind = SimisTokenKind.IntegerDWord;
break;
case "float":
if (token.EndsWith(",", StringComparison.Ordinal))
{
token = token.Substring(0, token.Length - 1);
}
try {
rv.Float = float.Parse(token, CultureInfo.InvariantCulture);
} catch (FormatException ex) {
throw new ReaderException(Reader, false, PinReaderChanged(), "SimisReader failed to parse '" + token + "' as '" + rv.Type + "'.", ex);
} catch (OverflowException ex) {
throw new ReaderException(Reader, false, PinReaderChanged(), "SimisReader failed to parse '" + token + "' as '" + rv.Type + "'.", ex);
}
rv.Kind = SimisTokenKind.Float;
break;
case "string":
rv.String = token;
rv.Kind = SimisTokenKind.String;
break;
default:
throw new ReaderException(Reader, false, PinReaderChanged(), "SimisReader found unexpected data type '" + rv.Type + "'.", new BnfStateException(BnfState, ""));
}
return(rv);
}
SimisToken ReadTokenAsBinary()
{
SimisToken rv = new SimisToken();
var validStates = (BnfState == null ? new string[] { } : BnfState.ValidStates);
//if (validStates.Length == 0) throw new ReaderException(BinaryReader, true, PinReaderChanged(), "SimisReader found no non-meta states available.", new BNFStateException(BNFState, ""));
// If we have any valid data types, we read that instead of a block start. They should all be the same data type, too.
var dataType = validStates.FirstOrDefault(s => DataTypes.Contains(s));
if (dataType != null)
{
if (!validStates.All(s => s == dataType || !DataTypes.Contains(s)))
{
throw new ReaderException(Reader, true, PinReaderChanged(), "SimisReader found inconsistent data types available.", new BnfStateException(BnfState, ""));
}
rv.Type = dataType;
switch (rv.Type)
{
case "uint":
rv.IntegerUnsigned = Reader.ReadUInt32();
rv.Kind = SimisTokenKind.IntegerUnsigned;
break;
case "sint":
rv.IntegerSigned = Reader.ReadInt32();
rv.Kind = SimisTokenKind.IntegerSigned;
break;
case "dword":
rv.IntegerDWord = Reader.ReadUInt32();
rv.Kind = SimisTokenKind.IntegerDWord;
break;
case "word":
rv.IntegerDWord = Reader.ReadUInt16();
rv.Kind = SimisTokenKind.IntegerWord;
break;
case "byte":
rv.IntegerDWord = Reader.ReadByte();
rv.Kind = SimisTokenKind.IntegerByte;
break;
case "float":
rv.Float = Reader.ReadSingle();
rv.Kind = SimisTokenKind.Float;
break;
case "string":
var stringLength = Reader.ReadUInt16();
if (stringLength > 10000)
{
throw new ReaderException(Reader, true, PinReaderChanged(), "SimisReader found a string longer than 10,000 characters.", new BnfStateException(BnfState, ""));
}
if (Reader.BaseStream.Position + stringLength * 2 > Reader.BaseStream.Length)
{
throw new ReaderException(Reader, true, PinReaderChanged(), "SimisReader found a string extending beyond the end of the file.", new BnfStateException(BnfState, ""));
}
for (var i = 0; i < stringLength; i++)
{
rv.String += (char)Reader.ReadUInt16();
}
rv.Kind = SimisTokenKind.String;
break;
case "buffer":
var bufferLength = BlockEndOffsets.Peek() - Reader.BaseStream.Position;
rv.String = String.Join("", Reader.ReadBytes((int)bufferLength).Select(b => b.ToString("X2", CultureInfo.InvariantCulture)).ToArray());
rv.Kind = SimisTokenKind.String;
break;
default:
throw new ReaderException(Reader, true, PinReaderChanged(), "SimisReader found unexpected data type '" + rv.Type + "'.", new BnfStateException(BnfState, ""));
}
}
else
{
var tokenID = Reader.ReadUInt16();
var tokenType = Reader.ReadUInt16();
var token = ((uint)tokenType << 16) + tokenID;
if (!SimisProvider.TokenNames.ContainsKey(token))
{
throw new ReaderException(Reader, true, PinReaderChanged(), String.Format(CultureInfo.CurrentCulture, "SimisReader got invalid block: id={0:X4}, type={1:X4}.", tokenID, tokenType), new BnfStateException(BnfState, ""));
}
if ((tokenType != 0x0000) && (tokenType != 0x0004))
{
throw new ReaderException(Reader, true, PinReaderChanged(), String.Format(CultureInfo.CurrentCulture, "SimisReader got invalid block: id={0:X4}, type={1:X4}, name={2}.", tokenID, tokenType, SimisProvider.TokenNames[token]), new BnfStateException(BnfState, ""));
}
rv.Type = SimisProvider.TokenNames[token];
rv.Kind = SimisTokenKind.Block;
if (BnfState == null)
{
try {
BnfState = new BnfState(JinxStreamFormat.Bnf);
} catch (UnknownSimisFormatException e) {
throw new ReaderException(Reader, true, PinReaderChanged(), "", e);
}
}
var contentsLength = Reader.ReadUInt32();
if (contentsLength == 0)
{
throw new ReaderException(Reader, true, PinReaderChanged(), "SimisReader got block with length of 0.", new BnfStateException(BnfState, ""));
}
if (Reader.BaseStream.Position + contentsLength > StreamLength)
{
throw new ReaderException(Reader, true, PinReaderChanged(), "SimisReader got block longer than stream.", new BnfStateException(BnfState, ""));
}
BlockEndOffsets.Push((uint)Reader.BaseStream.Position + contentsLength);
PendingTokens.Enqueue(new SimisToken()
{
Kind = SimisTokenKind.BlockBegin, String = BlockEndOffsets.Peek().ToString("X8", CultureInfo.CurrentCulture)
});
var nameLength = Reader.Read();
if (nameLength > 0)
{
if (Reader.BaseStream.Position + nameLength * 2 > StreamLength)
{
throw new ReaderException(Reader, true, PinReaderChanged(), "SimisReader got block with name longer than stream.", new BnfStateException(BnfState, ""));
}
for (var i = 0; i < nameLength; i++)
{
rv.Name += (char)Reader.ReadUInt16();
}
}
}
return(rv);
}
public void WriteToken(SimisToken token)
{
if (JinxStreamIsBinary)
{
switch (token.Kind)
{
case SimisTokenKind.Block:
var id = SimisProvider.TokenIds[token.Type];
Writer.Write((uint)id);
Writer.Write((uint)0x7F8F7F8F); // Length, set to sentinel value for now.
BlockStarts.Push(Writer.BaseStream.Position);
Writer.Write((byte)token.Name.Length);
foreach (var ch in token.Name.ToCharArray())
{
Writer.Write((ushort)ch);
}
break;
case SimisTokenKind.BlockBegin:
break;
case SimisTokenKind.BlockEnd:
var start = BlockStarts.Pop();
var length = (uint)(Writer.BaseStream.Position - start);
Writer.BaseStream.Seek(start - 4, SeekOrigin.Begin);
Writer.Write(length);
Writer.BaseStream.Seek(0, SeekOrigin.End);
break;
case SimisTokenKind.IntegerUnsigned:
Writer.Write(token.IntegerUnsigned);
break;
case SimisTokenKind.IntegerSigned:
Writer.Write(token.IntegerSigned);
break;
case SimisTokenKind.IntegerDWord:
Writer.Write(token.IntegerDWord);
break;
case SimisTokenKind.IntegerWord:
Writer.Write((ushort)token.IntegerDWord);
break;
case SimisTokenKind.IntegerByte:
Writer.Write((byte)token.IntegerDWord);
break;
case SimisTokenKind.Float:
Writer.Write((float)token.Float);
break;
case SimisTokenKind.String:
Writer.Write((ushort)token.String.Length);
foreach (var ch in token.String.ToCharArray())
{
Writer.Write((ushort)ch);
}
break;
default:
throw new InvalidDataException("SimisToken.Kind is invalid: " + token.Kind);
}
}
else
{
switch (token.Kind)
{
case SimisTokenKind.Block:
if (BnfState == null)
{
BnfState = new BnfState(JinxStreamFormat.Bnf);
}
if (token.Type.Length > 0)
{
BnfState.MoveTo(token.Type);
}
if (!TextBlocked)
{
Writer.Write("\r\n".ToCharArray());
}
for (var i = 0; i < TextIndent; i++)
{
Writer.Write('\t');
}
Writer.Write(token.Type.ToCharArray());
if (token.Name.Length > 0)
{
Writer.Write((" " + token.Name).ToCharArray());
}
TextBlocked = true;
break;
case SimisTokenKind.BlockBegin:
BnfState.EnterBlock();
Writer.Write(" (".ToCharArray());
TextIndent++;
TextBlocked = false;
TextBlockEmpty = true;
break;
case SimisTokenKind.BlockEnd:
BnfState.LeaveBlock();
if (BnfState.IsCompleted)
{
BnfState = null;
}
TextIndent--;
if (TextBlocked)
{
for (var i = 0; i < TextIndent; i++)
{
Writer.Write('\t');
}
}
else if (!TextBlockEmpty)
{
Writer.Write(' ');
}
Writer.Write(")\r\n".ToCharArray());
TextBlocked = true;
break;
case SimisTokenKind.IntegerUnsigned:
BnfState.MoveTo(token.Type);
if (TextBlocked)
{
for (var i = 0; i < TextIndent; i++)
{
Writer.Write('\t');
}
}
else
{
Writer.Write(' ');
}
Writer.Write(token.IntegerUnsigned.ToString(CultureInfo.InvariantCulture).ToCharArray());
if (TextBlocked)
{
Writer.Write("\r\n".ToCharArray());
}
TextBlockEmpty = false;
break;
case SimisTokenKind.IntegerSigned:
BnfState.MoveTo(token.Type);
if (TextBlocked)
{
for (var i = 0; i < TextIndent; i++)
{
Writer.Write('\t');
}
}
else
{
Writer.Write(' ');
}
Writer.Write(token.IntegerSigned.ToString(CultureInfo.InvariantCulture).ToCharArray());
if (TextBlocked)
{
Writer.Write("\r\n".ToCharArray());
}
TextBlockEmpty = false;
break;
case SimisTokenKind.IntegerDWord:
case SimisTokenKind.IntegerWord:
case SimisTokenKind.IntegerByte:
BnfState.MoveTo(token.Type);
if (TextBlocked)
{
for (var i = 0; i < TextIndent; i++)
{
Writer.Write('\t');
}
}
else
{
Writer.Write(' ');
}
Writer.Write(token.IntegerDWord.ToString(token.Kind == SimisTokenKind.IntegerDWord ? "X8" : token.Kind == SimisTokenKind.IntegerWord ? "X4" : "X2", CultureInfo.InvariantCulture).ToCharArray());
if (TextBlocked)
{
Writer.Write("\r\n".ToCharArray());
}
TextBlockEmpty = false;
break;
case SimisTokenKind.Float:
BnfState.MoveTo(token.Type);
if (TextBlocked)
{
for (var i = 0; i < TextIndent; i++)
{
Writer.Write('\t');
}
}
else
{
Writer.Write(' ');
}
if (token.Float.ToString("G6", CultureInfo.InvariantCulture).IndexOf("E", StringComparison.OrdinalIgnoreCase) >= 0)
{
Writer.Write(token.Float.ToString("0.#####e000", CultureInfo.InvariantCulture).ToCharArray());
}
else
{
Writer.Write(token.Float.ToString("G6", CultureInfo.InvariantCulture).ToCharArray());
}
if (TextBlocked)
{
Writer.Write("\r\n".ToCharArray());
}
TextBlockEmpty = false;
break;
case SimisTokenKind.String:
// If the token has no type, it was a specially-skipped input (comment, SKIP(...) block etc.).
if (token.Type.Length == 0)
{
if (!TextBlocked)
{
Writer.Write("\r\n".ToCharArray());
}
for (var i = 0; i < TextIndent; i++)
{
Writer.Write('\t');
}
Writer.Write(token.String.ToCharArray());
Writer.Write("\r\n".ToCharArray());
TextBlocked = true;
TextBlockEmpty = true;
}
else
{
BnfState.MoveTo(token.Type);
if (TextBlocked)
{
for (var i = 0; i < TextIndent; i++)
{
Writer.Write('\t');
}
}
else
{
Writer.Write(' ');
}
if ((token.String.Length > 0) && token.String.ToCharArray().All(c => SafeTokenCharacters.Contains(c)))
{
Writer.Write(token.String.ToCharArray());
}
else
{
var wrap = "\"+\r\n";
for (var i = 0; i < TextIndent; i++)
{
wrap += '\t';
}
wrap += " \"";
Writer.Write(('"' + token.String.Replace("\\", "\\\\").Replace("\"", "\\\"").Replace("\t", "\\t").Replace("\n\n", "\\n\n").Replace("\n", "\\n" + wrap) + '"').Replace(wrap + '"', "\"" + wrap.Substring(2, wrap.Length - 4)).ToCharArray());
}
if (TextBlocked)
{
Writer.Write("\r\n".ToCharArray());
}
TextBlockEmpty = false;
}
break;
default:
throw new InvalidDataException("SimisToken.Kind is invalid: " + token.Kind);
}
}
}
