private static void Expect(FastBinaryReader reader, char character)
{
var next = reader.Read();
if (next != character)
{
throw new UnexpectedCharacterException((char)next);
}
}
private static StepToken ParseNumber(FastBinaryReader reader, FastStringBuilder buffer)
{
//integer or real
while (IsDigit(reader.Peek()))
{
buffer.Append((char)reader.Read());
}
if (reader.Peek() == '.')
{
buffer.Append((char)reader.Read());
while (IsDigit(reader.Peek()))
{
//realValue +=
buffer.Append((char)reader.Read());
}
if (reader.Peek() == 'E')
{
reader.Read();
buffer.Append('E');
var next = reader.Peek();
if (next == '+' || next == '-')
{
buffer.Append((char)next);
reader.Read();
next = reader.Peek();
}
if (IsDigit(next))
{
buffer.Append((char)reader.Read());
}
else
{
throw new UnexpectedCharacterException((char)next);
}
while (IsDigit(reader.Peek()))
{
buffer.Append((char)reader.Read());
}
}
return(new StepToken(StepTokenKind.Real, buffer));
}
else
{
return(new StepToken(StepTokenKind.Integer, buffer));
}
}
private unsafe static StepTokenKind ParseStringKind(FastBinaryReader reader, FastStringBuilder buffer)
{
int encoding = 0;
int next;
while ((next = reader.Read()) != -1)
{
if (next == '\'')
{
if (reader.Peek() == '\'')
{
reader.Read();
buffer.Append('\'');
}
else
{
return(StepTokenKind.String);
}
}
else if (next == '\\')
{
switch (reader.Read())
{
case 'S':
{
next = reader.Read();
if (next != '\\')
{
throw new UnexpectedCharacterException((char)next);
}
next = reader.Read();
if (IsCharacter(next))
{
byte *ptr = stackalloc byte[1];
char *result = stackalloc char[1];
ptr[0] = (byte)(next + 128);
encodings_[encoding].GetChars(ptr, 1, result, 1);
buffer.Append(*result);
break;
}
else
{
throw new UnexpectedCharacterException((char)next);
}
}
case 'P':
{
next = reader.Read();
if (next >= 'A' && next <= 'I')
{
encoding = next - 'A';
}
else
{
throw new UnexpectedCharacterException((char)next);
}
next = reader.Read();
if (next != '\\')
{
throw new UnexpectedCharacterException((char)next);
}
break;
}
case 'X':
{
next = reader.Read();
if (next == '\\')
{
byte *ptr = stackalloc byte[1];
char *result = stackalloc char[1];
next = reader.Read();
if (!IsHex(next))
{
throw new UnexpectedCharacterException((char)next);
}
ptr[0] = (byte)(FromHex(next) << 8);
next = reader.Read();
ptr[0] |= (byte)FromHex(next);
encodings_[0].GetChars(ptr, 1, result, 1);
buffer.Append(*result);
break;
}
else if (next == '2')
{
byte *ptr = stackalloc byte[20];
char *chars = stackalloc char[10];
Expect(reader, '\\');
int offset = 0;
while (IsHex(reader.Peek()))
{
next = reader.Read();
ptr[offset] = (byte)(FromHex(next) << 24);
next = reader.Read();
ptr[offset] |= (byte)(FromHex(next) << 16);
offset++;
next = reader.Read();
ptr[offset] = (byte)(FromHex(next) << 8);
next = reader.Read();
ptr[offset] |= (byte)(FromHex(next));
offset++;
if (offset == 20)
{
Encoding.Unicode.GetChars(ptr, offset, chars, 20);
buffer.Append(chars, 10);
offset = 0;
}
}
var encoded = Encoding.Unicode.GetChars(ptr, offset, chars, 20);
buffer.Append(chars, encoded);
Expect(reader, '\\');
Expect(reader, 'X');
Expect(reader, '0');
Expect(reader, '\\');
break;
}
else if (next == '4')
{
throw new NotImplementedException();
}
else
{
throw new UnexpectedCharacterException((char)next);
}
}
case '\\':
buffer.Append('\\');
break;
case int ch:
throw new UnexpectedCharacterException((char)ch);
}
}
else if (IsCharacter(next))
{
buffer.Append((char)next);
}
else
{
throw new UnexpectedCharacterException((char)next);
}
}
return(StepTokenKind.String);
}
public static IEnumerable <StepToken> Tokenize(FastBinaryReader reader)
{
int current;
var buffers = new DoubleBuffered <FastStringBuilder>();
//var keywordBuilder = new KeywordStringBuilder();
while ((current = reader.Read()) != -1)
{
var ch = (char)current;
var buffer = buffers.Current;
switch (ch)
{
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
{
buffers.Swap();
buffer.Clear();
buffer.Append(ch);
yield return(ParseNumber(reader, buffer));
break;
};
case '-':
case '+':
{
if (IsDigit(reader.Peek()))
{
buffers.Swap();
buffer.Clear();
buffer.Append(ch);
yield return(ParseNumber(reader, buffer));
break;
}
else
{
throw new UnexpectedCharacterException(ch);
}
}
case '#':
{
if (!IsDigit(reader.Peek()))
{
throw new UnexpectedCharacterException(ch);
}
buffers.Swap();
buffer.Clear();
do
{
buffer.Append((char)reader.Read());
}while (IsDigit(reader.Peek()));
yield return(new StepToken(StepTokenKind.EntityInstanceName, buffer));
break;
}
case '"':
{
throw new NotImplementedException("binary");
}
case '\'':
{
buffers.Swap();
buffer.Clear();
yield return(ParseString(reader, buffer));
break;
}
case '!':
{
throw new NotImplementedException("user defined keyword");
}
case '.':
{
if (!IsUpper(reader.Peek()))
{
throw new UnexpectedCharacterException(ch);
}
buffers.Swap();
buffer.Clear();
do
{
buffer.Append((char)reader.Read());
} while (IsUpperOrDigit(reader.Peek()));
if (reader.Peek() == '.')
{
reader.Read();
yield return(new StepToken(StepTokenKind.Enumeration, buffer));
break;
}
else
{
throw new UnexpectedCharacterException((char)reader.Peek());
}
}
case '=':
yield return(StepToken.Assignment);
break;
case '*':
yield return(StepToken.Asterisk);
break;
case '$':
yield return(StepToken.Dollar);
break;
case ';':
yield return(StepToken.Semicolon);
break;
case '(':
yield return(StepToken.LeftParen);
break;
case ')':
yield return(StepToken.RightParen);
break;
case '/':
{
if (reader.Peek() == '*')
{
while ((current = reader.Read()) != -1)
{
if (current == '*' && reader.Peek() == '/')
{
reader.Read();
break;
}
}
}
else
{
yield return(StepToken.Solidus);
}
}
break;
case ',':
yield return(StepToken.Comma);
break;
case ' ':
break;
default:
{
if (IsUpper(ch))
{
buffers.Swap();
buffer.Clear();
buffer.Append(ch);
while (IsUpperOrDigitOrMinus(reader.Peek()))
{
buffer.Append((char)reader.Read());
}
if (buffer.Equals("ISO-10303-21"))
{
yield return(StepToken.Iso);
}
else if (buffer.Equals("END-ISO-10303-21"))
{
yield return(StepToken.EndIso);
}
else if (buffer.Equals("HEADER"))
{
yield return(StepToken.Header);
}
else if (buffer.Equals("ENDSEC"))
{
yield return(StepToken.EndSection);
}
else if (buffer.Equals("DATA"))
{
yield return(StepToken.Data);
}
else
{
yield return(new StepToken(StepTokenKind.StandardKeyword, buffer));
}
break;
}
else if (IsCharacter(ch))
{
throw new UnexpectedCharacterException(ch);
}
break;
}
}
}
yield return(StepToken.Eof);
}