private static void Expect(FastBinaryReader reader, char character) { var next = reader.Read(); if (next != character) { throw new UnexpectedCharacterException((char)next); } }
private static StepToken ParseNumber(FastBinaryReader reader, FastStringBuilder buffer) { //integer or real while (IsDigit(reader.Peek())) { buffer.Append((char)reader.Read()); } if (reader.Peek() == '.') { buffer.Append((char)reader.Read()); while (IsDigit(reader.Peek())) { //realValue += buffer.Append((char)reader.Read()); } if (reader.Peek() == 'E') { reader.Read(); buffer.Append('E'); var next = reader.Peek(); if (next == '+' || next == '-') { buffer.Append((char)next); reader.Read(); next = reader.Peek(); } if (IsDigit(next)) { buffer.Append((char)reader.Read()); } else { throw new UnexpectedCharacterException((char)next); } while (IsDigit(reader.Peek())) { buffer.Append((char)reader.Read()); } } return(new StepToken(StepTokenKind.Real, buffer)); } else { return(new StepToken(StepTokenKind.Integer, buffer)); } }
private unsafe static StepTokenKind ParseStringKind(FastBinaryReader reader, FastStringBuilder buffer) { int encoding = 0; int next; while ((next = reader.Read()) != -1) { if (next == '\'') { if (reader.Peek() == '\'') { reader.Read(); buffer.Append('\''); } else { return(StepTokenKind.String); } } else if (next == '\\') { switch (reader.Read()) { case 'S': { next = reader.Read(); if (next != '\\') { throw new UnexpectedCharacterException((char)next); } next = reader.Read(); if (IsCharacter(next)) { byte *ptr = stackalloc byte[1]; char *result = stackalloc char[1]; ptr[0] = (byte)(next + 128); encodings_[encoding].GetChars(ptr, 1, result, 1); buffer.Append(*result); break; } else { throw new UnexpectedCharacterException((char)next); } } case 'P': { next = reader.Read(); if (next >= 'A' && next <= 'I') { encoding = next - 'A'; } else { throw new UnexpectedCharacterException((char)next); } next = reader.Read(); if (next != '\\') { throw new UnexpectedCharacterException((char)next); } break; } case 'X': { next = reader.Read(); if (next == '\\') { byte *ptr = stackalloc byte[1]; char *result = stackalloc char[1]; next = reader.Read(); if (!IsHex(next)) { throw new UnexpectedCharacterException((char)next); } ptr[0] = (byte)(FromHex(next) << 8); next = reader.Read(); ptr[0] |= (byte)FromHex(next); encodings_[0].GetChars(ptr, 1, result, 1); buffer.Append(*result); break; } else if (next == '2') { byte *ptr = stackalloc byte[20]; char *chars = stackalloc char[10]; Expect(reader, '\\'); int offset = 0; while (IsHex(reader.Peek())) { next = reader.Read(); ptr[offset] = (byte)(FromHex(next) << 24); next = reader.Read(); ptr[offset] |= (byte)(FromHex(next) << 16); offset++; next = reader.Read(); ptr[offset] = (byte)(FromHex(next) << 8); next = reader.Read(); ptr[offset] |= (byte)(FromHex(next)); offset++; if (offset == 20) { Encoding.Unicode.GetChars(ptr, offset, chars, 20); buffer.Append(chars, 10); offset = 0; } } var encoded = Encoding.Unicode.GetChars(ptr, offset, chars, 20); buffer.Append(chars, encoded); Expect(reader, '\\'); Expect(reader, 'X'); Expect(reader, '0'); Expect(reader, '\\'); break; } else if (next == '4') { throw new NotImplementedException(); } else { throw new UnexpectedCharacterException((char)next); } } case '\\': buffer.Append('\\'); break; case int ch: throw new UnexpectedCharacterException((char)ch); } } else if (IsCharacter(next)) { buffer.Append((char)next); } else { throw new UnexpectedCharacterException((char)next); } } return(StepTokenKind.String); }
public static IEnumerable <StepToken> Tokenize(FastBinaryReader reader) { int current; var buffers = new DoubleBuffered <FastStringBuilder>(); //var keywordBuilder = new KeywordStringBuilder(); while ((current = reader.Read()) != -1) { var ch = (char)current; var buffer = buffers.Current; switch (ch) { case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': { buffers.Swap(); buffer.Clear(); buffer.Append(ch); yield return(ParseNumber(reader, buffer)); break; }; case '-': case '+': { if (IsDigit(reader.Peek())) { buffers.Swap(); buffer.Clear(); buffer.Append(ch); yield return(ParseNumber(reader, buffer)); break; } else { throw new UnexpectedCharacterException(ch); } } case '#': { if (!IsDigit(reader.Peek())) { throw new UnexpectedCharacterException(ch); } buffers.Swap(); buffer.Clear(); do { buffer.Append((char)reader.Read()); }while (IsDigit(reader.Peek())); yield return(new StepToken(StepTokenKind.EntityInstanceName, buffer)); break; } case '"': { throw new NotImplementedException("binary"); } case '\'': { buffers.Swap(); buffer.Clear(); yield return(ParseString(reader, buffer)); break; } case '!': { throw new NotImplementedException("user defined keyword"); } case '.': { if (!IsUpper(reader.Peek())) { throw new UnexpectedCharacterException(ch); } buffers.Swap(); buffer.Clear(); do { buffer.Append((char)reader.Read()); } while (IsUpperOrDigit(reader.Peek())); if (reader.Peek() == '.') { reader.Read(); yield return(new StepToken(StepTokenKind.Enumeration, buffer)); break; } else { throw new UnexpectedCharacterException((char)reader.Peek()); } } case '=': yield return(StepToken.Assignment); break; case '*': yield return(StepToken.Asterisk); break; case '$': yield return(StepToken.Dollar); break; case ';': yield return(StepToken.Semicolon); break; case '(': yield return(StepToken.LeftParen); break; case ')': yield return(StepToken.RightParen); break; case '/': { if (reader.Peek() == '*') { while ((current = reader.Read()) != -1) { if (current == '*' && reader.Peek() == '/') { reader.Read(); break; } } } else { yield return(StepToken.Solidus); } } break; case ',': yield return(StepToken.Comma); break; case ' ': break; default: { if (IsUpper(ch)) { buffers.Swap(); buffer.Clear(); buffer.Append(ch); while (IsUpperOrDigitOrMinus(reader.Peek())) { buffer.Append((char)reader.Read()); } if (buffer.Equals("ISO-10303-21")) { yield return(StepToken.Iso); } else if (buffer.Equals("END-ISO-10303-21")) { yield return(StepToken.EndIso); } else if (buffer.Equals("HEADER")) { yield return(StepToken.Header); } else if (buffer.Equals("ENDSEC")) { yield return(StepToken.EndSection); } else if (buffer.Equals("DATA")) { yield return(StepToken.Data); } else { yield return(new StepToken(StepTokenKind.StandardKeyword, buffer)); } break; } else if (IsCharacter(ch)) { throw new UnexpectedCharacterException(ch); } break; } } } yield return(StepToken.Eof); }