public Token(TokType type, string text, string value, int pos, int line, int col)
{
m_type = type;
m_value = value;
m_text = text;
m_pos = pos;
m_line = line;
m_col = col;
}
virtual public bool NextToken() {
int ch = 0;
do {
ch = file.Read();
} while (ch != -1 && IsWhitespace(ch));
if (ch == -1){
type = TokType.ENDOFFILE;
return false;
}
// Note: We have to initialize stringValue here, after we've looked for the end of the stream,
// to ensure that we don't lose the value of a token that might end exactly at the end
// of the stream
StringBuilder outBuf = new StringBuilder();
stringValue = EMPTY;
switch (ch) {
case '[':
type = TokType.START_ARRAY;
break;
case ']':
type = TokType.END_ARRAY;
break;
case '/': {
outBuf.Length = 0;
type = TokType.NAME;
while (true) {
ch = file.Read();
if (ch == -1 || IsDelimiter(ch) || IsWhitespace(ch))
break;
if (ch == '#') {
ch = (GetHex(file.Read()) << 4) + GetHex(file.Read());
}
outBuf.Append((char)ch);
}
BackOnePosition(ch);
break;
}
case '>':
ch = file.Read();
if (ch != '>')
ThrowError(MessageLocalization.GetComposedMessage("greaterthan.not.expected"));
type = TokType.END_DIC;
break;
case '<': {
int v1 = file.Read();
if (v1 == '<') {
type = TokType.START_DIC;
break;
}
outBuf.Length = 0;
type = TokType.STRING;
hexString = true;
int v2 = 0;
while (true) {
while (IsWhitespace(v1))
v1 = file.Read();
if (v1 == '>')
break;
v1 = GetHex(v1);
if (v1 < 0)
break;
v2 = file.Read();
while (IsWhitespace(v2))
v2 = file.Read();
if (v2 == '>') {
ch = v1 << 4;
outBuf.Append((char)ch);
break;
}
v2 = GetHex(v2);
if (v2 < 0)
break;
ch = (v1 << 4) + v2;
outBuf.Append((char)ch);
v1 = file.Read();
}
if (v1 < 0 || v2 < 0)
ThrowError(MessageLocalization.GetComposedMessage("error.reading.string"));
break;
}
case '%':
type = TokType.COMMENT;
do {
ch = file.Read();
} while (ch != -1 && ch != '\r' && ch != '\n');
break;
case '(': {
outBuf.Length = 0;
type = TokType.STRING;
hexString = false;
int nesting = 0;
while (true) {
ch = file.Read();
if (ch == -1)
break;
if (ch == '(') {
++nesting;
}
else if (ch == ')') {
--nesting;
}
else if (ch == '\\') {
bool lineBreak = false;
ch = file.Read();
switch (ch) {
case 'n':
ch = '\n';
break;
case 'r':
ch = '\r';
break;
case 't':
ch = '\t';
break;
case 'b':
ch = '\b';
break;
case 'f':
ch = '\f';
break;
case '(':
case ')':
case '\\':
break;
case '\r':
lineBreak = true;
ch = file.Read();
if (ch != '\n')
BackOnePosition(ch);
break;
case '\n':
lineBreak = true;
break;
default: {
if (ch < '0' || ch > '7') {
break;
}
int octal = ch - '0';
ch = file.Read();
if (ch < '0' || ch > '7') {
BackOnePosition(ch);
ch = octal;
break;
}
octal = (octal << 3) + ch - '0';
ch = file.Read();
if (ch < '0' || ch > '7') {
BackOnePosition(ch);
ch = octal;
break;
}
octal = (octal << 3) + ch - '0';
ch = octal & 0xff;
break;
}
}
if (lineBreak)
continue;
if (ch < 0)
break;
}
else if (ch == '\r') {
ch = file.Read();
if (ch < 0)
break;
if (ch != '\n') {
BackOnePosition(ch);
ch = '\n';
}
}
if (nesting == -1)
break;
outBuf.Append((char)ch);
}
if (ch == -1)
ThrowError(MessageLocalization.GetComposedMessage("error.reading.string"));
break;
}
default: {
outBuf.Length = 0;
if (ch == '-' || ch == '+' || ch == '.' || (ch >= '0' && ch <= '9')) {
type = TokType.NUMBER;
bool isReal = false;
int numberOfMinuses = 0;
if (ch == '-') {
// Take care of number like "--234". If Acrobat can read them so must we.
do {
++numberOfMinuses;
ch = file.Read();
} while (ch == '-');
outBuf.Append('-');
}
else {
outBuf.Append((char)ch);
// We don't need to check if the number is real over here as we need to know that fact only in case if there are any minuses.
ch = file.Read();
}
while (ch != -1 && ((ch >= '0' && ch <= '9') || ch == '.')) {
if (ch == '.')
isReal = true;
outBuf.Append((char)ch);
ch = file.Read();
}
if (numberOfMinuses > 1 && !isReal) {
// Numbers of integer type and with more than one minus before them
// are interpreted by Acrobat as zero.
outBuf.Length = 0;
outBuf.Append('0');
}
}
else {
type = TokType.OTHER;
do {
outBuf.Append((char)ch);
ch = file.Read();
} while (ch != -1 && !IsDelimiter(ch) && !IsWhitespace(ch));
}
if (ch != -1)
BackOnePosition(ch);
break;
}
}
if (outBuf != null)
stringValue = outBuf.ToString();
return true;
}
virtual public void NextValidToken() {
int level = 0;
string n1 = null;
string n2 = null;
long ptr = 0;
while (NextToken()) {
if (type == TokType.COMMENT)
continue;
switch (level) {
case 0: {
if (type != TokType.NUMBER)
return;
ptr = file.FilePointer;
n1 = stringValue;
++level;
break;
}
case 1: {
if (type != TokType.NUMBER) {
file.Seek(ptr);
type = TokType.NUMBER;
stringValue = n1;
return;
}
n2 = stringValue;
++level;
break;
}
default: {
if (type != TokType.OTHER || !stringValue.Equals("R")) {
file.Seek(ptr);
type = TokType.NUMBER;
stringValue = n1;
return;
}
type = TokType.REF;
reference = int.Parse(n1);
generation = int.Parse(n2);
return;
}
}
}
if (level == 1) { // if the level 1 check returns EOF, then we are still looking at a number - set the type back to NUMBER
type = TokType.NUMBER;
}
// if we hit here, the file is either corrupt (stream ended unexpectedly),
// or the last token ended exactly at the end of a stream. This last
// case can occur inside an Object Stream.
}
/// <summary>
/// Initializes a new instance of the <see cref="Token"/> class.
/// </summary>
/// <param name="tokType">The value.</param>
/// <param name="tokString">The token string value.</param>
private Token(TokType tokType, string tokString)
{
this.tokType = tokType;
this.tokString = tokString;
}
public bool NextToken()
{
int ch = 0;
do {
ch = file.Read();
} while (ch != -1 && IsWhitespace(ch));
if (ch == -1){
type = TokType.ENDOFFILE;
return false;
}
// Note: We have to initialize stringValue here, after we've looked for the end of the stream,
// to ensure that we don't lose the value of a token that might end exactly at the end
// of the stream
StringBuilder outBuf = null;
stringValue = EMPTY;
switch (ch) {
case '[':
type = TokType.START_ARRAY;
break;
case ']':
type = TokType.END_ARRAY;
break;
case '/': {
outBuf = new StringBuilder();
type = TokType.NAME;
while (true) {
ch = file.Read();
if (ch == -1 || IsDelimiter(ch) || IsWhitespace(ch))
break;
if (ch == '#') {
ch = (GetHex(file.Read()) << 4) + GetHex(file.Read());
}
outBuf.Append((char)ch);
}
BackOnePosition(ch);
break;
}
case '>':
ch = file.Read();
if (ch != '>')
ThrowError(MessageLocalization.GetComposedMessage("greaterthan.not.expected"));
type = TokType.END_DIC;
break;
case '<': {
int v1 = file.Read();
if (v1 == '<') {
type = TokType.START_DIC;
break;
}
outBuf = new StringBuilder();
type = TokType.STRING;
hexString = true;
int v2 = 0;
while (true) {
while (IsWhitespace(v1))
v1 = file.Read();
if (v1 == '>')
break;
v1 = GetHex(v1);
if (v1 < 0)
break;
v2 = file.Read();
while (IsWhitespace(v2))
v2 = file.Read();
if (v2 == '>') {
ch = v1 << 4;
outBuf.Append((char)ch);
break;
}
v2 = GetHex(v2);
if (v2 < 0)
break;
ch = (v1 << 4) + v2;
outBuf.Append((char)ch);
v1 = file.Read();
}
if (v1 < 0 || v2 < 0)
ThrowError(MessageLocalization.GetComposedMessage("error.reading.string"));
break;
}
case '%':
type = TokType.COMMENT;
do {
ch = file.Read();
} while (ch != -1 && ch != '\r' && ch != '\n');
break;
case '(': {
outBuf = new StringBuilder();
type = TokType.STRING;
hexString = false;
int nesting = 0;
while (true) {
ch = file.Read();
if (ch == -1)
break;
if (ch == '(') {
++nesting;
}
else if (ch == ')') {
--nesting;
}
else if (ch == '\\') {
bool lineBreak = false;
ch = file.Read();
switch (ch) {
case 'n':
ch = '\n';
break;
case 'r':
ch = '\r';
break;
case 't':
ch = '\t';
break;
case 'b':
ch = '\b';
break;
case 'f':
ch = '\f';
break;
case '(':
case ')':
case '\\':
break;
case '\r':
lineBreak = true;
ch = file.Read();
if (ch != '\n')
BackOnePosition(ch);
break;
case '\n':
lineBreak = true;
break;
default: {
if (ch < '0' || ch > '7') {
break;
}
int octal = ch - '0';
ch = file.Read();
if (ch < '0' || ch > '7') {
BackOnePosition(ch);
ch = octal;
break;
}
octal = (octal << 3) + ch - '0';
ch = file.Read();
if (ch < '0' || ch > '7') {
BackOnePosition(ch);
ch = octal;
break;
}
octal = (octal << 3) + ch - '0';
ch = octal & 0xff;
break;
}
}
if (lineBreak)
continue;
if (ch < 0)
break;
}
else if (ch == '\r') {
ch = file.Read();
if (ch < 0)
break;
if (ch != '\n') {
BackOnePosition(ch);
ch = '\n';
}
}
if (nesting == -1)
break;
outBuf.Append((char)ch);
}
if (ch == -1)
ThrowError(MessageLocalization.GetComposedMessage("error.reading.string"));
break;
}
default: {
outBuf = new StringBuilder();
if (ch == '-' || ch == '+' || ch == '.' || (ch >= '0' && ch <= '9')) {
type = TokType.NUMBER;
do {
outBuf.Append((char)ch);
ch = file.Read();
} while (ch != -1 && ((ch >= '0' && ch <= '9') || ch == '.'));
}
else {
type = TokType.OTHER;
do {
outBuf.Append((char)ch);
ch = file.Read();
} while (ch != -1 && !IsDelimiter(ch) && !IsWhitespace(ch));
}
BackOnePosition(ch);
break;
}
}
if (outBuf != null)
stringValue = outBuf.ToString();
return true;
}
/// <summary>
/// Expand a quote token by reading the expr and sticking it in a list after the appropriate symbol.
/// </summary>
/// <param name="tok">The quote token to expand.</param>
/// <param name="expr">The list of enclosed expressions.</param>
/// <returns>The expanded quote expression.</returns>
private SchemeObject Expand(TokType tok, SchemeObject expr)
{
var sym = QuoteTranslation[tok];
return List.MakeList(Symbol.New(sym, this.LineNumber), expr);
}
public bool IsTokOfType(TokType _tokenType)
{
return(tokenType == _tokenType);
}
