internal Tokenizer(char[] array)
{
this.BasicInitialization();
this._inChars = array;
this._inSize = array.Length;
this._inTokenSource = Tokenizer.TokenSource.CharArray;
}
internal Tokenizer(string input)
{
this.BasicInitialization();
this._inString = input;
this._inSize = input.Length;
this._inTokenSource = Tokenizer.TokenSource.String;
}
// Token: 0x06002BA8 RID: 11176 RVA: 0x000A3C4C File Offset: 0x000A1E4C
internal void ChangeFormat(Encoding encoding)
{
if (encoding == null)
{
return;
}
Tokenizer.TokenSource inTokenSource = this._inTokenSource;
if (inTokenSource > Tokenizer.TokenSource.ASCIIByteArray)
{
if (inTokenSource - Tokenizer.TokenSource.CharArray <= 2)
{
return;
}
}
else
{
if (encoding == Encoding.Unicode)
{
this._inTokenSource = Tokenizer.TokenSource.UnicodeByteArray;
return;
}
if (encoding == Encoding.UTF8)
{
this._inTokenSource = Tokenizer.TokenSource.UTF8ByteArray;
return;
}
if (encoding == Encoding.ASCII)
{
this._inTokenSource = Tokenizer.TokenSource.ASCIIByteArray;
return;
}
}
inTokenSource = this._inTokenSource;
Stream stream;
if (inTokenSource > Tokenizer.TokenSource.ASCIIByteArray)
{
if (inTokenSource - Tokenizer.TokenSource.CharArray <= 2)
{
return;
}
Tokenizer.StreamTokenReader streamTokenReader = this._inTokenReader as Tokenizer.StreamTokenReader;
if (streamTokenReader == null)
{
return;
}
stream = streamTokenReader._in.BaseStream;
string s = new string(' ', streamTokenReader.NumCharEncountered);
stream.Position = (long)streamTokenReader._in.CurrentEncoding.GetByteCount(s);
}
else
{
stream = new MemoryStream(this._inBytes, this._inIndex, this._inSize - this._inIndex);
}
this._inTokenReader = new Tokenizer.StreamTokenReader(new StreamReader(stream, encoding));
this._inTokenSource = Tokenizer.TokenSource.Other;
}
internal Tokenizer(string input, string[] searchStrings, string[] replaceStrings)
{
this.BasicInitialization();
this._inString = input;
this._inSize = this._inString.Length;
this._inTokenSource = Tokenizer.TokenSource.NestedStrings;
this._searchStrings = searchStrings;
this._replaceStrings = replaceStrings;
}
internal void BasicInitialization()
{
this.LineNo = 1;
this._inProcessingTag = 0;
this._inSavedCharacter = -1;
this._inIndex = 0;
this._inSize = 0;
this._inNestedSize = 0;
this._inNestedIndex = 0;
this._inTokenSource = Tokenizer.TokenSource.Other;
this._maker = SharedStatics.GetSharedStringMaker();
}
internal Tokenizer(byte[] array, Tokenizer.ByteTokenEncoding encoding, int startIndex)
{
this.BasicInitialization();
this._inBytes = array;
this._inSize = array.Length;
this._inIndex = startIndex;
switch (encoding)
{
case Tokenizer.ByteTokenEncoding.UnicodeTokens:
this._inTokenSource = Tokenizer.TokenSource.UnicodeByteArray;
break;
case Tokenizer.ByteTokenEncoding.UTF8Tokens:
this._inTokenSource = Tokenizer.TokenSource.UTF8ByteArray;
break;
case Tokenizer.ByteTokenEncoding.ByteTokens:
this._inTokenSource = Tokenizer.TokenSource.ASCIIByteArray;
break;
default:
throw new ArgumentException(Environment.GetResourceString("Arg_EnumIllegalVal", (object)encoding));
}
}
internal void ChangeFormat(Encoding encoding)
{
if (encoding == null)
{
return;
}
switch (this._inTokenSource)
{
case Tokenizer.TokenSource.UnicodeByteArray:
case Tokenizer.TokenSource.UTF8ByteArray:
case Tokenizer.TokenSource.ASCIIByteArray:
if (encoding == Encoding.Unicode)
{
this._inTokenSource = Tokenizer.TokenSource.UnicodeByteArray;
return;
}
if (encoding == Encoding.UTF8)
{
this._inTokenSource = Tokenizer.TokenSource.UTF8ByteArray;
return;
}
if (encoding == Encoding.ASCII)
{
this._inTokenSource = Tokenizer.TokenSource.ASCIIByteArray;
return;
}
break;
case Tokenizer.TokenSource.CharArray:
return;
case Tokenizer.TokenSource.String:
return;
case Tokenizer.TokenSource.NestedStrings:
return;
}
Stream stream;
switch (this._inTokenSource)
{
case Tokenizer.TokenSource.UnicodeByteArray:
case Tokenizer.TokenSource.UTF8ByteArray:
case Tokenizer.TokenSource.ASCIIByteArray:
stream = (Stream) new MemoryStream(this._inBytes, this._inIndex, this._inSize - this._inIndex);
break;
case Tokenizer.TokenSource.CharArray:
return;
case Tokenizer.TokenSource.String:
return;
case Tokenizer.TokenSource.NestedStrings:
return;
default:
Tokenizer.StreamTokenReader streamTokenReader = this._inTokenReader as Tokenizer.StreamTokenReader;
if (streamTokenReader == null)
{
return;
}
stream = streamTokenReader._in.BaseStream;
string s = new string(' ', streamTokenReader.NumCharEncountered);
stream.Position = (long)streamTokenReader._in.CurrentEncoding.GetByteCount(s);
break;
}
this._inTokenReader = (Tokenizer.ITokenReader) new Tokenizer.StreamTokenReader(new StreamReader(stream, encoding));
this._inTokenSource = Tokenizer.TokenSource.Other;
}
