/// <summary>
/// Start tokenization from string.
/// </summary>
/// <param name="text"></param>
/// <param name="settings"></param>
/// <returns></returns>
public static Tokenizer FromString
(
string text,
TokenizerSettings settings)
{
TextReader reader = new StringReader(text);
return(new Tokenizer(reader, settings));
}
/// <summary>
/// Start tokenization from stream.
/// </summary>
/// <param name="stream"></param>
/// <param name="encoding"></param>
/// <param name="settings"></param>
/// <returns></returns>
public static Tokenizer FromStream
(
Stream stream,
Encoding encoding,
TokenizerSettings settings)
{
StreamReader reader = new StreamReader(stream, encoding);
return(new Tokenizer(reader, settings));
}
/// <summary>
/// Start tokenization from memory.
/// </summary>
/// <param name="bytes"></param>
/// <param name="encoding"></param>
/// <param name="settings"></param>
/// <returns></returns>
public static Tokenizer FromMemory
(
byte[] bytes,
Encoding encoding,
TokenizerSettings settings)
{
MemoryStream stream = new MemoryStream(bytes, false);
StreamReader reader = new StreamReader(stream, encoding);
return(new Tokenizer(reader, settings));
}
/// <summary>
/// Start tokenization from file.
/// </summary>
/// <param name="fileName"></param>
/// <param name="encoding"></param>
/// <param name="settings"></param>
/// <returns></returns>
public static Tokenizer FromFile
(
string fileName,
Encoding encoding,
TokenizerSettings settings)
{
StreamReader reader = new StreamReader(fileName, encoding);
Tokenizer result = new Tokenizer(reader, settings);
result._ownReader = true;
return(result);
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="reader"></param>
/// <param name="settings"></param>
public Tokenizer
(
TextReader reader,
TokenizerSettings settings)
{
if (reader == null)
{
throw new ArgumentNullException();
}
if (settings == null)
{
settings = TokenizerSettings.Default;
}
_reader = reader;
_settings = settings;
}
