public void CsvTokenizerTest_DoubleQouteOnRecord()
{
using CsvTokenizer tokenizer = new CsvTokenizer(new StringReader("aaa\""));
CsvToken token = tokenizer.NextToken();
Assert.AreEqual(CsvTokenType.Eof, token.TokenType);
Assert.AreEqual("aaa\"", token.Value);
}
public CsvToken GetToken()
{
if (disposed)
{
throw new ObjectDisposedException("tokenizer");
}
StringBuilder text = null;
while (true)
{
switch (state)
{
case TokenizerState.Unknown:
{
input = reader.Read();
if (EndOfStream != input)
{
TextPosition = TextPosition.Begin();
//text = new StringBuilder();
state = TokenizerState.Reading;
break;
}
state = TokenizerState.EndOfDocument;
break;
}
case TokenizerState.FlushLastToken:
{
if (null != text && 0 < text.Length)
{
state = TokenizerState.EndOfDocument;
return(CsvToken.String(text.ToString()));
}
state = TokenizerState.Failed;
break;
}
case TokenizerState.EndOfDocument:
{
return(CsvToken.End);
}
case TokenizerState.Reading:
{
if (EndOfStream == input)
{
state = null == text ? TokenizerState.EndOfDocument : TokenizerState.FlushLastToken;
break;
}
if (null == text)
{
//TextPosition = TextPosition.Begin();
text = new StringBuilder();
}
var current = (char)input;
if (IsTerm(current))
{
if (0 < text.Length)
{
return(CsvToken.String(text.ToString()));
}
input = reader.Read();
return(CsvToken.Terminal(current));
}
text.Append(current);
input = reader.Read();
break;
}
default:
{
throw new Exception();
}
}
}
}
static CsvToken()
{
End = new CsvToken(CsvTokenType.End);
Unknown = new CsvToken(CsvTokenType.Unknown);
}