public void AnalyzeTest(int blockSize)
{
var analyzer = new CharacterAnalyzer();
var result = (CharacterAnalyzerResult)analyzer.Analyze(DataStream, blockSize);
Assert.AreEqual(CharacterCount, result.CharacterCount.Count);
Assert.AreEqual(PunctuationCount, result.Punctuation.Counter.Count);
Assert.AreEqual(WhitespaceCount, result.Whitespace.Counter.Count);
Assert.AreEqual(UpperCount, result.UpperCase.Counter.Count);
Assert.AreEqual(OtherCount, result.Other.Counter.Count);
}
public void AnalyzeAsyncTest(int blockSize)
{
var analyzer = new CharacterAnalyzer();
var result = (CharacterAnalyzerResult)Task.Run(async () => await analyzer.AnalyzeAsync(DataStream, blockSize).ConfigureAwait(false)).Result;
Assert.AreEqual(CharacterCount, result.CharacterCount.Count);
Assert.AreEqual(PunctuationCount, result.Punctuation.Counter.Count);
Assert.AreEqual(WhitespaceCount, result.Whitespace.Counter.Count);
Assert.AreEqual(UpperCount, result.UpperCase.Counter.Count);
Assert.AreEqual(OtherCount, result.Other.Counter.Count);
}
/// <summary>
/// Check if a scalar is valid.
/// </summary>
private void AnalyzeScalar(string value)
{
bool block_indicators = false;
bool flow_indicators = false;
bool line_breaks = false;
bool special_characters = false;
bool leading_space = false;
bool leading_break = false;
bool trailing_space = false;
bool trailing_break = false;
bool break_space = false;
bool space_break = false;
bool previous_space = false;
bool previous_break = false;
scalarData.value = value;
if (value.Length == 0)
{
scalarData.isMultiline = false;
scalarData.isFlowPlainAllowed = false;
scalarData.isBlockPlainAllowed = true;
scalarData.isSingleQuotedAllowed = true;
scalarData.isBlockAllowed = false;
return;
}
if (value.StartsWith("---", StringComparison.Ordinal) || value.StartsWith("...", StringComparison.Ordinal))
{
block_indicators = true;
flow_indicators = true;
}
bool preceeded_by_whitespace = true;
CharacterAnalyzer<StringLookAheadBuffer> buffer = new CharacterAnalyzer<StringLookAheadBuffer>(new StringLookAheadBuffer(value));
bool followed_by_whitespace = buffer.IsBlankOrBreakOrZero(1);
bool isFirst = true;
while (!buffer.EndOfInput)
{
if (isFirst)
{
if (buffer.Check(@"#,[]{}&*!|>\""%@`"))
{
flow_indicators = true;
block_indicators = true;
}
if (buffer.Check("?:"))
{
flow_indicators = true;
if (followed_by_whitespace)
{
block_indicators = true;
}
}
if (buffer.Check('-') && followed_by_whitespace)
{
flow_indicators = true;
block_indicators = true;
}
}
else
{
if (buffer.Check(",?[]{}"))
{
flow_indicators = true;
}
if (buffer.Check(':'))
{
flow_indicators = true;
if (followed_by_whitespace)
{
block_indicators = true;
}
}
if (buffer.Check('#') && preceeded_by_whitespace)
{
flow_indicators = true;
block_indicators = true;
}
}
if (!buffer.IsPrintable() || (!buffer.IsAscii() && !IsUnicode))
{
special_characters = true;
}
if (buffer.IsBreak())
{
line_breaks = true;
}
if (buffer.IsSpace())
{
if(isFirst)
{
leading_space = true;
}
if(buffer.Buffer.Position >= buffer.Buffer.Length - 1)
{
trailing_space = true;
}
if(previous_break) {
break_space = true;
}
previous_space = true;
previous_break = false;
}
else if (buffer.IsBreak())
{
if(isFirst)
{
leading_break = true;
}
if(buffer.Buffer.Position >= buffer.Buffer.Length - 1)
{
trailing_break = true;
}
if (previous_space)
{
space_break = true;
}
previous_space = false;
previous_break = true;
}
else
{
previous_space = false;
previous_break = false;
}
preceeded_by_whitespace = buffer.IsBlankOrBreakOrZero();
buffer.Skip(1);
if (!buffer.EndOfInput)
{
followed_by_whitespace = buffer.IsBlankOrBreakOrZero(1);
}
isFirst = false;
}
scalarData.isMultiline = line_breaks;
scalarData.isFlowPlainAllowed = true;
scalarData.isBlockPlainAllowed = true;
scalarData.isSingleQuotedAllowed = true;
scalarData.isBlockAllowed = true;
if (leading_space || leading_break || trailing_space || trailing_break)
{
scalarData.isFlowPlainAllowed = false;
scalarData.isBlockPlainAllowed = false;
}
if(trailing_space)
{
scalarData.isBlockAllowed = false;
}
if (break_space)
{
scalarData.isFlowPlainAllowed = false;
scalarData.isBlockPlainAllowed = false;
scalarData.isSingleQuotedAllowed = false;
}
if (space_break || special_characters)
{
scalarData.isFlowPlainAllowed = false;
scalarData.isBlockPlainAllowed = false;
scalarData.isSingleQuotedAllowed = false;
scalarData.isBlockAllowed = false;
}
if (line_breaks)
{
scalarData.isFlowPlainAllowed = false;
scalarData.isBlockPlainAllowed = false;
}
if (flow_indicators)
{
scalarData.isFlowPlainAllowed = false;
}
if (block_indicators)
{
scalarData.isBlockPlainAllowed = false;
}
}
private void WriteBlockScalarHints(string value)
{
var analyzer = new CharacterAnalyzer<StringLookAheadBuffer>(new StringLookAheadBuffer(value));
if(analyzer.IsSpace() || analyzer.IsBreak()) {
string indent_hint = string.Format(CultureInfo.InvariantCulture, "{0}\0", bestIndent);
WriteIndicator(indent_hint, false, false, false);
}
isOpenEnded = false;
string chomp_hint = null;
if(value.Length == 0 || !analyzer.IsBreak(value.Length - 1)) {
chomp_hint = "-";
}
else if(value.Length >= 2 && analyzer.IsBreak(value.Length - 2)) {
chomp_hint = "+";
isOpenEnded = true;
}
if(chomp_hint != null){
WriteIndicator(chomp_hint, false, false, false);
}
}
private void AnalyzeScalar(Scalar scalar)
{
var value = scalar.Value;
scalarData.value = value;
if (value.Length == 0)
{
if (scalar.Tag == "tag:yaml.org,2002:null")
{
scalarData.isMultiline = false;
scalarData.isFlowPlainAllowed = false;
scalarData.isBlockPlainAllowed = true;
scalarData.isSingleQuotedAllowed = false;
scalarData.isBlockAllowed = false;
}
else
{
scalarData.isMultiline = false;
scalarData.isFlowPlainAllowed = false;
scalarData.isBlockPlainAllowed = false;
scalarData.isSingleQuotedAllowed = true;
scalarData.isBlockAllowed = false;
}
return;
}
var flowIndicators = false;
var blockIndicators = false;
if (value.StartsWith("---", StringComparison.Ordinal) || value.StartsWith("...", StringComparison.Ordinal))
{
flowIndicators = true;
blockIndicators = true;
}
var buffer = new CharacterAnalyzer<StringLookAheadBuffer>(new StringLookAheadBuffer(value));
var preceededByWhitespace = true;
var followedByWhitespace = buffer.IsWhiteBreakOrZero(1);
var leadingSpace = false;
var leadingBreak = false;
var trailingSpace = false;
var trailingBreak = false;
var breakSpace = false;
var spaceBreak = false;
var previousSpace = false;
var previousBreak = false;
var lineBreaks = false;
var specialCharacters = !ValueIsRepresentableInOutputEncoding(value);
var isFirst = true;
while (!buffer.EndOfInput)
{
if (isFirst)
{
if (buffer.Check(@"#,[]{}&*!|>\""%@`"))
{
flowIndicators = true;
blockIndicators = true;
}
if (buffer.Check("?:"))
{
flowIndicators = true;
if (followedByWhitespace)
blockIndicators = true;
}
if (buffer.Check('-') && followedByWhitespace)
{
flowIndicators = true;
blockIndicators = true;
}
}
else
{
if (buffer.Check(",?[]{}"))
flowIndicators = true;
if (buffer.Check(':'))
{
flowIndicators = true;
if (followedByWhitespace)
blockIndicators = true;
}
if (buffer.Check('#') && preceededByWhitespace)
{
flowIndicators = true;
blockIndicators = true;
}
}
if (!specialCharacters && !buffer.IsPrintable())
{
specialCharacters = true;
}
if (buffer.IsBreak())
{
lineBreaks = true;
}
if (buffer.IsSpace())
{
if (isFirst)
leadingSpace = true;
if (buffer.Buffer.Position >= buffer.Buffer.Length - 1)
trailingSpace = true;
if (previousBreak)
breakSpace = true;
previousSpace = true;
previousBreak = false;
}
else if (buffer.IsBreak())
{
if (isFirst)
leadingBreak = true;
if (buffer.Buffer.Position >= buffer.Buffer.Length - 1)
trailingBreak = true;
if (previousSpace)
spaceBreak = true;
previousSpace = false;
previousBreak = true;
}
else
{
previousSpace = false;
previousBreak = false;
}
preceededByWhitespace = buffer.IsWhiteBreakOrZero();
buffer.Skip(1);
if (!buffer.EndOfInput)
followedByWhitespace = buffer.IsWhiteBreakOrZero(1);
isFirst = false;
}
scalarData.isFlowPlainAllowed = true;
scalarData.isBlockPlainAllowed = true;
scalarData.isSingleQuotedAllowed = true;
scalarData.isBlockAllowed = true;
if (leadingSpace || leadingBreak || trailingSpace || trailingBreak)
{
scalarData.isFlowPlainAllowed = false;
scalarData.isBlockPlainAllowed = false;
}
if (trailingSpace)
scalarData.isBlockAllowed = false;
if (breakSpace)
{
scalarData.isFlowPlainAllowed = false;
scalarData.isBlockPlainAllowed = false;
scalarData.isSingleQuotedAllowed = false;
}
if (spaceBreak || specialCharacters)
{
scalarData.isFlowPlainAllowed = false;
scalarData.isBlockPlainAllowed = false;
scalarData.isSingleQuotedAllowed = false;
scalarData.isBlockAllowed = false;
}
scalarData.isMultiline = lineBreaks;
if (lineBreaks)
{
scalarData.isFlowPlainAllowed = false;
scalarData.isBlockPlainAllowed = false;
}
if (flowIndicators)
scalarData.isFlowPlainAllowed = false;
if (blockIndicators)
scalarData.isBlockPlainAllowed = false;
}
/// <summary>
/// Initializes a new instance of the <see cref="Scanner"/> class.
/// </summary>
/// <param name="input">The input.</param>
public Scanner(TextReader input)
{
analyzer = new CharacterAnalyzer<LookAheadBuffer>(new LookAheadBuffer(input, MaxBufferLength));
mark.Column = 0;
mark.Line = 0;
}
/// <summary>
/// Initializes a new instance of the <see cref="Scanner"/> class.
/// </summary>
/// <param name="input">The input.</param>
/// <param name="skipComments">Indicates whether comments should be ignored</param>
public Scanner(TextReader input, bool skipComments = true)
{
analyzer = new CharacterAnalyzer<LookAheadBuffer>(new LookAheadBuffer(input, MaxBufferLength));
cursor = new Cursor();
SkipComments = skipComments;
}
/// <summary>
/// Initializes a new instance of the <see cref="Scanner"/> class.
/// </summary>
/// <param name="input">The input.</param>
public Scanner(TextReader input)
{
analyzer = new CharacterAnalyzer<LookAheadBuffer>(new LookAheadBuffer(input, MaxBufferLength));
cursor = new Cursor();
}
