/// <summary>
/// Get the next token in the stream. Returns an EOF at the end of document.
/// </summary>
/// <returns></returns>
public YamlToken ReadNext()
{
var pair = ReadNextWorker();
_lastPair = pair;
return(pair);
}
internal YamlToken CheckDuplicate(string propName, int currentLine)
{
int oldLine;
if (_previousProperties.TryGetValue(propName, out oldLine))
{
// Key is already present.
return(YamlToken.NewError(default(SourceLocation), $"Property '{propName}' is already defined on line {oldLine}."));
}
_previousProperties.Add(propName, currentLine);
return(null);
}
public YamlLexer(TextReader source, string filenameHint = null)
{
_reader = source;
_currentFileName = filenameHint; // For errort reporting
// We pretend the file is wrapped in a "object:" tag.
_lastPair = YamlToken.NewStartObj(default(SourceLocation), null);
_currentIndent = new Stack <Indent>()
{
new Indent
{
_oldIndentLevel = -1,
_lineStart = 0
}
};
}
private YamlToken Error(LineParser line, string message)
{
return(YamlToken.NewError(this.Loc(line), message));
}
private YamlToken ReadNextWorker()
{
// Warn on:
// # comment
// no escape (= or |)
// unallowed multiline escape
// blank lines
// different indent size
// duplicate keys
// empty properties (should have an indent)
// no tabs.
// don't allow --- documents
// Start of line.
// [Indent] [PropertyName] [Colon]
// [Indent] [PropertyName] [Colon] [space] [equals] [VALUE]
// [Indent] [PropertyName] [Colon] [space] [MultilineEscape]
LineParser line;
int indentLen;
while (true)
{
line = PeekLine();
if (line == null)
{
// End of File.
// Close any outstanding objects.
if (_currentIndent.Count > 1)
{
_currentIndent.Pop();
return(YamlToken.EndObj);
}
return(YamlToken.EndOfFile);
}
// get starting indent.
indentLen = line.EatIndent();
// Comment indent level doesn't matter - may not match object indent.
if (line.Current == '#')
{
// https://github.com/microsoft/PowerApps-Language-Tooling/issues/115
// Allow comment lines. These will get stripped (won't roundtrip).
if (!_commentStrippedWarning.HasValue)
{
_commentStrippedWarning = Loc(line);
}
MoveNextLine();
continue;
}
if (line.Current != 0)
{
break;
}
// Eat the newline and go to the next line.
MoveNextLine();
}
if (line.Current == '-')
{
return(Unsupported(line, "--- markers are not supported. Only a single document per file."));
}
if (line.Current == '\t')
{
// Helpful warning.
return(Error(line, "Use spaces, not tabs."));
}
var startColumn = line._idx + 1; // 1-based start column number.
// If last was 'start object', then this indent sets the new level,
if (_lastPair.Kind == YamlTokenKind.StartObj)
{
var lastIndent = _currentIndent.Peek()._oldIndentLevel;
if (indentLen == lastIndent) // Close immediate parent
{
_currentIndent.Pop();
return(YamlToken.EndObj);
}
else if (indentLen < lastIndent)
{
// Close current objects one at a time.
_currentIndent.Pop();
var prevIndent = _currentIndent.Peek()._oldIndentLevel;
// Indent must exactly match a previous one up the stack.
if (indentLen > prevIndent)
{
return(Error(line, "Property indent must align exactly with a previous indent level."));
}
return(YamlToken.EndObj);
}
if (indentLen <= lastIndent)
{
// Error. new object should be indented.
return(Unsupported(line, "Can't have null properties. Line should be indented to start a new property."));
}
_currentIndent.Peek()._newIndentLevel = indentLen;
}
else
{
// Subsequent properties should be at same indentation level.
var expectedIndent = _currentIndent.Peek()._newIndentLevel;
if (indentLen == expectedIndent)
{
// Good. Common case.
// Continue processing below to actually parse this property.
}
else if (indentLen > expectedIndent)
{
return(Error(line, "Property should be at same indent level"));
}
else
{
// Closing an object.
// Indent must exactly match a previous one up the stack.
if (indentLen > _currentIndent.Peek()._oldIndentLevel)
{
return(Error(line, "Property indent must align exactly with a previous indent level."));
}
// Close current objects one at a time.
_currentIndent.Pop();
return(YamlToken.EndObj);
}
}
// Get identifier
// toggle isEscaped on every ' appearance
bool isEscaped = false;
bool requiresClosingDoubleQuote = line.MaybeEat('"');
while (line.Current != ':' || isEscaped || requiresClosingDoubleQuote)
{
if (line.Current == '\'')
{
isEscaped = !isEscaped;
}
if (requiresClosingDoubleQuote && line.Current == '"' && line.Previous != '\\')
{
line._idx++;
break;
}
if (line.Current == 0) // EOL
{
if (isEscaped)
{
return(Unsupported(line, "Missing closing \'."));
}
if (requiresClosingDoubleQuote)
{
return(Unsupported(line, "Missing closing \"."));
}
return(Unsupported(line, "Missing ':'. If this is a multiline property, use |"));
}
line._idx++;
}
line.MaybeEat(':'); // skip colon.
// Prop name could have spaces, but no colons.
var propName = line._line.Substring(indentLen, line._idx - indentLen - 1).Trim();
if (requiresClosingDoubleQuote)
{
propName = propName.Replace("\\\"", "\"").Trim('"');
}
// If it's a property, must have at least 1 space.
// If it's start object, then ignore all spaces.
int iSpaces = 0;
while (line.MaybeEat(' '))
{
iSpaces++; // skip optional spaces
}
if (line.Current == 0) // EOL
{
var error = _currentIndent.Peek().CheckDuplicate(propName, _currentLine);
if (error != null)
{
error.Span = Loc(line);
return(error);
}
// New Object.
// Next line must begin an indent.
_currentIndent.Push(new Indent {
_lineStart = _currentLine,
_oldIndentLevel = indentLen
// newIndentLevel will be set at next line
});
MoveNextLine();
return(YamlToken.NewStartObj(Loc(startColumn, line), propName));
}
if (line.Current == '#')
{
return(UnsupportedComment(line));
}
if (iSpaces == 0)
{
return(Error(line, "Must have at least 1 space after colon."));
}
// Escape must be
string value = null;
if (line.MaybeEat('='))
{
// Single line. Property doesn't include \n at end.
value = line.RestOfLine;
if (value.IndexOf('#') >= 0)
{
return(UnsupportedComment(line));
}
MoveNextLine();
}
else if ((line.Current == '\"') || (line.Current == '\''))
{
// These are common YAml sequences, but extremely problematic and could be user error.
// Disallow them and force the user to explicit.
// Is "hello" a string or identifer?
// Foo: "Hello"
//
// Instead, have the user write:
// Foo: ="Hello" // String
// Foo: =Hello // identifier
// Foo: |
// ="Hello" // string
return(Unsupported(line, "Quote is not a supported escape sequence. Use = or |"));
}
else if (line.MaybeEat('>'))
{
// Unsupported Multiline escape.
return(Unsupported(line, "> is not a supported multiline escape. Use |"));
}
else if (line.MaybeEat('|'))
{
// Multiline
int multilineMode;
if (line.MaybeEat('-'))
{
multilineMode = 0; // 0 newlines at end
}
else if (line.MaybeEat('+'))
{
multilineMode = 2; // 1+ newlines.
}
else
{
multilineMode = 1; // exactly 1 newline at end.
}
iSpaces = 0;
while (line.MaybeEat(' '))
{
iSpaces++; // skip optional spaces
}
if (line.Current == '#')
{
return(UnsupportedComment(line));
}
else if (line.Current != 0) // EOL, catch all error.
{
return(Error(line, "Content for | escape must start on next line."));
}
MoveNextLine();
value = ReadMultiline(multilineMode);
if (value.Length == 0)
{
return(Unsupported(line, "Can't have empty multiline expressions."));
}
if (value[0] != '=')
{
return(Unsupported(line, "Property value must start with an '='"));
}
value = value.Substring(1); // move past '='
}
else
{
// Warn on legal yaml escapes (>) that we don't support in our subset here.
return(Error(line, "Expected either '=' for a single line expression or '|' to begin a multiline expression"));
}
{
var error = _currentIndent.Peek().CheckDuplicate(propName, _currentLine);
if (error != null)
{
error.Span = Loc(line);
return(error);
}
}
int endIndex = line._line.Length + 1;
return(YamlToken.NewProperty(LocWorker(startColumn, endIndex), propName, value));
}
