public void SetText(string text, int offset, bool keepLineNumbering) {
Text = text;
//For line-by-line input, automatically increment line# for every new line
var line = keepLineNumbering ? _location.Line + 1 : 0;
Location = new SourceLocation(offset, line, 0);
_nextNewLinePosition = text.IndexOfAny(_scannerData.LineTerminatorsArray, offset);
}
public override IEnumerable<Token> BeginFiltering(ParsingContext context, IEnumerable<Token> tokens)
{
this.pcontext = context;
foreach (Token t in tokens)
{
Console.WriteLine(" -> {0} ({1})", t, bracket_indent_level);
currentLoc = t.Location;
if (t.Terminal == grammar.Eof)
{
Console.WriteLine("CLI: {0}, CIL: {1}", current_line_indent, current_indent_level);
current_line_indent = 0;
while(current_indent_level > 0)
{
current_indent_level--;
Console.WriteLine(" <- DEDENT");
yield return new Token(grammar.Dedent,currentLoc,string.Empty,null);
}
yield return t;
break;
}
if (bracket_indent_level == 0)
{
while (ProcessToken(t)) { ;}
while (OutputTokens.Count > 0)
yield return OutputTokens.Pop();
continue;
}
if (t.Terminal == opening_bracket) { bracket_indent_level++; yield return new Token(grammar.Indent, currentLoc, string.Empty, null); continue; }
else if (t.Terminal == closing_bracket) { Debug.Assert(bracket_indent_level > 0); bracket_indent_level--; yield return new Token(grammar.Dedent, currentLoc, string.Empty, null); continue; }
yield return t;
}
}
public static SourceLocation ToLocation(this Irony.Parsing.SourceLocation srcLoc)
{
// somehow Irony's location line and column are zero based
return(new SourceLocation()
{
Line = srcLoc.Line + 1, Column = srcLoc.Column + 1
});
}
public override void Reset() {
base.Reset();
Indents.Clear();
Indents.Push(0);
OutputTokens.Clear();
PreviousToken = null;
CurrentToken = null;
PreviousTokenLocation = new SourceLocation();
}
public SourceStream(string text, bool caseSensitive, int tabWidth, SourceLocation initialLocation) {
_text = text;
_textLength = _text.Length;
_chars = Text.ToCharArray();
_stringComparison = caseSensitive ? StringComparison.InvariantCulture : StringComparison.InvariantCultureIgnoreCase;
_tabWidth = tabWidth;
_location = initialLocation;
_previewPosition = _location.Position;
if (_tabWidth <= 1)
_tabWidth = 8;
}
protected TextSpan CreateSpanFor(SourceLocation location, int endPosition, Source source)
{
var span = new TextSpan
{
iStartLine = location.Line,
iStartIndex = location.Column,
};
if (source != null)
source.GetLineIndexOfPosition(endPosition, out span.iEndLine, out span.iEndIndex);
return span;
}
public ScriptExtent(SourceSpan origSpan, int startOffset, int endOffset)
{
if (startOffset == 0 && endOffset == 0)
{
_span = origSpan;
return;
}
// create a new span with offsets
var location = origSpan.Location;
// we don't adjust line and column for artifical offsets. This is hopefully okay.
var newLocation = new SourceLocation(location.Position + startOffset, location.Line, location.Column);
// don't forget to subtract the start offset here
_span = new SourceSpan(newLocation, origSpan.Length + endOffset - startOffset);
}
public SourceStream(string text, bool caseSensitive, int tabWidth, SourceLocation initialLocation)
{
Text = text;
_textLength = Text.Length;
_chars = Text.ToCharArray();
#if DNXCORE50
_stringComparison = caseSensitive ? StringComparison.CurrentCulture : StringComparison.CurrentCultureIgnoreCase;
#else
_stringComparison = caseSensitive
? StringComparison.InvariantCulture
: StringComparison.InvariantCultureIgnoreCase;
#endif
_tabWidth = tabWidth;
Location = initialLocation;
PreviewPosition = Location.Position;
if (_tabWidth <= 1)
_tabWidth = 8;
}
internal void Reset()
{
CurrentParserState = Parser.InitialState;
CurrentParserInput = null;
CurrentCommentTokens = new TokenList();
ParserStack.Clear();
HasErrors = false;
ParserStack.Push(new ParseTreeNode(CurrentParserState));
CurrentParseTree = null;
OpenBraces.Clear();
ParserTrace.Clear();
CurrentTerminals.Clear();
CurrentToken = null;
PreviousToken = null;
PreviousLineStart = new SourceLocation(0, -1, 0);
BufferedTokens.Clear();
PreviewTokens.Clear();
Values.Clear();
foreach (var filter in TokenFilters)
filter.Reset();
}
/// <summary>
/// Begins the preview.
/// </summary>
public virtual void BeginPreview()
{
Context.Status = ParserStatus.Previewing;
previewTokens.Clear();
previewStartLocation = Location;
}
public ScriptException(string message, Exception inner, SourceLocation location, ScriptStackTrace stack)
: base(message, inner) {
Location = location;
ScriptStackTrace = stack;
}
public SourceSpan(SourceLocation location, int length) {
Location = location;
Length = length;
}
private void PushOutlineToken(Terminal term, SourceLocation location)
{
OutputTokens.Push(new Token(term, location, string.Empty, null));
}
public void AddParserMessage(ErrorLevel level, SourceLocation location, string message, params object[] args)
{
if (CurrentParseTree == null) return;
if (CurrentParseTree.ParserMessages.Count >= MaxErrors) return;
if (args != null && args.Length > 0)
message = string.Format(message, args);
CurrentParseTree.ParserMessages.Add(new LogMessage(level, location, message, CurrentParserState));
if (TracingEnabled)
AddTrace(true, message);
}
//Computes the Location info (line, col) for a new source position.
private void SetNewPosition(int newPosition) {
if (newPosition < Position)
throw new Exception(Resources.ErrCannotMoveBackInSource);
int p = Position;
int col = Location.Column;
int line = Location.Line;
while(p < newPosition) {
var curr = _chars[p];
switch (curr) {
case '\n': line++; col = 0; break;
case '\r': break;
case '\t': col = (col / _tabWidth + 1) * _tabWidth; break;
default: col++; break;
} //switch
p++;
}
Location = new SourceLocation(p, line, col);
}
//Switches Scanner into preview mode
public void BeginPreview()
{
Context.Status = ParserStatus.Previewing;
_previewStartLocation = Context.SourceStream.Location;
Context.PreviewTokens.Clear();
}
public static int ByLocation(ParserMessage x, ParserMessage y)
{
return(SourceLocation.Compare(x.Location, y.Location));
}
public void AddParserMessage(ParserErrorLevel level, SourceLocation location, string message, params object[] args) {
if (CurrentParseTree == null) return;
if (CurrentParseTree.ParserMessages.Count >= MaxErrors) return;
if (args != null && args.Length > 0)
message = string.Format(message, args);
CurrentParseTree.ParserMessages.Add(new ParserMessage(level, location, message, CurrentParserState));
if (OptionIsSet(ParseOptions.TraceParser))
ParserTrace.Add( new ParserTraceEntry(CurrentParserState, ParserStack.Top, CurrentParserInput, message, true));
}
private Token CompleteMatch(ParsingContext context, ISourceStream source, byte level)
{
string text = source.Text.Substring(source.PreviewPosition);
var matches = Regex.Matches(text, @"\](=*)\]");
foreach(Match match in matches)
{
if (match.Groups[1].Value.Length == (int)level)
{
source.PreviewPosition += match.Index + match.Length;
if (context.VsLineScanState.Value != 0)
{
SourceLocation tokenStart = new SourceLocation();
tokenStart.Position = 0;
string lexeme = source.Text.Substring(0, source.PreviewPosition);
context.VsLineScanState.Value = 0;
return new Token(this, tokenStart, lexeme, null);
}
else
{
return source.CreateToken(this.OutputTerminal);
}
}
}
context.VsLineScanState.TerminalIndex = this.MultilineIndex;
context.VsLineScanState.TokenSubType = level;
return null;
}
}//method
public void ResetLocationAndClearInput(SourceLocation location, int position) {
Context.CurrentParserInput = null;
Context.ParserInputStack.Clear();
Context.SetSourceLocation(location);
}
private void PushOutlineToken(Terminal term, SourceLocation location)
{
OutputTokens.Push(new Token(term, location, string.Empty, null));
}
private void ShowSourceLocation(SourceLocation location, int length)
{
if (location.Position < 0) return;
txtSource.SelectionStart = location.Position;
txtSource.SelectionLength = length;
//txtSource.Select(location.Position, length);
txtSource.ScrollToCaret();
if (tabGrammar.SelectedTab != pageTest)
tabGrammar.SelectedTab = pageTest;
txtSource.Focus();
//lblLoc.Text = location.ToString();
}
public void AddMessage(ErrorLevel level, SourceLocation location, string message, params object[] args) {
if (args != null && args.Length > 0)
message = string.Format(message, args);
Messages.Add(new LogMessage(level, location, message, null));
}
}//method
public void CopyMessages(ParserMessageList others, SourceLocation baseLocation, string messagePrefix) {
foreach(var other in others)
this.ParserMessages.Add(new ParserMessage(other.Level, baseLocation + other.Location, messagePrefix + other.Message, other.ParserState));
}//
private bool NeedLineStartToken(SourceLocation forLocation)
{
return(_grammar.FlagIsSet(LanguageFlags.EmitLineStartToken) && forLocation.Line > Context.PreviousLineStart.Line);
}
//Switches Scanner into preview mode
public void BeginPreview() {
Context.Status = ParserStatus.Previewing;
_previewStartLocation = Context.SourceStream.Location;
Context.PreviewTokens.Clear();
}
public RuntimeException(string message, Exception inner, SourceLocation location) : base(message, inner) {
Location = location;
}
}//method
public void ResetLocationAndClearInput(SourceLocation location, int position)
{
Context.CurrentParserInput = null;
Context.ParserInputStack.Clear();
Context.SetSourceLocation(location);
}
private bool NeedLineStartToken(SourceLocation forLocation) {
return Grammar.FlagIsSet(LanguageFlags.EmitLineStartToken) && forLocation.Line > Context.PreviousLineStart.Line;
}
} //method
private void SetCurrentToken(Token token)
{
_doubleEof = CurrentToken != null && CurrentToken.Terminal == _grammar.Eof
&& token.Terminal == _grammar.Eof;
//Copy CurrentToken to PreviousToken
if (CurrentToken != null && CurrentToken.Category == TokenCategory.Content)
{
//remember only content tokens
PreviousToken = CurrentToken;
_prevIsContinuation = _isContinuation;
_prevIsOperator = _isOperator;
if (PreviousToken != null)
PreviousTokenLocation = PreviousToken.Location;
}
CurrentToken = token;
_isContinuation = (token.Terminal == ContinuationTerminal && ContinuationTerminal != null);
_isOperator = token.Terminal.Flags.IsSet(TermFlags.IsOperator);
if (!_isContinuation)
OutputTokens.Push(token); //by default input token goes to output, except continuation symbol
}
//Switches Scanner into preview mode
public override void BeginPreview() {
base.BeginPreview();
_previewStartLocation = SourceStream.Location;
}
private Token CompleteMatch(ParsingContext context, ISourceStream source, byte commentLevel)
{
if (commentLevel == 0)
{
var line_breaks = new char[] { '\n', '\r', '\v' };
var firstCharPos = source.Text.IndexOfAny(line_breaks, source.PreviewPosition);
if (firstCharPos > 0)
{
source.PreviewPosition = firstCharPos;
}
else
{
source.PreviewPosition = source.Text.Length;
}
return source.CreateToken(this.OutputTerminal);
}
while (!source.EOF())
{
string text = source.Text.Substring(source.PreviewPosition);
var matches = Regex.Matches(text, @"\](=*)\]");
foreach (Match match in matches)
{
if (match.Groups[1].Value.Length == (int)commentLevel - 1)
{
source.PreviewPosition += match.Index + match.Length;
if (context.VsLineScanState.Value != 0)
{
SourceLocation tokenStart = new SourceLocation();
tokenStart.Position = 0;
string lexeme = source.Text.Substring(0, source.PreviewPosition);
context.VsLineScanState.Value = 0;
return new Token(this, tokenStart, lexeme, null);
}
else
{
return source.CreateToken(this.OutputTerminal);
}
}
}
source.PreviewPosition++;
}
context.VsLineScanState.TokenSubType = commentLevel;
return null;
}
public static int Compare(SourceLocation x, SourceLocation y) {
if (x.Position < y.Position) return -1;
if (x.Position == y.Position) return 0;
return 1;
}
public void VsSetSource(string text, int offset) {
var line = Context.Source==null ? 0 : Context.Source.Location.Line;
var newLoc = new SourceLocation(offset, line + 1, 0);
Context.Source = new SourceStream(text, Context.Language.Grammar.CaseSensitive, Context.TabWidth, newLoc);
}
private void ShowSourceLocationAndTraceToken(SourceLocation location, int length)
{
ShowSourceLocation(location, length);
//find token in trace
for (int i = 0; i < lstTokens.Items.Count; i++) {
var tkn = lstTokens.Items[i] as Token;
if (tkn.Location.Position == location.Position) {
lstTokens.SelectedIndex = i;
return;
}//if
}//for i
}
public void SetSourceLocation(SourceLocation location) {
foreach (var filter in TokenFilters)
filter.OnSetSourceLocation(location);
SourceStream.Location = location;
}
public MultiToken(Terminal term, SourceLocation location, TokenList childTokens) : base(term, location, string.Empty, null)
{
ChildTokens = childTokens;
}