public override TokenInfo ReadToken()
{
Token token = tokenizer.GetNext ();
SourceLocation location = new SourceLocation(token.StartPosition,token.StartLine,token.StartColumn);
SS.SourceSpan span = new SS.SourceSpan (ConvertToSSSrcLocation(location), ConvertToSSSrcLocation(tokenizer.Position));
TokenTriggers trigger = GetTrigger(token.Kind);
TokenCategory category = GetCategory (token.Kind);
return new TokenInfo (span, category, trigger);
}
public HappySourceLocation(SourceUnit unit, SourceLocation start, SourceLocation end)
{
ContractUtils.RequiresNotNull(unit, "unit");
ContractUtils.RequiresNotNull(start, "start");
ContractUtils.RequiresNotNull(end, "end");
//ContractUtils.Requires(isBefore(start, end), "start and end must be well ordered");
if(!isBefore(start, end))
throw new ApplicationException("start and end must be well ordered");
this.Unit = unit;
this.Span = new SourceSpan(start, end);
}
private static void ValidateLocations(SourceLocation start, SourceLocation end) {
if (start.IsValid && end.IsValid) {
if (start > end) {
throw new ArgumentException("Start and End must be well ordered");
}
} else {
if (start.IsValid || end.IsValid) {
throw new ArgumentException("Start and End must both be valid or both invalid");
}
}
}
public void Init()
{
mockCompiler = new MockPythonCompiler();
compiler = new DummyPythonCompilerTask(mockCompiler, @"C:\Projects\MyProject");
compiler.TargetType = "Exe";
compiler.OutputAssembly = "test.exe";
TaskItem sourceFile = new TaskItem(@"D:\Projects\MyProject\PythonApp.Program.py");
compiler.Sources = new ITaskItem[] {sourceFile};
SourceUnit source = DefaultContext.DefaultPythonContext.CreateSourceUnit(NullTextContentProvider.Null, @"PythonApp\Program", SourceCodeKind.InteractiveCode);
SourceLocation start = new SourceLocation(0, 1, 1);
SourceLocation end = new SourceLocation(0, 2, 3);
SourceSpan span = new SourceSpan(start, end);
SyntaxErrorException ex = new SyntaxErrorException("Error", source, span, 1000, Severity.FatalError);
mockCompiler.ThrowExceptionAtCompile = ex;
success = compiler.Execute();
}
public MSAst.Expression TransformMaybeSingleLineSuite(Statement body, SourceLocation prevStart) {
MSAst.Expression res;
if (body.Start.Line == prevStart.Line) {
// avoid creating and throwing away as much line number goo as we can...
res = body.Transform(this);
} else {
res = Transform(body);
}
MSAst.BlockExpression block = res as MSAst.BlockExpression;
if (block != null && block.Expressions.Count > 0) {
MSAst.DebugInfoExpression dbgInfo = block.Expressions[0] as MSAst.DebugInfoExpression;
// body on the same line as an if, don't generate a 2nd sequence point
if (dbgInfo != null && dbgInfo.StartLine == prevStart.Line) {
// we remove the debug info based upon how it's generated in DebugStatement.AddDebugInfo which is
// the helper method which adds the debug info.
if (block.Type == typeof(void)) {
Debug.Assert(block.Expressions.Count == 3);
Debug.Assert(block.Expressions[2] is MSAst.DebugInfoExpression && ((MSAst.DebugInfoExpression)block.Expressions[2]).IsClear);
res = block.Expressions[1];
} else {
Debug.Assert(block.Expressions.Count == 4);
Debug.Assert(block.Expressions[3] is MSAst.DebugInfoExpression && ((MSAst.DebugInfoExpression)block.Expressions[2]).IsClear);
Debug.Assert(block.Expressions[1] is MSAst.BinaryExpression && ((MSAst.BinaryExpression)block.Expressions[2]).NodeType == MSAst.ExpressionType.Assign);
res = ((MSAst.BinaryExpression)block.Expressions[1]).Right;
}
}
}
if (res.Type != typeof(void)) {
res = AstUtils.Void(res);
}
return res;
}
internal static MSAst.Expression TransformForStatement(AstGenerator ag, MSAst.ParameterExpression enumerator,
Expression list, Expression left, MSAst.Expression body,
Statement else_, SourceSpan span, SourceLocation header,
MSAst.LabelTarget breakLabel, MSAst.LabelTarget continueLabel) {
// enumerator = PythonOps.GetEnumeratorForIteration(list)
MSAst.Expression init = Ast.Assign(
enumerator,
ag.Operation(
typeof(IEnumerator),
PythonOperationKind.GetEnumeratorForIteration,
ag.TransformAsObject(list)
)
);
// while enumerator.MoveNext():
// left = enumerator.Current
// body
// else:
// else
MSAst.Expression ls = AstUtils.Loop(
ag.AddDebugInfo(Ast.Call(
enumerator,
typeof(IEnumerator).GetMethod("MoveNext")
), left.Span),
null,
Ast.Block(
left.TransformSet(
ag,
SourceSpan.None,
Ast.Call(
enumerator,
typeof(IEnumerator).GetProperty("Current").GetGetMethod()
),
PythonOperationKind.None
),
body,
ag.UpdateLineNumber(list.Start.Line),
AstUtils.Empty()
),
ag.Transform(else_),
breakLabel,
continueLabel
);
return Ast.Block(
init,
ls,
AstUtils.Empty()
);
}
internal MSAst.Expression/*!*/ AddDebugInfo(MSAst.Expression/*!*/ expression, SourceLocation start, SourceLocation end) {
if (PyContext.PythonOptions.GCStress != null) {
expression = Ast.Block(
Ast.Call(
typeof(GC).GetMethod("Collect", new[] { typeof(int) }),
Ast.Constant(PyContext.PythonOptions.GCStress.Value)
),
expression
);
}
return Utils.AddDebugInfo(expression, _document, start, end);
}
internal void ReportSyntaxError(SourceLocation start, SourceLocation end, string message, int errorCode) {
// save the first one, the next error codes may be induced errors:
if (_errorCode == 0) {
_errorCode = errorCode;
}
_errors.Add(_sourceUnit, message, new SourceSpan(start, end), errorCode, Severity.FatalError);
}
public SourceLocation MakeLocation(SourceLocation loc)
{
return(new SourceLocation(loc.Index, MapLine(loc.Line), loc.Column));
}
public override void Initialize(object state, TextReader/*!*/ reader, SourceUnit sourceUnit, SourceLocation initialLocation) {
ContractUtils.RequiresNotNull(reader, "reader");
_sourceUnit = sourceUnit;
_input = reader;
_initialLocation = initialLocation;
_currentLine = _initialLocation.Line;
_currentLineIndex = _initialLocation.Index;
_tokenSpan = new SourceSpan(initialLocation, initialLocation);
SetState(LexicalState.EXPR_BEG);
_tokenValue = new TokenValue();
_eofReached = false;
_unterminatedToken = false;
DumpBeginningOfUnit();
}
private Expression FinishSlice(Expression e0, SourceLocation start) {
Expression e1 = null;
Expression e2 = null;
bool stepProvided = false;
switch (PeekToken().Kind) {
case TokenKind.Comma:
case TokenKind.RightBracket:
break;
case TokenKind.Colon:
// x[?::?]
stepProvided = true;
NextToken();
e2 = ParseSliceEnd();
break;
default:
// x[?:val:?]
e1 = ParseExpression();
if (MaybeEat(TokenKind.Colon)) {
stepProvided = true;
e2 = ParseSliceEnd();
}
break;
}
SliceExpression ret = new SliceExpression(e0, e1, e2, stepProvided);
ret.SetLoc(start, GetEnd());
return ret;
}
public Token OutputBuffer(Symbol symbol)
{
var loc1 = new SourceLocation(storedIndex, storedLine, storedColumn);
var loc2 = new SourceLocation(index, Line, Column);
return new Token(symbol, new SourceSpan(loc1, loc2), buffer.ToString());
}
Token ExpectMatch(Symbol right, Symbol left, SourceLocation leftStart)
{
if (Current.Symbol != right)
{
if (Current.Line == leftStart.Line)
{
throw ReportSyntaxErrorNear("'{0}' expected",
right.ToTokenString());
}
throw ReportSyntaxErrorNear("'{0}' expected (to close '{1}' at line {2})",
right.ToTokenString(),
left.ToTokenString(),
leftStart.Line);
}
return Consume();
}
/// <summary>
/// Gets the body region for a class or a method.
/// </summary>
/// <remarks>
/// Note that SharpDevelop line numbers are zero based but the
/// DomRegion values are one based. IronPython columns and lines are one based.
/// </remarks>
/// <param name="body">The body statement.</param>
/// <param name="header">The location of the header. This gives the end location for the
/// method or class definition up to the colon.</param>
public static DomRegion GetBodyRegion(Statement body, SourceLocation header)
{
int columnAfterColonCharacter = header.Column + 1;
return new DomRegion(header.Line, header.Column + 1, body.End.Line, body.End.Column);
}
/// <summary>
/// Constructs a new span with a specific start and end location.
/// </summary>
/// <param name="start">The beginning of the span.</param>
/// <param name="end">The end of the span.</param>
public SourceSpan(SourceLocation start, SourceLocation end) {
ValidateLocations(start, end);
this._start = start;
this._end = end;
}
private void MarkTokenStart() {
_currentTokenStart = GetCurrentLocation();
_currentTokenStartIndex = _bufferPos;
}
private void MarkMultiLineTokenEnd() {
_currentTokenEnd = GetCurrentLocation();
}
private void MarkSingleLineTokenEnd() {
Debug.Assert(_lineBuffer == null || Array.IndexOf(_lineBuffer, '\n', _currentTokenStartIndex, _bufferPos - _currentTokenStartIndex) == -1);
_currentTokenEnd = GetCurrentLocation();
}
internal MSAst.Expression TransformLoopBody(Statement body, SourceLocation headerStart, out LabelTarget breakLabel, out LabelTarget continueLabel) {
// Save state
bool savedInFinally = _inFinally;
LabelTarget savedBreakLabel = _breakLabel;
LabelTarget savedContinueLabel = _continueLabel;
int loopId = ++_loopOrFinallyId;
LoopOrFinallyIds.Add(loopId, false);
_inFinally = false;
breakLabel = _breakLabel = Ast.Label("break");
continueLabel = _continueLabel = Ast.Label("continue");
MSAst.Expression result;
try {
result = TransformMaybeSingleLineSuite(body, headerStart);
} finally {
_inFinally = savedInFinally;
_breakLabel = savedBreakLabel;
_continueLabel = savedContinueLabel;
LoopOrFinallyIds.Remove(loopId);
}
return result;
}
public void SetLoc(SourceLocation start, SourceLocation header, SourceLocation end) {
Start = start;
_header = header;
End = end;
}
public Token Output(Symbol symbol)
{
var loc = new SourceLocation(index, Line, Column);
return new Token(symbol, new SourceSpan(loc, loc));
}
public void AddError(string message, SourceLocation start, SourceLocation end, Severity severity, int errorCode)
{
_errors.Add(SourceUnit, message, new SourceSpan(start, end), errorCode, severity);
}
internal Expression/*!*/ AddDebugInfo(Expression/*!*/ expression, SourceLocation start, SourceLocation end)
{
//if (TotemContext.PythonOptions.GCStress != null)
//{
// expression = Ast.Block(
// Ast.Call(
// typeof(GC).GetMethod("Collect", new[] { typeof(int) }),
// Ast.Constant(PyContext.PythonOptions.GCStress.Value)
// ),
// expression
// );
//}
return Utils.AddDebugInfo(expression, Document, start, end);
}
public void SetLoc(SourceLocation start, SourceLocation end) {
_start = start;
_end = end;
}
internal MSAst.Expression/*!*/ AddDebugInfo(MSAst.Expression/*!*/ expression, SourceLocation start, SourceLocation end) {
return Utils.AddDebugInfo(expression, _document, start, end);
}
public void SetLoc(SourceSpan span) {
_start = span.Start;
_end = span.End;
}
internal static MSAst.Expression TransformMaybeSingleLineSuite(Statement body, SourceLocation prevStart) {
if (body.GlobalParent.IndexToLocation(body.StartIndex).Line != prevStart.Line) {
return body;
}
MSAst.Expression res = body.Reduce();
res = RemoveDebugInfo(prevStart.Line, res);
if (res.Type != typeof(void)) {
res = AstUtils.Void(res);
}
return res;
}
public PythonProperty(IClass declaringType, string name, SourceLocation location)
: base(declaringType, name)
{
GetPropertyRegion(location);
declaringType.Properties.Add(this);
}
internal void ReportSyntaxError(SourceLocation start, SourceLocation end, string message) {
ReportSyntaxError(start, end, message, ErrorCodes.SyntaxError);
}
void GetPropertyRegion(SourceLocation location)
{
int line = location.Line;
int column = location.Column;
Region = new DomRegion(line, column, line, column);
}
internal static MSAst.Expression TransformFor(ScopeStatement parent, MSAst.ParameterExpression enumerator,
Expression list, Expression left, MSAst.Expression body,
Statement else_, SourceSpan span, SourceLocation header,
MSAst.LabelTarget breakLabel, MSAst.LabelTarget continueLabel, bool isStatement) {
// enumerator, isDisposable = Dynamic(GetEnumeratorBinder, list)
MSAst.Expression init = Ast.Assign(
enumerator,
new PythonDynamicExpression1<KeyValuePair<IEnumerator, IDisposable>>(
Binders.UnaryOperationBinder(
parent.GlobalParent.PyContext,
PythonOperationKind.GetEnumeratorForIteration
),
parent.GlobalParent.CompilationMode,
AstUtils.Convert(list, typeof(object))
)
);
// while enumerator.MoveNext():
// left = enumerator.Current
// body
// else:
// else
MSAst.Expression ls = AstUtils.Loop(
parent.GlobalParent.AddDebugInfo(
Ast.Call(
Ast.Property(
enumerator,
typeof(KeyValuePair<IEnumerator, IDisposable>).GetProperty("Key")
),
typeof(IEnumerator).GetMethod("MoveNext")
),
left.Span
),
null,
Ast.Block(
left.TransformSet(
SourceSpan.None,
Ast.Call(
Ast.Property(
enumerator,
typeof(KeyValuePair<IEnumerator, IDisposable>).GetProperty("Key")
),
typeof(IEnumerator).GetProperty("Current").GetGetMethod()
),
PythonOperationKind.None
),
body,
isStatement ? UpdateLineNumber(parent.GlobalParent.IndexToLocation(list.StartIndex).Line) : AstUtils.Empty(),
AstUtils.Empty()
),
else_,
breakLabel,
continueLabel
);
return Ast.Block(
init,
Ast.TryFinally(
ls,
Ast.Call(AstMethods.ForLoopDispose, enumerator)
)
);
}
/// <summary>
/// Constructs a new span with a specific start and end location.
/// </summary>
/// <param name="start">The beginning of the span.</param>
/// <param name="end">The end of the span.</param>
public SourceSpan(SourceLocation start, SourceLocation end)
{
ValidateLocations(start, end);
Start = start;
End = end;
}