private Token ReadFloatPostDot()
{
while (true)
{
int ch = NextChar();
switch (ch)
{
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
continue;
case 'e':
case 'E':
return(ReadFloatPostE());
case 'j':
case 'J':
SetEnd();
return(new ConstantValueToken(LiteralParser.ParseImaginary(GetImage())));
default:
Backup();
SetEnd();
return(new ConstantValueToken(ParseFloat(GetImage())));
}
}
}
private static void ValuesShouldRoundTrip <T>(LiteralFormatter formatter, LiteralParser parser, params T[] values)
{
foreach (T value in values)
{
ValueShouldRoundTrip(value, formatter, parser);
}
}
private static BigInteger ParseBigIntegerSign(string s, int radix, int start = 0) {
try {
return LiteralParser.ParseBigIntegerSign(s, radix, start);
} catch (ArgumentException e) {
throw PythonOps.ValueError(e.Message);
}
}
public void TryParseLiteralWithDateForParenthesesKeyDelimiterShouldReturnValidDate()
{
var parser = LiteralParser.ForKeys(false /*keyAsSegment*/);
object output;
Assert.True(parser.TryParseLiteral(typeof(Date), "2015-09-28", out output));
Assert.Equal(new Date(2015, 09, 28), output);
}
public void TryParseLiteralWithDurationLiteralForSlashKeyDelimiterShouldReturnValidTimeSpan()
{
var parser = LiteralParser.ForKeys(true /*keyAsSegment*/);
object output;
Assert.True(parser.TryParseLiteral(typeof(TimeSpan), "P1D", out output));
Assert.Equal(new TimeSpan(1, 0, 0, 0), output);
}
public void TryParseLiteralWithDurationLiteralForDefaultUrlConventionsShouldReturnValidTimeSpan()
{
var parser = LiteralParser.ForKeys(false /*keyAsSegment*/);
object output;
parser.TryParseLiteral(typeof(TimeSpan), "duration'P1D'", out output).Should().BeTrue();
output.ShouldBeEquivalentTo(new TimeSpan(1, 0, 0, 0));
}
public void LargeInt32ValueReturnsFalse()
{
// regression test for: [UriParser] When int32 key value is too big for int32 throws System.OverflowException
LiteralParser parser = LiteralParser.ForETags;
object output;
parser.TryParseLiteral(typeof(int), "23500000000000000", out output).Should().BeFalse();
}
public void TryParseLiteralWithDurationLiteralForSlashKeyDelimiterShouldReturnValidTimeSpan()
{
var parser = LiteralParser.ForKeys(true /*keyAsSegment*/);
object output;
parser.TryParseLiteral(typeof(TimeSpan), "P1D", out output).Should().BeTrue();
output.ShouldBeEquivalentTo(new TimeSpan(1, 0, 0, 0));
}
public void ParseLiterals_count(string source, int expectedMatchCount)
{
var sb = new StringBuilder(source);
var subject = new LiteralParser();
var actual = subject.ParseLiterals(sb);
Assert.Equal(expectedMatchCount, actual.Count());
}
public void TryParseLiteralWithDateForParenthesesKeyDelimiterShouldReturnValidDate()
{
var parser = LiteralParser.ForKeys(false /*keyAsSegment*/);
object output;
parser.TryParseLiteral(typeof(Date), "2015-09-28", out output).Should().BeTrue();
output.ShouldBeEquivalentTo(new Date(2015, 09, 28));
}
private static object ParseFloat(string x)
{
try {
return((float)LiteralParser.ParseFloat(x));
} catch (FormatException) {
throw PythonOps.ValueError("invalid literal for Single(): {0}", x);
}
}
private object ParseInteger(string s, int radix)
{
try {
return(LiteralParser.ParseInteger(s, radix));
} catch (ArgumentException e) {
ReportSyntaxError(e.Message);
}
return(0);
}
private object ParseFloat(string s)
{
try {
return(LiteralParser.ParseFloat(s));
} catch (Exception e) {
ReportSyntaxError(e.Message);
return(0.0);
}
}
public void ParseLiterals_source_line_and_column_number(string source, int lineNumber, int columnNumber)
{
var sb = new StringBuilder(source);
var subject = new LiteralParser();
var actual = subject.ParseLiterals(sb);
var first = actual.First();
Assert.Equal(lineNumber, first.SourceLineNumber);
Assert.Equal(columnNumber, first.SourceColumnNumber);
}
static Complex ParseComplex(string s, object?imag)
{
if (imag is Missing)
{
return(LiteralParser.ParseComplex(s));
}
else
{
throw PythonOps.TypeError($"complex() can't take second arg if first is a string");
}
}
public void ParseLiterals_bracket_mismatch(
string source,
int expectedOpenBraceCount,
int expectedCloseBraceCount)
{
var sb = new StringBuilder(source);
var subject = new LiteralParser();
var ex = Assert.Throws <UnbalancedBracesException>(() => subject.ParseLiterals(sb));
Assert.Equal(expectedOpenBraceCount, ex.OpenBraceCount);
Assert.Equal(expectedCloseBraceCount, ex.CloseBraceCount);
}
public void ParseLiterals_unclosed_escape_sequence(
string source,
int expectedLineNumber,
int expectedColumnNumber)
{
var sb = new StringBuilder(source);
var subject = new LiteralParser();
var ex = Assert.Throws <MalformedLiteralException>(() => subject.ParseLiterals(sb));
Assert.Equal(expectedLineNumber, ex.LineNumber);
Assert.Equal(expectedColumnNumber, ex.ColumnNumber);
}
public override bool TryBypass(CompilationPool compilationPool)
{
var line = 1;
while (compilationPool.CodePosition < compilationPool.Code.Length)
{
if (Constraints.Instance.Tokens.SkippedSymbols.Contains(
compilationPool.Code[compilationPool.CodePosition]))
{
if (compilationPool.Code[compilationPool.CodePosition] == '\n')
{
line++;
}
compilationPool.CodePosition++;
continue;
}
if (_stateMachines.Any(stateMachine => stateMachine.FindToken(compilationPool)))
{
OnTokenFounded(compilationPool.Tokens.Last(), compilationPool.FileName, line);
continue;
}
if (LiteralParser.IsLiteral(compilationPool))
{
OnTokenFounded(compilationPool.Tokens.Last(), compilationPool.FileName, line);
continue;
}
if (IdentifierParser.IsIndentifier(compilationPool))
{
OnTokenFounded(compilationPool.Tokens.Last(), compilationPool.FileName, line);
continue;
}
Errors.Add(string.Format(Resources.Messages.UnexpectedSymbol, compilationPool.FileName,
GetNextPartOfLexem(compilationPool), line));
return(false);
}
DetermineIdentifierTypes(compilationPool);
compilationPool.Identifiers.ForEach(identifier =>
{
Messages.Add(string.Format(Resources.Messages.IndentifierFounded,
compilationPool.FileName,
identifier.Type,
identifier.Identity));
});
return(true);
}
private Token ReadNumber(char start)
{
int b = 10;
if (start == '0')
{
if (NextChar('x') || NextChar('X'))
{
return(ReadHexNumber());
}
b = 8;
}
while (true)
{
int ch = NextChar();
switch (ch)
{
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
continue;
case '.': return(ReadFloatPostDot());
case 'e':
case 'E':
return(ReadFloatPostE());
case 'j':
case 'J':
SetEnd();
return(new ConstantValueToken(LiteralParser.ParseImaginary(GetImage())));
case 'l':
case 'L':
SetEnd();
return(new ConstantValueToken(LiteralParser.ParseBigInteger(GetImage(), b)));
default:
Backup();
SetEnd();
return(new ConstantValueToken(ParseInteger(GetImage(), b)));
}
}
}
public static object __new__(CodeContext context, PythonType cls, string s, int radix)
{
var value = LiteralParser.ParseBigIntegerSign(s, radix, Int32Ops.FindStart(s, radix));
if (cls == TypeCache.BigInteger)
{
return(value);
}
else
{
return(cls.CreateInstance(context, value));
}
}
public static object __new__(PythonType cls, string s, int radix)
{
ValidateType(cls);
// radix 16/8/2 allows a 0x/0o/0b preceding it... We either need a whole new
// integer parser, or special case it here.
if (radix == 16 || radix == 8 || radix == 2)
{
s = TrimRadix(s, radix);
}
return(LiteralParser.ParseIntegerSign(s, radix));
}
private static double ParseFloat(string x)
{
try {
double?res = TryParseSpecialFloat(x);
if (res != null)
{
return(res.Value);
}
return(LiteralParser.ParseFloat(x));
} catch (FormatException) {
throw PythonOps.ValueError("invalid literal for float(): {0}", x);
}
}
public static object __new__(PythonType cls, string s, int @base)
{
ValidateType(cls);
// radix 16/8/2 allows a 0x/0o/0b preceding it... We either need a whole new
// integer parser, or special case it here.
int start = 0;
if (@base == 16 || @base == 8 || @base == 2)
{
start = s.Length - TrimRadix(s, @base).Length;
}
return(LiteralParser.ParseIntegerSign(s, @base, start));
}
private void LoadString()
{
string repr = ReadLineNoNewline();
if (repr.Length < 2 ||
!(
repr[0] == '"' && repr[repr.Length - 1] == '"' ||
repr[0] == '\'' && repr[repr.Length - 1] == '\''
)
)
{
throw new ArgumentException(String.Format("while executing STRING, expected string that starts and ends with quotes {0}", repr));
}
_stack.Add(LiteralParser.ParseString(repr.Substring(1, repr.Length - 2), false, false));
}
private Token ReadHexNumber()
{
while (true)
{
int ch = NextChar();
string s;
switch (ch)
{
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
case 'a':
case 'b':
case 'c':
case 'd':
case 'e':
case 'f':
case 'A':
case 'B':
case 'C':
case 'D':
case 'E':
case 'F':
continue;
case 'l':
case 'L':
SetEnd();
s = GetImage();
s = s.Substring(2, s.Length - 3);
return(new ConstantValueToken(LiteralParser.ParseBigInteger(s, 16)));
default:
Backup();
SetEnd();
s = GetImage();
s = s.Substring(2, s.Length - 2);
return(new ConstantValueToken(ParseInteger(s, 16)));
}
}
}
public void ParseLiterals_position_and_inner_text(string source, int[] position, string expectedInnerText)
{
var sb = new StringBuilder(source);
var subject = new LiteralParser();
var actual = subject.ParseLiterals(sb);
var first = actual.First();
string innerText = first.InnerText.ToString();
Assert.Equal(expectedInnerText, innerText);
Assert.Equal(position[0], first.StartIndex);
// Makes up for line-ending differences due to Git.
var expectedEndIndex = position[1] + source.Count(c => c == '\r');
var expectedSourceColumnNumber = first.StartIndex + 1;
Assert.Equal(expectedEndIndex, first.EndIndex);
Assert.Equal(expectedSourceColumnNumber, first.SourceColumnNumber);
}
public SyntaxParseResult Parse(List <SyntaxToken> tokens)
{
this.tokens = tokens;
position = 0;
diagnostics.Clear();
if (tokens.Count == 0)
{
return(null);
}
functionDeclarationParser = new FunctionDeclarationParser(this);
variableDeclarationParser = new VariableDeclarationParser(this);
literalParser = new LiteralParser(this);
binaryExpressionParser = new BinaryExpressionParser(this);
CodeBlockNode module = new CodeBlockNode();
while (CurrentToken.Isnt(SyntaxTokenKind.EndOfFile))
{
if (CurrentToken.Is(SyntaxTokenKind.Invalid))
{
Advance();
continue;
}
Statement declaration = ParseDeclaration();
if (declaration != null)
{
module.Children.Add(declaration);
}
else
{
Advance();
}
}
return(new SyntaxParseResult(module, diagnostics));
}
public void TestLiteralParser()
{
// Arrange.
string sampleLiteral = "\"Hello, world!\"";
var testedInstance = new LiteralParser();
var expectedBytes = new byte[] {
0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x2c, 0x20, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0x21
};
// Act.
byte[] parsedBytes = testedInstance.Parse(sampleLiteral);
// Assert.
Assert.AreEqual(expectedBytes.Length, parsedBytes.Length);
for (int i = 0; i < expectedBytes.Length; i++)
{
Assert.AreEqual(expectedBytes[i], parsedBytes[i]);
}
}
public static object __new__(CodeContext /*!*/ context, PythonType cls, IList <byte> s, int @base = 10)
{
object value;
IPythonObject po = s as IPythonObject;
if (po == null ||
!PythonTypeOps.TryInvokeUnaryOperator(DefaultContext.Default, po, "__long__", out value))
{
value = LiteralParser.ParseBigIntegerSign(s.MakeString(), @base);
}
if (cls == TypeCache.BigInteger)
{
return(value);
}
else
{
// derived long creation...
return(cls.CreateInstance(context, value));
}
}
public static string __repr__(CodeContext /*!*/ context, double self)
{
if (Double.IsNaN(self))
{
return("nan");
}
// first format using Python's specific formatting rules...
StringFormatter sf = new StringFormatter(context, "%.17g", self);
sf._TrailingZeroAfterWholeFloat = true;
string res = sf.Format();
if (LiteralParser.ParseFloat(res) == self)
{
return(res);
}
// if it's not round trippable though use .NET's round-trip format
return(self.ToString("R", CultureInfo.InvariantCulture));
}
