private static Token CreateString(RootNode rootNode, SourceReader Source, FastStringBuilder builder, char quote, TokenType literalType)
{
var token = rootNode.ProvideToken();
builder.Clear();
token.Type = literalType;
//Opening quote
Source.Pop();
token.Start = Source.Location;
while (Source.Peek() != quote)
{
if (Source.Peek() == '\\')
{
builder.Append(Source.Pop());
}
builder.Append(Source.Pop());
}
;
//Closing quote
Source.Pop();
token.End = Source.Location;
token.Content = new FastString(rootNode, builder);
return(token);
}
public void Append_AddsACharacter()
{
var builder = new FastStringBuilder(10);
builder.Append('a');
builder.Append('b');
builder.Append('c');
Assert.AreEqual("abc", builder.ToString());
}
// This needs to be done properly!
protected override IEnumerable <string> GetBatches(ISQLBuilder sqlBuilder)
{
yield return("SET XACT_ABORT ON;");
yield return("BEGIN TRANSACTION;");
// TODO: issue batches in smaller chunks
var currentBatch = new FastStringBuilder();
foreach (var statement in sqlBuilder.Statements.Where(x => x.Type != SQLStatementType.TCL))
{
if (statement.Type == SQLStatementType.DDL)
{
if (currentBatch.Length > 0)
{
yield return(currentBatch.ToString());
currentBatch.Clear();
}
yield return(statement.SQL);
}
else
{
currentBatch.Append(statement.SQL);
}
}
if (currentBatch.Length > 0)
{
yield return(currentBatch.ToString());
}
yield return("COMMIT TRANSACTION;");
}
public void Append_WhenCapacityThresholdIsHit(string initial, char append, string expected)
{
var a = new FastStringBuilder(initial);
a.Append(append);
Assert.AreEqual(expected, a.ToString());
}
public void FastStringBuilder_ShouldParseDouble()
{
var builder = new FastStringBuilder();
builder.Append("1.2e-17");
Assert.AreEqual(1.2e-17, builder.ParseDouble());
}
public void GlyphFixer_FixesYah_WhenFarsiIsRequired()
{
var text = new FastStringBuilder(10);
text.Append((char)ArabicGeneralLetters.Yeh);
GlyphFixer.FixYah(text, true);
Assert.AreEqual(((char)ArabicGeneralLetters.FarsiYeh).ToString(), text.ToString());
}
public void FastStirngBuilder_ShouldEqualsWork()
{
var builder = new FastStringBuilder();
builder.Append("Test");
Assert.IsTrue(builder.Equals("Test"));
Assert.IsFalse(builder.Equals("Tes"));
Assert.IsFalse(builder.Equals("Test1"));
}
private static FastStringBuilder GetFarsiNumbers()
{
var text = new FastStringBuilder(10);
var farsiNumbers = Enum.GetValues(typeof(FarsiNumbers));
foreach (int farsiNumber in farsiNumbers)
{
text.Append((char)farsiNumber);
}
return(text);
}
public void FastStringBuilder_ShouldParseInt()
{
var builder = new FastStringBuilder();
builder.Append("1233321");
Assert.AreEqual(1233321, builder.ParseInt());
builder.Clear();
builder.Append("+1235321");
Assert.AreEqual(1235321, builder.ParseInt());
builder.Clear();
builder.Append("-1234321");
Assert.AreEqual(-1234321, builder.ParseInt());
}
private static FastStringBuilder GetHinduNumbers()
{
var text = new FastStringBuilder(10);
var hinduNumbers = Enum.GetValues(typeof(HinduNumbers));
foreach (int hinduNumber in hinduNumbers)
{
text.Append((char)hinduNumber);
}
return(text);
}
private static FastStringBuilder GetEnglishNumbers()
{
var text = new FastStringBuilder(10);
var englishNumbers = Enum.GetValues(typeof(EnglishNumbers));
foreach (int englishNumber in englishNumbers)
{
text.Append((char)englishNumber);
}
return(text);
}
private static Token WrapSymbol(RootNode rootNode, SourceReader Source, FastStringBuilder builder, TokenType type)
{
var token = rootNode.ProvideToken();
builder.Clear();
token.Type = type;
token.Start = Source.Location;
builder.Append(Source.Pop());
token.End = Source.Location;
token.Content = new FastString(rootNode, builder);
return(token);
}
private static StepToken ParseNumber(FastBinaryReader reader, FastStringBuilder buffer)
{
//integer or real
while (IsDigit(reader.Peek()))
{
buffer.Append((char)reader.Read());
}
if (reader.Peek() == '.')
{
buffer.Append((char)reader.Read());
while (IsDigit(reader.Peek()))
{
//realValue +=
buffer.Append((char)reader.Read());
}
if (reader.Peek() == 'E')
{
reader.Read();
buffer.Append('E');
var next = reader.Peek();
if (next == '+' || next == '-')
{
buffer.Append((char)next);
reader.Read();
next = reader.Peek();
}
if (IsDigit(next))
{
buffer.Append((char)reader.Read());
}
else
{
throw new UnexpectedCharacterException((char)next);
}
while (IsDigit(reader.Peek()))
{
buffer.Append((char)reader.Read());
}
}
return(new StepToken(StepTokenKind.Real, buffer));
}
else
{
return(new StepToken(StepTokenKind.Integer, buffer));
}
}
public void GlyphFixer_FixesYah_WhenFarsiIsRequired()
{
var text = new FastStringBuilder(10);
text.Append((char)GeneralLetters.Ya);
Debug.Log("Before: " + text);
GlyphFixer.FixYah(text, true);
Debug.Log("After: " + text);
Assert.AreEqual(((char)GeneralLetters.PersianYa).ToString(), text.ToString());
}
public override string ToString()
{
var query = base.ToString();
if (StatementCount <= 1)
{
query = query.Trim(); // only trim for small queries as memory may be wasted otherwise
if (query.EndsWith(StatementTerminator))
{
query = query.Substring(0, query.Length - StatementTerminator.Length);
}
}
else
{
var stringBuilder = new FastStringBuilder();
stringBuilder.AppendFormat("SET TERM {0};{1}", StatementTerminator, Environment.NewLine);
if (_variableDeclarations.Count > 0)
{
stringBuilder.AppendLine("EXECUTE BLOCK AS");
stringBuilder.AppendLine("BEGIN");
foreach (var variable in _variableDeclarations.Keys)
{
stringBuilder.AppendFormat("DECLARE VARIABLE {0} {1}{2}{3}", variable, ConvertTypeToSQLType(_variableDeclarations[variable]), StatementTerminator, Environment.NewLine);
}
}
stringBuilder.Append(query);
if (_variableDeclarations.Count > 0)
{
stringBuilder.Append("END");
}
query = stringBuilder.ToString();
}
return(query);
}
/// <summary>
/// Gets a number from the reader, which can be integer, floating point or scientific notation
/// Examples: 123343, -123232, 12.345, -45.3434, 3.45E+10
/// </summary>
/// <param name="start">the starting character</param>
/// <param name="buffer">buffer to hold input</param>
/// <returns>number token</returns>
private Token GetNumber(char start, FastStringBuilder buffer)
{
char ch = start;
buffer.Append(ch);
int i = (start == '.') ? 1 : 0;
while (i < 3)
{
switch (i)
{
case 0: // first part of integer
GetIntegerPart(buffer);
ch = (char)_reader.Peek();
if (ch == '.')
{
i=1; // try to read fractional now
buffer.Append((char)_reader.Read());
}
else if (ch == 'e' || ch == 'E')
{
i = 2; // try to read exponent now
buffer.Append((char)_reader.Read());
}
else
{
i = 4; //break out
break;
}
break;
case 1: // fractional part
GetIntegerPart(buffer);
ch = (char)_reader.Peek();
if (ch == '.')
{
throw new ParseException("Invalid number exception");
}
else if (ch == 'e' || ch == 'E')
{
i = 2; // read exponent
buffer.Append((char)_reader.Read());
}
else
{
i = 3; // break out
}
break;
case 2: // scientific notation
ch = (char)_reader.Peek();
//check for an optional sign
if (ch == '+' || ch == '-')
{
buffer.Append((char)_reader.Read());
}
GetIntegerPart(buffer);
ch = (char)_reader.Peek();
if (ch == '.')
{
throw new ParseException("Invalid number exception");
}
else
{
i = 3; // break out
}
break;
}
}
return new Token(TokenType.Number, buffer.ToString());
}
private unsafe static StepTokenKind ParseStringKind(FastBinaryReader reader, FastStringBuilder buffer)
{
int encoding = 0;
int next;
while ((next = reader.Read()) != -1)
{
if (next == '\'')
{
if (reader.Peek() == '\'')
{
reader.Read();
buffer.Append('\'');
}
else
{
return(StepTokenKind.String);
}
}
else if (next == '\\')
{
switch (reader.Read())
{
case 'S':
{
next = reader.Read();
if (next != '\\')
{
throw new UnexpectedCharacterException((char)next);
}
next = reader.Read();
if (IsCharacter(next))
{
byte *ptr = stackalloc byte[1];
char *result = stackalloc char[1];
ptr[0] = (byte)(next + 128);
encodings_[encoding].GetChars(ptr, 1, result, 1);
buffer.Append(*result);
break;
}
else
{
throw new UnexpectedCharacterException((char)next);
}
}
case 'P':
{
next = reader.Read();
if (next >= 'A' && next <= 'I')
{
encoding = next - 'A';
}
else
{
throw new UnexpectedCharacterException((char)next);
}
next = reader.Read();
if (next != '\\')
{
throw new UnexpectedCharacterException((char)next);
}
break;
}
case 'X':
{
next = reader.Read();
if (next == '\\')
{
byte *ptr = stackalloc byte[1];
char *result = stackalloc char[1];
next = reader.Read();
if (!IsHex(next))
{
throw new UnexpectedCharacterException((char)next);
}
ptr[0] = (byte)(FromHex(next) << 8);
next = reader.Read();
ptr[0] |= (byte)FromHex(next);
encodings_[0].GetChars(ptr, 1, result, 1);
buffer.Append(*result);
break;
}
else if (next == '2')
{
byte *ptr = stackalloc byte[20];
char *chars = stackalloc char[10];
Expect(reader, '\\');
int offset = 0;
while (IsHex(reader.Peek()))
{
next = reader.Read();
ptr[offset] = (byte)(FromHex(next) << 24);
next = reader.Read();
ptr[offset] |= (byte)(FromHex(next) << 16);
offset++;
next = reader.Read();
ptr[offset] = (byte)(FromHex(next) << 8);
next = reader.Read();
ptr[offset] |= (byte)(FromHex(next));
offset++;
if (offset == 20)
{
Encoding.Unicode.GetChars(ptr, offset, chars, 20);
buffer.Append(chars, 10);
offset = 0;
}
}
var encoded = Encoding.Unicode.GetChars(ptr, offset, chars, 20);
buffer.Append(chars, encoded);
Expect(reader, '\\');
Expect(reader, 'X');
Expect(reader, '0');
Expect(reader, '\\');
break;
}
else if (next == '4')
{
throw new NotImplementedException();
}
else
{
throw new UnexpectedCharacterException((char)next);
}
}
case '\\':
buffer.Append('\\');
break;
case int ch:
throw new UnexpectedCharacterException((char)ch);
}
}
else if (IsCharacter(next))
{
buffer.Append((char)next);
}
else
{
throw new UnexpectedCharacterException((char)next);
}
}
return(StepTokenKind.String);
}
/// <summary>
/// Gets an identifier from the reader such as a variable reference, null, true, or false.
/// Follows C# rules, non-qouted string starting with a letter or "_" followed by letters digits or "_"
/// </summary>
/// <param name="start">the starting character</param>
/// <param name="buffer">a buffer to hold input</param>
/// <returns>identifier token</returns>
private Token GetIdentifier(char start, FastStringBuilder buffer)
{
buffer.Append(start);
int c;
char ch;
while ((c = _reader.Peek()) != -1)
{
ch = (char)c;
if (char.IsLetterOrDigit(ch) || ch == '_')
{
buffer.Append(ch);
}
else
{
return new Token(TokenType.Identifier, buffer.ToString());
}
_reader.Read();
}
return new Token(TokenType.Identifier, buffer.ToString());
}
public static StepTokenKind ReadNext(FastBinaryReader reader, FastStringBuilder buffer)
{
int current;
while ((current = reader.Read()) != -1)
{
var ch = (char)current;
switch (ch)
{
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
{
buffer.Clear();
buffer.Append(ch);
return(ParseNumberKind(reader, buffer));
};
case '-':
case '+':
{
if (IsDigit(reader.Peek()))
{
buffer.Clear();
buffer.Append(ch);
return(ParseNumberKind(reader, buffer));
}
else
{
throw new UnexpectedCharacterException(ch);
}
}
case '#':
{
if (!IsDigit(reader.Peek()))
{
throw new UnexpectedCharacterException(ch);
}
buffer.Clear();
do
{
buffer.Append((char)reader.Read());
}while (IsDigit(reader.Peek()));
return(StepTokenKind.EntityInstanceName);
}
case '"':
{
throw new NotImplementedException("binary");
}
case '\'':
{
buffer.Clear();
return(ParseStringKind(reader, buffer));
}
case '!':
{
throw new NotImplementedException("user defined keyword");
}
case '.':
{
if (!IsUpper(reader.Peek()))
{
throw new UnexpectedCharacterException(ch);
}
buffer.Clear();
do
{
buffer.Append((char)reader.Read());
} while (IsUpperOrDigit(reader.Peek()));
if (reader.Peek() == '.')
{
reader.Read();
return(StepTokenKind.Enumeration);
}
else
{
throw new UnexpectedCharacterException((char)reader.Peek());
}
}
case '=':
return(StepTokenKind.Assignment);
case '*':
return(StepTokenKind.Asterisk);
case '$':
return(StepTokenKind.Dollar);
case ';':
return(StepTokenKind.Semicolon);
case '(':
return(StepTokenKind.LeftParen);
case ')':
return(StepTokenKind.RightParen);
case '/':
{
if (reader.Peek() == '*')
{
while ((current = reader.Read()) != -1)
{
if (current == '*' && reader.Peek() == '/')
{
reader.Read();
break;
}
}
}
else
{
return(StepTokenKind.Solidus);
}
}
break;
case ',':
return(StepTokenKind.Comma);
case ' ':
break;
default:
{
if (IsUpper(ch))
{
buffer.Clear();
buffer.Append(ch);
while (IsUpperOrDigitOrMinus(reader.Peek()))
{
buffer.Append((char)reader.Read());
}
if (buffer.Equals("ISO-10303-21"))
{
return(StepTokenKind.Iso);
}
else if (buffer.Equals("END-ISO-10303-21"))
{
return(StepTokenKind.EndIso);
}
else if (buffer.Equals("HEADER"))
{
return(StepTokenKind.Header);
}
else if (buffer.Equals("ENDSEC"))
{
return(StepTokenKind.EndSection);
}
else if (buffer.Equals("DATA"))
{
return(StepTokenKind.Data);
}
else
{
return(StepTokenKind.StandardKeyword);
}
}
else if (IsCharacter(ch))
{
throw new UnexpectedCharacterException(ch);
}
break;
}
}
}
return(StepTokenKind.Eof);
}
/// <summary>
/// Gets an integer portion of a number, stopping at a "." or the start of an exponent "e" or "E"
/// </summary>
/// <param name="buffer">buffer to store input</param>
private void GetIntegerPart(FastStringBuilder buffer)
{
int c;
char ch;
while ((c = _reader.Peek()) != -1)
{
ch = (char)c;
if (char.IsNumber(ch))
{
buffer.Append(ch);
}
else if (ch == '.' || ch == 'e' || ch == 'E' || IsSymbolStart(ch) || char.IsWhiteSpace(ch))
{
break;
}
else
{
throw new ParseException("Invalid number, unexpected character: " + ch);
}
_reader.Read();
}
}
/// <summary>
/// Gets a single or double qouted string from the reader, handling and escape characters
/// </summary>
/// <param name="start">the starting character</param>
/// <param name="buffer">buffer for input</param>
/// <returns>string token</returns>
private Token GetQuotedString(char start, FastStringBuilder buffer)
{
char quoteChar = start;
bool escape = false;
char ch;
int c;
while ((c = _reader.Read()) != -1) {
ch = (char) c;
if (escape)
{
switch (ch)
{
case 't': // horizantal tab
buffer.Append('\t');
break;
case 'n': // newline
buffer.Append('\n');
break;
case '\\': // reverse solidus
buffer.Append('\\');
break;
case '/': // solidus
buffer.Append('/');
break;
case 'b': // backspace
buffer.Append('\b');
break;
case 'f': // formfeed
buffer.Append('\f');
break;
case 'r': // carriage return
buffer.Append('\r');
break;
case 'u': // unicode escape sequence \unnnn
{
char[] ucodeChar = new char[4];
int nRead = _reader.Read(ucodeChar, 0, 4);
if (nRead != 4)
throw new ParseException("Invalid unicode escape sequence, expecting \"\\unnnn\", but got " + (new string(ucodeChar, 0, nRead)));
buffer.Append((char)uint.Parse(new string(ucodeChar), System.Globalization.NumberStyles.HexNumber));
}
break;
default:
buffer.Append(ch);
break;
}
escape = false;
}
else
{
if (ch == '\\')
{
escape = true;
}
else if (ch == quoteChar)
{
return new Token(quoteChar == '"' ? TokenType.DoubleQuotedString : TokenType.SingleQuotedString, buffer.ToString());
buffer.Length = 0;
}
else
{
buffer.Append(ch);
}
}
}
throw new ParseException("Unterminated string constant");
}
