public void Insert_DoesNotThrowsException_IfTwoFastStringBuildersContainsSameThing()
{
var a = new FastStringBuilder("test");
var b = new FastStringBuilder("test");
Assert.DoesNotThrow(() => a.Insert(0, b, 0, 1));
}
public void StepParser_ShouldTokenizeFileFast2(string path)
{
var sw = Stopwatch.StartNew();
using (var file = new FastBinaryReader(path))
{
var tokens = new int[Enum.GetValues(typeof(StepTokenKind)).Length];
StepTokenKind token;
var buffer = new FastStringBuilder();
while ((token = StepTokenizer.ReadNext(file, buffer)) != StepTokenKind.Eof)
{
tokens[(int)token]++;
}
Console.WriteLine(sw.ElapsedMilliseconds);
foreach (StepTokenKind kind in Enum.GetValues(typeof(StepTokenKind)))
{
Console.WriteLine($"{kind}: {tokens[(int)kind]}");
}
Console.WriteLine($"total: {tokens.Sum()}");
}
}
/// <summary>
/// Handles the special Lam-Alef connection in the text.
/// </summary>
/// <param name="input"></param>
/// <param name="output"></param>
/// <param name="i">Index of Lam letter</param>
/// <returns><see langword="true" /> if special connection has been made.</returns>
private static bool HandleSpecialLam(FastStringBuilder input, FastStringBuilder output, int i)
{
bool isFixed;
switch (input.Get(i + 1))
{
case (char)GeneralLetters.AlefMaksoor:
output.Set(i, (char)0xFEF7);
isFixed = true;
break;
case (char)GeneralLetters.Alef:
output.Set(i, (char)0xFEF9);
isFixed = true;
break;
case (char)GeneralLetters.AlefHamza:
output.Set(i, (char)0xFEF5);
isFixed = true;
break;
case (char)GeneralLetters.AlefMad:
output.Set(i, (char)0xFEF3);
isFixed = true;
break;
default:
isFixed = false;
break;
}
if (isFixed)
{
output.Set(i + 1, (char)0xFFFF);
}
return isFixed;
}
public void Length_Set(string initial, int len, string expected)
{
var a = new FastStringBuilder(initial);
a.Length = len;
Assert.AreEqual(expected, a.ToString());
}
/// <summary>
/// Fixes the provided string
/// </summary>
/// <param name="input">Text to fix</param>
/// <param name="output">Fixed text</param>
/// <param name="fixTextTags"></param>
/// <param name="preserveNumbers"></param>
/// <param name="farsi"></param>
/// <returns>Fixed text</returns>
public static void FixRTL(
string input,
FastStringBuilder output,
bool farsi = true,
bool fixTextTags = true,
bool preserveNumbers = false)
{
inputBuilder.SetValue(input);
TashkeelFixer.RemoveTashkeel(inputBuilder);
// The shape of the letters in shapeFixedLetters is fixed according to their position in word. But the flow of the text is not fixed.
GlyphFixer.Fix(inputBuilder, glyphFixerOutput, preserveNumbers, farsi, fixTextTags);
//Restore tashkeel to their places.
TashkeelFixer.RestoreTashkeel(glyphFixerOutput);
// Fix flow of the text and put the result in FinalLetters field
LigatureFixer.Fix(glyphFixerOutput, output, farsi, fixTextTags, preserveNumbers);
if (fixTextTags)
{
RichTextFixer.Fix(output);
}
inputBuilder.Clear();
}
// This needs to be done properly!
protected override IEnumerable <string> GetBatches(ISQLBuilder sqlBuilder)
{
yield return("SET XACT_ABORT ON;");
yield return("BEGIN TRANSACTION;");
// TODO: issue batches in smaller chunks
var currentBatch = new FastStringBuilder();
foreach (var statement in sqlBuilder.Statements.Where(x => x.Type != SQLStatementType.TCL))
{
if (statement.Type == SQLStatementType.DDL)
{
if (currentBatch.Length > 0)
{
yield return(currentBatch.ToString());
currentBatch.Clear();
}
yield return(statement.SQL);
}
else
{
currentBatch.Append(statement.SQL);
}
}
if (currentBatch.Length > 0)
{
yield return(currentBatch.ToString());
}
yield return("COMMIT TRANSACTION;");
}
private static Token CreateString(RootNode rootNode, SourceReader Source, FastStringBuilder builder, char quote, TokenType literalType)
{
var token = rootNode.ProvideToken();
builder.Clear();
token.Type = literalType;
//Opening quote
Source.Pop();
token.Start = Source.Location;
while (Source.Peek() != quote)
{
if (Source.Peek() == '\\')
{
builder.Append(Source.Pop());
}
builder.Append(Source.Pop());
}
;
//Closing quote
Source.Pop();
token.End = Source.Location;
token.Content = new FastString(rootNode, builder);
return(token);
}
/// <summary>
/// Fixes the shape of letters based on their position.
/// </summary>
/// <param name="input"></param>
/// <param name="output"></param>
/// <param name="preserveNumbers"></param>
/// <param name="farsi"></param>
/// <returns></returns>
public static void Fix(FastStringBuilder input, FastStringBuilder output, bool preserveNumbers, bool farsi)
{
FixYah(input, farsi);
output.SetValue(input);
for (int i = 0; i < input.Length; i++)
{
bool skipNext = false;
char iChar = input.Get(i);
// For special Lam Letter connections.
if (iChar == (char)GeneralLetters.Lam)
{
if (i < input.Length - 1)
{
skipNext = HandleSpecialLam(input, output, i);
if (skipNext)
iChar = output.Get(i);
}
}
// We don't want to fix tatweel or zwnj character
if (iChar == (int)GeneralLetters.ArabicTatweel ||
iChar == (int)GeneralLetters.ZeroWidthNoJoiner)
{
continue;
}
if (TextUtils.IsRTLCharacter(iChar))
{
char converted = GlyphTable.Convert(iChar);
if (IsMiddleLetter(input, i))
{
output.Set(i, (char)(converted + 3));
}
else if (IsFinishingLetter(input, i))
{
output.Set(i, (char)(converted + 1));
}
else if (IsLeadingLetter(input, i))
{
output.Set(i, (char)(converted + 2));
}
}
// If this letter as Lam and special Lam-Alef connection was made, We want to skip the Alef
// (Lam-Alef occupies 1 space)
if (skipNext)
{
i++;
}
}
if (!preserveNumbers)
{
FixNumbers(output, farsi);
}
}
public void Append_WhenCapacityThresholdIsHit(string initial, char append, string expected)
{
var a = new FastStringBuilder(initial);
a.Append(append);
Assert.AreEqual(expected, a.ToString());
}
public void Replace_CanReplaceStrings(string text, string oldStr, string newStr, string expected)
{
var fastString = new FastStringBuilder(text);
fastString.Replace(oldStr, newStr);
Assert.AreEqual(expected, fastString.ToString());
}
public void FastStringBuilder_ShouldParseDouble()
{
var builder = new FastStringBuilder();
builder.Append("1.2e-17");
Assert.AreEqual(1.2e-17, builder.ParseDouble());
}
public void SetValue_IncreasesCapacity_AsNeeded()
{
var builder = new FastStringBuilder(0);
builder.SetValue("abcd");
Assert.AreEqual("abcd", builder.ToString());
}
public void SetValue_SetsTheValueOfStringBuilderWithProvidedText()
{
var builder = new FastStringBuilder(10);
builder.SetValue("abcd");
Assert.AreEqual("abcd", builder.ToString());
}
public void GlyphFixer_PreservesNumbers_WhenPreserveNumberIsTrue()
{
var text = GetEnglishNumbers();
var output = new FastStringBuilder(10);
GlyphFixer.Fix(text, output, true, false, false);
Assert.AreEqual(text.ToString(), output.ToString());
}
public void SetValue_CanCopyValueOfAnotherStringBuilder()
{
var builder1 = new FastStringBuilder("my holy moly text");
var builder2 = new FastStringBuilder(10);
builder2.SetValue(builder1);
Assert.AreEqual(builder1.ToString(), builder2.ToString());
}
public void GlyphFixer_ConvertsNumbers_Hindu()
{
var text = GetEnglishNumbers();
var output = new FastStringBuilder(10);
GlyphFixer.Fix(text, output, false, false, false);
Assert.AreEqual(GetHinduNumbers().ToString(), output.ToString());
}
/// <summary>
/// Create an instance of the token stream to read from the given reader.
/// </summary>
/// <param name="reader"></param>
public TokenStream(TextReader reader)
{
_reader = reader;
//_capacity = 16;
//_tokens = new Queue<Token>(_capacity);
_symbols = "[]<>():,{}.$".ToCharArray();
Array.Sort<char>(_symbols);
_buffer = new FastStringBuilder();
//Fill();
}
public void FastStirngBuilder_ShouldEqualsWork()
{
var builder = new FastStringBuilder();
builder.Append("Test");
Assert.IsTrue(builder.Equals("Test"));
Assert.IsFalse(builder.Equals("Tes"));
Assert.IsFalse(builder.Equals("Test1"));
}
public void GlyphFixer_FixesYah_WhenFarsiIsRequired()
{
var text = new FastStringBuilder(10);
text.Append((char)ArabicGeneralLetters.Yeh);
GlyphFixer.FixYah(text, true);
Assert.AreEqual(((char)ArabicGeneralLetters.FarsiYeh).ToString(), text.ToString());
}
public void Append_AddsACharacter()
{
var builder = new FastStringBuilder(10);
builder.Append('a');
builder.Append('b');
builder.Append('c');
Assert.AreEqual("abc", builder.ToString());
}
public void Reverse_CanReverseTheWholeString_WhenLengthIsOdd()
{
// Arrange
var builder = new FastStringBuilder("abcdefg");
// Act
builder.Reverse(0, builder.Length);
// Assert
Assert.AreEqual("gfedcba", builder.ToString());
}
private static FastStringBuilder GetHinduNumbers()
{
var text = new FastStringBuilder(10);
var hinduNumbers = Enum.GetValues(typeof(HinduNumbers));
foreach (int hinduNumber in hinduNumbers)
{
text.Append((char)hinduNumber);
}
return(text);
}
private static FastStringBuilder GetFarsiNumbers()
{
var text = new FastStringBuilder(10);
var farsiNumbers = Enum.GetValues(typeof(FarsiNumbers));
foreach (int farsiNumber in farsiNumbers)
{
text.Append((char)farsiNumber);
}
return(text);
}
public void Fix_Reverses_SimpleOpenAndClosingTagWithValue()
{
// Arrange
var text = new FastStringBuilder("text </open> text <open=134>");
// Act
RichTextFixer.Fix(text);
// Assert
Assert.AreEqual("text >nepo/< text >431=nepo<", text.ToString());
}
public void Fix_Reverses_OpenTag_ThatDoesntHaveClosingTag()
{
// Arrange
var text = new FastStringBuilder("text <open> text");
// Act
RichTextFixer.Fix(text);
// Assert
Assert.AreEqual("text >nepo< text", text.ToString());
}
public void Fix_Reverses_SelfContainedTags()
{
// Arrange
var text = new FastStringBuilder("text <self-contained/> text");
// Act
RichTextFixer.Fix(text);
// Assert
Assert.AreEqual("text >/deniatnoc-fles< text", text.ToString());
}
public void GlyphFixer_ConvertsNumbers_Farsi()
{
var text = GetEnglishNumbers();
var output = new FastStringBuilder(10);
GlyphFixer.Fix(text, output, false, true, false);
Debug.Log("Output: " + output);
Assert.AreEqual(GetFarsiNumbers().ToString(), output.ToString());
}
public void Reverse_CanReversePartOfString_WhenLengthIsOdd(int start, int length, string expected)
{
// Arrange
var builder = new FastStringBuilder("abcdefg");
// Act
builder.Reverse(start, length);
// Assert
Assert.AreEqual(expected, builder.ToString());
}
private static FastStringBuilder GetEnglishNumbers()
{
var text = new FastStringBuilder(10);
var englishNumbers = Enum.GetValues(typeof(EnglishNumbers));
foreach (int englishNumber in englishNumbers)
{
text.Append((char)englishNumber);
}
return(text);
}
public void Remove_RemovesAllOccurrencesOfCharacter()
{
// Arrange
var builder = new FastStringBuilder("cabccdececgc");
// Act
builder.RemoveAll('c');
// Assert
Assert.AreEqual("abdeeg", builder.ToString());
}
public void SetValue_MakesIndependentCopies()
{
var builder1 = new FastStringBuilder("my holy moly text");
var builder2 = new FastStringBuilder(10);
builder2.SetValue(builder1);
builder1.Set(5, 'f');
Assert.AreNotEqual(builder1.ToString(), builder2.ToString());
}
/// <summary>
/// Gets a number from the reader, which can be integer, floating point or scientific notation
/// Examples: 123343, -123232, 12.345, -45.3434, 3.45E+10
/// </summary>
/// <param name="start">the starting character</param>
/// <param name="buffer">buffer to hold input</param>
/// <returns>number token</returns>
private Token GetNumber(char start, FastStringBuilder buffer)
{
char ch = start;
buffer.Append(ch);
int i = (start == '.') ? 1 : 0;
while (i < 3)
{
switch (i)
{
case 0: // first part of integer
GetIntegerPart(buffer);
ch = (char)_reader.Peek();
if (ch == '.')
{
i=1; // try to read fractional now
buffer.Append((char)_reader.Read());
}
else if (ch == 'e' || ch == 'E')
{
i = 2; // try to read exponent now
buffer.Append((char)_reader.Read());
}
else
{
i = 4; //break out
break;
}
break;
case 1: // fractional part
GetIntegerPart(buffer);
ch = (char)_reader.Peek();
if (ch == '.')
{
throw new ParseException("Invalid number exception");
}
else if (ch == 'e' || ch == 'E')
{
i = 2; // read exponent
buffer.Append((char)_reader.Read());
}
else
{
i = 3; // break out
}
break;
case 2: // scientific notation
ch = (char)_reader.Peek();
//check for an optional sign
if (ch == '+' || ch == '-')
{
buffer.Append((char)_reader.Read());
}
GetIntegerPart(buffer);
ch = (char)_reader.Peek();
if (ch == '.')
{
throw new ParseException("Invalid number exception");
}
else
{
i = 3; // break out
}
break;
}
}
return new Token(TokenType.Number, buffer.ToString());
}
/// <summary>
/// Gets an integer portion of a number, stopping at a "." or the start of an exponent "e" or "E"
/// </summary>
/// <param name="buffer">buffer to store input</param>
private void GetIntegerPart(FastStringBuilder buffer)
{
int c;
char ch;
while ((c = _reader.Peek()) != -1)
{
ch = (char)c;
if (char.IsNumber(ch))
{
buffer.Append(ch);
}
else if (ch == '.' || ch == 'e' || ch == 'E' || IsSymbolStart(ch) || char.IsWhiteSpace(ch))
{
break;
}
else
{
throw new ParseException("Invalid number, unexpected character: " + ch);
}
_reader.Read();
}
}
/// <summary>
/// Gets an identifier from the reader such as a variable reference, null, true, or false.
/// Follows C# rules, non-qouted string starting with a letter or "_" followed by letters digits or "_"
/// </summary>
/// <param name="start">the starting character</param>
/// <param name="buffer">a buffer to hold input</param>
/// <returns>identifier token</returns>
private Token GetIdentifier(char start, FastStringBuilder buffer)
{
buffer.Append(start);
int c;
char ch;
while ((c = _reader.Peek()) != -1)
{
ch = (char)c;
if (char.IsLetterOrDigit(ch) || ch == '_')
{
buffer.Append(ch);
}
else
{
return new Token(TokenType.Identifier, buffer.ToString());
}
_reader.Read();
}
return new Token(TokenType.Identifier, buffer.ToString());
}
/// <summary>
/// Gets a single or double qouted string from the reader, handling and escape characters
/// </summary>
/// <param name="start">the starting character</param>
/// <param name="buffer">buffer for input</param>
/// <returns>string token</returns>
private Token GetQuotedString(char start, FastStringBuilder buffer)
{
char quoteChar = start;
bool escape = false;
char ch;
int c;
while ((c = _reader.Read()) != -1) {
ch = (char) c;
if (escape)
{
switch (ch)
{
case 't': // horizantal tab
buffer.Append('\t');
break;
case 'n': // newline
buffer.Append('\n');
break;
case '\\': // reverse solidus
buffer.Append('\\');
break;
case '/': // solidus
buffer.Append('/');
break;
case 'b': // backspace
buffer.Append('\b');
break;
case 'f': // formfeed
buffer.Append('\f');
break;
case 'r': // carriage return
buffer.Append('\r');
break;
case 'u': // unicode escape sequence \unnnn
{
char[] ucodeChar = new char[4];
int nRead = _reader.Read(ucodeChar, 0, 4);
if (nRead != 4)
throw new ParseException("Invalid unicode escape sequence, expecting \"\\unnnn\", but got " + (new string(ucodeChar, 0, nRead)));
buffer.Append((char)uint.Parse(new string(ucodeChar), System.Globalization.NumberStyles.HexNumber));
}
break;
default:
buffer.Append(ch);
break;
}
escape = false;
}
else
{
if (ch == '\\')
{
escape = true;
}
else if (ch == quoteChar)
{
return new Token(quoteChar == '"' ? TokenType.DoubleQuotedString : TokenType.SingleQuotedString, buffer.ToString());
buffer.Length = 0;
}
else
{
buffer.Append(ch);
}
}
}
throw new ParseException("Unterminated string constant");
}
