public void JsonTabularWriter_Escaping()
{
StringBuilder allEscapedValues = new StringBuilder();
// All 0x00 - 0x1F are escaped
for (int i = 0; i < 32; ++i)
{
allEscapedValues.Append((char)i);
}
// Backslash is escaped
allEscapedValues.Append('\\');
// Quote is escaped
allEscapedValues.Append('"');
string all = allEscapedValues.ToString();
String8 all8 = String8.Convert(all, new byte[String8.GetLength(all)]);
using (ITabularWriter w = new JsonTabularWriter("Escaping.json"))
{
w.SetColumns(new string[] { "Bad" });
w.Write(all8);
w.NextRow();
}
string content = File.ReadAllText("Escaping.json");
Assert.IsTrue(content.IndexOf("\\u0000\\u0001\\u0002\\u0003\\u0004\\u0005\\u0006\\u0007\\u0008\\u0009\\u000A\\u000B\\u000C\\u000D\\u000E\\u000F\\u0010\\u0011\\u0012\\u0013\\u0014\\u0015\\u0016\\u0017\\u0018\\u0019\\u001A\\u001B\\u001C\\u001D\\u001E\\u001F\\\\\\\"") >= 0);
}
public static int FindByPath(ItemTree tree, StringStore strings, string path, char delimiter = '\\')
{
String8 path8 = String8.Convert(path, new byte[String8.GetLength(path)]);
String8Set pathSplit8 = path8.Split(delimiter, new int[String8Set.GetLength(path8, delimiter)]);
return(tree.FindByPath(0, pathSplit8, strings));
}
private void EnsureComparesConsistent(string left, string right)
{
String8 left8 = String8.Convert(left, new byte[String8.GetLength(left)]);
String8 right8 = String8.Convert(right, new byte[String8.GetLength(right)]);
CompareResult caseSensitiveExpected = ToResult(String.Compare(left, right, StringComparison.Ordinal));
CompareResult caseInsensitiveExpected = ToResult(String.Compare(left, right, StringComparison.OrdinalIgnoreCase));
Assert.AreEqual(caseSensitiveExpected, ToResult(left8.CompareTo(right8)), "Case sensitive comparison result incorrect.");
Assert.AreEqual(caseInsensitiveExpected, ToResult(left8.CompareTo(right8, true)), "Case insensitive comparison result incorrect.");
Assert.AreEqual(caseSensitiveExpected, ToResult(left8.CompareTo(right)), "Case sensitive String8 to string comparison result incorrect.");
Assert.AreEqual(caseInsensitiveExpected, ToResult(left8.CompareTo(right, true)), "Case insensitive String8 to string comparison result incorrect.");
// StartsWith and CompareAsPrefixTo
Assert.AreEqual(left.StartsWith(right), left8.StartsWith(right8));
Assert.AreEqual(left.StartsWith(right, StringComparison.OrdinalIgnoreCase), left8.StartsWith(right8, true));
Assert.AreEqual(right.StartsWith(left), right8.StartsWith(left8));
Assert.AreEqual(right.StartsWith(left, StringComparison.OrdinalIgnoreCase), right8.StartsWith(left8, true));
// Case Insensitive Stable is the insensitive order, then the sensitive order for ties
CompareResult caseInsensitiveStableExpected = (caseInsensitiveExpected == CompareResult.Equal ? caseSensitiveExpected : caseInsensitiveExpected);
Assert.AreEqual(caseInsensitiveStableExpected, ToResult(left8.CompareCaseInsensitiveStableTo(right8)), "Case insensitive stable String8 to string comparison result incorrect.");
}
public static string SplitAndJoin(string value)
{
String8 value8 = String8.Convert(value, new byte[String8.GetLength(value)]);
PartialArray <int> boundaryArray = new PartialArray <int>();
String8Set set = AlphanumericSplitter.Split(value8, ref boundaryArray);
bool firstPart = true;
StringBuilder result = new StringBuilder();
using (StringWriter writer = new StringWriter(result))
{
for (int i = 0; i < set.Count; ++i)
{
String8 part = set[i];
if (!part.IsEmpty() && AlphanumericSplitter.IsAlphaNumeric(part[0]))
{
if (!firstPart)
{
writer.Write("|");
}
firstPart = false;
part.WriteTo(writer);
}
}
}
return(result.ToString());
}
private static void ITabularValue_Basics(object value)
{
string valueString = null;
if (value != null)
{
if (value is DateTime)
{
valueString = ((DateTime)value).ToString("u");
}
else
{
valueString = value.ToString();
}
}
String8 value8 = String8.Convert(valueString, new byte[String8.GetLength(valueString) + 1], 1);
String8TabularValue itv8 = new String8TabularValue();
itv8.SetValue(value8);
ITabularValue_Basics(valueString ?? "", value8, itv8);
ObjectTabularValue otv = new ObjectTabularValue(new String8Block());
otv.SetValue(value);
ITabularValue_Basics(valueString, value8, otv);
}
private static void MatchContains(ITabularReader reader, ITabularWriter writer, WhereResult result)
{
string valueString = (string)result.Value;
String8 value = String8.Convert(valueString, new byte[String8.GetLength(valueString)]);
while (reader.NextRow())
{
// Ensure the row has enough columns
if (reader.CurrentRowColumns <= result.ColumnIndex)
{
continue;
}
// Match the value
if (reader.Current(result.ColumnIndex).ToString8().IndexOf(value) == -1)
{
continue;
}
result.MatchCount++;
// If this is the matching row, write it
EchoRow(reader, writer);
}
}
private string CsvSplitAndJoin(string value)
{
String8 value8 = String8.Convert(value, new byte[String8.GetLength(value)]);
String8Set set = value8.SplitAndDecodeCsvCells(new PartialArray <int>());
String8 joined8 = set.Join(UTF8.Pipe, new byte[set.Value.Length]);
return(joined8.ToString());
}
private string SplitOutsideQuotesAndJoin(string value, byte delimiter)
{
String8 value8 = String8.Convert(value, new byte[String8.GetLength(value)]);
String8Set set = value8.SplitOutsideQuotes(delimiter, new PartialArray <int>());
String8 joined8 = set.Join(UTF8.Pipe, new byte[set.Value.Length]);
return(joined8.ToString());
}
public void String8_IndexOf()
{
string binaryName = "System.Collections.Generic.List";
String8 binaryName8 = String8.Convert(binaryName, new byte[String8.GetLength(binaryName)]);
Assert.AreEqual(binaryName.IndexOf('.'), binaryName8.IndexOf((byte)'.'));
Assert.AreEqual(binaryName.IndexOf('.', 18), binaryName8.IndexOf((byte)'.', 18));
Assert.AreEqual(binaryName.LastIndexOf('.'), binaryName8.LastIndexOf((byte)'.'));
Assert.AreEqual(binaryName.LastIndexOf('.', 18), binaryName8.LastIndexOf((byte)'.', 18));
}
public IXColumn Build(IXTable source, XDatabaseContext context)
{
IXColumn text = context.Parser.NextColumn(source, context, typeof(String8));
string value = context.Parser.NextString();
String8 value8 = String8.Convert(value, new byte[String8.GetLength(value)]);
return(SimpleTransformFunction <String8, int> .Build(
Name,
source,
text,
(string8) => string8.IndexOf(value8)));
}
public IXColumn Build(IXTable source, XDatabaseContext context)
{
IXColumn value = context.Parser.NextColumn(source, context, typeof(String8));
string prefix = context.Parser.NextString();
String8 prefix8 = String8.Convert(prefix, new byte[String8.GetLength(prefix)]);
return(SimpleTransformFunction <String8, String8> .Build(
Name,
source,
value,
(string8) => AfterFirst(string8, prefix8)));
}
private static void WriteSampleFileWithIssues(Stream stream, Func <Stream, ITabularWriter> buildWriter)
{
Random r = new Random();
string huge = new string('Z', 100000);
String8 huge8 = String8.Convert(huge, new byte[String8.GetLength(huge)]);
String8 abcdef = String8.Convert("ABCDEF", new byte[6]);
using (ITabularWriter writer = buildWriter(stream))
{
writer.SetColumns(new string[] { "LineNumber", "Count", "Description" });
for (int i = writer.RowCountWritten + 1; i <= 10000; ++i)
{
if (i % 100 == 99)
{
// Write an empty row (1/100)
long rowStartPosition = stream.Position;
// Make the writer think everything is ok (it'll throw if you don't write enough values)
writer.Write(String8.Empty);
writer.Write(String8.Empty);
writer.WriteValueStart();
writer.WriteValueEnd();
// Wipe out what was written
stream.Seek(rowStartPosition, SeekOrigin.Begin);
}
else if (i == 5000)
{
// Write a huge row
writer.Write(i);
writer.WriteValueStart();
writer.WriteValuePart(r.Next(100000));
writer.WriteValueEnd();
writer.Write(huge8);
}
else
{
// Write a normal row
writer.Write(i);
writer.Write(r.Next(100000));
writer.Write(abcdef);
}
writer.NextRow();
}
}
}
private DateTime?TryToDateTime(string value)
{
String8 value8 = String8.Convert(value, new byte[String8.GetLength(value) + 1], 1);
DateTime?result = null;
DateTime parsed = DateTime.MinValue;
if (value8.TryToDateTime(out parsed))
{
result = parsed;
}
return(result);
}
private int?TryToInteger(string value)
{
String8 value8 = String8.Convert(value, new byte[String8.GetLength(value) + 1], 1);
int?result = null;
int parsed = 0;
if (value8.TryToInteger(out parsed))
{
result = parsed;
}
return(result);
}
private bool?TryToBoolean(string value)
{
String8 value8 = String8.Convert(value, new byte[String8.GetLength(value) + 1], 1);
bool?result = null;
bool parsed = false;
if (value8.TryToBoolean(out parsed))
{
result = parsed;
}
return(result);
}
public StringToString8Converter(object defaultValue)
{
if (defaultValue == null)
{
_defaultValue = String8.Empty;
}
else if (defaultValue is String8)
{
_defaultValue = (String8)defaultValue;
}
else
{
string defaultAsString = defaultValue.ToString();
_defaultValue = String8.Convert(defaultAsString, new byte[String8.GetLength(defaultAsString)]);
}
}
/// <summary>
/// Translate a single literal value to the sanitized form, given the value
/// and which column it is from. Uses the mapper configured for the column in
/// the spec file.
/// </summary>
/// <param name="value">Value to convert</param>
/// <param name="columnName">ColumnName value is from</param>
/// <returns>Sanitized version of value</returns>
public string Translate(string value, string columnName)
{
IColumnHandler handler;
// If there's no handler, there's no re-mapping
if (!this.HandlersByColumn.TryGetValue(columnName, out handler))
{
handler = new KeepColumnHandler();
}
// Convert and return the value
String8 value8 = String8.Convert(value, new byte[String8.GetLength(value)]);
String8 replacement = this.HandlersByColumn[columnName].Sanitize(value8);
return(replacement.ToString());
}
public void SetColumns(IEnumerable <string> columnNames)
{
if (_columnCount != 0)
{
throw new InvalidOperationException("SetColumns may only be called once on a JsonTabularWriter.");
}
// {
// "colIndex": {
s_beforeColumnNames.WriteTo(_stream);
int columnIndex = 0;
foreach (string columnName in columnNames)
{
int length = String8.GetLength(columnName);
if (_typeConversionBuffer == null || _typeConversionBuffer.Length < length)
{
_typeConversionBuffer = new byte[length];
}
// ,
if (columnIndex > 0)
{
s_valueDelimiter.WriteTo(_stream);
}
// "ColumnName"
_stream.WriteByte(UTF8.Quote);
WriteEscaped(String8.Convert(columnName, _typeConversionBuffer));
_stream.WriteByte(UTF8.Quote);
// : 0
_stream.WriteByte(UTF8.Colon);
_stream.WriteByte(UTF8.Space);
String8.FromInteger(columnIndex, _typeConversionBuffer).WriteTo(_stream);
columnIndex++;
}
// },
// "rows": {
s_afterColumnNames.WriteTo(_stream);
_columnCount = columnIndex;
}
private static string GetRangeValuesAsString(string prefix, IStringStore store)
{
Range matches;
if (!store.TryGetRangeStartingWith(String8.Convert(prefix, new byte[String8.GetLength(prefix)]), out matches))
{
return(string.Empty);
}
if (matches.Start == matches.End)
{
return(store[matches.Start].ToString());
}
else
{
return(string.Format("{0}-{1}", store[matches.Start], store[matches.End]));
}
}
public Sanitizer(string specFilePath, string hashKey)
{
this.Provider = new SanitizerProvider();
this.HashKeyHash = Hashing.Hash(String8.Convert(hashKey, new byte[String8.GetLength(hashKey)]), 0);
this.SpecFilePath = specFilePath;
// Track values to Echo. Empty string is always Echoed.
this.EchoValues = new HashSet <String8>();
this.EchoValues.Add(String8.Empty);
// Track columns to drop (exclude from output)
this.DropColumns = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
// Track the map handler for columns being mapped
this.HandlersByColumn = new Dictionary <string, IColumnHandler>(StringComparer.OrdinalIgnoreCase);
LoadSpec();
}
public IXColumn NextColumn(IXTable source, XDatabaseContext context, Type requiredType = null)
{
if (source == null)
{
throw new ArgumentNullException("source");
}
IXColumn result = null;
if (_scanner.Current.Type == TokenType.Value)
{
object value = String8.Convert(_scanner.Current.Value, new byte[String8.GetLength(_scanner.Current.Value)]);
if (requiredType != null && requiredType != typeof(String8))
{
value = TypeConverterFactory.ConvertSingle(value, requiredType);
}
result = new ConstantColumn(source, value, (requiredType == null ? typeof(String8) : requiredType), !_scanner.Current.IsWrapped);
_scanner.Next();
}
else if (_scanner.Current.Type == TokenType.FunctionName)
{
result = NextFunction(source, context);
}
else if (_scanner.Current.Type == TokenType.ColumnName)
{
result = source.Columns.Find(NextColumnName(source, requiredType));
}
if (result == null || (requiredType != null && result.ColumnDetails.Type != requiredType))
{
Throw("Col|Func|Const", EscapedColumnList(source, requiredType).Concat(EscapedFunctionList(requiredType)));
}
if (_scanner.Current.Value.Equals("as", StringComparison.OrdinalIgnoreCase))
{
_scanner.Next();
string columnName = NextOutputColumnName(source);
result = RenamedColumn.Build(result, columnName);
}
return(result);
}
public ToString8Converter(object defaultValue, int bytesPerItem, Func <T, byte[], int, String8> converter)
{
if (defaultValue == null)
{
_defaultValue = String8.Empty;
}
else if (defaultValue is String8)
{
_defaultValue = (String8)defaultValue;
}
else
{
string defaultAsString = defaultValue.ToString();
_defaultValue = String8.Convert(defaultAsString, new byte[String8.GetLength(defaultAsString)]);
}
_converter = converter;
_bytesPerItem = bytesPerItem;
}
public void SetLocation(int memberIndex, string filePath, ushort line, ushort charInLine)
{
// TODO: Handle '/' or '\' in ItemTree to avoid canonicalizing
if (!String.IsNullOrEmpty(filePath) && (filePath.StartsWith("http:", StringComparison.OrdinalIgnoreCase) || filePath.StartsWith("https:", StringComparison.OrdinalIgnoreCase)))
{
filePath = filePath.Replace('/', '\\');
}
// Find (or add) the file path to the File Tree
int fileIndex = 0;
if (!String.IsNullOrEmpty(filePath))
{
String8 path8 = String8.Convert(filePath, new byte[String8.GetLength(filePath)]);
String8Set splitPath8 = path8.Split('\\', new int[String8Set.GetLength(path8, '\\')]);
fileIndex = this.FileTree.AddPath(0, splitPath8, this.StringStore);
}
// Write the updated location
SymbolLocation location;
location.FileIndex = fileIndex;
location.Line = line;
location.CharInLine = charInLine;
if (this.DeclaredMemberLocations.Count == memberIndex)
{
this.DeclaredMemberLocations.Add(location);
}
else if (this.DeclaredMemberLocations.Count > memberIndex)
{
this.DeclaredMemberLocations[memberIndex] = location;
}
else
{
throw new InvalidOperationException(String.Format(Resources.DatabaseArraysOutOfSync, "DeclaredMemberLocations"));
}
}
private bool TryFindAssembly(string name, out Symbol assembly)
{
assembly = default(Symbol);
String8 name8 = String8.Convert(name, new byte[String8.GetLength(name)]);
Range nameRange;
if (!this.Previous.StringStore.TryFindString(name8, out nameRange))
{
return(false);
}
int matchIndex;
if (!this.Previous.DeclaredMembers.TryFindChildByName(1, nameRange, out matchIndex))
{
return(false);
}
assembly = new Symbol(this.Previous, matchIndex);
return(true);
}
public MutableSymbol FindOrAddPath(string path, char delimiter, SymbolType pathPartType)
{
String8 path8 = String8.Convert(path, new byte[String8.GetLength(path)]);
String8Set splitPath8 = path8.Split(delimiter, new int[String8Set.GetLength(path8, delimiter)]);
int currentIndex = _index;
for (int i = 0; i < splitPath8.Count; ++i)
{
String8 part = splitPath8[i];
int partNameIdentifier = _database.StringStore.FindOrAddString(part);
int foundNode;
if (!_database.DeclaredMembers.TryFindChildByName(currentIndex, partNameIdentifier, out foundNode))
{
foundNode = _database.DeclaredMembers.Add(currentIndex, partNameIdentifier);
_database.DeclaredMemberDetails.Add(new SymbolDetails() { Type = pathPartType });
_database.DeclaredMemberLocations.Add(default(SymbolLocation));
}
currentIndex = foundNode;
}
return new MutableSymbol(_database, currentIndex);
}
private static void ConcatenateColumn(string inputFilePath, string outputFilePath, string columnName1, string separator, string columnName2, string outputColumnName)
{
String8 separator8 = String8.Convert(separator, new byte[String8.GetLength(separator)]);
using (ITabularReader reader = TabularFactory.BuildReader(inputFilePath))
{
// Find the columns to concatenate
int columnIndex1 = reader.ColumnIndex(columnName1);
int columnIndex2 = reader.ColumnIndex(columnName2);
// Build an output column list and mapping from output order to input index, with '-1' for the concatenated value
List <string> outputColumns = new List <string>();
int[] indexMapping = new int[reader.Columns.Count - 1];
bool hasConcatenatedColumn = false;
for (int i = 0; i < reader.Columns.Count; ++i)
{
string columnName = reader.Columns[i];
// If this is a column to concatenate...
if (columnName.Equals(reader.Columns[columnIndex1], StringComparison.OrdinalIgnoreCase) ||
columnName.Equals(reader.Columns[columnIndex2], StringComparison.OrdinalIgnoreCase))
{
// .. if it's the first one, the output column will appear at this position
if (!hasConcatenatedColumn)
{
hasConcatenatedColumn = true;
indexMapping[outputColumns.Count] = -1;
outputColumns.Add(outputColumnName);
}
}
else
{
// Otherwise, copy this column through
indexMapping[outputColumns.Count] = i;
outputColumns.Add(columnName);
}
}
using (ITabularWriter writer = TabularFactory.BuildWriter(outputFilePath))
{
writer.SetColumns(outputColumns);
while (reader.NextRow())
{
// Write columns in mapped order
for (int i = 0; i < indexMapping.Length; ++i)
{
int sourceColumnIndex = indexMapping[i];
if (sourceColumnIndex == -1)
{
// Write concatenated column
writer.WriteValueStart();
writer.WriteValuePart(reader.Current(columnIndex1).ToString8());
writer.WriteValuePart(separator8);
writer.WriteValuePart(reader.Current(columnIndex2).ToString8());
writer.WriteValueEnd();
}
else
{
writer.Write(reader.Current(sourceColumnIndex).ToString8());
}
}
writer.NextRow();
}
WriteSizeSummary(reader, writer);
}
}
}
public void String8_Basics()
{
// Create a shared byte buffer for String8 conversions
byte[] buffer = new byte[1024 + 4];
// Null and Empty conversion
Assert.IsTrue(((string)null).TestConvert().IsEmpty());
Assert.IsTrue(string.Empty.TestConvert().IsEmpty());
// Null and Empty comparison [both ways]
Assert.AreEqual(0, new String8(null, 0, 0).CompareTo(string.Empty.TestConvert()));
Assert.AreEqual(0, string.Empty.TestConvert().CompareTo(new String8(null, 0, 0)));
// Sort sample strings by ordinal
string[] samples = new string[] { "A", "AA", "AAA", "short", "four", "!@#$%^&*()", "A\U00020213C", "Diagnostics", "NonePadded", new string('z', 1024) };
Array.Sort(samples, StringComparer.Ordinal);
// Convert each string into a new array [allocation is obvious in API]
String8[] converted = new String8[samples.Length];
for (int i = 0; i < samples.Length; ++i)
{
int length = String8.GetLength(samples[i]);
converted[i] = String8.Convert(samples[i], new byte[length + 1], 1);
}
// Verify round-trip
StringBuilder sb = new StringBuilder();
using (StringWriter writer = new StringWriter(sb))
{
for (int i = 0; i < samples.Length; ++i)
{
// Round-Trip via ToString()
Assert.AreEqual(samples[i], converted[i].ToString());
// Round-Trip via byte[]
int length = converted[i].WriteTo(buffer, 0);
Assert.AreEqual(samples[i], Encoding.UTF8.GetString(buffer, 0, length));
// Round-Trip via TextWriter
converted[i].WriteTo(writer);
Assert.AreEqual(samples[i], sb.ToString());
sb.Clear();
}
}
// Verify non-empty and length
for (int i = 0; i < samples.Length; ++i)
{
Assert.IsFalse(converted[i].IsEmpty());
int expectedLength = Encoding.UTF8.GetByteCount(samples[i]);
Assert.AreEqual(expectedLength, converted[i].Length);
}
// Verify they compare correctly
for (int i = 1; i < samples.Length; ++i)
{
// This is less than the one after it
Assert.IsTrue(converted[i - 1].CompareTo(converted[i]) < 0);
// This is more than the one before it
Assert.IsTrue(converted[i].CompareTo(converted[i - 1]) > 0);
// This equals itself
Assert.IsTrue(converted[i].CompareTo(converted[i]) == 0);
// This equals a new instance of the same value
Assert.IsTrue(converted[i].CompareTo(samples[i].TestConvert()) == 0);
// This is greater than a prefix of itself
Assert.IsTrue(converted[i].CompareTo(String8.Convert(samples[i], 0, samples[i].Length - 1, buffer)) > 0);
// This is less than itself plus another character
Assert.IsTrue(converted[i].CompareTo(String8.Convert(samples[i] + "A", buffer)) < 0);
}
}
public static String8 TestConvert(this string value)
{
// Convert the string to a String8, *but not at the very start of the buffer*.
// This is a common source of bugs.
return(String8.Convert(value, new byte[String8.GetLength(value) + 1], 1));
}
public MutableSymbol FindByFullName(string path, char delimiter)
{
String8 path8 = String8.Convert(path, new byte[String8.GetLength(path)]);
String8Set splitPath8 = path8.Split(delimiter, new int[String8Set.GetLength(path8, delimiter)]);
return new MutableSymbol(_database, _database.DeclaredMembers.FindByPath(_index, splitPath8, _database.StringStore));
}
public String8 GetPackageName(int memberIndex)
{
// Return the package name from the PackageIdentity
return(String8.Convert(this.Identity.PackageName, new byte[String8.GetLength(this.Identity.PackageName)]));
}