public void JsonTabularWriter_Escaping()
{
StringBuilder allEscapedValues = new StringBuilder();
// All 0x00 - 0x1F are escaped
for (int i = 0; i < 32; ++i)
{
allEscapedValues.Append((char)i);
}
// Backslash is escaped
allEscapedValues.Append('\\');
// Quote is escaped
allEscapedValues.Append('"');
string all = allEscapedValues.ToString();
String8 all8 = String8.Convert(all, new byte[String8.GetLength(all)]);
using (ITabularWriter w = new JsonTabularWriter("Escaping.json"))
{
w.SetColumns(new string[] { "Bad" });
w.Write(all8);
w.NextRow();
}
string content = File.ReadAllText("Escaping.json");
Assert.IsTrue(content.IndexOf("\\u0000\\u0001\\u0002\\u0003\\u0004\\u0005\\u0006\\u0007\\u0008\\u0009\\u000A\\u000B\\u000C\\u000D\\u000E\\u000F\\u0010\\u0011\\u0012\\u0013\\u0014\\u0015\\u0016\\u0017\\u0018\\u0019\\u001A\\u001B\\u001C\\u001D\\u001E\\u001F\\\\\\\"") >= 0);
}
private static void Compare(string oldFilePath, string newFilePath, string outputFilePath, string columnIdentifier)
{
String8Block block = new String8Block();
HashSet <String8> oldValues = new HashSet <String8>();
HashSet <String8> newValues = new HashSet <String8>();
using (ITabularReader oldReader = TabularFactory.BuildReader(oldFilePath))
{
int leftColumnIndex = oldReader.ColumnIndex(columnIdentifier);
while (oldReader.NextRow())
{
oldValues.Add(block.GetCopy(oldReader.Current(leftColumnIndex)));
}
Trace.WriteLine(String.Format("Old: {0:n0} values for \"{1}\" in {2:n0} rows.", oldValues.Count, columnIdentifier, oldReader.RowCountRead));
}
using (ITabularReader newReader = TabularFactory.BuildReader(newFilePath))
{
int rightColumnIndex = newReader.ColumnIndex(columnIdentifier);
while (newReader.NextRow())
{
newValues.Add(block.GetCopy(newReader.Current(rightColumnIndex)));
}
Trace.WriteLine(String.Format("New: {0:n0} values for \"{1}\" in {2:n0} rows.", newValues.Count, columnIdentifier, newReader.RowCountRead));
}
HashSet <String8> oldOnly = new HashSet <String8>(oldValues);
oldOnly.ExceptWith(newValues);
HashSet <String8> newOnly = new HashSet <String8>(newValues);
newOnly.ExceptWith(oldValues);
Trace.WriteLine(String.Format("{0:n0} values were only in \"{1}\".\r\n{2:n0} values were only in \"{3}\".", oldOnly.Count, oldFilePath, newOnly.Count, newFilePath));
String8 leftMarker = String8.Convert("-", new byte[1]);
String8 rightMarker = String8.Convert("+", new byte[1]);
using (ITabularWriter writer = TabularFactory.BuildWriter(outputFilePath))
{
writer.SetColumns(new string[] { "In", columnIdentifier });
foreach (String8 value in oldOnly)
{
writer.Write(leftMarker);
writer.Write(value);
writer.NextRow();
}
foreach (String8 value in newOnly)
{
writer.Write(rightMarker);
writer.Write(value);
writer.NextRow();
}
}
}
public static int FindByPath(ItemTree tree, StringStore strings, string path, char delimiter = '\\')
{
String8 path8 = String8.Convert(path, new byte[String8.GetLength(path)]);
String8Set pathSplit8 = path8.Split(delimiter, new int[String8Set.GetLength(path8, delimiter)]);
return(tree.FindByPath(0, pathSplit8, strings));
}
private void EnsureComparesConsistent(string left, string right)
{
String8 left8 = String8.Convert(left, new byte[String8.GetLength(left)]);
String8 right8 = String8.Convert(right, new byte[String8.GetLength(right)]);
CompareResult caseSensitiveExpected = ToResult(String.Compare(left, right, StringComparison.Ordinal));
CompareResult caseInsensitiveExpected = ToResult(String.Compare(left, right, StringComparison.OrdinalIgnoreCase));
Assert.AreEqual(caseSensitiveExpected, ToResult(left8.CompareTo(right8)), "Case sensitive comparison result incorrect.");
Assert.AreEqual(caseInsensitiveExpected, ToResult(left8.CompareTo(right8, true)), "Case insensitive comparison result incorrect.");
Assert.AreEqual(caseSensitiveExpected, ToResult(left8.CompareTo(right)), "Case sensitive String8 to string comparison result incorrect.");
Assert.AreEqual(caseInsensitiveExpected, ToResult(left8.CompareTo(right, true)), "Case insensitive String8 to string comparison result incorrect.");
// StartsWith and CompareAsPrefixTo
Assert.AreEqual(left.StartsWith(right), left8.StartsWith(right8));
Assert.AreEqual(left.StartsWith(right, StringComparison.OrdinalIgnoreCase), left8.StartsWith(right8, true));
Assert.AreEqual(right.StartsWith(left), right8.StartsWith(left8));
Assert.AreEqual(right.StartsWith(left, StringComparison.OrdinalIgnoreCase), right8.StartsWith(left8, true));
// Case Insensitive Stable is the insensitive order, then the sensitive order for ties
CompareResult caseInsensitiveStableExpected = (caseInsensitiveExpected == CompareResult.Equal ? caseSensitiveExpected : caseInsensitiveExpected);
Assert.AreEqual(caseInsensitiveStableExpected, ToResult(left8.CompareCaseInsensitiveStableTo(right8)), "Case insensitive stable String8 to string comparison result incorrect.");
}
private static void ITabularValue_Basics(object value)
{
string valueString = null;
if (value != null)
{
if (value is DateTime)
{
valueString = ((DateTime)value).ToString("u");
}
else
{
valueString = value.ToString();
}
}
String8 value8 = String8.Convert(valueString, new byte[String8.GetLength(valueString) + 1], 1);
String8TabularValue itv8 = new String8TabularValue();
itv8.SetValue(value8);
ITabularValue_Basics(valueString ?? "", value8, itv8);
ObjectTabularValue otv = new ObjectTabularValue(new String8Block());
otv.SetValue(value);
ITabularValue_Basics(valueString, value8, otv);
}
public void Writer_WriteValidUsingAllOverloads(Stream stream, Func <Stream, ITabularWriter> buildWriter)
{
String8Set names = String8Set.Split(String8.Convert("Jeff,Bill,Todd,\\Barry\\", new byte[30]), UTF8.Comma, new int[5]);
using (ITabularWriter w = buildWriter(stream))
{
Assert.AreEqual(0, w.RowCountWritten);
w.SetColumns(new string[] { "ID", "IsEven", "Backslash", "Today", "Name", "Description" });
Assert.AreEqual(0, w.RowCountWritten);
for (int i = 0; i < 10; ++i)
{
w.Write(i);
w.Write(i % 2 == 0);
w.Write(UTF8.Backslash);
w.Write(new DateTime(2017, 05, 03, 0, 0, 0, DateTimeKind.Utc));
w.Write(names[i % names.Count]);
w.WriteValueStart();
w.WriteValuePart(i + 1);
w.WriteValuePart(i % 2 == 1);
w.WriteValuePart(UTF8.Quote);
w.WriteValuePart(new DateTime(2017, 05, 01, 0, 0, 0, DateTimeKind.Utc));
w.WriteValuePart(names[i % names.Count]);
w.WriteValueEnd();
Assert.AreEqual(i, w.RowCountWritten);
w.NextRow();
Assert.AreEqual(i + 1, w.RowCountWritten);
Assert.AreEqual(stream.Position, w.BytesWritten);
}
}
}
public void String8_ComparePerformance()
{
// Ten sample strings
string[] strings = { null, "", "Array", "ArrayList", "Boolean", "Collections", "Dictionary", "Dictionary<string, int>", "System.Collections.Generic.Array", "System.Collections.Generic.ArrayList" };
String8[] values = new String8[strings.Length];
// Convert into two buffers, half to each
byte[] buffer = new byte[1024];
byte[] buffer2 = new byte[1024];
int usedSpace = 0;
for (int i = 0; i < strings.Length; ++i)
{
values[i] = String8.Convert(strings[i], (i % 2 == 0 ? buffer : buffer2), usedSpace);
usedSpace += values[i].Length;
}
// Compare every combination of values (half within and half across buffers)
// Goal: 500k/sec [case sensitive]
int iterations = 100000;
Verify.PerformanceByOperation(500 * LongExtensions.Thousand, () =>
{
RunAllComparisons(values, false, iterations);
return(iterations * values.Length * values.Length);
});
// Goal: 400k/sec [case insensitive]
Verify.PerformanceByOperation(400 * LongExtensions.Thousand, () =>
{
RunAllComparisons(values, true, iterations);
return(iterations * values.Length * values.Length);
});
}
private static void MatchContains(ITabularReader reader, ITabularWriter writer, WhereResult result)
{
string valueString = (string)result.Value;
String8 value = String8.Convert(valueString, new byte[String8.GetLength(valueString)]);
while (reader.NextRow())
{
// Ensure the row has enough columns
if (reader.CurrentRowColumns <= result.ColumnIndex)
{
continue;
}
// Match the value
if (reader.Current(result.ColumnIndex).ToString8().IndexOf(value) == -1)
{
continue;
}
result.MatchCount++;
// If this is the matching row, write it
EchoRow(reader, writer);
}
}
public static string SplitAndJoin(string value)
{
String8 value8 = String8.Convert(value, new byte[String8.GetLength(value)]);
PartialArray <int> boundaryArray = new PartialArray <int>();
String8Set set = AlphanumericSplitter.Split(value8, ref boundaryArray);
bool firstPart = true;
StringBuilder result = new StringBuilder();
using (StringWriter writer = new StringWriter(result))
{
for (int i = 0; i < set.Count; ++i)
{
String8 part = set[i];
if (!part.IsEmpty() && AlphanumericSplitter.IsAlphaNumeric(part[0]))
{
if (!firstPart)
{
writer.Write("|");
}
firstPart = false;
part.WriteTo(writer);
}
}
}
return(result.ToString());
}
public void Utf8ToUtf16()
{
byte[] buffer = null;
String8 value = String8.Convert("abc def", ref buffer);
String8 replacement;
// Bounds
Assert.Equal(0, String8.Utf8ToUtf16(0, value));
Assert.Equal(value.Length, String8.Utf8ToUtf16(value.Length, value));
// All single byte: Verify all indices are the same
for (int i = 0; i < value.Length; ++i)
{
Assert.Equal(i, String8.Utf8ToUtf16(i, value));
if (i > 0)
{
Assert.Equal(i, String8.Utf8ToUtf16(i, value, i - 1, i - 1));
}
}
// Place a two-byte character right after 'abc' [¼]
replacement = String8.Convert("\u00BC", ref buffer, 3);
Assert.Equal(2, replacement.Length);
// 0123345678 [Index 3-4 are char 3]
// abc.. def
Assert.Equal(2, String8.Utf8ToUtf16(2, value));
Assert.Equal(3, String8.Utf8ToUtf16(3, value));
Assert.Equal(3, String8.Utf8ToUtf16(4, value));
Assert.Equal(4, String8.Utf8ToUtf16(5, value));
Assert.Equal(5, String8.Utf8ToUtf16(6, value));
// Place a three byte character right after 'Two' [ᚠ]
replacement = String8.Convert("\u16A0", ref buffer, 3);
Assert.Equal(3, replacement.Length);
// 0123334567 [Index 3-5 are char 3]
// abc... def
Assert.Equal(2, String8.Utf8ToUtf16(2, value));
Assert.Equal(3, String8.Utf8ToUtf16(3, value));
Assert.Equal(3, String8.Utf8ToUtf16(4, value));
Assert.Equal(3, String8.Utf8ToUtf16(5, value));
Assert.Equal(4, String8.Utf8ToUtf16(6, value));
Assert.Equal(5, String8.Utf8ToUtf16(7, value));
// Place a four byte character right after 'Two' [𐤈]
replacement = String8.Convert("\U00010908", ref buffer, 3);
Assert.Equal(4, replacement.Length);
// 0123333567 [Index 3-6 are char 3 AND it's length 2, so next is index 5]
// abc....def
Assert.Equal(2, String8.Utf8ToUtf16(2, value));
Assert.Equal(3, String8.Utf8ToUtf16(3, value));
Assert.Equal(3, String8.Utf8ToUtf16(4, value));
Assert.Equal(3, String8.Utf8ToUtf16(5, value));
Assert.Equal(3, String8.Utf8ToUtf16(6, value));
Assert.Equal(5, String8.Utf8ToUtf16(7, value));
Assert.Equal(6, String8.Utf8ToUtf16(8, value));
}
private string CsvSplitAndJoin(string value)
{
String8 value8 = String8.Convert(value, new byte[String8.GetLength(value)]);
String8Set set = value8.SplitAndDecodeCsvCells(new PartialArray <int>());
String8 joined8 = set.Join(UTF8.Pipe, new byte[set.Value.Length]);
return(joined8.ToString());
}
private string SplitOutsideQuotesAndJoin(string value, byte delimiter)
{
String8 value8 = String8.Convert(value, new byte[String8.GetLength(value)]);
String8Set set = value8.SplitOutsideQuotes(delimiter, new PartialArray <int>());
String8 joined8 = set.Join(UTF8.Pipe, new byte[set.Value.Length]);
return(joined8.ToString());
}
public void String8_Prefix()
{
String8 full = String8.Convert("One.Two.Three", new byte[13]);
String8 start = String8.Convert("One", new byte[3]);
String8 part = String8.Convert("Two", new byte[3]);
String8 startInsensitive = String8.Convert("ONE", new byte[3]);
Assert.AreEqual(0, start.CompareAsPrefixTo(full));
}
public void FindAllMatches(string input)
{
byte[] buffer = null;
String8 input8 = String8.Convert(input, ref buffer);
var e = Regex2.Matches(input8, Regex).GetEnumerator();
while (e.MoveNext())
{
}
}
public void String8_IndexOf()
{
string binaryName = "System.Collections.Generic.List";
String8 binaryName8 = String8.Convert(binaryName, new byte[String8.GetLength(binaryName)]);
Assert.AreEqual(binaryName.IndexOf('.'), binaryName8.IndexOf((byte)'.'));
Assert.AreEqual(binaryName.IndexOf('.', 18), binaryName8.IndexOf((byte)'.', 18));
Assert.AreEqual(binaryName.LastIndexOf('.'), binaryName8.LastIndexOf((byte)'.'));
Assert.AreEqual(binaryName.LastIndexOf('.', 18), binaryName8.LastIndexOf((byte)'.', 18));
}
public void String8_ToUpper()
{
// Verify no exception
String8.Empty.ToUpperInvariant();
String8 sample = String8.Convert("abcABC", new byte[6]);
sample.ToUpperInvariant();
Assert.AreEqual("ABCABC", sample.ToString());
}
public void ImmutableStringStore_Basic()
{
// Set of strings to index [out of order to verify GetSerializationIdentifier]
string[] strings = { "Boolean", "Abacus", "Array", "ArrayList", "Editor", "Collections", "Dictionary" };
int[] identifiers = new int[strings.Length];
// Add values to a mutable store and track identifiers
MutableStringStore store = new MutableStringStore();
for (int i = 0; i < strings.Length; ++i)
{
identifiers[i] = store.FindOrAddString(strings[i]);
}
// Convert to immutable
IStringStore iStore = Convert(store, identifiers);
// Verify each value is found at the expected position
byte[] buffer = new byte[32];
for (int i = 0; i < strings.Length; ++i)
{
String8 value = String8.Convert(strings[i], buffer);
Range foundAtIndex;
Assert.IsTrue(iStore.TryFindString(value, out foundAtIndex), "ImmutableStore didn't contain added value \"{0}\"", strings[i]);
Assert.AreEqual(identifiers[i], foundAtIndex.Start, "ImmutableStore didn't find value at SerializationIdentifier position");
Assert.AreEqual(value, iStore[foundAtIndex.Start], "ImmutableStore didn't rebuild string with same value");
}
// Verify values not in collection aren't found and (single) insertion positions are returned
Range matches;
Assert.IsFalse(iStore.TryFindString("ZZ AfterLastValue", out matches));
Assert.AreEqual((FindIdentifier("Editor", strings, identifiers) + 1).ToString(), matches.ToString());
Assert.IsFalse(iStore.TryFindString("AA BeforeFirstValue", out matches));
Assert.AreEqual(FindIdentifier("Abacus", strings, identifiers).ToString(), matches.ToString());
Assert.IsFalse(iStore.TryFindString("Bz Between Boolean and Collections", out matches));
Assert.AreEqual(FindIdentifier("Collections", strings, identifiers).ToString(), matches.ToString());
Assert.IsFalse(iStore.TryFindString("Cz Between Collections and Dictionary", out matches));
Assert.AreEqual(FindIdentifier("Dictionary", strings, identifiers).ToString(), matches.ToString());
// Check range searches
Assert.AreEqual("Array-ArrayList", GetRangeValuesAsString("Arr", iStore), "Prefix covering multiple items should match all");
Assert.AreEqual("Array-ArrayList", GetRangeValuesAsString("Array", iStore), "Prefix equalling an item should include it");
Assert.AreEqual("ArrayList", GetRangeValuesAsString("ArrayL", iStore), "Prefix of only one item has only it");
Assert.AreEqual(string.Empty, GetRangeValuesAsString("ArrayList2", iStore), "Prefix longer than item should not match");
Assert.AreEqual("Abacus-ArrayList", GetRangeValuesAsString("A", iStore), "Prefix containing first item should include it");
Assert.AreEqual("Editor", GetRangeValuesAsString("Edit", iStore), "Prefix containing last item should include it");
Assert.AreEqual(string.Empty, GetRangeValuesAsString("AA", iStore), "Prefix before first item has empty range");
Assert.AreEqual(string.Empty, GetRangeValuesAsString("ZZ", iStore), "Prefix after last item has empty range");
}
public IXColumn Build(IXTable source, XDatabaseContext context)
{
IXColumn value = context.Parser.NextColumn(source, context, typeof(String8));
string prefix = context.Parser.NextString();
String8 prefix8 = String8.Convert(prefix, new byte[String8.GetLength(prefix)]);
return(SimpleTransformFunction <String8, String8> .Build(
Name,
source,
value,
(string8) => AfterFirst(string8, prefix8)));
}
public IXColumn Build(IXTable source, XDatabaseContext context)
{
IXColumn text = context.Parser.NextColumn(source, context, typeof(String8));
string value = context.Parser.NextString();
String8 value8 = String8.Convert(value, new byte[String8.GetLength(value)]);
return(SimpleTransformFunction <String8, int> .Build(
Name,
source,
text,
(string8) => string8.IndexOf(value8)));
}
private static void WriteSampleFileWithIssues(Stream stream, Func <Stream, ITabularWriter> buildWriter)
{
Random r = new Random();
string huge = new string('Z', 100000);
String8 huge8 = String8.Convert(huge, new byte[String8.GetLength(huge)]);
String8 abcdef = String8.Convert("ABCDEF", new byte[6]);
using (ITabularWriter writer = buildWriter(stream))
{
writer.SetColumns(new string[] { "LineNumber", "Count", "Description" });
for (int i = writer.RowCountWritten + 1; i <= 10000; ++i)
{
if (i % 100 == 99)
{
// Write an empty row (1/100)
long rowStartPosition = stream.Position;
// Make the writer think everything is ok (it'll throw if you don't write enough values)
writer.Write(String8.Empty);
writer.Write(String8.Empty);
writer.WriteValueStart();
writer.WriteValueEnd();
// Wipe out what was written
stream.Seek(rowStartPosition, SeekOrigin.Begin);
}
else if (i == 5000)
{
// Write a huge row
writer.Write(i);
writer.WriteValueStart();
writer.WriteValuePart(r.Next(100000));
writer.WriteValueEnd();
writer.Write(huge8);
}
else
{
// Write a normal row
writer.Write(i);
writer.Write(r.Next(100000));
writer.Write(abcdef);
}
writer.NextRow();
}
}
}
public void String8Block_Basics()
{
String8Block block = new String8Block();
byte[] buffer = new byte[4096];
String8 value = String8.Convert("hello", buffer);
// Verify copies are persistent when the original buffer is overwritten
String8 valueCopy = block.GetCopy(value);
String8.Convert("there", buffer);
Assert.AreEqual("there", value.ToString());
Assert.AreEqual("hello", valueCopy.ToString());
// Verify copy of String8.Empty works
String8 emptyCopy = block.GetCopy(String8.Empty);
Assert.IsTrue(emptyCopy.IsEmpty());
// Verify large strings are copied correctly (stored individually)
value = String8.Convert(new string('A', 4096), buffer);
valueCopy = block.GetCopy(value);
Assert.IsTrue(value.Equals(valueCopy));
String8.Convert(new string('B', 4096), buffer);
Assert.IsFalse(value.Equals(valueCopy));
// Verify storage uses multiple blocks correctly
for (int i = 0; i < 1000; ++i)
{
value = String8.Convert(new string((char)('0' + (i % 10)), 100), buffer);
valueCopy = block.GetCopy(value);
Assert.IsTrue(value.Equals(valueCopy));
}
// Verify conversion of strings
String8 directConversion = block.GetCopy("Regular String");
Assert.AreEqual("Regular String", directConversion.ToString());
// Verify null/empty string conversion
directConversion = block.GetCopy((string)null);
Assert.IsTrue(directConversion.IsEmpty());
directConversion = block.GetCopy(String.Empty);
Assert.IsTrue(directConversion.IsEmpty());
// Verify clear works (doesn't throw, GetCopy works afterward)
block.Clear();
valueCopy = block.GetCopy("Third");
Assert.AreEqual("Third", valueCopy.ToString());
}
// Get the integer ID of the cached copy of the Regex from the native side; cache it if it hasn't been parsed.
private static unsafe int BuildRegex(ParsedRegexCache cache, string expression, RegexOptions options)
{
if (string.IsNullOrEmpty(expression))
{
throw new ArgumentNullException(nameof(expression));
}
try
{
var key = Tuple.Create <string, RegexOptions>(expression, options);
int expressionIndex;
if (!cache.TryGetValue(key, out expressionIndex))
{
// Remove named groups from the expression (on add only, so once per Regex only)
expression = RemoveNamedGroups(expression);
byte[] buffer = null;
var expression8 = String8.Convert(expression, ref buffer);
// The native BuildRegex code is thread-safe for creating compiled expressions.
fixed(byte *expressionPtr = expression8.Array)
{
expressionIndex = NativeMethods.BuildRegex(new String8Interop(expressionPtr, expression8.Index, expression8.Length), (int)options);
}
// Throw if RE2 couldn't parse the regex.
// Error Text is native allocated and so not returned; it's written to the console by RE2.
if (expressionIndex == -1)
{
throw new ArgumentException($"RE2 could not parse regular expression \"{expression}\".");
}
// Throw if RE2 couldn't support a passed RegexOption.
if (expressionIndex == -2)
{
throw new ArgumentException($"RE2 doesn't support a passed RegexOption. Supported Options: Singleline, IgnoreCase. Options passed: {options}");
}
cache[key] = expressionIndex;
}
return(expressionIndex);
}
catch (DllNotFoundException ex)
{
// Throw a clearer exception if RE2.Native.*.dll wasn't found in any of the DLL loading paths.
throw new InvalidOperationException($"RE2.Native.*.dll was not found. It's required for RE2.Managed to run. Place RE2.Native.*.dll next to RE2.Managed.dll in '{Assembly.GetExecutingAssembly().Location}'. HR: {ex.HResult}", ex);
}
}
private DateTime?TryToDateTime(string value)
{
String8 value8 = String8.Convert(value, new byte[String8.GetLength(value) + 1], 1);
DateTime?result = null;
DateTime parsed = DateTime.MinValue;
if (value8.TryToDateTime(out parsed))
{
result = parsed;
}
return(result);
}
private int?TryToInteger(string value)
{
String8 value8 = String8.Convert(value, new byte[String8.GetLength(value) + 1], 1);
int?result = null;
int parsed = 0;
if (value8.TryToInteger(out parsed))
{
result = parsed;
}
return(result);
}
private bool?TryToBoolean(string value)
{
String8 value8 = String8.Convert(value, new byte[String8.GetLength(value) + 1], 1);
bool?result = null;
bool parsed = false;
if (value8.TryToBoolean(out parsed))
{
result = parsed;
}
return(result);
}
/// <summary>
/// Translate a single literal value to the sanitized form, given the value
/// and which column it is from. Uses the mapper configured for the column in
/// the spec file.
/// </summary>
/// <param name="value">Value to convert</param>
/// <param name="columnName">ColumnName value is from</param>
/// <returns>Sanitized version of value</returns>
public string Translate(string value, string columnName)
{
IColumnHandler handler;
// If there's no handler, there's no re-mapping
if (!this.HandlersByColumn.TryGetValue(columnName, out handler))
{
handler = new KeepColumnHandler();
}
// Convert and return the value
String8 value8 = String8.Convert(value, new byte[String8.GetLength(value)]);
String8 replacement = this.HandlersByColumn[columnName].Sanitize(value8);
return(replacement.ToString());
}
public StringToString8Converter(object defaultValue)
{
if (defaultValue == null)
{
_defaultValue = String8.Empty;
}
else if (defaultValue is String8)
{
_defaultValue = (String8)defaultValue;
}
else
{
string defaultAsString = defaultValue.ToString();
_defaultValue = String8.Convert(defaultAsString, new byte[String8.GetLength(defaultAsString)]);
}
}
public void Regex2_Timeout()
{
byte[] buffer = null;
var sample = String8.Convert("using Microsoft.VisualStudio.TestTools.UnitTesting;", ref buffer);
// Long timeout: Verify all matches returned
var timeout = Timeout.Start(TimeSpan.FromSeconds(10));
Assert.Equal(51, Regex2.Matches(sample, ".", RegexOptions.None, timeout).Count());
Assert.False(timeout.IsExpired);
// Tiny timeout: Verify early stop
timeout = Timeout.Start(TimeSpan.FromTicks(1));
Assert.NotEqual(51, Regex2.Matches(sample, ".", RegexOptions.None, timeout).Count());
Assert.True(timeout.IsExpired);
}
public void SetColumns(IEnumerable <string> columnNames)
{
if (_columnCount != 0)
{
throw new InvalidOperationException("SetColumns may only be called once on a JsonTabularWriter.");
}
// {
// "colIndex": {
s_beforeColumnNames.WriteTo(_stream);
int columnIndex = 0;
foreach (string columnName in columnNames)
{
int length = String8.GetLength(columnName);
if (_typeConversionBuffer == null || _typeConversionBuffer.Length < length)
{
_typeConversionBuffer = new byte[length];
}
// ,
if (columnIndex > 0)
{
s_valueDelimiter.WriteTo(_stream);
}
// "ColumnName"
_stream.WriteByte(UTF8.Quote);
WriteEscaped(String8.Convert(columnName, _typeConversionBuffer));
_stream.WriteByte(UTF8.Quote);
// : 0
_stream.WriteByte(UTF8.Colon);
_stream.WriteByte(UTF8.Space);
String8.FromInteger(columnIndex, _typeConversionBuffer).WriteTo(_stream);
columnIndex++;
}
// },
// "rows": {
s_afterColumnNames.WriteTo(_stream);
_columnCount = columnIndex;
}
public Sanitizer(string specFilePath, string hashKey)
{
this.Provider = new SanitizerProvider();
this.HashKeyHash = Hashing.Hash(String8.Convert(hashKey, new byte[String8.GetLength(hashKey)]), 0);
this.SpecFilePath = specFilePath;
// Track values to Echo. Empty string is always Echoed.
this.EchoValues = new HashSet <String8>();
this.EchoValues.Add(String8.Empty);
// Track columns to drop (exclude from output)
this.DropColumns = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
// Track the map handler for columns being mapped
this.HandlersByColumn = new Dictionary <string, IColumnHandler>(StringComparer.OrdinalIgnoreCase);
LoadSpec();
}
