public void Initialize()
{
String8Block block = new String8Block();
String8[] names = new String8[] { block.GetCopy("Scott"), block.GetCopy("Mike"), block.GetCopy("Jeff"), block.GetCopy("Sophie") };
using (ITabularWriter sample = TabularFactory.BuildWriter(s_sampleFilePath))
{
sample.SetColumns(new string[] { "ID", "IsEven", "Count", "WhenAdded", "Name" });
int sum = 0;
for (int i = 0; i < 1000; ++i)
{
sum += i;
sample.Write(i);
sample.Write((i & 0x1) == 0);
sample.Write(sum);
sample.Write(new DateTime(2017, 05, 23).AddMinutes(i));
sample.Write(names[i % names.Length]);
sample.NextRow();
}
}
}
public unsafe void Hashing_Basic()
{
String8Block block = new String8Block();
HashSet <ulong> values = new HashSet <ulong>();
// Integers: Verify hashing, verify hashes all unique for a large set
for (ulong i = 0; i < 100000; ++i)
{
ulong partHash = Hashing.Hash(i, 0);
Assert.IsFalse(values.Contains(partHash));
values.Add(partHash);
}
// String and Guid -> String hashes: Verify values same as a copy
CompareHashAndCopy(block.GetCopy("Reasonably Long Sample Value"));
CompareHashAndCopy(block.GetCopy(Guid.NewGuid().ToString()));
// Verify hash is really using last bytes correctly (pointer math bugs)
EnsureHashesDifferent(
block.GetCopy("9a6953f0-2db7-41ac-aea8-6e9defdbfa50"),
block.GetCopy("9a6953f0-2db7-41ac-aea8-6e9defdbfa51")
);
// Guid: Verify hashes work, different values different
Guid g1 = Guid.NewGuid();
ulong g1Hash = Hashing.Hash(g1, 0);
ulong g1HashCopy = Hashing.Hash(g1, 0);
Assert.AreEqual(g1Hash, g1HashCopy);
Guid g2 = Guid.NewGuid();
ulong g2Hash = Hashing.Hash(g2, 0);
Assert.AreNotEqual(g1Hash, g2Hash);
// DateTime: Verify hashes work, different values different
DateTime d1 = DateTime.UtcNow;
ulong d1Hash = Hashing.Hash(d1, 0);
ulong d1HashCopy = Hashing.Hash(d1, 0);
Assert.AreEqual(d1Hash, d1HashCopy);
DateTime d2 = d1.AddMilliseconds(5);
ulong d2Hash = Hashing.Hash(d2, 0);
Assert.AreNotEqual(d1Hash, d2Hash);
// TimeSpan: Verify hashes work, different values different
TimeSpan t1 = TimeSpan.FromMilliseconds(567);
ulong t1Hash = Hashing.Hash(t1, 0);
ulong t1HashCopy = Hashing.Hash(t1, 0);
Assert.AreEqual(t1Hash, t1HashCopy);
TimeSpan t2 = TimeSpan.FromMinutes(1);
ulong t2Hash = Hashing.Hash(t2, 0);
Assert.AreNotEqual(t1Hash, t2Hash);
}
public void Function_Between()
{
String8Block block = new String8Block();
String8[] values = new String8[]
{
String8.Empty,
block.GetCopy(" "),
block.GetCopy(@"C:\Code\elfie-arriba\XForm\XForm.Test\bin\x64\Release"),
block.GetCopy(@"XForm.Test\bin\x64\Release"),
block.GetCopy(@"XForm.Test\bin"),
block.GetCopy(@"XForm.Test"),
};
String8[] expected = new String8[]
{
String8.Empty,
block.GetCopy(" "),
block.GetCopy(@"\bin\x64\"),
block.GetCopy(@"\bin\x64\"),
block.GetCopy(@"\bin"),
String8.Empty
};
RunQueryAndVerify(values, "Name", expected, "Name", "set [Name] Between([Name], \"XForm.Test\", \"Release\")");
}
public TabularFileReader(IStreamProvider streamProvider, string filePath)
{
_streamProvider = streamProvider;
_filePath = filePath;
_block = new String8Block();
Reset();
}
public void WriteTo(ITabularWriter writer, String8Block block)
{
if (writer.RowCountWritten == 0)
{
writer.SetColumns(new string[] {
"UserID",
"UserGuid",
"EmailAddress",
"Region",
"Browser",
"OS",
"IsPremiumUser",
"JoinDate"
});
}
block.Clear();
writer.Write(this.ID);
writer.Write(block.GetCopy(this.Guid.ToString()));
writer.Write(block.GetCopy(this.EmailAddress));
writer.Write(block.GetCopy(this.Region));
writer.Write(block.GetCopy(this.Browser));
writer.Write(block.GetCopy(this.OS));
writer.Write(this.IsPremiumUser);
writer.Write(this.JoinDate);
writer.NextRow();
}
/// <summary>
/// Identify the columns to be written.
/// Must be called before anything else.
/// </summary>
/// <param name="columnNames">Set of column names each row will write.</param>
public void SetColumns(IEnumerable <string> columnNames)
{
if (_columnCount != 0)
{
throw new InvalidOperationException("SetColumns may only be called once on a JsonTabularWriter.");
}
_columnCount = columnNames.Count();
// Write header row
if (this._writeHeaderRow)
{
String8Block buffer = new String8Block();
foreach (string columnName in columnNames)
{
Write(buffer.GetCopy(columnName));
}
NextRow();
// Header row shouldn't count toward row count written.
_rowCountWritten = 0;
}
if (_columnCount == 0)
{
throw new InvalidOperationException("No columns were passed to contructor. TSV must have at least one column.");
}
}
public void String8_ShiftBack()
{
String8Block block = new String8Block();
// Goal: Split on semi-colon, collapse semi-colon and spaces in-place
String8 shiftable = "One; Two;Three; Four".TestConvert();
int totalShift = 0;
String8Set parts = shiftable.Split(UTF8.Semicolon, new PartialArray <int>(5, false));
for (int i = 0; i < parts.Count; ++i)
{
String8 part = parts[i];
totalShift++;
if (part.StartsWith(UTF8.Space))
{
part = part.Substring(1);
totalShift++;
}
String8 beforeShift = block.GetCopy(part);
String8 shifted = part.ShiftBack(totalShift);
Assert.AreEqual(beforeShift, shifted);
}
String8 result = shiftable.Substring(0, shiftable.Length - totalShift);
Assert.AreNotEqual("OneTwoThreeFour", result.ToString());
}
public void Sanitizer_Hashing_Basics()
{
// Verify Extract can extract 2x16 bits, 4x8 bits, 8x4 bits
Assert.AreEqual(2, ExtractCount(uint.MaxValue, 65536));
Assert.AreEqual(4, ExtractCount(uint.MaxValue, 256));
Assert.AreEqual(8, ExtractCount(uint.MaxValue, 16));
String8Block block = new String8Block();
String8 one = block.GetCopy("One");
String8 two = block.GetCopy("Two");
// Verify hashes are stable
Assert.AreEqual(Hashing.Hash(String8.Empty, 0), Hashing.Hash(String8.Empty, 0));
Assert.AreEqual(Hashing.Hash(one, 0), Hashing.Hash(one, 0));
Assert.AreEqual(Hashing.Hash(two, 0), Hashing.Hash(two, 0));
// Verify hashes vary based on hashKey
Assert.AreNotEqual(Hashing.Hash(String8.Empty, 1), Hashing.Hash(String8.Empty, 2));
Assert.AreNotEqual(Hashing.Hash(one, 1), Hashing.Hash(one, 2));
Assert.AreNotEqual(Hashing.Hash(two, 1), Hashing.Hash(two, 2));
// Verify hashes vary based on value
Assert.AreNotEqual(Hashing.Hash(String8.Empty, 3), Hashing.Hash(one, 3));
Assert.AreNotEqual(Hashing.Hash(String8.Empty, 3), Hashing.Hash(two, 3));
Assert.AreNotEqual(Hashing.Hash(one, 3), Hashing.Hash(two, 3));
}
private static void Compare(string oldFilePath, string newFilePath, string outputFilePath, string columnIdentifier)
{
String8Block block = new String8Block();
HashSet <String8> oldValues = new HashSet <String8>();
HashSet <String8> newValues = new HashSet <String8>();
using (ITabularReader oldReader = TabularFactory.BuildReader(oldFilePath))
{
int leftColumnIndex = oldReader.ColumnIndex(columnIdentifier);
while (oldReader.NextRow())
{
oldValues.Add(block.GetCopy(oldReader.Current(leftColumnIndex)));
}
Trace.WriteLine(String.Format("Old: {0:n0} values for \"{1}\" in {2:n0} rows.", oldValues.Count, columnIdentifier, oldReader.RowCountRead));
}
using (ITabularReader newReader = TabularFactory.BuildReader(newFilePath))
{
int rightColumnIndex = newReader.ColumnIndex(columnIdentifier);
while (newReader.NextRow())
{
newValues.Add(block.GetCopy(newReader.Current(rightColumnIndex)));
}
Trace.WriteLine(String.Format("New: {0:n0} values for \"{1}\" in {2:n0} rows.", newValues.Count, columnIdentifier, newReader.RowCountRead));
}
HashSet <String8> oldOnly = new HashSet <String8>(oldValues);
oldOnly.ExceptWith(newValues);
HashSet <String8> newOnly = new HashSet <String8>(newValues);
newOnly.ExceptWith(oldValues);
Trace.WriteLine(String.Format("{0:n0} values were only in \"{1}\".\r\n{2:n0} values were only in \"{3}\".", oldOnly.Count, oldFilePath, newOnly.Count, newFilePath));
String8 leftMarker = String8.Convert("-", new byte[1]);
String8 rightMarker = String8.Convert("+", new byte[1]);
using (ITabularWriter writer = TabularFactory.BuildWriter(outputFilePath))
{
writer.SetColumns(new string[] { "In", columnIdentifier });
foreach (String8 value in oldOnly)
{
writer.Write(leftMarker);
writer.Write(value);
writer.NextRow();
}
foreach (String8 value in newOnly)
{
writer.Write(rightMarker);
writer.Write(value);
writer.NextRow();
}
}
}
public void Function_IndexOf()
{
String8Block block = new String8Block();
String8[] values = new String8[]
{
String8.Empty,
block.GetCopy(" "),
block.GetCopy(@"C:\Code\elfie-arriba\XForm\XForm.Test\bin\x64\Release"),
block.GetCopy(@"XForm.Test\bin\x64\Release"),
block.GetCopy(@"XForm.Test\bin"),
block.GetCopy(@"bin"),
};
int[] expected = new int[]
{
-1,
-1,
38,
11,
11,
0
};
RunQueryAndVerify(values, "Name", expected, "Name", "set [Name] IndexOf([Name], \"bin\")");
}
private void WriteException(ErrorContext context, bool isValid, ITabularWriter writer)
{
String8Block block = new String8Block();
writer.SetColumns(new string[] { "Valid", "Usage", "ItemCategory", "ErrorMessage", "Values", "InvalidToken", "InvalidTokenIndex" });
writer.Write(isValid);
writer.Write(block.GetCopy(context.Usage));
writer.Write(block.GetCopy(context.InvalidValueCategory));
writer.Write(block.GetCopy(context.ErrorMessage));
String8 values = String8.Empty;
if (context.ValidValues != null)
{
foreach (string value in context.ValidValues)
{
values = block.Concatenate(values, s_delimiter, block.GetCopy(value));
}
}
writer.Write(values);
writer.Write(block.GetCopy(context.InvalidValue ?? ""));
writer.Write(context.InvalidTokenIndex);
writer.NextRow();
}
private static void Distinct(string inputFilePath, string outputFilePath, string columnIdentifier)
{
String8Block block = new String8Block();
HashSet <String8> distinctValues = new HashSet <String8>();
using (ITabularReader reader = TabularFactory.BuildReader(inputFilePath))
{
int columnIndex = reader.ColumnIndex(columnIdentifier);
using (ITabularWriter writer = TabularFactory.BuildWriter(outputFilePath))
{
writer.SetColumns(new string[] { reader.Columns[columnIndex] });
while (reader.NextRow())
{
String8 value = reader.Current(columnIndex).ToString8();
if (!distinctValues.Contains(value))
{
distinctValues.Add(block.GetCopy(value));
writer.Write(value);
writer.NextRow();
}
}
WriteSizeSummary(reader, writer);
}
}
}
public static void Write(IStreamProvider streamProvider, string tableRootPath, TableMetadata metadata)
{
String8Block block = new String8Block();
using (ITabularWriter sw = TabularFactory.BuildWriter(streamProvider.OpenWrite(Path.Combine(tableRootPath, SchemaFileName)), SchemaFileName))
{
sw.SetColumns(new string[] { "Name", "Type" });
foreach (ColumnDetails column in metadata.Schema)
{
sw.Write(block.GetCopy(column.Name));
sw.Write(block.GetCopy(column.Type.Name.ToString()));
sw.NextRow();
}
}
using (ITabularWriter mw = TabularFactory.BuildWriter(streamProvider.OpenWrite(Path.Combine(tableRootPath, MetadataFileName)), MetadataFileName))
{
mw.SetColumns(new string[] { "Name", "Context", "Value" });
mw.Write(block.GetCopy("RowCount"));
mw.Write(String8.Empty);
mw.Write(metadata.RowCount);
mw.NextRow();
}
streamProvider.WriteAllText(Path.Combine(tableRootPath, ConfigQueryPath), metadata.Query);
s_Cache.Add($"{streamProvider}|{tableRootPath}", metadata);
}
public IXColumn Build(IXTable source, XDatabaseContext context)
{
// Strings in XForm are stored in the type 'String8'.
// Make a String8Block (like StringBuilder) to allocate space for the new strings for the resolved IP addresses.
String8Block block = new String8Block();
return(SimpleTransformFunction <String8, String8> .Build(Name, source,
context.Parser.NextColumn(source, context, typeof(String8)),
(name8) =>
{
// Convert the value passed for the name column to a string
string machineName = name8.ToString();
// Run the code to do the lookup
string firstIpAddress = LookupName(machineName);
// Convert back to XForm's String8 type
return block.GetCopy(firstIpAddress);
},
() =>
{
// Before each page, clear the String8Block to reuse the memory
block.Clear();
}
));
}
private static void Generate_WebRequestSample(string basePath, int randomSeed, int userCount, int eventCount, int numberOfDays)
{
Random r = new Random(randomSeed);
DateTime asOfDate = DateTime.UtcNow.Date;
String8Block block = new String8Block();
WebRequestGenerator generator;
string path;
// Generate a set of users and write them out [for a week ago]
asOfDate = asOfDate.AddDays(-8);
path = Path.Combine(basePath, $"Users.{asOfDate:yyyyMMdd}.r{randomSeed}.{userCount}.csv");
Console.WriteLine($"Writing {path}...");
UserGenerator userGenerator = new UserGenerator(r, asOfDate);
List <User> users = userGenerator.Next(userCount);
using (ITabularWriter writer = TabularFactory.BuildWriter(path))
{
foreach (User user in users)
{
user.WriteTo(writer, block);
}
}
File.SetLastWriteTimeUtc(path, asOfDate);
// Generate WebRequest Data [for a week ago]
generator = new WebRequestGenerator(users, r, asOfDate, (eventCount < 1001 ? 10 : 100));
BuildWebRequests(basePath, generator, eventCount, WebRequestWriteMode.All);
asOfDate = asOfDate.AddDays(8);
// Generate batches of WebRequest sample data [current]
for (int day = 0; day < numberOfDays; ++day)
{
generator = new WebRequestGenerator(users, r, asOfDate, (eventCount < 1001 ? 10 : 100));
if (day == 0)
{
generator.Issue = new PremiumUserOutage(asOfDate.AddMinutes(18), asOfDate.AddMinutes(104), r);
}
BuildWebRequests(basePath, generator, eventCount, WebRequestWriteMode.UserIdentityOnly);
asOfDate = asOfDate.AddDays(-1);
}
// Generate one big joinable batch
eventCount = 10 * 1000 * 1000;
generator = new WebRequestGenerator(users, r, asOfDate, 1000);
generator.Issue = new PortRangeBlocked(asOfDate.AddMinutes(1), asOfDate.AddMinutes(180), 11450, 11480);
BuildWebRequests(basePath, generator, eventCount, WebRequestWriteMode.UserIdentityOnly);
asOfDate = asOfDate.AddDays(-1);
// Generate one huge minimal batch
eventCount = 100 * 1000 * 1000;
generator = new WebRequestGenerator(users, r, asOfDate, 1000);
generator.Issue = new UncachedSlowness(asOfDate.AddMinutes(4), asOfDate.AddMinutes(36), r);
BuildWebRequests(basePath, generator, eventCount, WebRequestWriteMode.Minimal);
Console.WriteLine("Done.");
}
public RegexColumnHandler(string regex, MapColumnHandler inner)
{
// Pre-Compile the Regex
this.Regex = new Regex(regex, RegexOptions.Compiled);
this.Inner = inner;
this.Block = new String8Block();
}
public Logger(IStreamProvider streamProvider, string outputFilePath)
{
string logFilePath = Path.Combine(outputFilePath, "Log.csv");
_writer = TabularFactory.BuildWriter(streamProvider.OpenWrite(logFilePath), logFilePath);
_writer.SetColumns(new string[] { "WhenUtc", "MessageType", "SourceComponent", "Message" });
_block = new String8Block();
}
private static void Concatenate(string inputFilePath, string outputFilePath, String8 delimiter)
{
using (ITabularReader reader = TabularFactory.BuildReader(inputFilePath))
{
using (ITabularWriter writer = TabularFactory.BuildWriter(outputFilePath))
{
writer.SetColumns(reader.Columns);
String8Block block = new String8Block();
String8[] lastValues = new String8[reader.CurrentRowColumns];
String8[] combinedValues = new String8[reader.CurrentRowColumns];
while (reader.NextRow())
{
String8 firstColumn = reader.Current(0).ToString8();
if (reader.RowCountRead == 2)
{
// First Row - Get the first ID only
combinedValues[0] = block.GetCopy(firstColumn);
}
else if (firstColumn.CompareTo(combinedValues[0], true) != 0)
{
// If we have a new ID (and not first row)
// Write concatenated values for previous ID
WriteCombinedRow(writer, combinedValues);
// Reset for this ID
block.Clear();
combinedValues[0] = block.GetCopy(firstColumn);
for (int i = 1; i < combinedValues.Length; ++i)
{
combinedValues[i] = String8.Empty;
}
}
// Concatenate non-duplicate values to "row in progress"
for (int i = 1; i < reader.CurrentRowColumns; ++i)
{
String8 value = reader.Current(i).ToString8();
if (lastValues[i] != value)
{
lastValues[i] = value;
combinedValues[i] = block.Concatenate(combinedValues[i], delimiter, value);
}
}
}
// After last row, write out values so far
WriteCombinedRow(writer, combinedValues);
WriteSizeSummary(reader, writer);
}
}
}
public void TsvWriter_Basics()
{
String8Block block = new String8Block();
String8 valueNoEscaping = block.GetCopy("Sample Description");
String8 valueEscaping = block.GetCopy("Value\tWith\nIssues");
String8 t = block.GetCopy("T");
using (TsvWriter writer = new TsvWriter("TsvWriter.tsv"))
{
writer.SetColumns(new string[] { "LineNumber", "IsEven", "Code", "When", "WhenDay", "Count", "Description", "Source" });
Assert.AreEqual(0, writer.RowCountWritten);
int sum = 0;
for (int i = 0; i < 10; ++i)
{
Assert.AreEqual(i, writer.RowCountWritten);
sum += i;
// Test write for non-String8 types
writer.Write(i);
writer.Write((i % 2 == 0));
writer.Write((byte)'Z');
writer.Write(new DateTime(2016, 02, 03, 04, 05, 06, DateTimeKind.Utc));
writer.Write(new DateTime(2016, 12, 23, 0, 0, 0, DateTimeKind.Utc));
// Test writing partial values
writer.WriteValueStart();
writer.WriteValuePart((byte)'[');
writer.WriteValuePart(sum);
writer.WriteValuePart(true);
writer.WriteValuePart(t);
writer.WriteValuePart((byte)']');
writer.WriteValueEnd();
// Test String8 writing with and without values to escape
writer.Write(valueNoEscaping);
writer.Write(valueEscaping);
writer.NextRow();
}
}
string tsvContent = File.ReadAllText("TsvWriter.tsv");
Trace.WriteLine(tsvContent);
// Verify header is as expected
Assert.IsTrue(tsvContent.StartsWith("LineNumber\tIsEven\tCode\tWhen\tWhenDay\tCount\tDescription\tSource\r\n"));
// Verify illegal characters are stripped
Assert.IsTrue(tsvContent.Contains("ValueWithIssues"));
// Verify the first row fully
Assert.IsTrue(tsvContent.Contains("1\tfalse\tZ\t2016-02-03T04:05:06Z\t2016-12-23\t[1trueT]\tSample Description\tValueWithIssues\r\n"));
}
public void String8Column_Basic()
{
StringStore store = new StringStore();
String8Block block = new String8Block();
String8[] samples = new String8[] { String8.Empty, block.GetCopy("One"), block.GetCopy("two"), block.GetCopy("three") };
String8Column column = new String8Column(store);
Assert.AreEqual(0, column.Count);
Verify.Exception <ArgumentOutOfRangeException>(() => { String8 value = column[0]; });
// Set values and verify they are read back consistently (all UTC)
for (int i = 0; i < samples.Length; ++i)
{
column.Add();
column[i] = samples[i];
Assert.AreEqual(samples[i], column[i]);
}
Assert.AreEqual(samples.Length, column.Count);
// Round-Trip the column
column.ConvertToImmutable();
StringStore readStore = new StringStore();
Verify.RoundTrip(store, readStore);
String8Column readColumn = new String8Column(readStore);
Verify.RoundTrip <String8Column>(column, readColumn);
Assert.AreEqual(column.Count, readColumn.Count);
for (int i = 0; i < column.Count; ++i)
{
Assert.AreEqual(column[i], readColumn[i]);
}
// Verify asking for raw identifiers works
int firstStringIdentifier = column.IdentifierFor(1);
Assert.AreEqual(store[firstStringIdentifier], column[1]);
// Verify clear works
column.Clear();
Assert.AreEqual(0, column.Count);
Verify.Exception <ArgumentOutOfRangeException>(() => { String8 value = column[0]; });
// Verify SetCount works
column.SetCount(2);
Assert.AreEqual(2, column.Count);
Assert.AreEqual(String8.Empty, column[1]);
}
private static void HtmlEscapeAndVerify(string htmlContent, string expectedEscapedValue)
{
String8Block block = new String8Block();
using (StringLastValueWriter writer = new StringLastValueWriter())
{
Program.WriteHtmlEscaped(block.GetCopy(htmlContent), writer);
Assert.AreEqual(expectedEscapedValue, writer.LastValue.ToString());
}
}
public void Writer_Performance(Func <Stream, ITabularWriter> buildWriter)
{
String8Block block = new String8Block();
String8 d1 = block.GetCopy("Description 1");
String8 d2 = block.GetCopy("Description 2");
String8 s1 = block.GetCopy("Source: Internal");
String8 s2 = block.GetCopy("Source: External");
using (MemoryStream s = new MemoryStream())
{
int iterations = 50;
long bytesWritten = 0;
int rowsWritten = 0;
// Tsv Write goal: 100MB/sec [Surface Book]
// NOTE: Tsv Write performance is very sensitive the mix of text and numbers written. Writing integers is slower.
Verify.PerformanceByBytes(50 * LongExtensions.Megabyte, () =>
{
for (int iteration = 0; iteration < iterations; ++iteration)
{
s.Seek(0, SeekOrigin.Begin);
ITabularWriter writer = buildWriter(s);
writer.SetColumns(new string[] { "LineNumber", "Count", "Description", "Source" });
int sum = 0;
for (int row = 1; row < 10000; ++row)
{
sum += row;
writer.Write(row);
writer.Write(sum);
if (row % 2 == 0)
{
writer.Write(d1);
writer.Write(s1);
}
else
{
writer.Write(d2);
writer.Write(s2);
}
writer.NextRow();
}
bytesWritten += writer.BytesWritten;
rowsWritten += writer.RowCountWritten;
}
return(bytesWritten);
});
}
}
private static IEnumerable <DataBlock> ReadAsDataBlockBatch(ITabularReader reader, IList <string> columnNames)
{
// Build a DataBlock to hold a batch of rows
int columnCount = columnNames.Count;
DataBlock result = new DataBlock(columnNames, BatchSize);
Value[][] columnArrays = new Value[columnCount][];
for (int i = 0; i < columnCount; ++i)
{
columnArrays[i] = new Value[BatchSize];
for (int j = 0; j < BatchSize; ++j)
{
columnArrays[i][j] = Value.Create(null);
}
result.SetColumn(i, columnArrays[i]);
}
// Look up indices of the columns
int[] columnIndices = new int[columnCount];
for (int i = 0; i < columnCount; ++i)
{
columnIndices[i] = reader.ColumnIndex(columnNames[i]);
}
// Fill blocks with rows as we go
int currentRowCount = 0;
String8Block block = new String8Block();
while (reader.NextRow())
{
for (int i = 0; i < columnCount; ++i)
{
String8 cell = block.GetCopy(reader.Current(columnIndices[i]).ToString8());
columnArrays[i][currentRowCount].Assign(new ByteBlock(cell.Array, cell.Index, cell.Length));
//columnArrays[i][currentRowCount].Assign(cell.ToString());
}
currentRowCount++;
if (currentRowCount == BatchSize)
{
yield return(result);
currentRowCount = 0;
block.Clear();
}
}
if (currentRowCount > 0)
{
yield return(result);
}
}
public DataReaderTabularReader(IDataReader reader)
{
this._reader = reader;
_columnNames = new List <string>();
foreach (DataRow row in _reader.GetSchemaTable().Rows)
{
_columnNames.Add(row.Field <string>("ColumnName"));
}
_block = new String8Block();
}
public static String8[] ToString8(String[] values)
{
String8Block block = new String8Block();
String8[] result = new String8[values.Length];
for (int i = 0; i < values.Length; ++i)
{
result[i] = block.GetCopy(values[i]);
}
return(result);
}
public void String8Block_Basics()
{
String8Block block = new String8Block();
byte[] buffer = new byte[4096];
String8 value = String8.Convert("hello", buffer);
// Verify copies are persistent when the original buffer is overwritten
String8 valueCopy = block.GetCopy(value);
String8.Convert("there", buffer);
Assert.AreEqual("there", value.ToString());
Assert.AreEqual("hello", valueCopy.ToString());
// Verify copy of String8.Empty works
String8 emptyCopy = block.GetCopy(String8.Empty);
Assert.IsTrue(emptyCopy.IsEmpty());
// Verify large strings are copied correctly (stored individually)
value = String8.Convert(new string('A', 4096), buffer);
valueCopy = block.GetCopy(value);
Assert.IsTrue(value.Equals(valueCopy));
String8.Convert(new string('B', 4096), buffer);
Assert.IsFalse(value.Equals(valueCopy));
// Verify storage uses multiple blocks correctly
for (int i = 0; i < 1000; ++i)
{
value = String8.Convert(new string((char)('0' + (i % 10)), 100), buffer);
valueCopy = block.GetCopy(value);
Assert.IsTrue(value.Equals(valueCopy));
}
// Verify conversion of strings
String8 directConversion = block.GetCopy("Regular String");
Assert.AreEqual("Regular String", directConversion.ToString());
// Verify null/empty string conversion
directConversion = block.GetCopy((string)null);
Assert.IsTrue(directConversion.IsEmpty());
directConversion = block.GetCopy(String.Empty);
Assert.IsTrue(directConversion.IsEmpty());
// Verify clear works (doesn't throw, GetCopy works afterward)
block.Clear();
valueCopy = block.GetCopy("Third");
Assert.AreEqual("Third", valueCopy.ToString());
}
public static XArray ToPercentageStrings(XArray counts, int total, Func <int, int, string> formatter)
{
int[] countArray = (int[])counts.Array;
String8Block block = new String8Block();
String8[] percentages = new String8[counts.Count];
for (int i = 0; i < counts.Count; ++i)
{
// Convert to a percentage, string, and then String8
percentages[i] = block.GetCopy(formatter(countArray[counts.Index(i)], total));
}
return(XArray.All(percentages, counts.Count));
}
public void SampleStructure_Basic()
{
SampleSet set;
SampleItem item;
// Empty Set: Round trip and verify no errors
set = new SampleSet();
Assert.AreEqual(0, set.Count);
SampleSet readSet = new SampleSet();
Verify.RoundTrip(set, readSet);
Assert.AreEqual(0, readSet.Count);
// Rebuild and add items
DateTime now = DateTime.UtcNow;
set = new SampleSet();
String8Block block = new String8Block();
String8 name = block.GetCopy("Name");
String8 target = block.GetCopy("Target");
for (int i = 0; i < 100; ++i)
{
item = set.Add();
item.Name = name;
item.Target = target;
item.Type = (i % 2 == 0 ? SampleItemType.Basic : SampleItemType.Complex);
item.EventTime = now.AddMinutes(i);
}
Assert.AreEqual(100, set.Count);
// Serialize and load
readSet = new SampleSet();
Verify.RoundTrip(set, readSet);
Assert.AreEqual(100, readSet.Count);
// Verify items are correct
for (int i = 0; i < readSet.Count; ++i)
{
item = readSet[i];
Assert.AreEqual(name, item.Name);
Assert.AreEqual(target, item.Target);
Assert.AreEqual((i % 2 == 0 ? SampleItemType.Basic : SampleItemType.Complex), item.Type);
Assert.AreEqual(now.AddMinutes(i), item.EventTime);
}
}
private void WriteException(Exception ex, ITabularWriter writer, bool isValid = false)
{
String8Block block = new String8Block();
if (ex is UsageException)
{
UsageException ue = ex as UsageException;
WriteException(ue.Context, isValid, writer);
}
else
{
writer.SetColumns(new string[] { "Valid", "Message", "Stack" });
writer.Write(false);
writer.Write(block.GetCopy(ex.Message));
writer.Write(block.GetCopy(ex.StackTrace));
writer.NextRow();
}
}
public void String8Block_Concatenate()
{
String8Block block = new String8Block();
String8 delimiter = block.GetCopy("; ");
String8 one = block.GetCopy("One");
String8 two = block.GetCopy("Two");
String8 three = block.GetCopy("Three");
// Verify Concatenate returns only one side if the other is empty
Assert.AreEqual(String8.Empty, block.Concatenate(String8.Empty, delimiter, String8.Empty));
Assert.AreEqual(two, block.Concatenate(String8.Empty, delimiter, two));
Assert.AreEqual(two, block.Concatenate(two, delimiter, String8.Empty));
// Verify a regular concatenation
String8 oneTwo = block.Concatenate(one, delimiter, two);
Assert.AreEqual("One; Two", oneTwo.ToString());
// Verify re-concatenating the last item re-uses memory and doesn't mess up previous item
String8 oneTwoThree = block.Concatenate(oneTwo, delimiter, three);
Assert.AreEqual(oneTwo._buffer, oneTwoThree._buffer);
Assert.AreEqual(oneTwo._index, oneTwoThree._index);
Assert.AreEqual("One; Two", oneTwo.ToString());
Assert.AreEqual("One; Two; Three", oneTwoThree.ToString());
// Verify re-concatenating doesn't overwrite a following item
String8 four = block.GetCopy("Four");
String8 oneTwoThreeFour = block.Concatenate(oneTwoThree, delimiter, four);
Assert.AreEqual("One; Two; Three", oneTwoThree.ToString());
Assert.AreEqual("One; Two; Three; Four", oneTwoThreeFour.ToString());
// Concatenate over the 64K limit and ensure reasonable behavior
String8 eight = block.GetCopy("12345678");
String8 eightSet = String8.Empty;
for (int i = 0; i < 10000; ++i)
{
eightSet = block.Concatenate(eightSet, delimiter, eight);
}
}
