public void String8Block_Basics()
{
String8Block block = new String8Block();
byte[] buffer = new byte[4096];
String8 value = String8.Convert("hello", buffer);
// Verify copies are persistent when the original buffer is overwritten
String8 valueCopy = block.GetCopy(value);
String8.Convert("there", buffer);
Assert.AreEqual("there", value.ToString());
Assert.AreEqual("hello", valueCopy.ToString());
// Verify copy of String8.Empty works
String8 emptyCopy = block.GetCopy(String8.Empty);
Assert.IsTrue(emptyCopy.IsEmpty());
// Verify large strings are copied correctly (stored individually)
value = String8.Convert(new string('A', 4096), buffer);
valueCopy = block.GetCopy(value);
Assert.IsTrue(value.Equals(valueCopy));
String8.Convert(new string('B', 4096), buffer);
Assert.IsFalse(value.Equals(valueCopy));
// Verify storage uses multiple blocks correctly
for (int i = 0; i < 1000; ++i)
{
value = String8.Convert(new string((char)('0' + (i % 10)), 100), buffer);
valueCopy = block.GetCopy(value);
Assert.IsTrue(value.Equals(valueCopy));
}
// Verify conversion of strings
String8 directConversion = block.GetCopy("Regular String");
Assert.AreEqual("Regular String", directConversion.ToString());
// Verify null/empty string conversion
directConversion = block.GetCopy((string)null);
Assert.IsTrue(directConversion.IsEmpty());
directConversion = block.GetCopy(String.Empty);
Assert.IsTrue(directConversion.IsEmpty());
// Verify clear works (doesn't throw, GetCopy works afterward)
block.Clear();
valueCopy = block.GetCopy("Third");
Assert.AreEqual("Third", valueCopy.ToString());
}
/// <summary>
/// Move the reader to the next row. This must be called before
/// reading the first row.
/// </summary>
/// <returns>True if another row exists, False if the TSV is out of content</returns>
public virtual bool NextRow()
{
String8 row = _reader.NextRow();
if (row.IsEmpty())
{
return(false);
}
// Split the line into cells
_currentRowColumns = SplitCells(row, _cellPositionArray);
this.RowCountRead++;
// Allocate a set of reusable String8TabularValues to avoid per-cell-value allocation or boxing.
if (_valueBoxes == null || _valueBoxes.Length < _currentRowColumns.Count)
{
_valueBoxes = new String8TabularValue[_currentRowColumns.Count];
for (int i = 0; i < _valueBoxes.Length; ++i)
{
_valueBoxes[i] = new String8TabularValue();
}
}
return(true);
}
private void ReadColumnLine(String8 line)
{
// Skip empty lines
if (line.Length == 0)
{
return;
}
// Skip record separator, continuation lines, comments, and grouping lines
byte first = line[0];
if (first == UTF8.CR || first == UTF8.Space || first == UTF8.Pound || first == UTF8.Dash)
{
return;
}
// Find the column name part of the line
String8 columnName = line.BeforeFirst(UTF8.Colon);
// If we haven't seen this column name before, add it to our collection
if (!columnName.IsEmpty() && !_columnIndices8.ContainsKey(columnName))
{
int columnIndex = _columnIndices8.Count;
_columnIndices8[_columnNamesBlock.GetCopy(columnName)] = columnIndex;
string columnNameString = columnName.ToString();
_columnNames.Add(columnNameString);
_columnIndices[columnNameString] = columnIndex;
}
}
private void SetColumnValue(String8 currentPropertyName, String8 currentPropertyValue, bool isEncoded)
{
if (currentPropertyName.IsEmpty())
{
return;
}
if (isEncoded)
{
currentPropertyValue = DecodeBase64(currentPropertyValue);
}
if (currentPropertyName.CompareTo("objectSid", true) == 0)
{
currentPropertyValue = DecodeSid(currentPropertyValue);
}
// TODO: Decode ".0Z" time format
// TODO: Decode NT file time format
int columnIndex;
if (_columnIndices8.TryGetValue(currentPropertyName, out columnIndex))
{
String8 previousValue = _currentRowValues[columnIndex].ToString8();
if (!previousValue.IsEmpty())
{
currentPropertyValue = _currentRowBlock.Concatenate(previousValue, MultiValueDelimiter, currentPropertyValue);
}
_currentRowValues[columnIndex].SetValue(currentPropertyValue);
}
}
public static string SplitAndJoin(string value)
{
String8 value8 = String8.Convert(value, new byte[String8.GetLength(value)]);
PartialArray <int> boundaryArray = new PartialArray <int>();
String8Set set = AlphanumericSplitter.Split(value8, ref boundaryArray);
bool firstPart = true;
StringBuilder result = new StringBuilder();
using (StringWriter writer = new StringWriter(result))
{
for (int i = 0; i < set.Count; ++i)
{
String8 part = set[i];
if (!part.IsEmpty() && AlphanumericSplitter.IsAlphaNumeric(part[0]))
{
if (!firstPart)
{
writer.Write("|");
}
firstPart = false;
part.WriteTo(writer);
}
}
}
return(result.ToString());
}
private bool IsCommentRow()
{
if (this.CurrentRowColumns == 0)
{
return(false);
}
String8 firstCell = this.Current(0).ToString8();
if (firstCell.IsEmpty())
{
return(false);
}
return(firstCell[0] == UTF8.Pound);
}
private void ReadColumns(Stream stream)
{
// Make a reader to split the input on newlines
BufferedRowReader reader = new BufferedRowReader(stream, (block, array) => block.Split(UTF8.Newline, array));
// Scan the lines for column names (something before a colon)
while (true)
{
String8 line = reader.NextRow();
if (line.IsEmpty())
{
break;
}
ReadColumnLine(line);
}
// Reset the stream for the second read
stream.Seek(0, SeekOrigin.Begin);
}
public bool IsNullOrEmpty()
{
return(_value.IsEmpty());
}
public bool NextRow()
{
_currentRowBlock.Clear();
String8 row = _reader.NextRow();
if (row.IsEmpty())
{
return(false);
}
// Clear values for row
for (int i = 0; i < _currentRowValues.Length; ++i)
{
_currentRowValues[i].SetValue(String8.Empty);
}
// Read available complete lines
String8 currentPropertyName = String8.Empty;
String8 currentPropertyValue = String8.Empty;
bool currentIsBase64 = false;
for (; _nextLineIndex < _blockLines.Count; ++_nextLineIndex)
{
String8 line = _blockLines[_nextLineIndex];
// Skip comment lines and grouping lines
if (line.StartsWith(UTF8.Pound) || line.StartsWith(UTF8.Dash))
{
continue;
}
// Trim trailing CR, if found
if (line.EndsWith(UTF8.CR))
{
line = line.Substring(0, line.Length - 1);
}
// An empty line or out of lines for the row range
if (line.Length == 0 || line.Index >= row.Index + row.Length)
{
break;
}
// Look for a wrapped line
if (line[0] == UTF8.Space)
{
// If found, concatenate the value after the space onto the value so far
line = line.Substring(1);
currentPropertyValue = _currentRowBlock.Concatenate(currentPropertyValue, String8.Empty, line);
}
else
{
// Set or Append the value just completed
SetColumnValue(currentPropertyName, currentPropertyValue, currentIsBase64);
// Split the property name and value [value is after colon and optional space]
currentPropertyName = line.BeforeFirst(UTF8.Colon);
currentPropertyValue = line.Substring(currentPropertyName.Length + 1);
if (currentPropertyValue.StartsWith(UTF8.Space))
{
currentPropertyValue = currentPropertyValue.Substring(1);
}
// Determine if the value is encoded
currentIsBase64 = (line[currentPropertyName.Length + 1] == UTF8.Colon);
if (currentIsBase64)
{
currentPropertyValue = currentPropertyValue.Substring(1);
}
}
}
// Set the last property value
SetColumnValue(currentPropertyName, currentPropertyValue, currentIsBase64);
// The next row starts after the row separator line
_nextLineIndex++;
this.RowCountRead++;
return(true);
}
