public void String8Block_Basics()
{
String8Block block = new String8Block();
byte[] buffer = new byte[4096];
String8 value = String8.Convert("hello", buffer);
// Verify copies are persistent when the original buffer is overwritten
String8 valueCopy = block.GetCopy(value);
String8.Convert("there", buffer);
Assert.AreEqual("there", value.ToString());
Assert.AreEqual("hello", valueCopy.ToString());
// Verify copy of String8.Empty works
String8 emptyCopy = block.GetCopy(String8.Empty);
Assert.IsTrue(emptyCopy.IsEmpty());
// Verify large strings are copied correctly (stored individually)
value = String8.Convert(new string('A', 4096), buffer);
valueCopy = block.GetCopy(value);
Assert.IsTrue(value.Equals(valueCopy));
String8.Convert(new string('B', 4096), buffer);
Assert.IsFalse(value.Equals(valueCopy));
// Verify storage uses multiple blocks correctly
for (int i = 0; i < 1000; ++i)
{
value = String8.Convert(new string((char)('0' + (i % 10)), 100), buffer);
valueCopy = block.GetCopy(value);
Assert.IsTrue(value.Equals(valueCopy));
}
// Verify conversion of strings
String8 directConversion = block.GetCopy("Regular String");
Assert.AreEqual("Regular String", directConversion.ToString());
// Verify null/empty string conversion
directConversion = block.GetCopy((string)null);
Assert.IsTrue(directConversion.IsEmpty());
directConversion = block.GetCopy(String.Empty);
Assert.IsTrue(directConversion.IsEmpty());
// Verify clear works (doesn't throw, GetCopy works afterward)
block.Clear();
valueCopy = block.GetCopy("Third");
Assert.AreEqual("Third", valueCopy.ToString());
}
public void String8_ToUpperToLower()
{
// Verify no exception
String8.Empty.ToUpperInvariant();
String8.Empty.ToLowerInvariant();
String8 sample = "abcABC@[`{".TestConvert();
sample.ToUpperInvariant();
Assert.AreEqual("ABCABC@[`{", sample.ToString());
sample = "abcABC@[`{".TestConvert();
sample.ToLowerInvariant();
Assert.AreEqual("abcabc@[`{", sample.ToString());
}
public void String8_ShiftBack()
{
String8Block block = new String8Block();
// Goal: Split on semi-colon, collapse semi-colon and spaces in-place
String8 shiftable = "One; Two;Three; Four".TestConvert();
int totalShift = 0;
String8Set parts = shiftable.Split(UTF8.Semicolon, new PartialArray <int>(5, false));
for (int i = 0; i < parts.Count; ++i)
{
String8 part = parts[i];
totalShift++;
if (part.StartsWith(UTF8.Space))
{
part = part.Substring(1);
totalShift++;
}
String8 beforeShift = block.GetCopy(part);
String8 shifted = part.ShiftBack(totalShift);
Assert.AreEqual(beforeShift, shifted);
}
String8 result = shiftable.Substring(0, shiftable.Length - totalShift);
Assert.AreNotEqual("OneTwoThreeFour", result.ToString());
}
private void ReadColumnLine(String8 line)
{
// Skip empty lines
if (line.Length == 0)
{
return;
}
// Skip record separator, continuation lines, comments, and grouping lines
byte first = line[0];
if (first == UTF8.CR || first == UTF8.Space || first == UTF8.Pound || first == UTF8.Dash)
{
return;
}
// Find the column name part of the line
String8 columnName = line.BeforeFirst(UTF8.Colon);
// If we haven't seen this column name before, add it to our collection
if (!columnName.IsEmpty() && !_columnIndices8.ContainsKey(columnName))
{
int columnIndex = _columnIndices8.Count;
_columnIndices8[_columnNamesBlock.GetCopy(columnName)] = columnIndex;
string columnNameString = columnName.ToString();
_columnNames.Add(columnNameString);
_columnIndices[columnNameString] = columnIndex;
}
}
private string CsvSplitAndJoin(string value)
{
String8 value8 = String8.Convert(value, new byte[String8.GetLength(value)]);
String8Set set = value8.SplitAndDecodeCsvCells(new PartialArray <int>());
String8 joined8 = set.Join(UTF8.Pipe, new byte[set.Value.Length]);
return(joined8.ToString());
}
private string SplitOutsideQuotesAndJoin(string value, byte delimiter)
{
String8 value8 = String8.Convert(value, new byte[String8.GetLength(value)]);
String8Set set = value8.SplitOutsideQuotes(delimiter, new PartialArray <int>());
String8 joined8 = set.Join(UTF8.Pipe, new byte[set.Value.Length]);
return(joined8.ToString());
}
public void String8_TrimEnd()
{
String8 sample = "Interesting ".TestConvert();
Assert.AreEqual("Interesting", sample.TrimEnd(UTF8.Space).ToString());
Assert.AreEqual(sample.ToString(), sample.TrimEnd(UTF8.Tab).ToString());
Assert.AreEqual(string.Empty, String8.Empty.TrimEnd(UTF8.Space).ToString());
Assert.AreEqual(string.Empty, " ".TestConvert().TrimEnd(UTF8.Space).ToString());
Assert.AreEqual("A", "A ".TestConvert().TrimEnd(UTF8.Space).ToString());
}
public void String8_ToUpper()
{
// Verify no exception
String8.Empty.ToUpperInvariant();
String8 sample = String8.Convert("abcABC", new byte[6]);
sample.ToUpperInvariant();
Assert.AreEqual("ABCABC", sample.ToString());
}
public void String8Block_Concatenate()
{
String8Block block = new String8Block();
String8 delimiter = block.GetCopy("; ");
String8 one = block.GetCopy("One");
String8 two = block.GetCopy("Two");
String8 three = block.GetCopy("Three");
// Verify Concatenate returns only one side if the other is empty
Assert.AreEqual(String8.Empty, block.Concatenate(String8.Empty, delimiter, String8.Empty));
Assert.AreEqual(two, block.Concatenate(String8.Empty, delimiter, two));
Assert.AreEqual(two, block.Concatenate(two, delimiter, String8.Empty));
// Verify a regular concatenation
String8 oneTwo = block.Concatenate(one, delimiter, two);
Assert.AreEqual("One; Two", oneTwo.ToString());
// Verify re-concatenating the last item re-uses memory and doesn't mess up previous item
String8 oneTwoThree = block.Concatenate(oneTwo, delimiter, three);
Assert.AreEqual(oneTwo._buffer, oneTwoThree._buffer);
Assert.AreEqual(oneTwo._index, oneTwoThree._index);
Assert.AreEqual("One; Two", oneTwo.ToString());
Assert.AreEqual("One; Two; Three", oneTwoThree.ToString());
// Verify re-concatenating doesn't overwrite a following item
String8 four = block.GetCopy("Four");
String8 oneTwoThreeFour = block.Concatenate(oneTwoThree, delimiter, four);
Assert.AreEqual("One; Two; Three", oneTwoThree.ToString());
Assert.AreEqual("One; Two; Three; Four", oneTwoThreeFour.ToString());
// Concatenate over the 64K limit and ensure reasonable behavior
String8 eight = block.GetCopy("12345678");
String8 eightSet = String8.Empty;
for (int i = 0; i < 10000; ++i)
{
eightSet = block.Concatenate(eightSet, delimiter, eight);
}
}
/// <summary>
/// Translate a single literal value to the sanitized form, given the value
/// and which column it is from. Uses the mapper configured for the column in
/// the spec file.
/// </summary>
/// <param name="value">Value to convert</param>
/// <param name="columnName">ColumnName value is from</param>
/// <returns>Sanitized version of value</returns>
public string Translate(string value, string columnName)
{
IColumnHandler handler;
// If there's no handler, there's no re-mapping
if (!this.HandlersByColumn.TryGetValue(columnName, out handler))
{
handler = new KeepColumnHandler();
}
// Convert and return the value
String8 value8 = String8.Convert(value, new byte[String8.GetLength(value)]);
String8 replacement = this.HandlersByColumn[columnName].Sanitize(value8);
return(replacement.ToString());
}
private static string SplitAndJoin(string content)
{
StringBuilder result = new StringBuilder();
byte[] buffer = new byte[content.Length];
using (BufferedReader reader = BufferedReader.FromString(content, ref buffer))
{
bool isWord = WordSplitter.IsLetterOrDigit(reader.Buffer[reader.Index]);
int length = 0;
while (!reader.EndOfStream)
{
// Read the next word
length = WordSplitter.NextWordLength(reader, isWord);
String8 word = String8.Reference(reader.Buffer, reader.Index, length);
// Set state to read next word
reader.Index += length;
isWord = !isWord;
if (reader.Index < reader.Length || reader.EndOfStream)
{
// If this is word is definitely complete, write it
if (result.Length > 0)
{
result.Append("|");
}
result.Append(word.ToString());
}
else if (!reader.EndOfStream)
{
// Reset state to re-read this word
reader.Index -= length;
isWord = !isWord;
// If end of buffer but not stream, request more
reader.EnsureSpace(length * 2);
}
}
}
return(result.ToString());
}
public String8 Sanitize(String8 value8)
{
this.Block.Clear();
StringBuilder result = new StringBuilder();
string value = value8.ToString();
int nextIndexToWrite = 0;
foreach (Match m in this.Regex.Matches(value))
{
// Replace the whole expression if no groups, otherwise the first parenthesized group
Group g = m.Groups[0];
if (m.Groups.Count > 1)
{
g = m.Groups[1];
}
// Write content before this match
result.Append(value.Substring(nextIndexToWrite, g.Index - nextIndexToWrite));
// Convert and write the match
String8 part = this.Inner.Sanitize(this.Block.GetCopy(g.Value));
result.Append(part.ToString());
// Set the next non-match we need to write
nextIndexToWrite = g.Index + g.Length;
}
// Write anything after the last match
if (nextIndexToWrite < value.Length)
{
result.Append(value.Substring(nextIndexToWrite));
}
return(this.Block.GetCopy(result.ToString()));
}
private bool AddUniqueMembers(
PackageDatabase source,
int sourceMemberIndex,
int targetMemberIndex,
DatabaseAddResult result,
ArdbVersion version)
{
int pathLength = result.CurrentPath.Length;
SymbolDetails memberDetails = source.DeclaredMemberDetails[sourceMemberIndex];
bool hasUniqueDescendants = false;
int indexToAddChildrenUnder = targetMemberIndex;
// Add public types and namespaces which contain either
bool addMember = false;
if (memberDetails.Type == SymbolType.FrameworkTarget)
{
String8 encodedFrameworkNames = source.StringStore[source.DeclaredMembers.GetNameIdentifier(sourceMemberIndex)];
result.CurrentFrameworkNames = new HashSet <string>(encodedFrameworkNames.ToString().ToFrameworkNames());
}
else if (memberDetails.Type == SymbolType.Namespace)
{
addMember = ContainsPublics(source, sourceMemberIndex);
}
else if (memberDetails.Type == SymbolType.Assembly)
{
if (result.CurrentFrameworkNames != null)
{
result.CurrentFrameworkNames.Clear();
}
}
else if (memberDetails.Type.IsType())
{
addMember = memberDetails.Modifiers.HasFlag(SymbolModifier.Public);
}
// Add the member if it is a public type or contains them [and set 'indexInTarget' to the added member]
if (addMember)
{
result.PublicTypeCount++;
// Remap name for target StringStore [all unique names will be added anyway]
int memberName = source.DeclaredMembers.GetNameIdentifier(sourceMemberIndex);
String8 memberNameText = source.StringStore[memberName];
result.CurrentPath.Append((result.CurrentPath.Length > 0 ? "." : "") + memberNameText.ToString());
memberName = this.StringStore.FindOrAddString(memberNameText);
string fullTypeName;
HashSet <string> frameworkTargets;
// See if this name is already found in the merged namespace tree
if (!this.MergedMembers.TryFindChildByName(targetMemberIndex, memberName, out indexToAddChildrenUnder))
{
// If not, add it, and the tree is unique because this name is
indexToAddChildrenUnder = this.MergedMembers.Add(targetMemberIndex, memberName);
// Identify the package which added it
int packageNameIdentifier = this.StringStore.FindOrAddString(source.Identity.PackageName);
this.MergedMemberSourcePackageIdentifier.Add(packageNameIdentifier);
this.MergedMemberDuplicateCount.Add(1);
hasUniqueDescendants = true;
result.MergedTypeCount++;
if (version != ArdbVersion.V1 && memberDetails.Type.IsType())
{
// We have encountered a fully-qualified type name for the
// very first time. We will associate this name with the
// current framework target. We will also remember this
// sourceMemberIndex, in order to use it as the item
// to add when populating the ARDB.
fullTypeName = result.CurrentPath.ToString();
result.TypeToFrameworkTargetsMap.Add(fullTypeName, new HashSet <string>());
if (result.CurrentFrameworkNames != null)
{
result.TypeToFrameworkTargetsMap[fullTypeName].UnionWith(result.CurrentFrameworkNames);
}
result.TypeNameToTypeIndexMap.Add(fullTypeName, sourceMemberIndex);
}
}
else
{
// Otherwise, if this is a duplicate with another entry in the same package, still include it (different framework targets)
String8 sourcePackage = this.StringStore[this.MergedMemberSourcePackageIdentifier[indexToAddChildrenUnder]];
if (version != ArdbVersion.V1 &&
result.CurrentFrameworkNames != null &&
sourcePackage.Equals(source.Identity.PackageName))
{
hasUniqueDescendants = true;
result.MergedTypeCount++;
if (version != ArdbVersion.V1 && memberDetails.Type.IsType())
{
// We have encountered a fully-qualified type name that we've seen
// before. We will record the current framework target but
// will already have a source member index to use to add
// to the ARDB later.
fullTypeName = result.CurrentPath.ToString();
frameworkTargets = result.TypeToFrameworkTargetsMap[fullTypeName];
frameworkTargets.UnionWith(result.CurrentFrameworkNames);
Debug.Assert(result.TypeNameToTypeIndexMap.ContainsKey(fullTypeName));
}
}
else
{
// If this isn't unique, increment the count of copies
this.MergedMemberDuplicateCount[indexToAddChildrenUnder] += 1;
}
}
}
if (!memberDetails.Type.IsType())
{
int childIndex = source.DeclaredMembers.GetFirstChild(sourceMemberIndex);
while (childIndex > 0)
{
hasUniqueDescendants |= AddUniqueMembers(source, childIndex, indexToAddChildrenUnder, result, version);
childIndex = source.DeclaredMembers.GetNextSibling(childIndex);
}
}
// Add the package which had this member to results if it wasn't unique
if (hasUniqueDescendants == false)
{
result.SourcePackageNames[sourceMemberIndex] = this.StringStore[this.MergedMemberSourcePackageIdentifier[indexToAddChildrenUnder]];
}
// Record whether this member was unique (and added)
result.WasMemberAdded[sourceMemberIndex] = hasUniqueDescendants;
result.CurrentPath.Length = pathLength;
return(hasUniqueDescendants);
}
public override string ToString()
{
return(_value.ToString());
}
private String8 DecodeBase64(String8 value)
{
// Horrible
byte[] decoded = Convert.FromBase64String(value.ToString());
return(_currentRowBlock.GetCopy(new String8(decoded, 0, decoded.Length)));
}