/// <summary>
/// Split a string into alphanumeric words without allocation by using
/// the same PartialArray<int> in a loop. When traversing over the
/// returned String8Set, check if each first letter IsAlphaNumeric to
/// decide whether to include it.
/// </summary>
/// <param name="value">String8 to split</param>
/// <param name="buffer">PartialArray to contain split positions [allows reuse without allocations]</param>
/// <returns>String8Set containing value split at alpha-to-non-alpha boundaries</returns>
public static String8Set Split(String8 value, ref PartialArray <int> buffer)
{
// Reset the buffer for our use
buffer.Clear();
if (!value.IsEmpty())
{
// Add the beginning as a part
buffer.Add(0);
bool inWord = IsAlphaNumeric(value[0]);
for (int i = 1; i < value.Length; ++i)
{
bool charIsAlpha = IsAlphaNumeric(value[i]);
if (inWord != charIsAlpha)
{
// Add a boundary at each alpha to non-alpha transition
buffer.Add(i);
inWord = charIsAlpha;
}
}
// Add the remaining part of the string
buffer.Add(value.Length);
}
return(new String8Set(value, 0, buffer));
}
private String8Set SplitRows(String8 block, PartialArray <int> rowPositionArray)
{
// Split the block into lines (and save the split for use splitting columns)
_blockLines = block.Split(UTF8.Newline, _lineArray);
// Reset where which line the next row begins with
_nextLineIndex = 0;
rowPositionArray.Clear();
rowPositionArray.Add(0);
for (int i = 0; i < _blockLines.Count - 1; ++i)
{
String8 line = _blockLines[i];
// An empty line (or \n\r\n) indicates a new logical row
if (line.Length == 0 || (line.Length == 1 && line[0] == UTF8.CR))
{
rowPositionArray.Add(_lineArray[i + 1]);
}
}
rowPositionArray.Add(block.Length + 1);
return(new String8Set(block, 1, rowPositionArray));
}
public ImmutableStringStore ConvertToImmutable(out int[] identifierToSerializedIdentifier)
{
int valueCount = _values.Count;
// Build an array of string indexes sorted by the strings
int[] valueIndexesSorted = new int[valueCount];
for (int i = 0; i < valueCount; ++i)
{
valueIndexesSorted[i] = i;
}
// Sort index array to point to values in sorted order
// NOTE: Sort is case insensitive stable (case insensitive search and minimal diff size)
Array.Sort(_values.ToArray(), valueIndexesSorted, s_comparer);
// Build a map from the original identifiers (ascending order by when inserted) to the new ones (sorted order) to fix references
int[] map = new int[valueCount];
for (int i = 0; i < valueCount; ++i)
{
int oldIdentifier = valueIndexesSorted[i];
int newIdentifier = i;
map[oldIdentifier] = newIdentifier;
}
// Walk in sorted order and determine the byte position where each value will be written
int totalLength = 0;
PartialArray <int> positions = new PartialArray <int>(new int[valueIndexesSorted.Length + 1]);
for (int i = 0; i < valueIndexesSorted.Length; ++i)
{
positions.Add(totalLength);
// Compute space needed for this value and delimiter (\r\n delimited)
totalLength += String8.GetLength(_values[valueIndexesSorted[i]]) + 2;
}
positions.Add(totalLength);
// Build byte[] with the concatenated values
byte[] sortedValueBytes = new byte[totalLength];
int nextWritePosition = 0;
for (int i = 0; i < valueIndexesSorted.Length; ++i)
{
// Copy value to output array
String8 newValue = String8.Convert(_values[valueIndexesSorted[i]], sortedValueBytes, nextWritePosition);
nextWritePosition += newValue.Length;
sortedValueBytes[nextWritePosition] = UTF8.CR;
sortedValueBytes[nextWritePosition + 1] = UTF8.LF;
nextWritePosition += 2;
}
// Build the ImmutableStore equivalent of this and an array to translate identifiers
identifierToSerializedIdentifier = map;
return(new ImmutableStringStore(new String8Set(new String8(sortedValueBytes, 0, sortedValueBytes.Length), 2, positions)));
}
/// <summary>
/// Split a string on a given delimiter only outside matching double quotes.
/// Used to split CSV content where the delimiters are ignored within quotes.
/// </summary>
/// <param name="value">String8 value to split</param>
/// <param name="delimiter">Delimiter to split on</param>
/// <param name="positions">PartialArray<int> to contain split positions</param>
/// <returns>String8Set containing split value</returns>
public static String8Set SplitOutsideQuotes(String8 value, byte delimiter, PartialArray <int> positions)
{
if (value.IsEmpty())
{
return(String8Set.Empty);
}
// Clear any previous values in the array
positions.Clear();
// The first part always begins at the start of the string
positions.Add(0);
byte[] array = value._buffer;
int i = value._index;
int end = i + value._length;
// Walk the string. Find and mark delimiters outside of quotes only
while (i < end)
{
// Outside Quotes
for (; i < end; ++i)
{
// If a quote is found, we're now inside quotes
if (array[i] == UTF8.Quote)
{
i++;
break;
}
// If a delimiter is found, add another split position
if (array[i] == delimiter)
{
positions.Add(i - value._index + 1);
}
}
// Inside Quotes
for (; i < end; ++i)
{
// If a quote was found, we're now outside quotes
if (array[i] == UTF8.Quote)
{
i++;
break;
}
}
}
// The last part always ends at the end of the string
positions.Add(value.Length + 1);
return(new String8Set(value, 1, positions));
}
public int Add(int parentIndex, int nameIdentifier)
{
if (parentIndex < -1 || parentIndex >= Count)
{
throw new ArgumentOutOfRangeException("parentIndex");
}
// If parent was not found from a search, parent is the root
if (parentIndex == -1)
{
parentIndex = 0;
}
// Make a new node, pointing to the parent, with the parent's first child as a sibling
int newNodeIndex = Count;
_parentIndex.Add(parentIndex);
_firstChildIndex.Add(-1);
_nextSiblingIndex.Add(_firstChildIndex[parentIndex]);
_nameIdentifier.Add(nameIdentifier);
// Make the parent's first child now the new node
_firstChildIndex[parentIndex] = newNodeIndex;
return(newNodeIndex);
}
public void PartialArray_MembersProvided()
{
// Build a partially filled array
int[] array = new int[10];
for (int i = 0; i < 5; ++i)
{
array[i] = i;
}
// Build a PartialArray around it
PartialArray <int> a = new PartialArray <int>(array, 5, true);
// Verify it knows it's partially full already
Assert.AreEqual(5, a.Count);
Assert.AreEqual(3, a[3]);
Assert.AreEqual(10, a.Capacity);
Assert.IsTrue(a.IsStaticSize);
Assert.IsFalse(a.IsFull);
// Add another value; verify it goes to the right place
a.Add(5);
Assert.AreEqual(6, a.Count);
Assert.AreEqual(5, a[5]);
Assert.AreEqual(5, array[5]);
}
public ItemTree(int rootIdentifier)
{
_parentIndex = new PartialArray <int>();
_firstChildIndex = new PartialArray <int>();
_nextSiblingIndex = new PartialArray <int>();
_nameIdentifier = new PartialArray <int>();
// Add sentinel root
_parentIndex.Add(-1);
_firstChildIndex.Add(-1);
_nextSiblingIndex.Add(-1);
_nameIdentifier.Add(rootIdentifier);
}
/// <summary>
/// Split a string on a given delimiter into a provided byte[]. Used
/// to split strings without allocation when a large byte[] is created
/// and reused for many strings.
/// </summary>
/// <param name="value">String8 value to split</param>
/// <param name="delimiter">Delimiter to split on</param>
/// <param name="positions">PartialArray<int> to contain split positions</param>
/// <returns>String8Set containing split value</returns>
public static String8Set Split(String8 value, byte delimiter, PartialArray <int> positions)
{
// Ensure the delimiter is single byte
if (delimiter >= 128)
{
throw new ArgumentException(String.Format(Resources.UnableToSupportMultibyteCharacter, delimiter));
}
if (value.IsEmpty())
{
return(String8Set.Empty);
}
// Clear any previous values in the array
positions.Clear();
// Record each delimiter position
positions.Add(0);
// Get the String8 array directly and loop from index to (index + length)
// 3x faster than String8[index].
byte[] array = value._buffer;
int end = value._index + value._length;
for (int i = value._index; i < end; ++i)
{
if (array[i] == delimiter)
{
// Next start position is after this delimiter
positions.Add(i - value._index + 1);
}
}
positions.Add(value.Length + 1);
return(new String8Set(value, 1, positions));
}
private void Sort(int parentNodeIndex, IComparer <int> nodeIndexComparer, ref PartialArray <int> buffer)
{
int currentChild = _firstChildIndex[parentNodeIndex];
// If no children, return
if (currentChild <= 0)
{
return;
}
// Add all children of the current element to the buffer
buffer.Clear();
while (currentChild > 0)
{
buffer.Add(currentChild);
currentChild = _nextSiblingIndex[currentChild];
}
// Sort the children by the compare function
buffer.Sort(nodeIndexComparer);
// Modify the FirstChild and NextSibling pointers to be in sorted order
currentChild = buffer[0];
_firstChildIndex[parentNodeIndex] = currentChild;
for (int i = 1; i < buffer.Count; ++i)
{
int nextChild = buffer[i];
_nextSiblingIndex[currentChild] = nextChild;
currentChild = nextChild;
}
_nextSiblingIndex[currentChild] = -1;
// Recurse on the children
currentChild = _firstChildIndex[parentNodeIndex];
while (currentChild > 0)
{
Sort(currentChild, nodeIndexComparer, ref buffer);
currentChild = _nextSiblingIndex[currentChild];
}
}
public void PartialArray_NonResizable()
{
// Create a fixed PartialArray
PartialArray <int> a = new PartialArray <int>(10, true);
// Verify it was allocated already
Assert.AreEqual(10, a.Capacity);
// Try to add too many values to it (checking IsFull)
for (int i = 0; i < 100; ++i)
{
if (a.IsFull)
{
break;
}
a.Add(i);
}
// Verify the first Capacity were added, and no resize happened
Assert.AreEqual(10, a.Count);
Assert.AreEqual(10, a.Capacity);
}
/// <summary>
/// Split a CSV row into cells. This method splits and unencodes quoted values together.
/// It changes the underlying buffer in the process.
/// </summary>
/// <param name="row">String8 containing a CSV row</param>
/// <param name="positions">PartialArray<int> to contain split positions</param>
/// <returns>String8Set containing unencoded cell values</returns>
public static String8Set SplitAndDecodeCsvCells(String8 row, PartialArray <int> positions)
{
// If row is empty, return empty set
if (row.IsEmpty())
{
return(String8Set.Empty);
}
// Clear any previous values in the array
positions.Clear();
// The first part always begins at the start of the (shifted) string
positions.Add(0);
byte[] array = row._buffer;
int i = row._index;
int end = i + row._length;
// We're shifting values in the string to overwrite quotes around cells
// and doubled quotes. copyTo is where we've written to in the unescaped
// string.
int copyTo = i;
// Walk each cell, handling quoted and unquoted cells.
while (i < end)
{
bool inQuote = (array[i] == UTF8.Quote);
if (!inQuote)
{
// Unquoted cell. Copy until next comma.
for (; i < end; ++i, ++copyTo)
{
// Copy everything as-is (no unescaping)
array[copyTo] = array[i];
// If a delimiter is found, add another split position
if (array[i] == UTF8.Comma)
{
positions.Add(copyTo - row._index + 1);
i++; copyTo++;
break;
}
}
}
else
{
// Quoted cell.
// Overwrite opening quote
i++;
// Look for end quote (undoubled quote)
for (; i < end; ++i, ++copyTo)
{
if (array[i] != UTF8.Quote)
{
// Copy everything that wasn't an escaped quote
array[copyTo] = array[i];
}
else
{
// Quote found. End of cell, escaped quote, or unescaped quote (error)?
i++;
// End of cell [end of line]
if (i == end)
{
break;
}
if (array[i] == UTF8.Comma)
{
// End of cell [comma]. Copy comma, end of cell.
positions.Add(copyTo - row._index + 1);
array[copyTo] = array[i];
i++; copyTo++;
break;
}
else if (array[i] == UTF8.Quote)
{
// Escaped quote. Copy the second quote, continue cell.
array[copyTo] = array[i];
}
else
{
// Unescaped quote. Abort; caller will see incomplete row and can throw
return(new String8Set(row, 1, positions));
}
}
}
}
}
// The last part always ends at the end of the (shifted) string
positions.Add(copyTo - row._index + 1);
// Overwrite duplicate values left from shifting to make bugs clearer
for (; copyTo < end; ++copyTo)
{
array[copyTo] = UTF8.Null;
}
return(new String8Set(row, 1, positions));
}
public void Add()
{
_identifiers.Add();
}
/// <summary>
/// Add a link from one item to another. Links must be added to items in
/// order of insertion.
/// </summary>
/// <param name="groupIndex">Index of item from which to link</param>
/// <param name="memberIndex">Index of item to which to link</param>
public void AddLink(int groupIndex, int memberIndex)
{
_groupIndices.Add(groupIndex);
_memberIndices.Add(memberIndex);
}
public void Add()
{
_ticksValues.Add();
}
public void PartialArray_Basics()
{
// Default Constructor - no fixed size
PartialArray <int> a = new PartialArray <int>();
// Verify empty to start
Assert.AreEqual(0, a.Count);
Assert.AreEqual(0, a.Capacity);
Assert.IsFalse(a.IsStaticSize);
Assert.IsFalse(a.IsFull);
// Verify Add doesn't throw
for (int i = 0; i < 100; ++i)
{
a.Add(i);
}
// Verify count and capacity are right, IsFull is still false
Assert.AreEqual(100, a.Count);
Assert.IsTrue(a.Capacity >= 100);
Assert.IsFalse(a.IsFull);
// Verify we can get values back
for (int i = 0; i < 100; ++i)
{
Assert.AreEqual(i, a[i]);
}
// Verify changing a value works
a[0] = 50;
Assert.AreEqual(50, a[0]);
a[0] = 0;
// Verify round trip works [Primitives only]
PartialArray <int> readArray = new PartialArray <int>();
Verify.RoundTrip <PartialArray <int> >(a, readArray);
a = readArray;
// Verify count and capacity are right, IsFull is still false
Assert.AreEqual(100, a.Count);
Assert.IsTrue(a.Capacity >= 100);
Assert.IsFalse(a.IsFull);
// Verify we can get values back
for (int i = 0; i < 100; ++i)
{
Assert.AreEqual(i, a[i]);
}
// Verify clear works
a.Clear();
Assert.AreEqual(0, a.Count);
Assert.IsTrue(a.Capacity >= 100);
Assert.IsFalse(a.IsFull);
// Verify Add after Clear works
a.Add(10);
Assert.AreEqual(1, a.Count);
Assert.AreEqual(10, a[0]);
}
/// <summary>
/// Search the given IMemberDatabase for matches to this query and put
/// results into the results array provided. The capacity of the results
/// array determines how many results are returned.
/// </summary>
/// <param name="db">Database to search</param>
/// <param name="results">PartialArray to contain results, sized for the count desired.</param>
/// <returns>True if results were added, False otherwise</returns>
public bool TryFindMembers(IMemberDatabase db, ref PartialArray <Symbol> results)
{
// Ensure strings must be found again so that benchmarks are realistic
ForceReresolve();
// Clear results from a previous query
results.Clear();
// If there was no query, return with no results
if (String.IsNullOrEmpty(SymbolName))
{
return(false);
}
// Get required members from database
StringStore strings = db.StringStore;
ItemTree declaredMembers = db.DeclaredMembers;
MemberIndex index = db.Index;
// Map strings to the local StringStore. Stop immediately if any values aren't found.
if (!ResolveStringsTo(strings))
{
return(false);
}
// Cache whether this query needs details to match
bool usesDetails = !this.Parameters8.IsEmpty() || this.Type != SymbolType.Any || this.Modifiers != SymbolModifier.None;
int[] matches;
int matchesIndex, matchesCount;
if (SplitSymbolName8.Count == 1)
{
// Find the set of symbols with names in range. If no symbols in index, return nothing
if (!index.TryGetMatchesInRange(SymbolNameSuffixIdentifiers, out matches, out matchesIndex, out matchesCount))
{
return(false);
}
// If there was just one name part searched for, all matches count
for (int i = matchesIndex; i < matchesIndex + matchesCount; ++i)
{
if ((usesDetails ? MatchesDetailed(declaredMembers, strings, db, matches[i]) : Matches(declaredMembers, strings, matches[i])))
{
results.Add(new Symbol(db, matches[i]));
if (results.IsFull)
{
return(true);
}
}
}
}
else
{
// Find all entries with exactly the second-to-last name
if (!index.TryGetMatchesInRange(SymbolNamePrefixIdentifiers[SymbolNamePrefixIdentifiers.Length - 1], out matches, out matchesIndex, out matchesCount))
{
return(false);
}
for (int i = matchesIndex; i < matchesIndex + matchesCount; ++i)
{
int currentMatchIndex = matches[i];
// First, do all previous name parts in the query match?
int currentAncestorIndex = currentMatchIndex;
int namePartIndex = SymbolNamePrefixIdentifiers.Length - 2;
for (; namePartIndex >= 0; --namePartIndex)
{
currentAncestorIndex = declaredMembers.GetParent(currentAncestorIndex);
int currentAncestorNameIdentifier = declaredMembers.GetNameIdentifier(currentAncestorIndex);
if (!SymbolNamePrefixIdentifiers[namePartIndex].Contains(currentAncestorNameIdentifier))
{
break;
}
}
if (namePartIndex != -1)
{
continue;
}
// If this was a full match, are we out of namespaces?
if (IsFullNamespace)
{
currentAncestorIndex = declaredMembers.GetParent(currentAncestorIndex);
SymbolType symbolAboveFullNameType = db.GetMemberType(currentAncestorIndex);
if (!symbolAboveFullNameType.IsAboveNamespace())
{
return(false);
}
}
// Next, find children of this item which match the last part typed
int leafId = declaredMembers.GetFirstChild(currentMatchIndex);
while (leafId > 0)
{
if ((usesDetails ? MatchesDetailed(declaredMembers, strings, db, leafId) : Matches(declaredMembers, strings, leafId)))
{
results.Add(new Symbol(db, leafId));
if (results.IsFull)
{
return(true);
}
}
leafId = declaredMembers.GetNextSibling(leafId);
}
}
}
return(results.Count > 0);
}