public void BigStrings()
{
var st = new StringTable(0);
var s1 = new string('x', StringTable.BytesPerBuffer - 1);
var s2 = new string('y', StringTable.BytesPerBuffer);
var s3 = new string('z', StringTable.BytesPerBuffer + 1);
StringId id1 = st.AddString(s1);
StringId id2 = st.AddString(s2);
StringId id3 = st.AddString(s3);
XAssert.AreEqual(s1.Length, st.GetLength(id1));
XAssert.AreEqual(s2.Length, st.GetLength(id2));
XAssert.AreEqual(s3.Length, st.GetLength(id3));
var buf = new char[StringTable.BytesPerBuffer + 1];
st.CopyString(id1, buf, 0);
for (int j = 0; j < s1.Length; j++)
{
XAssert.AreEqual(s1[j], buf[j]);
}
st.CopyString(id2, buf, 0);
for (int j = 0; j < s2.Length; j++)
{
XAssert.AreEqual(s2[j], buf[j]);
}
st.CopyString(id3, buf, 0);
for (int j = 0; j < s3.Length; j++)
{
XAssert.AreEqual(s3[j], buf[j]);
}
XAssert.IsTrue(st.CaseInsensitiveEquals(id1, st.AddString(s1)));
XAssert.IsTrue(st.CaseInsensitiveEquals(id2, st.AddString(s2)));
XAssert.IsTrue(st.CaseInsensitiveEquals(id3, st.AddString(s3)));
XAssert.IsTrue(st.Equals(s1, id1));
XAssert.IsTrue(st.Equals(s2, id2));
XAssert.IsTrue(st.Equals(s3, id3));
XAssert.IsFalse(st.CaseInsensitiveEquals(id1, st.AddString(s2)));
XAssert.IsFalse(st.CaseInsensitiveEquals(id1, st.AddString(s3)));
XAssert.IsFalse(st.CaseInsensitiveEquals(id2, st.AddString(s1)));
XAssert.IsFalse(st.CaseInsensitiveEquals(id2, st.AddString(s3)));
XAssert.IsFalse(st.CaseInsensitiveEquals(id3, st.AddString(s1)));
XAssert.IsFalse(st.CaseInsensitiveEquals(id3, st.AddString(s2)));
XAssert.IsFalse(st.Equals(s2, id1));
XAssert.IsFalse(st.Equals(s3, id1));
XAssert.IsFalse(st.Equals(s1, id2));
XAssert.IsFalse(st.Equals(s3, id2));
XAssert.IsFalse(st.Equals(s1, id3));
XAssert.IsFalse(st.Equals(s2, id3));
}
/// <summary>
/// Returns the length of a StringId when rendered to string using given <paramref name="escaping"/>.
/// </summary>
public int GetLength(StringId stringId, PipDataFragmentEscaping escaping)
{
switch (escaping)
{
case PipDataFragmentEscaping.CRuntimeArgumentRules:
// - passing null as StringBuilder to AppendEscapedCommandLineWord is ok
// - the return value of AppendEscapedCommandLineWord indicates how many characters would be appended
return(CommandLineEscaping.AppendEscapedCommandLineWord(builder: null, word: Render(stringId)));
case PipDataFragmentEscaping.NoEscaping:
return(StringTable.GetLength(stringId)); // instead of retrieving the whole string, StringTable.GetLength is faster.
default:
Contract.Assert(false, I($"Unhandled fragmentEscaping: {escaping}"));
return(0);
}
}
public void CharMix()
{
// make sure all characters are handled properly
var st = new StringTable(0);
var map = new Dictionary <StringId, string>();
var sb = new StringBuilder(3000000);
// zip through small sizes
for (int i = 0; i < 65535; i++)
{
sb.Length = 0;
sb.Append((char)i, (i % 255) + 1);
string str = sb.ToString();
StringId sd = st.AddString(str);
map.Add(sd, str);
}
for (int i = 0; i < 2; i++)
{
// make sure all the right strings come out
var buf = new char[65535];
foreach (StringId sd in map.Keys)
{
// get the actual string we kept
string str = map[sd];
// does the table report the right length?
int length = st.GetLength(sd);
XAssert.AreEqual(str.Length, length);
// extract the chars from the table
st.CopyString(sd, buf, 0);
// make sure we got all the characters out that we should have
for (int j = 0; j < str.Length; j++)
{
XAssert.AreEqual(str[j], buf[j]);
}
}
// make sure the same behavior occurs after freezing
st.Freeze();
}
}
public async Task RunCommonTestsAsync()
{
XAssert.AreNotEqual(StringTable.Count, AddedStrings.Count, "Ensure strings are added through harness");
StringTable deserializedStringTable = null;
using (MemoryStream ms = new MemoryStream())
{
using (BuildXLWriter writer = new BuildXLWriter(true, ms, true, logStats: true))
{
StringTable.Serialize(writer);
}
if (MaxSize < StringTable.BytesPerBuffer)
{
XAssert.IsTrue(ms.Position < StringTable.BytesPerBuffer,
"Small StringTable should not serialize a full empty byte buffer.");
}
ms.Position = 0;
using (BuildXLReader reader = new BuildXLReader(true, ms, true))
{
deserializedStringTable = await StringTable.DeserializeAsync(reader);
}
}
foreach (var entry in AddedStrings)
{
// Test deserialization behaviors
XAssert.AreEqual(entry.Value.Length, deserializedStringTable.GetLength(entry.Key));
XAssert.AreEqual(entry.Value.Length, deserializedStringTable.GetBinaryString(entry.Key).Length);
XAssert.AreEqual(entry.Value, deserializedStringTable.GetString(entry.Key));
XAssert.AreEqual(entry.Key, deserializedStringTable.AddString(entry.Value));
// Test original string table behaviors
XAssert.AreEqual(entry.Key, StringTable.AddString(entry.Value));
XAssert.AreEqual(entry.Value.Length, StringTable.GetLength(entry.Key));
XAssert.AreEqual(entry.Value.Length, StringTable.GetBinaryString(entry.Key).Length);
XAssert.AreEqual(entry.Value, StringTable.GetString(entry.Key));
}
}
public void SizeMix()
{
var st = new StringTable(0);
var map = new Dictionary <StringId, string>();
var sb = new StringBuilder(3000000);
// zip through small sizes
for (int i = 0; i < 3000; i++)
{
sb.Length = 0;
sb.Append('x', i);
string str = sb.ToString();
StringId sd = st.AddString(str);
map.Add(sd, str);
}
// now use increasingly large amounts, including exceeding the size of a single buffer's worth
for (int i = 0; i < 10; i++)
{
sb.Length = 0;
sb.Append('x', i * 10000);
string str = sb.ToString();
StringId sd = st.AddString(str);
if (!map.ContainsKey(sd))
{
map.Add(sd, str);
}
}
for (int i = 0; i < 2; i++)
{
// make sure all the right strings come out
int startBias = 0;
var buf = new char[4000000];
foreach (StringId sd in map.Keys)
{
// get the actual string we kept
string str = map[sd];
// does the table report the right length?
int length = st.GetLength(sd);
XAssert.AreEqual(str.Length, length);
// put sentinel bytes in the char buffer, extract the string from the table, and check the sentinels
if (startBias > 0)
{
buf[startBias - 1] = (char)42;
}
buf[startBias + length] = (char)31415;
st.CopyString(sd, buf, startBias);
if (startBias > 0)
{
XAssert.AreEqual(42, buf[startBias - 1]);
}
XAssert.AreEqual(31415, buf[startBias + length]);
// make sure we got all the characters out that we should have
for (int j = 0; j < str.Length; j++)
{
XAssert.AreEqual(str[j], buf[startBias + j]);
}
startBias++;
}
// make sure the same behavior occurs after freezing
st.Freeze();
}
}
public async Task RunCommonTestsAsync()
{
XAssert.AreNotEqual(StringTable.Count, AddedStrings.Count, "Ensure strings are added through harness");
StringTable deserializedStringTable = null;
using (MemoryStream ms = new MemoryStream())
{
using (Stream stream = createWrapperStream(ms, compress: true))
{
using (BuildXLWriter writer = new BuildXLWriter(true, stream, true, logStats: true))
{
StringTable.Serialize(writer);
}
if (MaxSize < StringTable.BytesPerBuffer)
{
XAssert.IsTrue(stream.Position < StringTable.BytesPerBuffer,
"Small StringTable should not serialize a full empty byte buffer.");
}
}
// We can't set a position on a wrapper stream, setting it for the memory stream that contains the data.
using (MemoryStream ms2 = new MemoryStream(ms.ToArray()))
using (Stream readStream = createWrapperStream(ms2, compress: false))
using (BuildXLReader reader = new BuildXLReader(true, readStream, true))
{
deserializedStringTable = await StringTable.DeserializeAsync(reader);
}
}
foreach (var entry in AddedStrings)
{
// Test deserialization behaviors
XAssert.AreEqual(entry.Value.Length, deserializedStringTable.GetLength(entry.Key));
XAssert.AreEqual(entry.Value.Length, deserializedStringTable.GetBinaryString(entry.Key).Length);
XAssert.AreEqual(entry.Value, deserializedStringTable.GetString(entry.Key));
XAssert.AreEqual(entry.Key, deserializedStringTable.AddString(entry.Value));
// Test original string table behaviors
XAssert.AreEqual(entry.Key, StringTable.AddString(entry.Value));
XAssert.AreEqual(entry.Value.Length, StringTable.GetLength(entry.Key));
XAssert.AreEqual(entry.Value.Length, StringTable.GetBinaryString(entry.Key).Length);
XAssert.AreEqual(entry.Value, StringTable.GetString(entry.Key));
}
Stream createWrapperStream(MemoryStream memoryStream, bool compress)
{
if (m_useDeflateStream)
{
return(new TrackedStream(
new BufferedStream(
compress ? new DeflateStream(memoryStream, CompressionLevel.Fastest, leaveOpen: true)
: new DeflateStream(memoryStream, CompressionMode.Decompress, leaveOpen: true),
bufferSize: 64 << 10),
// Need to close the underlying stream to flush the content.
leaveOpen: false));
}
return(memoryStream);
}
}
public async Task SizeMixParallel()
{
var st = new StringTable(0);
ConcurrentDictionary <StringId, string> map = new ConcurrentDictionary <StringId, string>();
var sb = new StringBuilder(3000000);
var strings = new BlockingCollection <string>();
var stringIds = new BlockingCollection <StringId>();
var createStringsTask = Task.Run(() =>
{
// zip through small sizes
for (int i = 0; i < 3000; i++)
{
sb.Length = 0;
sb.Append('x', i);
string str = sb.ToString();
strings.Add(str);
}
// now use increasingly large amounts, including exceeding the size of a single buffer's worth
for (int i = 0; i < 100; i++)
{
sb.Length = 0;
sb.Append('x', i * 10000);
string str = sb.ToString();
}
strings.CompleteAdding();
});
var validateStringsTask = Task.Run(() =>
{
for (int i = 0; i < 2; i++)
{
// make sure all the right strings come out
int startBias = 0;
var buf = new char[4000000];
foreach (StringId sd in stringIds.GetConsumingEnumerable())
{
// get the actual string we kept
string str = map[sd];
// does the table report the right length?
int length = st.GetLength(sd);
XAssert.AreEqual(str.Length, length);
// put sentinel bytes in the char buffer, extract the string from the table, and check the sentinels
if (startBias > 0)
{
buf[startBias - 1] = (char)42;
}
buf[startBias + length] = (char)31415;
st.CopyString(sd, buf, startBias);
if (startBias > 0)
{
XAssert.AreEqual(42, buf[startBias - 1]);
}
XAssert.AreEqual(31415, buf[startBias + length]);
// make sure we got all the characters out that we should have
for (int j = 0; j < str.Length; j++)
{
XAssert.AreEqual(str[j], buf[startBias + j]);
}
startBias++;
}
// make sure the same behavior occurs after freezing
st.Freeze();
}
});
Parallel.ForEach(strings.GetConsumingEnumerable(), str =>
{
StringId sd = st.AddString(str);
map.TryAdd(sd, str);
stringIds.Add(sd);
});
stringIds.CompleteAdding();
await createStringsTask;
await validateStringsTask;
}
