public void BigStrings() { var st = new StringTable(0); var s1 = new string('x', StringTable.BytesPerBuffer - 1); var s2 = new string('y', StringTable.BytesPerBuffer); var s3 = new string('z', StringTable.BytesPerBuffer + 1); StringId id1 = st.AddString(s1); StringId id2 = st.AddString(s2); StringId id3 = st.AddString(s3); XAssert.AreEqual(s1.Length, st.GetLength(id1)); XAssert.AreEqual(s2.Length, st.GetLength(id2)); XAssert.AreEqual(s3.Length, st.GetLength(id3)); var buf = new char[StringTable.BytesPerBuffer + 1]; st.CopyString(id1, buf, 0); for (int j = 0; j < s1.Length; j++) { XAssert.AreEqual(s1[j], buf[j]); } st.CopyString(id2, buf, 0); for (int j = 0; j < s2.Length; j++) { XAssert.AreEqual(s2[j], buf[j]); } st.CopyString(id3, buf, 0); for (int j = 0; j < s3.Length; j++) { XAssert.AreEqual(s3[j], buf[j]); } XAssert.IsTrue(st.CaseInsensitiveEquals(id1, st.AddString(s1))); XAssert.IsTrue(st.CaseInsensitiveEquals(id2, st.AddString(s2))); XAssert.IsTrue(st.CaseInsensitiveEquals(id3, st.AddString(s3))); XAssert.IsTrue(st.Equals(s1, id1)); XAssert.IsTrue(st.Equals(s2, id2)); XAssert.IsTrue(st.Equals(s3, id3)); XAssert.IsFalse(st.CaseInsensitiveEquals(id1, st.AddString(s2))); XAssert.IsFalse(st.CaseInsensitiveEquals(id1, st.AddString(s3))); XAssert.IsFalse(st.CaseInsensitiveEquals(id2, st.AddString(s1))); XAssert.IsFalse(st.CaseInsensitiveEquals(id2, st.AddString(s3))); XAssert.IsFalse(st.CaseInsensitiveEquals(id3, st.AddString(s1))); XAssert.IsFalse(st.CaseInsensitiveEquals(id3, st.AddString(s2))); XAssert.IsFalse(st.Equals(s2, id1)); XAssert.IsFalse(st.Equals(s3, id1)); XAssert.IsFalse(st.Equals(s1, id2)); XAssert.IsFalse(st.Equals(s3, id2)); XAssert.IsFalse(st.Equals(s1, id3)); XAssert.IsFalse(st.Equals(s2, id3)); }
/// <summary> /// Returns the length of a StringId when rendered to string using given <paramref name="escaping"/>. /// </summary> public int GetLength(StringId stringId, PipDataFragmentEscaping escaping) { switch (escaping) { case PipDataFragmentEscaping.CRuntimeArgumentRules: // - passing null as StringBuilder to AppendEscapedCommandLineWord is ok // - the return value of AppendEscapedCommandLineWord indicates how many characters would be appended return(CommandLineEscaping.AppendEscapedCommandLineWord(builder: null, word: Render(stringId))); case PipDataFragmentEscaping.NoEscaping: return(StringTable.GetLength(stringId)); // instead of retrieving the whole string, StringTable.GetLength is faster. default: Contract.Assert(false, I($"Unhandled fragmentEscaping: {escaping}")); return(0); } }
public void CharMix() { // make sure all characters are handled properly var st = new StringTable(0); var map = new Dictionary <StringId, string>(); var sb = new StringBuilder(3000000); // zip through small sizes for (int i = 0; i < 65535; i++) { sb.Length = 0; sb.Append((char)i, (i % 255) + 1); string str = sb.ToString(); StringId sd = st.AddString(str); map.Add(sd, str); } for (int i = 0; i < 2; i++) { // make sure all the right strings come out var buf = new char[65535]; foreach (StringId sd in map.Keys) { // get the actual string we kept string str = map[sd]; // does the table report the right length? int length = st.GetLength(sd); XAssert.AreEqual(str.Length, length); // extract the chars from the table st.CopyString(sd, buf, 0); // make sure we got all the characters out that we should have for (int j = 0; j < str.Length; j++) { XAssert.AreEqual(str[j], buf[j]); } } // make sure the same behavior occurs after freezing st.Freeze(); } }
public async Task RunCommonTestsAsync() { XAssert.AreNotEqual(StringTable.Count, AddedStrings.Count, "Ensure strings are added through harness"); StringTable deserializedStringTable = null; using (MemoryStream ms = new MemoryStream()) { using (BuildXLWriter writer = new BuildXLWriter(true, ms, true, logStats: true)) { StringTable.Serialize(writer); } if (MaxSize < StringTable.BytesPerBuffer) { XAssert.IsTrue(ms.Position < StringTable.BytesPerBuffer, "Small StringTable should not serialize a full empty byte buffer."); } ms.Position = 0; using (BuildXLReader reader = new BuildXLReader(true, ms, true)) { deserializedStringTable = await StringTable.DeserializeAsync(reader); } } foreach (var entry in AddedStrings) { // Test deserialization behaviors XAssert.AreEqual(entry.Value.Length, deserializedStringTable.GetLength(entry.Key)); XAssert.AreEqual(entry.Value.Length, deserializedStringTable.GetBinaryString(entry.Key).Length); XAssert.AreEqual(entry.Value, deserializedStringTable.GetString(entry.Key)); XAssert.AreEqual(entry.Key, deserializedStringTable.AddString(entry.Value)); // Test original string table behaviors XAssert.AreEqual(entry.Key, StringTable.AddString(entry.Value)); XAssert.AreEqual(entry.Value.Length, StringTable.GetLength(entry.Key)); XAssert.AreEqual(entry.Value.Length, StringTable.GetBinaryString(entry.Key).Length); XAssert.AreEqual(entry.Value, StringTable.GetString(entry.Key)); } }
public void SizeMix() { var st = new StringTable(0); var map = new Dictionary <StringId, string>(); var sb = new StringBuilder(3000000); // zip through small sizes for (int i = 0; i < 3000; i++) { sb.Length = 0; sb.Append('x', i); string str = sb.ToString(); StringId sd = st.AddString(str); map.Add(sd, str); } // now use increasingly large amounts, including exceeding the size of a single buffer's worth for (int i = 0; i < 10; i++) { sb.Length = 0; sb.Append('x', i * 10000); string str = sb.ToString(); StringId sd = st.AddString(str); if (!map.ContainsKey(sd)) { map.Add(sd, str); } } for (int i = 0; i < 2; i++) { // make sure all the right strings come out int startBias = 0; var buf = new char[4000000]; foreach (StringId sd in map.Keys) { // get the actual string we kept string str = map[sd]; // does the table report the right length? int length = st.GetLength(sd); XAssert.AreEqual(str.Length, length); // put sentinel bytes in the char buffer, extract the string from the table, and check the sentinels if (startBias > 0) { buf[startBias - 1] = (char)42; } buf[startBias + length] = (char)31415; st.CopyString(sd, buf, startBias); if (startBias > 0) { XAssert.AreEqual(42, buf[startBias - 1]); } XAssert.AreEqual(31415, buf[startBias + length]); // make sure we got all the characters out that we should have for (int j = 0; j < str.Length; j++) { XAssert.AreEqual(str[j], buf[startBias + j]); } startBias++; } // make sure the same behavior occurs after freezing st.Freeze(); } }
public async Task RunCommonTestsAsync() { XAssert.AreNotEqual(StringTable.Count, AddedStrings.Count, "Ensure strings are added through harness"); StringTable deserializedStringTable = null; using (MemoryStream ms = new MemoryStream()) { using (Stream stream = createWrapperStream(ms, compress: true)) { using (BuildXLWriter writer = new BuildXLWriter(true, stream, true, logStats: true)) { StringTable.Serialize(writer); } if (MaxSize < StringTable.BytesPerBuffer) { XAssert.IsTrue(stream.Position < StringTable.BytesPerBuffer, "Small StringTable should not serialize a full empty byte buffer."); } } // We can't set a position on a wrapper stream, setting it for the memory stream that contains the data. using (MemoryStream ms2 = new MemoryStream(ms.ToArray())) using (Stream readStream = createWrapperStream(ms2, compress: false)) using (BuildXLReader reader = new BuildXLReader(true, readStream, true)) { deserializedStringTable = await StringTable.DeserializeAsync(reader); } } foreach (var entry in AddedStrings) { // Test deserialization behaviors XAssert.AreEqual(entry.Value.Length, deserializedStringTable.GetLength(entry.Key)); XAssert.AreEqual(entry.Value.Length, deserializedStringTable.GetBinaryString(entry.Key).Length); XAssert.AreEqual(entry.Value, deserializedStringTable.GetString(entry.Key)); XAssert.AreEqual(entry.Key, deserializedStringTable.AddString(entry.Value)); // Test original string table behaviors XAssert.AreEqual(entry.Key, StringTable.AddString(entry.Value)); XAssert.AreEqual(entry.Value.Length, StringTable.GetLength(entry.Key)); XAssert.AreEqual(entry.Value.Length, StringTable.GetBinaryString(entry.Key).Length); XAssert.AreEqual(entry.Value, StringTable.GetString(entry.Key)); } Stream createWrapperStream(MemoryStream memoryStream, bool compress) { if (m_useDeflateStream) { return(new TrackedStream( new BufferedStream( compress ? new DeflateStream(memoryStream, CompressionLevel.Fastest, leaveOpen: true) : new DeflateStream(memoryStream, CompressionMode.Decompress, leaveOpen: true), bufferSize: 64 << 10), // Need to close the underlying stream to flush the content. leaveOpen: false)); } return(memoryStream); } }
public async Task SizeMixParallel() { var st = new StringTable(0); ConcurrentDictionary <StringId, string> map = new ConcurrentDictionary <StringId, string>(); var sb = new StringBuilder(3000000); var strings = new BlockingCollection <string>(); var stringIds = new BlockingCollection <StringId>(); var createStringsTask = Task.Run(() => { // zip through small sizes for (int i = 0; i < 3000; i++) { sb.Length = 0; sb.Append('x', i); string str = sb.ToString(); strings.Add(str); } // now use increasingly large amounts, including exceeding the size of a single buffer's worth for (int i = 0; i < 100; i++) { sb.Length = 0; sb.Append('x', i * 10000); string str = sb.ToString(); } strings.CompleteAdding(); }); var validateStringsTask = Task.Run(() => { for (int i = 0; i < 2; i++) { // make sure all the right strings come out int startBias = 0; var buf = new char[4000000]; foreach (StringId sd in stringIds.GetConsumingEnumerable()) { // get the actual string we kept string str = map[sd]; // does the table report the right length? int length = st.GetLength(sd); XAssert.AreEqual(str.Length, length); // put sentinel bytes in the char buffer, extract the string from the table, and check the sentinels if (startBias > 0) { buf[startBias - 1] = (char)42; } buf[startBias + length] = (char)31415; st.CopyString(sd, buf, startBias); if (startBias > 0) { XAssert.AreEqual(42, buf[startBias - 1]); } XAssert.AreEqual(31415, buf[startBias + length]); // make sure we got all the characters out that we should have for (int j = 0; j < str.Length; j++) { XAssert.AreEqual(str[j], buf[startBias + j]); } startBias++; } // make sure the same behavior occurs after freezing st.Freeze(); } }); Parallel.ForEach(strings.GetConsumingEnumerable(), str => { StringId sd = st.AddString(str); map.TryAdd(sd, str); stringIds.Add(sd); }); stringIds.CompleteAdding(); await createStringsTask; await validateStringsTask; }