public void Test_ReadSharedStrings()
{
var t = new StringsTable <SharedString>
{
String1 = "string",
String2 = "foo",
String3 = "string",
};
byte[] destination = new byte[1024];
var serializer = FlatBufferSerializer.Default.Compile <StringsTable <SharedString> >()
.WithSettings(new SerializerSettings
{
SharedStringWriterFactory = () => new SharedStringWriter(100),
SharedStringReaderFactory = () => SharedStringReader.Create(100),
});
int bytesWritten = serializer.Write(SpanWriter.Instance, destination, t);
var table = serializer.Parse(destination);
Assert.AreEqual("string", (string)table.String1);
Assert.AreEqual("foo", (string)table.String2);
Assert.AreEqual("string", (string)table.String3);
Assert.IsTrue(object.ReferenceEquals(table.String1.String, table.String3.String));
}
public override void GlobalSetup()
{
this.regularStringSerializer = FlatBufferSerializer.Default.Compile <VectorTable <string> >();
this.sharedStringSerializer = FlatBufferSerializer.Default.Compile <VectorTable <SharedString> >()
.WithSettings(new SerializerSettings
{
SharedStringReaderFactory = () => SharedStringReader.Create(this.CacheSize),
SharedStringWriterFactory = () => new SharedStringWriter(this.CacheSize),
});
this.sharedStringThreadSafeSerializer = FlatBufferSerializer.Default.Compile <VectorTable <SharedString> >()
.WithSettings(new SerializerSettings
{
SharedStringReaderFactory = () => SharedStringReader.CreateThreadSafe(this.CacheSize),
SharedStringWriterFactory = () => new SharedStringWriter(this.CacheSize),
});
Random random = new Random();
List <string> guids =
Enumerable.Range(0, this.VectorLength)
.Select(i => Guid.NewGuid().ToString()).ToList();
List <string> randomGuids = Enumerable.Range(0, this.VectorLength)
.Select(i => guids[random.Next(0, guids.Count)]).ToList();
List <string> guassianGuids = Enumerable.Range(0, this.VectorLength)
.Select(i => guids[NextGaussianInt(random, 0, guids.Count - 1)]).ToList();
nonSharedStringVector = new VectorTable <string> {
Vector = randomGuids.ToList()
};
randomSharedStringVector = new VectorTable <SharedString> {
Vector = randomGuids.Select(SharedString.Create).ToList()
};
nonRandomSharedStringVector = new VectorTable <SharedString> {
Vector = guassianGuids.Select(SharedString.Create).ToList()
};
int nonSharedSize = this.Serialize_RandomStringVector_WithRegularString();
int cacheSize = this.Serialize_NonRandomStringVector_WithSharing();
Console.WriteLine($"NonSharedSize = {nonSharedSize}");
Console.WriteLine($"DirectMapSize = {cacheSize}");
}
public static void Run()
{
// Create a matrix of 1000 rows.
Matrix matrix = new Matrix()
{
Rows = Enumerable.Range(0, 100).Select(x => CreateRow()).ToArray(),
};
// String deduplication is off by default.
ISerializer <Matrix> defaultSerializer = Matrix.Serializer;
// We can create a new serializer based on the current one with shared strings turned on.
// These factory delegates configure the Shared String reader / writer. For SharedStringReaders,
// it's very important to think about thread safety, since deserialized objects sometimes maintain a reference
// to the input buffer object, which can result in race conditions if the deserialized object
// is shared between threads. The recommendation is to disable string sharing on parse or to use
// a thread safe shared string reader.
ISerializer <Matrix> sharedStringSerializer = Matrix.Serializer.WithSettings(
new SerializerSettings
{
// This can be null to disable shared string reading:
// SharedStringReaderFactory = null,
SharedStringReaderFactory = () => SharedStringReader.CreateThreadSafe(),
SharedStringWriterFactory = () => new SharedStringWriter(),
});
// We can also create our own shared string providers (defined at the bottom of this file).
// These two use normal dictionaries internally.
ISerializer <Matrix> customSharedStringSerializer = Matrix.Serializer.WithSettings(
new SerializerSettings
{
SharedStringReaderFactory = () => new PerfectSharedStringReader(),
SharedStringWriterFactory = () => new PerfectSharedStringWriter(),
});
byte[] defaultBuffer = new byte[defaultSerializer.GetMaxSize(matrix)];
byte[] sharedBuffer = new byte[sharedStringSerializer.GetMaxSize(matrix)];
byte[] customBuffer = new byte[customSharedStringSerializer.GetMaxSize(matrix)];
int defaultBytesWritten = defaultSerializer.Write(defaultBuffer, matrix);
int sharedBytesWritten = sharedStringSerializer.Write(sharedBuffer, matrix);
int customBytesWritten = customSharedStringSerializer.Write(customBuffer, matrix);
Console.WriteLine($"Serialized size without shared strings: {defaultBytesWritten}");
// These will be the same since there are so few shared strings. For large numbers,
// the custom provider will give smaller outputs while being considerably slower.
Console.WriteLine($"Serialized size with shared strings: {sharedBytesWritten}");
Console.WriteLine($"Serialized size with custom shared strings: {customBytesWritten}");
Matrix nonSharedMatrix = defaultSerializer.Parse(defaultBuffer);
#pragma warning disable CS8602 // Dereference of a possibly null reference.
Debug.Assert(
nonSharedMatrix.Rows[0].Values[0].ColumnName == nonSharedMatrix.Rows[1].Values[0].ColumnName,
"Without string sharing, the contents of the strings in the same column in different rows are the same.");
Debug.Assert(
!object.ReferenceEquals(nonSharedMatrix.Rows[0].Values[0].ColumnName, nonSharedMatrix.Rows[1].Values[0].ColumnName),
"...but the object references are different, showing these are two different strings.");
Matrix sharedMatrix = sharedStringSerializer.Parse(sharedBuffer);
Debug.Assert(
sharedMatrix.Rows[0].Values[0].ColumnName == sharedMatrix.Rows[1].Values[0].ColumnName,
"With string sharing on, the values of the same column in different rows are still the same.");
Debug.Assert(
object.ReferenceEquals(sharedMatrix.Rows[0].Values[0].ColumnName, sharedMatrix.Rows[1].Values[0].ColumnName),
"...and the object references are also the same, showing that we have only parsed the string once.");
// same as the above.
Matrix customSharedMatrix = customSharedStringSerializer.Parse(customBuffer);
Debug.Assert(
customSharedMatrix.Rows[0].Values[0].ColumnName == customSharedMatrix.Rows[1].Values[0].ColumnName,
"With string sharing on, the values of the same column in different rows are still the same.");
Debug.Assert(
object.ReferenceEquals(customSharedMatrix.Rows[0].Values[0].ColumnName, customSharedMatrix.Rows[1].Values[0].ColumnName),
"...and the object references are also the same, showing that we have only parsed the string once.");
#pragma warning restore CS8602 // Dereference of a possibly null reference.
}
