public unsafe void Can_define_compressed_table_and_read_write_large()
{
using (var tx = Env.WriteTransaction())
{
Slice.From(tx.Allocator, "Compression", out var etagIndexName);
var fixedSizedIndex = new TableSchema.FixedSizeSchemaIndexDef
{
Name = etagIndexName,
IsGlobal = true,
StartIndex = 1,
};
var tableSchema = new TableSchema()
.CompressValues(fixedSizedIndex, true)
.DefineFixedSizeIndex(fixedSizedIndex)
.DefineKey(new TableSchema.SchemaIndexDef
{
StartIndex = 0,
Count = 1,
});
tableSchema.Create(tx, "Items", 16);
var itemsTable = tx.OpenTable(tableSchema, "Items");
const long number = 1L;
var str = Enumerable.Range(0, 10_000)
.Aggregate(new StringBuilder(), (sb, i) => sb.Append(i))
.ToString();
using (itemsTable.Allocate(out TableValueBuilder builder))
using (Slice.From(tx.Allocator, "val1", out var key))
using (Slice.From(tx.Allocator, str, out var val))
{
builder.Add(key);
builder.Add(Bits.SwapBytes(number));
builder.Add(val);
long id = itemsTable.Insert(builder);
long allocatedSize = itemsTable.GetAllocatedSize(id);
Assert.True(allocatedSize > 8192 && allocatedSize < str.Length);
}
using (Slice.From(tx.Allocator, "val1", out var key))
{
Assert.True(itemsTable.ReadByKey(key, out var reader));
var ptr = reader.Read(2, out var size);
using var _ = Slice.External(tx.Allocator, ptr, size, out var slice);
Assert.Equal(str, slice.ToString());
}
using (Slice.From(tx.Allocator, "val1", out var key))
{
Assert.True(itemsTable.ReadByKey(key, out var reader));
Assert.True(itemsTable.ReadByKey(key, out var reader2));
Assert.Equal(new IntPtr(reader.Pointer), new IntPtr(reader2.Pointer));
}
}
}
public void ErrorsOnInvalidSchemaWithSingleFixedIndex()
{
using (var tx = Env.WriteTransaction())
{
var expectedSchema = new TableSchema();
var def = new TableSchema.FixedSizeSchemaIndexDef
{
StartIndex = 2,
};
Slice.From(tx.Allocator, "Test Name", ByteStringType.Immutable, out def.Name);
expectedSchema.DefineFixedSizeIndex(def);
var actualSchema = new TableSchema();
def = new TableSchema.FixedSizeSchemaIndexDef
{
StartIndex = 4,
};
Slice.From(tx.Allocator, "Bad Test Name", ByteStringType.Immutable, out def.Name);
actualSchema.DefineFixedSizeIndex(def);
Assert.Throws <ArgumentNullException>(delegate { expectedSchema.Validate(null); });
Assert.Throws <ArgumentException>(delegate { expectedSchema.Validate(actualSchema); });
}
}
public void delete_by_fixed_sized_index()
{
using (var tx = Env.WriteTransaction())
{
Slice.From(tx.Allocator, "EtagIndexName", out var etagIndexName);
var fixedSizedIndex = new TableSchema.FixedSizeSchemaIndexDef
{
Name = etagIndexName,
IsGlobal = true,
StartIndex = 1,
};
var tableSchema = new TableSchema()
.DefineFixedSizeIndex(fixedSizedIndex)
.DefineKey(new TableSchema.SchemaIndexDef
{
StartIndex = 0,
Count = 1,
});
tableSchema.Create(tx, "Items", 16);
var itemsTable = tx.OpenTable(tableSchema, "Items");
const long number1 = 1L;
const long number2 = 2L;
const long number3 = 3L;
using (itemsTable.Allocate(out TableValueBuilder builder))
using (Slice.From(tx.Allocator, "val1", out var key))
{
builder.Add(key);
builder.Add(Bits.SwapBytes(number1));
itemsTable.Set(builder);
}
using (itemsTable.Allocate(out TableValueBuilder builder))
using (Slice.From(tx.Allocator, "val2", out var key))
{
builder.Add(key);
builder.Add(Bits.SwapBytes(number2));
itemsTable.Set(builder);
}
using (itemsTable.Allocate(out TableValueBuilder builder))
using (Slice.From(tx.Allocator, "val3", out var key))
{
builder.Add(key);
builder.Add(Bits.SwapBytes(number3));
itemsTable.Set(builder);
}
Assert.False(itemsTable.DeleteByIndex(fixedSizedIndex, 0L));
Assert.True(itemsTable.DeleteByIndex(fixedSizedIndex, 2L));
Assert.True(itemsTable.DeleteByIndex(fixedSizedIndex, 3L));
Assert.True(itemsTable.DeleteByIndex(fixedSizedIndex, 1L));
}
}
public void Can_update_compressed_value()
{
using (var tx = Env.WriteTransaction())
{
Slice.From(tx.Allocator, "Compression", out var etagIndexName);
var fixedSizedIndex = new TableSchema.FixedSizeSchemaIndexDef
{
Name = etagIndexName,
IsGlobal = true,
StartIndex = 1,
};
var tableSchema = new TableSchema()
.CompressValues(fixedSizedIndex, true)
.DefineFixedSizeIndex(fixedSizedIndex)
.DefineKey(new TableSchema.SchemaIndexDef
{
StartIndex = 0,
Count = 1,
});
tableSchema.Create(tx, "Items", 16);
var itemsTable = tx.OpenTable(tableSchema, "Items");
const long number = 1L;
using (itemsTable.Allocate(out TableValueBuilder builder))
using (Slice.From(tx.Allocator, "val1", out var key))
using (Slice.From(tx.Allocator, new string('a', 1024 * 16), out var val))
{
builder.Add(key);
builder.Add(Bits.SwapBytes(number));
builder.Add(val);
long id = itemsTable.Insert(builder);
var allocatedSize = itemsTable.GetAllocatedSize(id);
Assert.True(allocatedSize < 128);
}
using (itemsTable.Allocate(out TableValueBuilder builder))
using (Slice.From(tx.Allocator, "val1", out var key))
using (Slice.From(tx.Allocator, new string('a', 1024 * 32), out var val))
{
builder.Add(key);
builder.Add(Bits.SwapBytes(number));
builder.Add(val);
itemsTable.Set(builder);
}
using (Slice.From(tx.Allocator, "val1", out var key))
{
Assert.True(itemsTable.ReadByKey(key, out var reader));
Assert.True(itemsTable.GetAllocatedSize(reader.Id) < 128);
}
}
}
private void FixedSchemaIndexDefEqual(TableSchema.FixedSizeSchemaIndexDef expectedIndex,
TableSchema.FixedSizeSchemaIndexDef actualIndex)
{
if (expectedIndex == null)
{
Assert.Equal(null, actualIndex);
}
else
{
Assert.Equal(expectedIndex.IsGlobal, actualIndex.IsGlobal);
Assert.True(SliceComparer.Equals(expectedIndex.Name, actualIndex.Name));
Assert.Equal(expectedIndex.StartIndex, actualIndex.StartIndex);
}
}
public void CanSerializeMultiIndexSchema()
{
using (var tx = Env.WriteTransaction())
{
var def1 = new TableSchema.SchemaIndexDef
{
StartIndex = 2,
Count = 1,
};
Slice.From(tx.Allocator, "Test Name 1", ByteStringType.Immutable, out def1.Name);
var def2 = new TableSchema.SchemaIndexDef
{
StartIndex = 1,
Count = 1,
};
Slice.From(tx.Allocator, "Test Name 2", ByteStringType.Immutable, out def2.Name);
var def3 = new TableSchema.FixedSizeSchemaIndexDef()
{
StartIndex = 2,
IsGlobal = true,
};
Slice.From(tx.Allocator, "Test Name 3", ByteStringType.Immutable, out def3.Name);
var tableSchema = new TableSchema()
.DefineIndex(def1)
.DefineIndex(def2)
.DefineFixedSizeIndex(def3)
.DefineKey(new TableSchema.SchemaIndexDef
{
StartIndex = 3,
Count = 1,
});
byte[] serialized = tableSchema.SerializeSchema();
fixed(byte *ptr = serialized)
{
var actualTableSchema = TableSchema.ReadFrom(tx.Allocator, ptr, serialized.Length);
// This checks that reserializing is the same
Assert.Equal(serialized, actualTableSchema.SerializeSchema());
// This checks that what was deserialized is correct
SchemaDefEqual(tableSchema, actualTableSchema);
tableSchema.Validate(actualTableSchema);
}
}
}
public void Insert_same_value_to_fixed_sized_index_throws()
{
using (var tx = Env.WriteTransaction())
{
Slice.From(tx.Allocator, "EtagIndexName", out var etagIndexName);
var fixedSizedIndex = new TableSchema.FixedSizeSchemaIndexDef
{
Name = etagIndexName,
IsGlobal = true,
StartIndex = 1,
};
var tableSchema = new TableSchema()
.DefineFixedSizeIndex(fixedSizedIndex)
.DefineKey(new TableSchema.SchemaIndexDef
{
StartIndex = 0,
Count = 1,
});
tableSchema.Create(tx, "Items", 16);
var itemsTable = tx.OpenTable(tableSchema, "Items");
const long number = 1L;
using (itemsTable.Allocate(out TableValueBuilder builder))
using (Slice.From(tx.Allocator, "val1", out var key))
{
builder.Add(key);
builder.Add(Bits.SwapBytes(number));
itemsTable.Set(builder);
}
using (itemsTable.Allocate(out TableValueBuilder builder))
using (Slice.From(tx.Allocator, "val2", out var key))
{
builder.Add(key);
builder.Add(Bits.SwapBytes(number));
var exception = Assert.Throws <VoronErrorException>(() => itemsTable.Set(builder));
Assert.True(exception.Message.StartsWith("Attempt to add duplicate value"));
}
}
}
public void ErrorsOnInvalidFixedSizeDef()
{
using (var tx = Env.WriteTransaction())
{
var expectedIndex = new TableSchema.FixedSizeSchemaIndexDef
{
StartIndex = 2,
};
Slice.From(tx.Allocator, "Test Name", ByteStringType.Immutable, out expectedIndex.Name);
var actualIndex = new TableSchema.FixedSizeSchemaIndexDef
{
StartIndex = 5,
};
Slice.From(tx.Allocator, "Test Name", ByteStringType.Immutable, out actualIndex.Name);
Assert.Throws <ArgumentNullException>(delegate { expectedIndex.Validate(null); });
Assert.Throws <ArgumentException>(delegate { expectedIndex.Validate(actualIndex); });
}
}
public void CanSerializeFixedIndex()
{
using (var tx = Env.WriteTransaction())
{
var expectedIndex = new TableSchema.FixedSizeSchemaIndexDef
{
StartIndex = 2,
IsGlobal = true,
};
Slice.From(tx.Allocator, "Test Name 2", ByteStringType.Immutable, out expectedIndex.Name);
byte[] serialized = expectedIndex.Serialize();
fixed(byte *serializedPtr = serialized)
{
var actualIndex = TableSchema.FixedSizeSchemaIndexDef.ReadFrom(tx.Allocator, serializedPtr, serialized.Length);
Assert.Equal(serialized, actualIndex.Serialize());
FixedSchemaIndexDefEqual(expectedIndex, actualIndex);
expectedIndex.Validate(actualIndex);
}
}
}
public void WillNotRememberOldDictionariesAfterRestart()
{
RequireFileBasedPager();
using var allocator = new ByteStringContext(SharedMultipleUseFlag.None);
using var ____ = Slice.From(allocator, "PK", out var pk);
using var ___ = Slice.From(allocator, "Etags", out var etags);
using var __ = Slice.From(allocator, "Table", out var tbl);
var idx = new TableSchema.FixedSizeSchemaIndexDef {
Name = etags, IsGlobal = false, StartIndex = 0
};
var schema = new TableSchema()
.DefineKey(new TableSchema.SchemaIndexDef
{
Name = pk,
IsGlobal = false,
StartIndex = 0,
Count = 1
})
.DefineFixedSizeIndex(idx)
.CompressValues(idx, true);
using (var wtx = Env.WriteTransaction())
{
schema.Create(wtx, tbl, null);
wtx.Commit();
}
using (var wtx = Env.WriteTransaction())
{
var table = wtx.OpenTable(schema, tbl);
using var _ = Slice.From(allocator, LongRandomString(), out var val);
for (long i = 0; i < 1024 * 4; i++)
{
using (table.Allocate(out var builder))
{
builder.Add(Bits.SwapBytes(i));
builder.Add(val);
table.Insert(builder);
}
}
wtx.Commit();
}
using (var wtx = Env.WriteTransaction())
{
var table = wtx.OpenTable(schema, tbl);
using var _ = Slice.From(allocator, LongRandomString(), out var val);
for (long i = 20_000; i < 20_000 + 1024 * 4; i++)
{
using (table.Allocate(out var builder))
{
builder.Add(Bits.SwapBytes(i));
builder.Add(val);
table.Insert(builder);
}
table.ReadLast(idx); // force to read the new dictionary
}
// explicitly discard the change
//wtx.Commit();
}
using (var wtx = Env.WriteTransaction())
{
var table = wtx.OpenTable(schema, tbl);
using var _ = Slice.From(allocator, LongRandomString(), out var val);
for (long i = 20_000; i < 20_000 + 1020 * 4; i++)
{
using (table.Allocate(out var builder))
{
builder.Add(Bits.SwapBytes(i));
builder.Add(val);
table.Insert(builder);
}
}
wtx.Commit();
}
RestartDatabase();
using (var rtx = Env.ReadTransaction())
{
var table = rtx.OpenTable(schema, tbl);
using (var it = table.SeekForwardFrom(idx, 0, 0).GetEnumerator())
{
while (it.MoveNext())
{
}
}
}
}
public unsafe void OnDataMoveShouldForgetOldCompressionIds()
{
var random = new Random(357);
using (var tx = Env.WriteTransaction())
{
Slice.From(tx.Allocator, "Compression", out var etagIndexName);
var fixedSizedIndex = new TableSchema.FixedSizeSchemaIndexDef
{
Name = etagIndexName,
IsGlobal = true,
StartIndex = 1,
};
var tableSchema = new TableSchema()
.CompressValues(fixedSizedIndex, true)
.DefineFixedSizeIndex(fixedSizedIndex)
.DefineKey(new TableSchema.SchemaIndexDef
{
StartIndex = 0,
Count = 1,
});
tableSchema.Create(tx, "Items", 16);
var itemsTable = tx.OpenTable(tableSchema, "Items");
var rnd = Enumerable.Range(1, 32)
.Select(i => Compression.RandomString(random, 123))
.ToList();
for (int i = 0; i < 32 * 1024; i++)
{
using (itemsTable.Allocate(out TableValueBuilder builder))
using (Slice.From(tx.Allocator, "val" + i, out var key))
using (Slice.From(tx.Allocator, rnd[i % rnd.Count], out var val))
{
builder.Add(key);
builder.Add(Bits.SwapBytes((long)i));
builder.Add(val);
itemsTable.Insert(builder);
}
}
AssertRemainingValues();
for (int i = 32 * 1024 - 1; i >= 0; i--)
{
using (Slice.From(tx.Allocator, "val" + i, out var key))
{
if (i % 10 != 0)
{
itemsTable.DeleteByKey(key);
}
}
}
AssertRemainingValues(validate: true);
void AssertRemainingValues(bool validate = false)
{
foreach (var reader in itemsTable.SeekBackwardFrom(fixedSizedIndex, long.MaxValue))
{
var p = reader.Reader.Read(1, out var sizeOfInt);
Assert.Equal(sizeOfInt, sizeof(long));
var i = Bits.SwapBytes(*(long *)p);
var keyPtr = reader.Reader.Read(0, out var keySize);
using (Slice.From(tx.Allocator, keyPtr, keySize, out var currentKey))
using (Slice.From(tx.Allocator, "val" + i, out var expectedKey))
{
Assert.True(SliceStructComparer.Instance.Equals(currentKey, expectedKey));
}
if (validate)
{
if (i % 10 != 0)
{
Assert.True(false, $"Oops {i}");
}
}
}
}
}
}
public void Can_force_small_value_to_compress_to_large()
{
var random = new Random(222);
using (var tx = Env.WriteTransaction())
{
Slice.From(tx.Allocator, "Compression", out var etagIndexName);
var fixedSizedIndex = new TableSchema.FixedSizeSchemaIndexDef
{
Name = etagIndexName,
IsGlobal = true,
StartIndex = 1,
};
var tableSchema = new TableSchema()
.CompressValues(fixedSizedIndex, true)
.DefineFixedSizeIndex(fixedSizedIndex)
.DefineKey(new TableSchema.SchemaIndexDef
{
StartIndex = 0,
Count = 1,
});
tableSchema.Create(tx, "Items", 16);
var itemsTable = tx.OpenTable(tableSchema, "Items");
long number = 1L;
var rnd = Enumerable.Range(1, 32)
.Select(i => RandomString(random, 1024))
.ToList();
for (int i = 0; i < 16 * 1024; i++)
{
using (itemsTable.Allocate(out TableValueBuilder builder))
using (Slice.From(tx.Allocator, "val" + i, out var key))
using (Slice.From(tx.Allocator, rnd[i % rnd.Count], out var val))
{
builder.Add(key);
builder.Add(Bits.SwapBytes(++number));
builder.Add(val);
itemsTable.Insert(builder);
}
}
for (int i = 512; i < 768; i++)
{
using (Slice.From(tx.Allocator, "val" + i, out var key))
{
itemsTable.DeleteByKey(key);
}
}
var str = Enumerable.Range(0, 10_000)
.Aggregate(new StringBuilder(), (sb, i) => sb.Append((char)(i % 26 + 'A')))
.ToString();
for (int i = 0; i < 256; i++)
{
using (itemsTable.Allocate(out TableValueBuilder builder))
using (Slice.From(tx.Allocator, "val" + i, out var key))
using (Slice.From(tx.Allocator, str, out var val))
{
builder.Add(key);
builder.Add(Bits.SwapBytes(++number));
builder.Add(val);
itemsTable.Set(builder);
}
}
}
}
public unsafe void Can_get_better_compression_rate_after_training()
{
using (var tx = Env.WriteTransaction())
{
Slice.From(tx.Allocator, "Compression", out var etagIndexName);
var fixedSizedIndex = new TableSchema.FixedSizeSchemaIndexDef
{
Name = etagIndexName,
IsGlobal = true,
StartIndex = 1,
};
var tableSchema = new TableSchema()
.CompressValues(fixedSizedIndex, true)
.DefineFixedSizeIndex(fixedSizedIndex)
.DefineKey(new TableSchema.SchemaIndexDef
{
StartIndex = 0,
Count = 1,
});
tableSchema.Create(tx, "Items", 16);
var itemsTable = tx.OpenTable(tableSchema, "Items");
long number = 1L;
var random = new Random(49941);
var data = string.Join(", ", Enumerable.Range(0, 100).Select(_ =>
{
var bytes = new byte[16];
random.NextBytes(bytes);
return(new Guid(bytes).ToString());
}));
int firstAllocatedSize;
using (itemsTable.Allocate(out TableValueBuilder builder))
using (Slice.From(tx.Allocator, "val", out var key))
using (Slice.From(tx.Allocator, data, out var val))
{
builder.Add(key);
builder.Add(Bits.SwapBytes(number));
builder.Add(val);
long id = itemsTable.Insert(builder);
firstAllocatedSize = itemsTable.GetAllocatedSize(id);
}
var minAllocatedSize = 0;
for (int i = 0; i < 100; i++)
{
using (itemsTable.Allocate(out TableValueBuilder builder))
using (Slice.From(tx.Allocator, "val" + i, out var key))
using (Slice.From(tx.Allocator, data, out var val))
{
builder.Add(key);
builder.Add(Bits.SwapBytes(++number));
builder.Add(val);
long id = itemsTable.Insert(builder);
var allocatedSize = itemsTable.GetAllocatedSize(id);
minAllocatedSize = Math.Min(minAllocatedSize, allocatedSize);
}
}
Assert.True(minAllocatedSize < firstAllocatedSize);
}
}
