public override void Setup()
{
base.Setup();
using (var tx = Env.WriteTransaction())
{
Schema.Create(tx, TableNameSlice, 16);
tx.Commit();
}
var totalPairs = Utils.GenerateUniqueRandomSlicePairs(
NumberOfTransactions * NumberOfRecordsPerTransaction,
KeyLength,
RandomSeed == -1 ? null as int? : RandomSeed);
// This will sort just the KEYS
totalPairs.Sort((x, y) => SliceComparer.Compare(x.Item1, y.Item1));
// Distribute keys in such a way that _valueBuilders[i][k] <
// _valueBuilders[j][m] iff i < j, for all k and m.
_valueBuilders = new List <TableValueBuilder> [NumberOfTransactions];
for (int i = 0; i < NumberOfTransactions; i++)
{
var values = totalPairs.Take(NumberOfRecordsPerTransaction);
totalPairs.RemoveRange(0, NumberOfRecordsPerTransaction);
_valueBuilders[i] = new List <TableValueBuilder>();
foreach (var pair in values)
{
_valueBuilders[i].Add(new TableValueBuilder
{
pair.Item1,
pair.Item2
});
}
}
}
private unsafe void CreateSchema()
{
_errorsSchema.DefineIndex(new TableSchema.SchemaIndexDef
{
StartIndex = 0,
// there is just a single instance of this table
// but we need it to be local so we'll be able to compact it
IsGlobal = false,
Name = IndexSchema.ErrorTimestampsSlice
});
using (_contextPool.AllocateOperationContext(out TransactionOperationContext context))
using (var tx = context.OpenWriteTransaction())
{
_errorsSchema.Create(tx.InnerTransaction, "Errors", 16);
var typeInt = (int)_index.Type;
var statsTree = tx.InnerTransaction.CreateTree(IndexSchema.StatsTree);
using (Slice.External(context.Allocator, (byte *)&typeInt, sizeof(int), out Slice tmpSlice))
statsTree.Add(IndexSchema.TypeSlice, tmpSlice);
if (statsTree.Read(IndexSchema.CreatedTimestampSlice) == null)
{
var binaryDate = SystemTime.UtcNow.ToBinary();
using (Slice.External(context.Allocator, (byte *)&binaryDate, sizeof(long), out Slice tmpSlice))
statsTree.Add(IndexSchema.CreatedTimestampSlice, tmpSlice);
}
tx.InnerTransaction.CreateTree(IndexSchema.EtagsTree);
tx.InnerTransaction.CreateTree(IndexSchema.EtagsTombstoneTree);
tx.InnerTransaction.CreateTree(IndexSchema.References);
_index.Definition.Persist(context, _environment.Options);
tx.Commit();
}
}
public static TableSchema CopySeries(this IMslScript script, DataColumn fromCol, DataColumn toCol)
{
if (toCol == null)
{
TempTable.Create();
toCol = TempTable["value"];
}
var inputTable = Interpreter.Context.TomScripting.GetManager(fromCol.Table.TableName);
var outTable = Interpreter.Context.TomScripting.GetManager(toCol.Table.TableName);
var stock = Interpreter.Context.Scope.Stock;
var from = Interpreter.Context.Scope.From;
var to = Interpreter.Context.Scope.To;
long ownerId = stock.GetId(inputTable.Schema.OwnerIdColumn);
using (TransactionScope trans = new TransactionScope())
{
ScopedTable fromData = inputTable.Query(ownerId, new DateClause(from, to), OriginClause.Default);
var result = outTable.Query(ownerId);
foreach (DataRow row in fromData.Rows)
{
DataRow outRow = result.NewRow();
outRow[result.Schema.OwnerIdColumn] = ownerId;
outRow.SetDate(outTable.Schema, row.GetDate(fromData.Schema));
outRow[toCol.ColumnName] = row[fromCol.ColumnName];
result.AddOrUpdate(outRow, toCol.ColumnName);
}
trans.Complete();
}
return(Interpreter.Context.TomScripting.GetManager(toCol.Table.TableName).Schema);
}
public void Initialize(RavenTransaction tx, RavenConfiguration configuration, Logger log)
{
_logHistoryMaxEntries = configuration.Cluster.LogHistoryMaxEntries;
LogHistoryTable.Create(tx.InnerTransaction, RachisLogHistory.LogHistorySlice, 16);
_log = log;
}
public unsafe void CanDeleteTableWithLargeValues()
{
TableSchema schema = new TableSchema();
schema.DefineKey(new TableSchema.SchemaIndexDef
{
StartIndex = 0,
Count = 1,
Name = Etag,
IsGlobal = true
});
schema.DefineIndex(new TableSchema.SchemaIndexDef
{
StartIndex = 0,
Count = 1,
Name = Local,
IsGlobal = false
});
using (Transaction tx = Env.WriteTransaction())
{
schema.Create(tx, "first", 256);
schema.Create(tx, "second", 256);
Table fst = tx.OpenTable(schema, "first");
Table snd = tx.OpenTable(schema, "second");
for (var i = 0; i < 1000;)
{
var bytes = new byte[2 * RawDataSection.MaxItemSize + 100];
new Random(i).NextBytes(bytes);
TableValueBuilder tvb1 = new TableValueBuilder();
TableValueBuilder tvb2 = new TableValueBuilder();
tvb1.Add(i++);
tvb2.Add(i++);
fixed(byte *ptr = bytes)
{
tvb1.Add(ptr, bytes.Length);
tvb2.Add(ptr, bytes.Length);
fst.Insert(tvb1);
snd.Insert(tvb2);
}
}
tx.Commit();
}
using (var tx = Env.WriteTransaction())
{
var firstTable = tx.OpenTable(schema, "first");
firstTable.DeleteByPrimaryKey(Slices.BeforeAllKeys, x =>
{
if (schema.Key.IsGlobal)
{
return(firstTable.IsOwned(x.Reader.Id));
}
return(true);
});
var secondTable = tx.OpenTable(schema, "first");
secondTable.DeleteByPrimaryKey(Slices.BeforeAllKeys, x =>
{
if (schema.Key.IsGlobal)
{
return(secondTable.IsOwned(x.Reader.Id));
}
return(true);
});
tx.Commit();
}
}
public unsafe void CanBeSafelyModifiedOnEither()
{
using (var tx = Env.WriteTransaction())
{
Slice.From(tx.Allocator, "RevisionsChangeVector", ByteStringType.Immutable, out var changeVectorSlice);
Slice.From(tx.Allocator, "Etag", ByteStringType.Immutable, out var etag);
var revisionsSchema = new TableSchema();
revisionsSchema.DefineKey(new TableSchema.SchemaIndexDef
{
StartIndex = 0,
Count = 1,
Name = changeVectorSlice,
IsGlobal = false
});
var indexDef = new TableSchema.SchemaIndexDef
{
StartIndex = 1,
Name = etag,
IsGlobal = true
};
revisionsSchema.DefineIndex(indexDef);
revisionsSchema.Create(tx, "users", 32);
revisionsSchema.Create(tx, "people", 32);
var usersTbl = tx.OpenTable(revisionsSchema, "users");
var peopleTbl = tx.OpenTable(revisionsSchema, "people");
using (usersTbl.Allocate(out var builder))
using (Slice.From(tx.Allocator, Guid.NewGuid().ToString(), out var key))
{
builder.Add(key);
builder.Add(0L);
usersTbl.Insert(builder);
}
for (int i = 0; i < 127; i++)
{
using (peopleTbl.Allocate(out var builder))
using (Slice.From(tx.Allocator, Guid.NewGuid().ToString(), out var key))
{
builder.Add(key);
builder.Add(0L);
peopleTbl.Insert(builder);
}
}
using (peopleTbl.Allocate(out var builder))
using (Slice.From(tx.Allocator, Guid.NewGuid().ToString(), out var key))
{
builder.Add(key);
builder.Add(0L);
peopleTbl.Insert(builder);
}
using (Slice.From(tx.Allocator, new byte[8], out var empty))
{
var userIndex = usersTbl.GetFixedSizeTree(usersTbl.GetTree(indexDef), empty, 0, true);
var peopleIndex = peopleTbl.GetFixedSizeTree(usersTbl.GetTree(indexDef), empty, 0, true);
Assert.Equal(userIndex.NumberOfEntries, peopleIndex.NumberOfEntries);
Assert.Equal(userIndex.Type, peopleIndex.Type);
}
}
}
public void Update_same_value_to_fixed_sized_index_throws()
{
using (var tx = Env.WriteTransaction())
{
Slice.From(tx.Allocator, "EtagIndexName", out var etagIndexName);
var fixedSizedIndex = new TableSchema.FixedSizeSchemaIndexDef
{
Name = etagIndexName,
IsGlobal = true,
StartIndex = 1,
};
var tableSchema = new TableSchema()
.DefineFixedSizeIndex(fixedSizedIndex)
.DefineKey(new TableSchema.SchemaIndexDef
{
StartIndex = 0,
Count = 1,
});
tableSchema.Create(tx, "Items", 16);
var itemsTable = tx.OpenTable(tableSchema, "Items");
const long number1 = 1L;
const long number2 = 2L;
const long number3 = 3L;
using (itemsTable.Allocate(out TableValueBuilder builder))
using (Slice.From(tx.Allocator, "val1", out var key))
{
builder.Add(key);
builder.Add(Bits.SwapBytes(number1));
itemsTable.Set(builder);
}
using (itemsTable.Allocate(out TableValueBuilder builder))
using (Slice.From(tx.Allocator, "val2", out var key))
{
builder.Add(key);
builder.Add(Bits.SwapBytes(number2));
itemsTable.Set(builder);
}
using (itemsTable.Allocate(out TableValueBuilder builder))
using (Slice.From(tx.Allocator, "val1", out var key))
{
builder.Add(key);
builder.Add(Bits.SwapBytes(number3));
itemsTable.Set(builder);
}
using (itemsTable.Allocate(out TableValueBuilder builder))
using (Slice.From(tx.Allocator, "val2", out var key))
{
builder.Add(key);
builder.Add(Bits.SwapBytes(number3));
var exception = Assert.Throws <VoronErrorException>(() => itemsTable.Set(builder));
Assert.True(exception.Message.StartsWith("Attempt to add duplicate value"));
}
}
}
public void Can_force_small_value_to_compress_to_large()
{
var random = new Random(222);
using (var tx = Env.WriteTransaction())
{
Slice.From(tx.Allocator, "Compression", out var etagIndexName);
var fixedSizedIndex = new TableSchema.FixedSizeSchemaIndexDef
{
Name = etagIndexName,
IsGlobal = true,
StartIndex = 1,
};
var tableSchema = new TableSchema()
.CompressValues(fixedSizedIndex, true)
.DefineFixedSizeIndex(fixedSizedIndex)
.DefineKey(new TableSchema.SchemaIndexDef
{
StartIndex = 0,
Count = 1,
});
tableSchema.Create(tx, "Items", 16);
var itemsTable = tx.OpenTable(tableSchema, "Items");
long number = 1L;
var rnd = Enumerable.Range(1, 32)
.Select(i => RandomString(random, 1024))
.ToList();
for (int i = 0; i < 16 * 1024; i++)
{
using (itemsTable.Allocate(out TableValueBuilder builder))
using (Slice.From(tx.Allocator, "val" + i, out var key))
using (Slice.From(tx.Allocator, rnd[i % rnd.Count], out var val))
{
builder.Add(key);
builder.Add(Bits.SwapBytes(++number));
builder.Add(val);
itemsTable.Insert(builder);
}
}
for (int i = 512; i < 768; i++)
{
using (Slice.From(tx.Allocator, "val" + i, out var key))
{
itemsTable.DeleteByKey(key);
}
}
var str = Enumerable.Range(0, 10_000)
.Aggregate(new StringBuilder(), (sb, i) => sb.Append((char)(i % 26 + 'A')))
.ToString();
for (int i = 0; i < 256; i++)
{
using (itemsTable.Allocate(out TableValueBuilder builder))
using (Slice.From(tx.Allocator, "val" + i, out var key))
using (Slice.From(tx.Allocator, str, out var val))
{
builder.Add(key);
builder.Add(Bits.SwapBytes(++number));
builder.Add(val);
itemsTable.Set(builder);
}
}
}
}
public void WillNotRememberOldDictionariesAfterRestart()
{
RequireFileBasedPager();
using var allocator = new ByteStringContext(SharedMultipleUseFlag.None);
using var ____ = Slice.From(allocator, "PK", out var pk);
using var ___ = Slice.From(allocator, "Etags", out var etags);
using var __ = Slice.From(allocator, "Table", out var tbl);
var idx = new TableSchema.FixedSizeSchemaIndexDef {
Name = etags, IsGlobal = false, StartIndex = 0
};
var schema = new TableSchema()
.DefineKey(new TableSchema.SchemaIndexDef
{
Name = pk,
IsGlobal = false,
StartIndex = 0,
Count = 1
})
.DefineFixedSizeIndex(idx)
.CompressValues(idx, true);
using (var wtx = Env.WriteTransaction())
{
schema.Create(wtx, tbl, null);
wtx.Commit();
}
using (var wtx = Env.WriteTransaction())
{
var table = wtx.OpenTable(schema, tbl);
using var _ = Slice.From(allocator, LongRandomString(), out var val);
for (long i = 0; i < 1024 * 4; i++)
{
using (table.Allocate(out var builder))
{
builder.Add(Bits.SwapBytes(i));
builder.Add(val);
table.Insert(builder);
}
}
wtx.Commit();
}
using (var wtx = Env.WriteTransaction())
{
var table = wtx.OpenTable(schema, tbl);
using var _ = Slice.From(allocator, LongRandomString(), out var val);
for (long i = 20_000; i < 20_000 + 1024 * 4; i++)
{
using (table.Allocate(out var builder))
{
builder.Add(Bits.SwapBytes(i));
builder.Add(val);
table.Insert(builder);
}
table.ReadLast(idx); // force to read the new dictionary
}
// explicitly discard the change
//wtx.Commit();
}
using (var wtx = Env.WriteTransaction())
{
var table = wtx.OpenTable(schema, tbl);
using var _ = Slice.From(allocator, LongRandomString(), out var val);
for (long i = 20_000; i < 20_000 + 1020 * 4; i++)
{
using (table.Allocate(out var builder))
{
builder.Add(Bits.SwapBytes(i));
builder.Add(val);
table.Insert(builder);
}
}
wtx.Commit();
}
RestartDatabase();
using (var rtx = Env.ReadTransaction())
{
var table = rtx.OpenTable(schema, tbl);
using (var it = table.SeekForwardFrom(idx, 0, 0).GetEnumerator())
{
while (it.MoveNext())
{
}
}
}
}
public unsafe void CanDeleteTableWithLargeValues()
{
TableSchema schema = new TableSchema();
schema.DefineKey(new TableSchema.SchemaIndexDef
{
StartIndex = 0,
Count = 1,
Name = Etag,
IsGlobal = true
});
schema.DefineIndex(new TableSchema.SchemaIndexDef
{
StartIndex = 0,
Count = 1,
Name = Local,
IsGlobal = false
});
using (Transaction tx = Env.WriteTransaction())
{
schema.Create(tx, "first", 256);
schema.Create(tx, "second", 256);
Table fst = tx.OpenTable(schema, "first");
Table snd = tx.OpenTable(schema, "second");
for (var i = 0; i < 1000;)
{
var bytes = new byte[2 * RawDataSection.MaxItemSize + 100];
new Random(i).NextBytes(bytes);
TableValueBuilder tvb1 = new TableValueBuilder();
TableValueBuilder tvb2 = new TableValueBuilder();
tvb1.Add(i++);
tvb2.Add(i++);
fixed(byte *ptr = bytes)
{
tvb1.Add(ptr, bytes.Length);
tvb2.Add(ptr, bytes.Length);
fst.Insert(tvb1);
snd.Insert(tvb2);
}
}
tx.Commit();
}
using (var tx = Env.WriteTransaction())
{
tx.DeleteTable("first");
tx.DeleteTable("second");
tx.Commit();
}
using (var tx = Env.WriteTransaction())
{
Assert.Null(tx.LowLevelTransaction.RootObjects.Read("first"));
Assert.Null(tx.LowLevelTransaction.RootObjects.Read("second"));
}
}
public void Initialize()
{
_shutdownNotification = new CancellationTokenSource();
AbstractLowMemoryNotification.Initialize(ServerShutdown, Configuration);
if (_logger.IsInfoEnabled)
{
_logger.Info("Starting to open server store for " + (Configuration.Core.RunInMemory ? "<memory>" : Configuration.Core.DataDirectory));
}
var options = Configuration.Core.RunInMemory
? StorageEnvironmentOptions.CreateMemoryOnly(Configuration.Core.DataDirectory)
: StorageEnvironmentOptions.ForPath(System.IO.Path.Combine(Configuration.Core.DataDirectory, "System"));
options.SchemaVersion = 2;
try
{
StorageEnvironment.MaxConcurrentFlushes = Configuration.Storage.MaxConcurrentFlushes;
_env = new StorageEnvironment(options);
using (var tx = _env.WriteTransaction())
{
tx.DeleteTree("items");// note the different casing, we remove the old items tree
_itemsSchema.Create(tx, "Items", 16);
tx.Commit();
}
using (var tx = _env.ReadTransaction())
{
var table = tx.OpenTable(_itemsSchema, "Items");
var itemsFromBackwards = table.SeekBackwardFrom(_itemsSchema.FixedSizeIndexes[EtagIndexName], long.MaxValue);
var reader = itemsFromBackwards.FirstOrDefault();
if (reader == null)
{
_lastEtag = 0;
}
else
{
int size;
_lastEtag = Bits.SwapBytes(*(long *)reader.Read(3, out size));
}
}
}
catch (Exception e)
{
if (_logger.IsOperationsEnabled)
{
_logger.Operations(
"Could not open server store for " + (Configuration.Core.RunInMemory ? "<memory>" : Configuration.Core.DataDirectory), e);
}
options.Dispose();
throw;
}
ContextPool = new TransactionContextPool(_env);
_timer = new Timer(IdleOperations, null, _frequencyToCheckForIdleDatabases, TimeSpan.FromDays(7));
Alerts.Initialize(_env, ContextPool);
DatabaseInfoCache.Initialize(_env, ContextPool);
LicenseStorage.Initialize(_env, ContextPool);
}
public unsafe void OnDataMoveShouldForgetOldCompressionIds()
{
var random = new Random(357);
using (var tx = Env.WriteTransaction())
{
Slice.From(tx.Allocator, "Compression", out var etagIndexName);
var fixedSizedIndex = new TableSchema.FixedSizeSchemaIndexDef
{
Name = etagIndexName,
IsGlobal = true,
StartIndex = 1,
};
var tableSchema = new TableSchema()
.CompressValues(fixedSizedIndex, true)
.DefineFixedSizeIndex(fixedSizedIndex)
.DefineKey(new TableSchema.SchemaIndexDef
{
StartIndex = 0,
Count = 1,
});
tableSchema.Create(tx, "Items", 16);
var itemsTable = tx.OpenTable(tableSchema, "Items");
var rnd = Enumerable.Range(1, 32)
.Select(i => Compression.RandomString(random, 123))
.ToList();
for (int i = 0; i < 32 * 1024; i++)
{
using (itemsTable.Allocate(out TableValueBuilder builder))
using (Slice.From(tx.Allocator, "val" + i, out var key))
using (Slice.From(tx.Allocator, rnd[i % rnd.Count], out var val))
{
builder.Add(key);
builder.Add(Bits.SwapBytes((long)i));
builder.Add(val);
itemsTable.Insert(builder);
}
}
AssertRemainingValues();
for (int i = 32 * 1024 - 1; i >= 0; i--)
{
using (Slice.From(tx.Allocator, "val" + i, out var key))
{
if (i % 10 != 0)
{
itemsTable.DeleteByKey(key);
}
}
}
AssertRemainingValues(validate: true);
void AssertRemainingValues(bool validate = false)
{
foreach (var reader in itemsTable.SeekBackwardFrom(fixedSizedIndex, long.MaxValue))
{
var p = reader.Reader.Read(1, out var sizeOfInt);
Assert.Equal(sizeOfInt, sizeof(long));
var i = Bits.SwapBytes(*(long *)p);
var keyPtr = reader.Reader.Read(0, out var keySize);
using (Slice.From(tx.Allocator, keyPtr, keySize, out var currentKey))
using (Slice.From(tx.Allocator, "val" + i, out var expectedKey))
{
Assert.True(SliceStructComparer.Instance.Equals(currentKey, expectedKey));
}
if (validate)
{
if (i % 10 != 0)
{
Assert.True(false, $"Oops {i}");
}
}
}
}
}
}
public static TableSchema HighLow(this IMslScript script, DataColumn value, TimeGrouping grouping, DataColumn intoHigh, DataColumn intoLow)
{
if (intoHigh == null)
{
TempTable.Create();
intoHigh = TempTable["high"];
}
if (intoLow == null)
{
TempTable.Create();
intoLow = TempTable["low"];
}
if (intoHigh.Table.TableName != intoLow.Table.TableName)
{
throw new Exception("High and low columns need to belong to the same table");
}
var inputTable = Interpreter.Context.TomScripting.GetManager(value.Table.TableName);
var outTable = Interpreter.Context.TomScripting.GetManager(intoHigh.Table.TableName);
var stock = Interpreter.Context.Scope.Stock;
var from = Interpreter.Context.Scope.From;
var to = Interpreter.Context.Scope.To;
using (TransactionScope trans = new TransactionScope())
{
long ownerId = stock.GetId(outTable.Schema.OwnerIdColumn);
var result = outTable.Query(ownerId);
ScopedTable inputData = inputTable.Query(ownerId, new DateClause(from, to), OriginClause.Default);
DateTime?currentPeriod = null;
DateTime?newPeriod = null;
double low = double.MaxValue;
double high = double.MinValue;
// helper Lambda which adds a found result to the result table
Action AddResult = delegate()
{
// handle no min, max could be determined
if (low == double.MaxValue)
{
myLogger.Warning("stock_price_low could not be determined for: TradedStockId = {0}", stock.TradedStock.Id);
low = -1;
}
if (high == double.MinValue)
{
myLogger.Warning("stock_price_high could not be determined for: TradedStockId = {0}", stock.TradedStock.Id);
high = -1;
}
DataRow outRow = result.NewRow();
outRow[result.Schema.OwnerIdColumn] = ownerId;
outRow.SetDate(result.Schema, currentPeriod.Value);
outRow[intoHigh.ColumnName] = high;
outRow[intoLow.ColumnName] = low;
result.AddOrUpdate(outRow, intoHigh.ColumnName, intoLow.ColumnName);
};
foreach (DataRow row in inputData.Rows.OrderBy(row => row[inputTable.Schema.DateColumn]))
{
DateTime date = row.GetDate(inputTable.Schema);
// convert the new period
if (grouping == TimeGrouping.Year)
{
newPeriod = Blade.DateTimeExtensions.FirstOfYear(date.Year);
}
else if (grouping == TimeGrouping.Month)
{
newPeriod = new DateTime(date.Year, date.Month, 1);
}
else
{
throw new InvalidOperationException("Unsupported time period definition: " + grouping);
}
// new period start detected other than the first one?
if (currentPeriod != newPeriod && currentPeriod != null)
{
AddResult();
// reset temp variables
currentPeriod = newPeriod;
high = double.MinValue;
low = double.MaxValue;
}
else if (currentPeriod == null)
{
// handle first period
currentPeriod = newPeriod;
}
double close = (double)row[value.ColumnName];
if (close == -1)
{
myLogger.Warning("Ignoring invalid close price for: TradedStockId = {0}", stock.TradedStock.Id);
continue;
}
low = Math.Min(close, low);
high = Math.Max(close, high);
}
// we left the time scope but is there s.th. left to store?
if (low != double.MaxValue && high != double.MinValue)
{
AddResult();
}
trans.Complete();
}
return(Interpreter.Context.TomScripting.GetManager(intoHigh.Table.TableName).Schema);
}
private unsafe void CreateSchema(DocumentDatabase documentDatabase)
{
_errorsSchema.DefineIndex(new TableSchema.SchemaIndexDef
{
StartIndex = 0,
// there is just a single instance of this table
// but we need it to be local so we'll be able to compact it
IsGlobal = false,
Name = IndexSchema.ErrorTimestampsSlice
});
using (_contextPool.AllocateOperationContext(out TransactionOperationContext context))
using (var tx = context.OpenWriteTransaction())
{
_errorsSchema.Create(tx.InnerTransaction, "Errors", 16);
var typeInt = (int)_index.Type;
var statsTree = tx.InnerTransaction.CreateTree(IndexSchema.StatsTree);
using (Slice.External(context.Allocator, (byte *)&typeInt, sizeof(int), out Slice tmpSlice))
statsTree.Add(IndexSchema.TypeSlice, tmpSlice);
var sourceTypeInt = (int)_index.SourceType;
using (Slice.External(context.Allocator, (byte *)&sourceTypeInt, sizeof(int), out Slice tmpSlice))
statsTree.Add(IndexSchema.SourceTypeSlice, tmpSlice);
if (statsTree.Read(IndexSchema.CreatedTimestampSlice) == null)
{
var binaryDate = SystemTime.UtcNow.ToBinary();
using (Slice.External(context.Allocator, (byte *)&binaryDate, sizeof(long), out Slice tmpSlice))
statsTree.Add(IndexSchema.CreatedTimestampSlice, tmpSlice);
}
using (Slice.From(context.Allocator, documentDatabase.DbBase64Id, out var dbId))
statsTree.Add(IndexSchema.DatabaseIdSlice, dbId);
tx.InnerTransaction.CreateTree(IndexSchema.EtagsTree);
tx.InnerTransaction.CreateTree(IndexSchema.EtagsTombstoneTree);
tx.InnerTransaction.CreateTree(IndexSchema.References);
tx.InnerTransaction.CreateTree(IndexSchema.ReferencesForCompareExchange);
_lastDatabaseEtagOnIndexCreation = InitializeLastDatabaseEtagOnIndexCreation(context);
_index.Definition.Persist(context, _environment.Options);
PersistConfiguration();
tx.Commit();
void PersistConfiguration()
{
var configurationTree = tx.InnerTransaction.CreateTree(IndexSchema.ConfigurationTree);
AssertAndPersistAnalyzer(configurationTree, RavenConfiguration.GetKey(x => x.Indexing.DefaultAnalyzer), _index.Configuration.DefaultAnalyzer, Raven.Client.Constants.Documents.Indexing.Analyzers.Default);
AssertAndPersistAnalyzer(configurationTree, RavenConfiguration.GetKey(x => x.Indexing.DefaultExactAnalyzer), _index.Configuration.DefaultExactAnalyzer, Raven.Client.Constants.Documents.Indexing.Analyzers.DefaultExact);
AssertAndPersistAnalyzer(configurationTree, RavenConfiguration.GetKey(x => x.Indexing.DefaultSearchAnalyzer), _index.Configuration.DefaultSearchAnalyzer, Raven.Client.Constants.Documents.Indexing.Analyzers.DefaultSearch);
}
void AssertAndPersistAnalyzer(Tree configurationTree, string configurationKey, string expectedAnalyzer, string defaultAnalyzer)
{
var result = configurationTree.Read(configurationKey);
string persistedConfigurationValue = null;
if (result != null)
{
persistedConfigurationValue = result.Reader.ToStringValue();
}
else if (_index.Definition.Version < IndexDefinitionBaseServerSide.IndexVersion.Analyzers)
{
persistedConfigurationValue = defaultAnalyzer;
}
if (persistedConfigurationValue != null)
{
if (persistedConfigurationValue != expectedAnalyzer)
{
throw new InvalidOperationException($"Invalid analyzer. The index '{_index.Name}' was created with analyzer '{persistedConfigurationValue}' for '{configurationKey}' configuration, but current one is '{expectedAnalyzer}'. Please reset the index.");
}
return;
}
configurationTree.Add(configurationKey, expectedAnalyzer);
}
}
}
public unsafe void SameTransaction()
{
TableSchema schema = new TableSchema();
schema.DefineIndex(new TableSchema.SchemaIndexDef
{
StartIndex = 1,
Count = 1,
Name = Global,
IsGlobal = true
});
schema.DefineIndex(new TableSchema.SchemaIndexDef
{
StartIndex = 2,
Count = 1,
Name = Local,
IsGlobal = false
});
using (Transaction tx = Env.WriteTransaction())
{
schema.Create(tx, "test", 256);
Table tbl = tx.OpenTable(schema, "test");
schema.Create(tx, "snd", 256);
Table snd = tx.OpenTable(schema, "snd");
for (int i = 0; i < 20_000; i += 2)
{
var tvb = new TableValueBuilder();
tvb.Add(i);
tvb.Add(0);
tvb.Add(i);
tbl.Insert(tvb);
}
for (int i = 1; i < 20_000; i += 2)
{
TableValueBuilder tvb = new TableValueBuilder();
tvb.Add(i);
tvb.Add(0);
tvb.Add(i);
snd.Insert(tvb);
}
tx.Commit();
}
using (var tx = Env.WriteTransaction())
{
Table snd = tx.OpenTable(schema, "snd");
var a = snd.DeleteForwardFrom(schema.Indexes[Local], Slices.BeforeAllKeys, false, long.MaxValue);
Assert.Equal(10_000, a);
}
}
public bool Update(UpdateStep step)
{
ItemsSchema.Create(step.WriteTx, Items, 32);
var itemsTable = step.WriteTx.OpenTable(ItemsSchema, Items);
CertificatesSchema.Create(step.WriteTx, CertificatesSlice, 32);
var certsTable = step.WriteTx.OpenTable(CertificatesSchema, CertificatesSlice);
using (var context = JsonOperationContext.ShortTermSingleUse())
{
// First we'll update all the certs in the local state with the pinning hash
var localCertKeys = GetCertificateKeysFromLocalState(step.ReadTx).ToList();
foreach (var localCertKey in localCertKeys)
{
using (var localCertificate = GetLocalStateByThumbprint(step.ReadTx, context, localCertKey))
{
if (localCertificate == null)
{
continue;
}
using (localCertificate)
{
var def = JsonDeserializationServer.CertificateDefinition(localCertificate);
def.PublicKeyPinningHash = new X509Certificate2(Convert.FromBase64String(def.Certificate)).GetPublicKeyPinningHash();
using (var cert = context.ReadObject(def.ToJson(), "updated/certificate"))
PutLocalState(step.WriteTx, localCertKey, cert);
}
}
}
// Read all the certs from the items table, add the pinning hash and store them in the new table. Then delete the original.
var allClusterCerts = ItemsStartingWith(step.WriteTx, context, step.WriteTx.Allocator, Constants.Certificates.Prefix, 0, int.MaxValue).ToList();
foreach (var cert in allClusterCerts)
{
using (cert.Value)
{
var def = JsonDeserializationServer.CertificateDefinition(cert.Value);
def.PublicKeyPinningHash = new X509Certificate2(Convert.FromBase64String(def.Certificate)).GetPublicKeyPinningHash();
using (Slice.From(step.WriteTx.Allocator, def.PublicKeyPinningHash, out Slice hashSlice))
using (Slice.From(step.WriteTx.Allocator, cert.ItemName, out Slice oldKeySlice)) // includes the 'certificates/' prefix
using (Slice.From(step.WriteTx.Allocator, def.Thumbprint.ToLowerInvariant(), out Slice newKeySlice))
{
// in this update we trim 'certificates/' prefix from key name, CollectionPrimaryKey and CollectionSecondaryKeys
DropCertificatePrefixFromDefinition(def, out _);
using (var newCert = context.ReadObject(def.ToJson(), "certificate/new/schema"))
{
ClusterStateMachine.UpdateCertificate(certsTable, newKeySlice, hashSlice, newCert);
itemsTable.DeleteByKey(oldKeySlice);
}
}
}
}
}
return(true);
}
public unsafe void ShouldNotError()
{
TableSchema schema = new TableSchema();
schema.DefineIndex(new TableSchema.SchemaIndexDef
{
StartIndex = 0,
Count = 1,
Name = Etag,
IsGlobal = true
});
schema.DefineIndex(new TableSchema.SchemaIndexDef
{
StartIndex = 1,
Count = 1,
Name = Global,
IsGlobal = true
});
schema.DefineIndex(new TableSchema.SchemaIndexDef
{
StartIndex = 2,
Count = 1,
Name = Local,
IsGlobal = false
});
using (Transaction tx = Env.WriteTransaction())
{
schema.Create(tx, "test", 256);
Table fst = tx.OpenTable(schema, "test");
schema.Create(tx, "snd", 256);
for (int i = 0; i < 20_000; i++)
{
TableValueBuilder tvb = new TableValueBuilder();
tvb.Add(i);
tvb.Add(0);
tvb.Add(i);
fst.Insert(tvb);
}
tx.Commit();
}
using (Transaction tx = Env.WriteTransaction())
{
Table snd = tx.OpenTable(schema, "snd");
Table fst = tx.OpenTable(schema, "test");
for (int i = 1; i < 20_000; i += 2)
{
using (Slice.From(tx.Allocator, BitConverter.GetBytes(i), out var key))
{
var a = fst.DeleteForwardFrom(schema.Indexes[Etag], key, true, 1);
Assert.True(1 == a, $"{a} on {i}");
}
snd.Insert(new TableValueBuilder
{
i,
0,
i
});
}
tx.Commit();
}
using (var tx = Env.WriteTransaction())
{
Table snd = tx.OpenTable(schema, "snd");
for (int i = 1; i < 20_000; i += 2)
{
using (Slice.From(tx.Allocator, BitConverter.GetBytes(i), out var key))
{
var a = snd.DeleteForwardFrom(schema.Indexes[Etag], key, true, 1);
Assert.True(1 == a, $"{a} on {i}");
}
}
}
}
public static List <Slice> GenerateWornoutTable(
StorageEnvironment env,
Slice tableNameSlice,
TableSchema schema,
int generationTableSize,
int generationBatchSize,
int keyLength,
double generationDeletionProbability,
int?randomSeed
)
{
var tableKeys = new List <Slice>();
var generator = randomSeed.HasValue ? new Random(randomSeed.Value) : new Random();
bool hasTable;
using (var tx = env.ReadTransaction())
{
try
{
tx.OpenTable(schema, tableNameSlice);
hasTable = true;
}
catch (Exception)
{
hasTable = false;
}
tx.Commit();
}
if (hasTable)
{
using (var tx = env.ReadTransaction())
{
var table = tx.OpenTable(schema, tableNameSlice);
foreach (var reader in table.SeekByPrimaryKey(Slices.BeforeAllKeys, 0))
{
Slice key;
schema.Key.GetSlice(Configuration.Allocator, ref reader.Reader, out key);
tableKeys.Add(key);
}
tx.Commit();
}
}
else
{
// Create a table with enough wearing
using (var tx = env.WriteTransaction())
{
var values = new List <Tuple <Slice, Slice> >();
schema.Create(tx, tableNameSlice, 16);
var table = tx.OpenTable(schema, tableNameSlice);
while (table.NumberOfEntries < generationTableSize)
{
int deletions = 0;
// Add BatchSize new keys
for (int i = 0; i < generationBatchSize; i++)
{
// We might run out of values while creating the table, generate more.
if (values.Count == 0)
{
values = GenerateUniqueRandomSlicePairs(
generationTableSize,
keyLength,
randomSeed);
}
var pair = values[0];
values.RemoveAt(0);
// Add it to the table key set
tableKeys.Add(pair.Item1);
// Add it to the table
table.Insert(new TableValueBuilder
{
pair.Item1,
pair.Item2
});
// Simulate a binomial rv in the mean time
if (generator.NextDouble() < generationDeletionProbability)
{
deletions++;
}
}
// Delete the number of deletions given by the binomial rv
// We may have gone a little bit over the limit during
// insertion, but we rebalance here.
if (table.NumberOfEntries > generationTableSize)
{
while (table.NumberOfEntries > generationTableSize)
{
var keyIndex = generator.Next(tableKeys.Count);
// TODO: next two lines will run too slow for big datasets
table.DeleteByKey(tableKeys[keyIndex]);
tableKeys.RemoveAt(keyIndex);
}
}
else
{
while (deletions > 0 && table.NumberOfEntries > 0)
{
var keyIndex = generator.Next(tableKeys.Count);
// TODO: next two lines will run too slow for big datasets
table.DeleteByKey(tableKeys[keyIndex]);
tableKeys.RemoveAt(keyIndex);
deletions--;
}
}
}
tx.Commit();
}
}
return(tableKeys);
}
public bool Update(UpdateStep step)
{
var oldCompareExchangeSchema = new TableSchema().
DefineKey(new TableSchema.SchemaIndexDef
{
StartIndex = (int)ClusterStateMachine.CompareExchangeTable.Key,
Count = 1
});
var newCompareExchangeSchema = new TableSchema()
.DefineKey(new TableSchema.SchemaIndexDef
{
StartIndex = (int)ClusterStateMachine.CompareExchangeTable.Key,
Count = 1
}).DefineIndex(new TableSchema.SchemaIndexDef
{
StartIndex = (int)ClusterStateMachine.CompareExchangeTable.PrefixIndex,
Count = 1,
Name = ClusterStateMachine.CompareExchangeIndex
});
const string oldTableName = "CmpXchg";
using (Slice.From(step.WriteTx.Allocator, oldTableName, out var oldCompareExchangeTable))
{
var oldTable = step.ReadTx.OpenTable(oldCompareExchangeSchema, oldCompareExchangeTable);
if (oldTable == null)
{
return(true);
}
var newTableName = ClusterStateMachine.CompareExchange.ToString();
foreach (var db in SchemaUpgradeExtensions.GetDatabases(step))
{
// update CompareExchange
newCompareExchangeSchema.Create(step.WriteTx, newTableName, null);
var newTable = step.WriteTx.OpenTable(newCompareExchangeSchema, newTableName);
var compareExchangeOldKey = $"{db.ToLowerInvariant()}/";
using (Slice.From(step.ReadTx.Allocator, compareExchangeOldKey, out var keyPrefix))
{
foreach (var item in oldTable.SeekByPrimaryKeyPrefix(keyPrefix, Slices.Empty, 0))
{
var index = DocumentsStorage.TableValueToLong((int)ClusterStateMachine.CompareExchangeTable.Index, ref item.Value.Reader);
using (CompareExchangeCommandBase.GetPrefixIndexSlices(step.ReadTx.Allocator, db, index, out var buffer))
using (Slice.External(step.WriteTx.Allocator, buffer.Ptr, buffer.Length, out var prefixIndexSlice))
using (newTable.Allocate(out TableValueBuilder write))
using (var ctx = JsonOperationContext.ShortTermSingleUse())
{
using (var bjro = new BlittableJsonReaderObject(
item.Value.Reader.Read((int)ClusterStateMachine.CompareExchangeTable.Value, out var size1),
size1, ctx).Clone(ctx)
)
{
write.Add(item.Key);
write.Add(index);
write.Add(bjro.BasePointer, bjro.Size);
write.Add(prefixIndexSlice);
newTable.Set(write);
}
}
}
}
}
}
// delete the old table
step.WriteTx.DeleteTable(oldTableName);
// remove the remaining CompareExchange global index
if (step.WriteTx.LowLevelTransaction.RootObjects.Read(ClusterStateMachine.CompareExchangeIndex) != null)
{
step.WriteTx.DeleteTree(ClusterStateMachine.CompareExchangeIndex);
}
return(true);
}
public unsafe void ShouldPreserveTables(int entries, int seed)
{
// Create random docs to check everything is preserved
using (var allocator = new ByteStringContext(SharedMultipleUseFlag.None))
{
var create = new Dictionary <Slice, long>();
var delete = new List <Slice>();
var r = new Random(seed);
for (var i = 0; i < entries; i++)
{
Slice key;
Slice.From(allocator, "test" + i, out key);
create.Add(key, r.Next());
if (r.NextDouble() < 0.5)
{
delete.Add(key);
}
}
// Create the schema
var schema = new TableSchema()
.DefineKey(new TableSchema.SchemaIndexDef
{
StartIndex = 0,
Count = 1,
IsGlobal = false
});
using (var env = new StorageEnvironment(StorageEnvironmentOptions.ForPath(DataDir)))
{
// Create table in the environment
using (var tx = env.WriteTransaction())
{
schema.Create(tx, "test", 16);
var table = tx.OpenTable(schema, "test");
foreach (var entry in create)
{
var value = entry.Value;
table.Set(new TableValueBuilder
{
entry.Key,
value
});
}
tx.Commit();
}
using (var tx = env.ReadTransaction())
{
var table = tx.OpenTable(schema, "test");
Assert.Equal(table.NumberOfEntries, entries);
}
// Delete some of the entries (this is so that compaction makes sense)
using (var tx = env.WriteTransaction())
{
var table = tx.OpenTable(schema, "test");
foreach (var entry in delete)
{
table.DeleteByKey(entry);
}
tx.Commit();
}
}
var compactedData = Path.Combine(DataDir, "Compacted");
StorageCompaction.Execute(StorageEnvironmentOptions.ForPath(DataDir),
(StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions)
StorageEnvironmentOptions.ForPath(compactedData));
using (var compacted = new StorageEnvironment(StorageEnvironmentOptions.ForPath(compactedData)))
{
using (var tx = compacted.ReadTransaction())
{
var table = tx.OpenTable(schema, "test");
foreach (var entry in create)
{
TableValueReader reader;
var hasValue = table.ReadByKey(entry.Key, out reader);
if (delete.Contains(entry.Key))
{
// This key should not be here
Assert.False(hasValue);
}
else
{
// This key should be there
Assert.True(hasValue);
// Data should be the same
int size;
byte *ptr = reader.Read(0, out size);
Slice current;
using (Slice.External(allocator, ptr, size, out current))
Assert.True(SliceComparer.Equals(current, entry.Key));
ptr = reader.Read(1, out size);
Assert.Equal(entry.Value, *(long *)ptr);
}
}
tx.Commit();
}
}
}
}
public unsafe void Can_get_better_compression_rate_after_training()
{
using (var tx = Env.WriteTransaction())
{
Slice.From(tx.Allocator, "Compression", out var etagIndexName);
var fixedSizedIndex = new TableSchema.FixedSizeSchemaIndexDef
{
Name = etagIndexName,
IsGlobal = true,
StartIndex = 1,
};
var tableSchema = new TableSchema()
.CompressValues(fixedSizedIndex, true)
.DefineFixedSizeIndex(fixedSizedIndex)
.DefineKey(new TableSchema.SchemaIndexDef
{
StartIndex = 0,
Count = 1,
});
tableSchema.Create(tx, "Items", 16);
var itemsTable = tx.OpenTable(tableSchema, "Items");
long number = 1L;
var random = new Random(49941);
var data = string.Join(", ", Enumerable.Range(0, 100).Select(_ =>
{
var bytes = new byte[16];
random.NextBytes(bytes);
return(new Guid(bytes).ToString());
}));
int firstAllocatedSize;
using (itemsTable.Allocate(out TableValueBuilder builder))
using (Slice.From(tx.Allocator, "val", out var key))
using (Slice.From(tx.Allocator, data, out var val))
{
builder.Add(key);
builder.Add(Bits.SwapBytes(number));
builder.Add(val);
long id = itemsTable.Insert(builder);
firstAllocatedSize = itemsTable.GetAllocatedSize(id);
}
var minAllocatedSize = 0;
for (int i = 0; i < 100; i++)
{
using (itemsTable.Allocate(out TableValueBuilder builder))
using (Slice.From(tx.Allocator, "val" + i, out var key))
using (Slice.From(tx.Allocator, data, out var val))
{
builder.Add(key);
builder.Add(Bits.SwapBytes(++number));
builder.Add(val);
long id = itemsTable.Insert(builder);
var allocatedSize = itemsTable.GetAllocatedSize(id);
minAllocatedSize = Math.Min(minAllocatedSize, allocatedSize);
}
}
Assert.True(minAllocatedSize < firstAllocatedSize);
}
}
