public IRollback Commit(PartitionTxData pd)
{
if (!_isStorageInitialized)
{
return(null);
}
// Set respective append offset.
// Some serializers can skip null fields.
var count = pd.NextRowID;
for (int i = 0; i < _columnStorage.OpenFileCount; i++)
{
var file = _columnStorage.GetOpenedFileByID(i);
if (file != null)
{
var column = _metadata.GetColumnByID(file.ColumnID);
var size = StorageSizeUtils.GetRecordSize(column, file.DataType);
if (size > 0)
{
pd.AppendOffset[file.FileID] = count * size;
}
}
}
return(_txSupport.Commit(pd));
}
private SymbolMapColumn CreateSymbolColumn(PartitionTxData tx)
{
int fileID = 0;
_symTestD = RawFileStub.InMemoryFile(4096, fileID++);
_symTestK = RawFileStub.InMemoryFile(4096, fileID++);
_symTestR = RawFileStub.InMemoryFile(4096, fileID++);
_symd = new BufferBinaryReader(new byte[4096], fileID++);
_symi = RawFileStub.InMemoryFile(4096, fileID++);
_symrk = RawFileStub.InMemoryFile(4096, fileID++);
_symrr = RawFileStub.InMemoryFile(4096, fileID);
_symbolCatch = new SymbolCache();
var smc = new SymbolMapColumn(
0,
0,
data: _symTestD.Object,
datak: _symTestK.Object,
datar: _symTestR.Object,
symd: _symd,
symi: _symi.Object,
symk: _symrk.Object,
symr: _symrr.Object,
propertyName: "symTest",
capacity: 24,
recordCountHint: 100,
maxLen: 56);
tx.SymbolData[_symTestK.Object.FileID].KeyBlockOffset = 16;
tx.AppendOffset[_symTestK.Object.FileID] = 28;
tx.SymbolData[_symrk.Object.FileID].KeyBlockOffset = 16;
tx.AppendOffset[_symrk.Object.FileID] = 28;
return(smc);
}
private void CompressPartition(int partitionID, Func <IQueryable <T>, IEnumerable <T> > compressor)
{
using (var txCntx = _partitionManager.ReadTxLog(Metadata.PartitionTtl.Milliseconds))
{
var tx = new Query <T>(this, txCntx);
var date = txCntx.Partitions[partitionID].StartDate;
var version = txCntx.Partitions[partitionID].Version;
var resultSet = tx.Items.ByPartitionId(partitionID);
var tempPartition = _partitionManager.CreateTempPartition(partitionID, date, version);
var newTx = new PartitionTxData(_metadata.ColumnCount, partitionID, new ReadContext());
try
{
foreach (var record in compressor(resultSet))
{
tempPartition.Append(record, newTx);
}
}
catch (Exception ex)
{
Trace.TraceError("Failed to compress partition", ex);
_partitionManager.RemoveTempPartition(tempPartition);
}
_partitionManager.CommitTempPartition(tempPartition, newTx);
}
}
public long ReadKeyRecordOffsetSlow(int key, PartitionTxData tx)
{
var sd = tx.SymbolData[_fileID];
if (!sd.KeyBlockCreated)
{
CopyKeyBlock(tx);
}
long keyBlockOffset = sd.KeyBlockOffset;
CheckKeyBlockOffset(keyBlockOffset);
// -1 used for null values.
long keyOffset = keyBlockOffset + KEY_BLOCK_HEADER_SIZE +
(key + 1) * KEY_RECORD_ENTRY_SIZE;
if (keyOffset >= keyBlockOffset + sd.KeyBlockSize)
{
int newKeyBlockSize = (int)(keyOffset - keyBlockOffset) + KEY_RECORD_ENTRY_SIZE;
int zerosToWrite = newKeyBlockSize - sd.KeyBlockSize;
while (zerosToWrite > 0)
{
int sizeBytes = Math.Min(EMPTY_BUFFER_SIZE, zerosToWrite);
_kData.WriteBytes(keyBlockOffset + sd.KeyBlockSize, EMPTY_BUFFER, 0, sizeBytes);
zerosToWrite -= sizeBytes;
}
sd.KeyBlockSize = newKeyBlockSize;
}
return(keyOffset);
}
public void SetValue(long rowID, byte[] bitArray, PartitionTxData tx)
{
var offset = rowID * _sizeBytes + ISSET_HEADER_LENGTH;
_storage.WriteBytes(offset, bitArray, 0, _sizeBytes);
tx.AppendOffset[_fileID] = offset + _sizeBytes;
}
private ITransactionContext CreateNewTxContext(int fileCount)
{
var tx = new TransactionContext(fileCount);
var pd = new PartitionTxData(fileCount, 1, new ReadContext());
tx.AddPartition(pd, 0);
return(tx);
}
public void Commit(ITransactionContext tx, int partitionTtl)
{
if (Access != EFileAccess.ReadWrite)
{
throw new NFSdbCommitFailedException(
"Journal opened in readonly mode. Transaction commit is not allowed");
}
bool isUpdated = false;
try
{
PartitionTxData lastAppendPartition = tx.GetPartitionTx();
// Non-empty commit.
if (lastAppendPartition != null)
{
var lastPartitionID = lastAppendPartition.PartitionID;
var rowIdFrom = _lastTxRec != null ? _lastTxRec.JournalMaxRowID : 0;
for (int i = tx.Partitions.Count - 1; i >= 0; i--)
{
var partition = tx.Partitions[i];
if (partition != null)
{
if (tx.IsParitionUpdated(partition.PartitionID, _lastTransactionLog))
{
partition.Commit(tx.GetPartitionTx(partition.PartitionID));
tx.RemoveRef(partitionTtl);
isUpdated = true;
}
}
}
if (isUpdated)
{
var lastPartition = GetPartitionByID(tx, lastPartitionID);
var rec = new TxRec();
lastPartition.SetTxRec(tx, rec);
// _symbolTxSupport.SetTxRec(tx, rec);
_txLog.Create(rec);
_lastTxRec = rec;
var onCommited = OnCommited;
if (onCommited != null)
{
onCommited(rowIdFrom, _lastTxRec.JournalMaxRowID);
}
}
tx.SetCommited();
}
}
catch (Exception ex)
{
throw new NFSdbCommitFailedException(
"Error commiting transaction. See InnerException for details.", ex);
}
_lastTransactionLog = tx;
}
public void Write(object item, long rowID, PartitionTxData tx)
{
var readCache = tx.ReadCache;
var bitSetAddress = readCache.AllocateBitsetArray(_bitsetColSize);
var byteArray = new ByteArray(bitSetAddress);
_writeMethod(item, byteArray, _fixedColumns, rowID, _refTypeColumns, tx);
_issetColumn.SetValue(rowID, bitSetAddress, tx);
}
public void SetValue(long rowID, object value, PartitionTxData readContext)
{
if (FieldType == EFieldType.String ||
FieldType == EFieldType.Symbol)
{
SetString(rowID, (string)value, readContext);
return;
}
throw new System.NotImplementedException();
}
public void Append(object item, PartitionTxData pd)
{
if (!_isStorageInitialized)
{
InitializeStorage();
}
_fieldSerializer.Write(item, pd.NextRowID, pd);
pd.NextRowID++;
pd.IsAppended = true;
}
private IndexColumn CreateIndex(int size, PartitionTxData pd)
{
int fileID = 0;
_symrk = RawFileStub.InMemoryFile(size, fileID++);
_symrr = RawFileStub.InMemoryFile(size, fileID);
pd.AppendOffset[_symrk.Object.FileID] = 28;
pd.SymbolData[_symrk.Object.FileID].KeyBlockOffset = 12;
pd.NextRowID = long.MaxValue;
return(new IndexColumn(_symrk.Object, _symrr.Object, 100, 1000));
}
public IRollback Commit(PartitionTxData partitionTxData)
{
var processedFileOffsets = new List <CommitData>(_metadata.FileCount);
var actionRollaback = new CommitRollback(processedFileOffsets);
for (int i = 0; i < _storage.OpenFileCount; i++)
{
var file = _storage.GetOpenedFileByID(i);
if (file == null)
{
continue;
}
try
{
int fileID = file.FileID;
var oldOffset = file.GetAppendOffset();
var appendOffset = partitionTxData.AppendOffset[fileID];
if (file.DataType == EDataType.Symrk || file.DataType == EDataType.Datak)
{
// Key block.
var sd = partitionTxData.SymbolData[fileID];
var keyBlockOffset = sd.KeyBlockOffset;
var keyBlockSize = sd.KeyBlockSize;
var oldBlockOffset = file.ReadInt64(MetadataConstants.K_FILE_KEY_BLOCK_OFFSET);
file.WriteInt64(MetadataConstants.K_FILE_KEY_BLOCK_OFFSET, keyBlockOffset);
file.WriteInt64(keyBlockOffset, keyBlockSize);
// Append offset.
appendOffset = keyBlockOffset + keyBlockSize;
partitionTxData.AppendOffset[fileID] = appendOffset;
file.SetAppendOffset(appendOffset);
processedFileOffsets.Add(new CommitData(file, oldOffset, oldBlockOffset));
}
else
{
// Append offset.
file.SetAppendOffset(appendOffset);
processedFileOffsets.Add(new CommitData(file, oldOffset));
}
}
catch (Exception ex)
{
actionRollaback.Rollback();
throw new NFSdbCommitFailedException(
"SetAppendOffset failed in file {0}", ex, file.Filename);
}
}
return(actionRollaback);
}
public int CheckKeyQuick(string value, PartitionTxData tx)
{
var symbolCache = tx.ReadCache.GetCache(_partitionID, _columnId, _cacheCapacity);
var key = symbolCache.GetRowID(value);
if (key == MetadataConstants.SYMBOL_NOT_FOUND_VALUE)
{
var hashKey = HashKey(value, _capacity);
return(CheckKeySlow(value, hashKey, symbolCache, tx));
}
return(key);
}
public void CommitTempPartition(IPartition partition, PartitionTxData txData)
{
// partition.Commit();
// Close the files
partition.Dispose();
var partitionVersion = new PartitionDate(partition.StartDate, partition.Version,
_metadata.Settings.PartitionType);
var path = Path.Combine(_metadata.Settings.DefaultPath, partitionVersion.Name);
Directory.Move(partition.DirectoryPath, path);
}
public IEnumerable <long> GetValues(int valueKey, PartitionTxData tx)
{
if (_datarIndex == null)
{
throw new NFSdbConfigurationException("Index does not exists for column "
+ _globalSymColumn.PropertyName);
}
if (valueKey == NOT_FOUND_VALUE)
{
return(EMPTY_RESULT);
}
return(_datarIndex.GetValues(valueKey, tx));
}
public long GetCount(int valueKey, PartitionTxData tx)
{
var keyRecOffset = _indexAddress.ReadOnlyKeyRecordOffset(valueKey, tx);
if (keyRecOffset < 0)
{
return(0L);
}
// Key found.
var rowCount = _kData.ReadInt64(keyRecOffset + 8);
return(rowCount);
}
public long ReadKeyRecordOffset(int key, PartitionTxData tx)
{
// Check transaction.
var sd = tx.SymbolData[_fileID];
if (sd.KeyBlockCreated)
{
long keyBlockOffset = sd.KeyBlockOffset;
long keyOffset = keyBlockOffset + KEY_BLOCK_HEADER_SIZE + (key + 1) * KEY_RECORD_ENTRY_SIZE;
if (keyOffset > keyBlockOffset + sd.KeyBlockSize)
{
return(keyOffset);
}
}
return(ReadKeyRecordOffsetSlow(key, tx));
}
private void SetValue0(long rowID, string value, PartitionTxData tx)
{
var symbolCache = tx.ReadCache.GetCache(_partitionID, _columnId, _cacheCapacity);
var key = symbolCache.GetRowID(value);
if (key == NOT_FOUND_VALUE)
{
key = AddKey(value, symbolCache, tx);
}
_data.WriteInt32(rowID * INT32_SIZE, key);
if (_isIndexed)
{
_datarIndex.Add(key, rowID, tx);
}
}
private static PartitionTxData DeepClone(PartitionTxData p)
{
var r = new PartitionTxData(p.AppendOffset.Length, p.PartitionID, p.StartDate, p.EndDate, new ReadContext())
{
LastTimestamp = p.LastTimestamp,
NextRowID = p.NextRowID,
IsPartitionUpdated = p.IsPartitionUpdated
};
for (int i = 0; i < p.AppendOffset.Length; i++)
{
r.AppendOffset[i] = p.AppendOffset[i];
r.SymbolData[i] = p.SymbolData[i].DeepClone();
}
return(r);
}
public void Write(object item, long rowID, PartitionTxData pd)
{
ReadContext readCache = pd.ReadCache;
if (_issetColumn != null)
{
byte[] bitSetAddress = readCache.AllocateBitsetArray(_bitsetColSize);
var byteArray = new ByteArray(bitSetAddress);
_writeMethod(item, byteArray, _fixedColumns, rowID, _stringColumns, pd);
_issetColumn.SetValue(rowID, bitSetAddress, pd);
}
else
{
var byteArray = new ByteArray();
_writeMethod(item, byteArray, _fixedColumns, rowID, _stringColumns, pd);
}
}
public IEnumerable <long> GetValues(int hashKey, PartitionTxData tx)
{
var keyRecOffset = _indexAddress.ReadOnlyKeyRecordOffset(hashKey, tx);
if (keyRecOffset < 0)
{
yield break;
}
var blockOffset = _kData.ReadInt64(keyRecOffset);
var rowCount = _kData.ReadInt64(keyRecOffset + 8);
// Key found.
var rowBlockSize = _indexAddress.RowBlockSize;
var rowBlockCount = (int)(rowCount >> _rowBlockLenBitHint) + 1;
var len = (int)(rowCount & (_rowBlockLen - 1));
if (len == 0)
{
rowBlockCount--;
len = _rowBlockLen;
}
var rowBlockOffset = blockOffset;
var partMaxRowId = tx.NextRowID;
// Loop blocks.
for (int i = rowBlockCount - 1; i >= 0; i--)
{
// Loop cells.
for (int k = len - 1; k >= 0; k--)
{
var rowid = _rData.ReadInt64(rowBlockOffset - rowBlockSize + k * 8);
// yield return rowid;
// Check data visible.
if (partMaxRowId < 0 || rowid < partMaxRowId)
{
yield return(rowid);
}
}
// Next block.
rowBlockOffset = _rData.ReadInt64(rowBlockOffset - 8);
len = _rowBlockLen;
}
}
public long ReadOnlyKeyRecordOffset(int key, PartitionTxData tx)
{
// Check transaction.
var sd = tx.SymbolData[_fileID];
long keyBlockOffset = sd.KeyBlockOffset;
CheckKeyBlockOffset(keyBlockOffset);
// -1 used for null values.
long keyOffset = keyBlockOffset + KEY_BLOCK_HEADER_SIZE +
(key + 1) * KEY_RECORD_ENTRY_SIZE;
if (keyOffset >= keyBlockOffset + KEY_BLOCK_HEADER_SIZE + sd.KeyBlockSize)
{
return(-1);
}
return(keyOffset);
}
public void SetBytes(long rowID, byte[] value, PartitionTxData tx)
{
var offset = rowID * MetadataConstants.STRING_INDEX_FILE_RECORD_SIZE;
if (value != null)
{
var writeOffset = tx.AppendOffset[_data.FileID];
_index.WriteInt64(offset, writeOffset);
var size = value.Length;
WriteLength(writeOffset, size);
_data.WriteBytes(writeOffset + HEADER_SIZE, value, 0, size);
tx.AppendOffset[_data.FileID] = writeOffset + HEADER_SIZE + size;
}
else
{
_index.WriteInt64(offset, MetadataConstants.INDEX_NULL_DATA_VALUE);
}
}
public void AddPartition(PartitionTxData partitionData, int partitionID)
{
if (PartitionTx == null || partitionID >= PartitionTx.Count)
{
var oldParitions = PartitionTx;
PartitionTx = new PartitionTxData[partitionID + 1];
for (int i = 0; i < PartitionTx.Count; i++)
{
if (oldParitions != null && i < oldParitions.Count)
{
PartitionTx[i] = DeepClone(oldParitions[i]);
}
else
{
PartitionTx[i] = partitionData;
}
}
}
}
public unsafe void SetBytes(long rowID, byte *value, int length, PartitionTxData tx)
{
var offset = rowID * MetadataConstants.STRING_INDEX_FILE_RECORD_SIZE;
if (value != null)
{
var writeOffset = tx.AppendOffset[_data.FileID];
_index.WriteInt64(offset, writeOffset);
tx.AppendOffset[_index.FileID] = offset + MetadataConstants.STRING_INDEX_FILE_RECORD_SIZE;
WriteLength(writeOffset, length);
_data.WriteBytes(writeOffset + HEADER_SIZE, value, length);
tx.AppendOffset[_data.FileID] = writeOffset + HEADER_SIZE + length;
}
else
{
_index.WriteInt64(offset, MetadataConstants.INDEX_NULL_DATA_VALUE);
tx.AppendOffset[_index.FileID] = offset + MetadataConstants.STRING_INDEX_FILE_RECORD_SIZE;
}
}
private void Commit(PartitionTxData pd, int rowcount)
{
var sd = pd.SymbolData[_symrk.Object.FileID];
var keyBockOffset = sd.KeyBlockOffset;
var keyBockSize = sd.KeyBlockSize;
var symrAppOff = keyBockOffset + keyBockSize +
MetadataConstants.K_FILE_KEY_BLOCK_HEADER_SIZE;
_symrk.Object.WriteInt64(MetadataConstants.K_FILE_KEY_BLOCK_OFFSET, keyBockOffset);
_symrk.Object.WriteInt64(keyBockOffset, keyBockSize);
_symrk.Object.SetAppendOffset(symrAppOff);
pd.AppendOffset[_symrk.Object.FileID] = symrAppOff;
pd.NextRowID = rowcount;
long symrAppendOffset = pd.AppendOffset[_symrr.Object.FileID];
_symrr.Object.SetAppendOffset(symrAppendOffset);
}
private void CopyKeyBlock(PartitionTxData pd)
{
var sd = pd.SymbolData[_fileID];
var newBlockOffset = pd.AppendOffset[_fileID];
var currentBlockOffset = sd.KeyBlockOffset;
CheckKeyBlockOffset(currentBlockOffset);
var currentBlockLen = sd.KeyBlockSize;
if (currentBlockLen > 0)
{
var keyBlockBuff = pd.ReadCache.AllocateCopyKeyBlockArray(currentBlockLen);
_kData.ReadBytes(currentBlockOffset, keyBlockBuff, 0, currentBlockLen);
_kData.WriteBytes(newBlockOffset, keyBlockBuff, 0, currentBlockLen);
sd.KeyBlockOffset = newBlockOffset;
sd.KeyBlockCreated = true;
}
}
private int AddKey(string value, SymbolCache symbolCache, PartitionTxData tx)
{
var hashKey = HashKey(value, _capacity);
var key = CheckKeySlow(value, hashKey, symbolCache, tx);
if (key == NOT_FOUND_VALUE)
{
var appendOffset = tx.AppendOffset[_symiFileID];
key = (int)(appendOffset / STRING_INDEX_FILE_RECORD_SIZE);
_globalSymColumn.SetString(key, value, tx);
_symrIndex.Add(hashKey, key, tx);
if (value == null)
{
key = MetadataConstants.NULL_SYMBOL_VALUE;
}
symbolCache.SetRowID(value, key);
}
return(key);
}
public unsafe void SetString(long rowID, string value, PartitionTxData tx)
{
if (value != null)
{
#if BIGENDIAN
var byteArray = tx.ReadCache.AllocateByteArray2(value.Length * 2);
fixed(byte *src = byteArray)
{
var charlen = value.Length;
int pos = 0;
fixed(char *chars = value)
{
var strBytes = (byte *)&chars[0];
for (int i = 0; i < charlen; i++)
{
src[pos++] = strBytes[2 * i + 1];
src[pos++] = strBytes[2 * i];
}
}
DebugCheckEquals(pos, charlen * 2);
}
SetBytes(rowID, byteArray, tx);
#else
fixed(char *chars = value)
{
var strBytes = (byte *)chars;
SetBytes(rowID, strBytes, value.Length * 2, tx);
}
#endif
}
else
{
SetBytes(rowID, null, 0, tx);
}
}
private int CheckKeySlow(string value, int hashKey, SymbolCache symbolCache, PartitionTxData tx)
{
var values = _symrIndex.GetValues(hashKey, tx);
foreach (var possibleKey in values)
{
var key = (int)possibleKey;
string possibleValue;
if (!symbolCache.IsValueCached(hashKey, out possibleValue))
{
possibleValue = _globalSymColumn.GetString(key, tx.ReadCache);
symbolCache.SetRowID(possibleValue, key);
if (string.Equals(value, possibleValue, StringComparison.Ordinal))
{
if (value == null)
{
key = MetadataConstants.NULL_SYMBOL_VALUE;
}
return(key);
}
}
}
return(NOT_FOUND_VALUE);
}
