public IRollback Commit(PartitionTxData pd) { if (!_isStorageInitialized) { return(null); } // Set respective append offset. // Some serializers can skip null fields. var count = pd.NextRowID; for (int i = 0; i < _columnStorage.OpenFileCount; i++) { var file = _columnStorage.GetOpenedFileByID(i); if (file != null) { var column = _metadata.GetColumnByID(file.ColumnID); var size = StorageSizeUtils.GetRecordSize(column, file.DataType); if (size > 0) { pd.AppendOffset[file.FileID] = count * size; } } } return(_txSupport.Commit(pd)); }
private SymbolMapColumn CreateSymbolColumn(PartitionTxData tx) { int fileID = 0; _symTestD = RawFileStub.InMemoryFile(4096, fileID++); _symTestK = RawFileStub.InMemoryFile(4096, fileID++); _symTestR = RawFileStub.InMemoryFile(4096, fileID++); _symd = new BufferBinaryReader(new byte[4096], fileID++); _symi = RawFileStub.InMemoryFile(4096, fileID++); _symrk = RawFileStub.InMemoryFile(4096, fileID++); _symrr = RawFileStub.InMemoryFile(4096, fileID); _symbolCatch = new SymbolCache(); var smc = new SymbolMapColumn( 0, 0, data: _symTestD.Object, datak: _symTestK.Object, datar: _symTestR.Object, symd: _symd, symi: _symi.Object, symk: _symrk.Object, symr: _symrr.Object, propertyName: "symTest", capacity: 24, recordCountHint: 100, maxLen: 56); tx.SymbolData[_symTestK.Object.FileID].KeyBlockOffset = 16; tx.AppendOffset[_symTestK.Object.FileID] = 28; tx.SymbolData[_symrk.Object.FileID].KeyBlockOffset = 16; tx.AppendOffset[_symrk.Object.FileID] = 28; return(smc); }
private void CompressPartition(int partitionID, Func <IQueryable <T>, IEnumerable <T> > compressor) { using (var txCntx = _partitionManager.ReadTxLog(Metadata.PartitionTtl.Milliseconds)) { var tx = new Query <T>(this, txCntx); var date = txCntx.Partitions[partitionID].StartDate; var version = txCntx.Partitions[partitionID].Version; var resultSet = tx.Items.ByPartitionId(partitionID); var tempPartition = _partitionManager.CreateTempPartition(partitionID, date, version); var newTx = new PartitionTxData(_metadata.ColumnCount, partitionID, new ReadContext()); try { foreach (var record in compressor(resultSet)) { tempPartition.Append(record, newTx); } } catch (Exception ex) { Trace.TraceError("Failed to compress partition", ex); _partitionManager.RemoveTempPartition(tempPartition); } _partitionManager.CommitTempPartition(tempPartition, newTx); } }
public long ReadKeyRecordOffsetSlow(int key, PartitionTxData tx) { var sd = tx.SymbolData[_fileID]; if (!sd.KeyBlockCreated) { CopyKeyBlock(tx); } long keyBlockOffset = sd.KeyBlockOffset; CheckKeyBlockOffset(keyBlockOffset); // -1 used for null values. long keyOffset = keyBlockOffset + KEY_BLOCK_HEADER_SIZE + (key + 1) * KEY_RECORD_ENTRY_SIZE; if (keyOffset >= keyBlockOffset + sd.KeyBlockSize) { int newKeyBlockSize = (int)(keyOffset - keyBlockOffset) + KEY_RECORD_ENTRY_SIZE; int zerosToWrite = newKeyBlockSize - sd.KeyBlockSize; while (zerosToWrite > 0) { int sizeBytes = Math.Min(EMPTY_BUFFER_SIZE, zerosToWrite); _kData.WriteBytes(keyBlockOffset + sd.KeyBlockSize, EMPTY_BUFFER, 0, sizeBytes); zerosToWrite -= sizeBytes; } sd.KeyBlockSize = newKeyBlockSize; } return(keyOffset); }
public void SetValue(long rowID, byte[] bitArray, PartitionTxData tx) { var offset = rowID * _sizeBytes + ISSET_HEADER_LENGTH; _storage.WriteBytes(offset, bitArray, 0, _sizeBytes); tx.AppendOffset[_fileID] = offset + _sizeBytes; }
private ITransactionContext CreateNewTxContext(int fileCount) { var tx = new TransactionContext(fileCount); var pd = new PartitionTxData(fileCount, 1, new ReadContext()); tx.AddPartition(pd, 0); return(tx); }
public void Commit(ITransactionContext tx, int partitionTtl) { if (Access != EFileAccess.ReadWrite) { throw new NFSdbCommitFailedException( "Journal opened in readonly mode. Transaction commit is not allowed"); } bool isUpdated = false; try { PartitionTxData lastAppendPartition = tx.GetPartitionTx(); // Non-empty commit. if (lastAppendPartition != null) { var lastPartitionID = lastAppendPartition.PartitionID; var rowIdFrom = _lastTxRec != null ? _lastTxRec.JournalMaxRowID : 0; for (int i = tx.Partitions.Count - 1; i >= 0; i--) { var partition = tx.Partitions[i]; if (partition != null) { if (tx.IsParitionUpdated(partition.PartitionID, _lastTransactionLog)) { partition.Commit(tx.GetPartitionTx(partition.PartitionID)); tx.RemoveRef(partitionTtl); isUpdated = true; } } } if (isUpdated) { var lastPartition = GetPartitionByID(tx, lastPartitionID); var rec = new TxRec(); lastPartition.SetTxRec(tx, rec); // _symbolTxSupport.SetTxRec(tx, rec); _txLog.Create(rec); _lastTxRec = rec; var onCommited = OnCommited; if (onCommited != null) { onCommited(rowIdFrom, _lastTxRec.JournalMaxRowID); } } tx.SetCommited(); } } catch (Exception ex) { throw new NFSdbCommitFailedException( "Error commiting transaction. See InnerException for details.", ex); } _lastTransactionLog = tx; }
public void Write(object item, long rowID, PartitionTxData tx) { var readCache = tx.ReadCache; var bitSetAddress = readCache.AllocateBitsetArray(_bitsetColSize); var byteArray = new ByteArray(bitSetAddress); _writeMethod(item, byteArray, _fixedColumns, rowID, _refTypeColumns, tx); _issetColumn.SetValue(rowID, bitSetAddress, tx); }
public void SetValue(long rowID, object value, PartitionTxData readContext) { if (FieldType == EFieldType.String || FieldType == EFieldType.Symbol) { SetString(rowID, (string)value, readContext); return; } throw new System.NotImplementedException(); }
public void Append(object item, PartitionTxData pd) { if (!_isStorageInitialized) { InitializeStorage(); } _fieldSerializer.Write(item, pd.NextRowID, pd); pd.NextRowID++; pd.IsAppended = true; }
private IndexColumn CreateIndex(int size, PartitionTxData pd) { int fileID = 0; _symrk = RawFileStub.InMemoryFile(size, fileID++); _symrr = RawFileStub.InMemoryFile(size, fileID); pd.AppendOffset[_symrk.Object.FileID] = 28; pd.SymbolData[_symrk.Object.FileID].KeyBlockOffset = 12; pd.NextRowID = long.MaxValue; return(new IndexColumn(_symrk.Object, _symrr.Object, 100, 1000)); }
public IRollback Commit(PartitionTxData partitionTxData) { var processedFileOffsets = new List <CommitData>(_metadata.FileCount); var actionRollaback = new CommitRollback(processedFileOffsets); for (int i = 0; i < _storage.OpenFileCount; i++) { var file = _storage.GetOpenedFileByID(i); if (file == null) { continue; } try { int fileID = file.FileID; var oldOffset = file.GetAppendOffset(); var appendOffset = partitionTxData.AppendOffset[fileID]; if (file.DataType == EDataType.Symrk || file.DataType == EDataType.Datak) { // Key block. var sd = partitionTxData.SymbolData[fileID]; var keyBlockOffset = sd.KeyBlockOffset; var keyBlockSize = sd.KeyBlockSize; var oldBlockOffset = file.ReadInt64(MetadataConstants.K_FILE_KEY_BLOCK_OFFSET); file.WriteInt64(MetadataConstants.K_FILE_KEY_BLOCK_OFFSET, keyBlockOffset); file.WriteInt64(keyBlockOffset, keyBlockSize); // Append offset. appendOffset = keyBlockOffset + keyBlockSize; partitionTxData.AppendOffset[fileID] = appendOffset; file.SetAppendOffset(appendOffset); processedFileOffsets.Add(new CommitData(file, oldOffset, oldBlockOffset)); } else { // Append offset. file.SetAppendOffset(appendOffset); processedFileOffsets.Add(new CommitData(file, oldOffset)); } } catch (Exception ex) { actionRollaback.Rollback(); throw new NFSdbCommitFailedException( "SetAppendOffset failed in file {0}", ex, file.Filename); } } return(actionRollaback); }
public int CheckKeyQuick(string value, PartitionTxData tx) { var symbolCache = tx.ReadCache.GetCache(_partitionID, _columnId, _cacheCapacity); var key = symbolCache.GetRowID(value); if (key == MetadataConstants.SYMBOL_NOT_FOUND_VALUE) { var hashKey = HashKey(value, _capacity); return(CheckKeySlow(value, hashKey, symbolCache, tx)); } return(key); }
public void CommitTempPartition(IPartition partition, PartitionTxData txData) { // partition.Commit(); // Close the files partition.Dispose(); var partitionVersion = new PartitionDate(partition.StartDate, partition.Version, _metadata.Settings.PartitionType); var path = Path.Combine(_metadata.Settings.DefaultPath, partitionVersion.Name); Directory.Move(partition.DirectoryPath, path); }
public IEnumerable <long> GetValues(int valueKey, PartitionTxData tx) { if (_datarIndex == null) { throw new NFSdbConfigurationException("Index does not exists for column " + _globalSymColumn.PropertyName); } if (valueKey == NOT_FOUND_VALUE) { return(EMPTY_RESULT); } return(_datarIndex.GetValues(valueKey, tx)); }
public long GetCount(int valueKey, PartitionTxData tx) { var keyRecOffset = _indexAddress.ReadOnlyKeyRecordOffset(valueKey, tx); if (keyRecOffset < 0) { return(0L); } // Key found. var rowCount = _kData.ReadInt64(keyRecOffset + 8); return(rowCount); }
public long ReadKeyRecordOffset(int key, PartitionTxData tx) { // Check transaction. var sd = tx.SymbolData[_fileID]; if (sd.KeyBlockCreated) { long keyBlockOffset = sd.KeyBlockOffset; long keyOffset = keyBlockOffset + KEY_BLOCK_HEADER_SIZE + (key + 1) * KEY_RECORD_ENTRY_SIZE; if (keyOffset > keyBlockOffset + sd.KeyBlockSize) { return(keyOffset); } } return(ReadKeyRecordOffsetSlow(key, tx)); }
private void SetValue0(long rowID, string value, PartitionTxData tx) { var symbolCache = tx.ReadCache.GetCache(_partitionID, _columnId, _cacheCapacity); var key = symbolCache.GetRowID(value); if (key == NOT_FOUND_VALUE) { key = AddKey(value, symbolCache, tx); } _data.WriteInt32(rowID * INT32_SIZE, key); if (_isIndexed) { _datarIndex.Add(key, rowID, tx); } }
private static PartitionTxData DeepClone(PartitionTxData p) { var r = new PartitionTxData(p.AppendOffset.Length, p.PartitionID, p.StartDate, p.EndDate, new ReadContext()) { LastTimestamp = p.LastTimestamp, NextRowID = p.NextRowID, IsPartitionUpdated = p.IsPartitionUpdated }; for (int i = 0; i < p.AppendOffset.Length; i++) { r.AppendOffset[i] = p.AppendOffset[i]; r.SymbolData[i] = p.SymbolData[i].DeepClone(); } return(r); }
public void Write(object item, long rowID, PartitionTxData pd) { ReadContext readCache = pd.ReadCache; if (_issetColumn != null) { byte[] bitSetAddress = readCache.AllocateBitsetArray(_bitsetColSize); var byteArray = new ByteArray(bitSetAddress); _writeMethod(item, byteArray, _fixedColumns, rowID, _stringColumns, pd); _issetColumn.SetValue(rowID, bitSetAddress, pd); } else { var byteArray = new ByteArray(); _writeMethod(item, byteArray, _fixedColumns, rowID, _stringColumns, pd); } }
public IEnumerable <long> GetValues(int hashKey, PartitionTxData tx) { var keyRecOffset = _indexAddress.ReadOnlyKeyRecordOffset(hashKey, tx); if (keyRecOffset < 0) { yield break; } var blockOffset = _kData.ReadInt64(keyRecOffset); var rowCount = _kData.ReadInt64(keyRecOffset + 8); // Key found. var rowBlockSize = _indexAddress.RowBlockSize; var rowBlockCount = (int)(rowCount >> _rowBlockLenBitHint) + 1; var len = (int)(rowCount & (_rowBlockLen - 1)); if (len == 0) { rowBlockCount--; len = _rowBlockLen; } var rowBlockOffset = blockOffset; var partMaxRowId = tx.NextRowID; // Loop blocks. for (int i = rowBlockCount - 1; i >= 0; i--) { // Loop cells. for (int k = len - 1; k >= 0; k--) { var rowid = _rData.ReadInt64(rowBlockOffset - rowBlockSize + k * 8); // yield return rowid; // Check data visible. if (partMaxRowId < 0 || rowid < partMaxRowId) { yield return(rowid); } } // Next block. rowBlockOffset = _rData.ReadInt64(rowBlockOffset - 8); len = _rowBlockLen; } }
public long ReadOnlyKeyRecordOffset(int key, PartitionTxData tx) { // Check transaction. var sd = tx.SymbolData[_fileID]; long keyBlockOffset = sd.KeyBlockOffset; CheckKeyBlockOffset(keyBlockOffset); // -1 used for null values. long keyOffset = keyBlockOffset + KEY_BLOCK_HEADER_SIZE + (key + 1) * KEY_RECORD_ENTRY_SIZE; if (keyOffset >= keyBlockOffset + KEY_BLOCK_HEADER_SIZE + sd.KeyBlockSize) { return(-1); } return(keyOffset); }
public void SetBytes(long rowID, byte[] value, PartitionTxData tx) { var offset = rowID * MetadataConstants.STRING_INDEX_FILE_RECORD_SIZE; if (value != null) { var writeOffset = tx.AppendOffset[_data.FileID]; _index.WriteInt64(offset, writeOffset); var size = value.Length; WriteLength(writeOffset, size); _data.WriteBytes(writeOffset + HEADER_SIZE, value, 0, size); tx.AppendOffset[_data.FileID] = writeOffset + HEADER_SIZE + size; } else { _index.WriteInt64(offset, MetadataConstants.INDEX_NULL_DATA_VALUE); } }
public void AddPartition(PartitionTxData partitionData, int partitionID) { if (PartitionTx == null || partitionID >= PartitionTx.Count) { var oldParitions = PartitionTx; PartitionTx = new PartitionTxData[partitionID + 1]; for (int i = 0; i < PartitionTx.Count; i++) { if (oldParitions != null && i < oldParitions.Count) { PartitionTx[i] = DeepClone(oldParitions[i]); } else { PartitionTx[i] = partitionData; } } } }
public unsafe void SetBytes(long rowID, byte *value, int length, PartitionTxData tx) { var offset = rowID * MetadataConstants.STRING_INDEX_FILE_RECORD_SIZE; if (value != null) { var writeOffset = tx.AppendOffset[_data.FileID]; _index.WriteInt64(offset, writeOffset); tx.AppendOffset[_index.FileID] = offset + MetadataConstants.STRING_INDEX_FILE_RECORD_SIZE; WriteLength(writeOffset, length); _data.WriteBytes(writeOffset + HEADER_SIZE, value, length); tx.AppendOffset[_data.FileID] = writeOffset + HEADER_SIZE + length; } else { _index.WriteInt64(offset, MetadataConstants.INDEX_NULL_DATA_VALUE); tx.AppendOffset[_index.FileID] = offset + MetadataConstants.STRING_INDEX_FILE_RECORD_SIZE; } }
private void Commit(PartitionTxData pd, int rowcount) { var sd = pd.SymbolData[_symrk.Object.FileID]; var keyBockOffset = sd.KeyBlockOffset; var keyBockSize = sd.KeyBlockSize; var symrAppOff = keyBockOffset + keyBockSize + MetadataConstants.K_FILE_KEY_BLOCK_HEADER_SIZE; _symrk.Object.WriteInt64(MetadataConstants.K_FILE_KEY_BLOCK_OFFSET, keyBockOffset); _symrk.Object.WriteInt64(keyBockOffset, keyBockSize); _symrk.Object.SetAppendOffset(symrAppOff); pd.AppendOffset[_symrk.Object.FileID] = symrAppOff; pd.NextRowID = rowcount; long symrAppendOffset = pd.AppendOffset[_symrr.Object.FileID]; _symrr.Object.SetAppendOffset(symrAppendOffset); }
private void CopyKeyBlock(PartitionTxData pd) { var sd = pd.SymbolData[_fileID]; var newBlockOffset = pd.AppendOffset[_fileID]; var currentBlockOffset = sd.KeyBlockOffset; CheckKeyBlockOffset(currentBlockOffset); var currentBlockLen = sd.KeyBlockSize; if (currentBlockLen > 0) { var keyBlockBuff = pd.ReadCache.AllocateCopyKeyBlockArray(currentBlockLen); _kData.ReadBytes(currentBlockOffset, keyBlockBuff, 0, currentBlockLen); _kData.WriteBytes(newBlockOffset, keyBlockBuff, 0, currentBlockLen); sd.KeyBlockOffset = newBlockOffset; sd.KeyBlockCreated = true; } }
private int AddKey(string value, SymbolCache symbolCache, PartitionTxData tx) { var hashKey = HashKey(value, _capacity); var key = CheckKeySlow(value, hashKey, symbolCache, tx); if (key == NOT_FOUND_VALUE) { var appendOffset = tx.AppendOffset[_symiFileID]; key = (int)(appendOffset / STRING_INDEX_FILE_RECORD_SIZE); _globalSymColumn.SetString(key, value, tx); _symrIndex.Add(hashKey, key, tx); if (value == null) { key = MetadataConstants.NULL_SYMBOL_VALUE; } symbolCache.SetRowID(value, key); } return(key); }
public unsafe void SetString(long rowID, string value, PartitionTxData tx) { if (value != null) { #if BIGENDIAN var byteArray = tx.ReadCache.AllocateByteArray2(value.Length * 2); fixed(byte *src = byteArray) { var charlen = value.Length; int pos = 0; fixed(char *chars = value) { var strBytes = (byte *)&chars[0]; for (int i = 0; i < charlen; i++) { src[pos++] = strBytes[2 * i + 1]; src[pos++] = strBytes[2 * i]; } } DebugCheckEquals(pos, charlen * 2); } SetBytes(rowID, byteArray, tx); #else fixed(char *chars = value) { var strBytes = (byte *)chars; SetBytes(rowID, strBytes, value.Length * 2, tx); } #endif } else { SetBytes(rowID, null, 0, tx); } }
private int CheckKeySlow(string value, int hashKey, SymbolCache symbolCache, PartitionTxData tx) { var values = _symrIndex.GetValues(hashKey, tx); foreach (var possibleKey in values) { var key = (int)possibleKey; string possibleValue; if (!symbolCache.IsValueCached(hashKey, out possibleValue)) { possibleValue = _globalSymColumn.GetString(key, tx.ReadCache); symbolCache.SetRowID(possibleValue, key); if (string.Equals(value, possibleValue, StringComparison.Ordinal)) { if (value == null) { key = MetadataConstants.NULL_SYMBOL_VALUE; } return(key); } } } return(NOT_FOUND_VALUE); }