public ITransactionContext ReadTxLog(int partitionTtlMs) { // _tx file. if (!_directoryChecked) { CheckCreateDirectory(); } if (_lastTxRec == null || Access != EFileAccess.ReadWrite) { _lastTxRec = _txLog.Get(); } var state = GetTxState(); ReconcilePartitionsWithTxRec(state.Partitions, _lastTxRec); var tx = new DeferredTransactionContext(state, this, _lastTxRec, partitionTtlMs); if (_lastTxRec != null) { tx.PrevTxAddress = Math.Max(_lastTxRec.PrevTxAddress, TxLog.MIN_TX_ADDRESS) + _lastTxRec.Size(); } // Set state to inital. _lastTransactionLog = tx; tx.AddRefsAllPartitions(); return(tx); }
public PartitionTxData ReadTxLogFromPartition(ReadContext readCache, TxRec txRec = null) { if (txRec == null) { return(ReadTxLogFromFile(readCache)); } return(ReadTxLogFromFileAndTxRec(readCache, txRec)); }
public void Commit(ITransactionContext tx, int partitionTtl) { if (Access != EFileAccess.ReadWrite) { throw new NFSdbCommitFailedException( "Journal opened in readonly mode. Transaction commit is not allowed"); } bool isUpdated = false; try { PartitionTxData lastAppendPartition = tx.GetPartitionTx(); // Non-empty commit. if (lastAppendPartition != null) { var lastPartitionID = lastAppendPartition.PartitionID; var rowIdFrom = _lastTxRec != null ? _lastTxRec.JournalMaxRowID : 0; for (int i = tx.Partitions.Count - 1; i >= 0; i--) { var partition = tx.Partitions[i]; if (partition != null) { if (tx.IsParitionUpdated(partition.PartitionID, _lastTransactionLog)) { partition.Commit(tx.GetPartitionTx(partition.PartitionID)); tx.RemoveRef(partitionTtl); isUpdated = true; } } } if (isUpdated) { var lastPartition = GetPartitionByID(tx, lastPartitionID); var rec = new TxRec(); lastPartition.SetTxRec(tx, rec); // _symbolTxSupport.SetTxRec(tx, rec); _txLog.Create(rec); _lastTxRec = rec; var onCommited = OnCommited; if (onCommited != null) { onCommited(rowIdFrom, _lastTxRec.JournalMaxRowID); } } tx.SetCommited(); } } catch (Exception ex) { throw new NFSdbCommitFailedException( "Error commiting transaction. See InnerException for details.", ex); } _lastTransactionLog = tx; }
public PartitionTxData ReadTxLogFromPartition(ReadContext readCache, TxRec txRec = null) { if (!_isStorageInitialized) { InitializeStorage(); } return(_txSupport.ReadTxLogFromPartition(readCache, txRec)); }
public void Should_save_LagName(string name) { var txLog = CreateTxLog(); var rec = new TxRec { LagName = name }; txLog.Create(rec); var rec2 = txLog.Get(); Assert.That(rec2.LagName, Is.EqualTo(name)); }
public void Should_save_LagSize(long lagSize) { var txLog = CreateTxLog(); var rec = new TxRec { LagSize = lagSize }; txLog.Create(rec); var rec2 = txLog.Get(); Assert.That(rec2.LagSize, Is.EqualTo(lagSize)); }
public void Should_save_last_timestamp(long timestamp) { var txLog = CreateTxLog(); var rec = new TxRec { LastPartitionTimestamp = timestamp }; txLog.Create(rec); var rec2 = txLog.Get(); Assert.That(rec2.LastPartitionTimestamp, Is.EqualTo(timestamp)); }
public void Should_save_max_row_id(long maxRec) { var txLog = CreateTxLog(); var rec = new TxRec { JournalMaxRowID = maxRec }; txLog.Create(rec); var rec2 = txLog.Get(); Assert.That(rec2.JournalMaxRowID, Is.EqualTo(maxRec)); }
public void SetTxRec(ITransactionContext tx, TxRec rec) { rec.PrevTxAddress = tx.PrevTxAddress; rec.Command = TxRec.TX_NORMAL; var pd = tx.GetPartitionTx(_partitionID); var columCount = _metadata.Columns.Count(); rec.LastPartitionTimestamp = DateUtils.DateTimeToUnixTimeStamp(tx.LastAppendTimestamp); // Java NFSdb Journal has paritions from 0 with no reserved id for symbol parition. // Max row ID is rowcount + 1 for compatibility rec.JournalMaxRowID = RowIDUtil.ToRowID(_partitionID - 1, pd.NextRowID - 1) + 1; rec.IndexPointers = new long[columCount]; var symbolTableSize = new List <int>(); var symbolTableIndexPointers = new List <long>(); for (int i = 0; i < _storage.OpenFileCount; i++) { var f = _storage.GetOpenedFileByID(i); if (f != null && f.DataType == EDataType.Datak) { rec.IndexPointers[f.ColumnID] = pd.SymbolData[f.FileID].KeyBlockOffset; } else { var file = _storage.GetOpenedFileByID(i); if (file == null) { continue; } if (file.DataType == EDataType.Symi) { var indexSize = (int)(pd.AppendOffset[file.FileID] / 8); symbolTableSize.Add(indexSize); } else if (file.DataType == EDataType.Symrk) { var sd = pd.SymbolData[file.FileID]; symbolTableIndexPointers.Add(sd.KeyBlockOffset); } } } rec.SymbolTableIndexPointers = symbolTableIndexPointers.ToArray(); rec.SymbolTableSizes = symbolTableSize.ToArray(); }
public void Should_save_LagIndexPointers(string tableSizes) { var txLog = CreateTxLog(); var symbols = string.IsNullOrEmpty(tableSizes) ? null : tableSizes.Split(',').Select(long.Parse).ToArray(); var rec = new TxRec { LagIndexPointers = symbols }; txLog.Create(rec); var rec2 = txLog.Get(); var result = string.Join(",", rec2.LagIndexPointers); Assert.That(result, Is.EqualTo(tableSizes)); }
public void Should_write_tx_rec_last_partition_timestamp() { const int partitionID = 1; const int timestamp = 20200119; var ftx = CreateFileTxSupport(CreateFileMocks("i.d-8|s.d-8"), partitionID); var tx = new Mock <ITransactionContext>(); tx.Setup(t => t.LastAppendTimestamp).Returns(DateUtils.UnixTimestampToDateTime(timestamp)); tx.Setup(t => t.GetPartitionTx(partitionID)).Returns(new PartitionTxData(2, partitionID, new ReadContext())); var txRec = new TxRec(); // Act. ftx.SetTxRec(tx.Object, txRec); // Verify. Assert.That(txRec.LastPartitionTimestamp, Is.EqualTo(timestamp)); }
public void Should_write_tx_rec_journal_max_row_id() { const int partitionID = 1; const int localRowID = 1001; var ftx = CreateFileTxSupport(CreateFileMocks("i.d-8|s.d-8"), partitionID); var tx = new Mock <ITransactionContext>(); tx.Setup(t => t.GetPartitionTx(partitionID)).Returns(new PartitionTxData(2, partitionID, new ReadContext()) { NextRowID = localRowID }); var txRec = new TxRec(); // Act. ftx.SetTxRec(tx.Object, txRec); // Verify. var expected = RowIDUtil.ToRowID(partitionID - 1, localRowID - 1) + 1; Assert.That(txRec.JournalMaxRowID, Is.EqualTo(expected)); }
public void ClearTxLog() { _lastTransactionLog = null; _lastTxRec = null; _txLog.Clean(); }
private void ReconcilePartitionsWithTxRec(List <IPartition> partitions, TxRec txRec) { partitions.Clear(); var defaultPath = _settings.DefaultPath; // Symbols are the partition 0. partitions.Add(null); var nextPartitionID = 1; var subDirs = LatestPartitionVersions(defaultPath); foreach (var subDir in subDirs) { var fullPath = Path.Combine(defaultPath, subDir.Name); var startDate = subDir.Date; var partitionDir = subDir; PartitionConfig config = PartitionManagerUtils.ReadPartitionConfig(fullPath); if (config != null) { nextPartitionID = config.PartitionID; } if (txRec.IsCommited(startDate, nextPartitionID)) { var partitionID = nextPartitionID; var partition = _partitions.AddOrUpdate(startDate, // Add. sd => { var p = new Lazy <IPartition>(() => CreateNewParition(partitionDir, partitionID, fullPath, config)); if (config == null && Access == EFileAccess.ReadWrite) { p.Value.SaveConfig(); } return(p); }, // Update. (sd, existing) => { if (existing.Value.Version == subDir.Version) { return(existing); } existing.Value.MarkOverwritten(); var p = new Lazy <IPartition>(() => CreateNewParition(partitionDir, partitionID, fullPath, config)); if (config == null && Access == EFileAccess.ReadWrite) { p.Value.SaveConfig(); } return(p); }); partitions.SetToIndex(partitionID, partition.Value); nextPartitionID++; } else { Trace.TraceInformation( "Ignoring directory '{0}' for partition type '{1}' as fully rolled back partition.", fullPath, _settings.PartitionType); Lazy <IPartition> existingPartition; if (_partitions.TryRemove(startDate, out existingPartition)) { if (existingPartition.IsValueCreated) { existingPartition.Value.Dispose(); } } } } }
private PartitionTxData ReadTxLogFromFileAndTxRec(ReadContext readCache, TxRec txRec) { int symrRead = 0; var pd = new PartitionTxData(_metadata.FileCount, _partitionID, _startDate, _endTime, readCache); long nextRowID = RowIDUtil.ToLocalRowID(txRec.JournalMaxRowID - 1) + 1; pd.NextRowID = nextRowID; for (int i = 0; i < _storage.OpenFileCount; i++) { var file = _storage.GetOpenedFileByID(i); if (file == null) { continue; } try { if (file.DataType == EDataType.Symrk && txRec.SymbolTableIndexPointers != null) { var blockOffset = txRec.SymbolTableIndexPointers[symrRead++]; pd.SymbolData[file.FileID].KeyBlockOffset = blockOffset; long keyBlockSize = file.ReadInt64(blockOffset + MetadataConstants.K_FILE_ROW_BLOCK_LEN_OFFSET); pd.SymbolData[file.FileID].KeyBlockSize = (int)keyBlockSize; pd.AppendOffset[file.FileID] = blockOffset + keyBlockSize; } else if (file.DataType == EDataType.Datak && txRec.IndexPointers != null) { var blockOffset = txRec.IndexPointers[file.ColumnID]; pd.SymbolData[file.FileID].KeyBlockOffset = blockOffset; long keyBlockSize = file.ReadInt64(blockOffset + MetadataConstants.K_FILE_ROW_BLOCK_LEN_OFFSET); pd.SymbolData[file.FileID].KeyBlockSize = (int)keyBlockSize; pd.AppendOffset[file.FileID] = blockOffset + keyBlockSize; } else { var column = _metadata.GetColumnByID(file.ColumnID); var size = StorageSizeUtils.GetRecordSize(column, file.DataType); if (size > 0) { // Fixed column. pd.AppendOffset[file.FileID] = nextRowID * size; } else { // Variable column. pd.AppendOffset[file.FileID] = file.GetAppendOffset(); } if (_metadata.TimestampColumnID == column.ColumnID) { var timestamp = file.ReadInt64(pd.AppendOffset[file.FileID] - TIMESTAMP_DATA_SIZE); pd.LastTimestamp = timestamp; } } } catch (Exception ex) { if (ex is NFSdbPartitionException) { throw; } throw new NFSdbTransactionStateExcepton( string.Format("Error reading transaction state from file {0}", file.Filename), ex); } } pd.NextRowID = nextRowID; return(pd); }
public void SetTxRec(ITransactionContext tx, TxRec rec) { _txSupport.SetTxRec(tx, rec); }