public ITransactionContext ReadTxLog(int partitionTtlMs)
{
// _tx file.
if (!_directoryChecked)
{
CheckCreateDirectory();
}
if (_lastTxRec == null || Access != EFileAccess.ReadWrite)
{
_lastTxRec = _txLog.Get();
}
var state = GetTxState();
ReconcilePartitionsWithTxRec(state.Partitions, _lastTxRec);
var tx = new DeferredTransactionContext(state, this, _lastTxRec, partitionTtlMs);
if (_lastTxRec != null)
{
tx.PrevTxAddress = Math.Max(_lastTxRec.PrevTxAddress, TxLog.MIN_TX_ADDRESS) + _lastTxRec.Size();
}
// Set state to inital.
_lastTransactionLog = tx;
tx.AddRefsAllPartitions();
return(tx);
}
public PartitionTxData ReadTxLogFromPartition(ReadContext readCache, TxRec txRec = null)
{
if (txRec == null)
{
return(ReadTxLogFromFile(readCache));
}
return(ReadTxLogFromFileAndTxRec(readCache, txRec));
}
public void Commit(ITransactionContext tx, int partitionTtl)
{
if (Access != EFileAccess.ReadWrite)
{
throw new NFSdbCommitFailedException(
"Journal opened in readonly mode. Transaction commit is not allowed");
}
bool isUpdated = false;
try
{
PartitionTxData lastAppendPartition = tx.GetPartitionTx();
// Non-empty commit.
if (lastAppendPartition != null)
{
var lastPartitionID = lastAppendPartition.PartitionID;
var rowIdFrom = _lastTxRec != null ? _lastTxRec.JournalMaxRowID : 0;
for (int i = tx.Partitions.Count - 1; i >= 0; i--)
{
var partition = tx.Partitions[i];
if (partition != null)
{
if (tx.IsParitionUpdated(partition.PartitionID, _lastTransactionLog))
{
partition.Commit(tx.GetPartitionTx(partition.PartitionID));
tx.RemoveRef(partitionTtl);
isUpdated = true;
}
}
}
if (isUpdated)
{
var lastPartition = GetPartitionByID(tx, lastPartitionID);
var rec = new TxRec();
lastPartition.SetTxRec(tx, rec);
// _symbolTxSupport.SetTxRec(tx, rec);
_txLog.Create(rec);
_lastTxRec = rec;
var onCommited = OnCommited;
if (onCommited != null)
{
onCommited(rowIdFrom, _lastTxRec.JournalMaxRowID);
}
}
tx.SetCommited();
}
}
catch (Exception ex)
{
throw new NFSdbCommitFailedException(
"Error commiting transaction. See InnerException for details.", ex);
}
_lastTransactionLog = tx;
}
public PartitionTxData ReadTxLogFromPartition(ReadContext readCache, TxRec txRec = null)
{
if (!_isStorageInitialized)
{
InitializeStorage();
}
return(_txSupport.ReadTxLogFromPartition(readCache, txRec));
}
public void Should_save_LagName(string name)
{
var txLog = CreateTxLog();
var rec = new TxRec
{
LagName = name
};
txLog.Create(rec);
var rec2 = txLog.Get();
Assert.That(rec2.LagName, Is.EqualTo(name));
}
public void Should_save_LagSize(long lagSize)
{
var txLog = CreateTxLog();
var rec = new TxRec
{
LagSize = lagSize
};
txLog.Create(rec);
var rec2 = txLog.Get();
Assert.That(rec2.LagSize, Is.EqualTo(lagSize));
}
public void Should_save_last_timestamp(long timestamp)
{
var txLog = CreateTxLog();
var rec = new TxRec
{
LastPartitionTimestamp = timestamp
};
txLog.Create(rec);
var rec2 = txLog.Get();
Assert.That(rec2.LastPartitionTimestamp, Is.EqualTo(timestamp));
}
public void Should_save_max_row_id(long maxRec)
{
var txLog = CreateTxLog();
var rec = new TxRec
{
JournalMaxRowID = maxRec
};
txLog.Create(rec);
var rec2 = txLog.Get();
Assert.That(rec2.JournalMaxRowID, Is.EqualTo(maxRec));
}
public void SetTxRec(ITransactionContext tx, TxRec rec)
{
rec.PrevTxAddress = tx.PrevTxAddress;
rec.Command = TxRec.TX_NORMAL;
var pd = tx.GetPartitionTx(_partitionID);
var columCount = _metadata.Columns.Count();
rec.LastPartitionTimestamp = DateUtils.DateTimeToUnixTimeStamp(tx.LastAppendTimestamp);
// Java NFSdb Journal has paritions from 0 with no reserved id for symbol parition.
// Max row ID is rowcount + 1 for compatibility
rec.JournalMaxRowID = RowIDUtil.ToRowID(_partitionID - 1, pd.NextRowID - 1) + 1;
rec.IndexPointers = new long[columCount];
var symbolTableSize = new List <int>();
var symbolTableIndexPointers = new List <long>();
for (int i = 0; i < _storage.OpenFileCount; i++)
{
var f = _storage.GetOpenedFileByID(i);
if (f != null && f.DataType == EDataType.Datak)
{
rec.IndexPointers[f.ColumnID] = pd.SymbolData[f.FileID].KeyBlockOffset;
}
else
{
var file = _storage.GetOpenedFileByID(i);
if (file == null)
{
continue;
}
if (file.DataType == EDataType.Symi)
{
var indexSize = (int)(pd.AppendOffset[file.FileID] / 8);
symbolTableSize.Add(indexSize);
}
else if (file.DataType == EDataType.Symrk)
{
var sd = pd.SymbolData[file.FileID];
symbolTableIndexPointers.Add(sd.KeyBlockOffset);
}
}
}
rec.SymbolTableIndexPointers = symbolTableIndexPointers.ToArray();
rec.SymbolTableSizes = symbolTableSize.ToArray();
}
public void Should_save_LagIndexPointers(string tableSizes)
{
var txLog = CreateTxLog();
var symbols = string.IsNullOrEmpty(tableSizes) ?
null : tableSizes.Split(',').Select(long.Parse).ToArray();
var rec = new TxRec
{
LagIndexPointers = symbols
};
txLog.Create(rec);
var rec2 = txLog.Get();
var result = string.Join(",", rec2.LagIndexPointers);
Assert.That(result, Is.EqualTo(tableSizes));
}
public void Should_write_tx_rec_last_partition_timestamp()
{
const int partitionID = 1;
const int timestamp = 20200119;
var ftx = CreateFileTxSupport(CreateFileMocks("i.d-8|s.d-8"), partitionID);
var tx = new Mock <ITransactionContext>();
tx.Setup(t => t.LastAppendTimestamp).Returns(DateUtils.UnixTimestampToDateTime(timestamp));
tx.Setup(t => t.GetPartitionTx(partitionID)).Returns(new PartitionTxData(2, partitionID, new ReadContext()));
var txRec = new TxRec();
// Act.
ftx.SetTxRec(tx.Object, txRec);
// Verify.
Assert.That(txRec.LastPartitionTimestamp, Is.EqualTo(timestamp));
}
public void Should_write_tx_rec_journal_max_row_id()
{
const int partitionID = 1;
const int localRowID = 1001;
var ftx = CreateFileTxSupport(CreateFileMocks("i.d-8|s.d-8"), partitionID);
var tx = new Mock <ITransactionContext>();
tx.Setup(t => t.GetPartitionTx(partitionID)).Returns(new PartitionTxData(2, partitionID, new ReadContext())
{
NextRowID = localRowID
});
var txRec = new TxRec();
// Act.
ftx.SetTxRec(tx.Object, txRec);
// Verify.
var expected = RowIDUtil.ToRowID(partitionID - 1, localRowID - 1) + 1;
Assert.That(txRec.JournalMaxRowID, Is.EqualTo(expected));
}
public void ClearTxLog()
{
_lastTransactionLog = null;
_lastTxRec = null;
_txLog.Clean();
}
private void ReconcilePartitionsWithTxRec(List <IPartition> partitions, TxRec txRec)
{
partitions.Clear();
var defaultPath = _settings.DefaultPath;
// Symbols are the partition 0.
partitions.Add(null);
var nextPartitionID = 1;
var subDirs = LatestPartitionVersions(defaultPath);
foreach (var subDir in subDirs)
{
var fullPath = Path.Combine(defaultPath, subDir.Name);
var startDate = subDir.Date;
var partitionDir = subDir;
PartitionConfig config = PartitionManagerUtils.ReadPartitionConfig(fullPath);
if (config != null)
{
nextPartitionID = config.PartitionID;
}
if (txRec.IsCommited(startDate, nextPartitionID))
{
var partitionID = nextPartitionID;
var partition = _partitions.AddOrUpdate(startDate,
// Add.
sd =>
{
var p = new Lazy <IPartition>(() => CreateNewParition(partitionDir, partitionID, fullPath, config));
if (config == null && Access == EFileAccess.ReadWrite)
{
p.Value.SaveConfig();
}
return(p);
},
// Update.
(sd, existing) =>
{
if (existing.Value.Version == subDir.Version)
{
return(existing);
}
existing.Value.MarkOverwritten();
var p = new Lazy <IPartition>(() => CreateNewParition(partitionDir, partitionID, fullPath, config));
if (config == null && Access == EFileAccess.ReadWrite)
{
p.Value.SaveConfig();
}
return(p);
});
partitions.SetToIndex(partitionID, partition.Value);
nextPartitionID++;
}
else
{
Trace.TraceInformation(
"Ignoring directory '{0}' for partition type '{1}' as fully rolled back partition.",
fullPath, _settings.PartitionType);
Lazy <IPartition> existingPartition;
if (_partitions.TryRemove(startDate, out existingPartition))
{
if (existingPartition.IsValueCreated)
{
existingPartition.Value.Dispose();
}
}
}
}
}
private PartitionTxData ReadTxLogFromFileAndTxRec(ReadContext readCache, TxRec txRec)
{
int symrRead = 0;
var pd = new PartitionTxData(_metadata.FileCount, _partitionID, _startDate, _endTime, readCache);
long nextRowID = RowIDUtil.ToLocalRowID(txRec.JournalMaxRowID - 1) + 1;
pd.NextRowID = nextRowID;
for (int i = 0; i < _storage.OpenFileCount; i++)
{
var file = _storage.GetOpenedFileByID(i);
if (file == null)
{
continue;
}
try
{
if (file.DataType == EDataType.Symrk &&
txRec.SymbolTableIndexPointers != null)
{
var blockOffset = txRec.SymbolTableIndexPointers[symrRead++];
pd.SymbolData[file.FileID].KeyBlockOffset = blockOffset;
long keyBlockSize = file.ReadInt64(blockOffset
+ MetadataConstants.K_FILE_ROW_BLOCK_LEN_OFFSET);
pd.SymbolData[file.FileID].KeyBlockSize = (int)keyBlockSize;
pd.AppendOffset[file.FileID] = blockOffset + keyBlockSize;
}
else if (file.DataType == EDataType.Datak &&
txRec.IndexPointers != null)
{
var blockOffset = txRec.IndexPointers[file.ColumnID];
pd.SymbolData[file.FileID].KeyBlockOffset = blockOffset;
long keyBlockSize = file.ReadInt64(blockOffset
+ MetadataConstants.K_FILE_ROW_BLOCK_LEN_OFFSET);
pd.SymbolData[file.FileID].KeyBlockSize = (int)keyBlockSize;
pd.AppendOffset[file.FileID] = blockOffset + keyBlockSize;
}
else
{
var column = _metadata.GetColumnByID(file.ColumnID);
var size = StorageSizeUtils.GetRecordSize(column, file.DataType);
if (size > 0)
{
// Fixed column.
pd.AppendOffset[file.FileID] = nextRowID * size;
}
else
{
// Variable column.
pd.AppendOffset[file.FileID] = file.GetAppendOffset();
}
if (_metadata.TimestampColumnID == column.ColumnID)
{
var timestamp = file.ReadInt64(pd.AppendOffset[file.FileID] - TIMESTAMP_DATA_SIZE);
pd.LastTimestamp = timestamp;
}
}
}
catch (Exception ex)
{
if (ex is NFSdbPartitionException)
{
throw;
}
throw new NFSdbTransactionStateExcepton(
string.Format("Error reading transaction state from file {0}",
file.Filename), ex);
}
}
pd.NextRowID = nextRowID;
return(pd);
}
public void SetTxRec(ITransactionContext tx, TxRec rec)
{
_txSupport.SetTxRec(tx, rec);
}
