public void SaveConfig(IReadTransactionContext tx = null)
{
if (_access != EFileAccess.ReadWrite)
{
throw new NFSdbIOException("Attempt to save partition {0} config in read-only state", DirectoryPath);
}
if (!Directory.Exists(DirectoryPath))
{
Directory.CreateDirectory(DirectoryPath);
}
var config = _config;
if (config == null)
{
config = new PartitionConfig();
config.Columns = _metadata.Settings.BuildColumns();
}
config.PartitionID = PartitionID;
config.RecordHint = tx != null?tx.GetPartitionTx(PartitionID).NextRowID : _metadata.Settings.RecordHint;
using (var fs = File.Open(Path.Combine(DirectoryPath, PartitionSettingsFileName), FileMode.Create, FileAccess.ReadWrite))
{
ConfigurationSerializer.WritePartitionConfiguration(fs, config);
}
_config = config;
}
private IEnumerable <long> GetLatestByIndexedSymbol(IEnumerable <PartitionRowIDRange> partitions,
IReadTransactionContext tx, ERowIDSortDirection sort)
{
List <long> latestRowIDs;
if (_keys != null)
{
latestRowIDs = GetLatestByIndexedSymbolByKeys(partitions, tx);
}
else
{
latestRowIDs = GetAllLatestByIndexedSymbolByKeys(partitions, tx);
}
// RowID sort asc.
latestRowIDs.Sort();
int startIndex = latestRowIDs.BinarySearch(1L);
if (startIndex < 0)
{
startIndex = ~startIndex;
}
var result = new ArraySlice <long>(latestRowIDs, startIndex, latestRowIDs.Count - startIndex,
sort == ERowIDSortDirection.Asc);
return(result);
}
private void OptimizeFilters(IJournalCore journal, IReadTransactionContext tx)
{
if (_andFilters != null)
{
_andFilters.Sort(new DescendingCardinalityComparer(journal, tx));
}
}
public long Cardinality(IJournalCore journal, IReadTransactionContext tx)
{
if (!_cardinality.HasValue)
{
_cardinality = journal.QueryStatistics.GetColumnDistinctCardinality(tx, _column);
}
return(_cardinality.Value);
}
protected override SingleMultipleValues <T> GetAllMatchingValues(IReadTransactionContext tx)
{
if (_values != null)
{
return(SingleMultipleValues <T> .Multiple(_values));
}
return(SingleMultipleValues <T> .Single(_value));
}
public long GetCardinality(IJournalCore journal, IReadTransactionContext tx)
{
if (_keys == null)
{
return(_journal.QueryStatistics.GetColumnDistinctCardinality(tx, _column));
}
return(_keys.Count);
}
public Query(IJournal <T> journal, IReadTransactionContext transactionContext)
{
_transactionContext = transactionContext;
_journal = journal;
_queryable = new Lazy <JournalQueryable <T> >(
() => new JournalQueryable <T>(new JournalQueryProvider <T>(_journal, transactionContext)));
_recordQuery = new RecordQuery(journal, transactionContext);
}
private IEnumerable <long> GetLatestByNonIndexedField(IEnumerable <PartitionRowIDRange> partitions,
IReadTransactionContext tx, ERowIDSortDirection sort)
{
if (sort == ERowIDSortDirection.Desc)
{
return(GetLatestFromDescending(partitions, tx));
}
return(GetLatestFromDescending(partitions.Reverse(), tx).Reverse());
}
public IEnumerable <long> Filter(IEnumerable <PartitionRowIDRange> partitions,
IReadTransactionContext tx, ERowIDSortDirection sort)
{
if (_column.Indexed)
{
return(GetLatestByIndexedSymbol(partitions, tx, sort));
}
return(GetLatestByNonIndexedField(partitions, tx, sort));
}
public IEnumerable <long> Filter(IEnumerable <PartitionRowIDRange> partitions,
IReadTransactionContext tx, ERowIDSortDirection sortDirection)
{
if (_column.Indexed)
{
var values = GetAllMatchingValues(tx);
if (!values.IsNone)
{
if (!values.IsSigle)
{
var valueSet = values.Values.Distinct().ToList();
var items = new IEnumerable <long> [valueSet.Count];
return(partitions.SelectMany(part =>
{
var partition = tx.Read(part.PartitionID);
for (int v = 0; v < valueSet.Count; v++)
{
var symbolValue = valueSet[v];
var rowIDs = TakeFromTo(part, partition.GetSymbolRows(_column.ColumnID, symbolValue, tx));
if (sortDirection == ERowIDSortDirection.Asc)
{
// Todo: use tx.ReadContext and reuse buffers.
rowIDs = rowIDs.Reverse();
}
items[v] = rowIDs;
}
if (sortDirection == ERowIDSortDirection.None)
{
return items.SelectMany(i => i);
}
return MergeSorted(items, sortDirection);
}));
}
else
{
return(partitions.SelectMany(part =>
{
var partition = tx.Read(part.PartitionID);
var rowIDs = TakeFromTo(part, partition.GetSymbolRows(_column.ColumnID, values.Value, tx));
if (sortDirection == ERowIDSortDirection.Asc)
{
rowIDs = rowIDs.Reverse();
}
return rowIDs;
}));
}
}
}
return(partitions.SelectMany(part =>
{
var partition = tx.Read(part.PartitionID);
return IsPartitionMatch(tx, part, partition, sortDirection);
}));
}
public IEnumerable <long> Execute(IJournalCore journal, IReadTransactionContext tx, ERowIDSortDirection sortDirection)
{
var intervalFilter = new PartitionIntervalIterator();
var symbolFilter = new LatestByFilter <T>(journal, _column, _keys);
return(Timestamps.AllIntervals.Reverse().SelectMany(interval =>
symbolFilter.Filter(intervalFilter.IteratePartitions(
tx.ReverseReadPartitions, interval, tx), tx, sortDirection)
));
}
public IEnumerable <long> GetSymbolRowsByKey(int fieldID, int valueKey, IReadTransactionContext tx)
{
if (!_isStorageInitialized)
{
InitializeStorage();
}
var symb = (IIndexedColumnCore)(_columns[fieldID].Column);
return(symb.GetValues(valueKey, tx.GetPartitionTx(PartitionID)));
}
public int GetSymbolKey <TT>(int fieldID, TT value, IReadTransactionContext tx)
{
if (!_isStorageInitialized)
{
InitializeStorage();
}
var symb = (IIndexedColumn <TT>)(_columns[fieldID].Column);
var key = symb.CheckKeyQuick(value, tx.GetPartitionTx(PartitionID));
return(key);
}
public IEnumerable <long> GetSymbolRows <TT>(int fileID, TT value, IReadTransactionContext tx)
{
if (!_isStorageInitialized)
{
InitializeStorage();
}
var symb = (IIndexedColumn <TT>)(_columns[fileID].Column);
var key = symb.CheckKeyQuick(value, tx.GetPartitionTx(PartitionID));
return(symb.GetValues(key, tx.GetPartitionTx(PartitionID)));
}
public long GetCardinalityByColumnValue <T>(IReadTransactionContext tx, IColumnMetadata col, IList <T> values)
{
if (col.Indexed)
{
var part = _partitionManager.GetOpenPartitions().FirstOrDefault(p => p != null);
if (part != null)
{
return(values.Sum(value => part.GetSymbolRowCount(col.ColumnID, value, tx)));
}
}
return(_metadata.Settings.RecordHint / col.HintDistinctCount * values.Count);
}
public long GetSymbolRowCount <TT>(int fieldID, TT value, IReadTransactionContext tx)
{
if (!_isStorageInitialized)
{
return(0);
}
var symb = (IIndexedColumn <TT>)(_columns[fieldID].Column);
var key = symb.CheckKeyQuick(value, tx.GetPartitionTx(PartitionID));
return(symb.GetCount(key, tx.GetPartitionTx(PartitionID)));
}
public long Cardinality(IJournalCore journal, IReadTransactionContext tx)
{
if (!_caridnality.HasValue)
{
var intervalFilter = new PartitionIntervalIterator();
_caridnality =
Timestamps
.AllIntervals
.Sum(p => intervalFilter.IteratePartitions(tx.ReadPartitions, p, tx).Sum(p2 => p2.High - p2.Low));
}
return(_caridnality.Value);
}
public RecordSet(IEnumerable <long> rowIDs,
IReadTransactionContext tx,
IJournalMetadata metadata)
{
_rowIDs = rowIDs;
_tx = tx;
_metadata = metadata;
var bitsetColumn = _metadata.Columns.FirstOrDefault(c => c.DataType.ColumnType == EFieldType.BitSet);
if (bitsetColumn != null)
{
_bitSetColIndex = bitsetColumn.ColumnID;
}
}
public long Cardinality(IJournalCore journal, IReadTransactionContext tx)
{
if (!_cardin.HasValue)
{
if (_andFilters != null)
{
_cardin = _andFilters.Min(f => f.GetCardinality(journal, tx));
}
else
{
_cardin = long.MaxValue;
}
}
return(_cardin.Value);
}
public IEnumerable <long> Execute(IJournalCore journal, IReadTransactionContext tx, ERowIDSortDirection sort)
{
var intervalFilter = new PartitionIntervalIterator();
if (sort == ERowIDSortDirection.Desc)
{
return(Timestamps.AllIntervals.Reverse().SelectMany(
interval => GetIdsDesc(
intervalFilter.IteratePartitions(
tx.ReadPartitions.Reverse(), interval, tx))
));
}
return(Timestamps.AllIntervals.SelectMany(
interval => GetIdsAsc(
intervalFilter.IteratePartitions(tx.ReadPartitions, interval, tx))
));
}
public IEnumerable <long> Execute(IJournalCore journal, IReadTransactionContext tx, ERowIDSortDirection sort)
{
// Timestamp restricted
if (_left is TimestampRangePlanItem)
{
return(_right.Execute(journal, tx, sort));
}
if (_right is TimestampRangePlanItem)
{
return(_left.Execute(journal, tx, sort));
}
if (_left.Cardinality(journal, tx) < _right.Cardinality(journal, tx))
{
return(Intersect(_left.Execute(journal, tx, sort), _right.Execute(journal, tx, sort), sort));
}
return(Intersect(_right.Execute(journal, tx, sort), _left.Execute(journal, tx, sort), sort));
}
private IEnumerable <long> FilterRowsRange(IReadTransactionContext tx, List <PartitionRowIDRange> intervals, ERowIDSortDirection sort)
{
for (int i = 0; i < intervals.Count; i++)
{
var partitionRowIDRange = intervals[i];
var paritition = tx.Read(partitionRowIDRange.PartitionID);
long from = sort == ERowIDSortDirection.Asc ? partitionRowIDRange.Low : partitionRowIDRange.High;
long to = sort == ERowIDSortDirection.Asc ? partitionRowIDRange.High: partitionRowIDRange.Low;
long increment = sort == ERowIDSortDirection.Asc ? 1 : -1;
for (long rowID = from; rowID < to; rowID += increment)
{
if (MatchFilters(paritition, tx.ReadCache, rowID))
{
yield return(RowIDUtil.ToRowID(partitionRowIDRange.PartitionID, rowID));
}
}
}
}
private IEnumerable <long> ApplyFilters(IEnumerable <long> rowIDs, IReadTransactionContext tx)
{
int partitionIndex = -1;
IPartitionReader partition = null;
foreach (var globalRowID in rowIDs)
{
var rowPartitionIndex = RowIDUtil.ToPartitionIndex(globalRowID);
if (rowPartitionIndex != partitionIndex)
{
partition = tx.Read(rowPartitionIndex);
partitionIndex = rowPartitionIndex;
}
if (MatchFilters(partition, tx.ReadCache, RowIDUtil.ToLocalRowID(globalRowID)))
{
yield return(globalRowID);
}
}
}
public long BinarySearchTimestamp(DateTime value, IReadTransactionContext tx)
{
if (!_isStorageInitialized)
{
InitializeStorage();
}
if (_timestampColumn == null)
{
throw new NFSdbConfigurationException("timestampColumn is not configured for journal in "
+ DirectoryPath);
}
var hi = tx.GetRowCount(PartitionID);
var timestampType = _timestampColumn.FieldType;
long values = timestampType == EFieldType.Int64 ||
timestampType == EFieldType.DateTimeEpochMs
? DateUtils.DateTimeToUnixTimeStamp(value)
: DateUtils.ToUnspecifiedDateTicks(value);
return(ColumnValueBinarySearch.LongBinarySerach(_timestampColumn, values, 0L, hi));
}
public IComparer <long> GetColumnComparer(int columnId, IReadTransactionContext tx, bool ascending)
{
switch (ColumnType)
{
case EFieldType.Byte:
return(new ColumnValueComparer <byte>(columnId, tx, ascending));
case EFieldType.Bool:
return(new ColumnValueComparer <bool>(columnId, tx, ascending));
case EFieldType.Int16:
return(new ColumnValueComparer <short>(columnId, tx, ascending));
case EFieldType.Int32:
return(new ColumnValueComparer <int>(columnId, tx, ascending));
case EFieldType.Int64:
return(new ColumnValueComparer <long>(columnId, tx, ascending));
case EFieldType.Double:
return(new ColumnValueComparer <double>(columnId, tx, ascending));
case EFieldType.Symbol:
case EFieldType.String:
return(new ColumnValueComparer <string>(columnId, tx, ascending));
case EFieldType.DateTime:
case EFieldType.DateTimeEpochMs:
return(new ColumnValueComparer <DateTime>(columnId, tx, ascending));
case EFieldType.Binary:
return(new BinaryColumnValueComparer(columnId, tx, ascending));
default:
throw QueryExceptionExtensions.NotSupported("Column {0} of type {1} is not sortable.", columnId, columnId);
}
}
private IEnumerable <long> IsPartitionMatch(IReadTransactionContext tx, PartitionRowIDRange part,
IPartitionReader partition, ERowIDSortDirection sortDirection)
{
if (sortDirection == ERowIDSortDirection.Asc)
{
for (long rowId = part.Low; rowId <= part.High; rowId++)
{
if (IsMatch(partition, tx.ReadCache, rowId))
{
yield return(RowIDUtil.ToRowID(part.PartitionID, rowId));
}
}
}
else
{
for (long rowId = part.High; rowId >= part.Low; rowId--)
{
if (IsMatch(partition, tx.ReadCache, rowId))
{
yield return(RowIDUtil.ToRowID(part.PartitionID, rowId));
}
}
}
}
public IEnumerable <long> Execute(IJournalCore journal, IReadTransactionContext tx,
ERowIDSortDirection sort)
{
var intervalFilter = new PartitionIntervalIterator();
var intervals = Timestamps.AllIntervals.SelectMany(
i => intervalFilter.IteratePartitions(tx.ReadPartitions, i, tx)).ToList();
if (sort == ERowIDSortDirection.Desc)
{
intervals.Reverse();
}
if (_partitionFilter != null)
{
if (_andFilters == null)
{
return(_partitionFilter.Filter(intervals, tx, sort));
}
OptimizeFilters(journal, tx);
return(ApplyFilters(_partitionFilter.Filter(intervals, tx, sort), tx));
}
OptimizeFilters(journal, tx);
return(FilterRowsRange(tx, intervals, sort));
}
public long Cardinality(IJournalCore journal, IReadTransactionContext tx)
{
return(Math.Min(_left.Cardinality(journal, tx), _right.Cardinality(journal, tx)));
}
public override long GetCardinality(IJournalCore journal, IReadTransactionContext tx)
{
return(journal.QueryStatistics.GetCardinalityByColumnValue(tx, _column, _values ?? new[] { _value }));
}
public ColumnValueComparer(int fieldId, IReadTransactionContext tx, bool asc)
{
_filedId = fieldId;
_tx = tx;
_asc = asc;
}
