public override IEnumerable <KeyValuePair <K, int> > CompareNonIndexedKeyWithKey <K, T>(
DbColumn key, DbColumn column, TokenType comparer, T value)
{
if (column.Indexed || column.Table.Multikey)
{
throw new ArgumentException($"Column {column} must be un-indexed, and belong to a One-Key table");
}
//get the key column, index
var keyColumn = column.Table.Key;
int keyColumnIndex = column.Index;
K keyValue = default(K);
var keyCount = 0;
//read indexer offsets in their order
using (var keyReader =
new io.BinaryReader(
io.File.OpenRead($"{key.Table.Database.BinaryPath}\\{key.Hash}.offset")))
{
//amount of keys
keyCount = keyReader.ReadInt32();
var count = Table.Count;
var initialCount = column.Index;
var buffer = new byte[sizeof(UInt64)];
var bytes = Table.RowMaskLength;
var mainMask = Table.RowMask;
int columnIndex = 0;
int columnSize = 0;
int bytesToSkip = 0;
ulong bitMask = 0;
ulong recordMask = 0;
void SKIP(int cnt)
{
while (cnt-- > 0)
{
if ((recordMask & bitMask) == 0)
{
if (columnIndex == keyColumnIndex)
{
//read Key value into: keyValue
keyValue = GetColumnValue <K>();
}
else
{
//SKIP
//not null value, add according to column type
if (columnSize == 0)
{
throw new ArgumentException("unsupported type size");
}
else if (columnSize > 0)
{
bytesToSkip += columnSize;
}
else
{
//string, read length, and add chars
reader.BaseStream.Position += bytesToSkip;
//read new bytes to skip
bytesToSkip = reader.ReadByte();
}
}
}
bitMask >>= 1;
columnSize = ColumnTypeSizes[++columnIndex];
}
//position reader
if (bytesToSkip > 0)
{
reader.BaseStream.Position += bytesToSkip;
}
}
C GetColumnValue <C>()
{
//for string, read length byte
var size = (columnSize < 0) ? reader.ReadByte() : columnSize;
//create buffer
var columnBuffer = new byte[size];
//read it
reader.Read(columnBuffer, 0, size);
//convert it to generic value
return(columnBuffer.BitConvertTo <C>());
}
for (var ndx = 0; ndx < keyCount; ndx++)
{
var offset = keyReader.ReadInt32();
//point to record offset
reader.BaseStream.Position = offset;
//copy main mask
bitMask = mainMask;
//read record mask
reader.Read(buffer, 0, bytes);
recordMask = BitConverter.ToUInt64(buffer, 0);
//developer only
var binary = Convert.ToString((long)recordMask, 2);
//
columnSize = ColumnTypeSizes[columnIndex = 0];
bytesToSkip = 0;
//skip initial columns if any
if (initialCount > 0)
{
SKIP(initialCount);
}
//read column value to compare
var columnValue = GetColumnValue <T>();
if (comparer.Compare <T>(columnValue, value))
{
if (keyColumnIndex > columnIndex)
{
//key not read yet
SKIP(keyColumnIndex - columnIndex - 1);
//read Key value into: keyValue
keyValue = GetColumnValue <K>();
}
yield return(new KeyValuePair <K, int>(keyValue, offset));
}
}
}
}
public override IEnumerable <KeyValuePair <K, int> > CompareNonIndexedKeyWithKey <K, T>(DbColumn key, DbColumn column, TokenType comparer, T value)
{
throw new NotImplementedException();
}
/// <summary>
/// Expensive non-indexed column comparison by a constant value
/// </summary>
/// <typeparam name="K">type of column key</typeparam>
/// <typeparam name="T">type of column</typeparam>
/// <param name="key">table key</param>
/// <param name="column">column</param>
/// <param name="comparer">comparison operator</param>
/// <param name="value">constant value</param>
/// <returns>key value pair with key and offset</returns>
public abstract IEnumerable <KeyValuePair <K, int> > CompareNonIndexedKeyWithKey <K, T>(
DbColumn key, DbColumn column, TokenType comparer, T value)
where K : IComparable <K>
where T : IComparable <T>;
/// <summary>
/// Expensive non-indexed column comparison by a constant value
/// </summary>
/// <typeparam name="T">type of column</typeparam>
/// <param name="key">table key</param>
/// <param name="column">column to compare</param>
/// <param name="comparer">comparison operator</param>
/// <param name="value">constant value</param>
/// <returns></returns>
public override IEnumerable <int> CompareNonIndexedKey <T>(
DbColumn key, DbColumn column, TokenType comparer, T value)
{
//THIS CANNOT BE TOGETHER WITH CompareNonIndexedKeyWithKey BECAUSE
// RETURNING KEY MAKE AN EXTRA LOAD
var keyCount = 0;
//read indexer offsets in their order
using (var keyReader =
new io.BinaryReader(
io.File.OpenRead($"{key.Table.Database.BinaryPath}\\{key.Hash}.offset")))
{
keyCount = keyReader.ReadInt32();
var count = Table.Count;
var initialCount = column.Index;
var buffer = new byte[sizeof(UInt64)];
var bytes = Table.RowMaskLength;
var mainMask = Table.RowMask;
int columnIndex = 0;
int columnSize = 0;
int bytesToSkip = 0;
ulong bitMask = 0;
ulong recordMask = 0;
//build column type translator
var columnTypeSizes = Table.ColumnTypes.Select(t => t.GetSize()).ToArray();
void SKIP(int cnt)
{
while (cnt-- > 0)
{
if ((recordMask & bitMask) == 0)
{
//not null value, add according to column type
var size = columnTypeSizes[columnIndex];
if (size == 0)
{
throw new ArgumentException("unsupported type size");
}
else if (size > 0)
{
bytesToSkip += size;
}
else
{
//string, read length, and add chars
reader.BaseStream.Position += bytesToSkip;
//read new bytes to skip
bytesToSkip = reader.ReadByte();
}
}
bitMask >>= 1;
columnSize = ColumnTypeSizes[++columnIndex];
}
//position reader
if (bytesToSkip > 0)
{
reader.BaseStream.Position += bytesToSkip;
}
}
T GetColumnValue <T>()
{
//for string, read length byte
var size = (columnSize < 0) ? reader.ReadByte() : columnSize;
//create buffer
var columnBuffer = new byte[size];
//read it
reader.Read(columnBuffer, 0, size);
//convert it to generic value
return(columnBuffer.BitConvertTo <T>());
}
for (var ndx = 0; ndx < keyCount; ndx++)
{
var offset = keyReader.ReadInt32();
//point to record offset
reader.BaseStream.Position = offset;
//copy main mask
bitMask = mainMask;
//read record mask
reader.Read(buffer, 0, bytes);
recordMask = BitConverter.ToUInt64(buffer, 0);
//developer only
var binary = Convert.ToString((long)recordMask, 2);
columnSize = ColumnTypeSizes[columnIndex = 0];
bytesToSkip = 0;
//skip initial columns if any
if (initialCount > 0)
{
SKIP(initialCount);
}
//read column value to compare
var columnValue = GetColumnValue <T>();
//compare
if (comparer.Compare <T>(columnValue, value))
{
yield return(offset);
}
}
}
}
/// <summary>
/// Expensive non-indexed column comparison by a constant value
/// </summary>
/// <typeparam name="T">type of column</typeparam>
/// <param name="key">table key</param>
/// <param name="column">column</param>
/// <param name="comparer">comparison operator</param>
/// <param name="value">constant value</param>
/// <returns></returns>
public abstract IEnumerable <int> CompareNonIndexedKey <T>(
DbColumn key, DbColumn column, TokenType comparer, T value)
where T : IComparable <T>;
/// <summary>
/// Fast indexed column coomparison by a constant value
/// </summary>
/// <typeparam name="T">type of column</typeparam>
/// <param name="key">table key</param>
/// <param name="column">comlumn to compare</param>
/// <param name="comparer">comparison operator</param>
/// <param name="value">constant value</param>
/// <returns></returns>
internal IEnumerable <KeyValuePair <T, List <int> > > CompareIndexedKeyWithKey <T>(
DbColumn key, DbColumn column, TokenType comparer, T value)
where T : IComparable <T>
{
var nodeTree = column.IndexTree <T>();
int offset = -1;
var collection = Enumerable.Empty <KeyValuePair <T, List <int> > >();
if (nodeTree.Root == null)
{
//go to .bin file directly, it's ONE page of items
collection = column.IndexItems <T>()
.FindByOper(DbGenerator.ItemsPageStart, value, comparer);
}
else
{
//this's for tree node page structure
switch (comparer)
{
case TokenType.Equal: // "="
//find page with key
var baseNode = nodeTree.FindKey(value) as PageIndexNodeBase <T>;
if (baseNode != null)
{
switch (baseNode.Type)
{
case MetaIndexType.Node:
//it's in a tree node page
var nodePage = baseNode as PageIndexNode <T>;
if (nodePage != null && nodePage.Values != null)
{
collection = new KeyValuePair <T, List <int> >[] {
new KeyValuePair <T, List <int> >(value, nodePage.Values)
};
}
break;
case MetaIndexType.Items:
//it's in a items page
offset = ((PageIndexItems <T>)baseNode).Offset;
DbKeyValues <T> item = column.IndexItems <T>().Find(offset, value);
if (item != null && item.Values != null)
{
collection = new KeyValuePair <T, List <int> >[] {
new KeyValuePair <T, List <int> >(value, item.Values)
};
}
break;
}
}
break;
case TokenType.Less: // "<"
collection = nodeTree.FindLessKey(value, TokenType.Less);
break;
case TokenType.LessOrEqual: // "<="
collection = nodeTree.FindLessKey(value, TokenType.LessOrEqual);
break;
case TokenType.Greater: // ">"
collection = nodeTree.FindGreaterKey(value, TokenType.Greater);
break;
case TokenType.GreaterOrEqual: //">="
collection = nodeTree.FindGreaterKey(value, TokenType.GreaterOrEqual);
break;
//throw new ArgumentException($"Operator: {Expression.Operator.Name} not implemented yet!");
}
}
foreach (var ofs in collection)
{
yield return(ofs);
}
}
/// <summary>
/// Loads a database schema
/// </summary>
/// <param name="reader">binary reader</param>
/// <returns></returns>
public static DbSchemaConfig Load(io.BinaryReader reader)
{
var schema = new DbSchemaConfig()
{
Flags = (DbSchemaConfigType)reader.ReadUInt32(),
Version = reader.BinaryRead(),
Description = reader.BinaryRead(),
Name = reader.BinaryRead(),
PageSize = reader.ReadInt32(),
_tables = new Dictionary <string, DbTable>()
//Tables = new List<DbTable>()
};
//tables
var pageCount = reader.ReadInt32();
for (var t = 0; t < pageCount; t++)
{
var table = new DbTable()
{
Name = reader.BinaryRead(),
FileName = reader.BinaryRead(),
Generate = reader.ReadBoolean(),
Multikey = reader.ReadBoolean(),
Rows = reader.ReadInt32(),
RowMask = reader.ReadUInt64(),
RowMaskLength = reader.ReadInt32(),
Pager = DbTablePager.Load(reader),
Count = reader.ReadInt32(),
_columns = new Dictionary <string, DbColumn>()
};
//read columns
var columnNameList = new List <string>();
for (var c = 0; c < table.Count; c++)
{
var col = new DbColumn()
{
Indexer = reader.BinaryRead(),
Unique = reader.ReadBoolean(),
Name = reader.BinaryRead(),
Index = reader.ReadInt32(),
Type = reader.BinaryRead(),
Key = reader.ReadBoolean(),
Indexed = reader.ReadBoolean(),
//
NodePages = reader.ReadInt32(),
ItemPages = reader.ReadInt32()
};
if (!table.Add(col))
{
throw new ArgumentException($"duplicated column: {col.Name} on table {table.Name}");
}
columnNameList.Add(col.Name);
}
//check count
if (table.Count != table.Columns.Count())
{
throw new ArgumentException($"invalid table count on: {table.Name}");
}
table._columnNameList = columnNameList.ToArray();
//get key
table.Keys = table.Columns.Where(c => c.Key).ToArray();
var keyCount = table.Keys.Length;
if (table.Multikey)
{
//must have more than one key
table.Key = null;
if (keyCount < 2)
{
throw new ArgumentException($"table: {table} is multi-key and has less than 2 keys");
}
}
else
{
//must have just one key
if (keyCount != 1)
{
throw new ArgumentException($"table: {table} must have one key");
}
table.Key = table.Keys[0];
}
//schema.Tables.Add(table);
schema._tables.Add(table.Name, table);
}
return(schema);
}
void SaveBinaryIndex <T>(BTreePageBase <T> rootPage, DbColumn index)
where T : IComparable <T>
{
if (rootPage == null)
{
return;
}
var pageCount = rootPage.ChildrenCount;
Console.WriteLine($" saving ({pageCount}) page(s)");
//update column tree page count
index.NodePages = pageCount;
index.Table.Database.Modified = true;
string indexfilepath = io.Path.Combine(Database.BinaryPath, $"{index.Indexer}");
//page with items .bin
//this where the main info is
var indexBinFilePath = indexfilepath + ".bin";
using (var writer = new io.BinaryWriter(io.File.Create(indexBinFilePath)))
{
var collectionPage = GetNodeItemPages <T>(rootPage).ToList();
//update column item page count
index.ItemPages = collectionPage.Count;
//MAIN HEADER
Int32 pageCollectionCount = collectionPage.Count;
//write amount of pages
writer.Write(pageCollectionCount);
//write index key type
Int32 keyType = (int)index.TypeEnum;
writer.Write(keyType);
//pages
//sizeof: pageCollectionCount, keyType
var offset = DbGenerator.ItemsPageStart; // 8;
foreach (var page in collectionPage)
{
page.Offset = offset;
//save page
var buffer = page.ToBuffer();
//
offset += buffer.Length;
//
writer.Write(buffer, 0, buffer.Length);
}
}
//tree page indexer .index
//this is a light in-memory tree structure for fast search
using (var writer = new io.BinaryWriter(io.File.Create(indexfilepath)))
{
Int32 valueInt32 = 0;
//write Index main Header
PageIndexTreeHeader header = new PageIndexTreeHeader();
//
header.Value0 = 0;
//page count
header.PageCount = pageCount;
//index.Index
header.ColumnIndex = index.Index;
//index.Unique
//index.Key
valueInt32 =
(index.Unique ? Consts.IndexHeaderIsUnique : 0) |
(index.Key ? Consts.IndexHeaderIsKey : 0) |
(rootPage.IsLeaf ? Consts.IndexHeaderIsLeaf : 0);
//index.Type
//valueInt32 = valueInt32 << 8;
valueInt32 |= (byte)index.TypeEnum;
header.Flags = valueInt32;
//write Header
byte[] buffer = header.ToByteArray();
writer.Write(buffer, 0, buffer.Length);
//test
var bin = Convert.ToString(header.Flags, 2);
//write page tree
if (!rootPage.IsLeaf)
{
StoreTreePage <T>(rootPage, writer);
}
}
}
List <KeyValuePair <T, List <int> > > GenerateIndexCollectionKeysMultipleValues <T>(
DbColumn index,
Type indexType,
string sufix
)
{
//Creating the Type for Generic List.
Type kp = typeof(KeyValuePair <,>);
Type[] kpArgs = { indexType, typeof(int) };
//create generic list type
Type kpType = kp.MakeGenericType(kpArgs);
var list = new List <KeyValuePair <T, int> >();
var listAdd = list.GetType().GetMethod("Add");
string line;
//read index data in .TXT file
var textIndexPath = io.Path.Combine(Database.LogPath, $"{index.Indexer}{sufix}.txt");
using (var reader = new io.StreamReader(io.File.OpenRead(textIndexPath)))
{
while ((line = reader.ReadLine()) != null)
{
var columns = line.Split(new char[] { '|' }, StringSplitOptions.RemoveEmptyEntries);
//create kp object
var objvalue = System.Convert.ChangeType(columns[0], indexType);
var kpObj = Activator.CreateInstance(kpType,
new object[] {
System.Convert.ChangeType(columns[0], indexType),
Int32.Parse(columns[1])
});
listAdd.Invoke(list, new object[] { kpObj });
}
}
//return new list ordered by its Key
list = list.OrderBy(i => i.Key).ToList();
//save ordered index
using (var writer =
new io.StreamWriter(io.File.Create(textIndexPath)))
{
foreach (var pair in list)
{
writer.WriteLine($"{pair.Key.ToString()}|{pair.Value}");
}
}
//group by key, to see if any duplicates
// if any save save as: {index.Indexer}.duplicates.txt
var groupedList = (from pair in list
group pair by pair.Key into g
let values = g.Select(p => p.Value).ToList()
select new KeyValuePair <T, List <int> >(g.Key, values))
.ToList();
//save grouped index for reference
var duplicatedIndexPath = io.Path.Combine(Database.LogPath, $"{index.Indexer}{sufix}.duplicates.txt");
if (groupedList.Any(g => g.Value.Count > 1))
{
using (var writer =
new io.StreamWriter(io.File.Create(duplicatedIndexPath)))
{
foreach (var pair in groupedList)
{
var values = String.Join(", ", pair.Value);
writer.WriteLine($"{pair.Key.ToString()}|{values}");
}
}
}
else
{
//ensure if re-compile changed index keep it updated
if (io.File.Exists(duplicatedIndexPath))
{
io.File.Delete(duplicatedIndexPath);
}
}
//
return(groupedList);
}
