/// <summary>
/// 为指定的索引生成头结点和尾结点。
/// </summary>
/// <param name="indexBlock"></param>
/// <param name="maxLevel"></param>
private void InitHeadTailNodes(IndexBlock indexBlock, int maxLevel)
{
{
// 初始化头结点列。
indexBlock.SkipListHeadNodes = new SkipListNodeBlock[maxLevel];
/*SkipListHeadNodes[maxLevel - 1]↓*/
/*SkipListHeadNodes[.]↓*/
/*SkipListHeadNodes[.]↓*/
/*SkipListHeadNodes[2]↓*/
/*SkipListHeadNodes[1]↓*/
/*SkipListHeadNodes[0] */
SkipListNodeBlock current = new SkipListNodeBlock();
indexBlock.SkipListHeadNodes[0] = current;
for (int i = 1; i < maxLevel; i++)
{
SkipListNodeBlock block = new SkipListNodeBlock();
block.DownObj = current;
indexBlock.SkipListHeadNodes[i] = block;
current = block;
}
}
{
// 初始化尾结点。尾结点在序列化到数据库文件后，将永远不再变动，这给编码带来方便，也是我使用尾结点的原因。
// 这样也是完全把外存（硬盘）与内存编程相对应的思想。
indexBlock.SkipListTailNode = new SkipListNodeBlock();
}
{
// 头结点指向对应的尾结点。
for (int i = 0; i < maxLevel; i++)
{
indexBlock.SkipListHeadNodes[i].RightObj = indexBlock.SkipListTailNode;
}
}
}
private static SkipListNodeBlock[] FindRightMostNodes(IComparable key, IndexBlock indexBlock, FileDBContext db)
{
FileStream fs = db.fileStream;
int maxLevel = db.headerBlock.MaxLevelOfSkipList;
SkipListNodeBlock[] rightNodes = new SkipListNodeBlock[maxLevel];
// Start at the top list header node
SkipListNodeBlock currentNode = indexBlock.SkipListHeadNodes[indexBlock.CurrentLevel];
for (int i = indexBlock.CurrentLevel; i >= 0; i--)
{
IComparable rightKey = db.GetRightObjKey(db.fileStream, indexBlock, currentNode);
while ((currentNode.RightObj != indexBlock.SkipListTailNode) && (rightKey.CompareTo(key) < 0))
{
currentNode = currentNode.RightObj;
rightKey = db.GetRightObjKey(db.fileStream, indexBlock, currentNode);
}
rightNodes[i] = currentNode;
if (i > 0)
{
currentNode.TryLoadProperties(fs, SkipListNodeBlockLoadOptions.DownObj);
currentNode = currentNode.DownObj;
}
}
return(rightNodes);
}
/// <summary>
/// 根据[TableIndex]特性，为新表初始化索引。
/// </summary>
/// <param name="type"></param>
/// <param name="indexBlockHead"></param>
/// <returns></returns>
private Dictionary <string, IndexBlock> CreateIndexBlocks(Type type, IndexBlock indexBlockHead)
{
Dictionary <string, IndexBlock> indexBlockDict = new Dictionary <string, IndexBlock>();
PropertyInfo[] properties = type.GetProperties();// RULE: 规则：索引必须加在属性上，否则无效。
foreach (var property in properties)
{
TableIndexAttribute attr = property.GetCustomAttribute <TableIndexAttribute>();
if (attr != null)
{
IndexBlock indexBlock = new IndexBlock();
this.transaction.Add(indexBlock);// 加入事务，准备写入数据库。
indexBlock.BindMember = property.Name;
int maxLevel = this.headerBlock.MaxLevelOfSkipList;
InitHeadTailNodes(indexBlock, maxLevel);
indexBlock.NextObj = indexBlockHead.NextObj;
indexBlockHead.NextObj = indexBlock;
indexBlockDict.Add(property.Name, indexBlock);// indexBlockDict不含indexBlock链表的头结点。
for (int i = maxLevel - 1; i >= 0; i--)
{
this.transaction.Add(indexBlock.SkipListHeadNodes[i]);
} // 加入事务，准备写入数据库。
this.transaction.Add(indexBlock.SkipListTailNode); // 加入事务，准备写入数据库。
}
}
return(indexBlockDict);
}
private static void InitBlockSearch()
{
int j = -1;
int k = 0;
int[] a = new int[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 };
IndexBlock[] indexBlock = new IndexBlock[BLOCK_COUNT];
for (int i = 0; i < BLOCK_COUNT; i++)
{
indexBlock[i].start = j + 1; //确定每个块范围的起始值
j = j + 1;
indexBlock[i].end = j + 4; //确定每个块范围的结束值
j = j + 4;
indexBlock[i].max = a[j]; //确定每个块范围中元素的最大值
}
k = BlockSearch(12, a, indexBlock);
if (k >= 0)
{
// Console.WriteLine("查找成功！你要查找的数在数组中的索引是：{0}\n", k);
}
else
{
// Console.WriteLine("查找失败！你要查找的数不在数组中。\n");
}
}
private Dictionary <string, IndexBlock> GetIndexDict(FileStream fileStream, TableBlock tableBlock)
{
Dictionary <string, IndexBlock> indexDict = new Dictionary <string, IndexBlock>();
long indexBlockHeadPos = tableBlock.IndexBlockHeadPos;
IndexBlock currentIndexBlock = fileStream.ReadBlock <IndexBlock>(indexBlockHeadPos);
tableBlock.IndexBlockHead = currentIndexBlock;
currentIndexBlock.TryLoadNextObj(fileStream);// 此时的currentIndexBlock是索引块的头结点。
while (currentIndexBlock.NextObj != null)
{
SkipListNodeBlock[] headNodes = GetHeadNodesOfSkipListNodeBlock(fileStream, currentIndexBlock.NextObj);
currentIndexBlock.NextObj.SkipListHeadNodes = headNodes;
SkipListNodeBlock tailNode = GetTailNodeOfSkipListNodeBlock(fileStream, currentIndexBlock.NextObj);
currentIndexBlock.NextObj.SkipListTailNode = tailNode;
foreach (SkipListNodeBlock headNode in currentIndexBlock.NextObj.SkipListHeadNodes)
{
if (headNode.RightPos == tailNode.ThisPos)
{
headNode.RightObj = tailNode;
}
}
currentIndexBlock.NextObj.TryLoadNextObj(fileStream);
indexDict.Add(currentIndexBlock.NextObj.BindMember, currentIndexBlock.NextObj);
currentIndexBlock = currentIndexBlock.NextObj;
}
return(indexDict);
}
public IndexBlock <IndexType, DataType> CreateIndexBlock <IndexType, DataType>(Func <DataType, IndexType> IndexFunction, bool isAggregator, string name)
{
var block = new IndexBlock <IndexType, DataType>(EventProcessor, IndexFunction, isAggregator);
SetDebugName(block, name);
return(block);
}
/// <summary>
/// 删除数据库文件里的某个表及其所有索引、数据。
/// </summary>
/// <typeparam name="T"></typeparam>
public bool DeleteTable <T>() where T : Table
{
Type type = typeof(T);
if (!this.tableBlockDict.ContainsKey(type))
{
return(false);
}
// 删除表、索引和数据。
FileStream fs = this.fileStream;
Transaction ts = this.transaction;
ts.Begin();
try
{
TableBlock table = this.tableBlockDict[type];
ts.Delete(table);
long currentIndexPos = table.IndexBlockHeadPos;
IndexBlock IndexHead = fs.ReadBlock <IndexBlock>(currentIndexPos);// 此时指向IndexBlock头结点
ts.Delete(IndexHead);
// 删除数据块。
{
IndexHead.TryLoadNextObj(fs);
IndexBlock currentIndex = IndexHead.NextObj;// 此时指向PK
ts.Delete(currentIndex);
DeleteDataBlocks(currentIndex, fs, ts);
}
// 删除索引块和skip list node块。
{
IndexBlock currentIndex = IndexHead;
while (currentIndex.NextPos != 0)
{
currentIndex.TryLoadNextObj(fs);
ts.Delete(currentIndex.NextObj);
DeleteSkipListNodes(currentIndex, fs, ts);
currentIndex = currentIndex.NextObj;
}
}
this.tableBlockDict.Remove(type);
this.tableIndexBlockDict.Remove(type);
ts.Commit();
return(true);
}
catch (Exception ex)
{
ts.Rollback();
throw ex;
}
}
public void Read()
{
using (FileStream stream = new FileStream(BaseFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
using (var br = new BinaryReader(stream))
{
br.BaseStream.Position = 0;
//((Length - footer) / (chunk + toc + block_hash)) + 1
int blockcount = (int)((br.BaseStream.Length - 36) / (CHUNK_SIZE + 16 + 8)) + 1;
for (int i = 0; i < blockcount; i++)
{
long start = br.BaseStream.Position;
var block = new IndexBlock();
while ((br.BaseStream.Position - start) + (16 + 4 + 2 + 4) <= CHUNK_SIZE)
{
IndexEntry entry = new IndexEntry()
{
EKey = new MD5Hash(br),
Size = br.ReadUInt32BE(),
ArchiveIndex = br.ReadUInt16BE(),
Offset = br.ReadUInt32BE()
};
if (!entry.EKey.IsEmpty)
block.Entries.Add(entry);
}
Blocks.Add(block);
if (br.BaseStream.Position % CHUNK_SIZE != 0)
br.BaseStream.Position += CHUNK_SIZE - ((br.BaseStream.Position - start) % CHUNK_SIZE);
}
//TOC last entry hashes
br.BaseStream.Position += Blocks.Count * 16;
//Block hashes - lower_md5 all blocks except last
br.BaseStream.Position += (Blocks.Count - 1) * 8;
Footer = new IndexFooter()
{
TOCHash = br.ReadBytes(8),
Version = br.ReadByte(),
_11 = br.ReadByte(),
_12 = br.ReadByte(),
BlockSizeKb = br.ReadByte(),
Offset = br.ReadByte(),
Size = br.ReadByte(),
KeySize = br.ReadByte(),
ChecksumSize = br.ReadByte(),
EntryCount = br.ReadUInt32(),
};
Footer.FooterMD5 = br.ReadBytes(Footer.ChecksumSize);
}
}
public static void Delete(this IndexBlock indexBlock, Table record, FileDBContext db)
{
Type type = record.GetType();
PropertyInfo property = type.GetProperty(indexBlock.BindMember);
TableIndexAttribute attr = property.GetCustomAttribute <TableIndexAttribute>();
if (attr == null)
{
throw new Exception(string.Format("No TableIndexAttribute binded!"));
}
FileStream fs = db.fileStream;
// 准备Key。
var key = property.GetValue(record) as IComparable;
SkipListNodeBlock[] rightNodes = FindRightMostNodes(key, indexBlock, db);
IComparable rightKey = db.GetRightObjKey(fs, indexBlock, rightNodes[0]);
// See if we actually found the node
if ((rightNodes[0].RightObj != indexBlock.SkipListTailNode) && (rightKey.CompareTo(key) == 0))
{
for (int i = 0; i <= indexBlock.CurrentLevel; i++)
{
// Since the node is consecutive levels, as soon as we don't find it on the next
// level, we can stop.
if (rightNodes[i].RightObj == indexBlock.SkipListTailNode)
{
break;
}
rightNodes[i].RightObj.TryLoadProperties(fs, SkipListNodeBlockLoadOptions.Key);
rightKey = rightNodes[i].RightObj.Key.GetObject <IComparable>(fs);
if (rightKey.CompareTo(key) != 0)
{
break;
} // 要删除的结点的高度比CurrentLevel低，所以会走到这里。
rightNodes[i].RightObj.TryLoadProperties(fs, SkipListNodeBlockLoadOptions.RightObj);
db.transaction.Add(rightNodes[i]);
db.transaction.Delete(rightNodes[i].RightObj);
rightNodes[i].RightObj = rightNodes[i].RightObj.RightObj;
}
//return true;
}
else
{
//return false;
}
// //itemsCount++;// 有的在内存，有的在文件，因此itemsCount不好使了。
}
public void Open(long startPointer)
{
lock (this) {
// Set up the start area
startArea = Store.GetArea(startPointer);
int magic = startArea.ReadInt32();
if (magic != Magic)
{
throw new IOException("Magic value for index set does not match.");
}
int version = startArea.ReadInt32();
if (version != 1)
{
throw new IOException("Unknown version for index set.");
}
// Setup the index_header area
indexHeaderPointer = startArea.ReadInt64();
indexHeaderArea = Store.GetArea(indexHeaderPointer);
// Read the index header area
version = indexHeaderArea.ReadInt32(); // version
if (version != 1)
{
throw new IOException("Incorrect version");
}
indexHeaderArea.ReadInt32(); // reserved
int indexCount = (int)indexHeaderArea.ReadInt64();
indexBlocks = new IndexBlock[indexCount];
// Initialize each index block
for (int i = 0; i < indexCount; ++i)
{
int type = indexHeaderArea.ReadInt32();
int blockSize = indexHeaderArea.ReadInt32();
long indexBlockPointer = indexHeaderArea.ReadInt64();
if (type != 1)
{
throw new IOException("Do not understand index type: " + type);
}
indexBlocks[i] = new IndexBlock(this, i, blockSize, indexBlockPointer);
indexBlocks[i].AddReference();
}
}
}
private static void DrawDataBlocks(FileDBContext db, FileStream fs, Bitmap[] bmps)
{
TableBlock tableBlockHead = db.GetTableBlockHeadNode();
TableBlock previousTable = tableBlockHead;
while (previousTable.NextPos != 0)
{
TableBlock table = previousTable.NextObj;
IndexBlock currentIndex = table.IndexBlockHead;
while (currentIndex.NextObj != null)
{
SkipListNodeBlock currentNode = currentIndex.NextObj.SkipListHeadNodes[0];
while (currentNode != null)
{
if (currentNode.KeyPos != 0)
{
currentNode.TryLoadProperties(fs, SkipListNodeBlockLoadOptions.Key | SkipListNodeBlockLoadOptions.Value);
{
long index = currentNode.Key.ThisPos.PagePos() / Consts.pageSize;
Bitmap bmp = bmps[index];
Graphics g = Graphics.FromImage(bmp);
int startPos = (int)(currentNode.Key.ThisPos - Consts.pageSize * index);
int length = currentNode.Key.ToBytes().Length;
Brush brush = new LinearGradientBrush(new Point(startPos, 0), new Point(startPos + length, 0), Color.DarkGoldenrod, Color.LightGoldenrodYellow);
g.FillRectangle(brush, startPos, 1, length, bmpHeight - 1);
g.DrawRectangle(boundPen, startPos, 1, length, bmpHeight - 1);
g.Dispose();
}
foreach (var dataBlock in currentNode.Value)
{
long index = dataBlock.ThisPos.PagePos() / Consts.pageSize;
Bitmap bmp = bmps[index];
Graphics g = Graphics.FromImage(bmp);
int startPos = (int)(dataBlock.ThisPos - Consts.pageSize * index);
int length = dataBlock.ToBytes().Length;
Brush brush = new LinearGradientBrush(new Point(startPos, 0), new Point(startPos + length, 0), Color.DarkGoldenrod, Color.LightGoldenrodYellow);
g.FillRectangle(brush, startPos, 1, length, bmpHeight - 1);
g.DrawRectangle(boundPen, startPos, 1, length, bmpHeight - 1);
g.Dispose();
}
}
currentNode.TryLoadProperties(fs, SkipListNodeBlockLoadOptions.RightObj);
currentNode = currentNode.RightObj;
}
currentIndex = currentIndex.NextObj;
}
previousTable = table;
}
}
/// <summary>
/// 删除数据库内的一条记录。
/// </summary>
/// <param name="record"></param>
public bool Delete(Table record)
{
if (record.Id == null)
{
throw new Exception(string.Format("[{0}] is a new record!", record));
}
Type type = record.GetType();
if (!this.tableBlockDict.ContainsKey(type))// 添加表和索引数据。
{
throw new Exception(string.Format("No Table for type [{0}] is set!", type));
}
Transaction ts = this.transaction;
ts.Begin();
try
{
// 删除record。
IndexBlock indexBlock = this.tableIndexBlockDict[type][Consts.TableIdString];
SkipListNodeBlock downNode = FindSkipListNode(fileStream, indexBlock, record.Id);
if (downNode == null)// 此记录根本不存在或已经被删除过一次了。
{
throw new Exception(string.Format("no data blocks for [{0}]", record));
}
foreach (KeyValuePair <string, IndexBlock> item in this.tableIndexBlockDict[type])
{
item.Value.Delete(record, this);
}
downNode.TryLoadProperties(fileStream, SkipListNodeBlockLoadOptions.Key | SkipListNodeBlockLoadOptions.Value);
for (int i = 0; i < downNode.Value.Length; i++)
{
this.transaction.Delete(downNode.Value[i]);
} // 加入事务，准备写入数据库。
this.transaction.Delete(downNode.Key); // 加入事务，准备写入数据库。
this.transaction.Commit();
return(true);
}
catch (Exception ex)
{
ts.Rollback();
throw ex;
}
}
private static void GetSkipListInfo(IndexBlock indexBlock, FileDBContext db, out List <List <SkipListNodeBlock> > skipList, out Dictionary <long, DataBlock> keyDict, out Dictionary <long, DataBlock> valueDict, out Dictionary <long, SkipListNodeBlock> skipListNodeDict)
{
FileStream fs = db.GetFileStream();
skipList = new List <List <SkipListNodeBlock> >();
keyDict = new Dictionary <long, DataBlock>();
valueDict = new Dictionary <long, DataBlock>();
skipListNodeDict = new Dictionary <long, SkipListNodeBlock>();
long headNodePos = indexBlock.SkipListHeadNodePos;
long currentHeadNodePos = headNodePos;
while (currentHeadNodePos != 0)
{
List <SkipListNodeBlock> line = new List <SkipListNodeBlock>();
long currentNodePos = currentHeadNodePos;
while (currentNodePos != 0)
{
SkipListNodeBlock currentNode = fs.ReadBlock <SkipListNodeBlock>(currentNodePos);
line.Add(currentNode);
if (!skipListNodeDict.ContainsKey(currentNode.ThisPos))
{
skipListNodeDict.Add(currentNode.ThisPos, currentNode);
}
if (currentNode.KeyPos != 0)
{
if (!keyDict.ContainsKey(currentNode.KeyPos))
{
DataBlock keyBlock = fs.ReadBlock <DataBlock>(currentNode.KeyPos);
keyDict.Add(currentNode.KeyPos, keyBlock);
}
}
if (currentNode.ValuePos != 0)
{
if (!valueDict.ContainsKey(currentNode.ValuePos))
{
DataBlock valueBlock = fs.ReadBlock <DataBlock>(currentNode.ValuePos);
valueDict.Add(currentNode.ValuePos, valueBlock);
}
}
currentNodePos = currentNode.RightPos;
}
skipList.Add(line);
currentHeadNodePos = skipListNodeDict[currentHeadNodePos].DownPos;
}
}
private static void PrintSkipLists(FileDBContext db, StringBuilder builder, FileStream fs)
{
TableBlock tableHead = db.GetTableBlockHeadNode();// fs.ReadBlock<TableBlock>(fs.Position);
TableBlock currentTableBlock = tableHead;
while (currentTableBlock.NextPos != 0)// 依次Print各个表
{
TableBlock tableBlock = fs.ReadBlock <TableBlock>(currentTableBlock.NextPos);
builder.Append("Table:"); builder.AppendLine(tableBlock.TableType.ToString());
IndexBlock indexBlockHead = fs.ReadBlock <IndexBlock>(tableBlock.IndexBlockHeadPos);
IndexBlock currentIndexBlock = indexBlockHead;
IndexBlock indexBlock = fs.ReadBlock <IndexBlock>(currentIndexBlock.NextPos);
SkipListNodeBlock currentHeadNode = fs.ReadBlock <SkipListNodeBlock>(indexBlock.SkipListHeadNodePos);
int level = db.GetDBHeaderBlock().MaxLevelOfSkipList - 1;
SkipListNodeBlock current = currentHeadNode;
while (current != null)// 依次Print表的PK Index
{
StringBuilder levelBuilder = new StringBuilder();
levelBuilder.AppendLine(string.Format("level {0}", level--));
string str = Print(current);
levelBuilder.AppendLine(str);
int count = 1;
while (current.RightPos != 0)//依次Print PK Index的Level
{
current = fs.ReadBlock <SkipListNodeBlock>(current.RightPos);
str = Print(current);
levelBuilder.AppendLine(str);
count++;
}
if (count > 2)
{
builder.AppendLine(levelBuilder.ToString());
}
if (currentHeadNode.DownPos != 0)
{
currentHeadNode = fs.ReadBlock <SkipListNodeBlock>(currentHeadNode.DownPos);
}
else
{
currentHeadNode = null;
}
current = currentHeadNode;
}
currentTableBlock = tableBlock;
}
}
/// <summary>
/// 把指定的表包含的所有索引的所有skip list node和key value信息都画到图片上。
/// </summary>
/// <param name="nextTable"></param>
/// <param name="dir"></param>
/// <param name="db"></param>
private static void PrintTableBlock(TableBlock table, string dir, string prefix, FileDBContext db)
{
FileStream fs = db.GetFileStream();
long indexBlockHeadPos = table.IndexBlockHeadPos;
IndexBlock indexBlockHead = fs.ReadBlock <IndexBlock>(indexBlockHeadPos);
IndexBlock previousIndex = indexBlockHead;
while (previousIndex.NextPos != 0)
{
previousIndex.TryLoadNextObj(fs);
IndexBlock index = previousIndex.NextObj;
PrintIndexBlock(table, index, dir, prefix, db);
previousIndex = index;
}
}
/// <summary>
/// 标记要删除的skip list node块。
/// </summary>
/// <param name="currentIndex"></param>
/// <param name="fs"></param>
/// <param name="ts"></param>
private void DeleteSkipListNodes(IndexBlock currentIndex, FileStream fs, Transaction ts)
{
foreach (SkipListNodeBlock levelHead in currentIndex.SkipListHeadNodes)
{
SkipListNodeBlock current = levelHead;
while (current.ThisPos != currentIndex.SkipListTailNodePos)
{
ts.Delete(levelHead);
current.TryLoadProperties(fs, SkipListNodeBlockLoadOptions.RightObj);
current = levelHead.RightObj;
}
}
ts.Delete(currentIndex.SkipListTailNode);
}
private static void PrintIndexBlock(TableBlock table, IndexBlock index, string dir, string prefix, FileDBContext db)
{
List <List <SkipListNodeBlock> > skipList;
Dictionary <long, DataBlock> keyDict;
Dictionary <long, DataBlock> valueDict;
Dictionary <long, SkipListNodeBlock> skipListNodeDict;
GetSkipListInfo(index, db, out skipList, out keyDict, out valueDict, out skipListNodeDict);
if (keyDict.Count != valueDict.Count)
{
throw new Exception();
}
SaveToImage(table, index, skipList, keyDict, valueDict, skipListNodeDict, dir, prefix, db);
}
internal IComparable GetRightObjKey(FileStream fileStream, IndexBlock indexBlock, SkipListNodeBlock currentNode)
{
IComparable rightKey = null;
if (currentNode.RightPos == indexBlock.SkipListTailNodePos)
{
currentNode.RightObj = indexBlock.SkipListTailNode;
}
else
{
currentNode.TryLoadProperties(fileStream, SkipListNodeBlockLoadOptions.RightObj);
currentNode.RightObj.TryLoadProperties(fileStream, SkipListNodeBlockLoadOptions.Key);
rightKey = currentNode.RightObj.Key.GetObject <IComparable>(fileStream);
}
return(rightKey);
}
private static int PickRandomLevel(FileDBContext db, IndexBlock indexBlock)
{
int randomLevel = 0;
int maxLevel = db.headerBlock.MaxLevelOfSkipList;
double probability = db.headerBlock.ProbabilityOfSkipList;
while ((rand.NextDouble() < probability)
//&& (randomLevel <= indexBlock.CurrentLevel + 1)
&& (randomLevel < maxLevel))
{
randomLevel++;
}
return(randomLevel);
}
/// <summary>
/// 标记要删除的数据块。
/// </summary>
/// <param name="currentIndex"></param>
/// <param name="fs"></param>
/// <param name="ts"></param>
private void DeleteDataBlocks(IndexBlock currentIndex, FileStream fs, Transaction ts)
{
SkipListNodeBlock current = currentIndex.SkipListHeadNodes[0];
while (current.RightPos != currentIndex.SkipListTailNodePos)
{
current.TryLoadProperties(fs, SkipListNodeBlockLoadOptions.RightObj);
current.RightObj.TryLoadProperties(fs, SkipListNodeBlockLoadOptions.Key | SkipListNodeBlockLoadOptions.Value);
ts.Delete(current.RightObj.Key);
foreach (var item in current.RightObj.Value)
{
ts.Delete(item);
}
current = current.RightObj;
}
}
//static readonly Brush indexBrush = new LinearGradientBrush(new Rectangle(0, 0, Consts.indexBlockLength, bmpHeight), Color.DarkGoldenrod, Color.LightGoldenrodYellow, 0f);
private static void DrawIndex(FileDBContext db, Bitmap[] bmps)
{
TableBlock tableBlockHead = db.GetTableBlockHeadNode();
TableBlock previousTable = tableBlockHead;
while (previousTable.NextPos != 0)
{
//previousTable.TryLoadNextObj(fs);
TableBlock table = previousTable.NextObj;
IndexBlock currentIndex = table.IndexBlockHead;
while (currentIndex != null)
{
long index = currentIndex.ThisPos.PagePos() / Consts.pageSize;
Bitmap bmp = bmps[index];
Graphics g = Graphics.FromImage(bmp);
int startPos = (int)(currentIndex.ThisPos - Consts.pageSize * index);
int length = currentIndex.ToBytes().Length;
Brush brush = new LinearGradientBrush(new Point(startPos, 0), new Point(startPos + length, 0), Color.DarkGray, Color.LightGray);
g.FillRectangle(brush, startPos, 1, length, bmpHeight - 1);
g.DrawRectangle(boundPen, startPos, 1, length, bmpHeight - 1);
g.Dispose();
currentIndex.TryLoadNextObj(db.GetFileStream());
currentIndex = currentIndex.NextObj;
}
//while (currentIndex.NextPos != 0)
//{
// long index = currentIndex.NextObj.ThisPos.PagePos() / Consts.pageSize;
// Bitmap bmp = bmps[index];
// Graphics g = Graphics.FromImage(bmp);
// int startPos = (int)(currentIndex.NextObj.ThisPos - Consts.pageSize * index);
// int length = currentIndex.NextObj.ToBytes().Length;
// Brush brush = new LinearGradientBrush(new Point(startPos, 0), new Point(startPos + length, 0), Color.DarkGray, Color.LightGray);
// g.FillRectangle(brush, startPos, 1, length, bmpHeight - 1);
// g.DrawRectangle(boundPen, startPos, 1, length, bmpHeight - 1);
// g.Dispose();
// currentIndex = currentIndex.NextObj;
//}
previousTable = table;
}
}
public void CommitDropIndex(int indexNum)
{
lock (this) {
// The IndexBlock we are dropping
var curIndexBlock = indexBlocks[indexNum];
int blockSize = curIndexBlock.BlockSize;
try {
Store.Lock();
// Add all the elements to the deleted areas in the block
var allBlockPointers = curIndexBlock.GetBlockPointers();
foreach (long area in allBlockPointers)
{
curIndexBlock.AddDeletedArea(area);
}
// Mark the current block as deleted
curIndexBlock.MarkAsDeleted();
// Make up a new blank block list for this index set.
long blockP = CreateNewBlock();
// Now create a new IndexBlock object
var newIndexBlock = new IndexBlock(this, indexNum, blockSize, blockP);
// Add reference to the new one
newIndexBlock.AddReference();
// Remove reference to the old
curIndexBlock.RemoveReference();
// Update the index_blocks list
indexBlocks[indexNum] = newIndexBlock;
// Commit the new index header (index_blocks)
CommitIndexHeader();
} finally {
Store.Unlock();
}
}
}
/// <summary>
/// 查找具有指定的key值的结点列的最下方的结点。
/// </summary>
/// <param name="fileStream"></param>
/// <param name="key"></param>
/// <param name="indexBlock"></param>
/// <returns></returns>
//private SkipListNodeBlock FindSkipListNode(FileStream fileStream, IComparable key, IndexBlock indexBlock)
private SkipListNodeBlock FindSkipListNode(FileStream fileStream, IndexBlock indexBlock, IComparable key)
{
// Start at the top list header node
SkipListNodeBlock currentNode = indexBlock.SkipListHeadNodes[indexBlock.CurrentLevel];
IComparable rightKey = null;
while (true)
{
rightKey = GetRightObjKey(fileStream, indexBlock, currentNode);
while ((currentNode.RightObj != indexBlock.SkipListTailNode) && (rightKey.CompareTo(key) < 0))
{
currentNode = currentNode.RightObj;
rightKey = GetRightObjKey(fileStream, indexBlock, currentNode);
}
// Check if there is a next level, and if there is move down.
if (currentNode.DownPos == 0)
{
break;
}
else
{
currentNode.TryLoadProperties(fileStream, SkipListNodeBlockLoadOptions.DownObj);
currentNode = currentNode.DownObj;
}
}
// Do one final comparison to see if the key to the right equals this key.
// If it doesn't match, it would be bigger than this key.
if (rightKey.CompareTo(key) == 0)
{
return(currentNode.RightObj);
}
else
{
return(null);
}
}
public virtual void ReadChildData(BinaryReader reader)
{
int x = 0;
for (x = 0; (x < _indexBlock.Count); x = (x + 1))
{
IndexBlock.Add(new IndexBlockHandcodedBlock());
IndexBlock[x].Read(reader);
}
for (x = 0; (x < _indexBlock.Count); x = (x + 1))
{
IndexBlock[x].ReadChildData(reader);
}
for (x = 0; (x < _byteBlock.Count); x = (x + 1))
{
ByteBlock.Add(new ByteBlockHandcodedBlock());
ByteBlock[x].Read(reader);
}
for (x = 0; (x < _byteBlock.Count); x = (x + 1))
{
ByteBlock[x].ReadChildData(reader);
}
}
/// <summary>
/// 查找数据库内所有指定类型的记录。
/// </summary>
/// <typeparam name="T">要查找的类型。</typeparam>
/// <returns></returns>
public IEnumerable <T> FindAll <T>() where T : Table, new()
{
Type type = typeof(T);
if (this.tableBlockDict.ContainsKey(type))
{
TableBlock tableBlock = this.tableBlockDict[type];
IndexBlock firstIndex = tableBlock.IndexBlockHead.NextObj;// 第一个索引应该是Table.Id的索引。
FileStream fs = this.fileStream;
SkipListNodeBlock current = firstIndex.SkipListHeadNodes[0]; //currentHeadNode;
while (current.RightPos != firstIndex.SkipListTailNodePos)
{
current.TryLoadProperties(fs, SkipListNodeBlockLoadOptions.RightObj);
current.RightObj.TryLoadProperties(fs, SkipListNodeBlockLoadOptions.RightObj | SkipListNodeBlockLoadOptions.Value);
T item = current.RightObj.Value.GetObject <T>(this);
yield return(item);
current = current.RightObj;
}
}
}
/// <summary>
/// 获取数据库的所有表和表里的所有数据对象。
/// </summary>
/// <param name="db"></param>
/// <returns></returns>
public static SharpFileDBInfo GetDBInfo(this FileDBContext db)
{
SharpFileDBInfo info = new SharpFileDBInfo();
FileStream fs = db.GetFileStream();
//fs.ReadBlock<PageHeaderBlock>(0);
//fs.ReadBlock<DBHeaderBlock>(fs.Position);
//TableBlock tableHead = fs.ReadBlock<TableBlock>(fs.Position);
TableBlock currentTableBlock = db.GetTableBlockHeadNode();// tableHead;
while (currentTableBlock.NextPos != 0)
{
TableBlock tableBlock = fs.ReadBlock <TableBlock>(currentTableBlock.NextPos);
TableInfo tableInfo = new TableInfo(tableBlock);
info.Add(tableInfo);
IndexBlock indexBlockHead = fs.ReadBlock <IndexBlock>(tableBlock.IndexBlockHeadPos);
IndexBlock currentIndexBlock = indexBlockHead;
bool primaryKey = true;
while (currentIndexBlock.NextPos != 0)
{
IndexBlock indexBlock = fs.ReadBlock <IndexBlock>(currentIndexBlock.NextPos);
tableInfo.Add(indexBlock);
if (primaryKey)
{
long currentHeadNodePos = indexBlock.SkipListHeadNodePos;
SkipListNodeBlock currentHeadNode = fs.ReadBlock <SkipListNodeBlock>(currentHeadNodePos);
currentHeadNode.TryLoadProperties(fs, SkipListNodeBlockLoadOptions.DownObj);
while (currentHeadNode.DownObj != null)
{
currentHeadNode.DownObj.TryLoadProperties(fs, SkipListNodeBlockLoadOptions.DownObj);
currentHeadNode = currentHeadNode.DownObj;
}
SkipListNodeBlock current = currentHeadNode;
current.TryLoadProperties(fs, SkipListNodeBlockLoadOptions.RightObj);
while (current.RightObj != null && current.RightObj.ThisPos != indexBlock.SkipListTailNodePos)
{
current.RightObj.TryLoadProperties(fs, SkipListNodeBlockLoadOptions.Value);
Table item = current.RightObj.Value.GetObject <Table>(db);
tableInfo.Add(item);
current = current.RightObj;
current.TryLoadProperties(fs, SkipListNodeBlockLoadOptions.RightObj);
}
//while (current.RightObj.RightPos != 0)
//{
// current.RightObj.TryLoadProperties(fs, SkipListNodeBlockLoadOptions.Value);
// Table item = current.RightObj.Value.GetObject<Table>(db);
// tableInfo.Add(item);
// current = current.RightObj;
//}
primaryKey = false;
}
currentIndexBlock = indexBlock;
}
currentTableBlock = tableBlock;
}
return(info);
}
private SkipListNodeBlock[] GetHeadNodesOfSkipListNodeBlock(FileStream fileStream, IndexBlock indexBlock)
{
SkipListNodeBlock[] headNodes = new SkipListNodeBlock[this.headerBlock.MaxLevelOfSkipList];
long currentSkipListNodeBlockPos = indexBlock.SkipListHeadNodePos;
for (int i = headNodes.Length - 1; i >= 0; i--)
{
if (currentSkipListNodeBlockPos == 0)
{
throw new Exception(string.Format("max level [{0}] != real max level [{1}]", headNodes.Length, headNodes.Length - 1 - i));
}
SkipListNodeBlock skipListNodeBlock = fileStream.ReadBlock <SkipListNodeBlock>(currentSkipListNodeBlockPos);
headNodes[i] = skipListNodeBlock;
if (i != headNodes.Length - 1)
{
headNodes[i + 1].DownObj = headNodes[i];
}
currentSkipListNodeBlockPos = skipListNodeBlock.DownPos;
}
return(headNodes);
}
private SkipListNodeBlock GetTailNodeOfSkipListNodeBlock(FileStream fileStream, IndexBlock indexBlock)
{
long currentSkipListNodeBlockPos = indexBlock.SkipListTailNodePos;
if (currentSkipListNodeBlockPos == 0)
{
throw new Exception(string.Format("tail node not set for [{0}]", indexBlock));
}
SkipListNodeBlock tailNode = fileStream.ReadBlock <SkipListNodeBlock>(currentSkipListNodeBlockPos);
return(tailNode);
}
public static void Insert(this IndexBlock indexBlock, Table item, DataBlock[] dataBlocksForValue, FileDBContext db)
{
Type type = item.GetType();
PropertyInfo property = type.GetProperty(indexBlock.BindMember);
TableIndexAttribute attr = property.GetCustomAttribute <TableIndexAttribute>();
if (attr == null)
{
throw new Exception(string.Format("No TableIndexAttribute binded!"));
}
// 准备Key。
IComparable key = property.GetValue(item) as IComparable;
byte[] keyBytes = key.ToBytes();
if (keyBytes.Length > Consts.maxDataBytes)
{
throw new Exception(string.Format("Toooo long is the key [{0}]", key));
}
DataBlock dataBlockForKey = new DataBlock()
{
ObjectLength = keyBytes.Length, Data = keyBytes,
};
SkipListNodeBlock[] rightNodes = FindRightMostNodes(key, indexBlock, db);
// Check if the item allready exists in the list. If it does, throw an exception -
// we will not allow duplicate items here.
FileStream fs = db.fileStream;
int maxLevel = db.headerBlock.MaxLevelOfSkipList;
IComparable rightKey = db.GetRightObjKey(fs, indexBlock, rightNodes[0]);
if ((rightNodes[0].RightObj != indexBlock.SkipListTailNode) &&
(rightKey.CompareTo(key) == 0)) // key相等，说明Value相同。此处不再使用NGenerics的Comparer<TKey>.Default这种可指定外部比较工具的模式，是因为那会由于忘记编写合适的比较工具而带来隐藏地很深的bug。
{
throw new Exception("Item Already In List");
}
else
{
int newLevel = PickRandomLevel(db, indexBlock);
if (newLevel > indexBlock.CurrentLevel)
{
for (int i = indexBlock.CurrentLevel + 1; i <= newLevel; i++)
{
rightNodes[i] = indexBlock.SkipListHeadNodes[i];
}
indexBlock.CurrentLevel = newLevel;
}
List <SkipListNodeBlock> nodeList = new List <SkipListNodeBlock>();
SkipListNodeBlock newNode = new SkipListNodeBlock()
{
Key = dataBlockForKey, Value = dataBlocksForValue,
};
nodeList.Add(newNode);
// Insert the item in the first level
newNode.RightObj = rightNodes[0].RightObj;
rightNodes[0].RightObj = newNode;
// And now insert the node in the rest of the levels, making sure
// to update the the links
SkipListNodeBlock previousNode = null;
for (int i = 1; i <= indexBlock.CurrentLevel; i++)
{
previousNode = newNode;
newNode = new SkipListNodeBlock()
{
Key = dataBlockForKey, Value = dataBlocksForValue,
};
nodeList.Add(newNode);
newNode.RightObj = rightNodes[i].RightObj;
rightNodes[i].RightObj = newNode;
newNode.DownObj = previousNode;
}
for (int i = nodeList.Count - 1; i >= 0; i--)
{
db.transaction.Add(rightNodes[i]);
} // 加入事务，准备写入数据库。
for (int i = nodeList.Count - 1; i >= 0; i--)
{
db.transaction.Add(nodeList[i]);
} // 加入事务，准备写入数据库。
db.transaction.Add(dataBlockForKey); // 加入事务，准备写入数据库。
}
//itemsCount++;// 有的在内存，有的在文件，因此itemsCount不好使了。
}
public string Write()
{
using (var md5 = MD5.Create())
using (var ms = new MemoryStream())
using (var bw = new BinaryWriter(ms))
{
IndexBlock lastBlock = Blocks[Blocks.Count - 1];
List <byte[]> blockhashes = new List <byte[]>();
//Blocks
foreach (var block in Blocks)
{
//Entries
long posBlockStart = bw.BaseStream.Position;
block.Entries.Sort(new HashComparer());
foreach (var entry in block.Entries)
{
bw.Write(entry.EKey.Value);
bw.WriteUInt32BE(entry.Size);
bw.WriteUInt16BE(entry.ArchiveIndex);
bw.WriteUInt32BE(entry.Offset);
}
//Pad to CHUNK_SIZE
if (bw.BaseStream.Position % CHUNK_SIZE != 0)
{
bw.Write(new byte[CHUNK_SIZE - ((bw.BaseStream.Position - posBlockStart) % CHUNK_SIZE)]);
}
//Calc block's md5
if (block != lastBlock)
{
blockhashes.Add(GetBlockHash(bw, md5, posBlockStart));
}
}
//TOC
bw.Write(Blocks.SelectMany(x => x.Entries.Last().EKey.Value).ToArray());
//Block hashes
if (Blocks.Count > 1)
{
bw.Write(blockhashes.SelectMany(x => x).ToArray());
blockhashes.Clear();
}
//Footer Start
long posFooterStart = bw.BaseStream.Position;
//Calculate IndexBlockHash
// BOTTOM 8 bytes = BLOCK DATA
bw.BaseStream.Position = CHUNK_SIZE * (Blocks.Count - 1);
byte[] indexBlockHashBytes = new byte[CHUNK_SIZE];
bw.BaseStream.Read(indexBlockHashBytes, 0, indexBlockHashBytes.Length);
bw.BaseStream.Position = posFooterStart;
var lowerHash = md5.ComputeHash(indexBlockHashBytes).Take(8).ToArray();
bw.Write(lowerHash);
// TOP 8 bytes = TOC + lowerHash
bw.BaseStream.Position = CHUNK_SIZE * Blocks.Count;
byte[] tocHashBytes = new byte[8 + posFooterStart - bw.BaseStream.Position];
bw.BaseStream.Read(tocHashBytes, 0, tocHashBytes.Length);
var upperHash = md5.ComputeHash(tocHashBytes).Take(8).ToArray();
bw.Write(upperHash);
//Footer
bw.Write(Footer.Version);
bw.Write(Footer._11);
bw.Write(Footer._12);
bw.Write(Footer._13);
bw.Write(Footer.Offset);
bw.Write(Footer.Size);
bw.Write(Footer.KeySize);
bw.Write(Footer.ChecksumSize);
bw.Write((uint)Blocks.Sum(x => x.Entries.Count));
bw.Write(new byte[Footer.ChecksumSize]); //Footer MD5 hash placeholder
//Update Footer Md5 Hash
bw.BaseStream.Position = posFooterStart + 16;
byte[] footerBytes = new byte[bw.BaseStream.Length - bw.BaseStream.Position]; //From _10 to EOF
bw.BaseStream.Read(footerBytes, 0, footerBytes.Length);
bw.BaseStream.Position = bw.BaseStream.Length - Footer.ChecksumSize; //Update the MD5 hash
var footerHash = md5.ComputeHash(footerBytes).Take(8).ToArray();
bw.Write(footerHash);
//Save file to output
string filename = ComputeFilename(bw, md5, posFooterStart);
var path = Path.Combine(CASContainer.Settings.OutputPath, filename + ".index");
if (CASContainer.Settings.StaticMode)
{
path = Path.Combine(CASContainer.Settings.OutputPath, Helper.GetCDNPath(filename + ".index", "data"));
Directory.CreateDirectory(Path.GetDirectoryName(path));
}
using (var fs = new FileStream(path, FileMode.Create, FileAccess.Write, FileShare.Read))
{
ms.Position = 0;
ms.CopyTo(fs);
fs.Flush();
}
//Update CDN Config
CASContainer.CDNConfig.Set("archive-group", filename);
return(filename);
}
}
