public int LoadFromMem(byte[] Input) { int written = 0; var hashArr = FractHashTree.CreateRecsFromMemory(Input, MinHashSize, GetHP); var Count = hashArr.Length; using (var fs = new FileStream(DBFile, FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite, DB_READ_SIZE)) { // we need 2 pages now since were block reading and we might pick a hash that start's scan // at the very end of a page byte[] buff = new byte[DB_READ_SIZE]; byte[] zero = new byte[HASH_REC_BYTES]; int i = 0, firstIndex = 0, zeroIndex = 0; bool WriteBack = false; do { var Index = hashArr[i].Index; // convert Index to PageIndex var DBPage = (long)((Index & SortMask) & ~DB_PAGE_MASK); // find block offset for this hash fs.Seek(DBPage, SeekOrigin.Begin); fs.Read(buff, 0, DB_READ_SIZE); WriteBack = false; do { // skip duplicates if (i + 1 < Count && hashArr[i].Index == hashArr[i + 1].Index && hashArr[i].CompressedHash == hashArr[i + 1].CompressedHash) { i++; continue; } if (i < Count) { // re-read Inxex since we could be on the inner loop Index = hashArr[i].Index; // Index inside of a page var PageIndex = Index & DB_PAGE_MASK; // Hash to populate the DB with var toWrite = HashRec.ToByteArr(hashArr[i]); // do we already have this hash from disk? firstIndex = buff.SearchBytes(toWrite, (int)PageIndex, toWrite.Length); if (firstIndex < 0) { zeroIndex = buff.SearchBytes(zero, (int)PageIndex, zero.Length); if (zeroIndex >= 0) { // we want the modified buffer to get written back WriteBack = true; int j, k; // update buff with new hash entry for write back //Array.Copy(toWrite, 0, buff, zeroIndex, toWrite.Length); for (j = zeroIndex, k = 0; j < zeroIndex + toWrite.Length; j++, k++) { buff[j] = toWrite[k]; } written++; // set to the origional index, shift down since were bit aligned HDB.SetIdxBit(Index); } else if (zeroIndex < 0) { var strerr = $"HASH TABLE SATURATED! YOU NEED TO MAKE THE DB LARGER!!"; WriteColor(ConsoleColor.Red, strerr); throw new ApplicationException(strerr); } } } i++; // continue to next entry if it's in the same block } while (i < Count && (((hashArr[i].Index & SortMask) & ~DB_PAGE_MASK) == (ulong)DBPage)); if (WriteBack) { // reset seek position fs.Seek(DBPage, SeekOrigin.Begin); // only write back 1 page if we can help it fs.Write(buff, 0, DB_READ_SIZE); } } while (i < Count); } return(written); }
public static HashRec[] CreateRecsFromMemory(byte[] MemPage, int minBlockSize, Func <HashLib.IHash> getHP, int rID = 0, long VA = 0, int OnlySize = 0, bool PreSerialize = false) { if (MemPage == null) { return(null); } int RawSize = MemPage.Length; var topCnt = BlockCount(RawSize, PAGE_SIZE); if (getHP == null) { getHP = new Func <HashLib.IHash>(() => { return(HashLib.HashFactory.Crypto.CreateTiger2()); }); } var levelMap = LevelMaps(RawSize, minBlockSize); int LevelCount = levelMap.Count(); long TotalHashs = levelMap[LevelCount - 1].Item1 + levelMap[LevelCount - 1].Item2; HashLib.IHash[] localHashProv = new HashLib.IHash[LevelCount]; var sHash = new HashRec[TotalHashs]; if (OnlySize != 0) { LevelCount = 1; minBlockSize = OnlySize; TotalHashs = BlockCount(RawSize, minBlockSize); sHash = new HashRec[TotalHashs]; } // smallest to largest orginization for (int i = 0; i < LevelCount; i++) { localHashProv[i] = getHP(); } for (byte lvl = 0; lvl < LevelCount; lvl++) { var blockSize = minBlockSize << lvl; var blockCnt = BlockCount(RawSize, blockSize); var hashLevelIndex = levelMap[lvl].Item1; localHashProv[lvl].Initialize(); for (int arri = 0; arri < blockCnt; arri++) { localHashProv[lvl].TransformBytes(MemPage, arri * blockSize, blockSize); var hashBytes = localHashProv[lvl].TransformFinal().GetBytes(); sHash[hashLevelIndex + arri] = new HashRec(hashBytes, lvl, rID); // trying to reduce some load in the DB commit path if (PreSerialize) { sHash[hashLevelIndex + arri].Serialized = HashRec.ToByteArr(sHash[hashLevelIndex + arri]); } } } return(sHash); }
public static HashRec[] CreateRecsFromMemory(byte[] MemPage, int minBlockSize, Func <HashLib.IHash> getHP, int rID = 0, long VA = 0, int OnlySize = 0, bool PreSerialize = false, bool FullHashes = false) { if (MemPage == null) { return(null); } //var LevelCount = 1; int RawSize = MemPage.Length; var TotalHashs = BlockCount(RawSize, minBlockSize); var sHash = new HashRec[TotalHashs]; if (OnlySize != 0) { minBlockSize = OnlySize; TotalHashs = BlockCount(RawSize, minBlockSize); sHash = new HashRec[TotalHashs]; } //var topCnt = BlockCount(RawSize, PAGE_SIZE); if (getHP == null) { getHP = new Func <HashLib.IHash>(() => { return(HashLib.HashFactory.Crypto.CreateTiger2()); }); } /* * var levelMap = LevelMaps(RawSize, minBlockSize); * int LevelCount = levelMap.Count(); * long TotalHashs = levelMap[LevelCount - 1].Item1 + levelMap[LevelCount - 1].Item2; */ HashLib.IHash localHashProv; // smallest to largest orginization // for (int i = 0; i < LevelCount; i++) localHashProv = getHP(); //for (byte lvl = 0; lvl < LevelCount; lvl++) //{ //var blockSize = minBlockSize << lvl; //var blockCnt = BlockCount(RawSize, OnlySize); //ar hashLevelIndex = levelMap[lvl].Item1; localHashProv.Initialize(); for (int arri = 0; arri < TotalHashs; arri++) { localHashProv.TransformBytes(MemPage, arri * minBlockSize, minBlockSize); var hashBytes = localHashProv.TransformFinal().GetBytes(); sHash[arri] = new HashRec(hashBytes, 0, rID); if (VA != 0) { sHash[arri].Address = VA + (arri * minBlockSize); } //if (!FullHashes) // sHash[arri].FullHash = null; // trying to reduce some load in the DB commit path if (PreSerialize) { sHash[arri].Serialized = HashRec.ToByteArr(sHash[arri]); } } //} return(sHash); }
void DumpBufToDisk(ParallelOptions po) { Stopwatch sw; long TotalDBWrites = 0; long TotalRequested = 0; long DBPage = 0; SortMask = HDB.DBEntriesMask << HASH_SHIFT; do { var hashArrTpl = ReadyQueue.Take(po.CancellationToken); var hashArr = hashArrTpl.Item2; var Count = hashArrTpl.Item1; ParallelAlgorithms.Sort <HashRec>(hashArr, 0, Count, GetICompareer <HashRec>(SortByDBSizeMask)); TotalRequested += Count; if (Vtero.VerboseLevel >= 1) { WriteColor(ConsoleColor.Cyan, $"Hash entries to store: {Count:N0}"); } using (var fs = new FileStream(DBFile, FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite, DB_READ_SIZE)) { // we need 2 pages now since were block reading and we might pick a hash that start's scan // at the very end of a page byte[] buff = new byte[DB_READ_SIZE]; byte[] zero = new byte[HASH_REC_BYTES]; int i = 0, firstIndex = 0, zeroIndex = 0; bool WriteBack = false; sw = Stopwatch.StartNew(); do { var Index = hashArr[i].Index; // convert Index to PageIndex DBPage = (long)((Index & SortMask) & ~DB_PAGE_MASK); // find block offset for this hash fs.Seek(DBPage, SeekOrigin.Begin); fs.Read(buff, 0, DB_READ_SIZE); WriteBack = false; if (po.CancellationToken.IsCancellationRequested) { return; } po.CancellationToken.ThrowIfCancellationRequested(); do { // skip duplicates if (i + 1 < Count && hashArr[i].Index == hashArr[i + 1].Index) //&& UnsafeHelp.UnsafeCompare(hashArr[i].HashData, hashArr[i + 1].HashData)) { i++; continue; } if (i < Count) { // re-read Inxex since we could be on the inner loop Index = hashArr[i].Index; // Index inside of a page var PageIndex = (int)(Index & DB_PAGE_MASK); // Hash to populate the DB with var toWrite = BitConverter.GetBytes(hashArr[i].CompressedHash); // do we already have this hash from disk? firstIndex = buff.SearchBytes(toWrite, PageIndex, HASH_REC_BYTES); if (firstIndex < 0) { zeroIndex = buff.SearchBytes(zero, PageIndex, HASH_REC_BYTES); if (zeroIndex >= 0) { // we want the modified buffer to get written back WriteBack = true; toWrite = HashRec.ToByteArr(hashArr[i]); // update buff with new hash entry for write back //Array.Copy(toWrite, 0, buff, zeroIndex, toWrite.Length); for (int j = zeroIndex, k = 0; j < zeroIndex + toWrite.Length; j++, k++) { buff[j] = toWrite[k]; } TotalDBWrites++; // set to the origional index, shift down since were bit aligned HDB.SetIdxBit(Index); } else if (zeroIndex < 0) { var strerr = $"HASH TABLE SATURATED!!! ({DBPage:X}:{PageIndex:X}) YOU NEED TO MAKE THE DB LARGER!!"; WriteColor(ConsoleColor.Red, strerr); source.Cancel(); } } } i++; if (i % 100000 == 0 && sw.Elapsed.TotalSeconds > 0) { WriteColor(ConsoleColor.Cyan, $"DB commit entries: {i:N0} - per second {(i / sw.Elapsed.TotalSeconds):N0}"); } // continue to next entry if it's in the same block } while (i < Count && (((hashArr[i].Index & SortMask) & ~DB_PAGE_MASK) == (ulong)DBPage)); if (WriteBack) { if (po.CancellationToken.IsCancellationRequested) { return; } // reset seek position fs.Seek(DBPage, SeekOrigin.Begin); // only write back 1 page if we can help it fs.Write(buff, 0, DB_READ_SIZE); } } while (i < Count); WriteColor(ConsoleColor.Cyan, $"DB entries: {i:N0} - per second {(i / sw.Elapsed.TotalSeconds):N0}"); //aPool.Return(hashArr); } } while (!DoneHashLoad || ReadyQueue.Count() > 0); WriteColor(ConsoleColor.Cyan, $"Finished DB write {TotalDBWrites:N0} NEW entries. Requsted {TotalRequested:N0} (reduced count reflects de-duplication). Task time: {sw.Elapsed}"); }