/// <summary> /// Calls ComputeHahses() method of BlockHashFilter class to calculate phone's block hashes, calls GetHashRunId() to insert phone's /// record to the database, calls InsertHahses() of DatabaseAccess to insert the block hashes into the database. /// </summary> /// <param name="file">The BinaryFile reference corresponding to the phone's memory file.</param> /// <param name="blockSize">The size of a memory block on phone.</param> /// <param name="slideAmount">The distance between the starting offsets of two consecutive blocks in memory.</param> /// <returns>The PhoneId of the phone, i.e. the row ID of the phone's record in the database.</returns> public int ProcessFile(BinaryFile file, int blockSize, int slideAmount) { int phoneId = GetPhoneId(file); DateTime start = DateTime.Now; List <string> hashes = BlockHashFilter.ComputeHashes(file.Path, blockSize, slideAmount); DateTime end = DateTime.Now; TimeSpan duration = end - start; long numBlocks = hashes.Count; long timeToHashSeconds = Convert.ToInt32(Math.Round(duration.TotalSeconds)); string hashType = "sha1"; int hashRunId = GetHashRunId(blockSize, phoneId, file.MemoryId, slideAmount, numBlocks, timeToHashSeconds, hashType, ""); #if _USESQLSERVER_ List <HashInfo> hashInfos = new List <HashInfo>(); for (int k = 0; k < hashes.Count; k++) { hashInfos.Add(new HashInfo { BlockIndex = k, Hash = hashes[k], HashRunId = hashRunId }); } BulkInsertBase bulky = BulkInsertBase.Load(hashInfos); bulky.Flush(); #else DatabaseAccess.InsertHashes(hashes, hashRunId); #endif return(phoneId); }
public void GetEmptyHash() { byte[] emptyBytes = new byte[32768]; var hash = BlockHashFilter.GetBlockHash(emptyBytes); var hashString = Convert.ToBase64String(hash); }
// BL 8/4 private Block GetBlock(FileStream stream, long start, long end) { int length = (int)(end - start); var bytes = BlockHashFilter.GetBytes(stream, start, length); return(new Block() { Bytes = bytes, OffsetFile = start }); }
private void Dump_Filtered_Blocks(FilterResult result, string filepath) { FileStream stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read); List <Block> blocks = new List <Block>(result.UnfilteredBlocks); System.IO.FileStream FileStream = new System.IO.FileStream("blocks_left", System.IO.FileMode.Create, System.IO.FileAccess.Write); for (int i = 0; i < blocks.Count; i++) { Block b = blocks[i]; var bytes = BlockHashFilter.GetBytes(stream, b.OffsetFile, b.Length); FileStream.Write(bytes, 0, bytes.Length); } FileStream.Close(); }
private FilterResult RunBlockHashFilter(string fileSha1, int phoneId, List <BlockHashFilter.UnfilterdBlockResult> storedBlocks) { FilterResult filterResult = null; try { write("Starting block hash filtering"); DateTime dt = DateTime.Now; BlockHashFilter bhf = new BlockHashFilter(this.filePath, HashLoader.HashLoader.DefaultBlockSize, HashLoader.HashLoader.DefaultSlideAmount, fileSha1, true, storedBlocks); #if _USESQLSERVER_ filterResult = bhf.Filter(phoneId); #else SQLiteConnection sql = DatabaseAccess.OpenSql(true); try { filterResult = bhf.Filter(phoneId, sql, true); } finally { sql.Close(); sql.Dispose(); } #endif TimeSpan ts = DateTime.Now.Subtract(dt); write("Time elapsed for block hash filtering: {0}", ts.ToString("c")); long nBytes = filterResult.FilteredBytesCount + filterResult.UnfilteredBytesCount; write("Filtered {0} out of {1} bytes: {2:0.00###}", filterResult.FilteredBytesCount, nBytes, (nBytes - filterResult.UnfilteredBytesCount) / (double)nBytes); } catch (ThreadAbortException) { return(null); } catch (Exception ex) { DisplayExceptionMessages(ex, "Block Hash Filter"); return(null); } return(filterResult); }
/// <summary> /// Runs the Viterbi for each input block using Run(byte[], long) method. /// </summary> /// <param name="blocks">List of blocks for which Viterbi inference is to be done.</param> /// <param name="inputFile">Path to the phone's memory file.</param> /// <returns>The list of states in the inferred Viterbi path.</returns> public ViterbiResult Run(List <Block> blocks, string inputFile) { var start = DateTime.Now; FileStream stream = new FileStream(inputFile, FileMode.Open, FileAccess.Read, FileShare.Read); // BL 8/4 for (int i = 0; i < blocks.Count; i++) { //Let's get the block bytes here. They should be null if this is a field level run of viterbi. For record-level runs, //they should not be null. //var bytes = blocks[i].Bytes ?? BlockHashFilter.GetBytes(inputFile, blocks[i].OffsetFile, blocks[i].Length); var bytes = blocks[i].Bytes ?? BlockHashFilter.GetBytes(stream, blocks[i].OffsetFile, blocks[i].Length); // BL 8/4 Run(bytes, blocks[i].OffsetFile); } stream.Close(); // BL 8/4 return(new ViterbiResult { Fields = _fieldList, Duration = DateTime.Now - start }); }
/// <summary> /// Calls the Filter() method of BlockHashFilter class for calculating the phone's block hashes. /// </summary> /// <param name="inputFile">The path to phone's memory file.</param> /// <param name="blocksize">The size of a memory block on phone.</param> /// <param name="slideAmount">The distance between the starting offsets of two consecutive blocks in memory.</param> /// <param name="nextStreamPosition">The starting position in memory of the block to be hashed.</param> /// <returns>A list of block hashes for the phone.</returns> public List <string> ComputeHashes(string inputFile, int blocksize, int slideAmount, ref long nextStreamPosition) { List <string> hashes = new List <string>(); int countBlocks = 0; using (FileStream stream = File.Open(inputFile, FileMode.Open, FileAccess.Read, FileShare.Read)) { var block = new byte[blocksize]; //Start where we left off. Should be zero for the first run of the method. stream.Position = nextStreamPosition; while (stream.Position < stream.Length && countBlocks < BLOCK_BUFFER) { stream.Read(block, 0, blocksize); byte[] hash = BlockHashFilter.GetBlockHash(block); string hashString = Convert.ToBase64String(hash); hashes.Add(hashString); //Need this check. Otherwise it will not finish if (stream.Position < stream.Length) { stream.Position = stream.Position - blocksize + slideAmount; } countBlocks++; nextStreamPosition = stream.Position; } } return(hashes); }
public static void GetConstants() { int[] sizes = new int[] { 1, 4, 16 };//{32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768}; using (PhoneDbDataContext dataContext = Dalbase.GetDataContext()) { for (int i = 0; i < sizes.Length; i++) { for (int j = 0; j < 256; j++) { byte value = (byte)j; byte[] bytes = (new byte[sizes[i]].Select(r => value)).ToArray(); var hash = BlockHashFilter.GetBlockHash(bytes); var hashString = Convert.ToBase64String(hash); var valString = "0x" + Convert.ToString(value, 16).PadLeft(2, '0'); dataContext.usp_Constants_Insert(valString, sizes[i], hashString); } } } }
public FilterResult RunBlockHashFilter() { var filter = new BlockHashFilter(_inputFile, _blockSize, _slideAmount, _memoryId, _noFilter); return(filter.Filter(_phoneId)); }