public FilterResult Filter() { int count = 0; var fileInfo = new FileInfo(_imagePath); var blocks = new List<Block>(); while(count < fileInfo.Length) { int newLength = Math.Min(_blockSize, Convert.ToInt32(fileInfo.Length - count)); var block = new Block() { Length = newLength, OffsetFile = count }; blocks.Add(block); count += newLength; } var result = new FilterResult() {UnfilteredBlocks = blocks}; return result; }
private ViterbiResult RunViterbi(FilterResult filterResult, string fileSha1) { // Load any user-defined state machines. List<UserState> userStates; try { userStates = Loader.LoadUserStates(MainForm.Program.UserStatesEnabled, MainForm.Program.UserStatesPath); } catch (Exception ex) { DisplayExceptionMessages(ex, "User-Defined States"); return null; } ViterbiResult viterbiResultFields = null; try { write("Running Viterbi on fields"); DateTime dt = DateTime.Now; #if _GENERALPARSE Viterbi viterbi = new Viterbi(RunType.GeneralParse, false); viterbiResultFields = viterbi.Run(filterResult.UnfilteredBlocks, this.filePath); #else #if SKIPREALWORK viterbiResultFields = new ViterbiResult(); viterbiResultFields.Fields = new List<ViterbiField>(); #else if (filterResult.UnfilteredBlocks.Count > 0) { ThreadedViterbi tv = new ThreadedViterbi(filterResult.UnfilteredBlocks, RunType.GeneralParse, userStates, this.filePath, this.fileSha1); viterbiResultFields = tv.RunThreadedViterbi(); TimeSpan ts = DateTime.Now.Subtract(dt); write("Time elapsed for Viterbi fields: {0}", ts.ToString("c")); write("Field count: {0}", viterbiResultFields.Fields.Count); } #endif #endif filterResult.UnfilteredBlocks.Clear(); // Allow gc to clean things up } catch (ThreadAbortException) { return null; } catch (Exception ex) { DisplayExceptionMessages(ex, "Viterbi Fields"); return null; } return viterbiResultFields; }
private void Dump_Filtered_Blocks(FilterResult result, string filepath) { FileStream stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read); List<Block> blocks = new List<Block>(result.UnfilteredBlocks); System.IO.FileStream FileStream = new System.IO.FileStream("blocks_left", System.IO.FileMode.Create, System.IO.FileAccess.Write); for (int i = 0; i < blocks.Count; i++) { Block b = blocks[i]; var bytes = BlockHashFilter.GetBytes(stream, b.OffsetFile, b.Length); FileStream.Write(bytes, 0, bytes.Length); } FileStream.Close(); }
/// <summary> /// Attempts to log the filter results from the same directory as the input file /// </summary> /// <returns>Returns true if successful.</returns> private bool LoadFilterResults() { string resfile = _inputFile + ".fil"; if (File.Exists(resfile)) { using (Stream instream = File.OpenRead(resfile)) { BinaryFormatter serializer = new BinaryFormatter(); var filterResult = (FilterResult)serializer.Deserialize(instream); // Check if the results file was created from the same binary input file if (filterResult.MemoryId == _memoryId) _filterResult = filterResult; } } return _filterResult != null; }
/// <summary> /// Performs Block Hash Filtering for the phone by returning those blocks of the phone's memory that did not match the existing block hash library. /// </summary> /// <param name="phoneId">The unique identification for this phone in the database.</param> /// <param name="sqlConnect">Handler for the database that contains the block hashes for this phone.</param> /// <returns>The set of blocks remaining in the phone's memory after block hash filtering is complete.</returns> public FilterResult Filter(int phoneId, object sqlConnect) { var start = DateTime.Now; var blockInfos = new List<BlockInfo>(); var allInterestingBlocks = new List<Block>(); #if _USESQLSERVER_ List<usp_Hash_GetUnfilteredBlocksResult> unfilteredBlocks; using (PhoneDbDataContext dataContext = Dalbase.GetDataContext()) { // BL:TEMP added try catch for debugging try { ISingleResult<usp_Hash_GetUnfilteredBlocksResult> results = dataContext.usp_Hash_GetUnfilteredBlocks(_blockSize, _slideAmount, phoneId, _memoryId, _noFilter, _slideAmountLibrary); unfilteredBlocks = (from result in results select result).ToList(); } catch (Exception ex) { throw ex; } } #else DateTime started = DateTime.Now; // TODO: Clean up and remove unused methods. // Note: Only 7a supports blocks passed directly to us, blocks that are not in the DB. //List<UnfilterdBlockResult> unfilteredBlocks = HashGetUnfilteredBlocks5(phoneId, (SQLiteConnection)sqlConnect); //List<UnfilterdBlockResult> unfilteredBlocks = HashGetUnfilteredBlocks6(phoneId, (SQLiteConnection)sqlConnect); //List<UnfilterdBlockResult> unfilteredBlocks = HashGetUnfilteredBlocks7(phoneId, (SQLiteConnection)sqlConnect); List<UnfilterdBlockResult> unfilteredBlocks = HashGetUnfilteredBlocks7a(phoneId, (SQLiteConnection)sqlConnect); //List<UnfilterdBlockResult> unfilteredBlocks = HashGetUnfilteredBlocks8(phoneId, (SQLiteConnection)sqlConnect); DateTime finished = DateTime.Now; System.Console.WriteLine("Block Hash Filter took " + (finished - started).TotalSeconds + " seconds"); #endif long numBytes = new FileInfo(_inputFile).Length; //This is to cover a weird issue where the memory is not divisible by the block size long numBlocks = 0; if(numBytes < _blockSize) numBlocks = 1; else numBlocks = Convert.ToInt64(Math.Round((numBytes - _blockSize) / (double)_slideAmount + 1, MidpointRounding.AwayFromZero)); int unfilteredIndex = 0; long i = 0; long lastBlockIndex = long.MinValue; while (i < numBlocks && unfilteredIndex < unfilteredBlocks.Count) { //Check if this block was not filtered. if (unfilteredBlocks[unfilteredIndex].blockIndexFirst == i) { #if CHECK_HASH // We need to do a sanity check to make sure the logic works var blockBytes = GetBytes(_inputFile, unfilteredBlocks[unfilteredIndex].blockIndexFirst.Value * _slideAmount, _blockSize); string blockHash = Convert.ToBase64String(_hash.ComputeHash(blockBytes)); bool hashesMatch = (blockHash == unfilteredBlocks[unfilteredIndex].hash); if (!hashesMatch) { //throw new ApplicationException("Wha? Why did the hashes not match?!"); Console.WriteLine("Wha? Why did the hashes not match?!"); } #endif BlockInfo blockInfo = new BlockInfo(); byte[] interestingBytes; //if not the last block, add the first X bytes where X=slide_amount if (unfilteredBlocks[unfilteredIndex].blockIndexFirst < numBlocks - 1) { blockInfo.Offset = unfilteredBlocks[unfilteredIndex].blockIndexFirst.Value*_slideAmount; blockInfo.Length = _slideAmount; } //else add the whole block else { blockInfo.Offset = unfilteredBlocks[unfilteredIndex].blockIndexFirst.Value * _slideAmount; blockInfo.Length = _blockSize; } //Concatenate this block with the previous if they are contiguous and less than max block size if (unfilteredBlocks[unfilteredIndex].blockIndexFirst - 1 == lastBlockIndex && blockInfos[blockInfos.Count-1].Length < MAX_BLOCK_SIZE) { blockInfos[blockInfos.Count - 1].Length += blockInfo.Length; } else { blockInfos.Add(blockInfo); } lastBlockIndex = unfilteredBlocks[unfilteredIndex].blockIndexFirst.Value; unfilteredIndex++; i++; } //Check if this block is filtered else if (i < unfilteredBlocks[unfilteredIndex].blockIndexFirst) { //skip to end of matched block i = i + (_blockSize / _slideAmount); //We do not want to skip past the last block if (i > numBlocks - 1) i = numBlocks - 1; //Skip all unfiltered blocks after the matched block that overlap with the matched block while (unfilteredIndex < unfilteredBlocks.Count && unfilteredBlocks[unfilteredIndex].blockIndexFirst < i) { unfilteredIndex++; } } else if (i > unfilteredBlocks[unfilteredIndex].blockIndexFirst) throw new ApplicationException("This should never happen..."); } long count = 0; //Get the bytes for the blocks for (int j = 0; j < blockInfos.Count; j++) { //This is going to be an issue for the blocks that contain fewer bytes than the block size, e.g. the last block. //The length won't be calculated right in th count += blockInfos[j].Length; //var bytes = GetBytes(_inputFile, blockInfos[j].Offset, blockInfos[j].Length); allInterestingBlocks.Add(new Block {OffsetFile = blockInfos[j].Offset, Length = blockInfos[j].Length}); } //Count the number of bytes remaining /* BL for (int j = 0; j < allInterestingBlocks.Count; j++) { //count += allInterestingBlocks[j].Bytes.Length; } */ _filterResult = new FilterResult { FilteredBytesCount = numBytes - count, UnfilteredBytesCount = count, UnfilteredBlocks = allInterestingBlocks, Duration = DateTime.Now - start, MemoryId = _memoryId }; Console.WriteLine("Filtered {0} out of {1} bytes: {2}.", numBytes - count, numBytes, (numBytes - count) / (double)numBytes); return _filterResult; }