public FilterResult Filter()
{
int count = 0;
var fileInfo = new FileInfo(_imagePath);
var blocks = new List<Block>();
while(count < fileInfo.Length)
{
int newLength = Math.Min(_blockSize, Convert.ToInt32(fileInfo.Length - count));
var block = new Block() { Length = newLength, OffsetFile = count };
blocks.Add(block);
count += newLength;
}
var result = new FilterResult() {UnfilteredBlocks = blocks};
return result;
}
private ViterbiResult RunViterbi(FilterResult filterResult, string fileSha1)
{
// Load any user-defined state machines.
List<UserState> userStates;
try
{
userStates = Loader.LoadUserStates(MainForm.Program.UserStatesEnabled,
MainForm.Program.UserStatesPath);
}
catch (Exception ex)
{
DisplayExceptionMessages(ex, "User-Defined States");
return null;
}
ViterbiResult viterbiResultFields = null;
try
{
write("Running Viterbi on fields");
DateTime dt = DateTime.Now;
#if _GENERALPARSE
Viterbi viterbi = new Viterbi(RunType.GeneralParse, false);
viterbiResultFields = viterbi.Run(filterResult.UnfilteredBlocks, this.filePath);
#else
#if SKIPREALWORK
viterbiResultFields = new ViterbiResult();
viterbiResultFields.Fields = new List<ViterbiField>();
#else
if (filterResult.UnfilteredBlocks.Count > 0)
{
ThreadedViterbi tv = new ThreadedViterbi(filterResult.UnfilteredBlocks, RunType.GeneralParse, userStates, this.filePath, this.fileSha1);
viterbiResultFields = tv.RunThreadedViterbi();
TimeSpan ts = DateTime.Now.Subtract(dt);
write("Time elapsed for Viterbi fields: {0}", ts.ToString("c"));
write("Field count: {0}", viterbiResultFields.Fields.Count);
}
#endif
#endif
filterResult.UnfilteredBlocks.Clear(); // Allow gc to clean things up
}
catch (ThreadAbortException)
{
return null;
}
catch (Exception ex)
{
DisplayExceptionMessages(ex, "Viterbi Fields");
return null;
}
return viterbiResultFields;
}
private void Dump_Filtered_Blocks(FilterResult result, string filepath)
{
FileStream stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read);
List<Block> blocks = new List<Block>(result.UnfilteredBlocks);
System.IO.FileStream FileStream = new System.IO.FileStream("blocks_left", System.IO.FileMode.Create, System.IO.FileAccess.Write);
for (int i = 0; i < blocks.Count; i++)
{
Block b = blocks[i];
var bytes = BlockHashFilter.GetBytes(stream, b.OffsetFile, b.Length);
FileStream.Write(bytes, 0, bytes.Length);
}
FileStream.Close();
}
/// <summary>
/// Attempts to log the filter results from the same directory as the input file
/// </summary>
/// <returns>Returns true if successful.</returns>
private bool LoadFilterResults()
{
string resfile = _inputFile + ".fil";
if (File.Exists(resfile))
{
using (Stream instream = File.OpenRead(resfile))
{
BinaryFormatter serializer = new BinaryFormatter();
var filterResult = (FilterResult)serializer.Deserialize(instream);
// Check if the results file was created from the same binary input file
if (filterResult.MemoryId == _memoryId)
_filterResult = filterResult;
}
}
return _filterResult != null;
}
/// <summary>
/// Performs Block Hash Filtering for the phone by returning those blocks of the phone's memory that did not match the existing block hash library.
/// </summary>
/// <param name="phoneId">The unique identification for this phone in the database.</param>
/// <param name="sqlConnect">Handler for the database that contains the block hashes for this phone.</param>
/// <returns>The set of blocks remaining in the phone's memory after block hash filtering is complete.</returns>
public FilterResult Filter(int phoneId, object sqlConnect)
{
var start = DateTime.Now;
var blockInfos = new List<BlockInfo>();
var allInterestingBlocks = new List<Block>();
#if _USESQLSERVER_
List<usp_Hash_GetUnfilteredBlocksResult> unfilteredBlocks;
using (PhoneDbDataContext dataContext = Dalbase.GetDataContext())
{
// BL:TEMP added try catch for debugging
try {
ISingleResult<usp_Hash_GetUnfilteredBlocksResult> results = dataContext.usp_Hash_GetUnfilteredBlocks(_blockSize, _slideAmount, phoneId, _memoryId, _noFilter, _slideAmountLibrary);
unfilteredBlocks = (from result in results select result).ToList();
} catch (Exception ex) {
throw ex;
}
}
#else
DateTime started = DateTime.Now;
// TODO: Clean up and remove unused methods.
// Note: Only 7a supports blocks passed directly to us, blocks that are not in the DB.
//List<UnfilterdBlockResult> unfilteredBlocks = HashGetUnfilteredBlocks5(phoneId, (SQLiteConnection)sqlConnect);
//List<UnfilterdBlockResult> unfilteredBlocks = HashGetUnfilteredBlocks6(phoneId, (SQLiteConnection)sqlConnect);
//List<UnfilterdBlockResult> unfilteredBlocks = HashGetUnfilteredBlocks7(phoneId, (SQLiteConnection)sqlConnect);
List<UnfilterdBlockResult> unfilteredBlocks = HashGetUnfilteredBlocks7a(phoneId, (SQLiteConnection)sqlConnect);
//List<UnfilterdBlockResult> unfilteredBlocks = HashGetUnfilteredBlocks8(phoneId, (SQLiteConnection)sqlConnect);
DateTime finished = DateTime.Now;
System.Console.WriteLine("Block Hash Filter took " + (finished - started).TotalSeconds + " seconds");
#endif
long numBytes = new FileInfo(_inputFile).Length;
//This is to cover a weird issue where the memory is not divisible by the block size
long numBlocks = 0;
if(numBytes < _blockSize)
numBlocks = 1;
else
numBlocks = Convert.ToInt64(Math.Round((numBytes - _blockSize) / (double)_slideAmount + 1, MidpointRounding.AwayFromZero));
int unfilteredIndex = 0;
long i = 0;
long lastBlockIndex = long.MinValue;
while (i < numBlocks && unfilteredIndex < unfilteredBlocks.Count)
{
//Check if this block was not filtered.
if (unfilteredBlocks[unfilteredIndex].blockIndexFirst == i)
{
#if CHECK_HASH
// We need to do a sanity check to make sure the logic works
var blockBytes = GetBytes(_inputFile, unfilteredBlocks[unfilteredIndex].blockIndexFirst.Value * _slideAmount, _blockSize);
string blockHash = Convert.ToBase64String(_hash.ComputeHash(blockBytes));
bool hashesMatch = (blockHash == unfilteredBlocks[unfilteredIndex].hash);
if (!hashesMatch)
{
//throw new ApplicationException("Wha? Why did the hashes not match?!");
Console.WriteLine("Wha? Why did the hashes not match?!");
}
#endif
BlockInfo blockInfo = new BlockInfo();
byte[] interestingBytes;
//if not the last block, add the first X bytes where X=slide_amount
if (unfilteredBlocks[unfilteredIndex].blockIndexFirst < numBlocks - 1)
{
blockInfo.Offset = unfilteredBlocks[unfilteredIndex].blockIndexFirst.Value*_slideAmount;
blockInfo.Length = _slideAmount;
}
//else add the whole block
else
{
blockInfo.Offset = unfilteredBlocks[unfilteredIndex].blockIndexFirst.Value * _slideAmount;
blockInfo.Length = _blockSize;
}
//Concatenate this block with the previous if they are contiguous and less than max block size
if (unfilteredBlocks[unfilteredIndex].blockIndexFirst - 1 == lastBlockIndex && blockInfos[blockInfos.Count-1].Length < MAX_BLOCK_SIZE)
{
blockInfos[blockInfos.Count - 1].Length += blockInfo.Length;
}
else
{
blockInfos.Add(blockInfo);
}
lastBlockIndex = unfilteredBlocks[unfilteredIndex].blockIndexFirst.Value;
unfilteredIndex++;
i++;
}
//Check if this block is filtered
else if (i < unfilteredBlocks[unfilteredIndex].blockIndexFirst)
{
//skip to end of matched block
i = i + (_blockSize / _slideAmount);
//We do not want to skip past the last block
if (i > numBlocks - 1)
i = numBlocks - 1;
//Skip all unfiltered blocks after the matched block that overlap with the matched block
while (unfilteredIndex < unfilteredBlocks.Count && unfilteredBlocks[unfilteredIndex].blockIndexFirst < i)
{
unfilteredIndex++;
}
}
else if (i > unfilteredBlocks[unfilteredIndex].blockIndexFirst)
throw new ApplicationException("This should never happen...");
}
long count = 0;
//Get the bytes for the blocks
for (int j = 0; j < blockInfos.Count; j++)
{
//This is going to be an issue for the blocks that contain fewer bytes than the block size, e.g. the last block.
//The length won't be calculated right in th
count += blockInfos[j].Length;
//var bytes = GetBytes(_inputFile, blockInfos[j].Offset, blockInfos[j].Length);
allInterestingBlocks.Add(new Block {OffsetFile = blockInfos[j].Offset, Length = blockInfos[j].Length});
}
//Count the number of bytes remaining
/* BL
for (int j = 0; j < allInterestingBlocks.Count; j++)
{
//count += allInterestingBlocks[j].Bytes.Length;
}
*/
_filterResult = new FilterResult
{
FilteredBytesCount = numBytes - count,
UnfilteredBytesCount = count,
UnfilteredBlocks = allInterestingBlocks,
Duration = DateTime.Now - start,
MemoryId = _memoryId
};
Console.WriteLine("Filtered {0} out of {1} bytes: {2}.", numBytes - count, numBytes, (numBytes - count) / (double)numBytes);
return _filterResult;
}
