/// <summary>
/// Calls ComputeHahses() method of BlockHashFilter class to calculate phone's block hashes, calls GetHashRunId() to insert phone's
/// record to the database, calls InsertHahses() of DatabaseAccess to insert the block hashes into the database.
/// </summary>
/// <param name="file">The BinaryFile reference corresponding to the phone's memory file.</param>
/// <param name="blockSize">The size of a memory block on phone.</param>
/// <param name="slideAmount">The distance between the starting offsets of two consecutive blocks in memory.</param>
/// <returns>The PhoneId of the phone, i.e. the row ID of the phone's record in the database.</returns>
public int ProcessFile(BinaryFile file, int blockSize, int slideAmount)
{
int phoneId = GetPhoneId(file);
DateTime start = DateTime.Now;
List <string> hashes = BlockHashFilter.ComputeHashes(file.Path, blockSize, slideAmount);
DateTime end = DateTime.Now;
TimeSpan duration = end - start;
long numBlocks = hashes.Count;
long timeToHashSeconds = Convert.ToInt32(Math.Round(duration.TotalSeconds));
string hashType = "sha1";
int hashRunId = GetHashRunId(blockSize, phoneId, file.MemoryId, slideAmount, numBlocks, timeToHashSeconds,
hashType, "");
#if _USESQLSERVER_
List <HashInfo> hashInfos = new List <HashInfo>();
for (int k = 0; k < hashes.Count; k++)
{
hashInfos.Add(new HashInfo {
BlockIndex = k, Hash = hashes[k], HashRunId = hashRunId
});
}
BulkInsertBase bulky = BulkInsertBase.Load(hashInfos);
bulky.Flush();
#else
DatabaseAccess.InsertHashes(hashes, hashRunId);
#endif
return(phoneId);
}
public void GetEmptyHash()
{
byte[] emptyBytes = new byte[32768];
var hash = BlockHashFilter.GetBlockHash(emptyBytes);
var hashString = Convert.ToBase64String(hash);
}
// BL 8/4
private Block GetBlock(FileStream stream, long start, long end)
{
int length = (int)(end - start);
var bytes = BlockHashFilter.GetBytes(stream, start, length);
return(new Block()
{
Bytes = bytes, OffsetFile = start
});
}
private void Dump_Filtered_Blocks(FilterResult result, string filepath)
{
FileStream stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read);
List <Block> blocks = new List <Block>(result.UnfilteredBlocks);
System.IO.FileStream FileStream = new System.IO.FileStream("blocks_left", System.IO.FileMode.Create, System.IO.FileAccess.Write);
for (int i = 0; i < blocks.Count; i++)
{
Block b = blocks[i];
var bytes = BlockHashFilter.GetBytes(stream, b.OffsetFile, b.Length);
FileStream.Write(bytes, 0, bytes.Length);
}
FileStream.Close();
}
private FilterResult RunBlockHashFilter(string fileSha1, int phoneId, List <BlockHashFilter.UnfilterdBlockResult> storedBlocks)
{
FilterResult filterResult = null;
try
{
write("Starting block hash filtering");
DateTime dt = DateTime.Now;
BlockHashFilter bhf = new BlockHashFilter(this.filePath, HashLoader.HashLoader.DefaultBlockSize,
HashLoader.HashLoader.DefaultSlideAmount, fileSha1, true, storedBlocks);
#if _USESQLSERVER_
filterResult = bhf.Filter(phoneId);
#else
SQLiteConnection sql = DatabaseAccess.OpenSql(true);
try
{
filterResult = bhf.Filter(phoneId, sql, true);
}
finally
{
sql.Close();
sql.Dispose();
}
#endif
TimeSpan ts = DateTime.Now.Subtract(dt);
write("Time elapsed for block hash filtering: {0}", ts.ToString("c"));
long nBytes = filterResult.FilteredBytesCount + filterResult.UnfilteredBytesCount;
write("Filtered {0} out of {1} bytes: {2:0.00###}", filterResult.FilteredBytesCount,
nBytes, (nBytes - filterResult.UnfilteredBytesCount) / (double)nBytes);
}
catch (ThreadAbortException)
{
return(null);
}
catch (Exception ex)
{
DisplayExceptionMessages(ex, "Block Hash Filter");
return(null);
}
return(filterResult);
}
/// <summary>
/// Runs the Viterbi for each input block using Run(byte[], long) method.
/// </summary>
/// <param name="blocks">List of blocks for which Viterbi inference is to be done.</param>
/// <param name="inputFile">Path to the phone's memory file.</param>
/// <returns>The list of states in the inferred Viterbi path.</returns>
public ViterbiResult Run(List <Block> blocks, string inputFile)
{
var start = DateTime.Now;
FileStream stream = new FileStream(inputFile, FileMode.Open, FileAccess.Read, FileShare.Read); // BL 8/4
for (int i = 0; i < blocks.Count; i++)
{
//Let's get the block bytes here. They should be null if this is a field level run of viterbi. For record-level runs,
//they should not be null.
//var bytes = blocks[i].Bytes ?? BlockHashFilter.GetBytes(inputFile, blocks[i].OffsetFile, blocks[i].Length);
var bytes = blocks[i].Bytes ?? BlockHashFilter.GetBytes(stream, blocks[i].OffsetFile, blocks[i].Length); // BL 8/4
Run(bytes, blocks[i].OffsetFile);
}
stream.Close(); // BL 8/4
return(new ViterbiResult {
Fields = _fieldList, Duration = DateTime.Now - start
});
}
/// <summary>
/// Calls the Filter() method of BlockHashFilter class for calculating the phone's block hashes.
/// </summary>
/// <param name="inputFile">The path to phone's memory file.</param>
/// <param name="blocksize">The size of a memory block on phone.</param>
/// <param name="slideAmount">The distance between the starting offsets of two consecutive blocks in memory.</param>
/// <param name="nextStreamPosition">The starting position in memory of the block to be hashed.</param>
/// <returns>A list of block hashes for the phone.</returns>
public List <string> ComputeHashes(string inputFile, int blocksize, int slideAmount, ref long nextStreamPosition)
{
List <string> hashes = new List <string>();
int countBlocks = 0;
using (FileStream stream = File.Open(inputFile, FileMode.Open, FileAccess.Read, FileShare.Read)) {
var block = new byte[blocksize];
//Start where we left off. Should be zero for the first run of the method.
stream.Position = nextStreamPosition;
while (stream.Position < stream.Length && countBlocks < BLOCK_BUFFER)
{
stream.Read(block, 0, blocksize);
byte[] hash = BlockHashFilter.GetBlockHash(block);
string hashString = Convert.ToBase64String(hash);
hashes.Add(hashString);
//Need this check. Otherwise it will not finish
if (stream.Position < stream.Length)
{
stream.Position = stream.Position - blocksize + slideAmount;
}
countBlocks++;
nextStreamPosition = stream.Position;
}
}
return(hashes);
}
public static void GetConstants()
{
int[] sizes = new int[] { 1, 4, 16 };//{32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768};
using (PhoneDbDataContext dataContext = Dalbase.GetDataContext()) {
for (int i = 0; i < sizes.Length; i++)
{
for (int j = 0; j < 256; j++)
{
byte value = (byte)j;
byte[] bytes = (new byte[sizes[i]].Select(r => value)).ToArray();
var hash = BlockHashFilter.GetBlockHash(bytes);
var hashString = Convert.ToBase64String(hash);
var valString = "0x" + Convert.ToString(value, 16).PadLeft(2, '0');
dataContext.usp_Constants_Insert(valString, sizes[i], hashString);
}
}
}
}
public FilterResult RunBlockHashFilter()
{
var filter = new BlockHashFilter(_inputFile, _blockSize, _slideAmount, _memoryId, _noFilter);
return(filter.Filter(_phoneId));
}
