public AddressBookFeatureSet(int parseId)
{
using (var dataContext = Dalbase.GetDataContext())
{
_numbers = (from result in dataContext.usp_ParsedFields_GetAllRecordPhoneNumbers_ByParseId(parseId) select result.number).ToList();
_numbersWithType =
(from result in dataContext.usp_ParsedFields_GetAllRecordPhoneNumbersWithRecordType_ByParseId(parseId)
select new PhoneCrossRecordFeature.Tuple {
Number = result.number, RecordType = result.recordType
}).
ToList();
//Create the phone features
_areaCode = new AreaCodeFeature(_numbers);
_phoneCrossRecord = new PhoneCrossRecordFeature(_numbersWithType);
_phoneForm = new PhoneFormFeature(_numbers);
_bigram = new BigramPerLengthFeature();
var results = (from result in dataContext.usp_Decode_AddressBook_CompareAnswersToParse(parseId)
where result.name != null
select result).ToList();
for (int i = 0; i < results.Count; i++)
{
bool isCorrect = (results[i].answer_name != null);
CreateFeatureRecord(dataContext, isCorrect, results[i].name, results[i].number, parseId);
}
}
}
/// <summary>
/// Gets the ID of the phone by adding its information to the database table and getting the corresponding row ID.
/// </summary>
/// <param name="file">The BinaryFile reference for the phone's memory file.</param>
/// <returns>The ID corresponding to the phone, from the database.</returns>
public int GetPhoneId(BinaryFile file)
{
#if _USESQLSERVER_
using (PhoneDbDataContext dataContext = Dalbase.GetDataContext()) {
var results = dataContext.usp_Phone_Insert(file.Make, file.Model, file.SubId,
string.Format("Inserted on {0}", DateTime.Now));
return((from result in results select result.phoneId).Single());
}
#else
return(DatabaseAccess.PhoneInsert(file.Make, file.Model, file.SubId,
string.Format("Inserted on {0}", DateTime.Now)));
#endif
}
/// <summary>
/// Gets the row ID for the record it inserts in the database, consisting of phone's general and
/// block hash information viz. memoryID, slide amount, block size, notes etc.
/// </summary>
/// <param name="blocksize">The size of a memory block on phone.</param>
/// <param name="phoneId">The unique identification for this phone in the database.</param>
/// <param name="memoryId">The FileSHA1 of the memory file.</param>
/// <param name="slideAmount">The distance between the starting offsets of two consecutive blocks in memory.</param>
/// <param name="numBlocks">The total number of blocks in phone's memory.</param>
/// <param name="timeToHashSeconds">Time taken to calculate all block hashes of the phone.</param>
/// <param name="hashType">The kind of hash, for example SHA1.</param>
/// <param name="notes">General information regarding the phone.</param>
/// <returns>The row ID of the record just inserted for this phone in the table.</returns>
public int GetHashRunId(int blocksize, int phoneId, string memoryId, int slideAmount, long numBlocks, long timeToHashSeconds, string hashType, string notes)
{
#if _USESQLSERVER_
using (PhoneDbDataContext dataContext = Dalbase.GetDataContext()) {
var results = dataContext.usp_HashRun_Insert(DateTime.Now, phoneId, memoryId, slideAmount, blocksize, numBlocks,
timeToHashSeconds, hashType, notes);
return((from result in results select result.hashRunId).Single());
}
#else
return(DatabaseAccess.HashRunInsert(DateTime.Now, phoneId, memoryId, slideAmount, blocksize, (int)numBlocks,
(int)timeToHashSeconds, hashType, notes));
#endif
}
/// <summary>
/// Inserts a collection of composite fields into the database. The specific sproc is
/// dependent on the type of field.
/// </summary>
/// <param name="fieldId">fieldId is either the parseId or the phoneId depending on if
/// you are loading results for a Dec0de parse or the answers (fields known to be on the phone)</param>
/// <param name="fields">The collection of fields to be loaded</param>
/// <param name="isParse">Set to 'true' if this is a result from Dec0de's parse. Set to 'false'
/// if this is an answer/known field value</param>
/// <param name="source">The source of the data for the insert, e.g. xry</param>
public static void Insert(int fieldId, List <MetaField> fields, bool isParse, string source)
{
int insertCount = 0;
using (var dataContext = Dalbase.GetDataContext())
{
for (int i = 0; i < fields.Count; i++)
{
try
{
fields[i].Insert(fieldId, dataContext, isParse, source);
insertCount++;
}
catch (Exception ex)
{
Console.WriteLine(ex.Message + " : " + fields[i]);
}
}
}
Console.WriteLine("Inserted {0} records into the database. Failed on {1}.", insertCount, fields.Count - insertCount);
}
/// <summary>
/// Finds out if the phone's block hashes are already present in the database.
/// </summary>
/// <param name="blockSize">The size of a memory block on phone.</param>
/// <param name="slideAmount">The distance between the starting offsets of two consecutive blocks in memory.</param>
/// <returns></returns>
public bool AlreadyExists(int blockSize, int slideAmount)
{
int pid = -1;
try {
#if _USESQLSERVER_
using (PhoneDbDataContext dataContext = Dalbase.GetDataContext()) {
var results = dataContext.usp_Phoneid_GetByMemoryid(this.memoryId, blockSize, slideAmount);
pid = (from result in results select result.phoneId).Single();
}
#else
pid = DatabaseAccess.GetPhoneidByMemoryid(this.memoryIdSha1, blockSize, slideAmount);
#endif
}
catch (Exception) {
}
if (pid < 0)
{
return(false);
}
this.PhoneId = pid;
return(true);
}
public static void GetConstants()
{
int[] sizes = new int[] { 1, 4, 16 };//{32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768};
using (PhoneDbDataContext dataContext = Dalbase.GetDataContext()) {
for (int i = 0; i < sizes.Length; i++)
{
for (int j = 0; j < 256; j++)
{
byte value = (byte)j;
byte[] bytes = (new byte[sizes[i]].Select(r => value)).ToArray();
var hash = BlockHashFilter.GetBlockHash(bytes);
var hashString = Convert.ToBase64String(hash);
var valString = "0x" + Convert.ToString(value, 16).PadLeft(2, '0');
dataContext.usp_Constants_Insert(valString, sizes[i], hashString);
}
}
}
}
/// <summary>
/// Calls ComputeHahses() method of BlockHashFilter class to calculate phone's block hashes, calls GetHashRunId() to insert phone's
/// record to the database, calls InsertHahses() of DatabaseAccess to insert the block hashes into the database and HashRunUpdate() of
/// DatabaseAccess to update the number of blocks and time to hash fields in the database.
/// </summary>
/// <param name="file">The BinaryFile reference corresponding to the phone's memory file.</param>
/// <param name="blockSize">The size of a memory block on phone.</param>
/// <param name="slideAmount">The distance between the starting offsets of two consecutive blocks in memory.</param>
/// <param name="hashRunId">The ID of the row to which the block hashes of the phone are to be inserted.</param>
/// <param name="pid">Phone id to be used.</param>
/// <param name="memId">Memory id for the phone.</param>
public void ProcessFile(BinaryFile file, int blockSize, int slideAmount, int hashRunId, int pid, string memId)
{
//The starting point in the phone file stream. Needed to maintain position state across multiple calls of the method.
long nextStreamPosition = 0;
long lastStreamPositon = -1;
int blockIndex = 0;
DateTime start = DateTime.Now;
while (lastStreamPositon < nextStreamPosition)
{
lastStreamPositon = nextStreamPosition;
List <string> hashes = ComputeHashes(file.Path, blockSize, slideAmount, ref nextStreamPosition);
if (hashes.Count == 0)
{
continue;
}
#if _USESQLSERVER_
long numBlocks = hashes.Count;
List <HashInfo> hashInfos = new List <HashInfo>();
for (int k = 0; k < hashes.Count; k++)
{
hashInfos.Add(new HashInfo {
BlockIndex = blockIndex, Hash = hashes[k], HashRunId = hashRunId
});
blockIndex++;
}
BulkInsertBase bulky = BulkInsertBase.Load(hashInfos);
bulky.Flush();
#else
List <BlockHashFilter.UnfilterdBlockResult> stored = new List <BlockHashFilter.UnfilterdBlockResult>();
try {
blockIndex = DatabaseAccess.InsertHashes(hashes, hashRunId, stored);
} catch (ThreadAbortException e) {
DatabaseAccess.ForgetPhone(memId, pid, hashRunId);
throw e;
}
if (blockIndex < 0)
{
DatabaseAccess.ForgetPhone(memId, pid, hashRunId);
return;
}
storedBlocks = stored;
#endif
}
DateTime end = DateTime.Now;
TimeSpan duration = end - start;
long timeToHashSeconds = Convert.ToInt32(Math.Round(duration.TotalSeconds));
#if _USESQLSERVER_
using (PhoneDbDataContext dataContext = Dalbase.GetDataContext()) {
dataContext.usp_HashRun_Update(hashRunId, blockIndex, timeToHashSeconds);
}
#else
DatabaseAccess.HashRunUpdate(hashRunId, blockIndex, (int)timeToHashSeconds);
#endif
}
public void Run(RunType type)
{
//Console.WriteLine("\nLoading hashes. {0}", start);
//phoneId = HashLoaderProgram.ProcessFile(new BinaryFile(inputFile), blockSize, slideAmount);
Console.WriteLine("\n\nPhone {0}", _phoneId);
Console.WriteLine("Starting the Block Hash Filtering. {0}", DateTime.Now);
var filterResult = RunBlockHashFilter();
Console.WriteLine("Finished the Block Hash Filtering. Duration: {0}", filterResult.Duration);
Console.WriteLine("Starting Viterbi.");
var viterbiResult = RunViterbi(filterResult.UnfilteredBlocks, type);
Console.WriteLine("Finished Viterbi. Duration: {0}", viterbiResult.Duration);
var start = DateTime.Now;
Console.WriteLine("Inserting results into database. {0}", start);
int parseId;
using (var dataContext = Dalbase.GetDataContext())
{
var records = dataContext.usp_Parse_Insert(DateTime.UtcNow, _phoneId, _memoryId, _blockSize,
_slideAmount, filterResult.FilteredBytesCount,
filterResult.UnfilteredBytesCount,
Convert.ToInt32(filterResult.Duration.TotalSeconds),
Convert.ToInt32(viterbiResult.Duration.TotalSeconds),
Convert.ToString(type));
parseId = (from record in records select record.parseId).First();
Console.WriteLine("ParseId {0}", parseId);
var fields = viterbiResult.Fields;
for (int i = 0; i < fields.Count; i++)
{
dataContext.usp_ParsedFields_Insert(parseId, fields[i].OffsetFile,
Convert.ToString(fields[i].MachineName), fields[i].HexString,
fields[i].FieldString);
}
try
{
dataContext.usp_ParsedPhoneNum_AutoUpdate(parseId);
}
catch (Exception)
{
}
}
Console.WriteLine("Finished inserting. Duration: {0}", DateTime.Now - start);
Console.WriteLine("Beging Viterbi Record Parse");
RunMetaViterbi(parseId, viterbiResult.Fields);
}