public ThreadedViterbi(List <Block> UnfilteredBlocks, RunType runType, List <UserState> userStates, string file_path, string fileSha1)
{
_unfilteredBlocks = UnfilteredBlocks;
_userStates = userStates;
_filePath = file_path;
_machines = new List <StateMachine>();
_states = new List <State>();
_viterbiResults = new ViterbiResult();
_viterbiResults.Fields = new List <ViterbiField>();
_fileSha1 = fileSha1;
_viterbiResults.MemoryId = this._fileSha1;
_runType = runType;
//TODO: (RJW) I am not convinced this bit of code loads the machines as Shaksham intended.
// This call is to load _machines, _states, _startState and _userStates variables, so that we do not execute same code for every block to load values
// into these variables, since they are the same for all blocks.
if (_runType == RunType.GeneralParse)
{
Viterbi viterbi = new Viterbi(RunType.GeneralParse, true, ref _machines, ref _states, ref _startState, ref _userStates);
}
else if (_runType == RunType.Meta)
{
Viterbi viterbi = new Viterbi(RunType.Meta, false, ref _machines, ref _states, ref _startState, ref _userStates);
_unfilteredBlocks = Split_On_Binary_Large_Fields(_unfilteredBlocks[0].Bytes);
}
else
{
Viterbi viterbi = new Viterbi(_runType, false, ref _machines, ref _states, ref _startState, ref _userStates);
}
}
/// <summary>
/// Returns the emission probability of a particular byte. If the probability is explicitly defined it returns
/// that value. Otherwise it returns ALMOST_ZERO.
/// </summary>
/// <param name="value">List of byte outputs from Epoch1900Tuple state.</param>
/// <param name="index">Index of the byte in values, whose emission probability, given the Epoch1900Tuple state is required.</param>
/// <returns>The emission probability of an output byte, given the Epoch1900Tuple state.</returns>
public override double GetValueProbability(byte[] values, int index, Viterbi viterbi)
{
double baseProb = base.GetValueProbability(values, index, viterbi);
//If the probability is zero, no need to do all of this logic.
if (baseProb == ALMOST_ZERO)
return ALMOST_ZERO;
// Two 4-byte little-endian timestamps. Use the first timestamp.
byte[] input1 = new byte[4];
input1[3] = values[index - 4];
input1[2] = values[index - 5];
input1[1] = values[index - 6];
input1[0] = values[index - 7];
byte[] input2 = new byte[4];
input2[3] = values[index];
input2[2] = values[index - 1];
input2[1] = values[index - 2];
input2[0] = values[index - 3];
try {
uint seconds1 = BitConverter.ToUInt32(input1, 0);
uint seconds2 = BitConverter.ToUInt32(input2, 0);
if (Math.Abs(seconds2 - seconds1) > 172800) {
// Two days diff is too long.
return ALMOST_ZERO;
}
var dateTime1 = new DateTime(1900, 1, 1).AddSeconds(seconds1);
if (dateTime1.Year >= TimeConstants.START_YEAR && dateTime1.Year <= TimeConstants.END_YEAR) {
return baseProb;
}
return ALMOST_ZERO;
} catch (Exception) {
return ALMOST_ZERO;
}
}
public ThreadedViterbi(List<Block> UnfilteredBlocks, RunType runType, List<UserState> userStates, string file_path, string fileSha1)
{
_unfilteredBlocks = UnfilteredBlocks;
_userStates = userStates;
_filePath = file_path;
_machines = new List<StateMachine>();
_states = new List<State>();
_viterbiResults = new ViterbiResult();
_viterbiResults.Fields = new List<ViterbiField>();
_fileSha1 = fileSha1;
_viterbiResults.MemoryId = this._fileSha1;
_runType = runType;
//TODO: (RJW) I am not convinced this bit of code loads the machines as Shaksham intended.
// This call is to load _machines, _states, _startState and _userStates variables, so that we do not execute same code for every block to load values
// into these variables, since they are the same for all blocks.
if (_runType == RunType.GeneralParse)
{
Viterbi viterbi = new Viterbi(RunType.GeneralParse, true, ref _machines, ref _states, ref _startState, ref _userStates);
}
else if (_runType == RunType.Meta)
{
Viterbi viterbi = new Viterbi(RunType.Meta, false, ref _machines, ref _states, ref _startState, ref _userStates);
_unfilteredBlocks = Split_On_Binary_Large_Fields(_unfilteredBlocks[0].Bytes);
}
else
{
Viterbi viterbi = new Viterbi(_runType, false, ref _machines, ref _states, ref _startState, ref _userStates);
}
}
public override double GetValueProbability(byte[] values, int index, Viterbi viterbi)
{
double baseProb = base.GetValueProbability(values, index, viterbi);
// If the probability is zero, no need to do all of this logic.
if (baseProb == ALMOST_ZERO)
{
return(ALMOST_ZERO);
}
try {
// Allocate byte array for data.
byte[] input = new byte[UserStateObj.Bytes.Count];
for (int n = input.Length - 1, i = 0; n >= 0; n--, i++)
{
input[n] = values[index - i];
}
DateTime dateTime = GetDateTime(UserStateObj, input);
if (dateTime.Year >= TimeConstants.START_YEAR && dateTime.Year <= TimeConstants.END_YEAR)
{
return(baseProb);
}
else
{
return(ALMOST_ZERO);
}
} catch {
return(ALMOST_ZERO);
}
}
public override double GetValueProbability(byte[] values, int index, Viterbi viterbi)
{
var baseProb = base.GetValueProbability(values, index, viterbi);
if (baseProb == ALMOST_ZERO)
return baseProb;
var path = viterbi.FindPath(index - 1, viterbi.FromState, index - 1);
int lengthIndex = -1;
int distance = 0;
for (int i = path.Count - 1; i >= 0; i--)
{
distance++;
if (path[i] == LengthState)
{
lengthIndex = index - distance;
break;
}
}
if (lengthIndex == -1)
return ALMOST_ZERO;
int length = values[lengthIndex];
if (length < distance + 1)
return ALMOST_ZERO;
return base.GetValueProbability(values, index, viterbi);
}
/// <summary>
/// Returns the emission probability of a particular byte. This is used to ensure we
/// don't look further than defined by the assoicated length state. If we're beyond
/// the length return ALMOST_ZERO, else return the existing base probability.
/// </summary>
/// <param name="value">List of byte outputs from the State.</param>
/// <param name="index">Index of the byte in values, whose emission probability,
/// given the BcdDigitState State is required.</param>
/// <returns>The emission probability of an output byte, given the BcdDigitState state.</returns>
public override double GetValueProbability(byte[] values, int index, Viterbi viterbi)
{
var baseProb = base.GetValueProbability(values, index, viterbi);
if (baseProb == ALMOST_ZERO) return baseProb;
var path = viterbi.FindPath(index - 1, viterbi.FromState, 24);
int lengthIndex = -1;
int byteDistance = 0;
for (int i = path.Count - 1; i >= 0; i--) {
byteDistance++;
if (path[i] == LengthState) {
lengthIndex = index - byteDistance;
break;
}
}
if (lengthIndex == -1) return ALMOST_ZERO;
// Subtract 1 from the distance to allow for the Type of Address (TOA)
// byte between the length and start of BCD phone number.
int phoneNumLength = values[lengthIndex];
if (((phoneNumLength + 1) / 2) < (byteDistance - 1)) {
return ALMOST_ZERO;
}
return baseProb;
}
private void RunBlockThread(List <Block> BlockSet, ref List <ViterbiResult> ResultsOnBlocks, ref object mutex, ref int job_count, ref ManualResetEvent manual)
{
ViterbiResult viterbiResultFields = null;
if (_runType != RunType.Meta)
{
#if !_GENERAL_PARSE
AnchorViterbi anchor_viterbi = new AnchorViterbi(RunType.GeneralParse, _filePath, new List <UserState>(_userStates));
viterbiResultFields = anchor_viterbi.RunThreaded(BlockSet, ref _machines, ref _states, ref _startState, ref _userStates);
#else
Viterbi viterbi = new Viterbi(_runType, false, ref _machines, ref _states, ref _startState, ref _userStates);
viterbiResultFields = viterbi.Run(BlockSet, _filePath);
#endif
}
else
{
Viterbi viterbi = new Viterbi(RunType.Meta, false, ref _machines, ref _states, ref _startState, ref _userStates);
viterbiResultFields = viterbi.Run(BlockSet, _filePath);
}
lock (mutex)
{
ResultsOnBlocks.Add(viterbiResultFields);
job_count--;
if (job_count == 0)
{
manual.Set(); // signal that all threads are done
}
}
}
/// <summary>
/// Returns the emission probability of a particular byte. If the probability is explicitly defined it returns
/// that value. Otherwise it returns ALMOST_ZERO.
/// </summary>
/// <param name="value">List of byte outputs from SMS time state.</param>
/// <param name="index">Index of the byte in values, whose emission probability, given the SMS
/// with TZ time state is required.</param>
/// <returns>The emission probability of an output byte, given the SMS time state.</returns>
public override double GetValueProbability(byte[] values, int index, Viterbi viterbi)
{
double baseProb = base.GetValueProbability(values, index, viterbi);
//If the probability is zero, no need to do all of this logic.
if (baseProb == ALMOST_ZERO)
{
return(ALMOST_ZERO);
}
var input = new byte[7];
input[6] = values[index];
input[5] = values[index - 1];
input[4] = values[index - 2];
input[3] = values[index - 3];
input[2] = values[index - 4];
input[1] = values[index - 5];
input[0] = values[index - 6];
var year = 2000 + Convert.ToInt32(Printer.GetNibbleString(Printer.SwapNibbles(input[0])));
var month = Convert.ToInt32(Printer.GetNibbleString(Printer.SwapNibbles(input[1])));
var day = Convert.ToInt32(Printer.GetNibbleString(Printer.SwapNibbles(input[2])));
var hour = Convert.ToInt32(Printer.GetNibbleString(Printer.SwapNibbles(input[3])));
var minute = Convert.ToInt32(Printer.GetNibbleString(Printer.SwapNibbles(input[4])));
var second = Convert.ToInt32(Printer.GetNibbleString(Printer.SwapNibbles(input[5])));
byte bv = Printer.SwapNibbles(input[6]);
byte bcd = (byte)(bv & 0x37);
byte[] nibbles = Printer.GetNibbles(bcd);
int tz = (nibbles[0] * 10) + nibbles[1];
if ((bv & 0x80) != 0)
{
tz = -tz;
}
bool isValidYear = (year >= TimeConstants.START_YEAR && year <= TimeConstants.END_YEAR);
bool isValidMonth = (month >= 1 && month <= 12);
bool isValidDay = (day >= 1 && day <= TimeConstants.MonthDays[month - 1]);
bool isValidHour = (hour >= 0 && hour <= 23);
bool isValidMinute = (minute >= 0 && minute <= 59);
bool isValidSecond = (second >= 0 && second <= 59);
bool isValidTz = (tz > -96 && tz < 96);
if (isValidYear && isValidMonth && isValidHour && isValidDay && isValidMinute && isValidSecond && isValidTz)
{
if ((month == 2) && (day == 29))
{
try {
var timestamp = new DateTime(year, month, day, hour, minute, second);
} catch (Exception) {
return(ALMOST_ZERO);
}
}
return(baseProb);
}
return(ALMOST_ZERO);
}
/// <summary>
/// Returns the emission probability of a particular byte. If the probability is explicitly defined it returns
/// that value. Otherwise it returns ALMOST_ZERO.
/// </summary>
/// <param name="value">List of byte outputs from Nokia time state.</param>
/// <param name="index">Index of the byte in values, whose emission probability, given the Nokia time state is required.</param>
/// <returns>The emission probability of an output byte, given the Nokia time state.</returns>
public override double GetValueProbability(byte[] values, int index, Viterbi viterbi)
{
double baseProb = base.GetValueProbability(values, index, viterbi);
//If the probability is zero, no need to do all of this logic.
if (baseProb == ALMOST_ZERO)
{
return(ALMOST_ZERO);
}
var input = new byte[7];
input[6] = values[index];
input[5] = values[index - 1];
input[4] = values[index - 2];
input[3] = values[index - 3];
input[2] = values[index - 4];
input[1] = values[index - 5];
input[0] = values[index - 6];
var yearByte = new Byte[] { 0x00, 0x00, input[0], input[1] };
if (BitConverter.IsLittleEndian)
{
Array.Reverse(yearByte);
}
var year = BitConverter.ToInt32(yearByte, 0);
var month = (int)input[2];
var day = (int)input[3];
var hour = (int)input[4];
var minute = (int)input[5];
var second = (int)input[6];
bool isValidYear = (year >= TimeConstants.START_YEAR && year <= TimeConstants.END_YEAR);
bool isValidMonth = (month >= 1 && month <= 12);
bool isValidDay = (day >= 1 && day <= TimeConstants.MonthDays[month - 1]);
bool isValidHour = (hour >= 0 && hour <= 23);
bool isValidMinute = (minute >= 0 && minute <= 59);
bool isValidSecond = (second >= 0 && second <= 59);
//if (minute == second && second == 0)
// return ALMOST_ZERO;
if (isValidYear && isValidMonth && isValidHour && isValidDay && isValidMinute && isValidSecond)
{
if ((month == 2) && (day == 29))
{
try {
var timestamp = new DateTime(year, month, day, hour, minute, second);
} catch (Exception) {
return(ALMOST_ZERO);
}
}
return(baseProb);
}
return(ALMOST_ZERO);
}
/// <summary>
/// Returns the emission probability of a particular byte. If the probability is explicitly defined it returns
/// that value. Otherwise it returns ALMOST_ZERO.
/// </summary>
/// <param name="value">List of byte outputs from Samsung time state.</param>
/// <param name="index">Index of the byte in values, whose emission probability, given the Samsung time state is required.</param>
/// <returns>The emission probability of an output byte, given the Samsung time state.</returns>
public override double GetValueProbability(byte[] values, int index, Viterbi viterbi)
{
double baseProb = base.GetValueProbability(values, index, viterbi);
//If the probability is zero, no need to do all of this logic.
if (baseProb == ALMOST_ZERO)
{
return(ALMOST_ZERO);
}
var input = new byte[4];
input[3] = values[index];
input[2] = values[index - 1];
input[1] = values[index - 2];
input[0] = values[index - 3];
var yearBytes = new byte[] { input[3], input[2], 0x00, 0x00 };
var monthBytes = new byte[] { 0x00, (byte)(input[2] & 0x0F), 0x00, 0x00 };
var dayBytes = new byte[] { 0x00, 0x00, input[1], 0x00 };
var hourBytes = new byte[] { 0x00, 0x00, (byte)(input[1] & 0x07), input[0] };
var minuteBytes = new byte[] { 0x00, 0x00, 0x00, (byte)(input[0] & 0x3F) };
if (BitConverter.IsLittleEndian)
{
Array.Reverse(yearBytes);
Array.Reverse(monthBytes);
Array.Reverse(dayBytes);
Array.Reverse(hourBytes);
Array.Reverse(minuteBytes);
}
var year = BitConverter.ToInt32(yearBytes, 0) >> 20;
var month = BitConverter.ToInt32(monthBytes, 0) >> 16;
var day = BitConverter.ToInt32(dayBytes, 0) >> 11;
var hour = BitConverter.ToInt32(hourBytes, 0) >> 6;
var minute = BitConverter.ToInt32(minuteBytes, 0);
bool isValidYear = (year >= TimeConstants.START_YEAR && year <= TimeConstants.END_YEAR);
bool isValidMonth = (month >= 1 && month <= 12);
bool isValidDay = (day >= 1 && day <= TimeConstants.MonthDays[month - 1]);
bool isValidHour = (hour >= 0 && hour <= 23);
bool isValidMinute = (minute >= 0 && minute <= 59);
if (isValidYear && isValidMonth && isValidHour && isValidDay && isValidMinute)
{
if ((month == 2) && (day == 29))
{
try {
var timestamp = new DateTime(year, month, day, hour, minute, 0);
} catch (Exception) {
return(ALMOST_ZERO);
}
}
return(baseProb);
}
return(ALMOST_ZERO);
}
private void TestAll(Viterbi viterbi, List <Test> tests)
{
for (int i = 0; i < tests.Count; i++)
{
viterbi.Run(tests[i].RawBytes, 0);
PrintResult(viterbi, tests[i]);
viterbi.FieldStrings.Clear();
viterbi.Fields.Clear();
}
}
/// <summary>
/// Returns the emission probability of a particular byte. If the probability is explicitly defined it returns
/// that value. Otherwise it returns ALMOST_ZERO.
/// </summary>
/// <param name="values">List of byte outputs from SevenBit State.</param>
/// <param name="index">Index of the byte in values, whose emission probability, given the SevenBit State is required.</param>
/// <param name="viterbi"></param>
/// <returns>The emission probability of an output byte, given the SevenBit state.</returns>
public override double GetValueProbability(byte[] values, int index, Viterbi viterbi)
{
var baseProb = base.GetValueProbability(values, index, viterbi);
if (baseProb == ALMOST_ZERO)
return baseProb;
var path = viterbi.FindPath(index - 1, viterbi.FromState, 141);
int lengthIndex = -1;
int byteDistance = 0;
for (int i = path.Count - 1; i >= 0; i--) {
byteDistance++;
if (path[i] == LengthState) {
lengthIndex = index - byteDistance;
break;
}
}
if (lengthIndex == -1)
return ALMOST_ZERO;
int septetLength = values[lengthIndex];
//Convert from number of septets to number of bytes
int byteLength = (int)Math.Ceiling((double)septetLength * 7 / 8);
if (!IsEnd) {
if (byteLength < byteDistance) {
return ALMOST_ZERO;
} else if (byteLength > byteDistance) {
//Perform a partial decode of the bytes. If there are any invalid characters we want to quit now. This (should) prevent problems of long fields subsuming smaller fields.
if (!IsValid(byteDistance, index, values, byteDistance)) {
return ALMOST_ZERO;
}
}
} else {
if (byteLength != byteDistance) {
return ALMOST_ZERO;
} else {
if (!IsValid(byteLength, index, values, septetLength)) {
return ALMOST_ZERO;
}
}
}
return base.GetValueProbability(values, index, viterbi);
}
/// <summary>
/// Returns the emission probability of a particular byte. If the probability is explicitly defined it returns
/// that value. Otherwise it returns ALMOST_ZERO.
/// </summary>
/// <param name="value">List of byte outputs from the Bigram state.</param>
/// <param name="index">Index of the byte in values, whose emission probability, given the Bigram state is required.</param>
/// <returns>The emission probability of an output byte, given the Bigram state.</returns>
public override double GetValueProbability(byte[] values, int index, Viterbi viterbi)
{
//Make sure there is a previous value. If not, assume it is a space
byte prevValue = (index > 0) ? values[index - 1] : SPACE;
byte currentValue = values[index];
if (PossibleValues[prevValue, currentValue] == 0.0f)
return double.Epsilon;
//If the previous byte was a lower case character, and the current byte is an upper case character
if (IsLower(prevValue) && IsUpper(currentValue))
return double.Epsilon;
return PossibleValues[ConvertToLowerIfPossible(prevValue), ConvertToLowerIfPossible(currentValue)];
}
/// <summary>
/// Returns the emission probability of a particular byte. If the probability is explicitly defined it returns
/// that value. Otherwise it returns ALMOST_ZERO.
/// </summary>
/// <param name="value">List of byte outputs from the Bigram state.</param>
/// <param name="index">Index of the byte in values, whose emission probability, given the Bigram state is required.</param>
/// <returns>The emission probability of an output byte, given the Bigram state.</returns>
public override double GetValueProbability(byte[] values, int index, Viterbi viterbi)
{
//Make sure there is a previous value. If not, assume it is a space
byte prevValue = (index > 0) ? values[index - 1] : SPACE;
byte currentValue = values[index];
if (PossibleValues[prevValue, currentValue] == 0.0f)
{
return(double.Epsilon);
}
//If the previous byte was a lower case character, and the current byte is an upper case character
if (IsLower(prevValue) && IsUpper(currentValue))
{
return(double.Epsilon);
}
return(PossibleValues[ConvertToLowerIfPossible(prevValue), ConvertToLowerIfPossible(currentValue)]);
}
/// <summary>
/// Returns the emission probability of a particular byte. If the probability is explicitly defined it returns
/// that value. Otherwise it returns ALMOST_ZERO.
/// </summary>
/// <param name="value">List of byte outputs from Unix time state.</param>
/// <param name="index">Index of the byte in values, whose emission probability, given the Unix time state is required.</param>
/// <returns>The emission probability of an output byte, given the Unix time state.</returns>
public override double GetValueProbability(byte[] values, int index, Viterbi viterbi)
{
double baseProb = base.GetValueProbability(values, index, viterbi);
//If the probability is zero, no need to do all of this logic.
if (baseProb == ALMOST_ZERO)
{
return(ALMOST_ZERO);
}
var input = new byte[4];
input[3] = values[index];
input[2] = values[index - 1];
input[1] = values[index - 2];
input[0] = values[index - 3];
if (BitConverter.IsLittleEndian)
{
Array.Reverse(input);
}
var seconds = BitConverter.ToUInt32(input, 0); // B Lynn: make unsigned
try
{
var dateTime = new DateTime(1970, 1, 1).AddSeconds(seconds);
if (dateTime.Year >= TimeConstants.START_YEAR && dateTime.Year <= TimeConstants.END_YEAR)
{
return(baseProb);
}
else
{
return(ALMOST_ZERO);
}
}
catch (Exception)
{
return(ALMOST_ZERO);
}
}
/// <summary>
/// Returns the emission probability of a particular byte. If the probability is explicitly defined it returns
/// that value. Otherwise it returns ALMOST_ZERO.
/// </summary>
/// <param name="value">List of byte outputs from Motorola's SMS time state.</param>
/// <param name="index">Index of the byte in values, whose emission probability, given the Motorola's SMS time state is required.</param>
/// <returns>The emission probability of an output byte, given the Motorola's SMS time state.</returns>
public override double GetValueProbability(byte[] values, int index, Viterbi viterbi)
{
double baseProb = base.GetValueProbability(values, index, viterbi);
//If the probability is zero, no need to do all of this logic.
if (baseProb == ALMOST_ZERO)
return ALMOST_ZERO;
var input = new byte[6];
input[5] = values[index];
input[4] = values[index - 1];
input[3] = values[index - 2];
input[2] = values[index - 3];
input[1] = values[index - 4];
input[0] = values[index - 5];
var year = 1970 + Convert.ToInt32(input[0]);
var month = Convert.ToInt32(input[1]);
var day = Convert.ToInt32(input[2]);
var hour = Convert.ToInt32(input[3]);
var minute = Convert.ToInt32(input[4]);
var second = Convert.ToInt32(input[5]);
bool isValidYear = (year >= TimeConstants.START_YEAR && year <= TimeConstants.END_YEAR);
bool isValidMonth = (month >= 1 && month <= 12);
bool isValidDay = (day >= 1 && day <= TimeConstants.MonthDays[month - 1]);
bool isValidHour = (hour >= 0 && hour <= 23);
bool isValidMinute = (minute >= 0 && minute <= 59);
bool isValidSecond = (second >= 0 && second <= 59);
if (isValidYear && isValidMonth && isValidHour && isValidDay && isValidMinute && isValidSecond) {
if ((month == 2) && (day == 29)) {
try {
var timestamp = new DateTime(year, month, day, hour, minute, second);
} catch (Exception) {
return ALMOST_ZERO;
}
}
return baseProb;
}
return ALMOST_ZERO;
}
public override double GetValueProbability(byte[] values, int index, Viterbi viterbi)
{
double baseProb = base.GetValueProbability(values, index, viterbi);
// If the probability is zero, no need to do all of this logic.
if (baseProb == ALMOST_ZERO) return ALMOST_ZERO;
try {
// Allocate byte array for data.
byte[] input = new byte[UserStateObj.Bytes.Count];
for (int n = input.Length - 1, i = 0; n >= 0; n--, i++) {
input[n] = values[index - i];
}
if (Validate(UserStateObj, input)) {
return baseProb;
} else {
return ALMOST_ZERO;
}
} catch {
return ALMOST_ZERO;
}
}
public ViterbiTest()
{
StateMachine.GeneralParse(ref _generalMachines, ref _generalStates, ref _generalStartState);
//var testMachines = new List<StateMachine>
// {
// StateMachine.Get7BitString_WithLength(1),
// //StateMachine.GetPhoneNumber_All(6),
// //StateMachine.GetTimeStamp_All(1),
// };
//StateMachine.TestStateMachines(testMachines, ref _generalMachines, ref _generalStates, ref _generalStartState);
StateMachine.TestMeta(ref _metaMachines, ref _metaStates, ref _metaStartState);
_metaViterbi = new Viterbi(_metaMachines, _metaStates, _metaStartState);
_generalViterbi = new Viterbi(_generalMachines, _generalStates, _generalStartState);
}
/// <summary>
/// Returns the emission probability of a particular byte. If the probability is explicitly defined it returns
/// that value. Otherwise it returns ALMOST_ZERO.
/// </summary>
/// <param name="value">List of byte outputs from Epoch1900Tuple state.</param>
/// <param name="index">Index of the byte in values, whose emission probability, given the Epoch1900Tuple state is required.</param>
/// <returns>The emission probability of an output byte, given the Epoch1900Tuple state.</returns>
public override double GetValueProbability(byte[] values, int index, Viterbi viterbi)
{
double baseProb = base.GetValueProbability(values, index, viterbi);
//If the probability is zero, no need to do all of this logic.
if (baseProb == ALMOST_ZERO)
{
return(ALMOST_ZERO);
}
// Two 4-byte little-endian timestamps. Use the first timestamp.
byte[] input1 = new byte[4];
input1[3] = values[index - 4];
input1[2] = values[index - 5];
input1[1] = values[index - 6];
input1[0] = values[index - 7];
byte[] input2 = new byte[4];
input2[3] = values[index];
input2[2] = values[index - 1];
input2[1] = values[index - 2];
input2[0] = values[index - 3];
try {
uint seconds1 = BitConverter.ToUInt32(input1, 0);
uint seconds2 = BitConverter.ToUInt32(input2, 0);
if (Math.Abs(seconds2 - seconds1) > 172800)
{
// Two days diff is too long.
return(ALMOST_ZERO);
}
var dateTime1 = new DateTime(1900, 1, 1).AddSeconds(seconds1);
if (dateTime1.Year >= TimeConstants.START_YEAR && dateTime1.Year <= TimeConstants.END_YEAR)
{
return(baseProb);
}
return(ALMOST_ZERO);
} catch (Exception) {
return(ALMOST_ZERO);
}
}
// Using this method instead of the Run() method above.
public ViterbiResult RunThreaded(List<Block> unfilteredBlocks, ref List<StateMachine> machines,
ref List<State> states, ref State startState, ref List<UserState> userStates)
{
var anchorBlocks = GetAnchorPointBlocks(unfilteredBlocks);
Console.WriteLine("{0} Anchor block bytes.", Block.GetByteTotal(anchorBlocks));
var viterbi = new Viterbi(_runType, false, ref machines, ref states, ref startState, ref userStates);
var result = viterbi.Run(anchorBlocks, _filePath);
result.Duration = result.Duration.Add(_anchorResults.Duration);
return result;
}
/// <summary>
/// Returns the emission probability of a particular byte. If the probability is explicitly defined it returns
/// that value. Otherwise it returns ALMOST_ZERO.
/// </summary>
/// <param name="value">List of byte outputs from the state.</param>
/// <param name="index">Index of the byte in values, whose emission probability, given this state is required.</param>
/// <returns>The emission probability of an output byte, given this state.</returns>
public virtual double GetValueProbability(byte[] values, int index, Viterbi viterbi)
{
if (AllValuesPossible)
return UNIFORM_PROB;
return PossibleValueProbabilities[values[index]];
}
private ViterbiResult RunViterbi(FilterResult filterResult, string fileSha1)
{
// Load any user-defined state machines.
List<UserState> userStates;
try
{
userStates = Loader.LoadUserStates(MainForm.Program.UserStatesEnabled,
MainForm.Program.UserStatesPath);
}
catch (Exception ex)
{
DisplayExceptionMessages(ex, "User-Defined States");
return null;
}
ViterbiResult viterbiResultFields = null;
try
{
write("Running Viterbi on fields");
DateTime dt = DateTime.Now;
#if _GENERALPARSE
Viterbi viterbi = new Viterbi(RunType.GeneralParse, false);
viterbiResultFields = viterbi.Run(filterResult.UnfilteredBlocks, this.filePath);
#else
#if SKIPREALWORK
viterbiResultFields = new ViterbiResult();
viterbiResultFields.Fields = new List<ViterbiField>();
#else
if (filterResult.UnfilteredBlocks.Count > 0)
{
ThreadedViterbi tv = new ThreadedViterbi(filterResult.UnfilteredBlocks, RunType.GeneralParse, userStates, this.filePath, this.fileSha1);
viterbiResultFields = tv.RunThreadedViterbi();
TimeSpan ts = DateTime.Now.Subtract(dt);
write("Time elapsed for Viterbi fields: {0}", ts.ToString("c"));
write("Field count: {0}", viterbiResultFields.Fields.Count);
}
#endif
#endif
filterResult.UnfilteredBlocks.Clear(); // Allow gc to clean things up
}
catch (ThreadAbortException)
{
return null;
}
catch (Exception ex)
{
DisplayExceptionMessages(ex, "Viterbi Fields");
return null;
}
return viterbiResultFields;
}
public override double GetValueProbability(byte[] values, int index, Viterbi viterbi)
{
var baseProb = base.GetValueProbability(values, index, viterbi);
if (baseProb == ALMOST_ZERO)
return baseProb;
var path = viterbi.FindPath(index - 1, viterbi.FromState, index - 1);
int lengthIndex = -1;
int distance = 0;
for (int i = path.Count - 1; i >= 0; i--)
{
distance++;
if (path[i] == LengthState)
{
lengthIndex = index - distance;
break;
}
}
if (lengthIndex == -1)
return ALMOST_ZERO;
int headerLength = values[lengthIndex];
if (headerLength >= (distance))
return ALMOST_ZERO;
var serialTypes = new List<int>();
var varint = new List<byte>();
//Grab the varints that sum to the length of the record size (sans header)
for (int i = 0; i < headerLength-1; i++ )
{
byte b = values[lengthIndex + 1 + i];
varint.Add(b);
if( b < 0x80 || varint.Count() == 9)
{
serialTypes.Add(ParseVarint(varint.ToArray()));
varint.Clear();
}
}
int recordLength;
if (_lengthDict.ContainsKey(lengthIndex))
{
recordLength = _lengthDict[lengthIndex];
}
else
{
recordLength = CalculateRecordLength(serialTypes);
_lengthDict.Add(lengthIndex, recordLength);
}
if (recordLength != (distance + 1 - headerLength))
return ALMOST_ZERO;
return base.GetValueProbability(values, index, viterbi);
}
/// <summary>
/// Returns the emission probability of a particular byte. If the probability is explicitly defined it returns
/// that value. Otherwise it returns ALMOST_ZERO.
/// </summary>
/// <param name="value">List of byte outputs from Samsung time state.</param>
/// <param name="index">Index of the byte in values, whose emission probability, given the Samsung time state is required.</param>
/// <returns>The emission probability of an output byte, given the Samsung time state.</returns>
public override double GetValueProbability(byte[] values, int index, Viterbi viterbi)
{
double baseProb = base.GetValueProbability(values, index, viterbi);
//If the probability is zero, no need to do all of this logic.
if (baseProb == ALMOST_ZERO)
return ALMOST_ZERO;
var input = new byte[4];
input[3] = values[index];
input[2] = values[index-1];
input[1] = values[index - 2];
input[0] = values[index - 3];
var yearBytes = new byte[] { input[3], input[2], 0x00, 0x00 };
var monthBytes = new byte[] { 0x00, (byte)(input[2] & 0x0F), 0x00, 0x00 };
var dayBytes = new byte[] { 0x00, 0x00, input[1], 0x00 };
var hourBytes = new byte[] { 0x00, 0x00, (byte)(input[1] & 0x07), input[0] };
var minuteBytes = new byte[] { 0x00, 0x00, 0x00, (byte)(input[0] & 0x3F) };
if (BitConverter.IsLittleEndian)
{
Array.Reverse(yearBytes);
Array.Reverse(monthBytes);
Array.Reverse(dayBytes);
Array.Reverse(hourBytes);
Array.Reverse(minuteBytes);
}
var year = BitConverter.ToInt32(yearBytes, 0) >> 20;
var month = BitConverter.ToInt32(monthBytes, 0) >> 16;
var day = BitConverter.ToInt32(dayBytes, 0) >> 11;
var hour = BitConverter.ToInt32(hourBytes, 0) >> 6;
var minute = BitConverter.ToInt32(minuteBytes, 0);
bool isValidYear = (year >= TimeConstants.START_YEAR && year <= TimeConstants.END_YEAR);
bool isValidMonth = (month >= 1 && month <= 12);
bool isValidDay = (day >= 1 && day <= TimeConstants.MonthDays[month-1]);
bool isValidHour = (hour >= 0 && hour <= 23);
bool isValidMinute = (minute >= 0 && minute <= 59);
if (isValidYear && isValidMonth && isValidHour && isValidDay && isValidMinute) {
if ((month == 2) && (day == 29)) {
try {
var timestamp = new DateTime(year, month, day, hour, minute, 0);
} catch (Exception) {
return ALMOST_ZERO;
}
}
return baseProb;
}
return ALMOST_ZERO;
}
/// <summary>
/// Returns the emission probability of a particular byte. If the probability is explicitly defined it returns
/// that value. Otherwise it returns ALMOST_ZERO.
/// </summary>
/// <param name="value">List of byte outputs from SMS time state.</param>
/// <param name="index">Index of the byte in values, whose emission probability, given the SMS
/// with TZ time state is required.</param>
/// <returns>The emission probability of an output byte, given the SMS time state.</returns>
public override double GetValueProbability(byte[] values, int index, Viterbi viterbi)
{
double baseProb = base.GetValueProbability(values, index, viterbi);
//If the probability is zero, no need to do all of this logic.
if (baseProb == ALMOST_ZERO)
return ALMOST_ZERO;
var input = new byte[7];
input[6] = values[index];
input[5] = values[index - 1];
input[4] = values[index - 2];
input[3] = values[index - 3];
input[2] = values[index - 4];
input[1] = values[index - 5];
input[0] = values[index - 6];
var year = 2000 + Convert.ToInt32(Printer.GetNibbleString(Printer.SwapNibbles(input[0])));
var month = Convert.ToInt32(Printer.GetNibbleString(Printer.SwapNibbles(input[1])));
var day = Convert.ToInt32(Printer.GetNibbleString(Printer.SwapNibbles(input[2])));
var hour = Convert.ToInt32(Printer.GetNibbleString(Printer.SwapNibbles(input[3])));
var minute = Convert.ToInt32(Printer.GetNibbleString(Printer.SwapNibbles(input[4])));
var second = Convert.ToInt32(Printer.GetNibbleString(Printer.SwapNibbles(input[5])));
byte bv = Printer.SwapNibbles(input[6]);
byte bcd = (byte)(bv & 0x37);
byte[] nibbles = Printer.GetNibbles(bcd);
int tz = (nibbles[0]*10) + nibbles[1];
if ((bv & 0x80) != 0) tz = -tz;
bool isValidYear = (year >= TimeConstants.START_YEAR && year <= TimeConstants.END_YEAR);
bool isValidMonth = (month >= 1 && month <= 12);
bool isValidDay = (day >= 1 && day <= TimeConstants.MonthDays[month - 1]);
bool isValidHour = (hour >= 0 && hour <= 23);
bool isValidMinute = (minute >= 0 && minute <= 59);
bool isValidSecond = (second >= 0 && second <= 59);
bool isValidTz = (tz > -96 && tz < 96);
if (isValidYear && isValidMonth && isValidHour && isValidDay && isValidMinute && isValidSecond && isValidTz) {
if ((month == 2) && (day == 29)) {
try {
var timestamp = new DateTime(year, month, day, hour, minute, second);
} catch (Exception) {
return ALMOST_ZERO;
}
}
return baseProb;
}
return ALMOST_ZERO;
}
/// <summary>
/// Returns the emission probability of a particular byte. If the probability is explicitly defined it returns
/// that value. Otherwise it returns ALMOST_ZERO.
/// </summary>
/// <param name="value">List of byte outputs from Unix time state.</param>
/// <param name="index">Index of the byte in values, whose emission probability, given the Unix time state is required.</param>
/// <returns>The emission probability of an output byte, given the Unix time state.</returns>
public override double GetValueProbability(byte[] values, int index, Viterbi viterbi)
{
double baseProb = base.GetValueProbability(values, index, viterbi);
//If the probability is zero, no need to do all of this logic.
if (baseProb == ALMOST_ZERO)
return ALMOST_ZERO;
var input = new byte[4];
input[3] = values[index];
input[2] = values[index - 1];
input[1] = values[index - 2];
input[0] = values[index - 3];
if (BitConverter.IsLittleEndian)
Array.Reverse(input);
var seconds = BitConverter.ToUInt32(input, 0); // B Lynn: make unsigned
try
{
var dateTime = new DateTime(1970, 1, 1).AddSeconds(seconds);
if (dateTime.Year >= TimeConstants.START_YEAR && dateTime.Year <= TimeConstants.END_YEAR)
return baseProb;
else
{
return ALMOST_ZERO;
}
}
catch (Exception)
{
return ALMOST_ZERO;
}
}
public ViterbiResult RunViterbi(List<Block> unfilteredBlocks, RunType type)
{
if (type == RunType.AnchorPoints)
{
var viterbiAnchor = new AnchorViterbi(RunType.GeneralParse, _inputFile);
return viterbiAnchor.Run(unfilteredBlocks);
}
else
{
var viterbi = new Viterbi.Viterbi(type, false);
return viterbi.Run(unfilteredBlocks, _inputFile);
}
}
public ViterbiTest()
{
StateMachine.GeneralParse(ref _generalMachines, ref _generalStates, ref _generalStartState);
//var testMachines = new List<StateMachine>
// {
// StateMachine.Get7BitString_WithLength(1),
// //StateMachine.GetPhoneNumber_All(6),
// //StateMachine.GetTimeStamp_All(1),
// };
//StateMachine.TestStateMachines(testMachines, ref _generalMachines, ref _generalStates, ref _generalStartState);
StateMachine.TestMeta(ref _metaMachines, ref _metaStates, ref _metaStartState);
_metaViterbi = new Viterbi(_metaMachines, _metaStates, _metaStartState);
_generalViterbi = new Viterbi(_generalMachines, _generalStates, _generalStartState);
}
private void PrintResult(Viterbi viterbi, Test test)
{
TestResult result = TestResult.Failed;
int count = 0;
int countWrong = 0;
int countMachines = 0;
if (test.Answers.Length == 0 && test.WrongAnswers.Length == 0 && test.Machines.Length == 0)
{
Console.WriteLine("No answers defined for {0}!", test.Name);
return;
}
MachineList[] machineList = (from v in viterbi.Fields select v.MachineName).ToArray();
for (int i = 0; i < test.Machines.Length; i++)
{
if (machineList.Contains(test.Machines[i]))
countMachines++;
}
for (int i = 0; i < test.WrongAnswers.Length; i++)
{
if (viterbi.FieldStrings.Contains(test.WrongAnswers[i]))
countWrong++;
}
for (int i = 0; i < test.Answers.Length; i++)
{
if (viterbi.FieldStrings.Contains(test.Answers[i]))
count++;
}
var prevColor = Console.ForegroundColor;
//test Answers length could be zero
if (count == test.Answers.Length && countWrong == 0 && countMachines == test.Machines.Length)
{
result = TestResult.Passed;
Console.ForegroundColor = ConsoleColor.Green;
}
else if (count != test.Answers.Length && count > 0 && countWrong == 0 &&
countMachines != test.Machines.Length && countMachines > 0)
{
result = TestResult.Partial;
Console.ForegroundColor = ConsoleColor.Yellow;
}
else
{
result = TestResult.Failed;
Console.ForegroundColor = ConsoleColor.Red;
}
Console.WriteLine("Test \'{0}\': {1}", test.Name, result);
Console.ForegroundColor = prevColor;
}
private void RunBlockThread(List<Block> BlockSet, ref List<ViterbiResult> ResultsOnBlocks, ref object mutex, ref int job_count, ref ManualResetEvent manual)
{
ViterbiResult viterbiResultFields = null;
if (_runType != RunType.Meta)
{
#if !_GENERAL_PARSE
AnchorViterbi anchor_viterbi = new AnchorViterbi(RunType.GeneralParse, _filePath, new List<UserState>(_userStates));
viterbiResultFields = anchor_viterbi.RunThreaded(BlockSet, ref _machines, ref _states, ref _startState, ref _userStates);
#else
Viterbi viterbi = new Viterbi(_runType, false, ref _machines, ref _states, ref _startState, ref _userStates);
viterbiResultFields = viterbi.Run(BlockSet, _filePath);
#endif
}
else
{
Viterbi viterbi = new Viterbi(RunType.Meta, false, ref _machines, ref _states, ref _startState, ref _userStates);
viterbiResultFields = viterbi.Run(BlockSet, _filePath);
}
lock (mutex)
{
ResultsOnBlocks.Add(viterbiResultFields);
job_count--;
if (job_count == 0)
{
manual.Set(); // signal that all threads are done
}
}
}
private void PrintResult(Viterbi viterbi, Test test)
{
TestResult result = TestResult.Failed;
int count = 0;
int countWrong = 0;
int countMachines = 0;
if (test.Answers.Length == 0 && test.WrongAnswers.Length == 0 && test.Machines.Length == 0)
{
Console.WriteLine("No answers defined for {0}!", test.Name);
return;
}
MachineList[] machineList = (from v in viterbi.Fields select v.MachineName).ToArray();
for (int i = 0; i < test.Machines.Length; i++)
{
if (machineList.Contains(test.Machines[i]))
{
countMachines++;
}
}
for (int i = 0; i < test.WrongAnswers.Length; i++)
{
if (viterbi.FieldStrings.Contains(test.WrongAnswers[i]))
{
countWrong++;
}
}
for (int i = 0; i < test.Answers.Length; i++)
{
if (viterbi.FieldStrings.Contains(test.Answers[i]))
{
count++;
}
}
var prevColor = Console.ForegroundColor;
//test Answers length could be zero
if (count == test.Answers.Length && countWrong == 0 && countMachines == test.Machines.Length)
{
result = TestResult.Passed;
Console.ForegroundColor = ConsoleColor.Green;
}
else if (count != test.Answers.Length && count > 0 && countWrong == 0 &&
countMachines != test.Machines.Length && countMachines > 0)
{
result = TestResult.Partial;
Console.ForegroundColor = ConsoleColor.Yellow;
}
else
{
result = TestResult.Failed;
Console.ForegroundColor = ConsoleColor.Red;
}
Console.WriteLine("Test \'{0}\': {1}", test.Name, result);
Console.ForegroundColor = prevColor;
}
private void TestAll(Viterbi viterbi, List<Test> tests)
{
for (int i = 0; i < tests.Count; i++)
{
viterbi.Run(tests[i].RawBytes, 0);
PrintResult(viterbi, tests[i]);
viterbi.FieldStrings.Clear();
viterbi.Fields.Clear();
}
}
/// <summary>
/// Calls the Run() method of Viterbi to start field level inference.
/// </summary>
/// <param name="unfilteredBlocks">List of blocks over which inference is to be performed.</param>
/// <returns>The most likely Viterbi path of fields explaining the set of outputs.</returns>
public ViterbiResult Run(List<Block> unfilteredBlocks)
{
var anchorBlocks = GetAnchorPointBlocks(unfilteredBlocks);
Console.WriteLine("{0} Anchor block bytes.", Block.GetByteTotal(anchorBlocks));
var viterbi = new Viterbi(_runType, false, _userStates);
_fieldResults = viterbi.Run(anchorBlocks, _filePath);
_fieldResults.Duration = _fieldResults.Duration.Add(_anchorResults.Duration);
_fieldResults.MemoryId = MemoryId;
return _fieldResults;
}
public static void Main(string[] args)
{
List<StateMachine> machines = new List<StateMachine>();
List<State> states = new List<State>();
State startState = null;
List<string> textList = new List<string>();
//Y:/110410/Nokia3200b_C.bin
string filePath = "Y:/110410/Nokia3200b_C.bin"; //"Y://111210/random.bin"; //"Y:/110410/Nokia3200b.bin";//"Y:/092010/output_binary.bin";
string outputFilePath = string.Format("Y:/{0}/output_{0}_{1}.txt", DateTime.Now.ToString("MMddyy"), DateTime.Now.ToString("HHmm"));
var testMachines = new List<StateMachine>
{
//StateMachine.GetText(5),
StateMachine.GetPhoneNumber_All(6),
//StateMachine.GetTimeStamp_All(1),
//StateMachine.GetBinaryFF(),
//StateMachine.GetTimestamp_Unix(1)
};
StateMachine.TestStateMachines(testMachines, ref machines, ref states, ref startState);
//StateMachine.GeneralParse(ref machines, ref states, ref startState);
Viterbi viterbi = new Viterbi(machines, states, startState);
//viterbi.Run(Test_MotoTimeStamp, 0);
//Console.WriteLine();
//viterbi.Run(Test_MotoCallLog, 0);
//viterbi.Run(Test_UnicodeThree, 0);
var start = DateTime.Now;
var text = viterbi.Run(filePath);
Console.WriteLine("Runtime: {0}", start - DateTime.Now);
textList = text.Distinct().ToList();
File.WriteAllLines(outputFilePath, textList);
Console.ReadLine();
}
public List<Block> GetAnchorPointBlocks(List<Block> unfilteredBlocks)
{
var anchor = new Viterbi(RunType.AnchorPoints, true, _userStates);
var blocks = new List<Block>();
//The list of Fields in the result are the anchor points
_anchorResults = anchor.Run(unfilteredBlocks, _filePath);
if (_anchorResults.Fields.Count == 0)
//throw new ApplicationException("No anchor points found!");
return new List<Block>();
//The next index after the end of the anchor point
long nextIndex = _anchorResults.Fields[0].OffsetFile + _anchorResults.Fields[0].Length;
//Initialize the start of the first block to the first anchor point - X bytes. If this goes past the start of the file,
// we will just use the start of the file.
long blockStart = Math.Max(0, _anchorResults.Fields[0].OffsetFile - BLOCK_PADDING_BYTES);
//Initialize the end of the first block to be X bytes past the first anchor points. If this
// goes past the end of the file, we will just use the end of the file.
long blockEnd = Math.Min(_fileLength, _anchorResults.Fields[0].OffsetFile + _anchorResults.Fields[0].Length + BLOCK_PADDING_BYTES);
FileStream stream = new FileStream(_filePath, FileMode.Open, FileAccess.Read, FileShare.Read); // BL 8/4
//Get blocks on data based on the anchor points.
for (int i = 1; i < _anchorResults.Fields.Count; i++)
{
//The size of the gap in bytes between the two anchor points
long gap = _anchorResults.Fields[i].OffsetFile - nextIndex;
//If the gap is too long, we need to create a new block
if (gap > LONG_GAP_BYTES || i == _anchorResults.Fields.Count - 1)
{
//Create block using current block start and end
// blocks.Add(GetBlock(blockStart, blockEnd)); // BL 8/4
blocks.Add(GetBlock(stream, blockStart, blockEnd)); // BL 8/4
blockStart = Math.Max(0, _anchorResults.Fields[i].OffsetFile - BLOCK_PADDING_BYTES);
}
blockEnd = Math.Min(_fileLength, _anchorResults.Fields[i].OffsetFile + _anchorResults.Fields[i].Length + BLOCK_PADDING_BYTES);
nextIndex = _anchorResults.Fields[i].OffsetFile + _anchorResults.Fields[i].Length;
}
stream.Close(); // BL 8/4
return blocks;
}