private void RunBlockThread(List <Block> BlockSet, ref List <ViterbiResult> ResultsOnBlocks, ref object mutex, ref int job_count, ref ManualResetEvent manual)
{
ViterbiResult viterbiResultFields = null;
if (_runType != RunType.Meta)
{
#if !_GENERAL_PARSE
AnchorViterbi anchor_viterbi = new AnchorViterbi(RunType.GeneralParse, _filePath, new List <UserState>(_userStates));
viterbiResultFields = anchor_viterbi.RunThreaded(BlockSet, ref _machines, ref _states, ref _startState, ref _userStates);
#else
Viterbi viterbi = new Viterbi(_runType, false, ref _machines, ref _states, ref _startState, ref _userStates);
viterbiResultFields = viterbi.Run(BlockSet, _filePath);
#endif
}
else
{
Viterbi viterbi = new Viterbi(RunType.Meta, false, ref _machines, ref _states, ref _startState, ref _userStates);
viterbiResultFields = viterbi.Run(BlockSet, _filePath);
}
lock (mutex)
{
ResultsOnBlocks.Add(viterbiResultFields);
job_count--;
if (job_count == 0)
{
manual.Set(); // signal that all threads are done
}
}
}
public List <Block> GetAnchorPointBlocks(List <Block> unfilteredBlocks)
{
var anchor = new Viterbi(RunType.AnchorPoints, true, _userStates);
var blocks = new List <Block>();
//The list of Fields in the result are the anchor points
_anchorResults = anchor.Run(unfilteredBlocks, _filePath);
if (_anchorResults.Fields.Count == 0)
{
//throw new ApplicationException("No anchor points found!");
return(new List <Block>());
}
//The next index after the end of the anchor point
long nextIndex = _anchorResults.Fields[0].OffsetFile + _anchorResults.Fields[0].Length;
//Initialize the start of the first block to the first anchor point - X bytes. If this goes past the start of the file,
// we will just use the start of the file.
long blockStart = Math.Max(0, _anchorResults.Fields[0].OffsetFile - BLOCK_PADDING_BYTES);
//Initialize the end of the first block to be X bytes past the first anchor points. If this
// goes past the end of the file, we will just use the end of the file.
long blockEnd = Math.Min(_fileLength, _anchorResults.Fields[0].OffsetFile + _anchorResults.Fields[0].Length + BLOCK_PADDING_BYTES);
FileStream stream = new FileStream(_filePath, FileMode.Open, FileAccess.Read, FileShare.Read); // BL 8/4
//Get blocks on data based on the anchor points.
for (int i = 1; i < _anchorResults.Fields.Count; i++)
{
//The size of the gap in bytes between the two anchor points
long gap = _anchorResults.Fields[i].OffsetFile - nextIndex;
//If the gap is too long, we need to create a new block
if (gap > LONG_GAP_BYTES || i == _anchorResults.Fields.Count - 1)
{
//Create block using current block start and end
// blocks.Add(GetBlock(blockStart, blockEnd)); // BL 8/4
blocks.Add(GetBlock(stream, blockStart, blockEnd)); // BL 8/4
blockStart = Math.Max(0, _anchorResults.Fields[i].OffsetFile - BLOCK_PADDING_BYTES);
}
blockEnd = Math.Min(_fileLength, _anchorResults.Fields[i].OffsetFile + _anchorResults.Fields[i].Length + BLOCK_PADDING_BYTES);
nextIndex = _anchorResults.Fields[i].OffsetFile + _anchorResults.Fields[i].Length;
}
stream.Close(); // BL 8/4
return(blocks);
}
private void TestAll(Viterbi viterbi, List <Test> tests)
{
for (int i = 0; i < tests.Count; i++)
{
viterbi.Run(tests[i].RawBytes, 0);
PrintResult(viterbi, tests[i]);
viterbi.FieldStrings.Clear();
viterbi.Fields.Clear();
}
}
public ViterbiResult RunViterbi(List <Block> unfilteredBlocks, RunType type)
{
if (type == RunType.AnchorPoints)
{
var viterbiAnchor = new AnchorViterbi(RunType.GeneralParse, _inputFile);
return(viterbiAnchor.Run(unfilteredBlocks));
}
else
{
var viterbi = new Viterbi.Viterbi(type, false);
return(viterbi.Run(unfilteredBlocks, _inputFile));
}
}
// Using this method instead of the Run() method above.
public ViterbiResult RunThreaded(List <Block> unfilteredBlocks, ref List <StateMachine> machines,
ref List <State> states, ref State startState, ref List <UserState> userStates)
{
var anchorBlocks = GetAnchorPointBlocks(unfilteredBlocks);
Console.WriteLine("{0} Anchor block bytes.", Block.GetByteTotal(anchorBlocks));
var viterbi = new Viterbi(_runType, false, ref machines, ref states, ref startState, ref userStates);
var result = viterbi.Run(anchorBlocks, _filePath);
result.Duration = result.Duration.Add(_anchorResults.Duration);
return(result);
}
/// <summary>
/// Calls the Run() method of Viterbi to start field level inference.
/// </summary>
/// <param name="unfilteredBlocks">List of blocks over which inference is to be performed.</param>
/// <returns>The most likely Viterbi path of fields explaining the set of outputs.</returns>
public ViterbiResult Run(List <Block> unfilteredBlocks)
{
var anchorBlocks = GetAnchorPointBlocks(unfilteredBlocks);
Console.WriteLine("{0} Anchor block bytes.", Block.GetByteTotal(anchorBlocks));
var viterbi = new Viterbi(_runType, false, _userStates);
_fieldResults = viterbi.Run(anchorBlocks, _filePath);
_fieldResults.Duration = _fieldResults.Duration.Add(_anchorResults.Duration);
_fieldResults.MemoryId = MemoryId;
return(_fieldResults);
}
public static void Main(string[] args)
{
List <StateMachine> machines = new List <StateMachine>();
List <State> states = new List <State>();
State startState = null;
List <string> textList = new List <string>();
//Y:/110410/Nokia3200b_C.bin
string filePath = "Y:/110410/Nokia3200b_C.bin"; //"Y://111210/random.bin"; //"Y:/110410/Nokia3200b.bin";//"Y:/092010/output_binary.bin";
string outputFilePath = string.Format("Y:/{0}/output_{0}_{1}.txt", DateTime.Now.ToString("MMddyy"), DateTime.Now.ToString("HHmm"));
var testMachines = new List <StateMachine>
{
//StateMachine.GetText(5),
StateMachine.GetPhoneNumber_All(6),
//StateMachine.GetTimeStamp_All(1),
//StateMachine.GetBinaryFF(),
//StateMachine.GetTimestamp_Unix(1)
};
StateMachine.TestStateMachines(testMachines, ref machines, ref states, ref startState);
//StateMachine.GeneralParse(ref machines, ref states, ref startState);
Viterbi viterbi = new Viterbi(machines, states, startState);
//viterbi.Run(Test_MotoTimeStamp, 0);
//Console.WriteLine();
//viterbi.Run(Test_MotoCallLog, 0);
//viterbi.Run(Test_UnicodeThree, 0);
var start = DateTime.Now;
var text = viterbi.Run(filePath);
Console.WriteLine("Runtime: {0}", start - DateTime.Now);
textList = text.Distinct().ToList();
File.WriteAllLines(outputFilePath, textList);
Console.ReadLine();
}
public void TestString(string input)
{
Console.WriteLine("Testing input: {0}", input);
byte[] raw = Utilities.GetBytes(input);
_generalViterbi.Run(raw, 0);
for (int i = 0; i < _generalViterbi.FieldStrings.Count(); i++)
{
Console.WriteLine("{0} : {1}", _generalViterbi.Fields[i].FieldString,
_generalViterbi.Fields[i].MachineName);
}
_generalViterbi.FieldStrings.Clear();
_generalViterbi.Fields.Clear();
}
public static void Main(string[] args)
{
List<StateMachine> machines = new List<StateMachine>();
List<State> states = new List<State>();
State startState = null;
List<string> textList = new List<string>();
//Y:/110410/Nokia3200b_C.bin
string filePath = "Y:/110410/Nokia3200b_C.bin"; //"Y://111210/random.bin"; //"Y:/110410/Nokia3200b.bin";//"Y:/092010/output_binary.bin";
string outputFilePath = string.Format("Y:/{0}/output_{0}_{1}.txt", DateTime.Now.ToString("MMddyy"), DateTime.Now.ToString("HHmm"));
var testMachines = new List<StateMachine>
{
//StateMachine.GetText(5),
StateMachine.GetPhoneNumber_All(6),
//StateMachine.GetTimeStamp_All(1),
//StateMachine.GetBinaryFF(),
//StateMachine.GetTimestamp_Unix(1)
};
StateMachine.TestStateMachines(testMachines, ref machines, ref states, ref startState);
//StateMachine.GeneralParse(ref machines, ref states, ref startState);
Viterbi viterbi = new Viterbi(machines, states, startState);
//viterbi.Run(Test_MotoTimeStamp, 0);
//Console.WriteLine();
//viterbi.Run(Test_MotoCallLog, 0);
//viterbi.Run(Test_UnicodeThree, 0);
var start = DateTime.Now;
var text = viterbi.Run(filePath);
Console.WriteLine("Runtime: {0}", start - DateTime.Now);
textList = text.Distinct().ToList();
File.WriteAllLines(outputFilePath, textList);
Console.ReadLine();
}
private ViterbiResult RunViterbi(FilterResult filterResult, string fileSha1)
{
// Load any user-defined state machines.
List<UserState> userStates;
try
{
userStates = Loader.LoadUserStates(MainForm.Program.UserStatesEnabled,
MainForm.Program.UserStatesPath);
}
catch (Exception ex)
{
DisplayExceptionMessages(ex, "User-Defined States");
return null;
}
ViterbiResult viterbiResultFields = null;
try
{
write("Running Viterbi on fields");
DateTime dt = DateTime.Now;
#if _GENERALPARSE
Viterbi viterbi = new Viterbi(RunType.GeneralParse, false);
viterbiResultFields = viterbi.Run(filterResult.UnfilteredBlocks, this.filePath);
#else
#if SKIPREALWORK
viterbiResultFields = new ViterbiResult();
viterbiResultFields.Fields = new List<ViterbiField>();
#else
if (filterResult.UnfilteredBlocks.Count > 0)
{
ThreadedViterbi tv = new ThreadedViterbi(filterResult.UnfilteredBlocks, RunType.GeneralParse, userStates, this.filePath, this.fileSha1);
viterbiResultFields = tv.RunThreadedViterbi();
TimeSpan ts = DateTime.Now.Subtract(dt);
write("Time elapsed for Viterbi fields: {0}", ts.ToString("c"));
write("Field count: {0}", viterbiResultFields.Fields.Count);
}
#endif
#endif
filterResult.UnfilteredBlocks.Clear(); // Allow gc to clean things up
}
catch (ThreadAbortException)
{
return null;
}
catch (Exception ex)
{
DisplayExceptionMessages(ex, "Viterbi Fields");
return null;
}
return viterbiResultFields;
}
private void RunBlockThread(List<Block> BlockSet, ref List<ViterbiResult> ResultsOnBlocks, ref object mutex, ref int job_count, ref ManualResetEvent manual)
{
ViterbiResult viterbiResultFields = null;
if (_runType != RunType.Meta)
{
#if !_GENERAL_PARSE
AnchorViterbi anchor_viterbi = new AnchorViterbi(RunType.GeneralParse, _filePath, new List<UserState>(_userStates));
viterbiResultFields = anchor_viterbi.RunThreaded(BlockSet, ref _machines, ref _states, ref _startState, ref _userStates);
#else
Viterbi viterbi = new Viterbi(_runType, false, ref _machines, ref _states, ref _startState, ref _userStates);
viterbiResultFields = viterbi.Run(BlockSet, _filePath);
#endif
}
else
{
Viterbi viterbi = new Viterbi(RunType.Meta, false, ref _machines, ref _states, ref _startState, ref _userStates);
viterbiResultFields = viterbi.Run(BlockSet, _filePath);
}
lock (mutex)
{
ResultsOnBlocks.Add(viterbiResultFields);
job_count--;
if (job_count == 0)
{
manual.Set(); // signal that all threads are done
}
}
}
public List<Block> GetAnchorPointBlocks(List<Block> unfilteredBlocks)
{
var anchor = new Viterbi(RunType.AnchorPoints, true, _userStates);
var blocks = new List<Block>();
//The list of Fields in the result are the anchor points
_anchorResults = anchor.Run(unfilteredBlocks, _filePath);
if (_anchorResults.Fields.Count == 0)
//throw new ApplicationException("No anchor points found!");
return new List<Block>();
//The next index after the end of the anchor point
long nextIndex = _anchorResults.Fields[0].OffsetFile + _anchorResults.Fields[0].Length;
//Initialize the start of the first block to the first anchor point - X bytes. If this goes past the start of the file,
// we will just use the start of the file.
long blockStart = Math.Max(0, _anchorResults.Fields[0].OffsetFile - BLOCK_PADDING_BYTES);
//Initialize the end of the first block to be X bytes past the first anchor points. If this
// goes past the end of the file, we will just use the end of the file.
long blockEnd = Math.Min(_fileLength, _anchorResults.Fields[0].OffsetFile + _anchorResults.Fields[0].Length + BLOCK_PADDING_BYTES);
FileStream stream = new FileStream(_filePath, FileMode.Open, FileAccess.Read, FileShare.Read); // BL 8/4
//Get blocks on data based on the anchor points.
for (int i = 1; i < _anchorResults.Fields.Count; i++)
{
//The size of the gap in bytes between the two anchor points
long gap = _anchorResults.Fields[i].OffsetFile - nextIndex;
//If the gap is too long, we need to create a new block
if (gap > LONG_GAP_BYTES || i == _anchorResults.Fields.Count - 1)
{
//Create block using current block start and end
// blocks.Add(GetBlock(blockStart, blockEnd)); // BL 8/4
blocks.Add(GetBlock(stream, blockStart, blockEnd)); // BL 8/4
blockStart = Math.Max(0, _anchorResults.Fields[i].OffsetFile - BLOCK_PADDING_BYTES);
}
blockEnd = Math.Min(_fileLength, _anchorResults.Fields[i].OffsetFile + _anchorResults.Fields[i].Length + BLOCK_PADDING_BYTES);
nextIndex = _anchorResults.Fields[i].OffsetFile + _anchorResults.Fields[i].Length;
}
stream.Close(); // BL 8/4
return blocks;
}
// Using this method instead of the Run() method above.
public ViterbiResult RunThreaded(List<Block> unfilteredBlocks, ref List<StateMachine> machines,
ref List<State> states, ref State startState, ref List<UserState> userStates)
{
var anchorBlocks = GetAnchorPointBlocks(unfilteredBlocks);
Console.WriteLine("{0} Anchor block bytes.", Block.GetByteTotal(anchorBlocks));
var viterbi = new Viterbi(_runType, false, ref machines, ref states, ref startState, ref userStates);
var result = viterbi.Run(anchorBlocks, _filePath);
result.Duration = result.Duration.Add(_anchorResults.Duration);
return result;
}
/// <summary>
/// Calls the Run() method of Viterbi to start field level inference.
/// </summary>
/// <param name="unfilteredBlocks">List of blocks over which inference is to be performed.</param>
/// <returns>The most likely Viterbi path of fields explaining the set of outputs.</returns>
public ViterbiResult Run(List<Block> unfilteredBlocks)
{
var anchorBlocks = GetAnchorPointBlocks(unfilteredBlocks);
Console.WriteLine("{0} Anchor block bytes.", Block.GetByteTotal(anchorBlocks));
var viterbi = new Viterbi(_runType, false, _userStates);
_fieldResults = viterbi.Run(anchorBlocks, _filePath);
_fieldResults.Duration = _fieldResults.Duration.Add(_anchorResults.Duration);
_fieldResults.MemoryId = MemoryId;
return _fieldResults;
}
private void TestAll(Viterbi viterbi, List<Test> tests)
{
for (int i = 0; i < tests.Count; i++)
{
viterbi.Run(tests[i].RawBytes, 0);
PrintResult(viterbi, tests[i]);
viterbi.FieldStrings.Clear();
viterbi.Fields.Clear();
}
}
public ViterbiResult RunViterbi(List<Block> unfilteredBlocks, RunType type)
{
if (type == RunType.AnchorPoints)
{
var viterbiAnchor = new AnchorViterbi(RunType.GeneralParse, _inputFile);
return viterbiAnchor.Run(unfilteredBlocks);
}
else
{
var viterbi = new Viterbi.Viterbi(type, false);
return viterbi.Run(unfilteredBlocks, _inputFile);
}
}
