/// <summary>
/// Parse an XML document in as a SkipperFile
/// </summary>
/// <param name="xtr">XmlReader representing the document</param>
/// <returns>True if the file could be parsed, false otherwise</returns>
private bool Parse(XmlReader xtr)
{
if (xtr == null)
{
return(false);
}
try
{
bool valid = false;
xtr.MoveToContent();
while (!xtr.EOF)
{
if (xtr.NodeType != XmlNodeType.Element)
{
xtr.Read();
}
switch (xtr.Name.ToLowerInvariant())
{
case "detector":
valid = true;
xtr.Read();
break;
case "name":
Name = xtr.ReadElementContentAsString();
break;
case "author":
Author = xtr.ReadElementContentAsString();
break;
case "version":
Version = xtr.ReadElementContentAsString();
break;
case "rule":
SkipperRule rule = ParseRule(xtr);
if (rule != null)
{
Rules.Add(rule);
}
xtr.Skip();
break;
default:
xtr.Read();
break;
}
}
return(valid);
}
catch
{
return(false);
}
}
/// <summary>
/// Detect header skipper compliance and create an output file
/// </summary>
/// <param name="file">Name of the file to be parsed</param>
/// <param name="outDir">Output directory to write the file to, empty means the same directory as the input file</param>
/// <param name="nostore">True if headers should not be stored in the database, false otherwise</param>
/// <returns>True if the output file was created, false otherwise</returns>
public static bool DetectTransformStore(string file, string outDir, bool nostore)
{
// Create the output directory if it doesn't exist
DirectoryExtensions.Ensure(outDir, create: true);
logger.User($"\nGetting skipper information for '{file}'");
// Get the skipper rule that matches the file, if any
SkipperRule rule = GetMatchingRule(file, string.Empty);
// If we have an empty rule, return false
if (rule.Tests == null || rule.Tests.Count == 0 || rule.Operation != HeaderSkipOperation.None)
return false;
logger.User("File has a valid copier header");
// Get the header bytes from the file first
string hstr;
try
{
// Extract the header as a string for the database
#if NET_FRAMEWORK
using (var fs = FileExtensions.TryOpenRead(file))
{
#else
using var fs = FileExtensions.TryOpenRead(file);
#endif
byte[] hbin = new byte[(int)rule.StartOffset];
fs.Read(hbin, 0, (int)rule.StartOffset);
hstr = Utilities.ByteArrayToString(hbin);
#if NET_FRAMEWORK
}
#endif
}
catch
{
return false;
}
// Apply the rule to the file
string newfile = (string.IsNullOrWhiteSpace(outDir) ? Path.GetFullPath(file) + ".new" : Path.Combine(outDir, Path.GetFileName(file)));
rule.TransformFile(file, newfile);
// If the output file doesn't exist, return false
if (!File.Exists(newfile))
return false;
// Now add the information to the database if it's not already there
if (!nostore)
{
BaseFile baseFile = FileExtensions.GetInfo(newfile, hashes: Hash.SHA1, asFiles: TreatAsFile.NonArchive);
DatabaseTools.AddHeaderToDatabase(hstr, Utilities.ByteArrayToString(baseFile.SHA1), rule.SourceFile);
}
return true;
}
/// <summary>
/// Get the SkipperRule associated with a given stream
/// </summary>
/// <param name="input">Name of the file to be checked</param>
/// <param name="skipperName">Name of the skipper to be used, blank to find a matching skipper</param>
/// <param name="keepOpen">True if the underlying stream should be kept open, false otherwise</param>
/// <returns>The SkipperRule that matched the file</returns>
public static SkipperRule GetMatchingRule(Stream input, string skipperName, bool keepOpen = false)
{
SkipperRule skipperRule = new SkipperRule();
// If we have a null skipper name, we return since we're not matching skippers
if (skipperName == null)
return skipperRule;
// Loop through and find a Skipper that has the right name
logger.Verbose("Beginning search for matching header skip rules");
List<SkipperFile> tempList = new List<SkipperFile>();
tempList.AddRange(List);
// Loop through all known SkipperFiles
foreach (SkipperFile skipper in tempList)
{
skipperRule = skipper.GetMatchingRule(input, skipperName);
if (skipperRule != null)
break;
}
// If we're not keeping the stream open, dispose of the binary reader
if (!keepOpen)
input.Dispose();
// If the SkipperRule is null, make it empty
if (skipperRule == null)
skipperRule = new SkipperRule();
// If we have a blank rule, inform the user
if (skipperRule.Tests == null)
logger.Verbose("No matching rule found!");
else
logger.User("Matching rule found!");
return skipperRule;
}
/// <summary>
/// Create a Skipper object parsed from an input file
/// </summary>
/// <param name="filename">Name of the file to parse</param>
public Skipper(string filename)
{
Rules = new List <SkipperRule>();
SourceFile = Path.GetFileNameWithoutExtension(filename);
Logger logger = new Logger();
XmlReader xtr = Utilities.GetXmlTextReader(filename);
if (xtr == null)
{
return;
}
bool valid = false;
xtr.MoveToContent();
while (!xtr.EOF)
{
if (xtr.NodeType != XmlNodeType.Element)
{
xtr.Read();
}
switch (xtr.Name.ToLowerInvariant())
{
case "detector":
valid = true;
xtr.Read();
break;
case "name":
Name = xtr.ReadElementContentAsString();
break;
case "author":
Author = xtr.ReadElementContentAsString();
break;
case "version":
Version = xtr.ReadElementContentAsString();
break;
case "rule":
// Get the information from the rule first
SkipperRule rule = new SkipperRule
{
StartOffset = null,
EndOffset = null,
Operation = HeaderSkipOperation.None,
Tests = new List <SkipperTest>(),
SourceFile = Path.GetFileNameWithoutExtension(filename),
};
if (xtr.GetAttribute("start_offset") != null)
{
string offset = xtr.GetAttribute("start_offset");
if (offset.ToLowerInvariant() == "eof")
{
rule.StartOffset = null;
}
else
{
rule.StartOffset = Convert.ToInt64(offset, 16);
}
}
if (xtr.GetAttribute("end_offset") != null)
{
string offset = xtr.GetAttribute("end_offset");
if (offset.ToLowerInvariant() == "eof")
{
rule.EndOffset = null;
}
else
{
rule.EndOffset = Convert.ToInt64(offset, 16);
}
}
if (xtr.GetAttribute("operation") != null)
{
string operation = xtr.GetAttribute("operation");
switch (operation.ToLowerInvariant())
{
case "bitswap":
rule.Operation = HeaderSkipOperation.Bitswap;
break;
case "byteswap":
rule.Operation = HeaderSkipOperation.Byteswap;
break;
case "wordswap":
rule.Operation = HeaderSkipOperation.Wordswap;
break;
}
}
// Now read the individual tests into the Rule
XmlReader subreader = xtr.ReadSubtree();
if (subreader != null)
{
while (!subreader.EOF)
{
if (subreader.NodeType != XmlNodeType.Element)
{
subreader.Read();
}
// Get the test type
SkipperTest test = new SkipperTest
{
Offset = 0,
Value = new byte[0],
Result = true,
Mask = new byte[0],
Size = 0,
Operator = HeaderSkipTestFileOperator.Equal,
};
switch (subreader.Name.ToLowerInvariant())
{
case "data":
test.Type = HeaderSkipTest.Data;
break;
case "or":
test.Type = HeaderSkipTest.Or;
break;
case "xor":
test.Type = HeaderSkipTest.Xor;
break;
case "and":
test.Type = HeaderSkipTest.And;
break;
case "file":
test.Type = HeaderSkipTest.File;
break;
default:
subreader.Read();
break;
}
// Now populate all the parts that we can
if (subreader.GetAttribute("offset") != null)
{
string offset = subreader.GetAttribute("offset");
if (offset.ToLowerInvariant() == "eof")
{
test.Offset = null;
}
else
{
test.Offset = Convert.ToInt64(offset, 16);
}
}
if (subreader.GetAttribute("value") != null)
{
string value = subreader.GetAttribute("value");
// http://stackoverflow.com/questions/321370/how-can-i-convert-a-hex-string-to-a-byte-array
test.Value = new byte[value.Length / 2];
for (int index = 0; index < test.Value.Length; index++)
{
string byteValue = value.Substring(index * 2, 2);
test.Value[index] = byte.Parse(byteValue, NumberStyles.HexNumber, CultureInfo.InvariantCulture);
}
}
if (subreader.GetAttribute("result") != null)
{
string result = subreader.GetAttribute("result");
switch (result.ToLowerInvariant())
{
case "false":
test.Result = false;
break;
case "true":
default:
test.Result = true;
break;
}
}
if (subreader.GetAttribute("mask") != null)
{
string mask = subreader.GetAttribute("mask");
// http://stackoverflow.com/questions/321370/how-can-i-convert-a-hex-string-to-a-byte-array
test.Mask = new byte[mask.Length / 2];
for (int index = 0; index < test.Mask.Length; index++)
{
string byteValue = mask.Substring(index * 2, 2);
test.Mask[index] = byte.Parse(byteValue, NumberStyles.HexNumber, CultureInfo.InvariantCulture);
}
}
if (subreader.GetAttribute("size") != null)
{
string size = subreader.GetAttribute("size");
if (size.ToLowerInvariant() == "po2")
{
test.Size = null;
}
else
{
test.Size = Convert.ToInt64(size, 16);
}
}
if (subreader.GetAttribute("operator") != null)
{
string oper = subreader.GetAttribute("operator");
switch (oper.ToLowerInvariant())
{
case "less":
test.Operator = HeaderSkipTestFileOperator.Less;
break;
case "greater":
test.Operator = HeaderSkipTestFileOperator.Greater;
break;
case "equal":
default:
test.Operator = HeaderSkipTestFileOperator.Equal;
break;
}
}
// Add the created test to the rule
rule.Tests.Add(test);
subreader.Read();
}
}
// Add the created rule to the skipper
Rules.Add(rule);
xtr.Skip();
break;
default:
xtr.Read();
break;
}
}
// If we somehow have an invalid file, zero out the fields
if (!valid)
{
Name = null;
Author = null;
Version = null;
Rules = null;
SourceFile = null;
}
}
/// <summary>
/// Get the SkipperRule associated with a given stream
/// </summary>
/// <param name="input">Name of the file to be checked</param>
/// <param name="skipperName">Name of the skipper to be used, blank to find a matching skipper</param>
/// <param name="logger">Logger object for file and console output</param>
/// <param name="keepOpen">True if the underlying stream should be kept open, false otherwise</param>
/// <returns>The SkipperRule that matched the file</returns>
public static SkipperRule GetMatchingRule(Stream input, string skipperName, bool keepOpen = false)
{
SkipperRule skipperRule = new SkipperRule();
// If we have a null skipper name, we return since we're not matching skippers
if (skipperName == null)
{
return(skipperRule);
}
// Loop through and find a Skipper that has the right name
Globals.Logger.Verbose("Beginning search for matching header skip rules");
List <Skipper> tempList = new List <Skipper>();
tempList.AddRange(List);
foreach (Skipper skipper in tempList)
{
// If we're searching for the skipper OR we have a match to an inputted one
if (String.IsNullOrWhiteSpace(skipperName) ||
(!String.IsNullOrWhiteSpace(skipper.Name) && skipperName.ToLowerInvariant() == skipper.Name.ToLowerInvariant()) ||
(!String.IsNullOrWhiteSpace(skipper.Name) && skipperName.ToLowerInvariant() == skipper.SourceFile.ToLowerInvariant()))
{
// Loop through the rules until one is found that works
BinaryReader br = new BinaryReader(input);
foreach (SkipperRule rule in skipper.Rules)
{
// Always reset the stream back to the original place
input.Seek(0, SeekOrigin.Begin);
// For each rule, make sure it passes each test
bool success = true;
foreach (SkipperTest test in rule.Tests)
{
bool result = true;
switch (test.Type)
{
case HeaderSkipTest.Data:
// First seek to the correct position
if (test.Offset == null)
{
input.Seek(0, SeekOrigin.End);
}
else if (test.Offset > 0 && test.Offset <= input.Length)
{
input.Seek((long)test.Offset, SeekOrigin.Begin);
}
else if (test.Offset < 0 && Math.Abs((long)test.Offset) <= input.Length)
{
input.Seek((long)test.Offset, SeekOrigin.End);
}
// Then read and compare bytewise
result = true;
for (int i = 0; i < test.Value.Length; i++)
{
try
{
if (br.ReadByte() != test.Value[i])
{
result = false;
break;
}
}
catch
{
result = false;
break;
}
}
// Return if the expected and actual results match
success &= (result == test.Result);
break;
case HeaderSkipTest.Or:
case HeaderSkipTest.Xor:
case HeaderSkipTest.And:
// First seek to the correct position
if (test.Offset == null)
{
input.Seek(0, SeekOrigin.End);
}
else if (test.Offset > 0 && test.Offset <= input.Length)
{
input.Seek((long)test.Offset, SeekOrigin.Begin);
}
else if (test.Offset < 0 && Math.Abs((long)test.Offset) <= input.Length)
{
input.Seek((long)test.Offset, SeekOrigin.End);
}
result = true;
try
{
// Then apply the mask if it exists
byte[] read = br.ReadBytes(test.Mask.Length);
byte[] masked = new byte[test.Mask.Length];
for (int i = 0; i < read.Length; i++)
{
masked[i] = (byte)(test.Type == HeaderSkipTest.And ? read[i] & test.Mask[i] :
(test.Type == HeaderSkipTest.Or ? read[i] | test.Mask[i] : read[i] ^ test.Mask[i])
);
}
// Finally, compare it against the value
for (int i = 0; i < test.Value.Length; i++)
{
if (masked[i] != test.Value[i])
{
result = false;
break;
}
}
}
catch
{
result = false;
}
// Return if the expected and actual results match
success &= (result == test.Result);
break;
case HeaderSkipTest.File:
// First get the file size from stream
long size = input.Length;
// If we have a null size, check that the size is a power of 2
result = true;
if (test.Size == null)
{
// http://stackoverflow.com/questions/600293/how-to-check-if-a-number-is-a-power-of-2
result = (((ulong)size & ((ulong)size - 1)) == 0);
}
else if (test.Operator == HeaderSkipTestFileOperator.Less)
{
result = (size < test.Size);
}
else if (test.Operator == HeaderSkipTestFileOperator.Greater)
{
result = (size > test.Size);
}
else if (test.Operator == HeaderSkipTestFileOperator.Equal)
{
result = (size == test.Size);
}
// Return if the expected and actual results match
success &= (result == test.Result);
break;
}
}
// If we still have a success, then return this rule
if (success)
{
// If we're not keeping the stream open, dispose of the binary reader
if (!keepOpen)
{
input.Dispose();
}
Globals.Logger.User(" Matching rule found!");
return(rule);
}
}
}
}
// If we're not keeping the stream open, dispose of the binary reader
if (!keepOpen)
{
input.Dispose();
}
// If we have a blank rule, inform the user
if (skipperRule.Tests == null)
{
Globals.Logger.Verbose("No matching rule found!");
}
return(skipperRule);
}
/// <summary>
/// Parse an XML document in as a SkipperRule
/// </summary>
/// <param name="xtr">XmlReader representing the document</param>
/// <returns>Filled SkipperRule on success, null otherwise</returns>
private SkipperRule ParseRule(XmlReader xtr)
{
if (xtr == null)
{
return(null);
}
try
{
// Get the information from the rule first
SkipperRule rule = new SkipperRule
{
StartOffset = null,
EndOffset = null,
Operation = HeaderSkipOperation.None,
Tests = new List <SkipperTest>(),
SourceFile = this.SourceFile,
};
if (xtr.GetAttribute("start_offset") != null)
{
string offset = xtr.GetAttribute("start_offset");
if (offset.ToLowerInvariant() == "eof")
{
rule.StartOffset = null;
}
else
{
rule.StartOffset = Convert.ToInt64(offset, 16);
}
}
if (xtr.GetAttribute("end_offset") != null)
{
string offset = xtr.GetAttribute("end_offset");
if (offset.ToLowerInvariant() == "eof")
{
rule.EndOffset = null;
}
else
{
rule.EndOffset = Convert.ToInt64(offset, 16);
}
}
if (xtr.GetAttribute("operation") != null)
{
string operation = xtr.GetAttribute("operation");
switch (operation.ToLowerInvariant())
{
case "bitswap":
rule.Operation = HeaderSkipOperation.Bitswap;
break;
case "byteswap":
rule.Operation = HeaderSkipOperation.Byteswap;
break;
case "wordswap":
rule.Operation = HeaderSkipOperation.Wordswap;
break;
}
}
// Now read the individual tests into the Rule
XmlReader subreader = xtr.ReadSubtree();
if (subreader != null)
{
while (!subreader.EOF)
{
if (subreader.NodeType != XmlNodeType.Element)
{
subreader.Read();
}
SkipperTest test = ParseTest(xtr);
// Add the created test to the rule
rule.Tests.Add(test);
subreader.Read();
}
}
return(rule);
}
catch
{
return(null);
}
}