public InstructionInfo(int bitness, string hexBytes, Code code, DecoderOptions options)
{
Bitness = bitness;
HexBytes = hexBytes;
Code = code;
Options = options;
}
public DecoderTestInfo(int bitness, Code code, string hexBytes, DecoderOptions options)
{
Bitness = bitness;
Code = code;
HexBytes = hexBytes;
Options = options;
}
public InstructionInfo(int bitness, string hexBytes, Code code)
{
Bitness = bitness;
HexBytes = hexBytes;
Code = code;
Options = DecoderOptions.None;
}
public InstructionInfo(int codeSize, string hexBytes, Code code)
{
CodeSize = codeSize;
HexBytes = hexBytes;
Code = code;
Options = DecoderOptions.None;
}
Decoder CreateDecoder(int codeSize, byte[] hexBytes, DecoderOptions options)
{
var codeReader = new ByteArrayCodeReader(hexBytes);
var decoder = Decoder.Create(codeSize, codeReader, options);
switch (codeSize)
{
case 16:
decoder.IP = DecoderConstants.DEFAULT_IP16;
break;
case 32:
decoder.IP = DecoderConstants.DEFAULT_IP32;
break;
case 64:
decoder.IP = DecoderConstants.DEFAULT_IP64;
break;
default:
throw new ArgumentOutOfRangeException(nameof(codeSize));
}
Assert.Equal(codeSize, decoder.Bitness);
return(decoder);
}
protected void EncodeInvalidBase(int codeSize, Code code, string hexBytes, DecoderOptions options, int invalidCodeSize)
{
var origBytes = HexUtils.ToByteArray(hexBytes);
var decoder = CreateDecoder(codeSize, origBytes, options);
var origRip = decoder.IP;
var origInstr = decoder.Decode();
Assert.Equal(code, origInstr.Code);
Assert.Equal(origBytes.Length, origInstr.ByteLength);
Assert.True(origInstr.ByteLength <= Iced.Intel.DecoderConstants.MaxInstructionLength);
Assert.Equal((ushort)origRip, origInstr.IP16);
Assert.Equal((uint)origRip, origInstr.IP32);
Assert.Equal(origRip, origInstr.IP);
var afterRip = decoder.IP;
Assert.Equal((ushort)afterRip, origInstr.NextIP16);
Assert.Equal((uint)afterRip, origInstr.NextIP32);
Assert.Equal(afterRip, origInstr.NextIP);
var writer = new CodeWriterImpl();
var encoder = CreateEncoder(invalidCodeSize, writer);
var origInstrCopy = origInstr;
bool result = encoder.TryEncode(origInstr, origRip, out uint encodedInstrLen, out string errorMessage);
Assert.Equal(invalidCodeSize == 64 ? Encoder.ERROR_ONLY_1632_BIT_MODE : Encoder.ERROR_ONLY_64_BIT_MODE, errorMessage);
Assert.False(result);
Assert.True(Instruction.TEST_BitByBitEquals(origInstr, origInstrCopy));
}
public void TestDecode()
{
var decoder = new CRFDecoder();
var options = new DecoderOptions
{
ModelFileName = @"C:\Users\haipi\Documents\Projects\BotSharp\Data\CRF\ner_model"
};
//Load encoded model from file
decoder.LoadModel(options.ModelFileName);
//Create decoder tagger instance.
var tagger = decoder.CreateTagger(options.NBest, options.MaxWord);
tagger.set_vlevel(options.ProbLevel);
//Initialize result
var crf_out = new CRFSegOut[options.NBest];
for (var i = 0; i < options.NBest; i++)
{
crf_out[i] = new CRFSegOut(options.MaxWord);
}
var dataset = GetTestData();
//predict given string's tags
decoder.Segment(crf_out, tagger, dataset);
}
public InstructionInfo(int codeSize, string hexBytes, Code code, DecoderOptions options)
{
CodeSize = codeSize;
HexBytes = hexBytes;
Code = code;
Options = options;
}
static Decoder CreateDecoder(int codeSize, string hexBytes, DecoderOptions options, out ulong rip)
{
Decoder decoder;
var codeReader = new ByteArrayCodeReader(hexBytes);
switch (codeSize)
{
case 16:
decoder = Decoder.Create16(codeReader, options);
rip = DecoderConstants.DEFAULT_IP16;
break;
case 32:
decoder = Decoder.Create32(codeReader, options);
rip = DecoderConstants.DEFAULT_IP32;
break;
case 64:
decoder = Decoder.Create64(codeReader, options);
rip = DecoderConstants.DEFAULT_IP64;
break;
default:
throw new ArgumentOutOfRangeException(nameof(codeSize));
}
Assert.Equal(codeSize, decoder.Bitness);
decoder.InstructionPointer = rip;
return(decoder);
}
static bool TryParseDecoderOptions(string[] stringOptions, ref DecoderOptions options)
{
foreach (var opt in stringOptions)
{
switch (opt.Trim().ToLowerInvariant())
{
case InstructionInfoDecoderOptions.AmdBranches:
options |= DecoderOptions.AmdBranches;
break;
case InstructionInfoDecoderOptions.ForceReservedNop:
options |= DecoderOptions.ForceReservedNop;
break;
case InstructionInfoDecoderOptions.Umov:
options |= DecoderOptions.Umov;
break;
case InstructionInfoDecoderOptions.Xbts:
options |= DecoderOptions.Xbts;
break;
case InstructionInfoDecoderOptions.Cmpxchg486A:
options |= DecoderOptions.Cmpxchg486A;
break;
case InstructionInfoDecoderOptions.OldFpu:
options |= DecoderOptions.OldFpu;
break;
case InstructionInfoDecoderOptions.Pcommit:
options |= DecoderOptions.Pcommit;
break;
case InstructionInfoDecoderOptions.Loadall286:
options |= DecoderOptions.Loadall286;
break;
case InstructionInfoDecoderOptions.Loadall386:
options |= DecoderOptions.Loadall386;
break;
case InstructionInfoDecoderOptions.Cl1invmb:
options |= DecoderOptions.Cl1invmb;
break;
case InstructionInfoDecoderOptions.MovTr:
options |= DecoderOptions.MovTr;
break;
case InstructionInfoDecoderOptions.Jmpe:
options |= DecoderOptions.Jmpe;
break;
default:
return(false);
}
}
return(true);
}
public OpCodeHandler_Options(OpCodeHandler defaultHandler, OpCodeHandler handler1, DecoderOptions options1)
{
this.defaultHandler = defaultHandler ?? throw new ArgumentNullException(nameof(defaultHandler));
infos = new HandlerOptions[] {
new HandlerOptions(handler1, options1),
};
infoOptions = options1;
}
Decoder CreateDecoder(int bitness, byte[] hexBytes, ulong ip, DecoderOptions options)
{
var codeReader = new ByteArrayCodeReader(hexBytes);
var decoder = Decoder.Create(bitness, codeReader, options);
decoder.IP = ip;
Assert.Equal(bitness, decoder.Bitness);
return(decoder);
}
public OpCodeHandler_Options1632(OpCodeHandler defaultHandler, OpCodeHandler handler1, DecoderOptions options1, OpCodeHandler handler2, DecoderOptions options2)
{
this.defaultHandler = defaultHandler ?? throw new ArgumentNullException(nameof(defaultHandler));
infos = new HandlerOptions[] {
new HandlerOptions(handler1 ?? throw new ArgumentNullException(nameof(handler1)), options1),
new HandlerOptions(handler2 ?? throw new ArgumentNullException(nameof(handler2)), options2),
};
infoOptions = options1 | options2;
}
public DecoderTestInfo(uint id, int bitness, Code code, string hexBytes, string encodedHexBytes, DecoderOptions options, DecoderTestOptions testOptions)
{
Id = id;
Bitness = bitness;
Code = code;
HexBytes = hexBytes;
EncodedHexBytes = encodedHexBytes;
Options = options;
TestOptions = testOptions;
}
protected void EncodeBase(int codeSize, Code code, string hexBytes, DecoderOptions options)
{
var origBytes = HexUtils.ToByteArray(hexBytes);
var decoder = CreateDecoder(codeSize, origBytes, options);
var origRip = decoder.InstructionPointer;
var origInstr = decoder.Decode();
var origConstantOffsets = decoder.GetConstantOffsets(ref origInstr);
Assert.Equal(code, origInstr.Code);
Assert.Equal(origBytes.Length, origInstr.ByteLength);
Assert.True(origInstr.ByteLength <= Iced.Intel.DecoderConstants.MaxInstructionLength);
Assert.Equal((ushort)origRip, origInstr.IP16);
Assert.Equal((uint)origRip, origInstr.IP32);
Assert.Equal(origRip, origInstr.IP64);
var afterRip = decoder.InstructionPointer;
Assert.Equal((ushort)afterRip, origInstr.NextIP16);
Assert.Equal((uint)afterRip, origInstr.NextIP32);
Assert.Equal(afterRip, origInstr.NextIP64);
var writer = new CodeWriterImpl();
var encoder = decoder.CreateEncoder(writer);
Assert.Equal(codeSize, encoder.Bitness);
var origInstrCopy = origInstr;
bool result = encoder.TryEncode(ref origInstr, origRip, out uint encodedInstrLen, out string errorMessage);
Assert.True(errorMessage == null, "Unexpected ErrorMessage: " + errorMessage);
Assert.True(result, "Error, result from Encoder.TryEncode must be true");
var encodedConstantOffsets = encoder.GetConstantOffsets();
FixConstantOffsets(ref encodedConstantOffsets, origInstr.ByteLength, (int)encodedInstrLen);
Assert.True(Equals(ref origConstantOffsets, ref encodedConstantOffsets));
var encodedBytes = writer.ToArray();
Assert.Equal(encodedBytes.Length, (int)encodedInstrLen);
Assert.True(Instruction.TEST_BitByBitEquals(origInstr, origInstrCopy), "Instruction are differing: " + Instruction.TEST_DumpDiff(origInstr, origInstrCopy));
// We don't compare the generated bytes, bit by bit, for many reasons. Eg. prefixes
// can be put in any order. Unused prefixes. [xxxx] can be encoded w/ and w/o a sib
// byte in 32-bit mode. The decoder-tests test ignored bits by setting them to 1, but
// the encoder always writes 0.
// The instruction is decoded again and then it's compared against the old one to
// make sure it matches exactly, bit by bit.
var newInstr = CreateDecoder(codeSize, encodedBytes, options).Decode();
Assert.Equal(code, newInstr.Code);
Assert.Equal(encodedBytes.Length, newInstr.ByteLength);
newInstr.ByteLength = origInstr.ByteLength;
newInstr.NextIP64 = origInstr.NextIP64;
Assert.True(Instruction.TEST_BitByBitEquals(origInstr, newInstr), "Instruction are differing: " + Instruction.TEST_DumpDiff(origInstr, newInstr));
// Some tests use useless or extra prefixes, so we can't verify the exact length
Assert.True(encodedBytes.Length <= origBytes.Length, "Unexpected encoded prefixes: " + HexUtils.ToString(encodedBytes));
}
static bool TryParseDecoderOptions(string[] stringOptions, ref DecoderOptions options)
{
foreach (var opt in stringOptions)
{
if (!ToEnumConverter.TryDecoderOptions(opt.Trim(), out var decOpts))
{
return(false);
}
options |= decOpts;
}
return(true);
}
public async Task <bool> Predict(AgentBase agent, NlpDoc doc, PipeModel meta)
{
var decoder = new CRFDecoder();
var options = new DecoderOptions
{
ModelFileName = System.IO.Path.Combine(Settings.ModelDir, meta.Model)
};
//Load encoded model from file
decoder.LoadModel(options.ModelFileName);
//Create decoder tagger instance.
var tagger = decoder.CreateTagger(options.NBest, options.MaxWord);
tagger.set_vlevel(options.ProbLevel);
//Initialize result
var crf_out = new CRFSegOut[options.NBest];
for (var i = 0; i < options.NBest; i++)
{
crf_out[i] = new CRFSegOut(options.MaxWord);
}
doc.Sentences.ForEach(sent =>
{
List <List <String> > dataset = new List <List <string> >();
dataset.AddRange(sent.Tokens.Select(token => new List <String> {
token.Text, token.Pos
}).ToList());
//predict given string's tags
decoder.Segment(crf_out, tagger, dataset);
var entities = new List <NlpEntity>();
for (int i = 0; i < sent.Tokens.Count; i++)
{
var entity = crf_out[0].result_;
entities.Add(new NlpEntity
{
Entity = entity[i],
Start = doc.Sentences[0].Tokens[i].Start,
Value = doc.Sentences[0].Tokens[i].Text,
Confidence = 0,
Extrator = "BotSharpNER"
});
}
sent.Entities = MergeEntity(doc.Sentences[0].Text, entities);
});
return(true);
}
void Test64_Simple_1(string hexBytes, int byteLength, Code code, DecoderOptions options)
{
var decoder = CreateDecoder64(hexBytes, options);
var instr = decoder.Decode();
Assert.Equal(code, instr.Code);
Assert.Equal(0, instr.OpCount);
Assert.Equal(byteLength, instr.ByteLength);
Assert.False(instr.HasRepePrefix);
Assert.False(instr.HasRepnePrefix);
Assert.False(instr.HasLockPrefix);
Assert.Equal(Register.None, instr.SegmentPrefix);
}
public void Decode_IgnoreMetadataIsTrue_CommentsAreIgnored()
{
DecoderOptions options = new DecoderOptions()
{
IgnoreMetadata = true
};
TestFile testFile = TestFile.Create(TestImages.Gif.Rings);
using (Image <Rgba32> image = testFile.CreateImage(options))
{
Assert.Equal(0, image.MetaData.Properties.Count);
}
}
Decoder CreateDecoder(int bitness, byte[] hexBytes, DecoderOptions options)
{
var codeReader = new ByteArrayCodeReader(hexBytes);
var decoder = Decoder.Create(bitness, codeReader, options);
decoder.IP = bitness switch {
16 => DecoderConstants.DEFAULT_IP16,
32 => DecoderConstants.DEFAULT_IP32,
64 => DecoderConstants.DEFAULT_IP64,
_ => throw new ArgumentOutOfRangeException(nameof(bitness)),
};
Assert.Equal(bitness, decoder.Bitness);
return(decoder);
}
public void Decode_IgnoreMetadataIsFalse_ExifProfileIsRead()
{
DecoderOptions options = new DecoderOptions()
{
IgnoreMetadata = false
};
TestFile testFile = TestFile.Create(TestImages.Jpeg.Baseline.Floorplan);
using (Image <Rgba32> image = testFile.CreateImage(options))
{
Assert.NotNull(image.MetaData.ExifProfile);
}
}
public void Decode_IgnoreMetadataIsTrue_ExifProfileIgnored()
{
DecoderOptions options = new DecoderOptions()
{
IgnoreMetadata = true
};
TestFile testFile = TestFile.Create(TestImages.Jpeg.Baseline.Floorplan);
using (Image image = testFile.CreateImage(options))
{
Assert.Null(image.MetaData.ExifProfile);
}
}
public void Decode_IgnoreMetadataIsFalse_CommentsAreRead()
{
DecoderOptions options = new DecoderOptions()
{
IgnoreMetadata = false
};
TestFile testFile = TestFile.Create(TestImages.Gif.Rings);
using (Image <Rgba32> image = testFile.CreateImage(options))
{
Assert.Equal(1, image.MetaData.Properties.Count);
Assert.Equal("Comments", image.MetaData.Properties[0].Name);
Assert.Equal("ImageSharp", image.MetaData.Properties[0].Value);
}
}
void Test64_Jmp_rel8_64_1(string hexBytes, int byteLength, ulong target, DecoderOptions options)
{
var decoder = CreateDecoder64(hexBytes, options);
var instr = decoder.Decode();
Assert.Equal(Code.Jmp_rel8_64, instr.Code);
Assert.Equal(1, instr.OpCount);
Assert.Equal(byteLength, instr.ByteLength);
Assert.False(instr.HasRepePrefix);
Assert.False(instr.HasRepnePrefix);
Assert.False(instr.HasLockPrefix);
Assert.Equal(Register.None, instr.SegmentPrefix);
Assert.Equal(OpKind.NearBranch64, instr.Op0Kind);
Assert.Equal(target, instr.NearBranch64);
}
public static void FormatTest(int codeSize, string hexBytes, Code code, DecoderOptions options, string formattedString, Formatter formatter)
{
var decoder = CreateDecoder(codeSize, hexBytes, options, out ulong nextRip);
var instr = decoder.Decode();
Assert.Equal(code, instr.Code);
Assert.Equal((ushort)nextRip, instr.IP16);
Assert.Equal((uint)nextRip, instr.IP32);
Assert.Equal(nextRip, instr.IP);
nextRip += (uint)instr.ByteLength;
Assert.Equal(nextRip, decoder.IP);
Assert.Equal((ushort)nextRip, instr.NextIP16);
Assert.Equal((uint)nextRip, instr.NextIP32);
Assert.Equal(nextRip, instr.NextIP);
FormatTest(ref instr, formattedString, formatter);
}
public static void FormatTest(int bitness, string hexBytes, Code code, DecoderOptions options, string formattedString, FastFormatter formatter)
{
var decoder = CreateDecoder(bitness, hexBytes, options, out ulong nextRip);
var instruction = decoder.Decode();
Assert.Equal(code, instruction.Code);
Assert.Equal((ushort)nextRip, instruction.IP16);
Assert.Equal((uint)nextRip, instruction.IP32);
Assert.Equal(nextRip, instruction.IP);
nextRip += (uint)instruction.Length;
Assert.Equal(nextRip, decoder.IP);
Assert.Equal((ushort)nextRip, instruction.NextIP16);
Assert.Equal((uint)nextRip, instruction.NextIP32);
Assert.Equal(nextRip, instruction.NextIP);
FormatTest(instruction, formattedString, formatter);
}
internal static Instruction[] Decode(int bitness, ulong rip, byte[] data, DecoderOptions options)
{
var decoder = Decoder.Create(bitness, new ByteArrayCodeReader(data), options);
decoder.IP = rip;
var list = new List <Instruction>();
while ((decoder.IP - rip) < (uint)data.Length)
{
list.Add(decoder.Decode());
}
if (decoder.IP - rip != (uint)data.Length)
{
throw new InvalidOperationException();
}
return(list.ToArray());
}
/// <summary>
/// Creates a new wrapper around a raw stream containing x86 code.
/// </summary>
/// <param name="architecture">The x86 architecture.</param>
/// <param name="inputStream">The raw code stream.</param>
/// <param name="bitness">The bitness of the x86 code. This value must be either 16, 32 or 64.</param>
/// <param name="baseAddress">The base address of the code stream.</param>
/// <param name="decoderOptions">Additional decoder options that need to be passed onto the Iced decoder.</param>
public X86DecoderInstructionProvider(
IInstructionSetArchitecture <Instruction> architecture,
Stream inputStream,
int bitness,
ulong baseAddress,
DecoderOptions decoderOptions)
{
_inputStream = inputStream ?? throw new ArgumentNullException(nameof(inputStream));
if (!inputStream.CanRead)
{
throw new ArgumentException("Input stream must be readable.");
}
if (!inputStream.CanSeek)
{
throw new ArgumentException("Input stream must be seekable.");
}
_decoder = Decoder.Create(bitness, new StreamCodeReader(inputStream), decoderOptions);
Architecture = architecture;
BaseAddress = baseAddress;
}
public HandlerOptions(OpCodeHandler handler, DecoderOptions options)
{
this.handler = handler;
this.options = options;
}
void Test64_Jcc_Jw16_1(string hexBytes, int byteLength, Code code, ulong target, DecoderOptions options)
{
var decoder = CreateDecoder64(hexBytes, options);
var instr = decoder.Decode();
Assert.Equal(code, instr.Code);
Assert.Equal(1, instr.OpCount);
Assert.Equal(byteLength, instr.ByteLength);
Assert.False(instr.HasRepePrefix);
Assert.False(instr.HasRepnePrefix);
Assert.False(instr.HasLockPrefix);
Assert.Equal(Register.None, instr.SegmentPrefix);
Assert.Equal(OpKind.NearBranch16, instr.Op0Kind);
Assert.Equal((ushort)target, instr.NearBranch16);
}
