private static ExtendedCapabilities ParseExtendedCaps(ByteReader reader, BitWidth numWidth)
{
var header = ReadIntegers(reader, 5, BitWidth.TwoBytes);
// Read boolean caps
var booleans = ReadBools(reader, header[Ordinals.ExtBoolCount]);
// Read num caps
var nums = ReadNums(reader, header[Ordinals.ExtNumCount], numWidth);
var indices = new Span <int>(ReadIntegers(reader, header[Ordinals.ExtOffsetCount], BitWidth.TwoBytes));
var data = new Span <char>(Encoding.ASCII.GetString(reader.ReadBytes(header[Ordinals.ExtTableSize])).ToCharArray());
// Read string caps
var(strings, last) = ReadStrings(indices, data, header[Ordinals.ExtStringCount]);
indices = indices.Slice(header[Ordinals.ExtStringCount]);
data = data.Slice(last);
// Read bool names
var(booleanNames, _) = ReadStrings(indices, data, header[Ordinals.ExtBoolCount]);
indices = indices.Slice(header[Ordinals.ExtBoolCount]);
// Read num names
var(numNames, _) = ReadStrings(indices, data, header[Ordinals.ExtNumCount]);
indices = indices.Slice(header[Ordinals.ExtNumCount]);
// Read string names
var(stringNames, _) = ReadStrings(indices, data, header[Ordinals.ExtStringCount]);
return(new ExtendedCapabilities(
booleans, nums, strings,
booleanNames, numNames, stringNames));
}
private byte Align(BitWidth width)
{
var byteWidth = 1UL << (int)width;
_offset += BitWidthUtil.PaddingSize(_offset, byteWidth);
return((byte)byteWidth);
}
public void ProdTest()
{
BitWidth encoding = BitWidth.BV7;
List <string> regexes = new List <string>(SampleRegexes.regexes);
regexes.RemoveRange(6, regexes.Count - 6); //just consider the first 6 cases
int nonemptyCount = 0;
for (int i = 0; i < regexes.Count; i++)
{
for (int j = 0; j < regexes.Count; j++)
{
string regexA = regexes[i];
string regexB = regexes[j];
var z3p = new Z3Provider(encoding);
z3p.MainSolver.Push();
var A = z3p.RegexConverter.Convert(regexA, System.Text.RegularExpressions.RegexOptions.None);
var B = z3p.RegexConverter.Convert(regexB, System.Text.RegularExpressions.RegexOptions.None);
var A1 = z3p.CharSetProvider.Convert(regexA);
var B1 = z3p.CharSetProvider.Convert(regexB);
var C1 = Automaton <BDD> .MkProduct(A1, B1).Determinize().Minimize();
var C = Automaton <Expr> .MkProduct(A, B);
var C2 = new SFAz3(z3p, z3p.CharSort, C.Determinize().Minimize()).Concretize(200);
var equiv = C1.IsEquivalentWith(C1);
Assert.IsTrue(equiv);
if (i == j)
{
Assert.IsFalse(C.IsEmpty);
}
if (!C.IsEmpty)
{
if (i != j)
{
//z3p.CharSetProvider.ShowGraph(C1, "C1");
//z3p.CharSetProvider.ShowGraph(C2, "C2");
}
nonemptyCount += 1;
string s = GetMember(z3p, C);
Assert.IsTrue(Regex.IsMatch(s, regexA, RegexOptions.None), "regex mismatch");
Assert.IsTrue(Regex.IsMatch(s, regexB, RegexOptions.None), "regex mismatch");
}
z3p.MainSolver.Pop();
}
}
Assert.AreEqual <int>(10, nonemptyCount, "wrong number of empty intersections");
}
public UnicodeCategoryToHashSetProvider(HashSetSolver solver)
{
this.solver = solver;
this.encoding = solver.Encoding;
this.solverBDD = new CharSetSolver(solver.Encoding);
InitializeUnicodeCategoryDefinitions();
}
private void ProdExperimentBDD(BitWidth encoding)
{
var solver = new CharSetSolver(encoding);
var converter = solver;
string[] regexes = SampleRegexes.regexes;
int nonemptyCount = 0;
for (int i = 0; i < 6; i++)
{
for (int j = 0; j < 6; j++)
{
string regexA = regexes[i];
string regexB = regexes[j];
var A = converter.Convert(regexA, System.Text.RegularExpressions.RegexOptions.None);
var B = converter.Convert(regexB, System.Text.RegularExpressions.RegexOptions.None);
var C = Automaton <BDD> .MkProduct(A, B);
if (i == j)
{
Assert.IsFalse(C.IsEmpty);
}
if (!C.IsEmpty)
{
nonemptyCount += 1;
string s = converter.GenerateMember(C);
Assert.IsTrue(Regex.IsMatch(s, regexA, RegexOptions.None), "regex mismatch");
Assert.IsTrue(Regex.IsMatch(s, regexB, RegexOptions.None), "regex mismatch");
}
}
}
Assert.AreEqual <int>(10, nonemptyCount, "wrong number of empty intersections");
}
private static int[] ReadIntegers(ByteReader reader, int count, BitWidth width)
{
var bytes = (int)width / 8;
var length = count * bytes;
var buffer = reader.ReadBytes(length);
if (reader.Position % 2 != 0)
{
reader.ReadByte();
}
var result = new int[count];
for (int i = 0, j = 0; i < length; i += bytes, j++)
{
switch (bytes)
{
case 1:
result[i] = buffer[i];
break;
case 2:
result[j] = (short)(buffer[i + 1] << 8 | buffer[i]);
break;
case 4:
result[j] = buffer[i + 3] << 24 | buffer[i + 2] << 16 | buffer[i + 1] << 8 | buffer[i];
break;
}
}
return(result);
}
public RegexToSMTConverter(BitWidth encoding)
{
css = new CharSetSolver(encoding);
automConverter = css.RegexConverter;
maxChar = (encoding == BitWidth.BV16 ? '\uFFFF' :
(encoding == BitWidth.BV8 ? '\u00FF' : '\u007F'));
CHAR = string.Format("(_ BitVec {0})", (int)encoding);
}
/// <summary>
/// Construct a character set solver for the given character encoding (nr of bits).
/// </summary>
public CharSetSolver(BitWidth bits) : base()
{
if (!CharacterEncodingTool.IsSpecified(bits))
{
throw new AutomataException(AutomataExceptionKind.CharacterEncodingIsUnspecified);
}
_bw = (int)bits;
}
public RegexToSMTConverter(BitWidth encoding, string charSortAlias)
{
css = new CharSetSolver(encoding);
automConverter = css.RegexConverter;
maxChar = (encoding == BitWidth.BV16 ? '\uFFFF' :
(encoding == BitWidth.BV8 ? '\u00FF' : '\u007F'));
CHAR = charSortAlias;
}
public RegexToSMTConverter(BitWidth encoding)
{
css = new CharSetSolver(encoding);
automConverter = css.RegexConverter;
maxChar = (encoding == BitWidth.BV16 ? '\uFFFF' :
(encoding == BitWidth.BV8 ? '\u00FF' : '\u007F'));
CHAR = string.Format("(_ BitVec {0})", (int)encoding);
}
/// <summary>
/// Returns true iff encoding equals to one of the enums in CharacterEncoding.
/// </summary>
public static bool IsSpecified(BitWidth encoding)
{
return(encoding == BitWidth.BV7 ||
encoding == BitWidth.BV32 ||
encoding == BitWidth.BV8 ||
encoding == BitWidth.BV64 ||
encoding == BitWidth.BV16);
}
/// <summary>
/// Returns true iff encoding equals to one of the enums in CharacterEncoding.
/// </summary>
public static bool IsSpecified(BitWidth encoding)
{
return (encoding == BitWidth.BV7 ||
encoding == BitWidth.BV32 ||
encoding == BitWidth.BV8 ||
encoding == BitWidth.BV64 ||
encoding == BitWidth.BV16);
}
public RegexToSMTConverter(BitWidth encoding, string charSortAlias)
{
css = new CharSetSolver(encoding);
automConverter = css.RegexConverter;
maxChar = (encoding == BitWidth.BV16 ? '\uFFFF' :
(encoding == BitWidth.BV8 ? '\u00FF' : '\u007F'));
CHAR = charSortAlias;
}
public CharRangeSolver(BitWidth encoding)
{
this.encoding = encoding;
this.minCharacter = (char)0;
this.maxCharacter = (encoding == BitWidth.BV7 ? '\x007F' :
(encoding == BitWidth.BV8 ? '\x00FF' : '\xFFFF'));
mtg = new MintermGenerator<HashSet<Tuple<char, char>>>(this);
}
public static StackValue Value(long value, BitWidth width, Type type)
{
return(new StackValue
{
Width = width,
LValue = value,
ValueType = type
});
}
public BitWidth StoredWidth(BitWidth bitWidth = BitWidth.Width8)
{
if (TypesUtil.IsInline(ValueType))
{
return((BitWidth)Math.Max((int)bitWidth, (int)Width));
}
return(Width);
}
public override Int32 GetHashCode()
{
int value = 31415;
value = value * 7 + ByteOrder.GetHashCode();
value = value * 7 + BitOrder.GetHashCode();
value = value * 7 + BitWidth.GetHashCode();
value = value * 7 + Signed.GetHashCode();
return(value);
}
/// <summary>
/// Maps ASCII to 7, extended ASCII to 8, and other encodings to 16.
/// Throws AutomataException if IsSpecified(encoding) is false.
/// </summary>
/// <param name="encoding"></param>
/// <returns>either 7, 8, or 16</returns>
public static int Truncate(BitWidth encoding)
{
switch (encoding)
{
case BitWidth.BV7: return 7;
case BitWidth.BV8: return 8;
case BitWidth.BV16: return 16;
case BitWidth.BV32: return 16;
case BitWidth.BV64: return 16;
default:
throw new AutomataException(AutomataExceptionKind.CharacterEncodingIsUnspecified);
}
}
public HashSetSolver(BitWidth encoding)
{
this.encoding = encoding;
this.maxCharacter = (encoding == BitWidth.BV7 ? '\x007F' :
(encoding == BitWidth.BV8 ? '\x00FF' : '\xFFFF'));
sigma_ = new HashSet <char>();
for (char i = this.minCharacter; i < this.maxCharacter; i++)
{
sigma_.Add(i);
}
sigma_.Add(this.maxCharacter);
mtg = new MintermGenerator <HashSet <char> >(this);
}
private async void btnSearchMemory_Click(object sender, EventArgs e)
{
// start a new search if one hasn't already been started, search through all of memory
if (!searchStarted)
{
// find the heap blocks
List <CommittedMemoryBlock> heapBlocks = new List <CommittedMemoryBlock>();
foreach (CommittedMemoryBlock b in console.CommittedMemory)
{
if (b.Base == 0x82000000)
{
heapBlocks.Add(b);
}
}
scanner = new XboxMemoryScanner(console, heapBlocks);
// grab the user arguments from the UI
ulong value = ulong.Parse(txtSearchValue.Text, (rdoHex.Checked) ? NumberStyles.HexNumber : NumberStyles.Integer);
BitWidth bitWidth = (BitWidth)Math.Pow(2, cmbxBitwidth.SelectedIndex);
// update the UI while searching
btnSearchMemory.Enabled = false;
// add all of the results to the list view
foreach (uint instance in await scanner.FindValue(value, bitWidth))
{
lstMemScanResults.Items.Add("0x" + instance.ToString("X8"));
}
// update the UI so the user can perform another search
btnSearchMemory.Enabled = true;
tipMemoryScan.SetToolTip(btnSearchMemory, "Search through the results for a new value");
searchStarted = true;
}
// if a search has already been created then only search through the results already found for the updated value
else
{
btnSearchMemory.Enabled = false;
lstMemScanResults.Clear();
foreach (uint address in await scanner.NarrowResults(ulong.Parse(txtSearchValue.Text, (rdoHex.Checked) ? NumberStyles.HexNumber : NumberStyles.Integer)))
{
lstMemScanResults.Items.Add("0x" + address.ToString("X8"));
}
btnSearchMemory.Enabled = true;
}
}
private static int?[] ReadNums(ByteReader reader, int count, BitWidth width)
{
var buffer = ReadIntegers(reader, count, width);
var result = new int?[count];
for (var i = 0; i < count; i++)
{
result[i] = buffer[i];
if (buffer[i] == -2 || buffer[i] == -1)
{
result[i] = null;
}
}
return(result);
}
private static void LDiffExperimentZ3(BitWidth encoding)
{
List <string> regexes = new List <string>(SampleRegexes.regexes);
regexes.RemoveRange(2, regexes.Count - 2); //just consider the first 5 cases
long timeout = 5 * Microsoft.Automata.Internal.Utilities.HighTimer.Frequency; //5 sec
for (int i = 0; i < regexes.Count; i++)
{
for (int j = 0; j < regexes.Count; j++)
{
string regexA = regexes[i];
string regexB = regexes[j];
var z3p = new Z3Provider(encoding);
z3p.MainSolver.Push();
var A = z3p.RegexConverter.Convert(regexA, System.Text.RegularExpressions.RegexOptions.None);
var B = z3p.RegexConverter.Convert(regexB, System.Text.RegularExpressions.RegexOptions.None);
//A.ShowGraph();
try
{
List <Expr> witness;
var AmB = Automaton <Expr> .MkDifference(A, B, (int)timeout).Determinize().Minimize();
//AmB.ShowGraph();
bool isNonempty = Automaton <Expr> .CheckDifference(A, B, (int)timeout, out witness);
if (isNonempty)
{
string s = new String(Array.ConvertAll(witness.ToArray(), c => z3p.GetCharValue(z3p.MainSolver.FindOneMember(c).Value)));
Assert.IsTrue(Regex.IsMatch(s, regexA), s + " must match " + regexA);
Assert.IsFalse(Regex.IsMatch(s, regexB), s + " must not match " + regexB);
}
}
catch (TimeoutException)
{
Console.WriteLine("Timeout {0},{1}", i, j);
}
z3p.MainSolver.Pop();
}
}
}
public static void SaveAsRangeAutomaton(string regex, BitWidth encoding, string file, bool epsilonfree = false, bool determinize = false, int determinization_timeout_ms = 0)
{
var solver = new CharRangeSolver(encoding);
var converter = new RegexToAutomatonConverterRanges(solver);
var A = converter.Convert(regex, System.Text.RegularExpressions.RegexOptions.None);
if (epsilonfree)
{
A = A.RemoveEpsilons(solver.MkOr);
}
if (determinize)
{
A = A.RemoveEpsilons(solver.MkOr);
A.CheckDeterminism(solver);
A = A.Determinize(solver, determinization_timeout_ms);
}
//A.CheckDeterminism(solver,true);
//if (!A.isDeterministic)
// throw new AutomataException(AutomataExceptionKind.InternalError);
System.IO.StreamWriter sw = new System.IO.StreamWriter(file);
sw.WriteLine(A.InitialState);
foreach (var s in A.GetFinalStates())
{
sw.Write("{0} ", s);
}
sw.WriteLine();
foreach (var move in A.GetMoves())
{
if (move.IsEpsilon)
{
sw.WriteLine("{0} {1} {2} {3}", move.SourceState, -1, -1, move.TargetState);
}
else
{
foreach (var range in move.Label)
{
sw.WriteLine("{0} {1} {2} {3}", move.SourceState, (int)range.First, (int)range.Second, move.TargetState);
}
}
}
sw.Close();
}
/// <summary>
/// Maps ASCII to 7, extended ASCII to 8, and other encodings to 16.
/// Throws AutomataException if IsSpecified(encoding) is false.
/// </summary>
/// <param name="encoding"></param>
/// <returns>either 7, 8, or 16</returns>
public static int Truncate(BitWidth encoding)
{
switch (encoding)
{
case BitWidth.BV7: return(7);
case BitWidth.BV8: return(8);
case BitWidth.BV16: return(16);
case BitWidth.BV32: return(16);
case BitWidth.BV64: return(16);
default:
throw new AutomataException(AutomataExceptionKind.CharacterEncodingIsUnspecified);
}
}
private static void LDiffExperimentBDD(BitWidth encoding)
{
var solver = new CharSetSolver(encoding);
var converter = solver;
string[] regexes = SampleRegexes.regexes;
long timeout = 60 * Microsoft.Automata.Internal.Utilities.HighTimer.Frequency; //1 minute
int nonemptyCnt = 0;
for (int i = 0; i < 6; i++)
{
for (int j = 0; j < 6; j++)
{
string regexA = regexes[i];
string regexB = regexes[j];
long start = Microsoft.Automata.Internal.Utilities.HighTimer.Now;
var A = converter.Convert(regexA, System.Text.RegularExpressions.RegexOptions.None);
var B = converter.Convert(regexB, System.Text.RegularExpressions.RegexOptions.None);
try
{
List <BDD> witness;
bool isNonempty;
isNonempty = Automaton <BDD> .CheckDifference(A, B, (int)timeout, out witness);
if (isNonempty)
{
nonemptyCnt += 1;
string s = solver.ChooseString(witness);
Assert.IsTrue(System.Text.RegularExpressions.Regex.IsMatch(s, regexA), s + " must be a member of " + regexA);
Assert.IsFalse(System.Text.RegularExpressions.Regex.IsMatch(s, regexB), s + " must not be a member of " + regexB);
}
}
catch (TimeoutException)
{
Console.WriteLine("Timeout {0},{1}", i, j);
}
}
}
Assert.AreEqual <int>(30, nonemptyCnt, "unexpected number of nonempty differences");
}
private static void DiffExperimentZ3(BitWidth encoding)
{
List <string> regexes = new List <string>(SampleRegexes.regexes);
regexes.RemoveRange(3, regexes.Count - 3); //just consider the first few cases
long timeout = 5 * Microsoft.Automata.Internal.Utilities.HighTimer.Frequency; //5 sec
for (int i = 0; i < regexes.Count; i++)
{
if (i != 6)
{
for (int j = 0; j < regexes.Count; j++)
{
string regexA = regexes[i];
string regexB = regexes[j];
var z3p = new Z3Provider(encoding);
z3p.MainSolver.Push();
var A = z3p.RegexConverter.Convert(regexA, System.Text.RegularExpressions.RegexOptions.None);
var B = z3p.RegexConverter.Convert(regexB, System.Text.RegularExpressions.RegexOptions.None);
try
{
var C = Automaton <Expr> .MkDifference(A, B, (int)timeout);
if (!C.IsEmpty)
{
string s = GetMember(z3p, C);
Assert.IsTrue(Regex.IsMatch(s, regexA), s + " must match " + regexA);
Assert.IsFalse(Regex.IsMatch(s, regexB), s + " must not match " + regexB);
}
}
catch (TimeoutException)
{
Console.WriteLine("Timeout {0},{1}", i, j);
}
z3p.MainSolver.Pop();
}
}
}
}
public async Task<List<uint>> FindValue(ulong value, BitWidth bitWidth)
{
prevBitWidth = bitWidth;
prevScanResults = new List<uint>();
// if the caller didn't specify which memory blocks to search, we'll just search all of them
if (memoryBlocks == null)
{
foreach (CommittedMemoryBlock block in console.CommittedMemory)
prevScanResults.AddRange(await FindValue(block, value, bitWidth));
return prevScanResults;
}
else
{
foreach (CommittedMemoryBlock block in memoryBlocks)
prevScanResults.AddRange(await FindValue(block, value, bitWidth));
return prevScanResults;
}
}
private ulong GetValue(byte[] buffer, int offset, BitWidth bitWidth)
{
switch (bitWidth)
{
case BitWidth.Byte8:
return(buffer[offset]);
case BitWidth.Word16:
return(BitConverterBigEndian.ToUInt16(buffer, offset));
case BitWidth.Dword32:
return(BitConverterBigEndian.ToUInt32(buffer, offset));
case BitWidth.Qword64:
return(BitConverterBigEndian.ToUInt64(buffer, offset));
}
return(0);
}
private void LProdExperimentZ3(BitWidth encoding)
{
List <string> regexes = new List <string>(SampleRegexes.regexes);
regexes.RemoveRange(6, regexes.Count - 6); //just consider the first 100 cases
int nonemptyCount = 0;
for (int i = 0; i < regexes.Count; i++)
{
for (int j = 0; j < regexes.Count; j++)
{
string regexA = regexes[i];
string regexB = regexes[j];
var z3p = new Z3Provider(encoding);
z3p.MainSolver.Push();
var A = z3p.RegexConverter.Convert(regexA, System.Text.RegularExpressions.RegexOptions.None);
var B = z3p.RegexConverter.Convert(regexB, System.Text.RegularExpressions.RegexOptions.None);
List <Expr> witness;
bool C = Automaton <Expr> .CheckProduct(A, B, 0, out witness);
if (i == j)
{
Assert.IsTrue(C, "product must me nonempty");
}
if (C)
{
nonemptyCount += 1;
string s = new String(Array.ConvertAll(witness.ToArray(), cs => { return(z3p.GetCharValue(z3p.MainSolver.FindOneMember(cs).Value)); }));
Assert.IsTrue(Regex.IsMatch(s, regexA, RegexOptions.None), "regex mismatch");
Assert.IsTrue(Regex.IsMatch(s, regexB, RegexOptions.None), "regex mismatch");
}
z3p.MainSolver.Pop();
}
}
Assert.AreEqual <int>(10, nonemptyCount, "wrong number of empty intersections");
}
private void ProdExperimentZ3(BitWidth encoding)
{
List <string> regexes = new List <string>(SampleRegexes.regexes);
regexes.RemoveRange(6, regexes.Count - 6); //just consider the first 6 cases
int nonemptyCount = 0;
var z3p = new Z3Provider(encoding);
for (int i = 0; i < regexes.Count; i++)
{
for (int j = 0; j < regexes.Count; j++)
{
string regexA = regexes[i];
string regexB = regexes[j];
z3p.MainSolver.Push();
var A = z3p.RegexConverter.Convert(regexA, System.Text.RegularExpressions.RegexOptions.None);
var B = z3p.RegexConverter.Convert(regexB, System.Text.RegularExpressions.RegexOptions.None);
var C = Automaton <Expr> .MkProduct(A, B);
if (i == j)
{
Assert.IsFalse(C.IsEmpty);
}
if (!C.IsEmpty)
{
nonemptyCount += 1;
string s = GetMember(z3p, C);
Assert.IsTrue(Regex.IsMatch(s, regexA, RegexOptions.None), "regex mismatch");
Assert.IsTrue(Regex.IsMatch(s, regexB, RegexOptions.None), "regex mismatch");
}
z3p.MainSolver.Pop();
}
}
Assert.AreEqual <int>(10, nonemptyCount, "wrong number of empty intersections");
}
private static bool InitLoader(string directory, string name, BitWidth loaderWidth, out string?loaderPath)
{
loaderPath = Path.Combine(directory, name);
int attempts = 10;
while (File.Exists(loaderPath))
{
loaderPath = Path.Combine(directory, $"{Path.GetRandomFileName().Substring(0, 4)}_{name}");
if (attempts-- < 0)
{
Logging.RecordError("Unable to create loader due to pre-existing loaders with similar name");
return(false);
}
}
byte[]? loaderBytes = (loaderWidth is BitWidth.Arch32) ?
global::rgat.Properties.Resources.DllLoader32 :
global::rgat.Properties.Resources.DllLoader64;
if (loaderBytes is null)
{
string loaderName = (loaderWidth is BitWidth.Arch32) ? "DllLoader32" : "DllLoader64";
Logging.RecordError($"Unable to retrieve loader {loaderName} from resources");
return(false);
}
try
{
File.WriteAllBytes(loaderPath, loaderBytes);
}
catch (Exception e)
{
Logging.RecordException($"Failed to write loader to DLL directory: {e.Message}", e);
return(false);
}
return(true);
}
private async Task<List<uint>> FindValue(CommittedMemoryBlock memBlock, ulong value, BitWidth bitWidth)
{
List<uint> toReturn = new List<uint>();
byte[] buffer = new byte[0x10000];
uint bytesLeft = memBlock.Size;
uint pos = memBlock.Base;
while (bytesLeft != 0)
{
uint bytesToRead = (bytesLeft > 0x10000) ? 0x10000 : bytesLeft;
console.GetMemory(pos, bytesToRead, buffer);
// search for the value inside the buffer recieved from the console
for (int i = 0; i < bytesToRead / (int)bitWidth; i++)
{
// deterimine the offset into the buffer for the next value
int offset = i * (int)bitWidth;
// get the correct value at the current position
ulong curValue = GetValue(buffer, offset, bitWidth);
// if it matches, then return address it was found at
if (curValue == value)
toReturn.Add(pos + (uint)offset);
if (i % 250 == 0)
await Task.Delay(10);
}
await Task.Delay(20);
// advance to the next value in the buffer
pos += bytesToRead;
bytesLeft -= bytesToRead;
}
return toReturn;
}
private void LProdExperimentBDD(BitWidth encoding)
{
var solver = new CharSetSolver(encoding);
var converter = solver;
string[] regexes = SampleRegexes.regexes;
int nonemptyCount = 0;
for (int i = 0; i < 6; i++)
{
for (int j = 0; j < 6; j++)
{
string regexA = regexes[i];
string regexB = regexes[j];
long start = Microsoft.Automata.Internal.Utilities.HighTimer.Now;
var A = converter.Convert(regexA, System.Text.RegularExpressions.RegexOptions.None);
var B = converter.Convert(regexB, System.Text.RegularExpressions.RegexOptions.None);
List <BDD> witness;
bool C = Automaton <BDD> .CheckProduct(A, B, 0, out witness);
if (i == j)
{
Assert.IsTrue(C, "product must me nonempty");
}
if (C)
{
nonemptyCount += 1;
string s = new String(Array.ConvertAll(witness.ToArray(), cs => { return((char)solver.Choose(cs)); }));
Assert.IsTrue(Regex.IsMatch(s, regexA, RegexOptions.None), "regex mismatch");
Assert.IsTrue(Regex.IsMatch(s, regexB, RegexOptions.None), "regex mismatch");
}
}
}
Assert.AreEqual <int>(10, nonemptyCount, "wrong number of empty intersections");
}
public async Task <List <uint> > FindValue(ulong value, BitWidth bitWidth)
{
prevBitWidth = bitWidth;
prevScanResults = new List <uint>();
// if the caller didn't specify which memory blocks to search, we'll just search all of them
if (memoryBlocks == null)
{
foreach (CommittedMemoryBlock block in console.CommittedMemory)
{
prevScanResults.AddRange(await FindValue(block, value, bitWidth));
}
return(prevScanResults);
}
else
{
foreach (CommittedMemoryBlock block in memoryBlocks)
{
prevScanResults.AddRange(await FindValue(block, value, bitWidth));
}
return(prevScanResults);
}
}
public void TestRanges2()
{
BitWidth enc = BitWidth.BV7;
CharSetSolver solver = new CharSetSolver(enc);
BDD cond = solver.MkCharSetFromRegexCharClass(@"\w");
int nodes = cond.CountNodes();
Pair <uint, uint>[] ranges = solver.ToRanges(cond);
BDD cond2 = solver.MkCharSetFromRanges(ranges);
Assert.AreSame(cond, cond2);
int nodes2 = cond2.CountNodes();
Assert.AreEqual <uint>((uint)'0', ranges[0].First);
Assert.AreEqual <uint>((uint)'9', ranges[0].Second);
Assert.AreEqual <uint>((uint)'A', ranges[1].First);
Assert.AreEqual <uint>((uint)'Z', ranges[1].Second);
Assert.AreEqual <uint>((uint)'_', ranges[2].First);
Assert.AreEqual <uint>((uint)'_', ranges[2].Second);
Assert.AreEqual <uint>((uint)'a', ranges[3].First);
Assert.AreEqual <uint>((uint)'z', ranges[3].Second);
Assert.AreEqual <int>(4, ranges.Length);
}
public void TestRanges2b()
{
BitWidth enc = BitWidth.BV16;
CharSetSolver solver = new CharSetSolver(enc);
BDD cond = solver.MkCharSetFromRegexCharClass(@"\w");
var ranges1 = solver.ToRanges(cond);
var cond1 = solver.MkCharSetFromRanges(ranges1);
Tuple <uint, uint>[] ranges = solver.ToRanges(cond1);
var cond2 = solver.MkCharSetFromRanges(ranges);
Assert.AreSame(cond1, cond2);
Assert.AreSame(cond, cond1);
//cond.ToDot("cond.dot");
Assert.AreEqual <uint>((uint)'0', ranges[0].Item1);
Assert.AreEqual <uint>((uint)'9', ranges[0].Item2);
Assert.AreEqual <uint>((uint)'A', ranges[1].Item1);
Assert.AreEqual <uint>((uint)'Z', ranges[1].Item2);
Assert.AreEqual <uint>((uint)'_', ranges[2].Item1);
Assert.AreEqual <uint>((uint)'_', ranges[2].Item2);
Assert.AreEqual <uint>((uint)'a', ranges[3].Item1);
Assert.AreEqual <uint>((uint)'z', ranges[3].Item2);
}
private static void LDiffExperimentZ3(BitWidth encoding)
{
List<string> regexes = new List<string>(SampleRegexes.regexes);
regexes.RemoveRange(2, regexes.Count - 2); //just consider the first 5 cases
long timeout = 5 * Microsoft.Automata.Internal.Utilities.HighTimer.Frequency; //5 sec
for (int i = 0; i < regexes.Count; i++)
for (int j = 0; j < regexes.Count; j++)
{
string regexA = regexes[i];
string regexB = regexes[j];
var z3p = new Z3Provider(encoding);
z3p.MainSolver.Push();
var A = z3p.RegexConverter.Convert(regexA, System.Text.RegularExpressions.RegexOptions.None);
var B = z3p.RegexConverter.Convert(regexB, System.Text.RegularExpressions.RegexOptions.None);
//A.ShowGraph();
try
{
List<Expr> witness;
var AmB = Automaton<Expr>.MkDifference(A, B, (int)timeout).Determinize().Minimize();
//AmB.ShowGraph();
bool isNonempty = Automaton<Expr>.CheckDifference(A, B, (int)timeout, out witness);
if (isNonempty)
{
string s = new String(Array.ConvertAll(witness.ToArray(), c => z3p.GetCharValue(z3p.MainSolver.FindOneMember(c).Value)));
Assert.IsTrue(Regex.IsMatch(s, regexA), s + " must match " + regexA);
Assert.IsFalse(Regex.IsMatch(s, regexB), s + " must not match " + regexB);
}
}
catch (TimeoutException)
{
Console.WriteLine("Timeout {0},{1}", i, j);
}
z3p.MainSolver.Pop();
}
}
private ulong GetValue(byte[] buffer, int offset, BitWidth bitWidth)
{
switch (bitWidth)
{
case BitWidth.Byte8:
return buffer[offset];
case BitWidth.Word16:
return BitConverterBigEndian.ToUInt16(buffer, offset);
case BitWidth.Dword32:
return BitConverterBigEndian.ToUInt32(buffer, offset);
case BitWidth.Qword64:
return BitConverterBigEndian.ToUInt64(buffer, offset);
}
return 0;
}
/// <summary>
/// Creates and instance of RexSettings for the given regexes using default values for other settings.
/// </summary>
public RexSettings(string[] regexes)
{
if (regexes.Length < 1)
throw new AutomataException("At least one regex must be specified");
this.regexes = regexes;
k = 1;
encoding = BitWidth.BV16;
//seed = -1;
intersect = false;
}
/// <summary>
/// Creates and instance of RexSettings with default values for all settings.
/// </summary>
RexSettings()
{
k = 1;
encoding = BitWidth.BV16;
//seed = -1;
intersect = false;
game = false;
}
private static void DiffExperimentZ3(BitWidth encoding)
{
List<string> regexes = new List<string>(SampleRegexes.regexes);
regexes.RemoveRange(3, regexes.Count - 3); //just consider the first few cases
long timeout = 5 * Microsoft.Automata.Internal.Utilities.HighTimer.Frequency; //5 sec
for (int i = 0; i < regexes.Count; i++)
if (i != 6)
for (int j = 0; j < regexes.Count; j++)
{
string regexA = regexes[i];
string regexB = regexes[j];
var z3p = new Z3Provider(encoding);
z3p.MainSolver.Push();
var A = z3p.RegexConverter.Convert(regexA, System.Text.RegularExpressions.RegexOptions.None);
var B = z3p.RegexConverter.Convert(regexB, System.Text.RegularExpressions.RegexOptions.None);
try
{
var C = Automaton<Expr>.MkDifference(A, B, (int)timeout);
if (!C.IsEmpty)
{
string s = GetMember(z3p, C);
Assert.IsTrue(Regex.IsMatch(s, regexA), s + " must match " + regexA);
Assert.IsFalse(Regex.IsMatch(s, regexB), s + " must not match " + regexB);
}
}
catch (TimeoutException)
{
Console.WriteLine("Timeout {0},{1}", i, j);
}
z3p.MainSolver.Pop();
}
}
private void ProdExperimentZ3(BitWidth encoding)
{
List<string> regexes = new List<string>(SampleRegexes.regexes);
regexes.RemoveRange(6, regexes.Count - 6); //just consider the first 6 cases
int nonemptyCount = 0;
var z3p = new Z3Provider(encoding);
for (int i = 0; i < regexes.Count; i++)
for (int j = 0; j < regexes.Count; j++)
{
string regexA = regexes[i];
string regexB = regexes[j];
z3p.MainSolver.Push();
var A = z3p.RegexConverter.Convert(regexA, System.Text.RegularExpressions.RegexOptions.None);
var B = z3p.RegexConverter.Convert(regexB, System.Text.RegularExpressions.RegexOptions.None);
var C = Automaton<Expr>.MkProduct(A, B);
if (i == j)
Assert.IsFalse(C.IsEmpty);
if (!C.IsEmpty)
{
nonemptyCount += 1;
string s = GetMember(z3p, C);
Assert.IsTrue(Regex.IsMatch(s, regexA, RegexOptions.None), "regex mismatch");
Assert.IsTrue(Regex.IsMatch(s, regexB, RegexOptions.None), "regex mismatch");
}
z3p.MainSolver.Pop();
}
Assert.AreEqual<int>(10, nonemptyCount, "wrong number of empty intersections");
}
private void LProdExperimentZ3(BitWidth encoding)
{
List<string> regexes = new List<string>(SampleRegexes.regexes);
regexes.RemoveRange(6, regexes.Count - 6); //just consider the first 100 cases
int nonemptyCount = 0;
for (int i = 0; i < regexes.Count; i++)
for (int j = 0; j < regexes.Count; j++)
{
string regexA = regexes[i];
string regexB = regexes[j];
var z3p = new Z3Provider(encoding);
z3p.MainSolver.Push();
var A = z3p.RegexConverter.Convert(regexA, System.Text.RegularExpressions.RegexOptions.None);
var B = z3p.RegexConverter.Convert(regexB, System.Text.RegularExpressions.RegexOptions.None);
List<Expr> witness;
bool C = Automaton<Expr>.CheckProduct(A, B, 0, out witness);
if (i == j)
Assert.IsTrue(C, "product must me nonempty");
if (C)
{
nonemptyCount += 1;
string s = new String(Array.ConvertAll(witness.ToArray(), cs => { return z3p.GetCharValue(z3p.MainSolver.FindOneMember(cs).Value); }));
Assert.IsTrue(Regex.IsMatch(s, regexA, RegexOptions.None), "regex mismatch");
Assert.IsTrue(Regex.IsMatch(s, regexB, RegexOptions.None), "regex mismatch");
}
z3p.MainSolver.Pop();
}
Assert.AreEqual<int>(10, nonemptyCount, "wrong number of empty intersections");
}
///// <summary>
///// Create a regex member generator for the given character encoding and the given random seed
///// </summary>
///// <param name="encoding">character encoding</param>
///// <param name="randomSeed">if less than 0 then a randomly chosen random seed is used</param>
//public RexEngine(BitWidth encoding, int randomSeed)
//{
// //int t = System.Environment.TickCount;
// this.encoding = encoding;
// solver = new CharSetSolver(encoding);
// chooser = (randomSeed < 0 ? new Chooser() : new Chooser(randomSeed));
// converter = solver.regexConverter;
//}
/// <summary>
/// Create a regex member generator for the given character encoding.
/// </summary>
/// <param name="encoding">character encoding</param>
public RexEngine(BitWidth encoding)
{
//int t = System.Environment.TickCount;
this.encoding = encoding;
solver = new CharSetSolver(encoding);
chooser = new Chooser();
converter = solver.regexConverter;
}
private static void WriteRangeFields(BitWidth encoding, StreamWriter sw, string field)
{
int bits = (int)encoding;
int maxChar = (1 << bits) - 1;
var catMap = new Dictionary<UnicodeCategory, Ranges>();
for (int c = 0; c < 30; c++)
catMap[(UnicodeCategory)c] = new Ranges();
Ranges whitespace = new Ranges();
Ranges wordcharacter = new Ranges();
for (int i = 0; i <= maxChar; i++)
{
char ch = (char)i;
if (char.IsWhiteSpace(ch))
whitespace.Add(i);
UnicodeCategory cat = char.GetUnicodeCategory(ch);
catMap[cat].Add(i);
int catCode = (int)cat;
//in .NET 3.5
if (bits == 7)
if (catCode == 0 || catCode == 1 || catCode == 2 || catCode == 3 || catCode == 4 || catCode == 5 || catCode == 8 || catCode == 18)
wordcharacter.Add(i);
}
//generate bdd reprs for each of the category ranges
BDD[] catBDDs = new BDD[30];
CharSetSolver bddb = new CharSetSolver(encoding);
for (int c = 0; c < 30; c++)
catBDDs[c] = bddb.MkBddForIntRanges(catMap[(UnicodeCategory)c].ranges);
BDD whitespaceBdd = bddb.MkBddForIntRanges(whitespace.ranges);
//in .NET 3.5 category 5 was NOT a word character
//union of categories 0,1,2,3,4,8,18
BDD wordCharBdd = bddb.MkOr(catBDDs[0],
bddb.MkOr(catBDDs[1],
bddb.MkOr(catBDDs[2],
bddb.MkOr(catBDDs[3],
bddb.MkOr(catBDDs[4],
bddb.MkOr(catBDDs[5],
bddb.MkOr(catBDDs[8], catBDDs[18])))))));
if (bits == 7)
{
sw.WriteLine(@"/// <summary>
/// Array of 30 UnicodeCategory ranges. Each entry is a pair of integers.
/// corresponding to the lower and upper bounds of the unicodes of the characters
/// that have the given UnicodeCategory code (between 0 and 29).
/// </summary>");
sw.WriteLine("public static int[][][] " + field + " = new int[][][]{");
foreach (UnicodeCategory c in catMap.Keys)
{
sw.WriteLine("//{0}({1}):", c, (int)c);
if (catMap[c].Count == 0)
sw.WriteLine("null,");
else
{
sw.WriteLine("new int[][]{");
foreach (int[] range in catMap[c].ranges)
sw.WriteLine(" new int[]{" + string.Format("{0},{1}", range[0], range[1]) + "},");
sw.WriteLine("},");
}
}
sw.WriteLine("};");
}
sw.WriteLine(@"/// <summary>
/// Compact BDD encodings of the categories.
/// </summary>");
sw.WriteLine("public static int[][] " + field + "Bdd = new int[][]{");
foreach (UnicodeCategory c in catMap.Keys)
{
sw.WriteLine("//{0}({1}):", c, (int)c);
BDD catBdd = catBDDs[(int)c];
if (catBdd == null || catBdd.IsEmpty)
sw.WriteLine("null, //false");
else if (catBdd.IsFull)
sw.WriteLine("new int[]{0,0}, //true");
else
{
sw.WriteLine("new int[]{");
foreach (var arc in bddb.SerializeCompact(catBdd))
sw.WriteLine("{0},", arc);
sw.WriteLine("},");
}
}
sw.WriteLine("};");
if (bits == 7)
{
sw.WriteLine(@"/// <summary>
/// Whitespace character ranges.
/// </summary>");
sw.WriteLine("public static int[][] " + field + "Whitespace = new int[][]{");
foreach (int[] range in whitespace.ranges)
sw.WriteLine(" new int[]{" + string.Format("{0},{1}", range[0], range[1]) + "},");
sw.WriteLine("};");
sw.WriteLine(@"/// <summary>
/// Word character ranges.
/// </summary>");
sw.WriteLine("public static int[][] " + field + "WordCharacter = new int[][]{");
foreach (int[] range in wordcharacter.ranges)
sw.WriteLine(" new int[]{" + string.Format("{0},{1}", range[0], range[1]) + "},");
sw.WriteLine("};");
}
sw.WriteLine(@"/// <summary>
/// Compact BDD encoding of the whitespace characters.
/// </summary>");
sw.WriteLine("public static int[] " + field + "WhitespaceBdd = new int[]{");
foreach (var arc in bddb.SerializeCompact(whitespaceBdd))
sw.WriteLine("{0},", arc);
sw.WriteLine("};");
sw.WriteLine(@"/// <summary>
/// Compact BDD encoding of word characters
/// </summary>");
sw.WriteLine("public static int[] " + field + "WordCharacterBdd = new int[]{");
foreach (var arc in bddb.SerializeCompact(wordCharBdd))
sw.WriteLine("{0},", arc);
sw.WriteLine("};");
}
public static void SaveAsRangeAutomaton(string regex, BitWidth encoding, string file, bool epsilonfree = false, bool determinize = false, int determinization_timeout_ms = 0)
{
var solver = new CharRangeSolver(encoding);
var converter = new RegexToAutomatonConverterRanges(solver);
var A = converter.Convert(regex, System.Text.RegularExpressions.RegexOptions.None);
if (epsilonfree)
A = A.RemoveEpsilons();
if (determinize)
{
A = A.RemoveEpsilons();
A.CheckDeterminism();
A = A.Determinize(determinization_timeout_ms);
}
//A.CheckDeterminism(solver,true);
//if (!A.isDeterministic)
// throw new AutomataException(AutomataExceptionKind.InternalError);
System.IO.StreamWriter sw = new System.IO.StreamWriter(file);
sw.WriteLine(A.InitialState);
foreach (var s in A.GetFinalStates())
sw.Write("{0} ", s);
sw.WriteLine();
foreach (var move in A.GetMoves())
if (move.IsEpsilon)
sw.WriteLine("{0} {1} {2} {3}", move.SourceState, -1, -1, move.TargetState);
else
foreach (var range in move.Label)
sw.WriteLine("{0} {1} {2} {3}", move.SourceState, (int)range.First, (int)range.Second, move.TargetState);
sw.Close();
}