/// <summary>Create a Boolean classifier.</summary> /// <param name="solver">Character algebra (the algebra is not stored in the classifier)</param> /// <param name="bdd">Elements that map to true.</param> public BooleanClassifier(CharSetSolver solver, BDD bdd) { // We want to optimize for ASCII, so query the BDD for each ASCII character in // order to precompute a lookup table we'll use at match time. var ascii = new bool[128]; for (int i = 0; i < ascii.Length; i++) { ascii[i] = bdd.Contains(i); } // At this point, we'll never consult the BDD for ASCII characters, so as an // optimization we can remove them from the BDD in hopes of simplifying it and making // it faster to query for the non-ASCII characters we will use it for. However, while // this is typically an optimization, it isn't always: the act of removing some // characters from the BDD can actually make the branching more complicated. The // extreme case of this is when the BDD is True, meaning everything maps to True, which // is as simple a BDD as you can get. In such a case, even though it's rare, this would // definitively be a deoptimization, so we avoid doing so. Other trivial cases are handled // by And itself, e.g. if the BDD == False, then And will just return False. if (!bdd.IsFull) { bdd = solver.And(solver._nonAscii, bdd); } _ascii = ascii; _nonAscii = bdd; }
/// <summary>Create a classifier that maps a character to the ID of its associated minterm.</summary> /// <param name="minterms">A BDD for classifying all characters (ASCII and non-ASCII) to their corresponding minterm IDs.</param> public MintermClassifier(BDD[] minterms) { Debug.Assert(minterms.Length > 0, "Requires at least"); CharSetSolver solver = CharSetSolver.Instance; if (minterms.Length == 1) { // With only a single minterm, the mapping is trivial: everything maps to it (ID 0). // For ASCII, use an array containing all zeros. For non-ASCII, use a BDD that maps everything to 0. _ascii = AllAsciiIsZeroMintermArray; _nonAscii = solver.ReplaceTrue(BDD.True, 0); return; } // Create a multi-terminal BDD for mapping any character to its associated minterm. BDD anyCharacterToMintermId = BDD.False; for (int i = 0; i < minterms.Length; i++) { // Each supplied minterm BDD decides whether a given character maps to it or not. // We need to combine all of those into a multi-terminal BDD that decides which // minterm a character maps to. To do that, we take each minterm BDD and replace // its True result with the ID of the minterm, such that a character that would // have returned True for that BDD now returns the minterm ID. BDD charToTargetMintermId = solver.ReplaceTrue(minterms[i], i); // Now union this BDD with the multi-terminal BDD we've built up thus far. Unioning // is valid because every character belongs to exactly one minterm and thus will // only map to an ID instead of False in exactly one of the input BDDs. anyCharacterToMintermId = solver.Or(anyCharacterToMintermId, charToTargetMintermId); } // Now that we have our mapping that supports any input character, we want to optimize for // ASCII inputs. Rather than forcing every input ASCII character to consult the BDD at match // time, we precompute a lookup table, where each ASCII character can be used to index into the // array to determine the ID for its corresponding minterm. var ascii = new int[128]; for (int i = 0; i < ascii.Length; i++) { ascii[i] = anyCharacterToMintermId.Find(i); } _ascii = ascii; // We can also further optimize the BDD in two ways: // 1. We can now remove the ASCII characters from it, as we'll always consult the lookup table first // for ASCII inputs and thus will never use the BDD for them. While optional (skipping this step will not // affect correctness), removing the ASCII values from the BDD reduces the size of the multi-terminal BDD. // 2. We can check if every character now maps to the same minterm ID (the same terminal in the // multi-terminal BDD). This can be relatively common after (1) above is applied, as many // patterns don't distinguish between any non-ASCII characters (e.g. "[0-9]*"). If every character // in the BDD now maps to the same minterm, we can replace the BDD with a much simpler/faster/smaller one. BDD nonAsciiBDD = solver.And(anyCharacterToMintermId, solver._nonAscii); nonAsciiBDD = nonAsciiBDD.IsEssentiallyBoolean(out BDD? singleTerminalBDD) ? singleTerminalBDD : nonAsciiBDD; _nonAscii = nonAsciiBDD; }
static char ChooseChar(Random random, BDD bdd, BDD ascii, CharSetSolver charSetSolver) { Debug.Assert(!bdd.IsEmpty); // Select characters from the visible ASCII range whenever possible BDD bdd1 = charSetSolver.And(bdd, ascii); (uint, uint)range = Choose(random, BDDRangeConverter.ToRanges(bdd1.IsEmpty ? bdd : bdd1)); return((char)random.Next((int)range.Item1, (int)range.Item2 + 1)); }
/// <summary> /// Assumes that set is a union of some minterms (or empty). /// If null then null is returned. /// </summary> public BitVector ConvertFromBDD(BDD set, CharSetSolver solver) { BDD[] partition = _minterms; BitVector result = Empty; for (int i = 0; i < partition.Length; i++) { if (!solver.IsEmpty(solver.And(partition[i], set))) { result = BitVector.Or(result, _mintermVectors[i]); } } return(result); }
/// <summary> /// Assumes that set is a union of some minterms (or empty). /// If null then 0 is returned. /// </summary> public ulong ConvertFromBDD(BDD set, CharSetSolver solver) { BDD[] partition = _minterms; ulong result = 0; for (int i = 0; i < partition.Length; i++) { // Set the i'th bit if the i'th minterm is in the set. if (!solver.IsEmpty(solver.And(partition[i], set))) { result |= (ulong)1 << i; } } return(result); }
public SymbolicRegexSampler(SymbolicRegexNode <TSet> root, int randomseed, bool negative) { _root = negative ? root._builder.Not(root) : root; // Treat 0 as no seed and instead choose a random seed randomly RandomSeed = randomseed == 0 ? new Random().Next() : randomseed; _random = new Random(RandomSeed); _solver = root._builder._solver; _charSetSolver = new CharSetSolver(); _asciiWordCharacters = _charSetSolver.Or(new BDD[] { _charSetSolver.CreateSetFromRange('A', 'Z'), _charSetSolver.CreateSetFromRange('a', 'z'), _charSetSolver.CreateFromChar('_'), _charSetSolver.CreateSetFromRange('0', '9') }); // Visible ASCII range for input character generation _ascii = _charSetSolver.CreateSetFromRange('\x20', '\x7E'); _asciiNonWordCharacters = _charSetSolver.And(_ascii, _charSetSolver.Not(_asciiWordCharacters)); }
public SymbolicRegexSampler(SymbolicRegexNode <S> root, int randomseed, bool negative) { _root = negative ? root._builder.Not(root) : root; // Treat 0 as no seed and instead choose a random seed randomly RandomSeed = randomseed == 0 ? new Random().Next() : randomseed; _random = new Random(RandomSeed); _solver = root._builder._solver; CharSetSolver bddSolver = CharSetSolver.Instance; _asciiWordCharacters = bddSolver.Or(new BDD[] { bddSolver.RangeConstraint('A', 'Z'), bddSolver.RangeConstraint('a', 'z'), bddSolver.CharConstraint('_'), bddSolver.RangeConstraint('0', '9') }); // Visible ASCII range for input character generation _ascii = bddSolver.RangeConstraint('\x20', '\x7E'); _asciiNonWordCharacters = bddSolver.And(_ascii, bddSolver.Not(_asciiWordCharacters)); }
public override IEnumerable <string> SampleMatches(int k, int randomseed) { // Zero is treated as no seed, instead using a system provided one Random random = randomseed != 0 ? new Random(randomseed) : new Random(); ISolver <TSet> solver = _builder._solver; CharSetSolver charSetSolver = _builder._charSetSolver; // Create helper BDDs for handling anchors and preferentially generating ASCII inputs BDD asciiWordCharacters = charSetSolver.Or(new BDD[] { charSetSolver.CreateBDDFromRange('A', 'Z'), charSetSolver.CreateBDDFromRange('a', 'z'), charSetSolver.CreateBDDFromChar('_'), charSetSolver.CreateBDDFromRange('0', '9') }); // Visible ASCII range for input character generation BDD ascii = charSetSolver.CreateBDDFromRange('\x20', '\x7E'); BDD asciiNonWordCharacters = charSetSolver.And(ascii, charSetSolver.Not(asciiWordCharacters)); // Set up two sets of minterms, one with the additional special minterm for the last end-of-line Debug.Assert(_builder._minterms is not null); int[] mintermIdsWithoutZ = new int[_builder._minterms.Length]; int[] mintermIdsWithZ = new int[_builder._minterms.Length + 1]; for (int i = 0; i < _builder._minterms.Length; ++i) { mintermIdsWithoutZ[i] = i; mintermIdsWithZ[i] = i; } mintermIdsWithZ[_builder._minterms.Length] = _builder._minterms.Length; for (int i = 0; i < k; i++) { // Holds the generated input so far StringBuilder inputSoFar = new(); StringBuilder?latestCandidate = null; // Current set of states reached initially contains just the root NfaMatchingState states = new(_builder); // Here one could also consider previous characters for example for \b, \B, and ^ anchors // and initialize inputSoFar accordingly states.InitializeFrom(_initialStates[GetCharKind(ReadOnlySpan <char> .Empty, -1)]); CurrentState statesWrapper = new(states); // Used for end suffixes List <string> possibleEndings = new(); while (true) { Debug.Assert(states.NfaStateSet.Count > 0); // Gather the possible endings for satisfying nullability possibleEndings.Clear(); if (NfaStateHandler.CanBeNullable(ref statesWrapper)) { // Unconditionally final state or end of the input due to \Z anchor for example if (NfaStateHandler.IsNullable(ref statesWrapper) || NfaStateHandler.IsNullableFor(_builder, ref statesWrapper, CharKind.BeginningEnd)) { possibleEndings.Add(""); } // End of line due to end-of-line anchor if (NfaStateHandler.IsNullableFor(_builder, ref statesWrapper, CharKind.Newline)) { possibleEndings.Add("\n"); } // Related to wordborder due to \b or \B if (NfaStateHandler.IsNullableFor(_builder, ref statesWrapper, CharKind.WordLetter)) { possibleEndings.Add(ChooseChar(random, asciiWordCharacters, ascii, charSetSolver).ToString()); } // Related to wordborder due to \b or \B if (NfaStateHandler.IsNullableFor(_builder, ref statesWrapper, CharKind.General)) { possibleEndings.Add(ChooseChar(random, asciiNonWordCharacters, ascii, charSetSolver).ToString()); } } // If we have a possible ending, then store a candidate input if (possibleEndings.Count > 0) { latestCandidate ??= new(); latestCandidate.Clear(); latestCandidate.Append(inputSoFar); //Choose some suffix that allows some anchor (if any) to be nullable latestCandidate.Append(Choose(random, possibleEndings)); // Choose to stop here based on a coin-toss if (FlipBiasedCoin(random, SampleMatchesStoppingProbability)) { yield return(latestCandidate.ToString()); break; } } // Shuffle the minterms, including the last end-of-line marker if appropriate int[] mintermIds = NfaStateHandler.StartsWithLineAnchor(_builder, ref statesWrapper) ? Shuffle(random, mintermIdsWithZ) : Shuffle(random, mintermIdsWithoutZ); foreach (int mintermId in mintermIds) { bool success = NfaStateHandler.TakeTransition(_builder, ref statesWrapper, mintermId); Debug.Assert(success); if (states.NfaStateSet.Count > 0) { TSet minterm = _builder.GetMinterm(mintermId); // Append a random member of the minterm inputSoFar.Append(ChooseChar(random, ToBDD(minterm, solver, charSetSolver), ascii, charSetSolver)); break; } else { // The transition was a dead end, undo and continue to try another minterm NfaStateHandler.UndoTransition(ref statesWrapper); } } // In the case that there are no next states or input has become too large: stop here if (states.NfaStateSet.Count == 0 || inputSoFar.Length > SampleMatchesMaxInputLength) { // Ending up here without an ending is unlikely but possible for example for infeasible patterns // such as @"no\bway" or due to poor choice of c -- no anchor is enabled -- so this is a deadend. if (latestCandidate != null) { yield return(latestCandidate.ToString()); } break; } } }