public override void ReportAmbiguity(Parser recognizer, DFA dfa, int startIndex, int stopIndex, BitSet ambigAlts, ATNConfigSet configs) { string format = "reportAmbiguity d={0}: ambigAlts={1}, input='{2}'"; recognizer.NotifyErrorListeners(string.Format(format, GetDecisionDescription(recognizer , dfa.decision), ambigAlts, ((ITokenStream)recognizer.InputStream).GetText(Interval .Of(startIndex, stopIndex)))); }
public TextBlock(string text, BitSet containedTextElements, int numWords, int numWordsInAnchorText , int numWordsInWrappedLines, int numWrappedLines, int offsetBlocks) { this.text = text; this.containedTextElements = containedTextElements; this.numWords = numWords; this.numWordsInAnchorText = numWordsInAnchorText; this.numWordsInWrappedLines = numWordsInWrappedLines; this.numWrappedLines = numWrappedLines; this.offsetBlocksStart = offsetBlocks; this.offsetBlocksEnd = offsetBlocks; InitDensities(); }
public void TestCardinality1() { BitSet set = new BitSet(); Assert.AreEqual(0, set.Cardinality()); for (int i = 0; i < 128; i++) { set.Set(i); Assert.AreEqual(i + 1, set.Cardinality()); Assert.AreEqual(0, set.NextSetBit(0)); if (i > 0) Assert.AreEqual(i * 1 - 1, set.NextSetBit(i * 1 - 1)); } }
public TextBlock(string text, BitSet containedTextElements, int numWords, int numWordsInAnchorText , int numWordsInWrappedLines, int numWrappedLines, int offsetBlocks, string imageSrc = null) { this.text = text; this.containedTextElements = containedTextElements; this.numWords = numWords; this.numWordsInAnchorText = numWordsInAnchorText; this.numWordsInWrappedLines = numWordsInWrappedLines; this.numWrappedLines = numWrappedLines; this.offsetBlocksStart = offsetBlocks; this.offsetBlocksEnd = offsetBlocks; if (imageSrc != null) { this.nearbyImages = new List<Tuple<int, string>> { Tuple.Create(0, imageSrc) }; } InitDensities(); }
public void Or(BitSet set) { if (set == null) throw new ArgumentNullException("set"); if (set._data.Length > _data.Length) Array.Resize(ref _data, set._data.Length); for (int i = 0; i < set._data.Length; i++) _data[i] |= set._data[i]; }
public BitSet Clone() { BitSet result = new BitSet(); result._data = (ulong[])_data.Clone(); return result; }
protected internal virtual BitSet GetConflictingAlts(BitSet reportedAlts, ATNConfigSet configs) { if (reportedAlts != null) { return reportedAlts; } BitSet result = new BitSet(); foreach (ATNConfig config in configs) { result.Set(config.Alt); } return result; }
public virtual void SetContextSensitive(ATN atn) { lock (this) { System.Diagnostics.Debug.Assert(!configs.IsOutermostConfigSet); if (IsContextSensitive) { return; } contextSymbols = new BitSet(); contextEdges = new SingletonEdgeMap<DFAState>(-1, atn.states.Count - 1); } }
/// <exception cref="System.IO.IOException"></exception> private int BuildMatrix(ProgressMonitor pm) { // Allocate for the worst-case scenario where every pair has a // score that we need to consider. We might not need that many. // matrix = new long[srcs.Count * dsts.Count]; long[] srcSizes = new long[srcs.Count]; long[] dstSizes = new long[dsts.Count]; BitSet dstTooLarge = null; // Consider each pair of files, if the score is above the minimum // threshold we need record that scoring in the matrix so we can // later find the best matches. // int mNext = 0; for (int srcIdx = 0; srcIdx < srcs.Count; srcIdx++) { DiffEntry srcEnt = srcs[srcIdx]; if (!IsFile(srcEnt.oldMode)) { pm.Update(dsts.Count); continue; } SimilarityIndex s = null; for (int dstIdx = 0; dstIdx < dsts.Count; dstIdx++) { DiffEntry dstEnt = dsts[dstIdx]; if (!IsFile(dstEnt.newMode)) { pm.Update(1); continue; } if (!RenameDetector.SameType(srcEnt.oldMode, dstEnt.newMode)) { pm.Update(1); continue; } if (dstTooLarge != null && dstTooLarge.Get(dstIdx)) { pm.Update(1); continue; } long srcSize = srcSizes[srcIdx]; if (srcSize == 0) { srcSize = Size(DiffEntry.Side.OLD, srcEnt) + 1; srcSizes[srcIdx] = srcSize; } long dstSize = dstSizes[dstIdx]; if (dstSize == 0) { dstSize = Size(DiffEntry.Side.NEW, dstEnt) + 1; dstSizes[dstIdx] = dstSize; } long max = Math.Max(srcSize, dstSize); long min = Math.Min(srcSize, dstSize); if (min * 100 / max < renameScore) { // Cannot possibly match, as the file sizes are so different pm.Update(1); continue; } if (s == null) { try { s = Hash(DiffEntry.Side.OLD, srcEnt); } catch (SimilarityIndex.TableFullException) { tableOverflow = true; goto SRC_continue; } } SimilarityIndex d; try { d = Hash(DiffEntry.Side.NEW, dstEnt); } catch (SimilarityIndex.TableFullException) { if (dstTooLarge == null) { dstTooLarge = new BitSet(dsts.Count); } dstTooLarge.Set(dstIdx); tableOverflow = true; pm.Update(1); continue; } int contentScore = s.Score(d, 10000); // nameScore returns a value between 0 and 100, but we want it // to be in the same range as the content score. This allows it // to be dropped into the pretty formula for the final score. int nameScore = NameScore(srcEnt.oldPath, dstEnt.newPath) * 100; int score = (contentScore * 99 + nameScore * 1) / 10000; if (score < renameScore) { pm.Update(1); continue; } matrix[mNext++] = Encode(score, srcIdx, dstIdx); pm.Update(1); } SRC_continue: ; } SRC_break: ; // Sort everything in the range we populated, which might be the // entire matrix, or just a smaller slice if we had some bad low // scoring pairs. // Arrays.Sort(matrix, 0, mNext); return mNext; }
public virtual BitSet And(BitSet a) { BitSet s = (BitSet)this.Clone(); s.AndInPlace(a); return s; }
/// <summary> /// Compute set of tokens that can follow /// <code>s</code> /// in the ATN in the /// specified /// <code>ctx</code> /// . /// <p/> /// If /// <code>ctx</code> /// is /// <see cref="PredictionContext.EmptyLocal">PredictionContext.EmptyLocal</see> /// and /// <code>stopState</code> /// or the end of the rule containing /// <code>s</code> /// is reached, /// <see cref="TokenConstants.Epsilon"/> /// is added to the result set. If /// <code>ctx</code> /// is not /// <see cref="PredictionContext.EmptyLocal">PredictionContext.EmptyLocal</see> /// and /// <code>addEOF</code> /// is /// <code>true</code> /// and /// <code>stopState</code> /// or the end of the outermost rule is reached, /// <see cref="TokenConstants.Eof"/> /// is added to the result set. /// </summary> /// <param name="s">the ATN state.</param> /// <param name="stopState"> /// the ATN state to stop at. This can be a /// <see cref="BlockEndState">BlockEndState</see> /// to detect epsilon paths through a closure. /// </param> /// <param name="ctx"> /// The outer context, or /// <see cref="PredictionContext.EmptyLocal">PredictionContext.EmptyLocal</see> /// if /// the outer context should not be used. /// </param> /// <param name="look">The result lookahead set.</param> /// <param name="lookBusy"> /// A set used for preventing epsilon closures in the ATN /// from causing a stack overflow. Outside code should pass /// <code>new HashSet<ATNConfig></code> /// for this argument. /// </param> /// <param name="calledRuleStack"> /// A set used for preventing left recursion in the /// ATN from causing a stack overflow. Outside code should pass /// <code>new BitSet()</code> /// for this argument. /// </param> /// <param name="seeThruPreds"> /// /// <code>true</code> /// to true semantic predicates as /// implicitly /// <code>true</code> /// and "see through them", otherwise /// <code>false</code> /// to treat semantic predicates as opaque and add /// <see cref="HitPred">HitPred</see> /// to the /// result if one is encountered. /// </param> /// <param name="addEOF"> /// Add /// <see cref="TokenConstants.Eof"/> /// to the result if the end of the /// outermost context is reached. This parameter has no effect if /// <code>ctx</code> /// is /// <see cref="PredictionContext.EmptyLocal">PredictionContext.EmptyLocal</see> /// . /// </param> protected internal virtual void Look(ATNState s, ATNState stopState, PredictionContext ctx, IntervalSet look, HashSet<ATNConfig> lookBusy, BitSet calledRuleStack, bool seeThruPreds, bool addEOF) { // System.out.println("_LOOK("+s.stateNumber+", ctx="+ctx); ATNConfig c = ATNConfig.Create(s, 0, ctx); if (!lookBusy.Add(c)) { return; } if (s == stopState) { if (PredictionContext.IsEmptyLocal(ctx)) { look.Add(TokenConstants.Epsilon); return; } else { if (ctx.IsEmpty && addEOF) { look.Add(TokenConstants.Eof); return; } } } if (s is RuleStopState) { if (PredictionContext.IsEmptyLocal(ctx)) { look.Add(TokenConstants.Epsilon); return; } else { if (ctx.IsEmpty && addEOF) { look.Add(TokenConstants.Eof); return; } } for (int i = 0; i < ctx.Size; i++) { if (ctx.GetReturnState(i) != PredictionContext.EmptyFullStateKey) { ATNState returnState = atn.states[ctx.GetReturnState(i)]; // System.out.println("popping back to "+retState); for (int j = 0; j < ctx.Size; j++) { bool removed = calledRuleStack.Get(returnState.ruleIndex); try { calledRuleStack.Clear(returnState.ruleIndex); Look(returnState, stopState, ctx.GetParent(j), look, lookBusy, calledRuleStack, seeThruPreds , addEOF); } finally { if (removed) { calledRuleStack.Set(returnState.ruleIndex); } } } return; } } } int n = s.NumberOfTransitions; for (int i_1 = 0; i_1 < n; i_1++) { Transition t = s.Transition(i_1); if (t.GetType() == typeof(RuleTransition)) { if (calledRuleStack.Get(((RuleTransition)t).target.ruleIndex)) { continue; } PredictionContext newContext = ctx.GetChild(((RuleTransition)t).followState.stateNumber ); try { calledRuleStack.Set(((RuleTransition)t).target.ruleIndex); Look(t.target, stopState, newContext, look, lookBusy, calledRuleStack, seeThruPreds , addEOF); } finally { calledRuleStack.Clear(((RuleTransition)t).target.ruleIndex); } } else { if (t is AbstractPredicateTransition) { if (seeThruPreds) { Look(t.target, stopState, ctx, look, lookBusy, calledRuleStack, seeThruPreds, addEOF ); } else { look.Add(HitPred); } } else { if (t.IsEpsilon) { Look(t.target, stopState, ctx, look, lookBusy, calledRuleStack, seeThruPreds, addEOF ); } else { if (t.GetType() == typeof(WildcardTransition)) { look.AddAll(IntervalSet.Of(TokenConstants.MinUserTokenType, atn.maxTokenType)); } else { // System.out.println("adding "+ t); IntervalSet set = t.Label; if (set != null) { if (t is NotSetTransition) { set = set.Complement(IntervalSet.Of(TokenConstants.MinUserTokenType, atn.maxTokenType )); } look.AddAll(set); } } } } } } }
private void _handleSpillOverflow(int bindex, com.fasterxml.jackson.core.sym.BytesToNameCanonicalizer.Bucket newBucket) { if (_overflows == null) { _overflows = new Sharpen.BitSet(); _overflows.set(bindex); } else { if (_overflows.get(bindex)) { // Has happened once already, so not a coincident... if (_failOnDoS) { reportTooManyCollisions(MAX_COLL_CHAIN_LENGTH); } // but even if we don't fail, we will stop intern()ing _intern = false; } else { _overflows.set(bindex); } } // regardless, if we get this far, clear up the bucket, adjust size appropriately. _collList[bindex] = null; _count -= (newBucket.length); // we could calculate longest; but for now just mark as invalid _longestCollisionList = -1; }
private static bool TestTailCall(ATN atn, RuleTransition transition, bool optimizedPath ) { if (!optimizedPath && transition.tailCall) { return true; } if (optimizedPath && transition.optimizedTailCall) { return true; } BitSet reachable = new BitSet(atn.states.Count); Stack<ATNState> worklist = new Stack<ATNState>(); worklist.Push(transition.followState); while (worklist.Count > 0) { ATNState state = worklist.Pop(); if (reachable.Get(state.stateNumber)) { continue; } if (state is RuleStopState) { continue; } if (!state.OnlyHasEpsilonTransitions) { return false; } IList<Transition> transitions = optimizedPath ? state.optimizedTransitions : state .transitions; foreach (Transition t in transitions) { if (t.TransitionType != TransitionType.Epsilon) { return false; } worklist.Push(t.target); } } return true; }
public BitSet GetDescendants(int rule) { BitSet descendants = new BitSet(); descendants.Or(children[rule]); while (true) { int cardinality = descendants.Cardinality(); for (int i = descendants.NextSetBit(0); i >= 0; i = descendants.NextSetBit(i + 1)) { descendants.Or(children[i]); } if (descendants.Cardinality() == cardinality) { // nothing changed break; } } return descendants; }
public BitSet GetAncestors(int rule) { BitSet ancestors = new BitSet(); ancestors.Or(parents[rule]); while (true) { int cardinality = ancestors.Cardinality(); for (int i = ancestors.NextSetBit(0); i >= 0; i = ancestors.NextSetBit(i + 1)) { ancestors.Or(parents[i]); } if (ancestors.Cardinality() == cardinality) { // nothing changed break; } } return ancestors; }
/* * Subtract the elements of 'a' from 'this' in-place. * Basically, just turn off all bits of 'this' that are in 'a'. */ public virtual void SubtractInPlace(BitSet a) { if (a == null) return; // for all words of 'a', turn off corresponding bits of 'this' for (int i = 0; i < dataBits.Length && i < a.dataBits.Length; i++) { dataBits[i] &= ~a.dataBits[i]; } }
public static BitSet of(int el) { BitSet s = new BitSet(el + 1); s.Add(el); return s; }
public void FlushBlock() { if (inBody == 0) { if (inBody == 0 && Sharpen.Runtime.EqualsIgnoreCase ("TITLE", lastStartTag)) SetTitle (tokenBuilder.ToString ().Trim ()); textBuilder.Length = 0; tokenBuilder.Length = 0; return; } int length = tokenBuilder.Length; if (length == 0) { return; } else if (length == 1) { if (sbLastWasWhitespace) { textBuilder.Length = 0; tokenBuilder.Length = 0; return; } } string[] tokens = UnicodeTokenizer.Tokenize (tokenBuilder); int numWords = 0; int numLinkedWords = 0; int numWrappedLines = 0; int currentLineLength = -1; // don't count the first space int maxLineLength = 80; int numTokens = 0; int numWordsCurrentLine = 0; foreach (string token in tokens) { if (token == ANCHOR_TEXT_START) { inAnchorText = true; } else { if (token == ANCHOR_TEXT_END) { inAnchorText = false; } else { if (IsWord (token)) { numTokens++; numWords++; numWordsCurrentLine++; if (inAnchorText) { numLinkedWords++; } int tokenLength = token.Length; currentLineLength += tokenLength + 1; if (currentLineLength > maxLineLength) { numWrappedLines++; currentLineLength = tokenLength; numWordsCurrentLine = 1; } } else { numTokens++; } } } } if (numTokens == 0) { return; } int numWordsInWrappedLines; if (numWrappedLines == 0) { numWordsInWrappedLines = numWords; numWrappedLines = 1; } else { numWordsInWrappedLines = numWords - numWordsCurrentLine; } TextBlock tb = new TextBlock (textBuilder.ToString ().Trim (), currentContainedTextElements , numWords, numLinkedWords, numWordsInWrappedLines, numWrappedLines, offsetBlocks ); currentContainedTextElements = new BitSet (); offsetBlocks++; textBuilder.Length = 0; tokenBuilder.Length = 0; tb.SetTagLevel (blockTagLevel); AddTextBlock (tb); blockTagLevel = -1; }
public virtual void AndInPlace(BitSet a) { int min = (int)(Math.Min(dataBits.Length, a.dataBits.Length)); for (int i = min - 1; i >= 0; i--) { dataBits[i] &= a.dataBits[i]; } // clear all bits in this not present in a (if this bigger than a). for (int i = min; i < dataBits.Length; i++) { dataBits[i] = 0; } }
/// <summary> /// Given a start and stop index, return a /// <code>List</code> /// of all tokens in /// the token type /// <code>BitSet</code> /// . Return /// <code>null</code> /// if no tokens were found. This /// method looks at both on and off channel tokens. /// </summary> public virtual IList<IToken> GetTokens(int start, int stop, BitSet types) { LazyInit(); if (start < 0 || stop >= tokens.Count || stop < 0 || start >= tokens.Count) { throw new ArgumentOutOfRangeException("start " + start + " or stop " + stop + " not in 0.." + (tokens.Count - 1)); } if (start > stop) { return null; } // list = tokens[start:stop]:{T t, t.getType() in types} IList<IToken> filteredTokens = new List<IToken>(); for (int i = start; i <= stop; i++) { IToken t = tokens[i]; if (types == null || types.Get(t.Type)) { filteredTokens.Add(t); } } if (filteredTokens.Count == 0) { filteredTokens = null; } return filteredTokens; }
public virtual void ReportAttemptingFullContext(Parser recognizer, DFA dfa, int startIndex , int stopIndex, BitSet conflictingAlts, SimulatorState conflictState) { }
public virtual IList<IToken> GetTokens(int start, int stop, int ttype) { BitSet s = new BitSet(ttype); s.Set(ttype); return GetTokens(start, stop, s); }
public override void ReportAttemptingFullContext(Parser recognizer, DFA dfa, int startIndex, int stopIndex, BitSet conflictingAlts, SimulatorState conflictState ) { string format = "reportAttemptingFullContext d={0}, input='{1}'"; string decision = GetDecisionDescription(recognizer, dfa); string text = ((ITokenStream)recognizer.InputStream).GetText(Interval.Of(startIndex , stopIndex)); string message = string.Format(format, decision, text); recognizer.NotifyErrorListeners(message); }
public virtual object Clone() { BitSet s; try { s = new BitSet(); s.dataBits = new long[dataBits.Length]; Array.Copy(dataBits, 0, s.dataBits, 0, dataBits.Length); } catch //(System.Exception e) { throw new System.ApplicationException(); } return s; }
public override void ReportAmbiguity(Parser recognizer, DFA dfa, int startIndex, int stopIndex, bool exact, BitSet ambigAlts, ATNConfigSet configs) { if (exactOnly && !exact) { return; } string format = "reportAmbiguity d={0}: ambigAlts={1}, input='{2}'"; string decision = GetDecisionDescription(recognizer, dfa); BitSet conflictingAlts = GetConflictingAlts(ambigAlts, configs); string text = ((ITokenStream)recognizer.InputStream).GetText(Interval.Of(startIndex , stopIndex)); string message = string.Format(format, decision, conflictingAlts, text); recognizer.NotifyErrorListeners(message); }
/* Return this | a in a new set. */ public virtual BitSet Or(BitSet a) { BitSet s = (BitSet)this.Clone(); s.OrInPlace(a); return s; }
public void And(BitSet set) { if (set == null) throw new ArgumentNullException("set"); int length = Math.Min(_data.Length, set._data.Length); for (int i = 0; i < length; i++) _data[i] &= set._data[i]; for (int i = length; i < _data.Length; i++) _data[i] = 0; }
public virtual void OrInPlace(BitSet a) { // If this is smaller than a, grow this first if (a.dataBits.Length > dataBits.Length) { setSize((int)(a.dataBits.Length)); } int min = (int)(System.Math.Min(dataBits.Length, a.dataBits.Length)); for (int i = min - 1; i >= 0; i--) { dataBits[i] |= a.dataBits[i]; } }
public virtual void ReportAmbiguity(Parser recognizer, DFA dfa, int startIndex, int stopIndex, BitSet ambigAlts, ATNConfigSet configs) { }
/* Is this contained within a? */ public virtual bool Subset(BitSet a) { if (a == null) //(a == null || !(a is BitSet)) return false; return this.And(a).Equals(this); }