public override void ReportAmbiguity(Parser recognizer, DFA dfa, int startIndex,
int stopIndex, BitSet ambigAlts, ATNConfigSet configs)
{
string format = "reportAmbiguity d={0}: ambigAlts={1}, input='{2}'";
recognizer.NotifyErrorListeners(string.Format(format, GetDecisionDescription(recognizer
, dfa.decision), ambigAlts, ((ITokenStream)recognizer.InputStream).GetText(Interval
.Of(startIndex, stopIndex))));
}
public TextBlock(string text, BitSet containedTextElements, int numWords, int numWordsInAnchorText
, int numWordsInWrappedLines, int numWrappedLines, int offsetBlocks)
{
this.text = text;
this.containedTextElements = containedTextElements;
this.numWords = numWords;
this.numWordsInAnchorText = numWordsInAnchorText;
this.numWordsInWrappedLines = numWordsInWrappedLines;
this.numWrappedLines = numWrappedLines;
this.offsetBlocksStart = offsetBlocks;
this.offsetBlocksEnd = offsetBlocks;
InitDensities();
}
public void TestCardinality1()
{
BitSet set = new BitSet();
Assert.AreEqual(0, set.Cardinality());
for (int i = 0; i < 128; i++)
{
set.Set(i);
Assert.AreEqual(i + 1, set.Cardinality());
Assert.AreEqual(0, set.NextSetBit(0));
if (i > 0)
Assert.AreEqual(i * 1 - 1, set.NextSetBit(i * 1 - 1));
}
}
public TextBlock(string text, BitSet containedTextElements, int numWords, int numWordsInAnchorText
, int numWordsInWrappedLines, int numWrappedLines, int offsetBlocks, string imageSrc = null)
{
this.text = text;
this.containedTextElements = containedTextElements;
this.numWords = numWords;
this.numWordsInAnchorText = numWordsInAnchorText;
this.numWordsInWrappedLines = numWordsInWrappedLines;
this.numWrappedLines = numWrappedLines;
this.offsetBlocksStart = offsetBlocks;
this.offsetBlocksEnd = offsetBlocks;
if (imageSrc != null)
{
this.nearbyImages = new List<Tuple<int, string>> { Tuple.Create(0, imageSrc) };
}
InitDensities();
}
public void Or(BitSet set)
{
if (set == null)
throw new ArgumentNullException("set");
if (set._data.Length > _data.Length)
Array.Resize(ref _data, set._data.Length);
for (int i = 0; i < set._data.Length; i++)
_data[i] |= set._data[i];
}
public BitSet Clone()
{
BitSet result = new BitSet();
result._data = (ulong[])_data.Clone();
return result;
}
protected internal virtual BitSet GetConflictingAlts(BitSet reportedAlts, ATNConfigSet
configs)
{
if (reportedAlts != null)
{
return reportedAlts;
}
BitSet result = new BitSet();
foreach (ATNConfig config in configs)
{
result.Set(config.Alt);
}
return result;
}
public virtual void SetContextSensitive(ATN atn)
{
lock (this)
{
System.Diagnostics.Debug.Assert(!configs.IsOutermostConfigSet);
if (IsContextSensitive)
{
return;
}
contextSymbols = new BitSet();
contextEdges = new SingletonEdgeMap<DFAState>(-1, atn.states.Count - 1);
}
}
/// <exception cref="System.IO.IOException"></exception>
private int BuildMatrix(ProgressMonitor pm)
{
// Allocate for the worst-case scenario where every pair has a
// score that we need to consider. We might not need that many.
//
matrix = new long[srcs.Count * dsts.Count];
long[] srcSizes = new long[srcs.Count];
long[] dstSizes = new long[dsts.Count];
BitSet dstTooLarge = null;
// Consider each pair of files, if the score is above the minimum
// threshold we need record that scoring in the matrix so we can
// later find the best matches.
//
int mNext = 0;
for (int srcIdx = 0; srcIdx < srcs.Count; srcIdx++)
{
DiffEntry srcEnt = srcs[srcIdx];
if (!IsFile(srcEnt.oldMode))
{
pm.Update(dsts.Count);
continue;
}
SimilarityIndex s = null;
for (int dstIdx = 0; dstIdx < dsts.Count; dstIdx++)
{
DiffEntry dstEnt = dsts[dstIdx];
if (!IsFile(dstEnt.newMode))
{
pm.Update(1);
continue;
}
if (!RenameDetector.SameType(srcEnt.oldMode, dstEnt.newMode))
{
pm.Update(1);
continue;
}
if (dstTooLarge != null && dstTooLarge.Get(dstIdx))
{
pm.Update(1);
continue;
}
long srcSize = srcSizes[srcIdx];
if (srcSize == 0)
{
srcSize = Size(DiffEntry.Side.OLD, srcEnt) + 1;
srcSizes[srcIdx] = srcSize;
}
long dstSize = dstSizes[dstIdx];
if (dstSize == 0)
{
dstSize = Size(DiffEntry.Side.NEW, dstEnt) + 1;
dstSizes[dstIdx] = dstSize;
}
long max = Math.Max(srcSize, dstSize);
long min = Math.Min(srcSize, dstSize);
if (min * 100 / max < renameScore)
{
// Cannot possibly match, as the file sizes are so different
pm.Update(1);
continue;
}
if (s == null)
{
try
{
s = Hash(DiffEntry.Side.OLD, srcEnt);
}
catch (SimilarityIndex.TableFullException)
{
tableOverflow = true;
goto SRC_continue;
}
}
SimilarityIndex d;
try
{
d = Hash(DiffEntry.Side.NEW, dstEnt);
}
catch (SimilarityIndex.TableFullException)
{
if (dstTooLarge == null)
{
dstTooLarge = new BitSet(dsts.Count);
}
dstTooLarge.Set(dstIdx);
tableOverflow = true;
pm.Update(1);
continue;
}
int contentScore = s.Score(d, 10000);
// nameScore returns a value between 0 and 100, but we want it
// to be in the same range as the content score. This allows it
// to be dropped into the pretty formula for the final score.
int nameScore = NameScore(srcEnt.oldPath, dstEnt.newPath) * 100;
int score = (contentScore * 99 + nameScore * 1) / 10000;
if (score < renameScore)
{
pm.Update(1);
continue;
}
matrix[mNext++] = Encode(score, srcIdx, dstIdx);
pm.Update(1);
}
SRC_continue: ;
}
SRC_break: ;
// Sort everything in the range we populated, which might be the
// entire matrix, or just a smaller slice if we had some bad low
// scoring pairs.
//
Arrays.Sort(matrix, 0, mNext);
return mNext;
}
public virtual BitSet And(BitSet a)
{
BitSet s = (BitSet)this.Clone();
s.AndInPlace(a);
return s;
}
/// <summary>
/// Compute set of tokens that can follow
/// <code>s</code>
/// in the ATN in the
/// specified
/// <code>ctx</code>
/// .
/// <p/>
/// If
/// <code>ctx</code>
/// is
/// <see cref="PredictionContext.EmptyLocal">PredictionContext.EmptyLocal</see>
/// and
/// <code>stopState</code>
/// or the end of the rule containing
/// <code>s</code>
/// is reached,
/// <see cref="TokenConstants.Epsilon"/>
/// is added to the result set. If
/// <code>ctx</code>
/// is not
/// <see cref="PredictionContext.EmptyLocal">PredictionContext.EmptyLocal</see>
/// and
/// <code>addEOF</code>
/// is
/// <code>true</code>
/// and
/// <code>stopState</code>
/// or the end of the outermost rule is reached,
/// <see cref="TokenConstants.Eof"/>
/// is added to the result set.
/// </summary>
/// <param name="s">the ATN state.</param>
/// <param name="stopState">
/// the ATN state to stop at. This can be a
/// <see cref="BlockEndState">BlockEndState</see>
/// to detect epsilon paths through a closure.
/// </param>
/// <param name="ctx">
/// The outer context, or
/// <see cref="PredictionContext.EmptyLocal">PredictionContext.EmptyLocal</see>
/// if
/// the outer context should not be used.
/// </param>
/// <param name="look">The result lookahead set.</param>
/// <param name="lookBusy">
/// A set used for preventing epsilon closures in the ATN
/// from causing a stack overflow. Outside code should pass
/// <code>new HashSet<ATNConfig></code>
/// for this argument.
/// </param>
/// <param name="calledRuleStack">
/// A set used for preventing left recursion in the
/// ATN from causing a stack overflow. Outside code should pass
/// <code>new BitSet()</code>
/// for this argument.
/// </param>
/// <param name="seeThruPreds">
///
/// <code>true</code>
/// to true semantic predicates as
/// implicitly
/// <code>true</code>
/// and "see through them", otherwise
/// <code>false</code>
/// to treat semantic predicates as opaque and add
/// <see cref="HitPred">HitPred</see>
/// to the
/// result if one is encountered.
/// </param>
/// <param name="addEOF">
/// Add
/// <see cref="TokenConstants.Eof"/>
/// to the result if the end of the
/// outermost context is reached. This parameter has no effect if
/// <code>ctx</code>
/// is
/// <see cref="PredictionContext.EmptyLocal">PredictionContext.EmptyLocal</see>
/// .
/// </param>
protected internal virtual void Look(ATNState s, ATNState stopState, PredictionContext
ctx, IntervalSet look, HashSet<ATNConfig> lookBusy, BitSet calledRuleStack,
bool seeThruPreds, bool addEOF)
{
// System.out.println("_LOOK("+s.stateNumber+", ctx="+ctx);
ATNConfig c = ATNConfig.Create(s, 0, ctx);
if (!lookBusy.Add(c))
{
return;
}
if (s == stopState)
{
if (PredictionContext.IsEmptyLocal(ctx))
{
look.Add(TokenConstants.Epsilon);
return;
}
else
{
if (ctx.IsEmpty && addEOF)
{
look.Add(TokenConstants.Eof);
return;
}
}
}
if (s is RuleStopState)
{
if (PredictionContext.IsEmptyLocal(ctx))
{
look.Add(TokenConstants.Epsilon);
return;
}
else
{
if (ctx.IsEmpty && addEOF)
{
look.Add(TokenConstants.Eof);
return;
}
}
for (int i = 0; i < ctx.Size; i++)
{
if (ctx.GetReturnState(i) != PredictionContext.EmptyFullStateKey)
{
ATNState returnState = atn.states[ctx.GetReturnState(i)];
// System.out.println("popping back to "+retState);
for (int j = 0; j < ctx.Size; j++)
{
bool removed = calledRuleStack.Get(returnState.ruleIndex);
try
{
calledRuleStack.Clear(returnState.ruleIndex);
Look(returnState, stopState, ctx.GetParent(j), look, lookBusy, calledRuleStack, seeThruPreds
, addEOF);
}
finally
{
if (removed)
{
calledRuleStack.Set(returnState.ruleIndex);
}
}
}
return;
}
}
}
int n = s.NumberOfTransitions;
for (int i_1 = 0; i_1 < n; i_1++)
{
Transition t = s.Transition(i_1);
if (t.GetType() == typeof(RuleTransition))
{
if (calledRuleStack.Get(((RuleTransition)t).target.ruleIndex))
{
continue;
}
PredictionContext newContext = ctx.GetChild(((RuleTransition)t).followState.stateNumber
);
try
{
calledRuleStack.Set(((RuleTransition)t).target.ruleIndex);
Look(t.target, stopState, newContext, look, lookBusy, calledRuleStack, seeThruPreds
, addEOF);
}
finally
{
calledRuleStack.Clear(((RuleTransition)t).target.ruleIndex);
}
}
else
{
if (t is AbstractPredicateTransition)
{
if (seeThruPreds)
{
Look(t.target, stopState, ctx, look, lookBusy, calledRuleStack, seeThruPreds, addEOF
);
}
else
{
look.Add(HitPred);
}
}
else
{
if (t.IsEpsilon)
{
Look(t.target, stopState, ctx, look, lookBusy, calledRuleStack, seeThruPreds, addEOF
);
}
else
{
if (t.GetType() == typeof(WildcardTransition))
{
look.AddAll(IntervalSet.Of(TokenConstants.MinUserTokenType, atn.maxTokenType));
}
else
{
// System.out.println("adding "+ t);
IntervalSet set = t.Label;
if (set != null)
{
if (t is NotSetTransition)
{
set = set.Complement(IntervalSet.Of(TokenConstants.MinUserTokenType, atn.maxTokenType
));
}
look.AddAll(set);
}
}
}
}
}
}
}
private void _handleSpillOverflow(int bindex, com.fasterxml.jackson.core.sym.BytesToNameCanonicalizer.Bucket
newBucket)
{
if (_overflows == null)
{
_overflows = new Sharpen.BitSet();
_overflows.set(bindex);
}
else
{
if (_overflows.get(bindex))
{
// Has happened once already, so not a coincident...
if (_failOnDoS)
{
reportTooManyCollisions(MAX_COLL_CHAIN_LENGTH);
}
// but even if we don't fail, we will stop intern()ing
_intern = false;
}
else
{
_overflows.set(bindex);
}
}
// regardless, if we get this far, clear up the bucket, adjust size appropriately.
_collList[bindex] = null;
_count -= (newBucket.length);
// we could calculate longest; but for now just mark as invalid
_longestCollisionList = -1;
}
private static bool TestTailCall(ATN atn, RuleTransition transition, bool optimizedPath
)
{
if (!optimizedPath && transition.tailCall)
{
return true;
}
if (optimizedPath && transition.optimizedTailCall)
{
return true;
}
BitSet reachable = new BitSet(atn.states.Count);
Stack<ATNState> worklist = new Stack<ATNState>();
worklist.Push(transition.followState);
while (worklist.Count > 0)
{
ATNState state = worklist.Pop();
if (reachable.Get(state.stateNumber))
{
continue;
}
if (state is RuleStopState)
{
continue;
}
if (!state.OnlyHasEpsilonTransitions)
{
return false;
}
IList<Transition> transitions = optimizedPath ? state.optimizedTransitions : state
.transitions;
foreach (Transition t in transitions)
{
if (t.TransitionType != TransitionType.Epsilon)
{
return false;
}
worklist.Push(t.target);
}
}
return true;
}
public BitSet GetDescendants(int rule)
{
BitSet descendants = new BitSet();
descendants.Or(children[rule]);
while (true)
{
int cardinality = descendants.Cardinality();
for (int i = descendants.NextSetBit(0); i >= 0; i = descendants.NextSetBit(i + 1))
{
descendants.Or(children[i]);
}
if (descendants.Cardinality() == cardinality)
{
// nothing changed
break;
}
}
return descendants;
}
public BitSet GetAncestors(int rule)
{
BitSet ancestors = new BitSet();
ancestors.Or(parents[rule]);
while (true)
{
int cardinality = ancestors.Cardinality();
for (int i = ancestors.NextSetBit(0); i >= 0; i = ancestors.NextSetBit(i + 1))
{
ancestors.Or(parents[i]);
}
if (ancestors.Cardinality() == cardinality)
{
// nothing changed
break;
}
}
return ancestors;
}
/*
* Subtract the elements of 'a' from 'this' in-place.
* Basically, just turn off all bits of 'this' that are in 'a'.
*/
public virtual void SubtractInPlace(BitSet a)
{
if (a == null)
return;
// for all words of 'a', turn off corresponding bits of 'this'
for (int i = 0; i < dataBits.Length && i < a.dataBits.Length; i++)
{
dataBits[i] &= ~a.dataBits[i];
}
}
public static BitSet of(int el)
{
BitSet s = new BitSet(el + 1);
s.Add(el);
return s;
}
public void FlushBlock()
{
if (inBody == 0) {
if (inBody == 0 && Sharpen.Runtime.EqualsIgnoreCase ("TITLE", lastStartTag))
SetTitle (tokenBuilder.ToString ().Trim ());
textBuilder.Length = 0;
tokenBuilder.Length = 0;
return;
}
int length = tokenBuilder.Length;
if (length == 0) {
return;
} else if (length == 1) {
if (sbLastWasWhitespace) {
textBuilder.Length = 0;
tokenBuilder.Length = 0;
return;
}
}
string[] tokens = UnicodeTokenizer.Tokenize (tokenBuilder);
int numWords = 0;
int numLinkedWords = 0;
int numWrappedLines = 0;
int currentLineLength = -1; // don't count the first space
int maxLineLength = 80;
int numTokens = 0;
int numWordsCurrentLine = 0;
foreach (string token in tokens) {
if (token == ANCHOR_TEXT_START) {
inAnchorText = true;
} else {
if (token == ANCHOR_TEXT_END) {
inAnchorText = false;
} else {
if (IsWord (token)) {
numTokens++;
numWords++;
numWordsCurrentLine++;
if (inAnchorText) {
numLinkedWords++;
}
int tokenLength = token.Length;
currentLineLength += tokenLength + 1;
if (currentLineLength > maxLineLength) {
numWrappedLines++;
currentLineLength = tokenLength;
numWordsCurrentLine = 1;
}
} else {
numTokens++;
}
}
}
}
if (numTokens == 0) {
return;
}
int numWordsInWrappedLines;
if (numWrappedLines == 0) {
numWordsInWrappedLines = numWords;
numWrappedLines = 1;
} else {
numWordsInWrappedLines = numWords - numWordsCurrentLine;
}
TextBlock tb = new TextBlock (textBuilder.ToString ().Trim (), currentContainedTextElements
, numWords, numLinkedWords, numWordsInWrappedLines, numWrappedLines, offsetBlocks
);
currentContainedTextElements = new BitSet ();
offsetBlocks++;
textBuilder.Length = 0;
tokenBuilder.Length = 0;
tb.SetTagLevel (blockTagLevel);
AddTextBlock (tb);
blockTagLevel = -1;
}
public virtual void AndInPlace(BitSet a)
{
int min = (int)(Math.Min(dataBits.Length, a.dataBits.Length));
for (int i = min - 1; i >= 0; i--)
{
dataBits[i] &= a.dataBits[i];
}
// clear all bits in this not present in a (if this bigger than a).
for (int i = min; i < dataBits.Length; i++)
{
dataBits[i] = 0;
}
}
/// <summary>
/// Given a start and stop index, return a
/// <code>List</code>
/// of all tokens in
/// the token type
/// <code>BitSet</code>
/// . Return
/// <code>null</code>
/// if no tokens were found. This
/// method looks at both on and off channel tokens.
/// </summary>
public virtual IList<IToken> GetTokens(int start, int stop, BitSet types)
{
LazyInit();
if (start < 0 || stop >= tokens.Count || stop < 0 || start >= tokens.Count)
{
throw new ArgumentOutOfRangeException("start " + start + " or stop " + stop + " not in 0.."
+ (tokens.Count - 1));
}
if (start > stop)
{
return null;
}
// list = tokens[start:stop]:{T t, t.getType() in types}
IList<IToken> filteredTokens = new List<IToken>();
for (int i = start; i <= stop; i++)
{
IToken t = tokens[i];
if (types == null || types.Get(t.Type))
{
filteredTokens.Add(t);
}
}
if (filteredTokens.Count == 0)
{
filteredTokens = null;
}
return filteredTokens;
}
public virtual void ReportAttemptingFullContext(Parser recognizer, DFA dfa, int startIndex
, int stopIndex, BitSet conflictingAlts, SimulatorState conflictState)
{
}
public virtual IList<IToken> GetTokens(int start, int stop, int ttype)
{
BitSet s = new BitSet(ttype);
s.Set(ttype);
return GetTokens(start, stop, s);
}
public override void ReportAttemptingFullContext(Parser recognizer, DFA dfa, int
startIndex, int stopIndex, BitSet conflictingAlts, SimulatorState conflictState
)
{
string format = "reportAttemptingFullContext d={0}, input='{1}'";
string decision = GetDecisionDescription(recognizer, dfa);
string text = ((ITokenStream)recognizer.InputStream).GetText(Interval.Of(startIndex
, stopIndex));
string message = string.Format(format, decision, text);
recognizer.NotifyErrorListeners(message);
}
public virtual object Clone()
{
BitSet s;
try
{
s = new BitSet();
s.dataBits = new long[dataBits.Length];
Array.Copy(dataBits, 0, s.dataBits, 0, dataBits.Length);
}
catch //(System.Exception e)
{
throw new System.ApplicationException();
}
return s;
}
public override void ReportAmbiguity(Parser recognizer, DFA dfa, int startIndex,
int stopIndex, bool exact, BitSet ambigAlts, ATNConfigSet configs)
{
if (exactOnly && !exact)
{
return;
}
string format = "reportAmbiguity d={0}: ambigAlts={1}, input='{2}'";
string decision = GetDecisionDescription(recognizer, dfa);
BitSet conflictingAlts = GetConflictingAlts(ambigAlts, configs);
string text = ((ITokenStream)recognizer.InputStream).GetText(Interval.Of(startIndex
, stopIndex));
string message = string.Format(format, decision, conflictingAlts, text);
recognizer.NotifyErrorListeners(message);
}
/* Return this | a in a new set. */
public virtual BitSet Or(BitSet a)
{
BitSet s = (BitSet)this.Clone();
s.OrInPlace(a);
return s;
}
public void And(BitSet set)
{
if (set == null)
throw new ArgumentNullException("set");
int length = Math.Min(_data.Length, set._data.Length);
for (int i = 0; i < length; i++)
_data[i] &= set._data[i];
for (int i = length; i < _data.Length; i++)
_data[i] = 0;
}
public virtual void OrInPlace(BitSet a)
{
// If this is smaller than a, grow this first
if (a.dataBits.Length > dataBits.Length)
{
setSize((int)(a.dataBits.Length));
}
int min = (int)(System.Math.Min(dataBits.Length, a.dataBits.Length));
for (int i = min - 1; i >= 0; i--)
{
dataBits[i] |= a.dataBits[i];
}
}
public virtual void ReportAmbiguity(Parser recognizer, DFA dfa, int startIndex, int
stopIndex, BitSet ambigAlts, ATNConfigSet configs)
{
}
/* Is this contained within a? */
public virtual bool Subset(BitSet a)
{
if (a == null) //(a == null || !(a is BitSet))
return false;
return this.And(a).Equals(this);
}
