public virtual void BackpropDerivative(Tree tree, IList <string> words, IdentityHashMap <Tree, SimpleMatrix> nodeVectors, TwoDimensionalMap <string, string, SimpleMatrix> binaryW_dfs, IDictionary <string, SimpleMatrix> unaryW_dfs, TwoDimensionalMap
<string, string, SimpleMatrix> binaryScoreDerivatives, IDictionary <string, SimpleMatrix> unaryScoreDerivatives, IDictionary <string, SimpleMatrix> wordVectorDerivatives)
{
SimpleMatrix delta = new SimpleMatrix(op.lexOptions.numHid, 1);
BackpropDerivative(tree, words, nodeVectors, binaryW_dfs, unaryW_dfs, binaryScoreDerivatives, unaryScoreDerivatives, wordVectorDerivatives, delta);
}
// TODO: make this part of DVModel or DVParser?
public virtual double Score(Tree tree, IdentityHashMap <Tree, SimpleMatrix> nodeVectors)
{
IList <string> words = GetContextWords(tree);
// score of the entire tree is the sum of the scores of all of
// its nodes
// TODO: make the node vectors part of the tree itself?
IdentityHashMap <Tree, double> scores = new IdentityHashMap <Tree, double>();
try
{
ForwardPropagateTree(tree, words, nodeVectors, scores);
}
catch (AssertionError e)
{
log.Info("Failed to correctly process tree " + tree);
throw;
}
double score = 0.0;
foreach (Tree node in scores.Keys)
{
score += scores[node];
}
//log.info(Double.toString(score));
return(score);
}
public virtual DeepTree GetHighestScoringTree(Tree tree, double lambda)
{
IList <Tree> hypotheses = topParses[tree];
if (hypotheses == null || hypotheses.Count == 0)
{
throw new AssertionError("Failed to get any hypothesis trees for " + tree);
}
double bestScore = double.NegativeInfinity;
Tree bestTree = null;
IdentityHashMap <Tree, SimpleMatrix> bestVectors = null;
foreach (Tree hypothesis in hypotheses)
{
IdentityHashMap <Tree, SimpleMatrix> nodeVectors = new IdentityHashMap <Tree, SimpleMatrix>();
double scoreHyp = Score(hypothesis, nodeVectors);
double deltaMargin = 0;
if (lambda != 0)
{
//TODO: RS: Play around with this parameter to prevent blowing up of scores
deltaMargin = op.trainOptions.deltaMargin * lambda * GetMargin(tree, hypothesis);
}
scoreHyp = scoreHyp + deltaMargin;
if (bestTree == null || scoreHyp > bestScore)
{
bestTree = hypothesis;
bestScore = scoreHyp;
bestVectors = nodeVectors;
}
}
DeepTree returnTree = new DeepTree(bestTree, bestVectors, bestScore);
return(returnTree);
}
public DVParserCostAndGradient(IList <Tree> trainingBatch, IdentityHashMap <Tree, IList <Tree> > topParses, DVModel dvModel, Options op)
{
this.trainingBatch = trainingBatch;
this.topParses = topParses;
this.dvModel = dvModel;
this.op = op;
}
/// <summary>
/// Returns a new non-threadsafe context map.
/// </summary>
public static IDictionary <string, IndexSearcher> NewContext(IndexSearcher searcher)
{
var context = new IdentityHashMap <string, IndexSearcher>();
context["searcher"] = searcher;
return(context);
}
public _AbstractMap_119(COLL coremaps, IdentityHashMap <CM, bool> references, Type valueKey, ICollection <KeyValuePair <CM, V> > entrySet)
{
this.coremaps = coremaps;
this.references = references;
this.valueKey = valueKey;
this.entrySet = entrySet;
}
public IList <IEntityMetaData> GetMetaData(IList <Type> entityTypes)
{
IdentityHashMap <IMergeServiceExtension, IList <Type> > mseToEntityTypes = new IdentityHashMap <IMergeServiceExtension, IList <Type> >();
for (int a = entityTypes.Count; a-- > 0;)
{
Type entityType = entityTypes[a];
IMergeServiceExtension mergeServiceExtension = mergeServiceExtensions.GetExtension(entityType);
if (mergeServiceExtension == null)
{
continue;
}
IList <Type> groupedEntityTypes = mseToEntityTypes.Get(mergeServiceExtension);
if (groupedEntityTypes == null)
{
groupedEntityTypes = new List <Type>();
mseToEntityTypes.Put(mergeServiceExtension, groupedEntityTypes);
}
groupedEntityTypes.Add(entityType);
}
List <IEntityMetaData> metaDataResult = new List <IEntityMetaData>(entityTypes.Count);
foreach (Entry <IMergeServiceExtension, IList <Type> > entry in mseToEntityTypes)
{
IList <IEntityMetaData> groupedMetaData = entry.Key.GetMetaData(entry.Value);
metaDataResult.AddRange(groupedMetaData);
}
return(metaDataResult);
}
public static CacheDependencyNode AddRootCache(IRootCache rootCache, IdentityHashMap <IRootCache, CacheDependencyNode> rootCacheToNodeMap)
{
rootCache = rootCache.CurrentRootCache;
CacheDependencyNode node = rootCacheToNodeMap.Get(rootCache);
if (node != null)
{
return(node);
}
CacheDependencyNode parentNode = null;
IRootCache parent = ((RootCache)rootCache).Parent;
if (parent != null)
{
parentNode = AddRootCache(parent, rootCacheToNodeMap);
}
node = new CacheDependencyNode(rootCache);
node.parentNode = parentNode;
if (parentNode != null)
{
parentNode.childNodes.Add(node);
}
rootCacheToNodeMap.Put(rootCache, node);
return(node);
}
internal static IdentityHashMap <Tree, Tree> BuildParentMap(Tree tree)
{
IdentityHashMap <Tree, Tree> map = Generics.NewIdentityHashMap();
BuildParentMapHelper(tree, null, map);
return(map);
}
public virtual void TestReuseDocsEnumNoReuse()
{
Directory dir = NewDirectory();
Codec cp = TestUtil.AlwaysPostingsFormat(new Lucene40RWPostingsFormat());
RandomIndexWriter writer = new RandomIndexWriter(Random(), dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetCodec(cp));
int numdocs = AtLeast(20);
CreateRandomIndex(numdocs, writer, Random());
writer.Commit();
DirectoryReader open = DirectoryReader.Open(dir);
foreach (AtomicReaderContext ctx in open.Leaves)
{
AtomicReader indexReader = (AtomicReader)ctx.Reader;
Terms terms = indexReader.GetTerms("body");
TermsEnum iterator = terms.GetIterator(null);
IdentityHashMap <DocsEnum, bool?> enums = new IdentityHashMap <DocsEnum, bool?>();
MatchNoBits bits = new MatchNoBits(indexReader.MaxDoc);
while ((iterator.Next()) != null)
{
DocsEnum docs = iterator.Docs(Random().NextBoolean() ? bits : new MatchNoBits(indexReader.MaxDoc), null, Random().NextBoolean() ? DocsFlags.FREQS : DocsFlags.NONE);
enums[docs] = true;
}
Assert.AreEqual(terms.Count, enums.Count);
}
IOUtils.Close(writer, open, dir);
}
public virtual void TestReuseDocsEnumNoReuse()
{
Directory dir = NewDirectory();
Codec cp = TestUtil.AlwaysPostingsFormat(new Lucene40RWPostingsFormat());
RandomIndexWriter writer = new RandomIndexWriter(Random(), dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetCodec(cp));
int numdocs = AtLeast(20);
CreateRandomIndex(numdocs, writer, Random());
writer.Commit();
DirectoryReader open = DirectoryReader.Open(dir);
foreach (AtomicReaderContext ctx in open.Leaves())
{
AtomicReader indexReader = (AtomicReader)ctx.Reader();
Terms terms = indexReader.Terms("body");
TermsEnum iterator = terms.Iterator(null);
IdentityHashMap<DocsEnum, bool?> enums = new IdentityHashMap<DocsEnum, bool?>();
MatchNoBits bits = new MatchNoBits(indexReader.MaxDoc());
while ((iterator.Next()) != null)
{
DocsEnum docs = iterator.Docs(Random().NextBoolean() ? bits : new MatchNoBits(indexReader.MaxDoc()), null, Random().NextBoolean() ? DocsEnum.FLAG_FREQS : DocsEnum.FLAG_NONE);
enums[docs] = true;
}
Assert.AreEqual(terms.Size(), enums.Count);
}
IOUtils.Close(writer, open, dir);
}
/* Iterates over all application hooks creating a new thread for each
* to run in. Hooks are run concurrently and this method waits for
* them to finish.
*/
internal static void RunHooks()
{
Collection <Thread> threads;
lock (typeof(ApplicationShutdownHooks))
{
threads = Hooks.KeySet();
Hooks = null;
}
foreach (Thread hook in threads)
{
hook.Start();
}
foreach (Thread hook in threads)
{
try
{
hook.Join();
}
catch (InterruptedException)
{
}
}
}
protected CacheDependencyNode BuildCacheDependency()
{
IRootCache privilegedSecondLevelCache = SecondLevelCacheManager.SelectPrivilegedSecondLevelCache(true);
IRootCache nonPrivilegedSecondLevelCache = SecondLevelCacheManager.SelectNonPrivilegedSecondLevelCache(false);
IList <IWritableCache> selectedFirstLevelCaches = FirstLevelCacheManager.SelectFirstLevelCaches();
IdentityHashMap <IRootCache, CacheDependencyNode> secondLevelCacheToNodeMap = new IdentityHashMap <IRootCache, CacheDependencyNode>();
if (privilegedSecondLevelCache != null)
{
CacheDependencyNodeFactory.AddRootCache(privilegedSecondLevelCache.CurrentRootCache, secondLevelCacheToNodeMap);
}
if (nonPrivilegedSecondLevelCache != null)
{
CacheDependencyNodeFactory.AddRootCache(nonPrivilegedSecondLevelCache.CurrentRootCache, secondLevelCacheToNodeMap);
}
for (int a = selectedFirstLevelCaches.Count; a-- > 0;)
{
ChildCache childCache = (ChildCache)selectedFirstLevelCaches[a];
IRootCache parent = ((IRootCache)childCache.Parent).CurrentRootCache;
CacheDependencyNode node = CacheDependencyNodeFactory.AddRootCache(parent, secondLevelCacheToNodeMap);
node.directChildCaches.Add(childCache);
}
return(CacheDependencyNodeFactory.BuildRootNode(secondLevelCacheToNodeMap));
}
public override void varDumpImpl(Env env,
WriteStream @out,
int depth,
IdentityHashMap <Value, String> valueSet)
{
@out.print("int(" + toLong() + ")");
}
public _IEnumerator_80(COLL coremaps, IdentityHashMap <CM, bool> references, Type valueKey)
{
this.coremaps = coremaps;
this.references = references;
this.valueKey = valueKey;
this.it = coremaps.GetEnumerator();
this.last = null;
}
public void varDumpImpl(Env env,
WriteStream @out,
int depth,
IdentityHashMap <Value, String> valueSet)
{
@out.print(getClass().getName());
}
/// <summary>
/// Construct a new, empty IdentityHashSet whose backing IdentityHashMap
/// has the default expected maximum size (21);
/// </summary>
public IdentityHashSet()
{
// todo: The Java bug database notes that "From 1.6, an identity hash set can be created by Collections.newSetFromMap(new IdentityHashMap())."
// INSTANCE VARIABLES -------------------------------------------------
// the IdentityHashMap which backs this set
// CONSTRUCTORS ---------------------------------------------------------
map = new IdentityHashMap <E, bool>();
}
private AuxiliaryTree(Tree tree, Tree foot, IDictionary <string, Tree> namesToNodes, string originalTreeString)
{
this.originalTreeString = originalTreeString;
this.tree = tree;
this.foot = foot;
this.namesToNodes = namesToNodes;
nodesToNames = null;
}
protected override void printRImpl(Env env,
WriteStream @out,
int depth,
IdentityHashMap <Value, String> valueSet)
{
toLocalValue().printRImpl(env, @out, depth, valueSet);
}
public override void varDumpImpl(Env env,
WriteStream @out,
int depth,
IdentityHashMap <Value, String> valueSet)
{
int length = length();
if (length < 0)
{
length = 0;
}
@out.print("string(");
@out.print(length);
@out.print(") \"");
for (int i = 0; i < length; i++)
{
int ch = charAt(i);
@out.print((char)ch);
}
@out.print("\"");
/*
* int length = length();
*
* if (length < 0)
* length = 0;
*
* @out.print("string");
*
* @out.print("(");
* @out.print(length);
* @out.print(") \"");
*
* for (int i = 0; i < length; i++) {
* char ch = charAt(i);
*
* if (0x20 <= ch && ch <= 0x7f || ch == '\t' || ch == '\r' || ch == '\n')
* @out.print(ch);
* else if (ch <= 0xff)
* @out.print("\\x"
+ Integer.toHexString(ch / 16) + Integer.toHexString(ch % 16));
+ else {
+ @out.print("\\u"
+ Integer.toHexString((ch >> 12) & 0xf)
+ Integer.toHexString((ch >> 8) & 0xf)
+ Integer.toHexString((ch >> 4) & 0xf)
+ Integer.toHexString((ch) & 0xf));
+ }
+ }
+
+ @out.print("\"");
*/
}
public override void varDumpImpl(Env env,
WriteStream @out,
int depth,
IdentityHashMap <Value, String> valueSet)
{
@out.print("&");
getValue().varDump(env, @out, depth, valueSet);
}
public ParseRecord(IList <Word> sentence, Tree goldTree, Tree parse, SimpleMatrix rootVector, IdentityHashMap <Tree, SimpleMatrix> nodeVectors)
{
// TODO: parameter?
this.sentence = sentence;
this.goldTree = goldTree;
this.parse = parse;
this.rootVector = rootVector;
this.nodeVectors = nodeVectors;
}
protected ClassEntry <V> CopyStructure()
{
ClassEntry <V> newClassEntry = new ClassEntry <V>();
LinkedHashMap <Type, Object> newTypeToDefEntryMap = newClassEntry.typeToDefEntryMap;
LinkedHashMap <StrongKey <V>, List <DefEntry <V> > > newDefinitionReverseMap = newClassEntry.definitionReverseMap;
IdentityHashMap <DefEntry <V>, DefEntry <V> > originalToCopyMap = new IdentityHashMap <DefEntry <V>, DefEntry <V> >();
{
foreach (Entry <Type, Object> entry in classEntry.typeToDefEntryMap)
{
Type key = entry.Key;
Object value = entry.Value;
if (Object.ReferenceEquals(value, alreadyHandled))
{
newTypeToDefEntryMap.Put(key, alreadyHandled);
}
else
{
InterfaceFastList <DefEntry <V> > list = (InterfaceFastList <DefEntry <V> >)value;
InterfaceFastList <DefEntry <V> > newList = new InterfaceFastList <DefEntry <V> >();
IListElem <DefEntry <V> > pointer = list.First;
while (pointer != null)
{
DefEntry <V> defEntry = pointer.ElemValue;
DefEntry <V> newDefEntry = new DefEntry <V>(defEntry.extension, defEntry.type, defEntry.distance);
originalToCopyMap.Put(defEntry, newDefEntry);
newList.PushLast(newDefEntry);
pointer = pointer.Next;
}
newTypeToDefEntryMap.Put(key, newList);
}
TypeToDefEntryMapChanged(newClassEntry, key);
}
}
foreach (Entry <StrongKey <V>, List <DefEntry <V> > > entry in classEntry.definitionReverseMap)
{
List <DefEntry <V> > defEntries = entry.Value;
List <DefEntry <V> > newDefEntries = new List <DefEntry <V> >(defEntries.Count);
for (int a = 0, size = defEntries.Count; a < size; a++)
{
DefEntry <V> newDefEntry = originalToCopyMap.Get(defEntries[a]);
if (newDefEntry == null)
{
throw new Exception("Must never happen");
}
newDefEntries.Add(newDefEntry);
}
newDefinitionReverseMap.Put(entry.Key, newDefEntries);
}
return(newClassEntry);
}
public virtual bool RunGradientCheck(IList <Tree> sentences, IdentityHashMap <Tree, byte[]> compressedParses)
{
log.Info("Gradient check: converting " + sentences.Count + " compressed trees");
IdentityHashMap <Tree, IList <Tree> > topParses = CacheParseHypotheses.ConvertToTrees(sentences, compressedParses, op.trainOptions.trainingThreads);
log.Info("Done converting trees");
DVParserCostAndGradient gcFunc = new DVParserCostAndGradient(sentences, topParses, dvModel, op);
return(gcFunc.GradientCheck(1000, 50, dvModel.ParamsToVector()));
}
internal TregexMatcher(Tree root, Tree tree, IdentityHashMap <Tree, Tree> nodesToParents, IDictionary <string, Tree> namesToNodes, VariableStrings variableStrings, IHeadFinder headFinder)
{
// these things are used by "find"
this.root = root;
this.tree = tree;
this.nodesToParents = nodesToParents;
this.namesToNodes = namesToNodes;
this.variableStrings = variableStrings;
this.headFinder = headFinder;
}
public virtual IdentityHashMap <Tree, byte[]> ConvertToBytes(IdentityHashMap <Tree, IList <Tree> > uncompressed)
{
IdentityHashMap <Tree, byte[]> compressed = Generics.NewIdentityHashMap();
foreach (KeyValuePair <Tree, IList <Tree> > entry in uncompressed)
{
compressed[entry.Key] = ConvertToBytes(entry.Value);
}
return(compressed);
}
protected ICacheWalkerResult WalkIntern(IObjRef[] objRefs, IdentityHashSet <ICache> allCachesSet)
{
IdentityHashMap <ICache, List <ICache> > cacheToChildCaches = new IdentityHashMap <ICache, List <ICache> >();
IdentityHashMap <ICache, ICache> cacheToProxyCache = new IdentityHashMap <ICache, ICache>();
ICache currentCommittedRootCache = CommittedRootCache.CurrentCache;
if (!CacheProvider.IsNewInstanceOnCall)
{
allCachesSet.Add(FirstLevelCache.CurrentCache);
}
ICache[] allChildCaches = allCachesSet.ToArray();
allCachesSet.Add(currentCommittedRootCache);
foreach (ICache childCache in allChildCaches)
{
ICache child = childCache;
ICache parent = ((ChildCache)child).Parent;
while (parent != null)
{
ICache currentParent = parent.CurrentCache;
if (!allCachesSet.Add(currentParent))
{
// skip this cache. we handled it already
break;
}
CheckParentCache(parent, currentParent, child, cacheToChildCaches, cacheToProxyCache);
parent = ((IRootCache)currentParent).Parent;
child = currentParent;
}
CheckParentCache(CommittedRootCache, currentCommittedRootCache, child, cacheToChildCaches, cacheToProxyCache);
}
if (objRefs != allEntityRefs)
{
objRefs = new CHashSet <IObjRef>(objRefs).ToArray();
Array.Sort(objRefs, ObjRef.comparator);
}
CacheWalkerResult rootEntry = BuildWalkedEntry(currentCommittedRootCache, objRefs, cacheToChildCaches, cacheToProxyCache);
if (objRefs == allEntityRefs)
{
HashMap <IObjRef, int?> allObjRefs = new HashMap <IObjRef, int?>();
CollectAllObjRefs(rootEntry, allObjRefs);
objRefs = allObjRefs.Count > 0 ? ListUtil.ToArray(allObjRefs.KeyList()) : ObjRef.EMPTY_ARRAY;
Array.Sort(objRefs, ObjRef.comparator);
for (int a = objRefs.Length; a-- > 0;)
{
allObjRefs.Put(objRefs[a], a);
}
ReallocateObjRefsAndCacheValues(rootEntry, objRefs, allObjRefs);
}
return(rootEntry);
}
/// <summary>
/// <inheritDoc/>
/// This annotator will, in particular, set the
/// <see cref="EntailedSentencesAnnotation"/>
/// and
/// <see cref="RelationTriplesAnnotation"/>
/// annotations.
/// </summary>
public virtual void Annotate(Annotation annotation)
{
// Accumulate Coref data
IDictionary <int, CorefChain> corefChains;
IDictionary <CoreLabel, IList <CoreLabel> > canonicalMentionMap = new IdentityHashMap <CoreLabel, IList <CoreLabel> >();
if (resolveCoref && (corefChains = annotation.Get(typeof(CorefCoreAnnotations.CorefChainAnnotation))) != null)
{
foreach (CorefChain chain in corefChains.Values)
{
// Make sure it's a real chain and not a singleton
if (chain.GetMentionsInTextualOrder().Count < 2)
{
continue;
}
// Metadata
IList <CoreLabel> canonicalMention = null;
double canonicalMentionScore = double.NegativeInfinity;
ICollection <CoreLabel> tokensToMark = new HashSet <CoreLabel>();
IList <CorefChain.CorefMention> mentions = chain.GetMentionsInTextualOrder();
// Iterate over mentions
for (int i = 0; i < mentions.Count; ++i)
{
// Get some data on this mention
Pair <IList <CoreLabel>, double> info = GrokCorefMention(annotation, mentions[i]);
// Figure out if it should be the canonical mention
double score = info.second + ((double)i) / ((double)mentions.Count) + (mentions[i] == chain.GetRepresentativeMention() ? 1.0 : 0.0);
if (canonicalMention == null || score > canonicalMentionScore)
{
canonicalMention = info.first;
canonicalMentionScore = score;
}
// Register the participating tokens
if (info.first.Count == 1)
{
// Only mark single-node tokens!
Sharpen.Collections.AddAll(tokensToMark, info.first);
}
}
// Mark the tokens as coreferent
System.Diagnostics.Debug.Assert(canonicalMention != null);
foreach (CoreLabel token in tokensToMark)
{
IList <CoreLabel> existingMention = canonicalMentionMap[token];
if (existingMention == null || existingMention.IsEmpty() || "O".Equals(existingMention[0].Ner()))
{
// Don't clobber existing good mentions
canonicalMentionMap[token] = canonicalMention;
}
}
}
}
// Annotate each sentence
annotation.Get(typeof(CoreAnnotations.SentencesAnnotation)).ForEach(null);
}
public static Tree FindRootTree(IdentityHashMap <Tree, SimpleMatrix> vectors)
{
foreach (Tree tree in vectors.Keys)
{
if (tree.Label().Value().Equals("ROOT"))
{
return(tree);
}
}
throw new Exception("Could not find root");
}
public override void varDumpImpl(Env env,
WriteStream @out,
int depth,
IdentityHashMap <Value, String> valueSet)
{
@out.print(getClass().getName());
@out.print('[');
@out.print(_methodName);
@out.print(']');
}
// protected Collection inverseImages(Collection block, Object symbol) {
// List inverseImages = new ArrayList();
// for (Iterator nodeI = block.iterator(); nodeI.hasNext();) {
// Object node = nodeI.next();
// inverseImages.addAll(getUnminimizedFA().getInboundArcs(node, symbol));
// }
// return inverseImages;
// }
protected internal virtual IDictionary SortIntoBlocks(ICollection nodes)
{
IDictionary blockToMembers = new IdentityHashMap();
foreach (object o in nodes)
{
FastExactAutomatonMinimizer.Block block = GetBlock(o);
Maps.PutIntoValueHashSet(blockToMembers, block, o);
}
return(blockToMembers);
}
/// <summary>
/// tests reuse with Pulsing1(Pulsing2(Standard)) </summary>
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: public void testNestedPulsing() throws Exception
public virtual void testNestedPulsing()
{
// we always run this test with pulsing codec.
Codec cp = TestUtil.alwaysPostingsFormat(new NestedPulsingPostingsFormat());
BaseDirectoryWrapper dir = newDirectory();
RandomIndexWriter iw = new RandomIndexWriter(random(), dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).setCodec(cp));
Document doc = new Document();
doc.add(new TextField("foo", "a b b c c c d e f g g g h i i j j k l l m m m", Field.Store.NO));
// note: the reuse is imperfect, here we would have 4 enums (lost reuse when we get an enum for 'm')
// this is because we only track the 'last' enum we reused (not all).
// but this seems 'good enough' for now.
iw.addDocument(doc);
DirectoryReader ir = iw.Reader;
iw.close();
AtomicReader segment = getOnlySegmentReader(ir);
DocsEnum reuse = null;
IDictionary<DocsEnum, bool?> allEnums = new IdentityHashMap<DocsEnum, bool?>();
TermsEnum te = segment.terms("foo").iterator(null);
while (te.next() != null)
{
reuse = te.docs(null, reuse, DocsEnum.FLAG_NONE);
allEnums[reuse] = true;
}
assertEquals(4, allEnums.Count);
allEnums.Clear();
DocsAndPositionsEnum posReuse = null;
te = segment.terms("foo").iterator(null);
while (te.next() != null)
{
posReuse = te.docsAndPositions(null, posReuse);
allEnums[posReuse] = true;
}
assertEquals(4, allEnums.Count);
ir.close();
dir.close();
}
public virtual void TestSophisticatedReuse()
{
// we always run this test with pulsing codec.
Codec cp = TestUtil.AlwaysPostingsFormat(new Pulsing41PostingsFormat(1));
Directory dir = NewDirectory();
RandomIndexWriter iw = new RandomIndexWriter(Random(), dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetCodec(cp));
Document doc = new Document();
doc.Add(new TextField("foo", "a b b c c c d e f g g h i i j j k", Field.Store.NO));
iw.AddDocument(doc);
DirectoryReader ir = iw.Reader;
iw.Dispose();
AtomicReader segment = GetOnlySegmentReader(ir);
DocsEnum reuse = null;
IDictionary<DocsEnum, bool?> allEnums = new IdentityHashMap<DocsEnum, bool?>();
TermsEnum te = segment.Terms("foo").Iterator(null);
while (te.Next() != null)
{
reuse = te.Docs(null, reuse, DocsEnum.FLAG_NONE);
allEnums[reuse] = true;
}
assertEquals(2, allEnums.Count);
allEnums.Clear();
DocsAndPositionsEnum posReuse = null;
te = segment.Terms("foo").Iterator(null);
while (te.Next() != null)
{
posReuse = te.DocsAndPositions(null, posReuse);
allEnums[posReuse] = true;
}
assertEquals(2, allEnums.Count);
ir.Dispose();
dir.Dispose();
}
/// <summary>
/// Internal recursive traversal for conversion.
/// </summary>
private static Util.Automaton.State Convert(State s, IdentityHashMap<State, Lucene.Net.Util.Automaton.State> visited)
{
Util.Automaton.State converted = visited[s];
if (converted != null)
{
return converted;
}
converted = new Util.Automaton.State();
converted.Accept = s.Is_final;
visited[s] = converted;
int i = 0;
int[] labels = s.Labels;
foreach (DaciukMihovAutomatonBuilder.State target in s.States)
{
converted.AddTransition(new Transition(labels[i++], Convert(target, visited)));
}
return converted;
}
public virtual void TestReuseDocsEnumDifferentReader()
{
Directory dir = NewDirectory();
Codec cp = TestUtil.AlwaysPostingsFormat(new Lucene40RWPostingsFormat());
MockAnalyzer analyzer = new MockAnalyzer(Random());
analyzer.MaxTokenLength = TestUtil.NextInt(Random(), 1, IndexWriter.MAX_TERM_LENGTH);
RandomIndexWriter writer = new RandomIndexWriter(Random(), dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer).SetCodec(cp));
int numdocs = AtLeast(20);
CreateRandomIndex(numdocs, writer, Random());
writer.Commit();
DirectoryReader firstReader = DirectoryReader.Open(dir);
DirectoryReader secondReader = DirectoryReader.Open(dir);
IList<AtomicReaderContext> leaves = firstReader.Leaves;
IList<AtomicReaderContext> leaves2 = secondReader.Leaves;
foreach (AtomicReaderContext ctx in leaves)
{
Terms terms = ((AtomicReader)ctx.Reader).Terms("body");
TermsEnum iterator = terms.Iterator(null);
IdentityHashMap<DocsEnum, bool?> enums = new IdentityHashMap<DocsEnum, bool?>();
MatchNoBits bits = new MatchNoBits(firstReader.MaxDoc);
iterator = terms.Iterator(null);
DocsEnum docs = null;
BytesRef term = null;
while ((term = iterator.Next()) != null)
{
docs = iterator.Docs(null, RandomDocsEnum("body", term, leaves2, bits), Random().NextBoolean() ? DocsEnum.FLAG_FREQS : DocsEnum.FLAG_NONE);
enums[docs] = true;
}
Assert.AreEqual(terms.Size(), enums.Count);
iterator = terms.Iterator(null);
enums.Clear();
docs = null;
while ((term = iterator.Next()) != null)
{
docs = iterator.Docs(bits, RandomDocsEnum("body", term, leaves2, bits), Random().NextBoolean() ? DocsEnum.FLAG_FREQS : DocsEnum.FLAG_NONE);
enums[docs] = true;
}
Assert.AreEqual(terms.Size(), enums.Count);
}
IOUtils.Close(writer, firstReader, secondReader, dir);
}
