/// <summary>
/// Gets the fees where the session, term and the class is known
/// </summary>
/// <param name="session">session</param>
/// <param name="term">term</param>
/// <param name="classEnum">class</param>
/// <returns>ClassTermFees</returns>
public ClassTermFee GetFees(string session, TermEnum term, ClassEnum classEnum)
{
try
{
var fees =
_classTermFees.FirstOrDefault(x => x.Session == session && x.TermEnum == term && x.ClassEnum == classEnum);
return fees ?? new ClassTermFee();
}
catch (Exception ex)
{
_log.Error("Error", ex);
return null;
}
}
public static TermInfo[] GetHighFreqTerms(Directory dir,
Hashtable junkWords,
int numTerms,
String[] fields)
{
if (dir == null || fields == null)
{
return(new TermInfo[0]);
}
IndexReader reader = IndexReader.Open(dir, true);
TermInfoQueue tiq = new TermInfoQueue(numTerms);
TermEnum terms = reader.Terms();
int minFreq = 0;
while (terms.Next())
{
String field = terms.Term().Field();
if (fields != null && fields.Length > 0)
{
bool skip = true;
for (int i = 0; i < fields.Length; i++)
{
if (field.Equals(fields[i]))
{
skip = false;
break;
}
}
if (skip)
{
continue;
}
}
if (junkWords != null && junkWords[terms.Term().Text()] != null)
{
continue;
}
if (terms.DocFreq() > minFreq)
{
TermInfo top = (TermInfo)tiq.Add(new TermInfo(terms.Term(), terms.DocFreq()));
if (tiq.Size() >= numTerms) // if tiq overfull
{
tiq.Pop(); // remove lowest in tiq
minFreq = top.DocFreq; // reset minFreq
}
}
}
TermInfo[] res = new TermInfo[tiq.Size()];
for (int i = 0; i < res.Length; i++)
{
res[res.Length - i - 1] = (TermInfo)tiq.Pop();
}
reader.Close();
return(res);
}
public void Seek(TermEnum termEnum)
{
if (DEBUG) System.Diagnostics.Debug.WriteLine(".seekEnum");
Seek(termEnum.Term);
}
public TermsEnumCompatibility(IndexReader reader, String fieldName)
{
this.reader = reader;
this.fieldName = string.Intern(fieldName);
this.termEnum = reader.Terms(new Term(this.fieldName));
}
/// <summary>this is a dummy, it is not used by this class. </summary>
protected internal override void SetEnum(TermEnum tenum)
{
throw new NotSupportedException("not implemented");
}
public virtual void TestKnownSetOfDocuments()
{
System.String test1 = "eating chocolate in a computer lab"; //6 terms
System.String test2 = "computer in a computer lab"; //5 terms
System.String test3 = "a chocolate lab grows old"; //5 terms
System.String test4 = "eating chocolate with a chocolate lab in an old chocolate colored computer lab"; //13 terms
System.Collections.IDictionary test4Map = new System.Collections.Hashtable();
test4Map["chocolate"] = 3;
test4Map["lab"] = 2;
test4Map["eating"] = 1;
test4Map["computer"] = 1;
test4Map["with"] = 1;
test4Map["a"] = 1;
test4Map["colored"] = 1;
test4Map["in"] = 1;
test4Map["an"] = 1;
test4Map["computer"] = 1;
test4Map["old"] = 1;
Document testDoc1 = new Document();
SetupDoc(testDoc1, test1);
Document testDoc2 = new Document();
SetupDoc(testDoc2, test2);
Document testDoc3 = new Document();
SetupDoc(testDoc3, test3);
Document testDoc4 = new Document();
SetupDoc(testDoc4, test4);
Directory dir = new MockRAMDirectory();
try
{
IndexWriter writer = new IndexWriter(dir, new SimpleAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
Assert.IsTrue(writer != null);
writer.AddDocument(testDoc1);
writer.AddDocument(testDoc2);
writer.AddDocument(testDoc3);
writer.AddDocument(testDoc4);
writer.Close();
IndexSearcher knownSearcher = new IndexSearcher(dir);
TermEnum termEnum = knownSearcher.reader_ForNUnit.Terms();
TermDocs termDocs = knownSearcher.reader_ForNUnit.TermDocs();
//System.out.println("Terms: " + termEnum.size() + " Orig Len: " + termArray.length);
Similarity sim = knownSearcher.GetSimilarity();
while (termEnum.Next() == true)
{
Term term = termEnum.Term();
//System.out.println("Term: " + term);
termDocs.Seek(term);
while (termDocs.Next())
{
int docId = termDocs.Doc();
int freq = termDocs.Freq();
//System.out.println("Doc Id: " + docId + " freq " + freq);
TermFreqVector vector = knownSearcher.reader_ForNUnit.GetTermFreqVector(docId, "field");
float tf = sim.Tf(freq);
float idf = sim.Idf(term, knownSearcher);
//float qNorm = sim.queryNorm()
//This is fine since we don't have stop words
float lNorm = sim.LengthNorm("field", vector.GetTerms().Length);
//float coord = sim.coord()
//System.out.println("TF: " + tf + " IDF: " + idf + " LenNorm: " + lNorm);
Assert.IsTrue(vector != null);
System.String[] vTerms = vector.GetTerms();
int[] freqs = vector.GetTermFrequencies();
for (int i = 0; i < vTerms.Length; i++)
{
if (term.Text().Equals(vTerms[i]))
{
Assert.IsTrue(freqs[i] == freq);
}
}
}
//System.out.println("--------");
}
Query query = new TermQuery(new Term("field", "chocolate"));
ScoreDoc[] hits = knownSearcher.Search(query, null, 1000).scoreDocs;
//doc 3 should be the first hit b/c it is the shortest match
Assert.IsTrue(hits.Length == 3);
float score = hits[0].score;
/*System.out.println("Hit 0: " + hits.id(0) + " Score: " + hits.score(0) + " String: " + hits.doc(0).toString());
* System.out.println("Explain: " + knownSearcher.explain(query, hits.id(0)));
* System.out.println("Hit 1: " + hits.id(1) + " Score: " + hits.score(1) + " String: " + hits.doc(1).toString());
* System.out.println("Explain: " + knownSearcher.explain(query, hits.id(1)));
* System.out.println("Hit 2: " + hits.id(2) + " Score: " + hits.score(2) + " String: " + hits.doc(2).toString());
* System.out.println("Explain: " + knownSearcher.explain(query, hits.id(2)));*/
Assert.IsTrue(hits[0].doc == 2);
Assert.IsTrue(hits[1].doc == 3);
Assert.IsTrue(hits[2].doc == 0);
TermFreqVector vector2 = knownSearcher.reader_ForNUnit.GetTermFreqVector(hits[1].doc, "field");
Assert.IsTrue(vector2 != null);
//System.out.println("Vector: " + vector);
System.String[] terms = vector2.GetTerms();
int[] freqs2 = vector2.GetTermFrequencies();
Assert.IsTrue(terms != null && terms.Length == 10);
for (int i = 0; i < terms.Length; i++)
{
System.String term = terms[i];
//System.out.println("Term: " + term);
int freq = freqs2[i];
Assert.IsTrue(test4.IndexOf(term) != -1);
System.Int32 freqInt = -1;
try
{
freqInt = (System.Int32)test4Map[term];
}
catch (Exception)
{
Assert.IsTrue(false);
}
Assert.IsTrue(freqInt == freq);
}
SortedTermVectorMapper mapper = new SortedTermVectorMapper(new TermVectorEntryFreqSortedComparator());
knownSearcher.reader_ForNUnit.GetTermFreqVector(hits[1].doc, mapper);
System.Collections.Generic.SortedDictionary <object, object> vectorEntrySet = mapper.GetTermVectorEntrySet();
Assert.IsTrue(vectorEntrySet.Count == 10, "mapper.getTermVectorEntrySet() Size: " + vectorEntrySet.Count + " is not: " + 10);
TermVectorEntry last = null;
foreach (TermVectorEntry tve in vectorEntrySet.Keys)
{
if (tve != null && last != null)
{
Assert.IsTrue(last.GetFrequency() >= tve.GetFrequency(), "terms are not properly sorted");
System.Int32 expectedFreq = (System.Int32)test4Map[tve.GetTerm()];
//we expect double the expectedFreq, since there are two fields with the exact same text and we are collapsing all fields
Assert.IsTrue(tve.GetFrequency() == 2 * expectedFreq, "Frequency is not correct:");
}
last = tve;
}
FieldSortedTermVectorMapper fieldMapper = new FieldSortedTermVectorMapper(new TermVectorEntryFreqSortedComparator());
knownSearcher.reader_ForNUnit.GetTermFreqVector(hits[1].doc, fieldMapper);
System.Collections.IDictionary map = fieldMapper.GetFieldToTerms();
Assert.IsTrue(map.Count == 2, "map Size: " + map.Count + " is not: " + 2);
vectorEntrySet = (System.Collections.Generic.SortedDictionary <Object, Object>)map["field"];
Assert.IsTrue(vectorEntrySet != null, "vectorEntrySet is null and it shouldn't be");
Assert.IsTrue(vectorEntrySet.Count == 10, "vectorEntrySet Size: " + vectorEntrySet.Count + " is not: " + 10);
knownSearcher.Close();
}
catch (System.IO.IOException e)
{
System.Console.Error.WriteLine(e.StackTrace);
Assert.IsTrue(false);
}
}
protected internal override System.Object CreateValue(IndexReader reader, Entry entryKey)
{
System.String field = StringHelper.Intern(entryKey.field);
int[] retArray = new int[reader.MaxDoc];
System.String[] mterms = new System.String[reader.MaxDoc + 1];
TermDocs termDocs = reader.TermDocs();
TermEnum termEnum = reader.Terms(new Term(field));
int t = 0; // current term number
// an entry for documents that have no terms in this field
// should a document with no terms be at top or bottom?
// this puts them at the top - if it is changed, FieldDocSortedHitQueue
// needs to change as well.
mterms[t++] = null;
try
{
do
{
Term term = termEnum.Term;
if (term == null || term.Field != field || t >= mterms.Length)
{
break;
}
// store term text
mterms[t] = term.Text;
termDocs.Seek(termEnum);
while (termDocs.Next())
{
retArray[termDocs.Doc] = t;
}
t++;
}while (termEnum.Next());
}
finally
{
termDocs.Close();
termEnum.Close();
}
if (t == 0)
{
// if there are no terms, make the term array
// have a single null entry
mterms = new System.String[1];
}
else if (t < mterms.Length)
{
// if there are less terms than documents,
// trim off the dead array space
System.String[] terms = new System.String[t];
Array.Copy(mterms, 0, terms, 0, t);
mterms = terms;
}
StringIndex value_Renamed = new StringIndex(retArray, mterms);
return(value_Renamed);
}
private void AddTerms(IndexReader reader, FieldVals f)
{
if (f.queryString == null)
{
return;
}
TokenStream ts = analyzer.TokenStream(f.fieldName, new System.IO.StringReader(f.queryString));
ITermAttribute termAtt = ts.AddAttribute <ITermAttribute>();
int corpusNumDocs = reader.NumDocs();
Term internSavingTemplateTerm = new Term(f.fieldName); //optimization to avoid constructing new Term() objects
HashSet <string> processedTerms = new HashSet <string>();
while (ts.IncrementToken())
{
String term = termAtt.Term;
if (!processedTerms.Contains(term))
{
processedTerms.Add(term);
ScoreTermQueue variantsQ = new ScoreTermQueue(MAX_VARIANTS_PER_TERM); //maxNum variants considered for any one term
float minScore = 0;
Term startTerm = internSavingTemplateTerm.CreateTerm(term);
FuzzyTermEnum fe = new FuzzyTermEnum(reader, startTerm, f.minSimilarity, f.prefixLength);
TermEnum origEnum = reader.Terms(startTerm);
int df = 0;
if (startTerm.Equals(origEnum.Term))
{
df = origEnum.DocFreq(); //store the df so all variants use same idf
}
int numVariants = 0;
int totalVariantDocFreqs = 0;
do
{
Term possibleMatch = fe.Term;
if (possibleMatch != null)
{
numVariants++;
totalVariantDocFreqs += fe.DocFreq();
float score = fe.Difference();
if (variantsQ.Size() < MAX_VARIANTS_PER_TERM || score > minScore)
{
ScoreTerm st = new ScoreTerm(possibleMatch, score, startTerm);
variantsQ.InsertWithOverflow(st);
minScore = variantsQ.Top().Score; // maintain minScore
}
}
}while (fe.Next());
if (numVariants > 0)
{
int avgDf = totalVariantDocFreqs / numVariants;
if (df == 0) //no direct match we can use as df for all variants
{
df = avgDf; //use avg df of all variants
}
// take the top variants (scored by edit distance) and reset the score
// to include an IDF factor then add to the global queue for ranking
// overall top query terms
int size = variantsQ.Size();
for (int i = 0; i < size; i++)
{
ScoreTerm st = variantsQ.Pop();
st.Score = (st.Score * st.Score) * sim.Idf(df, corpusNumDocs);
q.InsertWithOverflow(st);
}
}
}
}
}
