internal SortingDocsEnum(int maxDoc, SortingDocsEnum reuse, DocsEnum input, bool withFreqs, Sorter.DocMap docMap)
: base(input)
{
this.maxDoc = maxDoc;
this.withFreqs = withFreqs;
if (reuse != null)
{
if (reuse.maxDoc == maxDoc)
{
sorter = reuse.sorter;
}
else
{
sorter = new DocFreqSorter(maxDoc);
}
docs = reuse.docs;
freqs = reuse.freqs; // maybe null
}
else
{
docs = new int[64];
sorter = new DocFreqSorter(maxDoc);
}
docIt = -1;
int i = 0;
int doc;
if (withFreqs)
{
if (freqs == null || freqs.Length < docs.Length)
{
freqs = new int[docs.Length];
}
while ((doc = input.NextDoc()) != DocIdSetIterator.NO_MORE_DOCS)
{
if (i >= docs.Length)
{
docs = ArrayUtil.Grow(docs, docs.Length + 1);
freqs = ArrayUtil.Grow(freqs, freqs.Length + 1);
}
docs[i] = docMap.OldToNew(doc);
freqs[i] = input.Freq;
++i;
}
}
else
{
freqs = null;
while ((doc = input.NextDoc()) != DocIdSetIterator.NO_MORE_DOCS)
{
if (i >= docs.Length)
{
docs = ArrayUtil.Grow(docs, docs.Length + 1);
}
docs[i++] = docMap.OldToNew(doc);
}
}
// TimSort can save much time compared to other sorts in case of
// reverse sorting, or when sorting a concatenation of sorted readers
sorter.Reset(docs, freqs);
sorter.Sort(0, i);
upto = i;
}
/// <summary>
/// Classify the specified input into a group.
/// </summary>
/// <param name="input">The input data.</param>
/// <returns>The group the data was classified into.</returns>
public int ComputeClassification(double[] input)
{
return(ArrayUtil.IndexOfLargest(ComputeRegression(input)));
}
public override Scorer GetScorer(AtomicReaderContext context, IBits acceptDocs)
{
Debug.Assert(outerInstance.termArrays.Count > 0);
AtomicReader reader = (context.AtomicReader);
IBits liveDocs = acceptDocs;
PhraseQuery.PostingsAndFreq[] postingsFreqs = new PhraseQuery.PostingsAndFreq[outerInstance.termArrays.Count];
Terms fieldTerms = reader.GetTerms(outerInstance.field);
if (fieldTerms == null)
{
return(null);
}
// Reuse single TermsEnum below:
TermsEnum termsEnum = fieldTerms.GetIterator(null);
for (int pos = 0; pos < postingsFreqs.Length; pos++)
{
Term[] terms = outerInstance.termArrays[pos];
DocsAndPositionsEnum postingsEnum;
int docFreq;
if (terms.Length > 1)
{
postingsEnum = new UnionDocsAndPositionsEnum(liveDocs, context, terms, termContexts, termsEnum);
// coarse -- this overcounts since a given doc can
// have more than one term:
docFreq = 0;
for (int termIdx = 0; termIdx < terms.Length; termIdx++)
{
Term term = terms[termIdx];
TermState termState = termContexts[term].Get(context.Ord);
if (termState == null)
{
// Term not in reader
continue;
}
termsEnum.SeekExact(term.Bytes, termState);
docFreq += termsEnum.DocFreq;
}
if (docFreq == 0)
{
// None of the terms are in this reader
return(null);
}
}
else
{
Term term = terms[0];
TermState termState = termContexts[term].Get(context.Ord);
if (termState == null)
{
// Term not in reader
return(null);
}
termsEnum.SeekExact(term.Bytes, termState);
postingsEnum = termsEnum.DocsAndPositions(liveDocs, null, DocsAndPositionsFlags.NONE);
if (postingsEnum == null)
{
// term does exist, but has no positions
Debug.Assert(termsEnum.Docs(liveDocs, null, DocsFlags.NONE) != null, "termstate found but no term exists in reader");
throw new InvalidOperationException("field \"" + term.Field + "\" was indexed without position data; cannot run PhraseQuery (term=" + term.Text() + ")");
}
docFreq = termsEnum.DocFreq;
}
postingsFreqs[pos] = new PhraseQuery.PostingsAndFreq(postingsEnum, docFreq, (int)outerInstance.positions[pos], terms);
}
// sort by increasing docFreq order
if (outerInstance.slop == 0)
{
ArrayUtil.TimSort(postingsFreqs);
}
if (outerInstance.slop == 0)
{
ExactPhraseScorer s = new ExactPhraseScorer(this, postingsFreqs, similarity.GetSimScorer(stats, context));
if (s.noDocs)
{
return(null);
}
else
{
return(s);
}
}
else
{
return(new SloppyPhraseScorer(this, postingsFreqs, outerInstance.slop, similarity.GetSimScorer(stats, context)));
}
}
private TermsFilter(FieldAndTermEnum iter, int length)
{
// TODO: maybe use oal.index.PrefixCodedTerms instead?
// If number of terms is more than a few hundred it
// should be a win
// TODO: we also pack terms in FieldCache/DocValues
// ... maybe we can refactor to share that code
// TODO: yet another option is to build the union of the terms in
// an automaton an call intersect on the termsenum if the density is high
int hash = 9;
var serializedTerms = Arrays.Empty <byte>();
this.offsets = new int[length + 1];
int lastEndOffset = 0;
int index = 0;
var termsAndFields = new List <TermsAndField>();
TermsAndField lastTermsAndField = null;
BytesRef previousTerm = null;
string previousField = null;
BytesRef currentTerm;
string currentField;
while (iter.MoveNext())
{
currentTerm = iter.Current;
currentField = iter.Field;
if (currentField == null)
{
throw new ArgumentException("Field must not be null");
}
if (previousField != null)
{
// deduplicate
if (previousField.Equals(currentField, StringComparison.Ordinal))
{
if (previousTerm.BytesEquals(currentTerm))
{
continue;
}
}
else
{
int _start = lastTermsAndField == null ? 0 : lastTermsAndField.end;
lastTermsAndField = new TermsAndField(_start, index, previousField);
termsAndFields.Add(lastTermsAndField);
}
}
hash = PRIME * hash + currentField.GetHashCode();
hash = PRIME * hash + currentTerm.GetHashCode();
if (serializedTerms.Length < lastEndOffset + currentTerm.Length)
{
serializedTerms = ArrayUtil.Grow(serializedTerms, lastEndOffset + currentTerm.Length);
}
Array.Copy(currentTerm.Bytes, currentTerm.Offset, serializedTerms, lastEndOffset, currentTerm.Length);
offsets[index] = lastEndOffset;
lastEndOffset += currentTerm.Length;
index++;
previousTerm = currentTerm;
previousField = currentField;
}
offsets[index] = lastEndOffset;
int start = lastTermsAndField == null ? 0 : lastTermsAndField.end;
lastTermsAndField = new TermsAndField(start, index, previousField);
termsAndFields.Add(lastTermsAndField);
this.termsBytes = ArrayUtil.Shrink(serializedTerms, lastEndOffset);
this.termsAndFields = termsAndFields.ToArray();
this.hashCode = hash;
}
internal readonly bool isSegmentPrivate; // set to true iff this frozen packet represents
// a segment private deletes. in that case is should
// only have Queries
public FrozenBufferedUpdates(BufferedUpdates deletes, bool isSegmentPrivate)
{
this.isSegmentPrivate = isSegmentPrivate;
Debug.Assert(!isSegmentPrivate || deletes.terms.Count == 0, "segment private package should only have del queries");
Term[] termsArray = deletes.terms.Keys.ToArray(/*new Term[deletes.Terms.Count]*/);
termCount = termsArray.Length;
ArrayUtil.TimSort(termsArray);
PrefixCodedTerms.Builder builder = new PrefixCodedTerms.Builder();
foreach (Term term in termsArray)
{
builder.Add(term);
}
terms = builder.Finish();
queries = new Query[deletes.queries.Count];
queryLimits = new int[deletes.queries.Count];
int upto = 0;
foreach (KeyValuePair <Query, int?> ent in deletes.queries)
{
queries[upto] = ent.Key;
if (ent.Value.HasValue)
{
queryLimits[upto] = ent.Value.Value;
}
else
{
// LUCENENET NOTE: According to this: http://stackoverflow.com/a/13914344
// we are supposed to throw an exception in this case, rather than
// silently fail.
throw new NullReferenceException();
}
upto++;
}
// TODO if a Term affects multiple fields, we could keep the updates key'd by Term
// so that it maps to all fields it affects, sorted by their docUpto, and traverse
// that Term only once, applying the update to all fields that still need to be
// updated.
IList <NumericDocValuesUpdate> allNumericUpdates = new List <NumericDocValuesUpdate>();
int numericUpdatesSize = 0;
foreach (var numericUpdates in deletes.numericUpdates.Values)
{
foreach (NumericDocValuesUpdate update in numericUpdates.Values)
{
allNumericUpdates.Add(update);
numericUpdatesSize += update.GetSizeInBytes();
}
}
numericDVUpdates = allNumericUpdates.ToArray();
// TODO if a Term affects multiple fields, we could keep the updates key'd by Term
// so that it maps to all fields it affects, sorted by their docUpto, and traverse
// that Term only once, applying the update to all fields that still need to be
// updated.
IList <BinaryDocValuesUpdate> allBinaryUpdates = new List <BinaryDocValuesUpdate>();
int binaryUpdatesSize = 0;
foreach (var binaryUpdates in deletes.binaryUpdates.Values)
{
foreach (BinaryDocValuesUpdate update in binaryUpdates.Values)
{
allBinaryUpdates.Add(update);
binaryUpdatesSize += update.GetSizeInBytes();
}
}
binaryDVUpdates = allBinaryUpdates.ToArray();
bytesUsed = (int)terms.GetSizeInBytes() + queries.Length * BYTES_PER_DEL_QUERY + numericUpdatesSize + numericDVUpdates.Length * RamUsageEstimator.NUM_BYTES_OBJECT_REF + binaryUpdatesSize + binaryDVUpdates.Length * RamUsageEstimator.NUM_BYTES_OBJECT_REF;
numTermDeletes = deletes.numTermDeletes;
}
/// <summary>
/// Add a new position & payload </summary>
public override void AddPosition(int position, BytesRef payload, int startOffset, int endOffset)
{
// if (DEBUG) {
// System.out.println("FPW.addPosition pos=" + position + " posBufferUpto=" + posBufferUpto + (fieldHasPayloads ? " payloadByteUpto=" + payloadByteUpto: ""));
// }
posDeltaBuffer[posBufferUpto] = position - lastPosition;
if (fieldHasPayloads)
{
if (payload == null || payload.Length == 0)
{
// no payload
payloadLengthBuffer[posBufferUpto] = 0;
}
else
{
payloadLengthBuffer[posBufferUpto] = payload.Length;
if (payloadByteUpto + payload.Length > payloadBytes.Length)
{
payloadBytes = ArrayUtil.Grow(payloadBytes, payloadByteUpto + payload.Length);
}
Array.Copy(payload.Bytes, payload.Offset, payloadBytes, payloadByteUpto, payload.Length);
payloadByteUpto += payload.Length;
}
}
if (fieldHasOffsets)
{
if (Debugging.AssertsEnabled)
{
Debugging.Assert(startOffset >= lastStartOffset);
Debugging.Assert(endOffset >= startOffset);
}
offsetStartDeltaBuffer[posBufferUpto] = startOffset - lastStartOffset;
offsetLengthBuffer[posBufferUpto] = endOffset - startOffset;
lastStartOffset = startOffset;
}
posBufferUpto++;
lastPosition = position;
if (posBufferUpto == Lucene41PostingsFormat.BLOCK_SIZE)
{
// if (DEBUG) {
// System.out.println(" write pos bulk block @ fp=" + posOut.getFilePointer());
// }
forUtil.WriteBlock(posDeltaBuffer, encoded, posOut);
if (fieldHasPayloads)
{
forUtil.WriteBlock(payloadLengthBuffer, encoded, payOut);
payOut.WriteVInt32(payloadByteUpto);
payOut.WriteBytes(payloadBytes, 0, payloadByteUpto);
payloadByteUpto = 0;
}
if (fieldHasOffsets)
{
forUtil.WriteBlock(offsetStartDeltaBuffer, encoded, payOut);
forUtil.WriteBlock(offsetLengthBuffer, encoded, payOut);
}
posBufferUpto = 0;
}
}
public override int Merge(MergeState mergeState)
{
int docCount = 0;
int idx = 0;
foreach (AtomicReader reader in mergeState.Readers)
{
SegmentReader matchingSegmentReader = mergeState.MatchingSegmentReaders[idx++];
CompressingStoredFieldsReader matchingFieldsReader = null;
if (matchingSegmentReader != null)
{
StoredFieldsReader fieldsReader = matchingSegmentReader.FieldsReader;
// we can only bulk-copy if the matching reader is also a CompressingStoredFieldsReader
if (fieldsReader != null && fieldsReader is CompressingStoredFieldsReader)
{
matchingFieldsReader = (CompressingStoredFieldsReader)fieldsReader;
}
}
int maxDoc = reader.MaxDoc;
Bits liveDocs = reader.LiveDocs;
if (matchingFieldsReader == null || matchingFieldsReader.Version != VERSION_CURRENT || matchingFieldsReader.CompressionMode != CompressionMode || matchingFieldsReader.ChunkSize != ChunkSize) // the way data is decompressed depends on the chunk size - means reader version is not the same as the writer version
{
// naive merge...
for (int i = NextLiveDoc(0, liveDocs, maxDoc); i < maxDoc; i = NextLiveDoc(i + 1, liveDocs, maxDoc))
{
Document doc = reader.Document(i);
AddDocument(doc, mergeState.FieldInfos);
++docCount;
mergeState.checkAbort.Work(300);
}
}
else
{
int docID = NextLiveDoc(0, liveDocs, maxDoc);
if (docID < maxDoc)
{
// not all docs were deleted
CompressingStoredFieldsReader.ChunkIterator it = matchingFieldsReader.GetChunkIterator(docID);
int[] startOffsets = new int[0];
do
{
// go to the next chunk that contains docID
it.Next(docID);
// transform lengths into offsets
if (startOffsets.Length < it.ChunkDocs)
{
startOffsets = new int[ArrayUtil.Oversize(it.ChunkDocs, 4)];
}
for (int i = 1; i < it.ChunkDocs; ++i)
{
startOffsets[i] = startOffsets[i - 1] + it.Lengths[i - 1];
}
if (NumBufferedDocs == 0 && startOffsets[it.ChunkDocs - 1] < ChunkSize && startOffsets[it.ChunkDocs - 1] + it.Lengths[it.ChunkDocs - 1] >= ChunkSize && NextDeletedDoc(it.DocBase, liveDocs, it.DocBase + it.ChunkDocs) == it.DocBase + it.ChunkDocs) // no deletion in the chunk - chunk is large enough - chunk is small enough - starting a new chunk
{
Debug.Assert(docID == it.DocBase);
// no need to decompress, just copy data
IndexWriter.WriteIndex(it.ChunkDocs, FieldsStream.FilePointer);
WriteHeader(this.DocBase, it.ChunkDocs, it.NumStoredFields, it.Lengths);
it.CopyCompressedData(FieldsStream);
this.DocBase += it.ChunkDocs;
docID = NextLiveDoc(it.DocBase + it.ChunkDocs, liveDocs, maxDoc);
docCount += it.ChunkDocs;
mergeState.checkAbort.Work(300 * it.ChunkDocs);
}
else
{
// decompress
it.Decompress();
if (startOffsets[it.ChunkDocs - 1] + it.Lengths[it.ChunkDocs - 1] != it.Bytes.Length)
{
throw new CorruptIndexException("Corrupted: expected chunk size=" + startOffsets[it.ChunkDocs - 1] + it.Lengths[it.ChunkDocs - 1] + ", got " + it.Bytes.Length);
}
// copy non-deleted docs
for (; docID < it.DocBase + it.ChunkDocs; docID = NextLiveDoc(docID + 1, liveDocs, maxDoc))
{
int diff = docID - it.DocBase;
StartDocument(it.NumStoredFields[diff]);
BufferedDocs.WriteBytes(it.Bytes.Bytes, it.Bytes.Offset + startOffsets[diff], it.Lengths[diff]);
FinishDocument();
++docCount;
mergeState.checkAbort.Work(300);
}
}
} while (docID < maxDoc);
it.CheckIntegrity();
}
}
}
Finish(mergeState.FieldInfos, docCount);
return(docCount);
}
private void LoadTerms()
{
var posIntOutputs = PositiveInt32Outputs.Singleton;
var outputsInner = new PairOutputs <long?, long?>(posIntOutputs, posIntOutputs);
var outputs = new PairOutputs <long?, PairOutputs <long?, long?> .Pair>(posIntOutputs, outputsInner);
// honestly, wtf kind of generic mess is this.
var b = new Builder <PairOutputs <long?, PairOutputs <long?, long?> .Pair> .Pair>(FST.INPUT_TYPE.BYTE1, outputs);
var input = (IndexInput)_outerInstance._input.Clone();
input.Seek(_termsStart);
var lastTerm = new BytesRef(10);
long lastDocsStart = -1;
int docFreq = 0;
long totalTermFreq = 0;
var visitedDocs = new FixedBitSet(_maxDoc);
var scratchIntsRef = new Int32sRef();
while (true)
{
SimpleTextUtil.ReadLine(input, _scratch);
if (_scratch.Equals(SimpleTextFieldsWriter.END) || StringHelper.StartsWith(_scratch, SimpleTextFieldsWriter.FIELD))
{
if (lastDocsStart != -1)
{
b.Add(Util.ToInt32sRef(lastTerm, scratchIntsRef),
outputs.NewPair(lastDocsStart, outputsInner.NewPair(docFreq, totalTermFreq)));
_sumTotalTermFreq += totalTermFreq;
}
break;
}
if (StringHelper.StartsWith(_scratch, SimpleTextFieldsWriter.DOC))
{
docFreq++;
_sumDocFreq++;
UnicodeUtil.UTF8toUTF16(_scratch.Bytes, _scratch.Offset + SimpleTextFieldsWriter.DOC.Length, _scratch.Length - SimpleTextFieldsWriter.DOC.Length,
_scratchUtf16);
int docId = ArrayUtil.ParseInt32(_scratchUtf16.Chars, 0, _scratchUtf16.Length);
visitedDocs.Set(docId);
}
else if (StringHelper.StartsWith(_scratch, SimpleTextFieldsWriter.FREQ))
{
UnicodeUtil.UTF8toUTF16(_scratch.Bytes, _scratch.Offset + SimpleTextFieldsWriter.FREQ.Length,
_scratch.Length - SimpleTextFieldsWriter.FREQ.Length, _scratchUtf16);
totalTermFreq += ArrayUtil.ParseInt32(_scratchUtf16.Chars, 0, _scratchUtf16.Length);
}
else if (StringHelper.StartsWith(_scratch, SimpleTextFieldsWriter.TERM))
{
if (lastDocsStart != -1)
{
b.Add(Util.ToInt32sRef(lastTerm, scratchIntsRef),
outputs.NewPair(lastDocsStart, outputsInner.NewPair(docFreq, totalTermFreq)));
}
lastDocsStart = input.GetFilePointer();
int len = _scratch.Length - SimpleTextFieldsWriter.TERM.Length;
if (len > lastTerm.Length)
{
lastTerm.Grow(len);
}
Array.Copy(_scratch.Bytes, SimpleTextFieldsWriter.TERM.Length, lastTerm.Bytes, 0, len);
lastTerm.Length = len;
docFreq = 0;
_sumTotalTermFreq += totalTermFreq;
totalTermFreq = 0;
_termCount++;
}
}
_docCount = visitedDocs.Cardinality();
_fst = b.Finish();
}
private IEnumerable <long?> GetMergeSortedSetOrdsEnumerable(AtomicReader[] readers, SortedSetDocValues[] dvs, OrdinalMap map)
{
int readerUpto = -1;
int docIDUpto = 0;
AtomicReader currentReader = null;
IBits currentLiveDocs = null;
var ords = new long[8];
int ordUpto = 0;
int ordLength = 0;
while (true)
{
if (readerUpto == readers.Length)
{
yield break;
}
if (ordUpto < ordLength)
{
var value = ords[ordUpto];
ordUpto++;
yield return(value);
continue;
}
if (currentReader == null || docIDUpto == currentReader.MaxDoc)
{
readerUpto++;
if (readerUpto < readers.Length)
{
currentReader = readers[readerUpto];
currentLiveDocs = currentReader.LiveDocs;
}
docIDUpto = 0;
continue;
}
if (currentLiveDocs == null || currentLiveDocs.Get(docIDUpto))
{
Debug.Assert(docIDUpto < currentReader.MaxDoc);
SortedSetDocValues dv = dvs[readerUpto];
dv.SetDocument(docIDUpto);
ordUpto = ordLength = 0;
long ord;
while ((ord = dv.NextOrd()) != SortedSetDocValues.NO_MORE_ORDS)
{
if (ordLength == ords.Length)
{
ords = ArrayUtil.Grow(ords, ordLength + 1);
}
ords[ordLength] = map.GetGlobalOrd(readerUpto, ord);
ordLength++;
}
docIDUpto++;
continue;
}
docIDUpto++;
}
}
public override int NextDoc()
{
bool first = true;
_in.Seek(_nextDocStart);
long posStart = 0;
while (true)
{
long lineStart = _in.GetFilePointer();
SimpleTextUtil.ReadLine(_in, _scratch);
//System.out.println("NEXT DOC: " + scratch.utf8ToString());
if (StringHelper.StartsWith(_scratch, SimpleTextFieldsWriter.DOC))
{
if (!first && (_liveDocs == null || _liveDocs.Get(_docId)))
{
_nextDocStart = lineStart;
_in.Seek(posStart);
return(_docId);
}
UnicodeUtil.UTF8toUTF16(_scratch.Bytes, _scratch.Offset + SimpleTextFieldsWriter.DOC.Length, _scratch.Length - SimpleTextFieldsWriter.DOC.Length,
_scratchUtf16);
_docId = ArrayUtil.ParseInt32(_scratchUtf16.Chars, 0, _scratchUtf16.Length);
_tf = 0;
first = false;
}
else if (StringHelper.StartsWith(_scratch, SimpleTextFieldsWriter.FREQ))
{
UnicodeUtil.UTF8toUTF16(_scratch.Bytes, _scratch.Offset + SimpleTextFieldsWriter.FREQ.Length,
_scratch.Length - SimpleTextFieldsWriter.FREQ.Length, _scratchUtf16);
_tf = ArrayUtil.ParseInt32(_scratchUtf16.Chars, 0, _scratchUtf16.Length);
posStart = _in.GetFilePointer();
}
else if (StringHelper.StartsWith(_scratch, SimpleTextFieldsWriter.POS))
{
// skip
}
else if (StringHelper.StartsWith(_scratch, SimpleTextFieldsWriter.START_OFFSET))
{
// skip
}
else if (StringHelper.StartsWith(_scratch, SimpleTextFieldsWriter.END_OFFSET))
{
// skip
}
else if (StringHelper.StartsWith(_scratch, SimpleTextFieldsWriter.PAYLOAD))
{
// skip
}
else
{
if (Debugging.AssertsEnabled)
{
Debugging.Assert(StringHelper.StartsWith(_scratch, SimpleTextFieldsWriter.TERM) || StringHelper.StartsWith(_scratch, SimpleTextFieldsWriter.FIELD) ||
StringHelper.StartsWith(_scratch, SimpleTextFieldsWriter.END));
}
if (!first && (_liveDocs == null || _liveDocs.Get(_docId)))
{
_nextDocStart = lineStart;
_in.Seek(posStart);
return(_docId);
}
return(_docId = NO_MORE_DOCS);
}
}
}
public override int NextPosition()
{
int pos;
if (_readPositions)
{
SimpleTextUtil.ReadLine(_in, _scratch);
if (Debugging.AssertsEnabled)
{
Debugging.Assert(StringHelper.StartsWith(_scratch, SimpleTextFieldsWriter.POS), () => "got line=" + _scratch.Utf8ToString());
}
UnicodeUtil.UTF8toUTF16(_scratch.Bytes, _scratch.Offset + SimpleTextFieldsWriter.POS.Length, _scratch.Length - SimpleTextFieldsWriter.POS.Length,
_scratchUtf162);
pos = ArrayUtil.ParseInt32(_scratchUtf162.Chars, 0, _scratchUtf162.Length);
}
else
{
pos = -1;
}
if (_readOffsets)
{
SimpleTextUtil.ReadLine(_in, _scratch);
if (Debugging.AssertsEnabled)
{
Debugging.Assert(StringHelper.StartsWith(_scratch, SimpleTextFieldsWriter.START_OFFSET), () => "got line=" + _scratch.Utf8ToString());
}
UnicodeUtil.UTF8toUTF16(_scratch.Bytes, _scratch.Offset + SimpleTextFieldsWriter.START_OFFSET.Length,
_scratch.Length - SimpleTextFieldsWriter.START_OFFSET.Length, _scratchUtf162);
_startOffset = ArrayUtil.ParseInt32(_scratchUtf162.Chars, 0, _scratchUtf162.Length);
SimpleTextUtil.ReadLine(_in, _scratch);
if (Debugging.AssertsEnabled)
{
Debugging.Assert(StringHelper.StartsWith(_scratch, SimpleTextFieldsWriter.END_OFFSET), () => "got line=" + _scratch.Utf8ToString());
}
UnicodeUtil.UTF8toUTF16(_scratch.Bytes, _scratch.Offset + SimpleTextFieldsWriter.END_OFFSET.Length,
_scratch.Length - SimpleTextFieldsWriter.END_OFFSET.Length, _scratchUtf162);
_endOffset = ArrayUtil.ParseInt32(_scratchUtf162.Chars, 0, _scratchUtf162.Length);
}
long fp = _in.GetFilePointer();
SimpleTextUtil.ReadLine(_in, _scratch);
if (StringHelper.StartsWith(_scratch, SimpleTextFieldsWriter.PAYLOAD))
{
int len = _scratch.Length - SimpleTextFieldsWriter.PAYLOAD.Length;
if (_scratch2.Bytes.Length < len)
{
_scratch2.Grow(len);
}
Array.Copy(_scratch.Bytes, SimpleTextFieldsWriter.PAYLOAD.Length, _scratch2.Bytes, 0, len);
_scratch2.Length = len;
_payload = _scratch2;
}
else
{
_payload = null;
_in.Seek(fp);
}
return(pos);
}
public override int NextDoc()
{
if (_docId == NO_MORE_DOCS)
{
return(_docId);
}
bool first = true;
int termFreq = 0;
while (true)
{
long lineStart = _in.GetFilePointer();
SimpleTextUtil.ReadLine(_in, _scratch);
if (StringHelper.StartsWith(_scratch, SimpleTextFieldsWriter.DOC))
{
if (!first && (_liveDocs == null || _liveDocs.Get(_docId)))
{
_in.Seek(lineStart);
if (!_omitTf)
{
_tf = termFreq;
}
return(_docId);
}
UnicodeUtil.UTF8toUTF16(_scratch.Bytes, _scratch.Offset + SimpleTextFieldsWriter.DOC.Length, _scratch.Length - SimpleTextFieldsWriter.DOC.Length,
_scratchUtf16);
_docId = ArrayUtil.ParseInt32(_scratchUtf16.Chars, 0, _scratchUtf16.Length);
termFreq = 0;
first = false;
}
else if (StringHelper.StartsWith(_scratch, SimpleTextFieldsWriter.FREQ))
{
UnicodeUtil.UTF8toUTF16(_scratch.Bytes, _scratch.Offset + SimpleTextFieldsWriter.FREQ.Length,
_scratch.Length - SimpleTextFieldsWriter.FREQ.Length, _scratchUtf16);
termFreq = ArrayUtil.ParseInt32(_scratchUtf16.Chars, 0, _scratchUtf16.Length);
}
else if (StringHelper.StartsWith(_scratch, SimpleTextFieldsWriter.POS))
{
// skip termFreq++;
}
else if (StringHelper.StartsWith(_scratch, SimpleTextFieldsWriter.START_OFFSET))
{
// skip
}
else if (StringHelper.StartsWith(_scratch, SimpleTextFieldsWriter.END_OFFSET))
{
// skip
}
else if (StringHelper.StartsWith(_scratch, SimpleTextFieldsWriter.PAYLOAD))
{
// skip
}
else
{
if (Debugging.AssertsEnabled)
{
Debugging.Assert(
StringHelper.StartsWith(_scratch, SimpleTextFieldsWriter.TERM) || StringHelper.StartsWith(_scratch, SimpleTextFieldsWriter.FIELD) ||
StringHelper.StartsWith(_scratch, SimpleTextFieldsWriter.END), () => "scratch=" + _scratch.Utf8ToString());
}
if (!first && (_liveDocs == null || _liveDocs.Get(_docId)))
{
_in.Seek(lineStart);
if (!_omitTf)
{
_tf = termFreq;
}
return(_docId);
}
return(_docId = NO_MORE_DOCS);
}
}
}
/// <summary>
/// Low level api. Returns a token stream generated from a <see cref="Terms"/>. This
/// can be used to feed the highlighter with a pre-parsed token
/// stream. The <see cref="Terms"/> must have offsets available.
/// <para/>
/// In my tests the speeds to recreate 1000 token streams using this method are:
/// <list type="bullet">
/// <item><description>
/// with TermVector offset only data stored - 420 milliseconds
/// </description></item>
/// <item><description>
/// with TermVector offset AND position data stored - 271 milliseconds
/// (nb timings for TermVector with position data are based on a tokenizer with contiguous
/// positions - no overlaps or gaps)
/// </description></item>
/// <item><description>
/// The cost of not using TermPositionVector to store
/// pre-parsed content and using an analyzer to re-parse the original content:
/// - reanalyzing the original content - 980 milliseconds
/// </description></item>
/// </list>
///
/// The re-analyze timings will typically vary depending on -
/// <list type="number">
/// <item><description>
/// The complexity of the analyzer code (timings above were using a
/// stemmer/lowercaser/stopword combo)
/// </description></item>
/// <item><description>
/// The number of other fields (Lucene reads ALL fields off the disk
/// when accessing just one document field - can cost dear!)
/// </description></item>
/// <item><description>
/// Use of compression on field storage - could be faster due to compression (less disk IO)
/// or slower (more CPU burn) depending on the content.
/// </description></item>
/// </list>
/// </summary>
/// <param name="tpv"></param>
/// <param name="tokenPositionsGuaranteedContiguous">true if the token position numbers have no overlaps or gaps. If looking
/// to eek out the last drops of performance, set to true. If in doubt, set to false.</param>
/// <exception cref="ArgumentException">if no offsets are available</exception>
public static TokenStream GetTokenStream(Terms tpv,
bool tokenPositionsGuaranteedContiguous)
{
if (!tpv.HasOffsets)
{
throw new ArgumentException("Cannot create TokenStream from Terms without offsets");
}
if (!tokenPositionsGuaranteedContiguous && tpv.HasPositions)
{
return(new TokenStreamFromTermPositionVector(tpv));
}
bool hasPayloads = tpv.HasPayloads;
// code to reconstruct the original sequence of Tokens
TermsEnum termsEnum = tpv.GetIterator(null);
int totalTokens = 0;
while (termsEnum.Next() != null)
{
totalTokens += (int)termsEnum.TotalTermFreq;
}
Token[] tokensInOriginalOrder = new Token[totalTokens];
List <Token> unsortedTokens = null;
termsEnum = tpv.GetIterator(null);
BytesRef text;
DocsAndPositionsEnum dpEnum = null;
while ((text = termsEnum.Next()) != null)
{
dpEnum = termsEnum.DocsAndPositions(null, dpEnum);
if (dpEnum == null)
{
throw new ArgumentException("Required TermVector Offset information was not found");
}
string term = text.Utf8ToString();
dpEnum.NextDoc();
int freq = dpEnum.Freq;
for (int posUpto = 0; posUpto < freq; posUpto++)
{
int pos = dpEnum.NextPosition();
if (dpEnum.StartOffset < 0)
{
throw new ArgumentException("Required TermVector Offset information was not found");
}
Token token = new Token(term, dpEnum.StartOffset, dpEnum.EndOffset);
if (hasPayloads)
{
// Must make a deep copy of the returned payload,
// since D&PEnum API is allowed to re-use on every
// call:
token.Payload = BytesRef.DeepCopyOf(dpEnum.GetPayload());
}
if (tokenPositionsGuaranteedContiguous && pos != -1)
{
// We have positions stored and a guarantee that the token position
// information is contiguous
// This may be fast BUT wont work if Tokenizers used which create >1
// token in same position or
// creates jumps in position numbers - this code would fail under those
// circumstances
// tokens stored with positions - can use this to index straight into
// sorted array
tokensInOriginalOrder[pos] = token;
}
else
{
// tokens NOT stored with positions or not guaranteed contiguous - must
// add to list and sort later
if (unsortedTokens == null)
{
unsortedTokens = new List <Token>();
}
unsortedTokens.Add(token);
}
}
}
// If the field has been stored without position data we must perform a sort
if (unsortedTokens != null)
{
tokensInOriginalOrder = unsortedTokens.ToArray();
ArrayUtil.TimSort(tokensInOriginalOrder, new TokenComparer());
//tokensInOriginalOrder = tokensInOriginalOrder
// .OrderBy(t => t, new TokenComparer() )
// .ToArray();
}
return(new StoredTokenStream(tokensInOriginalOrder));
}
private void Trigger_Imp(object sender, object incomingArg, object outgoingArg, BlockingTriggerYieldInstruction instruction)
{
switch (this._activationType)
{
case TriggerActivationType.TriggerAllOnTarget:
{
if (_triggerAllCache == null)
{
//_triggerAllCache = _triggerable.GetComponentsAlt<ITriggerableMechanism>();
var go = GameObjectUtil.GetGameObjectFromSource(_triggerable);
if (go != null)
{
_triggerAllCache = go.GetComponents <ITriggerableMechanism>();
}
else if (_triggerable is ITriggerableMechanism)
{
_triggerAllCache = new ITriggerableMechanism[] { _triggerable as ITriggerableMechanism }
}
;
else
{
_triggerAllCache = ArrayUtil.Empty <ITriggerableMechanism>();
}
if (_triggerableArgs.Length > 1)
{
System.Array.Sort(_triggerableArgs, TriggerableMechanismOrderComparer.Default);
}
}
if (instruction != null)
{
foreach (var t in _triggerAllCache)
{
if (t.CanTrigger)
{
if (t is IBlockingTriggerableMechanism)
{
(t as IBlockingTriggerableMechanism).Trigger(sender, outgoingArg, instruction);
}
else
{
t.Trigger(sender, outgoingArg);
}
}
}
}
else
{
foreach (var t in _triggerAllCache)
{
if (t.CanTrigger)
{
t.Trigger(sender, outgoingArg);
}
}
}
}
break;
case TriggerActivationType.TriggerSelectedTarget:
{
//UnityEngine.Object targ = _triggerable;
//if (targ is IProxy) targ = (targ as IProxy).GetTarget(incomingArg);
//TriggerSelectedTarget(targ, sender, outgoingArg, instruction);
TriggerSelectedTarget(_triggerable, sender, outgoingArg, instruction);
}
break;
case TriggerActivationType.SendMessage:
{
object targ = _triggerable;
if (targ is IProxy)
{
targ = (targ as IProxy).GetTarget(incomingArg);
}
SendMessageToTarget(targ, _methodName, outgoingArg);
}
break;
case TriggerActivationType.CallMethodOnSelectedTarget:
{
CallMethodOnSelectedTarget(_triggerable, _methodName, _triggerableArgs);
}
break;
case TriggerActivationType.EnableTarget:
{
object targ = _triggerable;
if (targ is IProxy)
{
targ = (targ as IProxy).GetTarget(incomingArg);
}
EnableTarget(_triggerable, ConvertUtil.ToEnum <EnableMode>(_methodName));
}
break;
case TriggerActivationType.DestroyTarget:
{
object targ = _triggerable;
if (targ is IProxy)
{
targ = (targ as IProxy).GetTarget(incomingArg);
}
DestroyTarget(_triggerable);
}
break;
}
}
public void LibraryTransitionTest()
{
LibraryManager libraryManager;
TestDocumentContainer docContainer;
int startRev;
SrmDocument docLoaded = CreateNISTLibraryDocument(out libraryManager, out docContainer, out startRev);
// Test tolerance range
SrmSettings settings = docLoaded.Settings.ChangeTransitionLibraries(l =>
l.ChangeIonMatchTolerance(TransitionLibraries.MIN_MATCH_TOLERANCE));
SrmDocument docLowTol = docLoaded.ChangeSettings(settings);
// Use the original low tolerance for transition testing, since
// the new low tolerance is for high accuracy data.
docLowTol = docLowTol.ChangeSettings(settings.ChangeTransitionLibraries(l =>
l.ChangeIonMatchTolerance(0.1)));
settings = docLowTol.Settings.ChangeTransitionLibraries(l =>
l.ChangeIonMatchTolerance(TransitionLibraries.MAX_MATCH_TOLERANCE));
SrmDocument docHighTol = docLoaded.ChangeSettings(settings);
Assert.AreEqual(docLowTol.PeptideTransitionCount, docHighTol.PeptideTransitionCount);
var transLow = docLowTol.PeptideTransitions.ToArray();
var transHigh = docHighTol.PeptideTransitions.ToArray();
int diffCount = 0;
for (int i = 0; i < transLow.Length; i++)
{
if (!Equals(transLow[i], transHigh[i]))
{
diffCount++;
}
}
Assert.AreEqual(2, diffCount);
Assert.IsTrue(ArrayUtil.ReferencesEqual(docLoaded.PeptideTransitionGroups.ToArray(), docHighTol.PeptideTransitionGroups.ToArray()));
Assert.IsTrue(HasMaxTransitionRank(docHighTol, 3));
SrmSettings setThrow = settings;
AssertEx.ThrowsException <InvalidDataException>(() =>
setThrow.ChangeTransitionLibraries(l => l.ChangeIonMatchTolerance(TransitionLibraries.MAX_MATCH_TOLERANCE * 2)));
AssertEx.ThrowsException <InvalidDataException>(() =>
setThrow.ChangeTransitionLibraries(l => l.ChangeIonMatchTolerance(TransitionLibraries.MIN_MATCH_TOLERANCE / 2)));
// Picked transition count
settings = docLoaded.Settings.ChangeTransitionLibraries(l => l.ChangeIonCount(5));
SrmDocument docHighIons = docLoaded.ChangeSettings(settings);
AssertEx.IsDocumentState(docHighIons, ++startRev, 2, 4, 20);
Assert.IsTrue(HasMaxTransitionRank(docHighIons, 5));
Assert.IsFalse(HasMinTransitionOrdinal(docHighIons, 4));
settings = settings.ChangeTransitionLibraries(l => l.ChangePick(TransitionLibraryPick.none));
SrmDocument docFilteredIons = docHighIons.ChangeSettings(settings);
AssertEx.IsDocumentState(docFilteredIons, ++startRev, 2, 4, 15); // Proline ions
Assert.IsFalse(HasMaxTransitionRank(docFilteredIons, 5));
settings = settings.ChangeTransitionFilter(f => f.ChangeFragmentRangeFirstName("ion 4")
.ChangeFragmentRangeLastName("last ion").ChangeMeasuredIons(new MeasuredIon[0]));
settings = settings.ChangeTransitionLibraries(l => l.ChangePick(TransitionLibraryPick.filter));
SrmDocument docRankedFiltered = docFilteredIons.ChangeSettings(settings);
AssertEx.IsDocumentState(docRankedFiltered, ++startRev, 2, 4, 20);
Assert.IsTrue(HasMaxTransitionRank(docRankedFiltered, 5));
Assert.IsTrue(HasMinTransitionOrdinal(docRankedFiltered, 4));
AssertEx.Serializable(docRankedFiltered, (doc1, doc2) => ValidateLibraryDocs(doc1, doc2, libraryManager));
}
public int[] GetRandomAcceptedString(Random r)
{
IList <int?> soFar = new List <int?>();
if (a.IsSingleton)
{
// accepts only one
var s = a.Singleton;
int charUpto = 0;
while (charUpto < s.Length)
{
int cp = s.CodePointAt(charUpto);
charUpto += Character.CharCount(cp);
soFar.Add(cp);
}
}
else
{
var s = a.initial;
while (true)
{
if (s.accept)
{
if (s.numTransitions == 0)
{
// stop now
break;
}
else
{
if (r.NextBoolean())
{
break;
}
}
}
if (s.numTransitions == 0)
{
throw new Exception("this automaton has dead states");
}
bool cheat = r.NextBoolean();
Transition t;
if (cheat)
{
// pick a transition that we know is the fastest
// path to an accept state
IList <Transition> toAccept = new List <Transition>();
for (int i = 0; i < s.numTransitions; i++)
{
Transition t0 = s.TransitionsArray[i];
if (leadsToAccept.ContainsKey(t0))
{
toAccept.Add(t0);
}
}
if (toAccept.Count == 0)
{
// this is OK -- it means we jumped into a cycle
t = s.TransitionsArray[r.Next(s.numTransitions)];
}
else
{
t = toAccept[r.Next(toAccept.Count)];
}
}
else
{
t = s.TransitionsArray[r.Next(s.numTransitions)];
}
soFar.Add(AutomatonTestUtil.GetRandomCodePoint(r, t));
s = t.to;
}
}
return(ArrayUtil.ToInt32Array(soFar));
}
/// <summary>
/// Suggest similar words.
///
/// <para>Unlike <seealso cref="SpellChecker"/>, the similarity used to fetch the most
/// relevant terms is an edit distance, therefore typically a low value
/// for numSug will work very well.
///
/// </para>
/// </summary>
/// <param name="term"> Term you want to spell check on </param>
/// <param name="numSug"> the maximum number of suggested words </param>
/// <param name="ir"> IndexReader to find terms from </param>
/// <param name="suggestMode"> specifies when to return suggested words </param>
/// <param name="accuracy"> return only suggested words that match with this similarity </param>
/// <returns> sorted list of the suggested words according to the comparator </returns>
/// <exception cref="IOException"> If there is a low-level I/O error. </exception>
public virtual SuggestWord[] SuggestSimilar(Term term, int numSug, IndexReader ir, SuggestMode suggestMode, float accuracy)
{
CharsRef spare = new CharsRef();
string text = term.Text();
if (minQueryLength > 0 && text.CodePointCount(0, text.Length) < minQueryLength)
{
return(new SuggestWord[0]);
}
if (lowerCaseTerms)
{
term = new Term(term.Field(), text.ToLower(Locale.ROOT));
}
int docfreq = ir.DocFreq(term);
if (suggestMode == SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX && docfreq > 0)
{
return(new SuggestWord[0]);
}
int maxDoc = ir.MaxDoc();
if (maxQueryFrequency >= 1f && docfreq > maxQueryFrequency)
{
return(new SuggestWord[0]);
}
else if (docfreq > (int)Math.Ceiling(maxQueryFrequency * (float)maxDoc))
{
return(new SuggestWord[0]);
}
if (suggestMode != SuggestMode.SUGGEST_MORE_POPULAR)
{
docfreq = 0;
}
if (thresholdFrequency >= 1f)
{
docfreq = Math.Max(docfreq, (int)thresholdFrequency);
}
else if (thresholdFrequency > 0f)
{
docfreq = Math.Max(docfreq, (int)(thresholdFrequency * (float)maxDoc) - 1);
}
ICollection <ScoreTerm> terms = null;
int inspections = numSug * maxInspections;
// try ed=1 first, in case we get lucky
terms = suggestSimilar(term, inspections, ir, docfreq, 1, accuracy, spare);
if (maxEdits > 1 && terms.Count < inspections)
{
var moreTerms = new HashSet <ScoreTerm>();
moreTerms.AddAll(terms);
moreTerms.AddAll(suggestSimilar(term, inspections, ir, docfreq, maxEdits, accuracy, spare));
terms = moreTerms;
}
// create the suggestword response, sort it, and trim it to size.
var suggestions = new SuggestWord[terms.Count];
int index = suggestions.Length - 1;
foreach (ScoreTerm s in terms)
{
SuggestWord suggestion = new SuggestWord();
if (s.termAsString == null)
{
UnicodeUtil.UTF8toUTF16(s.term, spare);
s.termAsString = spare.ToString();
}
suggestion.@string = s.termAsString;
suggestion.score = s.score;
suggestion.freq = s.docfreq;
suggestions[index--] = suggestion;
}
ArrayUtil.TimSort(suggestions, Collections.ReverseOrder(comparator));
if (numSug < suggestions.Length)
{
SuggestWord[] trimmed = new SuggestWord[numSug];
Array.Copy(suggestions, 0, trimmed, 0, numSug);
suggestions = trimmed;
}
return(suggestions);
}
internal readonly bool isSegmentPrivate; // set to true iff this frozen packet represents
// a segment private deletes. in that case is should
// only have Queries
public FrozenBufferedUpdates(BufferedUpdates deletes, bool isSegmentPrivate)
{
this.isSegmentPrivate = isSegmentPrivate;
if (Debugging.AssertsEnabled)
{
Debugging.Assert(!isSegmentPrivate || deletes.terms.Count == 0, "segment private package should only have del queries");
}
Term[] termsArray = deletes.terms.Keys.ToArray(/*new Term[deletes.terms.Count]*/);
termCount = termsArray.Length;
ArrayUtil.TimSort(termsArray);
PrefixCodedTerms.Builder builder = new PrefixCodedTerms.Builder();
foreach (Term term in termsArray)
{
builder.Add(term);
}
terms = builder.Finish();
queries = new Query[deletes.queries.Count];
queryLimits = new int[deletes.queries.Count];
int upto = 0;
foreach (KeyValuePair <Query, int> ent in deletes.queries)
{
queries[upto] = ent.Key;
queryLimits[upto] = ent.Value;
upto++;
}
// TODO if a Term affects multiple fields, we could keep the updates key'd by Term
// so that it maps to all fields it affects, sorted by their docUpto, and traverse
// that Term only once, applying the update to all fields that still need to be
// updated.
IList <NumericDocValuesUpdate> allNumericUpdates = new JCG.List <NumericDocValuesUpdate>();
int numericUpdatesSize = 0;
foreach (var numericUpdates in deletes.numericUpdates.Values)
{
foreach (NumericDocValuesUpdate update in numericUpdates.Values)
{
allNumericUpdates.Add(update);
numericUpdatesSize += update.GetSizeInBytes();
}
}
numericDVUpdates = allNumericUpdates.ToArray();
// TODO if a Term affects multiple fields, we could keep the updates key'd by Term
// so that it maps to all fields it affects, sorted by their docUpto, and traverse
// that Term only once, applying the update to all fields that still need to be
// updated.
IList <BinaryDocValuesUpdate> allBinaryUpdates = new JCG.List <BinaryDocValuesUpdate>();
int binaryUpdatesSize = 0;
foreach (var binaryUpdates in deletes.binaryUpdates.Values)
{
foreach (BinaryDocValuesUpdate update in binaryUpdates.Values)
{
allBinaryUpdates.Add(update);
binaryUpdatesSize += update.GetSizeInBytes();
}
}
binaryDVUpdates = allBinaryUpdates.ToArray();
bytesUsed = (int)terms.GetSizeInBytes() + queries.Length * BYTES_PER_DEL_QUERY + numericUpdatesSize + numericDVUpdates.Length * RamUsageEstimator.NUM_BYTES_OBJECT_REF + binaryUpdatesSize + binaryDVUpdates.Length * RamUsageEstimator.NUM_BYTES_OBJECT_REF;
numTermDeletes = deletes.numTermDeletes;
}
public object[] GetSingleRowData()
{
return(ArrayUtil.ResizeArrayIfDifferent <object>(this.InitialiseNavigator().GetNext(), this.MetaData.GetColumnCount()));
}
/// <summary>
/// Expert: highlights the top-N passages from multiple fields,
/// for the provided int[] docids, to custom object as
/// returned by the <see cref="PassageFormatter"/>. Use
/// this API to render to something other than <see cref="string"/>.
/// </summary>
/// <param name="fieldsIn">field names to highlight. Must have a stored string value and also be indexed with offsets.</param>
/// <param name="query">query to highlight.</param>
/// <param name="searcher">searcher that was previously used to execute the query.</param>
/// <param name="docidsIn">containing the document IDs to highlight.</param>
/// <param name="maxPassagesIn">The maximum number of top-N ranked passages per-field used to form the highlighted snippets.</param>
/// <returns>
/// <see cref="T:IDictionary{string, object[]}"/> keyed on field name, containing the array of formatted snippets
/// corresponding to the documents in <paramref name="docidsIn"/>.
/// If no highlights were found for a document, the
/// first <paramref name="maxPassagesIn"/> from the field will
/// be returned.
/// </returns>
/// <exception cref="IOException">if an I/O error occurred during processing</exception>
/// <exception cref="ArgumentException">if <c>field</c> was indexed without <see cref="IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS"/></exception>
protected internal virtual IDictionary <string, object[]> HighlightFieldsAsObjects(string[] fieldsIn, Query query, IndexSearcher searcher, int[] docidsIn, int[] maxPassagesIn)
{
if (fieldsIn.Length < 1)
{
throw new ArgumentException("fieldsIn must not be empty");
}
if (fieldsIn.Length != maxPassagesIn.Length)
{
throw new ArgumentException("invalid number of maxPassagesIn");
}
IndexReader reader = searcher.IndexReader;
Query rewritten = Rewrite(query);
JCG.SortedSet <Term> queryTerms = new JCG.SortedSet <Term>();
rewritten.ExtractTerms(queryTerms);
IndexReaderContext readerContext = reader.Context;
IList <AtomicReaderContext> leaves = readerContext.Leaves;
// Make our own copies because we sort in-place:
int[] docids = new int[docidsIn.Length];
System.Array.Copy(docidsIn, 0, docids, 0, docidsIn.Length);
string[] fields = new string[fieldsIn.Length];
System.Array.Copy(fieldsIn, 0, fields, 0, fieldsIn.Length);
int[] maxPassages = new int[maxPassagesIn.Length];
System.Array.Copy(maxPassagesIn, 0, maxPassages, 0, maxPassagesIn.Length);
// sort for sequential io
ArrayUtil.TimSort(docids);
new InPlaceMergeSorterAnonymousHelper(fields, maxPassages).Sort(0, fields.Length);
// pull stored data:
IList <string[]> contents = LoadFieldValues(searcher, fields, docids, maxLength);
IDictionary <string, object[]> highlights = new Dictionary <string, object[]>();
for (int i = 0; i < fields.Length; i++)
{
string field = fields[i];
int numPassages = maxPassages[i];
Term floor = new Term(field, "");
Term ceiling = new Term(field, UnicodeUtil.BIG_TERM);
// LUCENENET: Call custom GetViewBetween overload to mimic Java's exclusive upper bound behavior.
var fieldTerms = queryTerms.GetViewBetween(floor, lowerValueInclusive: true, ceiling, upperValueInclusive: false);
// TODO: should we have some reasonable defaults for term pruning? (e.g. stopwords)
// Strip off the redundant field:
BytesRef[] terms = new BytesRef[fieldTerms.Count];
int termUpto = 0;
foreach (Term term in fieldTerms)
{
terms[termUpto++] = term.Bytes;
}
IDictionary <int, object> fieldHighlights = HighlightField(field, contents[i], GetBreakIterator(field), terms, docids, leaves, numPassages, query);
object[] result = new object[docids.Length];
for (int j = 0; j < docidsIn.Length; j++)
{
fieldHighlights.TryGetValue(docidsIn[j], out result[j]);
}
highlights[field] = result;
}
return(highlights);
}
public virtual void ResolveReferences(Session session, RangeVariable[] outerRanges)
{
this._leftQueryExpression.ResolveReferences(session, outerRanges);
this._rightQueryExpression.ResolveReferences(session, outerRanges);
this.AddUnresolvedExpressions(this._leftQueryExpression.UnresolvedExpressions);
this.AddUnresolvedExpressions(this._rightQueryExpression.UnresolvedExpressions);
if (this._unionCorresponding)
{
string[] columnNames = this._leftQueryExpression.GetColumnNames();
string[] array = this._rightQueryExpression.GetColumnNames();
if (this._unionCorrespondingColumns == null)
{
this._unionCorrespondingColumns = new OrderedHashSet <string>();
OrderedIntHashSet set = new OrderedIntHashSet();
OrderedIntHashSet set2 = new OrderedIntHashSet();
for (int i = 0; i < columnNames.Length; i++)
{
string str = columnNames[i];
int index = ArrayUtil.Find(array, str);
if ((str.Length > 0) && (index != -1))
{
if (!this._leftQueryExpression.AccessibleColumns[i])
{
throw Error.GetError(0x15ca);
}
if (!this._rightQueryExpression.AccessibleColumns[index])
{
throw Error.GetError(0x15ca);
}
set.Add(i);
set2.Add(index);
this._unionCorrespondingColumns.Add(str);
}
}
if (this._unionCorrespondingColumns.IsEmpty())
{
throw Error.GetError(0x15ca);
}
this._leftQueryExpression.UnionColumnMap = set.ToArray();
this._rightQueryExpression.UnionColumnMap = set2.ToArray();
}
else
{
this._leftQueryExpression.UnionColumnMap = new int[this._unionCorrespondingColumns.Size()];
this._rightQueryExpression.UnionColumnMap = new int[this._unionCorrespondingColumns.Size()];
for (int i = 0; i < this._unionCorrespondingColumns.Size(); i++)
{
string str2 = this._unionCorrespondingColumns.Get(i);
int index = ArrayUtil.Find(columnNames, str2);
if (index == -1)
{
throw Error.GetError(0x157d);
}
if (!this._leftQueryExpression.AccessibleColumns[index])
{
throw Error.GetError(0x15ca);
}
this._leftQueryExpression.UnionColumnMap[i] = index;
index = ArrayUtil.Find(array, str2);
if (index == -1)
{
throw Error.GetError(0x157d);
}
if (!this._rightQueryExpression.AccessibleColumns[index])
{
throw Error.GetError(0x15ca);
}
this._rightQueryExpression.UnionColumnMap[i] = index;
}
}
this._columnCount = this._unionCorrespondingColumns.Size();
this.UnionColumnTypes = new SqlType[this._columnCount];
this.UnionColumnNullability = new byte[this._columnCount];
this.ResolveColumnRefernecesInUnionOrderBy();
}
else
{
this._columnCount = this._leftQueryExpression.GetColumnCount();
int columnCount = this._rightQueryExpression.GetColumnCount();
if (this._columnCount != columnCount)
{
throw Error.GetError(0x15da);
}
this.UnionColumnTypes = new SqlType[this._columnCount];
this.UnionColumnNullability = new byte[this._columnCount];
this._leftQueryExpression.UnionColumnMap = this._rightQueryExpression.UnionColumnMap = new int[this._columnCount];
ArrayUtil.FillSequence(this._leftQueryExpression.UnionColumnMap);
this.ResolveColumnRefernecesInUnionOrderBy();
}
}
// algorithm: treat sentence snippets as miniature documents
// we can intersect these with the postings lists via BreakIterator.preceding(offset),s
// score each sentence as norm(sentenceStartOffset) * sum(weight * tf(freq))
private Passage[] HighlightDoc(string field, BytesRef[] terms, int contentLength, BreakIterator bi, int doc,
TermsEnum termsEnum, DocsAndPositionsEnum[] postings, int n)
{
PassageScorer scorer = GetScorer(field);
if (scorer == null)
{
throw new NullReferenceException("PassageScorer cannot be null");
}
JCG.PriorityQueue <OffsetsEnum> pq = new JCG.PriorityQueue <OffsetsEnum>();
float[] weights = new float[terms.Length];
// initialize postings
for (int i = 0; i < terms.Length; i++)
{
DocsAndPositionsEnum de = postings[i];
int pDoc;
if (de == EMPTY)
{
continue;
}
else if (de == null)
{
postings[i] = EMPTY; // initially
if (!termsEnum.SeekExact(terms[i]))
{
continue; // term not found
}
de = postings[i] = termsEnum.DocsAndPositions(null, null, DocsAndPositionsFlags.OFFSETS);
if (de == null)
{
// no positions available
throw new ArgumentException("field '" + field + "' was indexed without offsets, cannot highlight");
}
pDoc = de.Advance(doc);
}
else
{
pDoc = de.DocID;
if (pDoc < doc)
{
pDoc = de.Advance(doc);
}
}
if (doc == pDoc)
{
weights[i] = scorer.Weight(contentLength, de.Freq);
de.NextPosition();
pq.Add(new OffsetsEnum(de, i));
}
}
pq.Add(new OffsetsEnum(EMPTY, int.MaxValue)); // a sentinel for termination
JCG.PriorityQueue <Passage> passageQueue = new JCG.PriorityQueue <Passage>(n, Comparer <Passage> .Create((left, right) =>
{
if (left.score < right.score)
{
return(-1);
}
else if (left.score > right.score)
{
return(1);
}
else
{
return(left.startOffset - right.startOffset);
}
}));
Passage current = new Passage();
while (pq.TryDequeue(out OffsetsEnum off))
{
DocsAndPositionsEnum dp = off.dp;
int start = dp.StartOffset;
if (start == -1)
{
throw new ArgumentException("field '" + field + "' was indexed without offsets, cannot highlight");
}
int end = dp.EndOffset;
// LUCENE-5166: this hit would span the content limit... however more valid
// hits may exist (they are sorted by start). so we pretend like we never
// saw this term, it won't cause a passage to be added to passageQueue or anything.
Debug.Assert(EMPTY.StartOffset == int.MaxValue);
if (start < contentLength && end > contentLength)
{
continue;
}
if (start >= current.endOffset)
{
if (current.startOffset >= 0)
{
// finalize current
current.score *= scorer.Norm(current.startOffset);
// new sentence: first add 'current' to queue
if (passageQueue.Count == n && current.score < passageQueue.Peek().score)
{
current.Reset(); // can't compete, just reset it
}
else
{
passageQueue.Enqueue(current);
if (passageQueue.Count > n)
{
current = passageQueue.Dequeue();
current.Reset();
}
else
{
current = new Passage();
}
}
}
// if we exceed limit, we are done
if (start >= contentLength)
{
Passage[] passages = passageQueue.ToArray();
foreach (Passage p in passages)
{
p.Sort();
}
// sort in ascending order
ArrayUtil.TimSort(passages, Comparer <Passage> .Create((left, right) => left.startOffset - right.startOffset));
return(passages);
}
// advance breakiterator
Debug.Assert(BreakIterator.Done < 0);
current.startOffset = Math.Max(bi.Preceding(start + 1), 0);
current.endOffset = Math.Min(bi.Next(), contentLength);
}
int tf = 0;
while (true)
{
tf++;
BytesRef term = terms[off.id];
if (term == null)
{
// multitermquery match, pull from payload
term = off.dp.GetPayload();
Debug.Assert(term != null);
}
current.AddMatch(start, end, term);
if (off.pos == dp.Freq)
{
break; // removed from pq
}
else
{
off.pos++;
dp.NextPosition();
start = dp.StartOffset;
end = dp.EndOffset;
}
if (start >= current.endOffset || end > contentLength)
{
pq.Enqueue(off);
break;
}
}
current.score += weights[off.id] * scorer.Tf(tf, current.endOffset - current.startOffset);
}
// Dead code but compiler disagrees:
Debug.Assert(false);
return(null);
}
private void GenerateAtlasFromBinary()
{
UInt32 num = this.ATLAS_W * this.ATLAS_H;
Color32[] array = new Color32[num];
UInt32 num2 = 0u;
UInt32 num3 = 1u;
for (Int32 i = 0; i < (Int32)this.overlayCount; i++)
{
BGOVERLAY_DEF bGOVERLAY_DEF = this.overlayList[i];
for (Int32 j = 0; j < (Int32)bGOVERLAY_DEF.spriteCount; j++)
{
BGSPRITE_LOC_DEF bGSPRITE_LOC_DEF = bGOVERLAY_DEF.spriteList[j];
bGSPRITE_LOC_DEF.atlasX = (UInt16)num2;
bGSPRITE_LOC_DEF.atlasY = (UInt16)num3;
if (bGSPRITE_LOC_DEF.res == 0)
{
Int32 index = ArrayUtil.GetIndex(bGSPRITE_LOC_DEF.clutX * 16, bGSPRITE_LOC_DEF.clutY, (Int32)this.vram.width, (Int32)this.vram.height);
for (UInt32 num4 = 0u; num4 < (UInt32)bGSPRITE_LOC_DEF.h; num4 += 1u)
{
Int32 index2 = ArrayUtil.GetIndex(bGSPRITE_LOC_DEF.texX * 64 + bGSPRITE_LOC_DEF.u / 4, (Int32)(bGSPRITE_LOC_DEF.texY * 256u + bGSPRITE_LOC_DEF.v + num4), (Int32)this.vram.width, (Int32)this.vram.height);
Int32 index3 = ArrayUtil.GetIndex((Int32)num2, (Int32)(num3 + num4), (Int32)this.ATLAS_W, (Int32)this.ATLAS_H);
UInt32 num5 = 0u;
while (num5 < (UInt64)(bGSPRITE_LOC_DEF.w / 2))
{
Byte b = this.vram.rawData[index2 * 2 + (Int32)num5];
Byte b2 = (Byte)(b & 15);
Byte b3 = (Byte)(b >> 4 & 15);
Int32 num6 = (index + b2) * 2;
UInt16 num7 = (UInt16)(this.vram.rawData[num6] | this.vram.rawData[num6 + 1] << 8);
Int32 num8 = index3 + (Int32)(num5 * 2u);
ConvertColor16toColor32(num7, out array[num8]);
if (bGSPRITE_LOC_DEF.trans != 0 && num7 != 0)
{
if (bGSPRITE_LOC_DEF.alpha == 0)
{
array[num8].a = 127;
}
else if (bGSPRITE_LOC_DEF.alpha == 3)
{
array[num8].a = 63;
}
}
num6 = (index + b3) * 2;
num7 = (UInt16)(this.vram.rawData[num6] | this.vram.rawData[num6 + 1] << 8);
num8 = index3 + (Int32)(num5 * 2u) + 1;
ConvertColor16toColor32(num7, out array[num8]);
if (bGSPRITE_LOC_DEF.trans != 0 && num7 != 0)
{
if (bGSPRITE_LOC_DEF.alpha == 0)
{
array[num8].a = 127;
}
else if (bGSPRITE_LOC_DEF.alpha == 3)
{
array[num8].a = 63;
}
}
num5 += 1u;
}
}
}
else if (bGSPRITE_LOC_DEF.res == 1)
{
Int32 index4 = ArrayUtil.GetIndex(bGSPRITE_LOC_DEF.clutX * 16, bGSPRITE_LOC_DEF.clutY, (Int32)this.vram.width, (Int32)this.vram.height);
for (UInt32 num9 = 0u; num9 < (UInt32)bGSPRITE_LOC_DEF.h; num9 += 1u)
{
Int32 index5 = ArrayUtil.GetIndex(bGSPRITE_LOC_DEF.texX * 64 + bGSPRITE_LOC_DEF.u / 2, (Int32)(bGSPRITE_LOC_DEF.texY * 256u + bGSPRITE_LOC_DEF.v + num9), (Int32)this.vram.width, (Int32)this.vram.height);
Int32 index6 = ArrayUtil.GetIndex((Int32)num2, (Int32)(num3 + num9), (Int32)this.ATLAS_W, (Int32)this.ATLAS_H);
for (UInt32 num10 = 0u; num10 < (UInt32)bGSPRITE_LOC_DEF.w; num10 += 1u)
{
Byte b4 = this.vram.rawData[index5 * 2 + (Int32)num10];
Int32 num11 = (index4 + b4) * 2;
UInt16 num12 = (UInt16)(this.vram.rawData[num11] | this.vram.rawData[num11 + 1] << 8);
Int32 num13 = index6 + (Int32)num10;
ConvertColor16toColor32(num12, out array[num13]);
if (bGSPRITE_LOC_DEF.trans != 0 && num12 != 0)
{
if (bGSPRITE_LOC_DEF.alpha == 0)
{
array[num13].a = 127;
}
else if (bGSPRITE_LOC_DEF.alpha == 3)
{
array[num13].a = 63;
}
}
}
}
}
for (UInt32 num14 = 0u; num14 < (UInt32)bGSPRITE_LOC_DEF.h; num14 += 1u)
{
Int32 index7 = ArrayUtil.GetIndex((Int32)(num2 + this.SPRITE_W), (Int32)(num3 + num14), (Int32)this.ATLAS_W, (Int32)this.ATLAS_H);
array[index7] = array[index7 - 1];
}
for (UInt32 num15 = 0u; num15 < (UInt32)bGSPRITE_LOC_DEF.w; num15 += 1u)
{
Int32 index8 = ArrayUtil.GetIndex((Int32)(num2 + num15), (Int32)num3, (Int32)this.ATLAS_W, (Int32)this.ATLAS_H);
Int32 index9 = ArrayUtil.GetIndex((Int32)(num2 + num15), (Int32)(num3 - 1u), (Int32)this.ATLAS_W, (Int32)this.ATLAS_H);
array[index9] = array[index8];
}
Int32 index10 = ArrayUtil.GetIndex((Int32)(num2 + this.SPRITE_W - 1u), (Int32)num3, (Int32)this.ATLAS_W, (Int32)this.ATLAS_H);
Int32 index11 = ArrayUtil.GetIndex((Int32)(num2 + this.SPRITE_W), (Int32)(num3 - 1u), (Int32)this.ATLAS_W, (Int32)this.ATLAS_H);
array[index11] = array[index10];
num2 += this.SPRITE_W + 1u;
if (num2 >= this.ATLAS_W || this.ATLAS_W - num2 < this.SPRITE_W + 1u)
{
num2 = 0u;
num3 += this.SPRITE_H + 1u;
}
}
}
this.atlas.SetPixels32(array);
this.atlas.Apply();
}
public static string[] RootedPath(this AwsS3ClientConfig config, params string[] path)
{
return(ArrayUtil.Concat(new[] { config.Bucket }, config.RootPath.Split('/'), path));
}
protected override void Swap(int i, int j)
{
ArrayUtil.Swap(outerInstance.subSpansByDoc, i, j);
}
/// <summary>
/// Prints the filename and size of each file within a given compound file.
/// Add the -extract flag to extract files to the current working directory.
/// In order to make the extracted version of the index work, you have to copy
/// the segments file from the compound index into the directory where the extracted files are stored. </summary>
///// <param name="args"> Usage: org.apache.lucene.index.IndexReader [-extract] <cfsfile> </param>
public static void Main(string[] args)
{
string filename = null;
bool extract = false;
string dirImpl = null;
int j = 0;
while (j < args.Length)
{
string arg = args[j];
if ("-extract".Equals(arg, StringComparison.Ordinal))
{
extract = true;
}
else if ("-dir-impl".Equals(arg, StringComparison.Ordinal))
{
if (j == args.Length - 1)
{
// LUCENENET specific - our wrapper console shows the correct usage
throw new ArgumentException("ERROR: missing value for --directory-type option");
//Console.WriteLine("ERROR: missing value for -dir-impl option");
//Environment.Exit(1);
}
j++;
dirImpl = args[j];
}
else if (filename == null)
{
filename = arg;
}
j++;
}
if (filename == null)
{
// LUCENENET specific - our wrapper console shows the correct usage
throw new ArgumentException("ERROR: CFS-FILE is required");
//Console.WriteLine("Usage: org.apache.lucene.index.CompoundFileExtractor [-extract] [-dir-impl X] <cfsfile>");
//return;
}
Store.Directory dir = null;
CompoundFileDirectory cfr = null;
IOContext context = IOContext.READ;
try
{
FileInfo file = new FileInfo(filename);
string dirname = file.DirectoryName;
filename = file.Name;
if (dirImpl == null)
{
dir = FSDirectory.Open(new DirectoryInfo(dirname));
}
else
{
dir = CommandLineUtil.NewFSDirectory(dirImpl, new DirectoryInfo(dirname));
}
cfr = new CompoundFileDirectory(dir, filename, IOContext.DEFAULT, false);
string[] files = cfr.ListAll();
ArrayUtil.TimSort(files); // sort the array of filename so that the output is more readable
for (int i = 0; i < files.Length; ++i)
{
long len = cfr.FileLength(files[i]);
if (extract)
{
Console.WriteLine("extract " + files[i] + " with " + len + " bytes to local directory...");
using (IndexInput ii = cfr.OpenInput(files[i], context))
{
using (FileStream f = new FileStream(files[i], FileMode.Open, FileAccess.ReadWrite))
{
// read and write with a small buffer, which is more effective than reading byte by byte
byte[] buffer = new byte[1024];
int chunk = buffer.Length;
while (len > 0)
{
int bufLen = (int)Math.Min(chunk, len);
ii.ReadBytes(buffer, 0, bufLen);
f.Write(buffer, 0, bufLen);
len -= bufLen;
}
}
}
}
else
{
Console.WriteLine(files[i] + ": " + len + " bytes");
}
}
}
catch (IOException ioe)
{
Console.WriteLine(ioe.ToString());
//Console.Write(ioe.StackTrace);
}
finally
{
try
{
if (dir != null)
{
dir.Dispose();
}
if (cfr != null)
{
cfr.Dispose();
}
}
catch (IOException ioe)
{
Console.WriteLine(ioe.ToString());
//Console.Write(ioe.StackTrace);
}
}
}
//--- Class Methods ---
public static ServiceBE StartService(ServiceBE service, bool forceRefresh, bool disableOnFailure)
{
// create subordinate request id for service start
var dreamContext = DreamContext.Current;
var requestId = dreamContext.GetState <string>(DreamHeaders.DREAM_REQUEST_ID);
dreamContext.SetState(DreamHeaders.DREAM_REQUEST_ID, requestId + "-service_" + service.Id);
try {
var stopwatch = Stopwatch.StartNew();
service.ServiceLastStatus = string.Empty;
StopService(service.Id, service, ServiceStopType.Restart);
DekiContext context = DekiContext.Current;
bool dirtyServiceEntity = false;
XUri location;
ServiceRepository.IServiceInfo serviceInfo = null;
try {
// check if service is local
if (service.ServiceLocal)
{
if (string.IsNullOrEmpty(service.SID))
{
throw new Exception("missing SID");
}
// start service
if (IsLocalAuthService(service))
{
// this service is the built-in authentication provider; no need to start it
location = context.Deki.Self;
}
else
{
// convert local service configuration into an xdoc
XDoc config = new XDoc("config");
foreach (KeyValuePair <string, string> configEntry in ArrayUtil.AllKeyValues(service.Config))
{
config.InsertValueAt(configEntry.Key, configEntry.Value);
}
// if no apikey was provided, create a random one so that CreateService doesn't inject the parent one
if (config["apikey"].IsEmpty)
{
config.Elem("apikey", StringUtil.CreateAlphaNumericKey(16));
}
// add information for service to callback into deki
if (config["uri.deki"].IsEmpty)
{
config.Elem("uri.deki", context.Deki.Self);
config.Elem("wikiid.deki", context.Instance.Id);
// Providing master apikey to service for setups that don't use per instance keys
config.Elem("apikey.deki", context.Instance.ApiKey.IfNullOrEmpty(context.Deki.MasterApiKey));
}
// the service location must use the service ID and the instance ID
string servicePath = string.Format("services/{0}/{1}", context.Instance.Id, service.Id);
_log.DebugFormat("starting service '{0}' at path {1} w/ namespace {2}", service.SID, servicePath, service.Preferences["namespace"]);
serviceInfo = context.Instance.CreateLocalService(service, servicePath, config);
location = serviceInfo.ServiceUri;
}
// check if the service uri has changed since last invocation (happens when service is started for the first time or server GUID has changed)
if (!service.Uri.EqualsInvariantIgnoreCase(location.ToString()))
{
dirtyServiceEntity = true;
service.Uri = location.ToString();
}
}
else
{
_log.DebugFormat("registering remote service '{0}'", service.SID);
if (string.IsNullOrEmpty(service.Uri))
{
throw new Exception("missing URI");
}
location = new XUri(service.Uri);
serviceInfo = context.Instance.RegisterRemoteService(service, location);
}
// check if service is an Extension service
if (service.Type == ServiceType.EXT)
{
if (service.ServiceLocal)
{
_log.DebugFormat("registering service '{0}' as extension", service.SID);
}
ExtensionBL.StartExtensionService(context, service, serviceInfo, forceRefresh);
}
//Successfully starting a service enables it.
if (!service.ServiceEnabled)
{
dirtyServiceEntity = true;
service.ServiceEnabled = true;
}
} catch (Exception e) {
dirtyServiceEntity = true;
DreamMessage dm = null;
if (e is DreamResponseException)
{
dm = ((DreamResponseException)e).Response;
string message = dm.HasDocument ? dm.ToDocument()[".//message"].AsText.IfNullOrEmpty(e.Message) : dm.ToText();
service.ServiceLastStatus = string.Format("unable to initialize service ({0})", message);
}
else
{
service.ServiceLastStatus = e.GetCoroutineStackTrace();
}
if (serviceInfo != null)
{
try {
context.Instance.DeregisterService(service.Id);
} catch { }
}
// A service that fails to start becomes disabled if it's started explicitly (not during deki startup)
if (disableOnFailure)
{
service.ServiceEnabled = false;
}
_log.ErrorExceptionMethodCall(e, "StartService", string.Format("Unable to start local service id '{0}' with SID '{1}' Error: '{2}'", service.Id, service.SID, service.ServiceLastStatus));
if (dm != null)
{
throw new ExternalServiceResponseException(dm);
}
else
{
throw;
}
} finally {
// don't update remote services that haven't changed
if (dirtyServiceEntity)
{
service = UpdateService(service);
}
}
stopwatch.Stop();
_log.InfoFormat("Service '{0}' ({1}) started in {2}ms", service.Description, service.SID, stopwatch.ElapsedMilliseconds);
return(service);
} finally {
// restore the request id
dreamContext.SetState(DreamHeaders.DREAM_REQUEST_ID, requestId);
}
}
public void LibraryOnlyPeptidesTest()
{
LibraryManager libraryManager;
TestDocumentContainer docContainer;
int startRev;
SrmDocument docLoaded = CreateNISTLibraryDocument(out libraryManager, out docContainer, out startRev);
Assert.IsTrue(HasAllLibraryInfo(docLoaded));
AssertEx.Serializable(docLoaded, (doc1, doc2) => ValidateLibraryDocs(doc1, doc2, libraryManager));
SrmSettings settings = docLoaded.Settings.ChangePeptideFilter(f => f.ChangeMaxPeptideLength(14));
settings = settings.ChangePeptideLibraries(l => l.ChangePick(PeptidePick.filter));
SrmDocument docFilter = docLoaded.ChangeSettings(settings);
Assert.IsFalse(HasAllLibraryInfo(docFilter));
AssertEx.IsDocumentState(docFilter, ++startRev, 2, 14, 45);
settings = settings.ChangePeptideLibraries(l => l.ChangePick(PeptidePick.either));
docFilter = docFilter.ChangeSettings(settings);
Assert.IsFalse(HasAllLibraryInfo(docFilter));
AssertEx.IsDocumentState(docFilter, ++startRev, 2, 16, 51);
AssertEx.Serializable(docFilter, (doc1, doc2) => ValidateLibraryDocs(doc1, doc2, libraryManager));
settings = settings.ChangePeptideLibraries(l => l.ChangePick(PeptidePick.both));
docFilter = docFilter.ChangeSettings(settings);
Assert.IsTrue(HasAllLibraryInfo(docFilter));
AssertEx.IsDocumentState(docFilter, ++startRev, 2, 2, 6);
// Check all possible rankings
settings = docLoaded.Settings.ChangePeptideLibraries(l => l.ChangeRankId(LibrarySpec.PEP_RANK_COPIES));
SrmDocument docRank = docLoaded.ChangeSettings(settings);
CheckRanks(docRank, null);
settings = settings.ChangePeptideLibraries(l => l.ChangeRankId(LibrarySpec.PEP_RANK_TOTAL_INTENSITY));
SrmDocument docRank2 = docRank.ChangeSettings(settings);
CheckRanks(docRank2, docRank);
settings = settings.ChangePeptideLibraries(l => l.ChangeRankId(LibrarySpec.PEP_RANK_PICKED_INTENSITY));
// ReSharper disable InconsistentNaming
SrmDocument docRank3_1 = docRank2.ChangeSettings(settings);
// ReSharper restore InconsistentNaming
CheckRanks(docRank3_1, docRank2);
// Turns out TFRatio and picked intensity rank the same with these peptides
settings = settings.ChangePeptideLibraries(l => l.ChangeRankId(NistLibSpecBase.PEP_RANK_TFRATIO));
// ReSharper disable InconsistentNaming
SrmDocument docRank3_2 = docRank2.ChangeSettings(settings);
// ReSharper restore InconsistentNaming
CheckRanks(docRank3_2, docRank2);
AssertEx.Serializable(docRank3_2, (doc1, doc2) => ValidateLibraryDocs(doc1, doc2, libraryManager));
SrmSettings setThrow1 = settings;
AssertEx.ThrowsException <InvalidDataException>(() => setThrow1.ChangePeptideLibraries(l => l.ChangeRankId(XHunterLibSpec.PEP_RANK_EXPECT)));
AssertEx.ThrowsException <InvalidDataException>(() => setThrow1.ChangePeptideLibraries(l => l.ChangePick(PeptidePick.filter)));
AssertEx.ThrowsException <InvalidDataException>(() => setThrow1.ChangePeptideLibraries(l => l.ChangePick(PeptidePick.either)));
// Check peptide limits based on rank
settings = settings.ChangePeptideLibraries(l => l.ChangePeptideCount(1));
SrmDocument docLimited = docRank3_1.ChangeSettings(settings);
// Should now be 2 peptides with rank 1
AssertEx.IsDocumentState(docLimited, ++startRev, 2, 2, 6);
foreach (var nodePep in docLimited.Peptides)
{
Assert.AreEqual(1, nodePep.Rank ?? 0);
}
settings = settings.ChangePeptideLibraries(l => l.ChangePeptideCount(3));
docLimited = docRank3_1.ChangeSettings(settings);
Assert.IsTrue(ArrayUtil.ReferencesEqual(docRank3_1.Peptides.ToArray(), docLimited.Peptides.ToArray()));
AssertEx.Serializable(docLimited, (doc1, doc2) => ValidateLibraryDocs(doc1, doc2, libraryManager));
SrmSettings setThrow2 = settings;
AssertEx.ThrowsException <InvalidDataException>(() => setThrow2.ChangePeptideLibraries(l => l.ChangePick(PeptidePick.filter)));
AssertEx.ThrowsException <InvalidDataException>(() => setThrow2.ChangePeptideLibraries(l => l.ChangePick(PeptidePick.either)));
AssertEx.NoExceptionThrown <InvalidDataException>(() => setThrow2.ChangePeptideLibraries(l => l.ChangeRankId(null)));
settings = settings.ChangePeptideLibraries(l => l.ChangePeptideCount(null));
SrmDocument docUnlimited = docLimited.ChangeSettings(settings);
Assert.IsTrue(ArrayUtil.ReferencesEqual(docLimited.Peptides.ToArray(), docUnlimited.Peptides.ToArray()));
SrmSettings setThrow3 = settings;
AssertEx.ThrowsException <InvalidDataException>(() => setThrow3.ChangePeptideLibraries(l => l.ChangePeptideCount(PeptideLibraries.MIN_PEPTIDE_COUNT - 1)));
AssertEx.ThrowsException <InvalidDataException>(() => setThrow3.ChangePeptideLibraries(l => l.ChangePeptideCount(PeptideLibraries.MAX_PEPTIDE_COUNT + 1)));
}
public PeptideGroupDocNode ChangeSettings(SrmSettings settingsNew, SrmSettingsDiff diff)
{
if (diff.Monitor != null)
{
diff.Monitor.ProcessGroup(this);
}
if (diff.DiffPeptides && settingsNew.PeptideSettings.Filter.AutoSelect && AutoManageChildren)
{
IList <DocNode> childrenNew = new List <DocNode>();
int countPeptides = 0;
int countIons = 0;
Dictionary <int, DocNode> mapIndexToChild = CreateGlobalIndexToChildMap();
Dictionary <PeptideModKey, DocNode> mapIdToChild = CreatePeptideModToChildMap();
foreach (PeptideDocNode nodePep in GetPeptideNodes(settingsNew, true))
{
PeptideDocNode nodePepResult = nodePep;
SrmSettingsDiff diffNode = SrmSettingsDiff.ALL;
DocNode existing;
// Add values that existed before the change. First check for exact match by
// global index, which will happen when explicit modifications are added,
// and then by content identity.
if (mapIndexToChild.TryGetValue(nodePep.Id.GlobalIndex, out existing) ||
mapIdToChild.TryGetValue(nodePep.Key, out existing))
{
nodePepResult = (PeptideDocNode)existing;
diffNode = diff;
}
if (nodePepResult != null)
{
// Materialize children of the peptide.
nodePepResult = nodePepResult.ChangeSettings(settingsNew, diffNode);
childrenNew.Add(nodePepResult);
// Make sure a single peptide group does not exceed document limits.
countPeptides++;
countIons += nodePepResult.TransitionCount;
if (countIons > SrmDocument.MAX_TRANSITION_COUNT)
{
throw new InvalidDataException(String.Format(
Resources.PeptideGroupDocNode_ChangeSettings_The_current_document_settings_would_cause_the_number_of_targeted_transitions_to_exceed__0_n0___The_document_settings_must_be_more_restrictive_or_add_fewer_proteins_,
SrmDocument.MAX_TRANSITION_COUNT));
}
if (countPeptides > SrmDocument.MAX_PEPTIDE_COUNT)
{
throw new InvalidDataException(String.Format(
Resources.PeptideGroupDocNode_ChangeSettings_The_current_document_settings_would_cause_the_number_of_peptides_to_exceed__0_n0___The_document_settings_must_be_more_restrictive_or_add_fewer_proteins_,
SrmDocument.MAX_PEPTIDE_COUNT));
}
}
}
if (PeptideGroup.Sequence != null)
{
childrenNew = PeptideGroup.RankPeptides(childrenNew, settingsNew, true);
}
return((PeptideGroupDocNode)ChangeChildrenChecked(childrenNew));
}
else
{
var nodeResult = this;
if (diff.DiffPeptides && diff.SettingsOld != null)
{
// If variable modifications changed, remove all peptides with variable
// modifications which are no longer possible.
var modsNew = settingsNew.PeptideSettings.Modifications;
var modsVarNew = modsNew.VariableModifications.ToArray();
var modsOld = diff.SettingsOld.PeptideSettings.Modifications;
var modsVarOld = modsOld.VariableModifications.ToArray();
if (modsNew.MaxVariableMods < modsOld.MaxVariableMods ||
!ArrayUtil.EqualsDeep(modsVarNew, modsVarOld))
{
IList <DocNode> childrenNew = new List <DocNode>();
foreach (PeptideDocNode nodePeptide in nodeResult.Children)
{
if (nodePeptide.AreVariableModsPossible(modsNew.MaxVariableMods, modsVarNew))
{
childrenNew.Add(nodePeptide);
}
}
nodeResult = (PeptideGroupDocNode)nodeResult.ChangeChildrenChecked(childrenNew);
}
}
// Check for changes affecting children
if (diff.DiffPeptideProps || diff.DiffExplicit ||
diff.DiffTransitionGroups || diff.DiffTransitionGroupProps ||
diff.DiffTransitions || diff.DiffTransitionProps ||
diff.DiffResults)
{
IList <DocNode> childrenNew = new List <DocNode>();
// Enumerate the nodes making necessary changes.
foreach (PeptideDocNode nodePeptide in nodeResult.Children)
{
childrenNew.Add(nodePeptide.ChangeSettings(settingsNew, diff));
}
childrenNew = RankChildren(settingsNew, childrenNew);
nodeResult = (PeptideGroupDocNode)nodeResult.ChangeChildrenChecked(childrenNew);
}
return(nodeResult);
}
}
/// <summary>
/// NOTE: This was parseIntAt() in Lucene.
/// </summary>
private int ParseInt32At(int offset)
{
UnicodeUtil.UTF8toUTF16(_scratch.Bytes, _scratch.Offset + offset, _scratch.Length - offset, _scratchUtf16);
return(ArrayUtil.ParseInt32(_scratchUtf16.Chars, 0, _scratchUtf16.Length));
}
