internal override void Lookup(LookupResult result, string name, BinderContext original, LookupFilter filter)
{
Debug.Assert(result.IsClear());
var results = Compilation.GlobalNamespace.GetMembers(name);
original.FilterAccessibility(result, results, filter);
}
internal sealed override void LookupSymbolsInSingleBinder(
LookupResult result,
string name,
int arity,
ConsList<Symbol> basesBeingResolved,
LookupOptions options,
Binder originalBinder,
bool diagnose,
ref HashSet<DiagnosticInfo> useSiteDiagnostics)
{
if ((options & (LookupOptions.NamespaceAliasesOnly | LookupOptions.NamespacesOrTypesOnly | LookupOptions.LabelsOnly)) != 0)
{
return;
}
var local = this.LookupPlaceholder(name);
if ((object)local == null)
{
base.LookupSymbolsInSingleBinder(result, name, arity, basesBeingResolved, options, originalBinder, diagnose, ref useSiteDiagnostics);
}
else
{
result.MergeEqual(this.CheckViability(local, arity, options, null, diagnose, ref useSiteDiagnostics, basesBeingResolved));
}
}
internal override void LookupSymbolsInSingleBinder(LookupResult result, string name, int arity, ConsList<Symbol> basesBeingResolved, LookupOptions options, Binder originalBinder, bool diagnose, ref HashSet<DiagnosticInfo> useSiteDiagnostics)
{
_sourceBinder.LookupSymbolsInSingleBinder(result, name, arity, basesBeingResolved, options, this, diagnose, ref useSiteDiagnostics);
var symbols = result.Symbols;
for (int i = 0; i < symbols.Count; i++)
{
// Type parameters requiring mapping to the target type and
// should be found by WithMethodTypeParametersBinder instead.
var parameter = (ParameterSymbol)symbols[i];
Debug.Assert(parameter.ContainingSymbol == _sourceBinder.ContainingMemberOrLambda);
symbols[i] = _targetParameters[parameter.Ordinal + _parameterOffset];
}
}
internal sealed override void LookupSymbolsInSingleBinder(
LookupResult result,
string name,
int arity,
ConsList<Symbol> basesBeingResolved,
LookupOptions options,
Binder originalBinder,
bool diagnose,
ref HashSet<DiagnosticInfo> useSiteDiagnostics)
{
if ((options & (LookupOptions.NamespaceAliasesOnly | LookupOptions.NamespacesOrTypesOnly | LookupOptions.LabelsOnly)) != 0)
{
return;
}
if (name.StartsWith("0x", StringComparison.OrdinalIgnoreCase))
{
var valueText = name.Substring(2);
ulong address;
if (!ulong.TryParse(valueText, NumberStyles.AllowHexSpecifier, CultureInfo.InvariantCulture, out address))
{
// Invalid value should have been caught by Lexer.
throw ExceptionUtilities.UnexpectedValue(valueText);
}
var local = new ObjectAddressLocalSymbol(_containingMethod, name, this.Compilation.GetSpecialType(SpecialType.System_Object), address);
result.MergeEqual(this.CheckViability(local, arity, options, null, diagnose, ref useSiteDiagnostics, basesBeingResolved));
}
else
{
LocalSymbol lowercaseReturnValueAlias;
if (_lowercaseReturnValueAliases.TryGetValue(name, out lowercaseReturnValueAlias))
{
result.MergeEqual(this.CheckViability(lowercaseReturnValueAlias, arity, options, null, diagnose, ref useSiteDiagnostics, basesBeingResolved));
}
else
{
base.LookupSymbolsInSingleBinder(result, name, arity, basesBeingResolved, options, originalBinder, diagnose, ref useSiteDiagnostics);
}
}
}
internal override void LookupNamespaceOrType(LookupResult result, string name, BinderContext original, ConsList <Symbol> basesBeingResolved, LookupFilter filter)
{
Debug.Assert(result.IsClear());
original.FilterAccessibility(result, Compilation.GlobalNamespace.GetMembers(name).OfType <Symbol, NamespaceOrTypeSymbol>(), filter);
}
/// <summary>
/// Retrieve suggestions.
/// </summary>
public virtual IList <LookupResult> DoLookup(string key, IEnumerable <BytesRef> contexts, int num)
{
// LUCENENET: Added guard clause for null
if (key is null)
{
throw new ArgumentNullException(nameof(key));
}
if (contexts != null)
{
throw new ArgumentException("this suggester doesn't support contexts");
}
TokenStream ts = queryAnalyzer.GetTokenStream("", key);
try
{
ITermToBytesRefAttribute termBytesAtt = ts.AddAttribute <ITermToBytesRefAttribute>();
IOffsetAttribute offsetAtt = ts.AddAttribute <IOffsetAttribute>();
IPositionLengthAttribute posLenAtt = ts.AddAttribute <IPositionLengthAttribute>();
IPositionIncrementAttribute posIncAtt = ts.AddAttribute <IPositionIncrementAttribute>();
ts.Reset();
var lastTokens = new BytesRef[grams];
//System.out.println("lookup: key='" + key + "'");
// Run full analysis, but save only the
// last 1gram, last 2gram, etc.:
BytesRef tokenBytes = termBytesAtt.BytesRef;
int maxEndOffset = -1;
bool sawRealToken = false;
while (ts.IncrementToken())
{
termBytesAtt.FillBytesRef();
sawRealToken |= tokenBytes.Length > 0;
// TODO: this is somewhat iffy; today, ShingleFilter
// sets posLen to the gram count; maybe we should make
// a separate dedicated att for this?
int gramCount = posLenAtt.PositionLength;
if (Debugging.AssertsEnabled)
{
Debugging.Assert(gramCount <= grams);
}
// Safety: make sure the recalculated count "agrees":
if (CountGrams(tokenBytes) != gramCount)
{
throw new ArgumentException("tokens must not contain separator byte; got token=" + tokenBytes + " but gramCount=" + gramCount + " does not match recalculated count=" + CountGrams(tokenBytes));
}
maxEndOffset = Math.Max(maxEndOffset, offsetAtt.EndOffset);
lastTokens[gramCount - 1] = BytesRef.DeepCopyOf(tokenBytes);
}
ts.End();
if (!sawRealToken)
{
throw new ArgumentException("no tokens produced by analyzer, or the only tokens were empty strings");
}
// Carefully fill last tokens with _ tokens;
// ShingleFilter appraently won't emit "only hole"
// tokens:
int endPosInc = posIncAtt.PositionIncrement;
// Note this will also be true if input is the empty
// string (in which case we saw no tokens and
// maxEndOffset is still -1), which in fact works out OK
// because we fill the unigram with an empty BytesRef
// below:
bool lastTokenEnded = offsetAtt.EndOffset > maxEndOffset || endPosInc > 0;
//System.out.println("maxEndOffset=" + maxEndOffset + " vs " + offsetAtt.EndOffset);
if (lastTokenEnded)
{
//System.out.println(" lastTokenEnded");
// If user hit space after the last token, then
// "upgrade" all tokens. This way "foo " will suggest
// all bigrams starting w/ foo, and not any unigrams
// starting with "foo":
for (int i = grams - 1; i > 0; i--)
{
BytesRef token = lastTokens[i - 1];
if (token is null)
{
continue;
}
token.Grow(token.Length + 1);
token.Bytes[token.Length] = separator;
token.Length++;
lastTokens[i] = token;
}
lastTokens[0] = new BytesRef();
}
var arc = new FST.Arc <Int64>();
var bytesReader = fst.GetBytesReader();
// Try highest order models first, and if they return
// results, return that; else, fallback:
double backoff = 1.0;
JCG.List <LookupResult> results = new JCG.List <LookupResult>(num);
// We only add a given suffix once, from the highest
// order model that saw it; for subsequent lower order
// models we skip it:
var seen = new JCG.HashSet <BytesRef>();
for (int gram = grams - 1; gram >= 0; gram--)
{
BytesRef token = lastTokens[gram];
// Don't make unigram predictions from empty string:
if (token is null || (token.Length == 0 && key.Length > 0))
{
// Input didn't have enough tokens:
//System.out.println(" gram=" + gram + ": skip: not enough input");
continue;
}
if (endPosInc > 0 && gram <= endPosInc)
{
// Skip hole-only predictions; in theory we
// shouldn't have to do this, but we'd need to fix
// ShingleFilter to produce only-hole tokens:
//System.out.println(" break: only holes now");
break;
}
//System.out.println("try " + (gram+1) + " gram token=" + token.utf8ToString());
// TODO: we could add fuzziness here
// match the prefix portion exactly
//Pair<Long,BytesRef> prefixOutput = null;
Int64 prefixOutput = null;
try
{
prefixOutput = LookupPrefix(fst, bytesReader, token, arc);
}
catch (Exception bogus) when(bogus.IsIOException())
{
throw RuntimeException.Create(bogus);
}
//System.out.println(" prefixOutput=" + prefixOutput);
if (prefixOutput is null)
{
// This model never saw this prefix, e.g. the
// trigram model never saw context "purple mushroom"
backoff *= ALPHA;
continue;
}
// TODO: we could do this division at build time, and
// bake it into the FST?
// Denominator for computing scores from current
// model's predictions:
long contextCount = totTokens;
BytesRef lastTokenFragment = null;
for (int i = token.Length - 1; i >= 0; i--)
{
if (token.Bytes[token.Offset + i] == separator)
{
BytesRef context = new BytesRef(token.Bytes, token.Offset, i);
long? output = Lucene.Net.Util.Fst.Util.Get(fst, Lucene.Net.Util.Fst.Util.ToInt32sRef(context, new Int32sRef()));
if (Debugging.AssertsEnabled)
{
Debugging.Assert(output != null);
}
contextCount = DecodeWeight(output);
lastTokenFragment = new BytesRef(token.Bytes, token.Offset + i + 1, token.Length - i - 1);
break;
}
}
BytesRef finalLastToken;
if (lastTokenFragment is null)
{
finalLastToken = BytesRef.DeepCopyOf(token);
}
else
{
finalLastToken = BytesRef.DeepCopyOf(lastTokenFragment);
}
if (Debugging.AssertsEnabled)
{
Debugging.Assert(finalLastToken.Offset == 0);
}
CharsRef spare = new CharsRef();
// complete top-N
Util.Fst.Util.TopResults <Int64> completions = null;
try
{
// Because we store multiple models in one FST
// (1gram, 2gram, 3gram), we must restrict the
// search so that it only considers the current
// model. For highest order model, this is not
// necessary since all completions in the FST
// must be from this model, but for lower order
// models we have to filter out the higher order
// ones:
// Must do num+seen.size() for queue depth because we may
// reject up to seen.size() paths in acceptResult():
Util.Fst.Util.TopNSearcher <Int64> searcher = new TopNSearcherAnonymousClass(this, fst, num, num + seen.Count, weightComparer, seen, finalLastToken);
// since this search is initialized with a single start node
// it is okay to start with an empty input path here
searcher.AddStartPaths(arc, prefixOutput, true, new Int32sRef());
completions = searcher.Search();
if (Debugging.AssertsEnabled)
{
Debugging.Assert(completions.IsComplete);
}
}
catch (Exception bogus) when(bogus.IsIOException())
{
throw RuntimeException.Create(bogus);
}
int prefixLength = token.Length;
BytesRef suffix = new BytesRef(8);
//System.out.println(" " + completions.length + " completions");
foreach (Util.Fst.Util.Result <Int64> completion in completions)
{
token.Length = prefixLength;
// append suffix
Util.Fst.Util.ToBytesRef(completion.Input, suffix);
token.Append(suffix);
//System.out.println(" completion " + token.utf8ToString());
// Skip this path if a higher-order model already
// saw/predicted its last token:
BytesRef lastToken = token;
for (int i = token.Length - 1; i >= 0; i--)
{
if (token.Bytes[token.Offset + i] == separator)
{
if (Debugging.AssertsEnabled)
{
Debugging.Assert(token.Length - i - 1 > 0);
}
lastToken = new BytesRef(token.Bytes, token.Offset + i + 1, token.Length - i - 1);
break;
}
}
if (seen.Contains(lastToken))
{
//System.out.println(" skip dup " + lastToken.utf8ToString());
goto nextCompletionContinue;
}
seen.Add(BytesRef.DeepCopyOf(lastToken));
spare.Grow(token.Length);
UnicodeUtil.UTF8toUTF16(token, spare);
LookupResult result = new LookupResult(spare.ToString(),
// LUCENENET NOTE: We need to calculate this as decimal because when using double it can sometimes
// return numbers that are greater than long.MaxValue, which results in a negative long number.
(long)(long.MaxValue * (decimal)backoff * ((decimal)DecodeWeight(completion.Output)) / contextCount));
results.Add(result);
if (Debugging.AssertsEnabled)
{
Debugging.Assert(results.Count == seen.Count);
}
//System.out.println(" add result=" + result);
nextCompletionContinue :;
}
backoff *= ALPHA;
}
results.Sort(Comparer <Lookup.LookupResult> .Create((a, b) =>
{
if (a.Value > b.Value)
{
return(-1);
}
else if (a.Value < b.Value)
{
return(1);
}
else
{
// Tie break by UTF16 sort order:
return(a.Key.CompareToOrdinal(b.Key));
}
}));
if (results.Count > num)
{
results.RemoveRange(num, results.Count - num); // LUCENENET: Converted end index to length
}
return(results);
}
finally
{
IOUtils.DisposeWhileHandlingException(ts);
}
}
internal override void LookupMembers(LookupResult result, NamespaceOrTypeSymbol nsOrType, string name, int arity, Utilities.ConsList<Symbol> basesBeingResolved, LookupOptions options, Binder originalBinder, bool diagnose)
{
options |= LookupOptions.NamespacesOrTypesOnly | LookupOptions.MustNotBeNestedType;
base.LookupMembers(result, nsOrType, name, arity, basesBeingResolved, options, originalBinder, diagnose);
}
private static string Format(LookupResult tens, LookupResult units)
{
const string numberFormat = "{0} {1}";
return string.Format(numberFormat, tens.Description, units.Description);
}
internal override void LookupSymbols(LookupResult result, string name, int arity, Utilities.ConsList <Symbol> basesBeingResolved, LookupOptions options, bool diagnose)
{
options |= LookupOptions.NamespacesOrTypesOnly | LookupOptions.MustNotBeNestedType;
base.LookupSymbols(result, name, arity, basesBeingResolved, options, diagnose);
}
/// <summary>
/// Create the results based on the search hits.
/// Can be overridden by subclass to add particular behavior (e.g. weight transformation) </summary>
/// <exception cref="System.IO.IOException"> If there are problems reading fields from the underlying Lucene index. </exception>
protected internal virtual List<LookupResult> CreateResults(IndexSearcher searcher, TopFieldDocs hits, int num, string charSequence, bool doHighlight, IEnumerable<string> matchedTokens, string prefixToken)
{
BinaryDocValues textDV = MultiDocValues.GetBinaryValues(searcher.IndexReader, TEXT_FIELD_NAME);
// This will just be null if app didn't pass payloads to build():
// TODO: maybe just stored fields? they compress...
BinaryDocValues payloadsDV = MultiDocValues.GetBinaryValues(searcher.IndexReader, "payloads");
IList<AtomicReaderContext> leaves = searcher.IndexReader.Leaves;
List<LookupResult> results = new List<LookupResult>();
BytesRef scratch = new BytesRef();
for (int i = 0; i < hits.ScoreDocs.Length; i++)
{
FieldDoc fd = (FieldDoc)hits.ScoreDocs[i];
textDV.Get(fd.Doc, scratch);
string text = scratch.Utf8ToString();
long score = (long)fd.Fields[0];
BytesRef payload;
if (payloadsDV != null)
{
payload = new BytesRef();
payloadsDV.Get(fd.Doc, payload);
}
else
{
payload = null;
}
// Must look up sorted-set by segment:
int segment = ReaderUtil.SubIndex(fd.Doc, leaves);
SortedSetDocValues contextsDV = leaves[segment].AtomicReader.GetSortedSetDocValues(CONTEXTS_FIELD_NAME);
HashSet<BytesRef> contexts;
if (contextsDV != null)
{
contexts = new HashSet<BytesRef>();
contextsDV.Document = fd.Doc - leaves[segment].DocBase;
long ord;
while ((ord = contextsDV.NextOrd()) != SortedSetDocValues.NO_MORE_ORDS)
{
BytesRef context = new BytesRef();
contextsDV.LookupOrd(ord, context);
contexts.Add(context);
}
}
else
{
contexts = null;
}
LookupResult result;
if (doHighlight)
{
object highlightKey = Highlight(text, matchedTokens, prefixToken);
result = new LookupResult(highlightKey.ToString(), highlightKey, score, payload, contexts);
}
else
{
result = new LookupResult(text, score, payload, contexts);
}
results.Add(result);
}
return results;
}
private ExpressionNode BindMemberAccess(SyntaxNode node, ExpressionNode left, SimpleNameSyntax right, bool invoked)
{
Debug.Assert(node != null);
Debug.Assert(left != null);
Debug.Assert(right != null);
Debug.Assert(node != null);
// A member-access consists of a primary-expression, a predefined-type, or a
// qualified-alias-member, followed by a "." token, followed by an identifier,
// optionally followed by a type-argument-list.
// A member-access is either of the form E.I or of the form E.I<A1, ..., AK>,
// where E is a primary-expression, I is a single identifier and <A1, ..., AK>
// is an optional type-argument-list. When no type-argument-list is specified,
// consider K to be zero.
// UNDONE: A member-access with a primary-expression of type dynamic is dynamically bound.
// UNDONE: In this case the compiler classifies the member access as a property access of
// UNDONE: type dynamic. The rules below to determine the meaning of the member-access are
// UNDONE: then applied at run-time, using the run-time type instead of the compile-time
// UNDONE: type of the primary-expression. If this run-time classification leads to a method
// UNDONE: group, then the member access must be the primary-expression of an invocation-expression.
// The member-access is evaluated and classified as follows:
string rightName = right.PlainName;
int rightArity = right.Arity;
LookupResult lookupResult = new LookupResult();
if (left.Kind == NodeKind.NamespaceExpression)
{
// If K is zero and E is a namespace and E contains a nested namespace with name I,
// then the result is that namespace.
var ns = ((NamespaceExpression)left).NamespaceSymbol;
lookupResult = MemberLookupInNamespace(ns, rightName, rightArity);
// UNDONE: Report errors if more than one, or none.
if (lookupResult.IsViable) {
Symbol sym = lookupResult.Symbols.First();
if (sym.Kind == SymbolKind.Namespace)
return new NamespaceExpression(node, (NamespaceSymbol)sym);
else {
Debug.Assert(sym.Kind == SymbolKind.NamedType);
return new TypeExpression(node, (NamedTypeSymbol)sym);
}
}
else {
return null;
}
#if SLOW
if (node.Right.Arity == 0)
{
var childnamespaces = ns.GetMembers(node.Right.Identifier.ValueText).OfType<NamespaceSymbol>();
var childnamespace = childnamespaces.SingleOrDefault();
if (childnamespace != null)
{
return new NamespaceExpression(node, childnamespace);
}
}
// Otherwise, if E is a namespace and E contains an accessible type having name I and K type
// parameters, then the result is that type constructed with the given type arguments.
var childTypes = ns.GetMembers(node.Right.Identifier.Text).OfType<NamedTypeSymbol>().Where(s => s.Arity == node.Right.Arity && IsMemberAccessible(s));
var childType = childTypes.SingleOrDefault();
if (childType != null)
{
// UNDONE: Construct the child type if it is generic!
return new TypeExpression(node, childType);
}
#endif
}
// If E is a predefined-type or a primary-expression classified as a type, if E is not a
// type parameter, and if a member lookup of I in E with K type parameters produces a
// match, then E.I is evaluated and classified as follows:
else if (left.Kind == NodeKind.TypeExpression)
{
var type = ((TypeExpression)left).Type;
if (!(type is TypeParameterSymbol))
{
lookupResult = MemberLookup(type, rightName, rightArity, invoked);
if (lookupResult.IsViable)
{
return BindStaticMemberOfType(node, left, right, lookupResult);
}
}
}
// If E is a property access, indexer access, variable, or value, the type of which is T,
// and a member lookup of I in T with K type arguments produces a match, then E.I
// is evaluated and classified as follows:
// UNDONE: Classify E as prop access, indexer access, variable or value
else {
var type = ((ValueNode)left).Type;
lookupResult = MemberLookup(type, rightName, rightArity, invoked);
if (lookupResult.IsViable)
{
return BindInstanceMemberOfType(node, type, left, right, lookupResult);
}
}
// UNDONE: Otherwise, an attempt is made to process E.I as an extension method invocation.
// UNDONE: If this fails, E.I is an invalid member reference, and a binding-time error occurs.
return null;
}
public override IList <LookupResult> DoLookup(string key, IEnumerable <BytesRef> contexts, bool onlyMorePopular, int num)
{
Debug.Assert(num > 0);
if (onlyMorePopular)
{
throw new System.ArgumentException("this suggester only works with onlyMorePopular=false");
}
if (contexts != null)
{
throw new System.ArgumentException("this suggester doesn't support contexts");
}
if (fst == null)
{
return(new List <LookupResult>());
}
//System.out.println("lookup key=" + key + " num=" + num);
for (var i = 0; i < key.Length; i++)
{
if (key[i] == 0x1E)
{
throw new ArgumentException(
"lookup key cannot contain HOLE character U+001E; this character is reserved");
}
if (key[i] == 0x1F)
{
throw new ArgumentException(
"lookup key cannot contain unit separator character U+001F; this character is reserved");
}
}
var utf8Key = new BytesRef(key);
try
{
Automaton lookupAutomaton = ToLookupAutomaton(key);
var spare = new CharsRef();
//System.out.println(" now intersect exactFirst=" + exactFirst);
// Intersect automaton w/ suggest wFST and get all
// prefix starting nodes & their outputs:
//final PathIntersector intersector = getPathIntersector(lookupAutomaton, fst);
//System.out.println(" prefixPaths: " + prefixPaths.size());
FST.BytesReader bytesReader = fst.GetBytesReader();
var scratchArc = new FST.Arc <PairOutputs <long?, BytesRef> .Pair>();
IList <LookupResult> results = new List <LookupResult>();
IList <FSTUtil.Path <PairOutputs <long?, BytesRef> .Pair> > prefixPaths =
FSTUtil.IntersectPrefixPaths(ConvertAutomaton(lookupAutomaton), fst);
if (exactFirst)
{
int count = 0;
foreach (FSTUtil.Path <PairOutputs <long?, BytesRef> .Pair> path in prefixPaths)
{
if (fst.FindTargetArc(END_BYTE, path.FstNode, scratchArc, bytesReader) != null)
{
// This node has END_BYTE arc leaving, meaning it's an
// "exact" match:
count++;
}
}
// Searcher just to find the single exact only
// match, if present:
Util.Fst.Util.TopNSearcher <PairOutputs <long?, BytesRef> .Pair> searcher_Renamed;
searcher_Renamed = new Util.Fst.Util.TopNSearcher <PairOutputs <long?, BytesRef> .Pair>(fst, count * maxSurfaceFormsPerAnalyzedForm,
count * maxSurfaceFormsPerAnalyzedForm, weightComparer);
// NOTE: we could almost get away with only using
// the first start node. The only catch is if
// maxSurfaceFormsPerAnalyzedForm had kicked in and
// pruned our exact match from one of these nodes
// ...:
foreach (var path in prefixPaths)
{
if (fst.FindTargetArc(END_BYTE, path.FstNode, scratchArc, bytesReader) != null)
{
// This node has END_BYTE arc leaving, meaning it's an
// "exact" match:
searcher_Renamed.AddStartPaths(scratchArc, fst.Outputs.Add(path.Output, scratchArc.Output), false,
path.Input);
}
}
var completions_Renamed = searcher_Renamed.Search();
Debug.Assert(completions_Renamed.IsComplete);
// NOTE: this is rather inefficient: we enumerate
// every matching "exactly the same analyzed form"
// path, and then do linear scan to see if one of
// these exactly matches the input. It should be
// possible (though hairy) to do something similar
// to getByOutput, since the surface form is encoded
// into the FST output, so we more efficiently hone
// in on the exact surface-form match. Still, I
// suspect very little time is spent in this linear
// seach: it's bounded by how many prefix start
// nodes we have and the
// maxSurfaceFormsPerAnalyzedForm:
foreach (var completion in completions_Renamed)
{
BytesRef output2 = completion.Output.Output2;
if (SameSurfaceForm(utf8Key, output2))
{
results.Add(GetLookupResult(completion.Output.Output1, output2, spare));
break;
}
}
if (results.Count == num)
{
// That was quick:
return(results);
}
}
Util.Fst.Util.TopNSearcher <PairOutputs <long?, BytesRef> .Pair> searcher;
searcher = new TopNSearcherAnonymousInnerClassHelper(this, fst, num - results.Count,
num * maxAnalyzedPathsForOneInput, weightComparer, utf8Key, results);
prefixPaths = GetFullPrefixPaths(prefixPaths, lookupAutomaton, fst);
foreach (FSTUtil.Path <PairOutputs <long?, BytesRef> .Pair> path in prefixPaths)
{
searcher.AddStartPaths(path.FstNode, path.Output, true, path.Input);
}
var completions = searcher.Search();
Debug.Assert(completions.IsComplete);
foreach (Util.Fst.Util.Result <PairOutputs <long?, BytesRef> .Pair> completion in completions)
{
LookupResult result = GetLookupResult(completion.Output.Output1, completion.Output.Output2, spare);
// TODO: for fuzzy case would be nice to return
// how many edits were required
//System.out.println(" result=" + result);
results.Add(result);
if (results.Count == num)
{
// In the exactFirst=true case the search may
// produce one extra path
break;
}
}
return(results);
}
catch (IOException bogus)
{
throw new Exception(bogus.ToString(), bogus);
}
}
private ExpressionNode BindInstanceMemberOfType(SyntaxNode node, TypeSymbol type,
ExpressionNode left, SimpleNameSyntax right, LookupResult lookupResult)
{
Debug.Assert(left != null);
Debug.Assert(right != null);
Debug.Assert(node != null);
Debug.Assert(lookupResult.IsViable);
Debug.Assert(lookupResult.Symbols.Any());
// UNDONE: First, if E is a property or indexer access, then the value of the property or indexer access is obtained (§7.1.1) and E is reclassified as a value.
SymbolOrMethodGroup symbolOrMethods = GetSymbolOrMethodGroup(lookupResult);
if (symbolOrMethods.IsMethodGroup)
{
// If I identifies one or more methods, then the result is a method group with an associated
// instance expression of E. If a type argument list was specified, it is used in calling
// a generic method.
// UNDONE: Construct the type argument list if there is one.
return new MethodGroup(right, null, left, symbolOrMethods.MethodGroup);
}
// UNDONE: If I identifies an instance property, then the result is a property access with an associated instance expression of E.
// UNDONE: If T is a class-type and I identifies an instance field of that class-type:
// UNDONE: If the value of E is null, then a System.NullReferenceException is thrown.
// UNDONE: Otherwise, if the field is readonly and the reference occurs outside an instance constructor of the class in which the field is declared, then the result is a value, namely the value of the field I in the object referenced by E.
// UNDONE: Otherwise, the result is a variable, namely the field I in the object referenced by E.
// UNDONE: If T is a struct-type and I identifies an instance field of that struct-type:
// UNDONE: If E is a value, or if the field is readonly and the reference occurs outside an instance constructor of the struct in which the field is declared, then the result is a value, namely the value of the field I in the struct instance given by E.
// UNDONE: Otherwise, the result is a variable, namely the field I in the struct instance given by E.
// UNDONE: If I identifies an instance event:
// UNDONE: If the reference occurs within the class or struct in which the event is declared, and the event was declared without event-accessor-declarations (§10.8), then E.I is processed exactly as if I was an instance field.
// UNDONE: Otherwise, the result is an event access with an associated instance expression of E.
return null;
}
// Given a viable LookupResult, report any ambiguity errors and return either a single non-method symbols
// or a method group.
private SymbolOrMethodGroup GetSymbolOrMethodGroup(LookupResult result)
{
Debug.Assert(result.IsViable);
Symbol nonMethod = null;
List<MethodSymbol> methodGroup = null;
if (result.IsSingleton) {
Symbol sym = result.SingleSymbol;
if (sym.Kind == SymbolKind.Method) {
return new SymbolOrMethodGroup(new List<MethodSymbol> { (MethodSymbol)sym });
}
else {
return new SymbolOrMethodGroup(sym);
}
}
else {
foreach (Symbol sym in result.Symbols) {
if (sym.Kind == SymbolKind.Method) {
if (methodGroup == null) {
methodGroup = new List<MethodSymbol>();
}
methodGroup.Add((MethodSymbol)sym);
}
else {
if (nonMethod == null)
nonMethod = sym;
else {
// UNDONE: report ambiguity error between two non-methods.
}
}
}
if (nonMethod != null) {
if (methodGroup != null) {
// UNDONE: report ambiguity error between method and non-method.
}
return new SymbolOrMethodGroup(nonMethod);
}
else {
return new SymbolOrMethodGroup(methodGroup);
}
}
}
private ExpressionNode BindStaticMemberOfType(SyntaxNode node, ExpressionNode left, SimpleNameSyntax right, LookupResult lookupResult)
{
Debug.Assert(node != null);
Debug.Assert(left != null);
Debug.Assert(right != null);
Debug.Assert(lookupResult.IsViable);
Debug.Assert(lookupResult.Symbols.Any());
SymbolOrMethodGroup symbolOrMethods = GetSymbolOrMethodGroup(lookupResult);
if (symbolOrMethods.IsMethodGroup) {
// If I identifies one or more methods, then the result is a method group with no
// associated instance expression. If a type argument list was specified, it is used
// in calling a generic method.
// UNDONE: Construct the type argument list if there is one.
return new MethodGroup(right, null, left, symbolOrMethods.MethodGroup);
}
else {
Symbol symbol = symbolOrMethods.NonMethod;
switch (symbol.Kind) {
case SymbolKind.NamedType:
case SymbolKind.ErrorType:
// If I identifies a type, then the result is that type constructed with the given type arguments.
// UNDONE: Construct the child type if it is generic!
return new TypeExpression(node, (TypeSymbol)symbol);
case SymbolKind.Property:
// If I identifies a static property, then the result is a property access with no
// associated instance expression.
// UNDONE: give error if not static.
return null;
case SymbolKind.Field:
// If I identifies a static field:
// UNDONE: If the field is readonly and the reference occurs outside the static constructor of
// UNDONE: the class or struct in which the field is declared, then the result is a value, namely
// UNDONE: the value of the static field I in E.
// UNDONE: Otherwise, the result is a variable, namely the static field I in E.
// UNDONE: Need a way to mark an expression node as "I am a variable, not a value".
// UNDONE: Give error for non-static.
return null;
default:
Debug.Fail("Unexpected symbol kind");
return null;
}
}
}
protected override void LookupSymbolsInSingleBinder(LookupResult result, string name, int arity, ConsList<Symbol> basesBeingResolved, LookupOptions options, bool diagnose)
{
LookupResult tmp = LookupResult.GetInstance();
LookupResult nonViable = LookupResult.GetInstance();
// Member definitions of different kinds hide each other (field defs hide method defs, etc.).
// So even if the caller asks only for invocable members find any member first and then reject the result if a non-invokable is found.
LookupOptions anyMemberLookupOptions = options & ~LookupOptions.MustBeInvocableMember;
// TODO: optimize lookup (there might be many interactions in the chain)
for (ICompilation commonSubmission = Compilation.PreviousSubmission; commonSubmission != null; commonSubmission = commonSubmission.PreviousSubmission)
{
// TODO (tomat): cross-language binding - for now, skip non-C# submissions
Compilation submission = commonSubmission as Compilation;
if (submission == null)
{
continue;
}
tmp.Clear();
Imports imports = GetImports(submission);
imports.LookupSymbolInAliases(this, tmp, name, arity, basesBeingResolved, anyMemberLookupOptions, diagnose);
// If a viable using alias and a matching member are both defined in the submission an error is reported elsewhere.
// Ignore the member in such case.
if (!tmp.IsMultiViable && (options & LookupOptions.NamespaceAliasesOnly) == 0)
{
this.LookupMembers(tmp, submission.ScriptClass, name, arity, basesBeingResolved, anyMemberLookupOptions, diagnose);
}
// found a non-method in the current submission:
if (tmp.Symbols.Count > 0 && tmp.Symbols.First().Kind != SymbolKind.Method)
{
if (!tmp.IsMultiViable)
{
// skip non-viable members, but remember them in case no viable members are found in previous submissions:
nonViable.MergePrioritized(tmp);
continue;
}
if (result.Symbols.Count == 0)
{
result.MergeEqual(tmp);
}
break;
}
// merge overloads:
Debug.Assert(result.Symbols.Count == 0 || result.Symbols.All(s => s.Kind == SymbolKind.Method));
result.MergeEqual(tmp);
}
// Set a proper error if we found a symbol that is not invocable but were asked for invocable only.
// Only a single non-method can be present in the result; methods are always invocable.
if ((options & LookupOptions.MustBeInvocableMember) != 0 && result.Symbols.Count == 1)
{
Symbol symbol = result.Symbols.First();
AliasSymbol alias = symbol as AliasSymbol;
if (alias != null)
{
symbol = alias.GetAliasTarget(basesBeingResolved);
}
if (IsNonInvocableMember(symbol))
{
result.SetFrom(LookupResult.NotInvocable(symbol, result.Symbols.First(), diagnose));
}
}
else if (result.Symbols.Count == 0)
{
result.SetFrom(nonViable);
}
tmp.Free();
nonViable.Free();
}
/// <summary>
/// Create the results based on the search hits.
/// Can be overridden by subclass to add particular behavior (e.g. weight transformation) </summary>
/// <exception cref="System.IO.IOException"> If there are problems reading fields from the underlying Lucene index. </exception>
protected internal virtual List <LookupResult> CreateResults(IndexSearcher searcher, TopFieldDocs hits, int num, string charSequence, bool doHighlight, IEnumerable <string> matchedTokens, string prefixToken)
{
BinaryDocValues textDV = MultiDocValues.GetBinaryValues(searcher.IndexReader, TEXT_FIELD_NAME);
// This will just be null if app didn't pass payloads to build():
// TODO: maybe just stored fields? they compress...
BinaryDocValues payloadsDV = MultiDocValues.GetBinaryValues(searcher.IndexReader, "payloads");
IList <AtomicReaderContext> leaves = searcher.IndexReader.Leaves;
List <LookupResult> results = new List <LookupResult>();
BytesRef scratch = new BytesRef();
for (int i = 0; i < hits.ScoreDocs.Length; i++)
{
FieldDoc fd = (FieldDoc)hits.ScoreDocs[i];
textDV.Get(fd.Doc, scratch);
string text = scratch.Utf8ToString();
long score = (long)fd.Fields[0];
BytesRef payload;
if (payloadsDV != null)
{
payload = new BytesRef();
payloadsDV.Get(fd.Doc, payload);
}
else
{
payload = null;
}
// Must look up sorted-set by segment:
int segment = ReaderUtil.SubIndex(fd.Doc, leaves);
SortedSetDocValues contextsDV = leaves[segment].AtomicReader.GetSortedSetDocValues(CONTEXTS_FIELD_NAME);
HashSet <BytesRef> contexts;
if (contextsDV != null)
{
contexts = new HashSet <BytesRef>();
contextsDV.Document = fd.Doc - leaves[segment].DocBase;
long ord;
while ((ord = contextsDV.NextOrd()) != SortedSetDocValues.NO_MORE_ORDS)
{
BytesRef context = new BytesRef();
contextsDV.LookupOrd(ord, context);
contexts.Add(context);
}
}
else
{
contexts = null;
}
LookupResult result;
if (doHighlight)
{
object highlightKey = Highlight(text, matchedTokens, prefixToken);
result = new LookupResult(highlightKey.ToString(), highlightKey, score, payload, contexts);
}
else
{
result = new LookupResult(text, score, payload, contexts);
}
results.Add(result);
}
return(results);
}
internal static void OnStaticResourceResolved(object targetObject, object targetProperty, LookupResult result)
{
EventHandler <StaticResourceResolvedEventArgs> handler = StaticResourceResolved;
if (handler != null && result.Dictionary != null)
{
handler(null, new StaticResourceResolvedEventArgs(
targetObject,
targetProperty,
result.Dictionary,
result.Key));
}
RequestCacheCleanup(targetObject);
}
// Given two lookups done in the same scope, merge the results.
private LookupResult MergeLookupsInSameScope(LookupResult result1, LookupResult result2)
{
// TODO: Make sure that this merging gives the correct semantics.
return result1.MergeEqual(result2);
}
protected internal override IList <Lookup.LookupResult> CreateResults(IndexSearcher searcher, TopFieldDocs hits,
int num, string key, bool doHighlight, ICollection <string> matchedTokens, string prefixToken)
{
BinaryDocValues textDV = MultiDocValues.GetBinaryValues(searcher.IndexReader, TEXT_FIELD_NAME);
if (Debugging.AssertsEnabled)
{
Debugging.Assert(textDV != null);
}
// This will just be null if app didn't pass payloads to build():
// TODO: maybe just stored fields? they compress...
BinaryDocValues payloadsDV = MultiDocValues.GetBinaryValues(searcher.IndexReader, "payloads");
JCG.SortedSet <Lookup.LookupResult> results = new JCG.SortedSet <Lookup.LookupResult>(LOOKUP_COMP);
// we reduce the num to the one initially requested
int actualNum = num / numFactor;
BytesRef scratch = new BytesRef();
for (int i = 0; i < hits.ScoreDocs.Length; i++)
{
FieldDoc fd = (FieldDoc)hits.ScoreDocs[i];
textDV.Get(fd.Doc, scratch);
string text = scratch.Utf8ToString();
long weight = (J2N.Numerics.Int64)fd.Fields[0];
BytesRef payload;
if (payloadsDV != null)
{
payload = new BytesRef();
payloadsDV.Get(fd.Doc, payload);
}
else
{
payload = null;
}
double coefficient;
if (text.StartsWith(key.ToString(), StringComparison.Ordinal))
{
// if hit starts with the key, we don't change the score
coefficient = 1;
}
else
{
coefficient = CreateCoefficient(searcher, fd.Doc, matchedTokens, prefixToken);
}
long score = (long)(weight * coefficient);
LookupResult result;
if (doHighlight)
{
object highlightKey = Highlight(text, matchedTokens, prefixToken);
result = new LookupResult(highlightKey.ToString(), highlightKey, score, payload);
}
else
{
result = new LookupResult(text, score, payload);
}
BoundedTreeAdd(results, result, actualNum);
}
return(new JCG.List <LookupResult>(results.Reverse()));
}
// Given two looksup in two scopes, whereby viable results in resultHiding should hide results
// in resultHidden, merge the lookups.
private LookupResult MergeHidingLookups(LookupResult resultHiding, LookupResult resultHidden)
{
// Methods hide non-methods, non-methods hide everything. We do not implement hiding by signature
// here; that can be handled later in overload lookup. Doing this efficiently is a little complex...
if (resultHiding.IsViable && resultHidden.IsViable) {
if (resultHiding.IsSingleton) {
if (resultHiding.SingleSymbol.Kind != SymbolKind.Method)
return resultHiding;
}
else {
foreach (Symbol sym in resultHiding.Symbols) {
if (sym.Kind != SymbolKind.Method)
return resultHiding; // any non-method hides everything in the hiding scope.
}
}
// "resultHiding" only has methods. Hide all non-methods from resultHidden.
if (resultHidden.IsSingleton) {
if (resultHidden.SingleSymbol.Kind == SymbolKind.Method)
return resultHiding.MergeEqual(resultHidden);
else
return resultHiding;
}
else {
LookupResult result = resultHiding;
foreach (Symbol sym in resultHidden.Symbols) {
if (sym.Kind == SymbolKind.Method)
result = result.MergeEqual(LookupResult.Good(sym));
}
return result;
}
}
else {
return resultHiding.MergePrioritized(resultHidden);
}
}
protected override void LookupSymbolsInSingleBinder(LookupResult result, string name, int arity, ConsList <Symbol> basesBeingResolved, LookupOptions options, bool diagnose)
{
// TODO
}
// Give lookups in two scopes which should be ambiguous toward eachother; merge the results.
private LookupResult MergeAmbiguousLookups(LookupResult result1, LookupResult result2)
{
// TODO: Make sure that this merging works correctly.
return result1.MergeEqual(result2);
}
internal override void LookupMembers(LookupResult result, NamespaceOrTypeSymbol nsOrType, string name, int arity, Utilities.ConsList <Symbol> basesBeingResolved, LookupOptions options, Binder originalBinder, bool diagnose)
{
options |= LookupOptions.NamespacesOrTypesOnly | LookupOptions.MustNotBeNestedType;
base.LookupMembers(result, nsOrType, name, arity, basesBeingResolved, options, originalBinder, diagnose);
}
protected override void LookupSymbolsInSingleBinder(LookupResult result, string name, int arity, ConsList<Symbol> basesBeingResolved, LookupOptions options, bool diagnose)
{
// TODO
}
internal override void LookupSymbols(LookupResult result, string name, int arity, Utilities.ConsList<Symbol> basesBeingResolved, LookupOptions options, bool diagnose)
{
options |= LookupOptions.NamespacesOrTypesOnly | LookupOptions.MustNotBeNestedType;
base.LookupSymbols(result, name, arity, basesBeingResolved, options, diagnose);
}
private static string Format(LookupResult numberOfHundreds, LookupResult hundred)
{
const string numberFormat = "{0} {1}";
return string.Format(numberFormat, numberOfHundreds.Description, hundred.Description);
}
internal override void LookupType(LookupResult result, string name, int arity, BinderContext original, ConsList<Symbol> basesBeingResolved)
{
Debug.Assert(result.IsClear());
original.FilterAccessibility(result, Compilation.GlobalNamespace.GetTypeMembers(name, arity));
}
static async Task FindSymbolReferencesAsync(HashSet <SearchResult> antiDuplicatesSet, List <SearchResult> result, LookupResult lookup, MonoDevelopWorkspace workspace, ISymbol simSym)
{
foreach (var loc in simSym.Locations)
{
if (!loc.IsInSource)
{
continue;
}
var sr = new SearchResult(new FileProvider(loc.SourceTree.FilePath), loc.SourceSpan.Start, loc.SourceSpan.Length);
if (antiDuplicatesSet.Add(sr))
{
result.Add(sr);
}
}
foreach (var mref in await SymbolFinder.FindReferencesAsync(simSym, lookup.Solution).ConfigureAwait(false))
{
foreach (var loc in mref.Locations)
{
var fileName = loc.Document.FilePath;
var offset = loc.Location.SourceSpan.Start;
string projectedName;
int projectedOffset;
if (workspace.TryGetOriginalFileFromProjection(fileName, offset, out projectedName, out projectedOffset))
{
fileName = projectedName;
offset = projectedOffset;
}
var sr = new SearchResult(new FileProvider(fileName), offset, loc.Location.SourceSpan.Length);
if (antiDuplicatesSet.Add(sr))
{
result.Add(sr);
}
}
}
}
// Does a member lookup in a single type, without considering inheritance.
private LookupResult MemberLookupWithoutInheritance(TypeSymbol type, string name, int arity, bool invoked)
{
LookupResult result = new LookupResult();
IEnumerable<Symbol> members = type.GetMembers(name);
foreach (Symbol member in members) {
LookupResult resultOfThisMember;
// Do we need to exclude override members, or is that done later by overload resolution. It seems like
// not excluding them here can't lead to problems, because we will always find the overridden method as well.
SymbolKind memberKind = member.Kind;
DiagnosticInfo diagInfo;
if (WrongArity(member, arity, out diagInfo))
resultOfThisMember = LookupResult.WrongArity(member, diagInfo);
else if (invoked && !IsInvocable(member))
resultOfThisMember = LookupResult.Bad(member, new CSDiagnosticInfo(ErrorCode.ERR_NonInvocableMemberCalled, member.GetFullName()));
else if (!IsMemberAccessible(member))
resultOfThisMember = LookupResult.Inaccessible(member);
else
resultOfThisMember = LookupResult.Good(member);
result = MergeLookupsInSameScope(result, resultOfThisMember);
}
return result;
}
private void StartLookup()
{
DisplayStatusMessage = false;
IsLookupRunning = true;
// Reset statistic
DNSServerAndPort = string.Empty;
Questions = 0;
Answers = 0;
Authorities = 0;
Additionals = 0;
MessageSize = 0;
// Measure the time
StartTime = DateTime.Now;
stopwatch.Start();
dispatcherTimer.Tick += DispatcherTimer_Tick;
dispatcherTimer.Interval = new TimeSpan(0, 0, 0, 0, 100);
dispatcherTimer.Start();
EndTime = null;
// Reset the latest results
LookupResult.Clear();
HostnameOrIPAddressHistory = new List <string>(HistoryListHelper.Modify(HostnameOrIPAddressHistory, HostnameOrIPAddress, SettingsManager.Current.Application_HistoryListEntries));
DNSLookupOptions DNSLookupOptions = new DNSLookupOptions();
if (SettingsManager.Current.DNSLookup_UseCustomDNSServer)
{
if (!string.IsNullOrEmpty(SettingsManager.Current.DNSLookup_CustomDNSServer))
{
DNSLookupOptions.UseCustomDNSServer = SettingsManager.Current.DNSLookup_UseCustomDNSServer;
DNSLookupOptions.CustomDNSServer = SettingsManager.Current.DNSLookup_CustomDNSServer;
}
else
{
StatusMessage = Application.Current.Resources["String_CustomDNSServerIsEmptyCheckYourSettingsUseWindowsOwnDNSServer"] as string;
DisplayStatusMessage = true;
}
}
DNSLookupOptions.Class = SettingsManager.Current.DNSLookup_Class;
DNSLookupOptions.Type = SettingsManager.Current.DNSLookup_Type;
DNSLookupOptions.Recursion = SettingsManager.Current.DNSLookup_Recursion;
DNSLookupOptions.UseResolverCache = SettingsManager.Current.DNSLookup_UseResolverCache;
DNSLookupOptions.TransportType = SettingsManager.Current.DNSLookup_TransportType;
DNSLookupOptions.Attempts = SettingsManager.Current.DNSLookup_Attempts;
DNSLookupOptions.Timeout = SettingsManager.Current.DNSLookup_Timeout;
DNSLookupOptions.ResolveCNAME = SettingsManager.Current.DNSLookup_ResolveCNAME;
DNSLookup DNSLookup = new DNSLookup();
DNSLookup.RecordReceived += DNSLookup_RecordReceived;
DNSLookup.LookupError += DNSLookup_LookupError;
DNSLookup.LookupComplete += DNSLookup_LookupComplete;
string hostnameOrIPAddress = HostnameOrIPAddress;
string dnsSuffix = string.Empty;
// Detect hostname (usually they don't contain ".")
if (HostnameOrIPAddress.IndexOf(".", StringComparison.OrdinalIgnoreCase) == -1)
{
if (SettingsManager.Current.DNSLookup_AddDNSSuffix)
{
if (SettingsManager.Current.DNSLookup_UseCustomDNSSuffix)
{
dnsSuffix = SettingsManager.Current.DNSLookup_CustomDNSSuffix;
}
else
{
dnsSuffix = IPGlobalProperties.GetIPGlobalProperties().DomainName;
}
}
}
// Append dns suffix to hostname
if (!string.IsNullOrEmpty(dnsSuffix))
{
hostnameOrIPAddress += string.Format("{0}{1}", dnsSuffix.StartsWith(".") ? "" : ".", dnsSuffix);
}
DNSLookup.LookupAsync(hostnameOrIPAddress, DNSLookupOptions);
}
internal override void LookupNamespaceOrType(LookupResult result, string name, BinderContext original, ConsList<Symbol> basesBeingResolved, LookupFilter filter)
{
Debug.Assert(result.IsClear());
original.FilterAccessibility(result, Compilation.GlobalNamespace.GetMembers(name).OfType<Symbol, NamespaceOrTypeSymbol>(), filter);
}
// Lookup member in a class, struct, enum, delegate.
private LookupResult MemberLookupInClass(TypeSymbol type, string name, int arity, bool invoked)
{
Debug.Assert(type != null && type.TypeKind != TypeKind.Interface && type.TypeKind != TypeKind.TypeParameter);
TypeSymbol currentType = type;
LookupResult result = new LookupResult();
while (currentType != null) {
result = MergeHidingLookups(result, MemberLookupWithoutInheritance(currentType, name, arity, invoked));
currentType = currentType.BaseType;
}
return result;
}
private LookupResult MemberLookupInNamespace(NamespaceSymbol ns, string name, int arity)
{
LookupResult result = new LookupResult();
IEnumerable<Symbol> members = ns.GetMembers(name);
foreach (Symbol member in members) {
LookupResult resultOfThisMember;
DiagnosticInfo diagInfo;
if (WrongArity(member, arity, out diagInfo))
resultOfThisMember = LookupResult.WrongArity(member, diagInfo);
else if (!IsMemberAccessible(member))
resultOfThisMember = LookupResult.Inaccessible(member);
else
resultOfThisMember = LookupResult.Good(member);
result = MergeLookupsInSameScope(result, resultOfThisMember);
}
return result;
}
internal override void LookupType(LookupResult result, string name, int arity, BinderContext original, ConsList <Symbol> basesBeingResolved)
{
Debug.Assert(result.IsClear());
original.FilterAccessibility(result, Compilation.GlobalNamespace.GetTypeMembers(name, arity));
}
