public static void Main(String[] args)
{
try
{
ITransformer transformer = new Transformer();
IReader reader = new ConsoleReader();
IAnalyzer analyzer = new Analyzer(reader, transformer);
AnalysisResult analysisResult = analyzer.Analyze();
IDumper fileDumper = new ConsoleDumper(analysisResult);
// A better way to inform about the result would be an Enum with some codes (error dumping, error reading, error connecting to db, etc)
bool operationFinished = fileDumper.Dump();
string message = "";
if (operationFinished) { message = ""; }
else { message = "The application encountered an error. Please check the logs. Press Enter to exit."; }
Console.WriteLine(message);
Console.ReadLine();
}
catch (Exception ex)
{
ex.Log("");
Console.WriteLine("The application encountered an error. Please check the logs. Press Enter to exit.");
Console.ReadLine();
}
}
public static bool IsArcValid (Part from, Analyzer.Node.Part to) {
if (to == null) { return true; }
if (!MovementHelper.IsArcValid(from.movement, to.movement)) {
return false;
}
//Transition is valid if we consider movement only
//Consider hold notes
if (from.movement == Movement.ForceDownStart) {
if (to.movement == Analyzer.Node.Movement.StayDown) {
return from.panel == to.panel; //Continue hold note
} else if (to.movement == Analyzer.Node.Movement.Relax) {
return from.panel == to.panel; //End hold note
} else {
throw new ArgumentException();
}
} else if (from.movement == Movement.ForceDown) {
if (to.movement == Analyzer.Node.Movement.StayDown) {
return from.panel == to.panel; //Continue hold note
} else if (to.movement == Analyzer.Node.Movement.Relax) {
return from.panel == to.panel; //End continued hold note
} else {
throw new ArgumentException();
}
}
return true;
}
public static Part TransitionTo (Part from, Analyzer.Node.Part to, float cur_second) {
if (!IsArcValid(from, to)) {
throw new ArgumentException();
}
if (to == null) {
throw new ArgumentException();
/*return new Part(
from.movement,
from.panel,
cur_second,
from.cur_moved_second,
from.prv_moved_second
);*/
} else {
Movement nxt_movement = MovementHelper.TransitionTo(from.movement, to.movement);
if (nxt_movement == Movement.PassiveDown) {
if (from.movement == Movement.PassiveDown) {
return new Part(nxt_movement, from.panel, cur_second, from.cur_moved_second, from.prv_moved_second);
} else {
return new Part(nxt_movement, from.panel, cur_second, cur_second, from.cur_moved_second);
}
} else {
return new Part(nxt_movement, to.panel, cur_second, cur_second, from.cur_moved_second);
}
}
}
protected override Analyzer GetAnalyzer(CompilationStartAnalysisContext context, CompilationSecurityTypes types, Version targetFrameworkVersion)
{
Analyzer analyzer = new Analyzer(types, CSharpSyntaxNodeHelper.Default, targetFrameworkVersion);
context.RegisterSyntaxNodeAction(analyzer.AnalyzeNode, SyntaxKind.MethodDeclaration, SyntaxKind.ConstructorDeclaration);
return analyzer;
}
internal override Evaluation Analyze(Analyzer/*!*/ analyzer, ExInfoFromParent info)
{
access = info.Access;
Evaluation cond_eval = condExpr.Analyze(analyzer, ExInfoFromParent.DefaultExInfo);
if (cond_eval.HasValue)
{
if (Convert.ObjectToBoolean(cond_eval.Value))
{
if (trueExpr != null)
return trueExpr.Analyze(analyzer, ExInfoFromParent.DefaultExInfo);
else
return cond_eval; // condExpr ?: falseExpr // ternary shortcut
}
else
return falseExpr.Analyze(analyzer, ExInfoFromParent.DefaultExInfo);
}
else
{
if (trueExpr != null)
{
analyzer.EnterConditionalCode();
trueExpr = trueExpr.Analyze(analyzer, ExInfoFromParent.DefaultExInfo).Literalize();
analyzer.LeaveConditionalCode();
}
analyzer.EnterConditionalCode();
falseExpr = falseExpr.Analyze(analyzer, ExInfoFromParent.DefaultExInfo).Literalize();
analyzer.LeaveConditionalCode();
return new Evaluation(this);
}
}
internal override Evaluation Analyze(Analyzer/*!*/ analyzer, ExInfoFromParent info)
{
analyzer.AddCurrentRoutineProperty(RoutineProperties.ContainsLocalsWorker);
body.Analyze(analyzer);
return new Evaluation(this);
}
public override void Initialize(AnalysisContext analysisContext)
{
// this is stateless analyzer, can run concurrently
analysisContext.EnableConcurrentExecution();
// this has no meaning on running on generated code which user can't control
analysisContext.ConfigureGeneratedCodeAnalysis(GeneratedCodeAnalysisFlags.None);
analysisContext.RegisterCompilationStartAction(c =>
{
INamedTypeSymbol @string = WellKnownTypes.String(c.Compilation);
INamedTypeSymbol uri = WellKnownTypes.Uri(c.Compilation);
if (@string == null || uri == null)
{
// we don't have required types
return;
}
var analyzer = new Analyzer(c.Compilation, @string, uri, GetInvocationExpression);
// REVIEW: I need to do this thing because OperationAnalysisContext doesn't give me OwningSymbol
c.RegisterOperationBlockStartAction(sc =>
{
sc.RegisterOperationAction(oc => analyzer.Analyze(oc, sc.OwningSymbol), OperationKind.InvocationExpression);
});
});
}
public void NoReportItems()
{
analyzer = new Analyzer(CreateMockParser(0));
Assert.AreEqual(0, analyzer.ActualReportItems.Count);
analyzer.Run();
Assert.AreEqual(0, analyzer.ExpectedReportItems.Count);
}
/// <summary>
/// Build an analyzer that limits the maximum number of tokens per field. </summary>
/// <param name="delegate"> the analyzer to wrap </param>
/// <param name="maxTokenCount"> max number of tokens to produce </param>
/// <param name="consumeAllTokens"> whether all tokens from the delegate should be consumed even if maxTokenCount is reached. </param>
public LimitTokenCountAnalyzer(Analyzer @delegate, int maxTokenCount, bool consumeAllTokens)
: base(@delegate.Strategy)
{
this.@delegate = @delegate;
this.maxTokenCount = maxTokenCount;
this.consumeAllTokens = consumeAllTokens;
}
/// <summary>
/// Creates a new ShingleAnalyzerWrapper
/// </summary>
/// <param name="delegate"> Analyzer whose TokenStream is to be filtered </param>
/// <param name="minShingleSize"> Min shingle (token ngram) size </param>
/// <param name="maxShingleSize"> Max shingle size </param>
/// <param name="tokenSeparator"> Used to separate input stream tokens in output shingles </param>
/// <param name="outputUnigrams"> Whether or not the filter shall pass the original
/// tokens to the output stream </param>
/// <param name="outputUnigramsIfNoShingles"> Overrides the behavior of outputUnigrams==false for those
/// times when no shingles are available (because there are fewer than
/// minShingleSize tokens in the input stream)?
/// Note that if outputUnigrams==true, then unigrams are always output,
/// regardless of whether any shingles are available. </param>
/// <param name="fillerToken"> filler token to use when positionIncrement is more than 1 </param>
public ShingleAnalyzerWrapper(Analyzer @delegate, int minShingleSize, int maxShingleSize,
string tokenSeparator, bool outputUnigrams, bool outputUnigramsIfNoShingles, string fillerToken)
: base(@delegate.Strategy)
{
this.@delegate = @delegate;
if (maxShingleSize < 2)
{
throw new ArgumentOutOfRangeException("Max shingle size must be >= 2");
}
this.maxShingleSize = maxShingleSize;
if (minShingleSize < 2)
{
throw new ArgumentOutOfRangeException("Min shingle size must be >= 2");
}
if (minShingleSize > maxShingleSize)
{
throw new ArgumentOutOfRangeException("Min shingle size must be <= max shingle size");
}
this.minShingleSize = minShingleSize;
this.tokenSeparator = (tokenSeparator == null ? "" : tokenSeparator);
this.outputUnigrams = outputUnigrams;
this.outputUnigramsIfNoShingles = outputUnigramsIfNoShingles;
this.fillerToken = fillerToken;
}
/// <include file='Doc/Nodes.xml' path='doc/method[@name="Expression.Analyze"]/*'/>
internal override Evaluation Analyze(Analyzer/*!*/ analyzer, ExInfoFromParent info)
{
access = info.Access;
ExInfoFromParent operand_info = ExInfoFromParent.DefaultExInfo;
Evaluation left_eval = leftExpr.Analyze(analyzer, operand_info);
Evaluation right_eval;
// Boolean expression evaluation semantics:
if (operation == Operations.Or)
{
analyzer.EnterConditionalCode();
right_eval = rightExpr.Analyze(analyzer, operand_info);
analyzer.LeaveConditionalCode();
}
else
{
right_eval = rightExpr.Analyze(analyzer, operand_info);
}
Evaluation result = Evaluation.Evaluate(this, left_eval, out leftExpr, right_eval, out rightExpr);
// division by zero check:
if ((operation == Operations.Div || operation == Operations.Mod) && result.HasValue && result.Value is bool && (bool)result.Value == false)
{
analyzer.ErrorSink.Add(Warnings.DivisionByZero, analyzer.SourceUnit, rightExpr.Position);
}
else if ((operation == Operations.Div || operation == Operations.Mod) && right_eval.HasValue && right_eval.Value is int && (int)right_eval.Value == 0)
{
result = new Evaluation(this, false);
analyzer.ErrorSink.Add(Warnings.DivisionByZero, analyzer.SourceUnit, rightExpr.Position);
}
return result;
}
public static void Main(string[] args)
{
string inputname = args[0];
string outputname = args[1];
string[] lines = File.ReadAllLines(inputname);
int ncases = int.Parse(lines[0]);
int nline = 0;
IList<string> results = new List<string>();
Analyzer analyzer = new Analyzer();
for (int k = 0; k < ncases; k++)
{
Console.WriteLine(string.Format("Solving Case #{0}", k + 1));
nline++;
string[] words = lines[nline].Split(' ');
string text = words[0];
int length = int.Parse(words[1]);
int count = analyzer.Count(text, length);
results.Add(string.Format("Case #{0}: {1}", k + 1, count));
}
File.WriteAllLines(outputname, results.ToArray());
}
internal override Evaluation Analyze(Analyzer/*!*/ analyzer, ExInfoFromParent info)
{
bool already_resolved = constant != null;
if (!already_resolved)
{
access = info.Access;
ResolveName(analyzer);
}
if (constant.IsUnknown)
return new Evaluation(this);
KnownConstant known_const = (KnownConstant)constant;
if (known_const.HasValue)
{
// constant value is known:
return new Evaluation(this, known_const.Value);
}
else if (already_resolved)
{
// circular definition:
constant.ReportCircularDefinition(analyzer.ErrorSink);
return new Evaluation(this);
}
else
{
// value is not known yet, try to resolve it:
if (known_const.Node != null)
known_const.Node.Analyze(analyzer);
return (known_const.HasValue) ? new Evaluation(this, known_const.Value) : new Evaluation(this);
}
}
/// <summary>
/// Set up a new index in RAM with three test phrases and the supplied Analyzer.
/// </summary>
/// <exception cref="Exception"> if an error occurs with index writer or searcher </exception>
public override void SetUp()
{
base.SetUp();
analyzer = new ShingleAnalyzerWrapper(new MockAnalyzer(Random(), MockTokenizer.WHITESPACE, false), 2);
directory = NewDirectory();
IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(TEST_VERSION_CURRENT, analyzer));
Document doc;
doc = new Document();
doc.Add(new TextField("content", "please divide this sentence into shingles", Field.Store.YES));
writer.AddDocument(doc);
doc = new Document();
doc.Add(new TextField("content", "just another test sentence", Field.Store.YES));
writer.AddDocument(doc);
doc = new Document();
doc.Add(new TextField("content", "a sentence which contains no test", Field.Store.YES));
writer.AddDocument(doc);
writer.Dispose();
reader = DirectoryReader.Open(directory);
searcher = NewSearcher(reader);
}
public static bool IsArcValid (Movement from, Analyzer.Node.Movement to) {
if (from == Movement.Unknown) {
return
to == Analyzer.Node.Movement.JustDown ||
to == Analyzer.Node.Movement.JustDownOrStayDown;
} else if (from == Movement.Tap) {
return
to == Analyzer.Node.Movement.JustDown ||
to == Analyzer.Node.Movement.JustDownOrStayDown;
} else if (from == Movement.ForceDownStart) {
return
to == Analyzer.Node.Movement.StayDown ||
to == Analyzer.Node.Movement.Relax;
} else if (from == Movement.ForceDown) {
return
to == Analyzer.Node.Movement.StayDown ||
to == Analyzer.Node.Movement.Relax;
} else if (from == Movement.PassiveDown) {
return
to == Analyzer.Node.Movement.JustDown ||
to == Analyzer.Node.Movement.JustDownOrStayDown;
} else {
throw new ArgumentException();
}
}
public QueryTermVector(System.String queryString, Analyzer analyzer)
{
if (analyzer != null)
{
TokenStream stream = analyzer.TokenStream("", new System.IO.StringReader(queryString));
if (stream != null)
{
IList<string> terms = new List<string>();
try
{
bool hasMoreTokens = false;
stream.Reset();
ITermAttribute termAtt = stream.AddAttribute<ITermAttribute>();
hasMoreTokens = stream.IncrementToken();
while (hasMoreTokens)
{
terms.Add(termAtt.Term);
hasMoreTokens = stream.IncrementToken();
}
ProcessTerms(terms.ToArray());
}
catch (System.IO.IOException)
{
}
}
}
}
public AstCache(Analyzer analyzer, IFileSystem fs, ILogger logger, string cacheDir)
{
this.analyzer = analyzer;
this.fs = fs;
this.LOG = logger;
this.cacheDir = cacheDir;
this.cache = new Dictionary<string, Module>();
}
/// <include file='Doc/Nodes.xml' path='doc/method[@name="Expression.Analyze"]/*'/>
internal override Evaluation Analyze(Analyzer/*!*/ analyzer, ExInfoFromParent info)
{
access = info.Access;
fileNameEx = fileNameEx.Analyze(analyzer, ExInfoFromParent.DefaultExInfo).Literalize();
analyzer.AddCurrentRoutineProperty(RoutineProperties.ContainsInclude);
analyzer.CurrentScope = this.scope;
return new Evaluation(this);
}
protected override Analyzer GetAnalyzer(CompilationStartAnalysisContext context, CompilationSecurityTypes cryptTypes)
{
Analyzer analyzer = new Analyzer(cryptTypes);
context.RegisterSyntaxNodeAction(analyzer.AnalyzeNode,
SyntaxKind.InvocationExpression,
SyntaxKind.ObjectCreationExpression);
return analyzer;
}
private static void Verify(Compilation compilation, string context)
{
var analyzer = new Analyzer(s => context == s);
var diagnostics = compilation.GetAnalyzerDiagnostics(new DiagnosticAnalyzer[] { analyzer });
Assert.Equal(1, diagnostics.Length);
Assert.True(diagnostics[0].Descriptor.Description.ToString().IndexOf(analyzer.Info.GetContext()) >= 0);
}
public void GetConsonantStringsLength2Sample1()
{
Analyzer analyzer = new Analyzer();
var result = analyzer.GetConsonantPositions("abc", 2);
Assert.IsNotNull(result);
Assert.AreEqual(1, result.Count);
Assert.AreEqual(1, result[0]);
}
internal void PreAnalyze(Analyzer/*!*/ analyzer, GenericParameter/*!*/ parameter)
{
this.parameter = parameter;
PhpRoutine routine = parameter.DeclaringMember as PhpRoutine;
PhpType type = (routine != null) ? routine.DeclaringType as PhpType : parameter.DeclaringPhpType;
parameter.WriteUp(analyzer.ResolveType(defaultType, type, routine, position, false));
}
internal override Evaluation Analyze(Analyzer/*!*/ analyzer, ExInfoFromParent info)
{
access = info.Access;
foreach (Item i in items)
if (i != null) i.Analyze(analyzer);
return new Evaluation(this);
}
/// <summary>
/// Resolves generic arguments.
/// </summary>
/// <returns><B>true</B> iff all arguments are resolvable to types or constructed types (none is variable).</returns>
internal virtual bool Analyze(Analyzer/*!*/ analyzer)
{
bool result = true;
foreach (TypeRef arg in genericParams)
result &= arg.Analyze(analyzer);
return result;
}
public static bool IsArcValid (Limb from, Analyzer.Node.Limb to) {
if (to == null) { return true; }
for (int i = 0; i < Limb.PART_COUNT; ++i) {
if (!PartHelper.IsArcValid(from[i], to[i])) {
return false;
}
}
return true;
}
public void Analyze(ActualParam/*!*/node, Analyzer/*!*/ analyzer, bool isBaseCtorCallConstrained)
{
// TODO: isBaseCtorCallConstrained
ExInfoFromParent info = new ExInfoFromParent(node);
analyzer.EnterActParam();
if (node.IsVariadic) throw new NotImplementedException();
if (analyzer.ActParamDeclIsUnknown())
{
// we don't know whether the parameter will be passed by reference at run-time:
if (node.Expression.AllowsPassByReference)
{
info.Access = AccessType.ReadUnknown;
// Although we prepare to pass reference, value can be really passed.
// That's why we report warning when user use '&' in calling,
// because it has no influence.
if (node.Ampersand)
analyzer.ErrorSink.Add(Warnings.ActualParamWithAmpersand, analyzer.SourceUnit, node.Span);
}
else
{
info.Access = AccessType.Read;
}
}
else
{
if (analyzer.ActParamPassedByRef())
{
if (node.Expression.AllowsPassByReference)
{
info.Access = AccessType.ReadRef;
}
else
{
analyzer.ErrorSink.Add(Errors.NonVariablePassedByRef, analyzer.SourceUnit, node.Expression.Span);
analyzer.LeaveActParam();
return;
}
}
else
{
info.Access = AccessType.Read;
if (node.Ampersand) analyzer.ErrorSink.Add(Warnings.ActualParamWithAmpersand, analyzer.SourceUnit, node.Span);
}
}
node._expression = node.Expression.Analyze(analyzer, info).Literalize();
// TODO: if signature is known, act. param has type hint and expression has known type; check if type hint matches expression
analyzer.LeaveActParam();
}
public void Test_TagSourceDiffer()
{
using (var workspace = CSharpWorkspaceFactory.CreateWorkspaceFromFiles(new string[] { "class A { }", "class E { }" }, CSharpParseOptions.Default))
{
var registrationService = workspace.Services.GetService<ISolutionCrawlerRegistrationService>();
registrationService.Register(workspace);
var analyzer = new Analyzer();
var analyzerService = new TestDiagnosticAnalyzerService(
new Dictionary<string, ImmutableArray<DiagnosticAnalyzer>>() { { LanguageNames.CSharp, ImmutableArray.Create<DiagnosticAnalyzer>(analyzer) } }.ToImmutableDictionary());
var listener = new AsynchronousOperationListener();
var listeners = AsynchronousOperationListener.CreateListeners(
ValueTuple.Create(FeatureAttribute.DiagnosticService, listener),
ValueTuple.Create(FeatureAttribute.ErrorSquiggles, listener));
var diagnosticService = new DiagnosticService(SpecializedCollections.SingletonEnumerable<IDiagnosticUpdateSource>(analyzerService), listeners);
var provider = new DiagnosticsSquiggleTaggerProvider(
workspace.Services.GetService<IOptionService>(), diagnosticService,
workspace.GetService<IForegroundNotificationService>(), listeners);
var tagger = provider.CreateTagger<IErrorTag>(workspace.Documents.First().GetTextBuffer());
using (var disposable = tagger as IDisposable)
{
var service = workspace.Services.GetService<ISolutionCrawlerRegistrationService>() as SolutionCrawlerRegistrationService;
var incrementalAnalyzers = ImmutableArray.Create(analyzerService.CreateIncrementalAnalyzer(workspace));
// test first update
service.WaitUntilCompletion_ForTestingPurposesOnly(workspace, incrementalAnalyzers);
listener.CreateWaitTask().PumpingWait();
var snapshot = workspace.Documents.First().GetTextBuffer().CurrentSnapshot;
var spans = tagger.GetTags(new NormalizedSnapshotSpanCollection(new SnapshotSpan(snapshot, 0, snapshot.Length))).ToList();
Assert.True(spans.First().Span.Contains(new Span(0, 1)));
// test second update
analyzer.ChangeSeverity();
var document = workspace.CurrentSolution.GetDocument(workspace.Documents.First().Id);
var text = document.GetTextAsync().Result;
workspace.TryApplyChanges(document.WithText(text.WithChanges(new TextChange(new TextSpan(text.Length - 1, 1), string.Empty))).Project.Solution);
service.WaitUntilCompletion_ForTestingPurposesOnly(workspace, incrementalAnalyzers);
listener.CreateWaitTask().PumpingWait();
snapshot = workspace.Documents.First().GetTextBuffer().CurrentSnapshot;
spans = tagger.GetTags(new NormalizedSnapshotSpanCollection(new SnapshotSpan(snapshot, 0, snapshot.Length))).ToList();
Assert.True(spans.First().Span.Contains(new Span(0, 1)));
registrationService.Unregister(workspace);
}
}
}
public Computer(int hertz,
List<bool[]> instructions,
CPU16Bit.CPUAnalyzer cpuAnalyzer,
Analyzer analyzer
)
: this(new ClockCycle(Computer.ClockCycled, hertz), new ROM(instructions),cpuAnalyzer)
{
this.analyzer = analyzer;
}
public void VerifyExpectedAndActualReports()
{
analyzer = new FakeAnalyzer(CreateMockParser(2), 2);
analyzer.Run();
Assert.AreEqual(analyzer.ActualReportItems.Count, analyzer.ExpectedReportItems.Count);
analyzer = new FakeAnalyzer(CreateMockParser(2), 3);
analyzer.Run();
Assert.AreNotEqual(analyzer.ActualReportItems.Count, analyzer.ExpectedReportItems.Count);
}
/// <summary> Constructs with default analyzer and a map of analyzers to use for
/// specific fields.
///
/// </summary>
/// <param name="defaultAnalyzer">Any fields not specifically
/// defined to use a different analyzer will use the one provided here.
/// </param>
/// <param name="fieldAnalyzers">a Map (String field name to the Analyzer) to be
/// used for those fields
/// </param>
public PerFieldAnalyzerWrapper(Analyzer defaultAnalyzer, IEnumerable<KeyValuePair<string, Analyzer>> fieldAnalyzers)
{
this.defaultAnalyzer = defaultAnalyzer;
if (fieldAnalyzers != null)
{
foreach(var entry in fieldAnalyzers)
analyzerMap[entry.Key] = entry.Value;
}
SetOverridesTokenStreamMethod<PerFieldAnalyzerWrapper>();
}
/// <summary> Parses a query, searching on the fields specified. Use this if you need
/// to specify certain fields as required, and others as prohibited.
/// <p/>
/// Usage:
/// <code>
/// String[] query = {"query1", "query2", "query3"};
/// String[] fields = {"filename", "contents", "description"};
/// BooleanClause.Occur[] flags = {BooleanClause.Occur.SHOULD,
/// BooleanClause.Occur.MUST,
/// BooleanClause.Occur.MUST_NOT};
/// MultiFieldQueryParser.parse(query, fields, flags, analyzer);
/// </code>
/// <p/>
/// The code above would construct a query:
///
/// <code>
/// (filename:query1) +(contents:query2) -(description:query3)
/// </code>
///
/// </summary>
/// <param name="matchVersion">Lucene version to match; this is passed through to
/// QueryParser.
/// </param>
/// <param name="queries">Queries string to parse
/// </param>
/// <param name="fields">Fields to search on
/// </param>
/// <param name="flags">Flags describing the fields
/// </param>
/// <param name="analyzer">Analyzer to use
/// </param>
/// <throws> ParseException </throws>
/// <summary> if query parsing fails
/// </summary>
/// <throws> IllegalArgumentException </throws>
/// <summary> if the length of the queries, fields, and flags array differ
/// </summary>
public static Query Parse(Version matchVersion, System.String[] queries, System.String[] fields, Occur[] flags, Analyzer analyzer)
{
if (!(queries.Length == fields.Length && queries.Length == flags.Length))
{
throw new System.ArgumentException("queries, fields, and flags array have have different length");
}
BooleanQuery bQuery = new BooleanQuery();
for (int i = 0; i < fields.Length; i++)
{
QueryParser qp = new QueryParser(matchVersion, fields[i], analyzer);
Query q = qp.Parse(queries[i]);
if (q != null && (!(q is BooleanQuery) || ((BooleanQuery)q).GetClauses().Length > 0))
{
bQuery.Add(q, flags[i]);
}
}
return(bQuery);
}
public abstract Analyzer GenerateAnalyzerForQuerying(string indexName, string query, Analyzer previousAnalyzer);
/// <summary>
/// Parses a query, searching on the fields specified. Use this if you need
/// to specify certain fields as required, and others as prohibited.
/// <para/>
/// Usage:
/// <code>
/// string[] query = {"query1", "query2", "query3"};
/// string[] fields = {"filename", "contents", "description"};
/// Occur[] flags = {Occur.SHOULD,
/// Occur.MUST,
/// Occur.MUST_NOT};
/// MultiFieldQueryParser.Parse(query, fields, flags, analyzer);
/// </code>
/// <para/>
/// The code above would construct a query:
///
/// <code>
/// (filename:query1) +(contents:query2) -(description:query3)
/// </code>
///
/// </summary>
/// <param name="matchVersion">Lucene version to match; this is passed through to
/// <see cref="QueryParser"/>.</param>
/// <param name="queries">Queries string to parse</param>
/// <param name="fields">Fields to search on</param>
/// <param name="flags">Flags describing the fields</param>
/// <param name="analyzer">Analyzer to use</param>
/// <exception cref="ParseException">if query parsing fails</exception>
/// <exception cref="ArgumentException">if the length of the queries, fields, and flags array differ</exception>
public static Query Parse(LuceneVersion matchVersion, string[] queries, string[] fields, Occur[] flags, Analyzer analyzer)
{
// LUCENENET: Added null guard clauses
if (queries is null)
{
throw new ArgumentNullException(nameof(queries));
}
if (fields is null)
{
throw new ArgumentNullException(nameof(fields));
}
if (flags is null)
{
throw new ArgumentNullException(nameof(flags));
}
if (!(queries.Length == fields.Length && queries.Length == flags.Length))
{
throw new ArgumentException("queries, fields, and flags array have have different length");
}
BooleanQuery bQuery = new BooleanQuery();
for (int i = 0; i < fields.Length; i++)
{
QueryParser qp = new QueryParser(matchVersion, fields[i], analyzer);
Query q = qp.Parse(queries[i]);
if (q != null && (!(q is BooleanQuery booleanQuery) || booleanQuery.Clauses.Count > 0))
{
bQuery.Add(q, flags[i]);
}
}
return(bQuery);
}
/// <summary>
/// Creates a new <see cref="ExtendableQueryParser"/> instance
/// </summary>
/// <param name="matchVersion">the lucene version to use.</param>
/// <param name="f">the default query field</param>
/// <param name="a">the analyzer used to find terms in a query string</param>
/// <param name="ext">the query parser extensions</param>
public ExtendableQueryParser(LuceneVersion matchVersion, string f, Analyzer a, Extensions ext)
: base(matchVersion, f, a)
{
this.defaultField = f;
this.extensions = ext;
}
/// <summary>
/// Constructor to allow for creating an indexer at runtime
/// </summary>
/// <param name="luceneDirectory"></param>
/// <param name="dataService"></param>
/// <param name="analyzer"></param>
/// <param name="async"></param>
public PDFIndexer(Lucene.Net.Store.Directory luceneDirectory, IDataService dataService, Analyzer analyzer, bool async)
: base(
new IndexCriteria(Enumerable.Empty <IIndexField>(), Enumerable.Empty <IIndexField>(), Enumerable.Empty <string>(), Enumerable.Empty <string>(), null),
luceneDirectory, dataService, analyzer, async)
{
SupportedExtensions = new[] { ".pdf" };
UmbracoFileProperty = "umbracoFile";
}
/// <summary>
/// Creates a new config that with defaults that match the specified
/// <see cref="LuceneVersion"/> as well as the default
/// <see cref="Analyzer"/>. If <paramref name="matchVersion"/> is >=
/// <see cref="LuceneVersion.LUCENE_32"/>, <see cref="TieredMergePolicy"/> is used
/// for merging; else <see cref="LogByteSizeMergePolicy"/>.
/// Note that <see cref="TieredMergePolicy"/> is free to select
/// non-contiguous merges, which means docIDs may not
/// remain monotonic over time. If this is a problem you
/// should switch to <see cref="LogByteSizeMergePolicy"/> or
/// <see cref="LogDocMergePolicy"/>.
/// </summary>
public IndexWriterConfig(LuceneVersion matchVersion, Analyzer analyzer)
: base(analyzer, matchVersion)
{
}
public ShingleAnalyzerWrapper(Analyzer defaultAnalyzer, int maxShingleSize)
: this(defaultAnalyzer)
{
this.maxShingleSize = maxShingleSize;
}
/// <summary>
/// Constructor to allow for creating an indexer at runtime
/// </summary>
/// <param name="indexerData"></param>
/// <param name="indexPath"></param>
/// <param name="dataService"></param>
/// <param name="analyzer"></param>
/// <param name="async"></param>
public AzureContentIndexer(IIndexCriteria indexerData, DirectoryInfo indexPath, IDataService dataService, Analyzer analyzer, bool async)
: base(indexerData, indexPath, dataService, analyzer, async)
{
}
public SnLucParser()
{
//_analyzers = ContentRepository.Storage.StorageContext.Search.SearchEngine.GetAnalyzers();
_masterAnalyzer = IndexManager.GetAnalyzer();
}
private static Query BuildGeneralQuery(
bool doExactId,
string originalSearchText,
Analyzer analyzer,
IEnumerable <NuGetSearchTerm> generalTerms,
IEnumerable <Query> generalQueries)
{
// All terms in the multi-term query appear in at least one of the target fields.
var conjuctionQuery = new BooleanQuery();
conjuctionQuery.Boost = 2.0f;
// Some terms in the multi-term query appear in at least one of the target fields.
var disjunctionQuery = new BooleanQuery();
disjunctionQuery.Boost = 0.1f;
// Suffix wildcard search e.g. jquer*
var wildCardQuery = new BooleanQuery();
wildCardQuery.Boost = 0.5f;
string escapedExactId = originalSearchText.ToLowerInvariant();
Query exactIdQuery = null;
Query wildCardIdQuery = null;
if (doExactId)
{
exactIdQuery = new TermQuery(new Term("Id-Exact", escapedExactId));
exactIdQuery.Boost = 7.5f;
wildCardIdQuery = new WildcardQuery(new Term("Id-Exact", "*" + escapedExactId + "*"));
}
Query nearlyExactIdQuery = null;
if (generalTerms.Any())
{
string escapedApproximateId = string.Join(" ", generalTerms.Select(c => c.TermOrPhrase));
nearlyExactIdQuery = AnalysisHelper.GetFieldQuery(analyzer, "Id", escapedApproximateId);
nearlyExactIdQuery.Boost = 2.0f;
}
foreach (var termQuery in generalQueries)
{
conjuctionQuery.Add(termQuery, Occur.MUST);
disjunctionQuery.Add(termQuery, Occur.SHOULD);
}
var sanitizedTerms = generalTerms.Select(c => c.TermOrPhrase.ToLowerInvariant());
foreach (var sanitizedTerm in sanitizedTerms)
{
foreach (var field in Fields)
{
var wildCardTermQuery = new WildcardQuery(new Term(field, sanitizedTerm + "*"));
wildCardTermQuery.Boost = 0.7f;
wildCardQuery.Add(wildCardTermQuery, Occur.SHOULD);
}
}
// OR of all the applicable queries
var queries = new Query[]
{
exactIdQuery, wildCardIdQuery, nearlyExactIdQuery, conjuctionQuery, disjunctionQuery, wildCardQuery
};
var queriesToCombine = queries.Where(q => !IsDegenerateQuery(q));
var query = conjuctionQuery.Combine(queriesToCombine.ToArray());
return(query);
}
public PostingsHighlighterAnalyzerHelper(Analyzer analyzer)
{
this.analyzer = analyzer;
}
public SearchEngineService(Directory directory, Analyzer analyzer)
{
_directory = directory;
_analyzer = analyzer;
}
/// <summary>
/// Creates a MultiFieldQueryParser.
///
/// <para/>
/// It will, when <see cref="QueryParserBase.Parse(string)"/> is called, construct a query like this
/// (assuming the query consists of two terms and you specify the two fields
/// <c>title</c> and <c>body</c>):
/// <para/>
///
/// <code>
/// (title:term1 body:term1) (title:term2 body:term2)
/// </code>
///
/// <para/>
/// When <see cref="QueryParserBase.DefaultOperator"/> is set to <see cref="QueryParserBase.AND_OPERATOR"/>, the result will be:
/// <para/>
///
/// <code>
/// +(title:term1 body:term1) +(title:term2 body:term2)
/// </code>
///
/// <para/>
/// In other words, all the query's terms must appear, but it doesn't matter
/// in what fields they appear.
/// <para/>
/// </summary>
public MultiFieldQueryParser(LuceneVersion matchVersion, string[] fields, Analyzer analyzer)
: base(matchVersion, null, analyzer)
{
this.m_fields = fields;
}
protected BaseLuceneSearcher(Analyzer analyzer)
{
IndexingAnalyzer = analyzer;
}
/// <summary>
/// Creates a new <see cref="ExtendableQueryParser"/> instance
/// </summary>
/// <param name="matchVersion">the lucene version to use.</param>
/// <param name="f">the default query field</param>
/// <param name="a">the analyzer used to find terms in a query string</param>
public ExtendableQueryParser(LuceneVersion matchVersion, string f, Analyzer a)
: this(matchVersion, f, a, DEFAULT_EXTENSION)
{
}
public ShingleAnalyzerWrapper(Analyzer defaultAnalyzer)
{
this.defaultAnalyzer = defaultAnalyzer;
SetOverridesTokenStreamMethod <ShingleAnalyzerWrapper>();
}
/// <summary> Creates a MultiFieldQueryParser.
///
/// <p/>
/// It will, when parse(String query) is called, construct a query like this
/// (assuming the query consists of two terms and you specify the two fields
/// <c>title</c> and <c>body</c>):
/// <p/>
///
/// <code>
/// (title:term1 body:term1) (title:term2 body:term2)
/// </code>
///
/// <p/>
/// When setDefaultOperator(AND_OPERATOR) is set, the result will be:
/// <p/>
///
/// <code>
/// +(title:term1 body:term1) +(title:term2 body:term2)
/// </code>
///
/// <p/>
/// In other words, all the query's terms must appear, but it doesn't matter
/// in what fields they appear.
/// <p/>
/// </summary>
public MultiFieldQueryParser(Version matchVersion, System.String[] fields, Analyzer analyzer)
: base(matchVersion, null, analyzer)
{
this.fields = fields;
}
public void TestContainsCapture()
{
var RuleName = "Contains Capture";
var containsCaptureRule = new Rule(RuleName)
{
Clauses = new List <Clause>()
{
new Clause(Operation.Contains)
{
Data = new List <string>()
{
"Magic",
"Words"
},
Capture = true
}
}
};
var analyzer = new Analyzer();
var ruleList = new List <Rule>()
{
containsCaptureRule
};
var res = analyzer.GetCapture(containsCaptureRule, "ThisStringContainsSomeMagicWords", null);
Assert.IsTrue(res.Result?.Captures.Any(x => x is TypedClauseCapture <string> y && y.Result == "ThisStringContainsSomeMagicWords") is true);
res = analyzer.GetCapture(containsCaptureRule, new List <string>()
{
"Magic", "Words"
}, null);
Assert.IsTrue(res.Result?.Captures.Any(x => x is TypedClauseCapture <List <string> > y && y.Result.Contains("Magic")) is true);
res = analyzer.GetCapture(containsCaptureRule, Words.Magic | Words.Words, null);
Assert.IsTrue(res.Result?.Captures.Any(x => x is TypedClauseCapture <Enum> y && y.Result.HasFlag(Words.Magic)) is true);
containsCaptureRule = new Rule(RuleName)
{
Clauses = new List <Clause>()
{
new Clause(Operation.Contains)
{
Data = new List <string>()
{
"Magic",
"Words"
},
Capture = true,
Invert = true
}
}
};
res = analyzer.GetCapture(containsCaptureRule, "ThisStringHasNothing", null);
Assert.IsTrue(res.Result?.Captures.Any(x => x is TypedClauseCapture <string> y && y.Result == "ThisStringHasNothing") is true);
res = analyzer.GetCapture(containsCaptureRule, new List <string>()
{
"None", "Null"
}, null);
Assert.IsTrue(res.Result?.Captures.Any(x => x is TypedClauseCapture <List <string> > y && y.Result.Contains("Null")) is true);
res = analyzer.GetCapture(containsCaptureRule, Words.None, null);
Assert.IsTrue(res.Result?.Captures.Any(x => x is TypedClauseCapture <Enum> y && y.Result.HasFlag(Words.None)) is true);
var testdata = new Dictionary <string, string>()
{
{ "Version", "1.0" }, { "State", "Beta" }
};
containsCaptureRule = new Rule(RuleName)
{
Clauses = new List <Clause>()
{
new Clause(Operation.Contains)
{
DictData = testdata.ToList(),
Capture = true
}
}
};
res = analyzer.GetCapture(containsCaptureRule, testdata, null);
Assert.IsTrue(res.Result?.Captures.Any(x => x is TypedClauseCapture <List <KeyValuePair <string, string> > > y && y.Result.Any(x => x.Key == "Version")) is true);
var testlist = testdata.ToList();
res = analyzer.GetCapture(containsCaptureRule, testlist, null);
Assert.IsTrue(res.Result?.Captures.Any(x => x is TypedClauseCapture <List <KeyValuePair <string, string> > > y && y.Result.Any(x => x.Key == "Version")) is true);
containsCaptureRule = new Rule(RuleName)
{
Clauses = new List <Clause>()
{
new Clause(Operation.Contains)
{
DictData = testlist,
Capture = true,
Invert = true
}
}
};
testdata = new Dictionary <string, string>()
{
{ "Something", "Else" }, { "Other", "Keys" }
};
res = analyzer.GetCapture(containsCaptureRule, testdata, null);
Assert.IsTrue(res.Result?.Captures.Any(x => x is TypedClauseCapture <List <KeyValuePair <string, string> > > y && y.Result.Any(x => x.Key == "Something")) is true);
testlist = testdata.ToList();
res = analyzer.GetCapture(containsCaptureRule, testlist, null);
Assert.IsTrue(res.Result?.Captures.Any(x => x is TypedClauseCapture <List <KeyValuePair <string, string> > > y && y.Result.Any(x => x.Key == "Other")) is true);
}
private static ReflectionFieldMapper <T> BuildPrimitive <T>(PropertyInfo p, Type type, FieldAttribute metadata, Version version, Analyzer externalAnalyzer)
{
var fieldName = (metadata != null ? metadata.Field : null) ?? p.Name;
var converter = GetConverter(p, type, metadata);
var store = metadata != null ? metadata.Store : StoreMode.Yes;
var index = metadata != null ? metadata.IndexMode : IndexMode.Analyzed;
var termVectorMode = metadata != null ? metadata.TermVector : TermVectorMode.No;
var boost = metadata != null ? metadata.Boost : 1.0f;
var defaultParserOperator = metadata != null ? metadata.DefaultParserOperator : QueryParsers.QueryParser.Operator.OR;
var caseSensitive = GetCaseSensitivity(metadata, converter);
var analyzer = BuildAnalyzer(metadata, converter, version) ?? externalAnalyzer; // ?? BuildAnalyzer(metadata, converter, version);
return(new ReflectionFieldMapper <T>(p, store, index, termVectorMode, converter, fieldName, defaultParserOperator, caseSensitive, analyzer, boost));
}
/// <summary> Creates a MultiFieldQueryParser. Allows passing of a map with term to
/// Boost, and the boost to apply to each term.
///
/// <p/>
/// It will, when parse(String query) is called, construct a query like this
/// (assuming the query consists of two terms and you specify the two fields
/// <c>title</c> and <c>body</c>):
/// <p/>
///
/// <code>
/// (title:term1 body:term1) (title:term2 body:term2)
/// </code>
///
/// <p/>
/// When setDefaultOperator(AND_OPERATOR) is set, the result will be:
/// <p/>
///
/// <code>
/// +(title:term1 body:term1) +(title:term2 body:term2)
/// </code>
///
/// <p/>
/// When you pass a boost (title=>5 body=>10) you can get
/// <p/>
///
/// <code>
/// +(title:term1^5.0 body:term1^10.0) +(title:term2^5.0 body:term2^10.0)
/// </code>
///
/// <p/>
/// In other words, all the query's terms must appear, but it doesn't matter
/// in what fields they appear.
/// <p/>
/// </summary>
public MultiFieldQueryParser(Version matchVersion, string[] fields, Analyzer analyzer, IDictionary <string, float> boosts)
: this(matchVersion, fields, analyzer)
{
this.boosts = boosts;
}
/*
* Wraps {@link StandardAnalyzer}.
*/
public ShingleAnalyzerWrapper(Version matchVersion)
{
this.defaultAnalyzer = new StandardAnalyzer(matchVersion);
SetOverridesTokenStreamMethod <ShingleAnalyzerWrapper>();
}
internal override void Analyze(Analyzer /*!*/ analyzer)
{
base.Analyze(analyzer);
valueExpr = valueExpr.Analyze(analyzer, ExInfoFromParent.DefaultExInfo).Literalize();
}
public TermsFilterBuilder(Analyzer analyzer)
{
this.analyzer = analyzer;
}
public FuzzyLikeThisQueryBuilder(Analyzer analyzer)
{
this.analyzer = analyzer;
}
public override Analyzer GenerateAnalyzerForQuerying(string indexName, string query, Analyzer previousAnalyzer)
{
if (indexName == SpecificIndexName)
{
return(new WhitespaceAnalyzer());
}
return(previousAnalyzer);
}
/// <summary>
/// Detects untokenized fields and sets as NotAnalyzed in analyzer
/// </summary>
private static string PreProcessUntokenizedTerms(PerFieldAnalyzerWrapper analyzer, string query, Analyzer keywordAnlyzer)
{
var untokenizedMatches = untokenizedQuery.Matches(query);
if (untokenizedMatches.Count < 1)
{
return(query);
}
var sb = new StringBuilder(query);
// KeywordAnalyzer will not tokenize the values
// process in reverse order to leverage match string indexes
for (int i = untokenizedMatches.Count; i > 0; i--)
{
Match match = untokenizedMatches[i - 1];
// specify that term for this field should not be tokenized
analyzer.AddAnalyzer(match.Groups[1].Value, keywordAnlyzer);
Group term = match.Groups[2];
// remove enclosing "[[" "]]" from term value (again in reverse order)
sb.Remove(term.Index + term.Length - 2, 2);
sb.Remove(term.Index, 2);
}
return(sb.ToString());
}
public override Analyzer GenerateAnalyzerForIndexing(string indexName, Document document, Analyzer previousAnalyzer)
{
if (indexName == SpecificIndexName)
{
return(new WhitespaceAnalyzer());
}
return(previousAnalyzer);
}
public void TestContainsAnyCapture()
{
var RuleName = "Contains Any Capture";
var containsAnyCaptureRule = new Rule(RuleName)
{
Clauses = new List <Clause>()
{
new Clause(Operation.ContainsAny)
{
Data = new List <string>()
{
"Magic",
"Words",
"Kaboom"
},
Capture = true
}
}
};
var analyzer = new Analyzer();
var res = analyzer.GetCapture(containsAnyCaptureRule, "ThisStringContainsSomeMagic", null);
Assert.IsTrue(res.Result?.Captures.Any(x => x is TypedClauseCapture <string> y && y.Result == "ThisStringContainsSomeMagic") is true);
res = analyzer.GetCapture(containsAnyCaptureRule, new List <string>()
{
"Magic", "Words"
}, null);
Assert.IsTrue(res.Result?.Captures.Any(x => x is TypedClauseCapture <List <string> > y && y.Result.Contains("Magic")) is true);
res = analyzer.GetCapture(containsAnyCaptureRule, Words.Magic | Words.Words, null);
Assert.IsTrue(res.Result?.Captures.Any(x => x is TypedClauseCapture <Enum> y && y.Result.HasFlag(Words.Magic)) is true);
var testdata = new Dictionary <string, string>()
{
{ "Version", "1.0" }, { "State", "Beta" }
};
containsAnyCaptureRule = new Rule(RuleName)
{
Clauses = new List <Clause>()
{
new Clause(Operation.ContainsAny)
{
DictData = testdata.ToList(),
Capture = true
}
}
};
testdata.Remove("State");
res = analyzer.GetCapture(containsAnyCaptureRule, testdata, null);
Assert.IsTrue(res.Result?.Captures.Any(x => x is TypedClauseCapture <List <KeyValuePair <string, string> > > y && y.Result.Any(x => x.Key == "Version")) is true);
var testlist = testdata.ToList();
res = analyzer.GetCapture(containsAnyCaptureRule, testlist, null);
Assert.IsTrue(res.Result?.Captures.Any(x => x is TypedClauseCapture <List <KeyValuePair <string, string> > > y && y.Result.Any(x => x.Key == "Version")) is true);
containsAnyCaptureRule = new Rule(RuleName)
{
Clauses = new List <Clause>()
{
new Clause(Operation.ContainsAny)
{
Data = new List <string>()
{
"Magic",
"Words",
"Kaboom"
},
Capture = true,
Invert = true
}
}
};
res = analyzer.GetCapture(containsAnyCaptureRule, "ThisStringIsn'tSpecial", null);
Assert.IsTrue(res.Result?.Captures.Any(x => x is TypedClauseCapture <string> y && y.Result == "ThisStringIsn'tSpecial") is true);
res = analyzer.GetCapture(containsAnyCaptureRule, new List <string>()
{
"Pizza", "Taco"
}, null);
Assert.IsTrue(res.Result?.Captures.Any(x => x is TypedClauseCapture <List <string> > y && y.Result.Contains("Taco")) is true);
res = analyzer.GetCapture(containsAnyCaptureRule, Words.None, null);
Assert.IsTrue(res.Result?.Captures.Any(x => x is TypedClauseCapture <Enum> y && y.Result.HasFlag(Words.None)) is true);
testdata = new Dictionary <string, string>()
{
{ "Version", "1.0" }, { "State", "Beta" }
};
containsAnyCaptureRule = new Rule(RuleName)
{
Clauses = new List <Clause>()
{
new Clause(Operation.ContainsAny)
{
DictData = testdata.ToList(),
Capture = true,
Invert = true
}
}
};
testdata = new Dictionary <string, string>()
{
{ "Not matching", "1.0" }, { "Status", "Beta" }
};
res = analyzer.GetCapture(containsAnyCaptureRule, testdata, null);
Assert.IsTrue(res.Result?.Captures.Any(x => x is TypedClauseCapture <List <KeyValuePair <string, string> > > y && y.Result.Any(x => x.Key == "Status")) is true);
testlist = testdata.ToList();
res = analyzer.GetCapture(containsAnyCaptureRule, testlist, null);
Assert.IsTrue(res.Result?.Captures.Any(x => x is TypedClauseCapture <List <KeyValuePair <string, string> > > y && y.Result.Any(x => x.Key == "Status")) is true);
}
public abstract Analyzer GenerateAnalyzerForIndexing(string indexName, Document document, Analyzer previousAnalyzer);
public PostingsHighlighterAnalyzerAndFormatterHelper(Analyzer analyzer, PassageFormatter formatter)
: base(analyzer)
{
this.formatter = formatter;
}
