public void TestSimpleReductionParsing()
{
string sql = new StringBuilder()
.AppendLine("SELECT")
.AppendLine(" TOP 10 *")
.AppendLine("FROM")
.AppendLine(" Posts")
.ToString();
var query = new ParsedQuery(sql, null);
Assert.AreEqual("SELECT\nTOP 10 *\nFROM\nPosts", query.ExecutionSql);
}
public void TestBatchSplittingIgnoresEmptyBatches()
{
string sql = new StringBuilder()
.AppendLine("SELECT 1")
.AppendLine("GO")
.ToString();
var query = new ParsedQuery(sql, null);
var batches = query.ExecutionSqlBatches.ToArray();
Assert.AreEqual(1, batches.Length);
Assert.AreEqual("SELECT 1", batches[0]);
}
public void TestStringEncoding()
{
string sql = "SELECT * FROM Users WHERE Login = ##UserName:string##";
var parameters = new NameValueCollection
{
{ "UserName", "I'm a User's Name" }
};
var query = new ParsedQuery(sql, parameters);
Assert.AreEqual("SELECT * FROM Users WHERE Login = '******''m a User''s Name'", query.ExecutionSql);
Assert.IsTrue(query.IsExecutionReady);
}
private static void ValidateQuery(ParsedQuery query, Site site)
{
if (!query.IsExecutionReady)
{
throw new ApplicationException(
query.ErrorMessage.IsNullOrEmptyReturn("All parameters must be set!")
);
}
if (site == null)
{
throw new ApplicationException("Invalid site ID");
}
}
public void TestWeDetectMissingParameterValues()
{
string sql = "##a## ##b##";
var parameters = new NameValueCollection
{
{ "a", "1" },
{ "b", "" }
};
var query = new ParsedQuery(sql, parameters);
Assert.IsFalse(query.IsExecutionReady);
Assert.AreEqual("Missing value for b!", query.Errors[0]);
}
public void TestInvalidIntParameters()
{
string sql = "##a:int## ##b##";
var parameters = new NameValueCollection
{
{ "a", "hello" },
{ "b", "3" }
};
var query = new ParsedQuery(sql, parameters);
Assert.AreEqual("Expected value of a to be a int!", query.ErrorMessage);
Assert.IsFalse(query.IsExecutionReady);
}
private static QueryResults TranslateResults(ParsedQuery query, bool textResults, QueryResults results)
{
textResults = textResults || (results.ResultSets.Count != 1);
if (textResults)
{
results = results.ToTextResults();
}
if (query.IncludeExecutionPlan)
{
results = results.TransformQueryPlan();
}
return(results);
}
public void TestInvalidParameterType()
{
string sql = "##a:frog##";
var parameters = new NameValueCollection
{
{ "a", "thingadongdong" }
};
var query = new ParsedQuery(sql, parameters);
Assert.IsTrue(query.Parameters.ContainsKey("a"));
Assert.AreEqual("a has unknown parameter type frog!", query.ErrorMessage);
Assert.IsFalse(query.IsExecutionReady);
}
public void TestIntParameters()
{
string sql = "##a:int## ##b##";
var parameters = new NameValueCollection
{
{ "a", "1" },
{ "b", "3" }
};
var query = new ParsedQuery(sql, parameters);
Assert.AreEqual("int", query.Parameters["a"].Type);
Assert.AreEqual("1 3", query.ExecutionSql);
Assert.IsTrue(query.IsExecutionReady);
}
public void TestWeDetectAllParameters()
{
string sql = "##a## ##b##";
var parameters = new NameValueCollection
{
{ "a", "1" },
{ "b", "3" }
};
var query = new ParsedQuery(sql, parameters);
Assert.IsTrue(query.Parameters.ContainsKey("a"));
Assert.IsTrue(query.Parameters.ContainsKey("b"));
Assert.IsTrue(query.IsExecutionReady);
}
public QueryPlan BuildExecutionPlan(ParsedQuery parsedQuery, bool freeCache)
{
QueryPlan characterPlan = _charactersIndex.BuildExecutionPlan(new ParsedQuery(words: parsedQuery.Words), freeCache);
if (string.IsNullOrEmpty(parsedQuery.Numbers))
{
return(characterPlan);
}
QueryPlan numberPlan = _numberIndex.BuildExecutionPlan(new ParsedQuery(numbers: parsedQuery.Numbers), freeCache);
if (string.IsNullOrEmpty(parsedQuery.Words))
{
return(numberPlan);
}
return(QueryPlan.And(characterPlan, numberPlan));
}
public void TestValidFloats()
{
string sql = "##a:float## ##b:float##";
var parameters = new NameValueCollection
{
{ "a", "1" },
{ "b", "1.2" }
};
var query = new ParsedQuery(sql, parameters);
Assert.AreEqual("float", query.Parameters["a"].Type);
Assert.AreEqual(query.ExecutionSql, "1 1.2");
Assert.IsTrue(query.IsExecutionReady);
}
public void TestFailingQuery()
{
string sql = new StringBuilder()
.AppendLine("Select 'Select ''' + ''' As [DatabaseName], Case WHEN TagName IS NULL Then" +
" ''' + 'Null' + ''' WHEN TagName = '''' Then ''Empty'' ELSE ''Unexpected'' End As " +
"[Type], Count(*) As [Count] from ' ")
.AppendLine(" + '[' + ']' + '..Tags Where IsNull(TagName,'''')='''' Group By TagName UNION'")
.ToString();
var query = new ParsedQuery(sql, null);
Assert.AreEqual("Select 'Select ''' + ''' As [DatabaseName], Case WHEN TagName IS NULL Then '''" +
" + 'Null' + ''' WHEN TagName = '''' Then ''Empty'' ELSE ''Unexpected'' End As [Type]," +
" Count(*) As [Count] from '\n+ '[' + ']' + '..Tags Where IsNull(TagName,'''')='''' Group" +
" By TagName UNION'", query.ExecutionSql);
}
public ActionResult ShowSingleSiteCsv(string sitename, int revisionId, string slug)
{
Query query = QueryUtil.GetQueryForRevision(revisionId);
if (query == null)
{
return(PageNotFound());
}
Site site;
if (!TryGetSite(sitename, out site))
{
return(site == null ? (ActionResult)PageNotFound() : RedirectPermanent(string.Format("/{0}/csv/{1}{2}{3}",
site.TinyName.ToLower(), revisionId, slug.HasValue() ? "/" + slug : "", Request.Url.Query
)));
}
var parsedQuery = new ParsedQuery(query.QueryBody, Request.Params);
if (!parsedQuery.IsExecutionReady)
{
return(PageBadRequest());
}
CachedResult cachedResults = QueryUtil.GetCachedResults(
parsedQuery,
Site.Id
);
List <ResultSet> resultSets;
if (cachedResults != null)
{
resultSets = JsonConvert.DeserializeObject <List <ResultSet> >(cachedResults.Results, QueryResults.GetSettings());
}
else
{
resultSets = QueryRunner.GetResults(
parsedQuery,
site,
CurrentUser
).ResultSets;
}
return(new CsvResult(resultSets));
}
public static VisitedQuery VisitStroke(this ParsedQuery q, Func <Type, bool> isEntity)
{
List <SqlQueryExpression> positioned = new List <SqlQueryExpression>();
var visitor = new StrokeVisitor(q.InitialTableReferences, isEntity);
foreach (var pe in q.Arguments)
{
visitor.Visit(pe.Expression);
var sql = visitor.Retrieve();
sql.IsTop = true;
sql.Position = pe.Position;
sql.Index = pe.Index;
positioned.Add(sql);
}
return(new VisitedQuery(q.QueryStructure, positioned.ToArray(), visitor.UsedTypes));
}
private void SetParameters(ParsedQuery query)
{
if (query.Named)
{
foreach (var name in query.ParameterNames.Distinct())
{
_command.AddParameter(name, ParameterSource[name]);
}
}
else
{
for (var i = 0; i < query.ParameterNames.Count; i++)
{
var name = query.ParameterNames[i];
_command.AddParameter(i.ToString(), ParameterSource[name]);
}
}
}
public void TestWeParseParametersInMultiLineStrings()
{
string sql = new StringBuilder()
.AppendLine("SELECT * FROM Posts WHERE Body LIKE '%")
.AppendLine("##SearchTerm##'")
.ToString();
var parameters = new NameValueCollection
{
{ "SearchTerm", "foobar" }
};
var query = new ParsedQuery(sql, parameters);
Assert.IsTrue(query.Parameters.ContainsKey("SearchTerm"));
Assert.AreEqual("SELECT * FROM Posts WHERE Body LIKE '%\nfoobar'", query.ExecutionSql);
Assert.IsTrue(query.IsExecutionReady);
}
public void TestCommentReductionParsing()
{
string sql = new StringBuilder()
.AppendLine("-- A single line comment")
.AppendLine("SELECT")
.AppendLine(" TOP 10 * -- We only want the top 10")
.AppendLine("FROM")
.AppendLine("/* Posts */")
.AppendLine("/*")
.AppendLine(" Comments")
.AppendLine(" */")
.AppendLine(" Users")
.ToString();
var query = new ParsedQuery(sql, null);
Assert.AreEqual("SELECT\nTOP 10 *\nFROM\nUsers", query.ExecutionSql);
}
public ActionResult ShowSingleSiteCsv(string sitename, int revisionId)
{
Query query = QueryUtil.GetQueryForRevision(revisionId);
if (query == null)
{
return(PageNotFound());
}
var site = GetSite(sitename);
if (sitename == null)
{
return(PageNotFound());
}
var parsedQuery = new ParsedQuery(query.QueryBody, Request.Params);
if (!parsedQuery.IsExecutionReady)
{
return(PageBadRequest());
}
CachedResult cachedResults = QueryUtil.GetCachedResults(
parsedQuery,
Site.Id
);
List <ResultSet> resultSets;
if (cachedResults != null)
{
resultSets = JsonConvert.DeserializeObject <List <ResultSet> >(cachedResults.Results);
}
else
{
resultSets = QueryRunner.GetResults(
parsedQuery,
site,
CurrentUser
).ResultSets;
}
return(new CsvResult(resultSets));
}
public static void Main(string[] args)
{
if (args.Length < 2)
{
Console.WriteLine("Usage: IndexStats index-dir action [args]");
Environment.Exit(1);
}
var index = Index.Load(args[0]);
var action = args[1];
if (action == "termDocs")
{
var term = args[2];
var query = new ParsedQuery {
TermsWithSynonyms = new[] { new[] { term } }
};
Console.WriteLine(string.Join(" ", index.GetPostingsByQuery(query).Select(x => x.PostingId.ToString()).ToArray()));
}
else if (action == "playGame")
{
var urls = File.ReadAllLines("urls.txt");
using (var output = new StreamWriter("ranking.txt"))
{
var reqNum = 0;
foreach (var req in File.ReadAllLines("requests.txt"))
{
var parsedQuery = RawQueryParser.Parse(req, null, null, null, new MorphologyAnalyzer("127.0.0.1"));
var results = index
.GetPostingsByQuery(parsedQuery)
.Rank()
.Take(10);
var pos = 0;
foreach (var res in results)
{
output.WriteLine("{0} {1} {2}", reqNum, pos, urls[res.PostingId - 1]);
pos++;
}
reqNum++;
}
}
}
}
private Hit ConvertDocumentIdToHit(WeightedPosting posting, ParsedQuery query)
{
string formattedDocumentId = string.Format("{0:D6}", posting.PostingId);
var document = new Document(formattedDocumentId);
var snippet = new SnippetGenerator(query, document).GenerateSnippet();
return
(new Hit
{
DocumentId = formattedDocumentId,
Description = GetDescription(posting.PostingId, string.Format("№{0}", formattedDocumentId)),
DebugInfo =
new[]
{
string.Format("tf-idf = {0:F5} = {1}", Ranker.GetTfIdf(posting), FormatTfIdf(posting)),
string.Format("ExactWordMatchCount = {0}", posting.ExactWordMatchCount)
},
Snippet = string.Join(" … ", snippet.Skip(1).ToArray()),
Title = snippet.First(),
});
}
public void TestInvalidFloats()
{
string sql = "##a:float## ##b:float## ##c:float## ##d:float##";
var parameters = new NameValueCollection
{
{ "a", "1." },
{ "b", "1.2." },
{ "c", ".2" },
{ "d", "frog" }
};
var query = new ParsedQuery(sql, parameters);
Assert.AreEqual(4, query.Errors.Count);
Assert.AreEqual("Expected value of a to be a float!", query.Errors[0]);
Assert.AreEqual("Expected value of b to be a float!", query.Errors[1]);
Assert.AreEqual("Expected value of c to be a float!", query.Errors[2]);
Assert.AreEqual("Expected value of d to be a float!", query.Errors[3]);
Assert.IsFalse(query.IsExecutionReady);
}
private void RunQuery()
{
lock (_queryLock)
{
if (Database == null)
{
return;
}
if (String.IsNullOrEmpty(Query) && _lastResult.Count == 0)
{
return;
}
Stopwatch w = Stopwatch.StartNew();
ParsedQuery.SymbolName = Query;
ParsedQuery.IgnoreCase = Options.IgnoreCase;
ParsedQuery.TryFindMembers(Database, ref _lastResult);
w.Stop();
QueryStatus = String.Format("{0:n0} matches in {1} for \"{2}\"", _lastResult.Count, w.Elapsed.ToFriendlyString(), Query);
}
}
private IEnumerable <Hit> Search(ParsedQuery query, int page, bool rankResults)
{
var results = Global.Index.GetPostingsByQuery(query);
if (rankResults)
{
results = results.Rank();
}
int resultCount = results.Count();
int pageCount = resultCount / PageSize + 1;
if (resultCount % PageSize == 0)
{
pageCount--;
}
if (page >= pageCount)
{
page = 0;
}
IEnumerable <Hit> pagedResults = results.Skip(PageSize * page).Take(PageSize).Select(x => ConvertDocumentIdToHit(x, query));
if (pagedResults.Count() > 0)
{
lblPage.Visible = hrefPrev.Visible = hrefNext.Visible = true;
int firstResult = page * PageSize + 1;
int lastResult = firstResult + pagedResults.Count() - 1;
listResults.Attributes["start"] = firstResult.ToString();
lblPage.Text = string.Format("Показываем результаты <b>{0}</b>—<b>{1}</b> из <b>{2}</b>", firstResult, lastResult, resultCount);
if (page > 0)
{
BuildUrlForPage(page - 1, hrefPrev);
}
if (page + 1 < pageCount)
{
BuildUrlForPage(page + 1, hrefNext);
}
}
return(pagedResults);
}
public void TestWeIgnoreCommentedParameters()
{
string sql = new StringBuilder()
.AppendLine("SELECT")
.AppendLine(" TOP ##TopCount##")
.AppendLine("FROM")
.AppendLine(" Posts")
.AppendLine("--WHERE UserId == ##UserId##")
.ToString();
var parameters = new NameValueCollection
{
{ "TopCount", "10" },
{ "UserId", "1" }
};
var query = new ParsedQuery(sql, parameters);
Assert.IsTrue(query.Parameters.ContainsKey("TopCount"));
Assert.IsFalse(query.Parameters.ContainsKey("UserId"));
Assert.IsTrue(query.IsExecutionReady);
}
public ActionResult ShowPlan(string sitename, int revisionId, string slug)
{
Query query = QueryUtil.GetQueryForRevision(revisionId);
if (query == null)
{
return(PageNotFound());
}
Site site;
if (!TryGetSite(sitename, out site))
{
return(site == null ? (ActionResult)PageNotFound() : RedirectPermanent(string.Format("/{0}/plan/{1}{2}{3}",
site.TinyName.ToLower(), revisionId, slug.HasValue() ? "/" + slug : "", Request.Url.Query
)));
}
var parsedQuery = new ParsedQuery(query.QueryBody, Request.Params);
if (!parsedQuery.IsExecutionReady)
{
return(PageBadRequest());
}
CachedResult cache = QueryUtil.GetCachedResults(
parsedQuery,
site.Id
);
if (cache == null || cache.ExecutionPlan == null)
{
return(PageNotFound());
}
return(new QueryPlanResult(cache.ExecutionPlan));
}
public void TestCreateProcWithGoReduction()
{
string sql = new StringBuilder()
.AppendLine("CREATE TABLE #Test (id int IDENTITY(1,1) NOT NULL, v VARCHAR(MAX) NOT NULL);")
.AppendLine("go")
.AppendLine("CREATE INDEX #IX_T on #Test(id)")
.AppendLine("go")
.AppendLine("--BEGIN")
.AppendLine("-- RETURN GETDATE()")
.AppendLine("--END")
.AppendLine("CREATE PROCEDURE #T_P2 @v VARCHAR(MAX) AS")
.AppendLine(" PRINT @v")
.AppendLine("RETURN")
.AppendLine("GO")
.AppendLine("#T_P2 'test'")
.ToString();
var query = new ParsedQuery(sql, null);
Assert.IsTrue(query.ExecutionSqlBatches.Any());
Assert.AreEqual("CREATE TABLE #Test (id int IDENTITY(1,1) NOT NULL, v VARCHAR(MAX) NOT NULL);\n" +
"go\nCREATE INDEX #IX_T on #Test(id)\ngo\nCREATE PROCEDURE #T_P2 @v VARCHAR(MAX) AS\nPRINT @v\n" +
"RETURN\nGO\n#T_P2 'test'", query.ExecutionSql);
}
public SnippetGenerator(ParsedQuery query, Document document)
{
this.query = query;
this.document = document;
}
public ActionResult Save(string sql, string title, string description, int siteId, int?querySetId, bool?textResults, bool?withExecutionPlan, TargetSites?targetSites)
{
if (CurrentUser.IsAnonymous && !CaptchaController.CaptchaPassed(GetRemoteIP()))
{
return(Json(new { captcha = true }));
}
ActionResult response = null;
try
{
if (!ValidateTargetSites(targetSites))
{
throw new ApplicationException("Invalid target sites selection");
}
QuerySet querySet = null;
if (querySetId.HasValue)
{
querySet = Current.DB.QuerySets.Get(querySetId.Value);
if (querySet == null)
{
throw new ApplicationException("Invalid query set ID");
}
}
var parsedQuery = new ParsedQuery(
sql,
Request.Params,
withExecutionPlan == true,
targetSites ?? TargetSites.Current
);
QueryResults results = null;
Site site = GetSite(siteId);
ValidateQuery(parsedQuery, site);
if (title.HasValue() && title.Length > 100)
{
throw new ApplicationException("Title must be no more than 100 characters");
}
if (description.HasValue() && description.Length > 1000)
{
throw new ApplicationException("Description must be no more than 1000 characters");
}
var contextData = new QueryContextData
{
Title = title,
Description = description,
IsText = textResults == true,
QuerySet = querySet
};
var asyncResults = AsyncQueryRunner.Execute(parsedQuery, CurrentUser, site, contextData);
if (asyncResults.State == AsyncQueryRunner.AsyncState.Failure)
{
throw asyncResults.Exception;
}
if (asyncResults.State == AsyncQueryRunner.AsyncState.Success || asyncResults.State == AsyncQueryRunner.AsyncState.Cancelled)
{
results = asyncResults.QueryResults;
}
else
{
return(Json(new { running = true, job_id = asyncResults.JobId }));
}
response = CompleteResponse(results, parsedQuery, contextData, siteId);
}
catch (Exception ex)
{
response = TransformExecutionException(ex);
}
return(response);
}
public ActionResult Execute(int querySetId, int revisionId, int siteId, bool?textResults, bool?withExecutionPlan, TargetSites?targetSites)
{
if (CurrentUser.IsAnonymous && !CaptchaController.CaptchaPassed(GetRemoteIP()))
{
return(Json(new { captcha = true }));
}
ActionResult response = null;
try
{
if (!ValidateTargetSites(targetSites))
{
throw new ApplicationException("Invalid target sites selection");
}
QuerySet querySet = null;
querySet = Current.DB.QuerySets.Get(querySetId);
if (querySet == null)
{
throw new ApplicationException("Invalid query set ID");
}
Revision revision = Current.DB.Revisions.Get(revisionId);
if (revision == null)
{
throw new ApplicationException("Invalid revision ID");
}
Query query = Current.DB.Queries.Get(revision.QueryId);
var parsedQuery = new ParsedQuery(
query.QueryBody,
Request.Params,
withExecutionPlan == true,
targetSites ?? TargetSites.Current
);
QueryResults results = null;
Site site = GetSite(siteId);
ValidateQuery(parsedQuery, site);
var contextData = new QueryContextData
{
IsText = textResults == true,
QuerySet = querySet,
Revision = revision
};
var asyncResults = AsyncQueryRunner.Execute(parsedQuery, CurrentUser, site, contextData);
if (asyncResults.State == AsyncQueryRunner.AsyncState.Failure)
{
throw asyncResults.Exception;
}
if (asyncResults.State == AsyncQueryRunner.AsyncState.Success)
{
results = asyncResults.QueryResults;
}
else
{
return(Json(new { running = true, job_id = asyncResults.JobId }));
}
response = CompleteResponse(results, parsedQuery, contextData, siteId);
}
catch (Exception ex)
{
response = TransformExecutionException(ex);
}
return(response);
}