public void TestOrQueryWithFixedPostingList_4()
{
var e = "{[1,1,1]}, {[1,1,3]}";
var left = new TermQuery(PostingListArray.Parse("[1,1,1], [1,1,3]"));
var right = NullQuery.Instance;
using (var q = new OrQuery(left, right))
{
var r = q.ExecuteToString();
Assert.Equal(e, r);
Assert.Null(q.NextMatch());
}
}
/// <summary>
/// In some cases we need the order of returned elements to be stable between calls.
/// For example if using the Take or Skip operators
/// </summary>
/// <typeparam name="TItemType"></typeparam>
/// <param name="query"></param>
/// <returns></returns>
public IEnumerable <TItemType> GetMany <TItemType>(OrQuery query)
{
var clientResults = new IEnumerator <TItemType> [CacheClients.Count];
try
{
Parallel.ForEach(CacheClients, client =>
{
var resultsFromThisNode = client.GetMany <TItemType>(query);
clientResults[client.ShardIndex] = resultsFromThisNode.GetEnumerator();
});
}
catch (AggregateException e)
{
if (e.InnerException != null)
{
throw e.InnerException;
}
}
var count = 0;
while (true)
{
var allFinished = true;
foreach (var clientResult in clientResults)
{
// limit the number of returned object if Take linq extension method was used
if (query.Take > 0 && count >= query.Take)
{
yield break;
}
if (clientResult.MoveNext())
{
allFinished = false;
yield return(clientResult.Current);
count++;
}
}
if (allFinished)
{
yield break;
}
}
}
public void TestOrQueryWithDefaultIndex()
{
using (var index = TestHelper.PrepareIndexForSearch(new PersistentIndexName(TestFolder)))
{
using (var query = new OrQuery(
new TermQuery(TestHelper.GetPostingList(index, "this")),
new TermQuery(TestHelper.GetPostingList(index, "is"))))
{
var result = query.ExecuteToString();
var expected = "{[3,1,1]}, {[3,1,2]}, {[4,1,1]}, {[4,1,4]}, {[5,1,1]}, {[6,1,2]}, {[6,1,8]}, {[6,1,9]}";
Assert.Equal(expected, result);
}
}
}
/// <summary>
/// Count the items matching the query and check for data completeness.
/// </summary>
/// <param name="query"></param>
/// <returns>completeness, items count</returns>
public KeyValuePair <bool, int> EvalQuery(OrQuery query)
{
var request = new EvalRequest(query);
var response = Channel.SendRequest(request);
if (response is ExceptionResponse exResponse)
{
throw new CacheException("Error while getting server information", exResponse.Message,
exResponse.CallStack);
}
var concreteResponse = (EvalResponse)response;
return(new KeyValuePair <bool, int>(concreteResponse.Complete, concreteResponse.Items));
}
public void ParsingOrQueryTest_ReturnsOrQuery()
{
//Arrange
string query = "kiwi + banana";
OrQuery expected = new OrQuery(
new List <IQueryComponent> {
new TermLiteral("kiwi"), new TermLiteral("banana")
}
);
//Act
IQueryComponent actual = parser.ParseQuery(query);
//Assert
actual.Should().BeOfType(typeof(OrQuery));
((OrQuery)actual).Components.Should().HaveSameCount(expected.Components);
}
public void TestOrQueryWithFixedPostingList_1()
{
var e = "{[1,1,1]}, {[1,1,2]}, {[1,1,3]}, {[1,1,4]}, {[1,1,5]}, {[1,1,10]}, {[1,2,1]}, {[1,3,1]}, {[1,3,2]}, {[1,3,3]}, {[1,3,4]}, {[1,3,5]}, {[2,1,1]}, {[2,1,2]}";
var left = new TermQuery(PostingListArray
.Parse("[1,1,1], [1,1,5], [1,1,10], [1,2,1], [1,3,5], [2,1,2]"));
var right = new TermQuery(PostingListArray
.Parse("[1,1,2], [1,1,3], [1,1,4], [1,3,1], [1,3,2], [1,3,3], [1,3,4], [2,1,1]"));
using (var q = new OrQuery(left, right))
{
var r = q.ExecuteToString();
Assert.Equal(e, r);
Assert.Null(q.NextMatch());
}
}
/// <summary>
/// Return cached object without deserialization to .NET object. Used to process data at json level
/// </summary>
/// <param name="query"></param>
/// <returns></returns>
public IList <CachedObject> GetObjectDescriptions(OrQuery query)
{
var exceptions = new List <ExceptionResponse>();
var result = new List <CachedObject>();
var request = new GetDescriptionRequest(query);
Channel.SendStreamRequest(request, (CachedObject item, int currentItem, int totalItems) => result.Add(item),
exceptions.Add);
if (exceptions.Count > 0)
{
throw new CacheException("Error in GetObjectDescriptions", exceptions[0].Message,
exceptions[0].CallStack);
}
return(result);
}
public static OrQuery ToQuery(this Node node, CollectionSchema schema)
{
OrQuery result = null;
if (node.Token == "select" || node.Token == "count")
{
result = SelectToQuery(node, schema);
}
QueryHelper.OptimizeQuery(result);
if (node.Token == "count")
{
result !.CountOnly = true;
}
return(result);
}
public void OrQueryRun()
{
var postings1 = new long[] { 1, 2, 3, 4 };
var postings2 = new long[] { 3, 4, 5, 6 };
var runner = new Mock <IQueryRunner>(MockBehavior.Strict);
var query1 = new Mock <IQuery>();
query1.Setup(x => x.Run(runner.Object)).Returns(postings1);
var query2 = new Mock <IQuery>();
query2.Setup(x => x.Run(runner.Object)).Returns(postings2);
var orQuery = new OrQuery();
orQuery.Subqueries.Add(query1.Object);
orQuery.Subqueries.Add(query2.Object);
CollectionAssert.AreEquivalent(new[] { 1, 2, 3, 4, 5, 6 }, orQuery.Run(runner.Object));
}
private void VisitBinaryExpression(BinaryExpression binaryExpression, OrQuery rootExpression)
{
// manage AND expressions
if (binaryExpression.NodeType == ExpressionType.AndAlso)
{
var andExpression = new AndQuery();
rootExpression.Elements.Add(andExpression);
VisitAndExpression(binaryExpression, andExpression);
}
// manage OR expressions
else if (binaryExpression.NodeType == ExpressionType.OrElse)
{
VisitOrExpression(binaryExpression, rootExpression);
}
// manage simple expressions like a > 10
else if (IsLeafExpression(binaryExpression))
{
AndQuery andExpression;
if (!rootExpression.MultipleWhereClauses)
{
andExpression = new AndQuery();
rootExpression.Elements.Add(andExpression);
}
else // if multiple where clauses consider them as expressions linked by AND
{
andExpression = rootExpression.Elements[0];
}
andExpression.Elements.Add(VisitLeafExpression(binaryExpression));
}
else
{
throw new NotSupportedException("Query too complex");
}
}
public int RemoveMany(OrQuery query)
{
var sum = 0;
try
{
Parallel.ForEach(CacheClients, client =>
{
var removed = client.RemoveMany(query);
Interlocked.Add(ref sum, removed);
});
}
catch (AggregateException e)
{
if (e.InnerException != null)
{
throw e.InnerException;
}
}
return(sum);
}
//TODO add unit test for timestamp synchronization (coverage)
/// <summary>
/// Optimistic synchronization using a timestamp property
/// Works like an UpdateIf that checks the previous value of a property of type DateTime named "Timestamp"
/// It also updates this property withe DateTime.Now
/// If you use this you should never modify the timestamp manually when updating the object
/// </summary>
/// <param name="newValue"></param>
public void UpdateWithTimestampSynchronization <T>(T newValue)
{
var prop = newValue.GetType().GetProperty("Timestamp");
if (prop == null)
{
throw new CacheException($"No Timestamp property found on type {typeof(T).Name}");
}
if (!prop.CanWrite)
{
throw new CacheException($"The Timestamp property of type {typeof(T).Name} is not writable");
}
var oldTimestamp = prop.GetValue(newValue);
var kv = KeyInfo.ValueToKeyValue(oldTimestamp,
new KeyInfo(KeyDataType.IntKey, KeyType.ScalarIndex, "Timestamp"));
var q = new AtomicQuery(kv);
var andQuery = new AndQuery();
andQuery.Elements.Add(q);
var orq = new OrQuery(typeof(T));
orq.Elements.Add(andQuery);
var now = DateTime.Now;
var newTimestamp = now.AddTicks(1); // add one to be sure its different
prop.SetValue(newValue, newTimestamp);
var packed = Pack(newValue);
_itemsToPut.Add(packed);
_conditions.Add(orq);
}
public void UpdateIf(PackedObject newValue, OrQuery testAsQuery)
{
if (newValue == null)
{
throw new ArgumentNullException(nameof(newValue));
}
var request = new PutRequest(testAsQuery.CollectionName)
{
ExcludeFromEviction = true, Predicate = testAsQuery
};
request.Items.Add(newValue);
var response = Channel.SendRequest(request);
if (response is ExceptionResponse exResponse)
{
throw new CacheException("Error while writing an object to the cache", exResponse.Message,
exResponse.CallStack);
}
}
/// <summary>
/// remove all the items matching a query
/// </summary>
/// <param name="query"></param>
/// <returns>number of items effectively removed</returns>
public int RemoveMany(OrQuery query)
{
if (query == null)
{
throw new ArgumentNullException(nameof(query));
}
var request = new RemoveManyRequest(query);
var response = Channel.SendRequest(request);
if (response is ExceptionResponse exResponse)
{
throw new CacheException("Error in RemoveMany", exResponse.Message, exResponse.CallStack);
}
if (!(response is ItemsCountResponse countResponse))
{
throw new CacheException("Invalid type of response received in RemoveMany()");
}
return(countResponse.ItemsCount);
}
public void UpdateIf <T>(T newValue, OrQuery testAsQuery)
{
if (newValue == null)
{
throw new ArgumentNullException(nameof(newValue));
}
CachedObject packedItem;
if (typeof(T) == typeof(CachedObject))
{
packedItem = newValue as CachedObject;
}
else
{
var description = RegisterTypeIfNeeded(typeof(T));
packedItem = CachedObject.Pack(newValue, description);
}
var request = new PutRequest(typeof(T))
{
ExcludeFromEviction = true, Predicate = testAsQuery
};
request.Items.Add(packedItem);
var response = Channel.SendRequest(request);
if (response is ExceptionResponse exResponse)
{
throw new CacheException("Error while writing an object to the cache", exResponse.Message,
exResponse.CallStack);
}
}
/// <summary>
/// Optimistic synchronization using a timestamp property
/// Works like an UpdateIf that checks the previous value of a property of type DateTime named "Timestamp"
/// It also updates this property withe DateTime.Now
/// If you use this you should never modify the timestamp manually
/// </summary>
/// <param name="newValue"></param>
public void UpdateWithTimestampSynchronization(T newValue)
{
var prop = newValue.GetType().GetProperty("Timestamp");
if (prop == null)
{
throw new CacheException($"No Timestamp property found on type {typeof(T).Name}");
}
if (!prop.CanWrite)
{
throw new CacheException($"The Timestamp property of type {typeof(T).Name} is not writable");
}
var oldTimestamp = prop.GetValue(newValue);
var kv = new KeyValue(oldTimestamp,
new KeyInfo("Timestamp", 0, IndexType.Dictionary));
var q = new AtomicQuery(_collectionSchema.KeyByName(kv.KeyName), kv);
var andQuery = new AndQuery();
andQuery.Elements.Add(q);
var orq = new OrQuery(_collectionName);
orq.Elements.Add(andQuery);
var now = DateTime.Now;
var newTimestamp = now.AddTicks(1); // add one to be sure its different
prop.SetValue(newValue, newTimestamp);
_client.UpdateIf(Pack(newValue), orq);
}
internal PivotRequest(OrQuery query)
: base(DataAccessType.Read, query.CollectionName)
{
Query = query;
}
// [DBFlute-0.8.9.9]
// ===============================================================================
// OrScopeQuery
// ============
public void OrScopeQuery(OrQuery <DmyプロシージャCB> orQuery)
{
xorQ((DmyプロシージャCB)this, orQuery);
}
private IList <PackedObject> InternalProcessQuery(OrQuery query)
{
var executionPlan = new ExecutionPlan();
try
{
executionPlan.Begin();
// an empty query should return everything
if (query.IsEmpty())
{
var all = _dataStore.PrimaryIndex.GetAll().ToList();
if (query.OrderByProperty != null)
{
return(OrderBy(all.ToHashSet(), query.OrderByProperty, query.OrderByIsDescending, executionPlan));
}
return(all);
}
// simplified processing if it is an atomic query
var atomicQuery = AsAtomic(query);
if (atomicQuery != null)
{
var res = ProcessSimpleQuery(atomicQuery, executionPlan);
if (query.OrderByProperty != null)
{
return(OrderBy(res.ToHashSet(), query.OrderByProperty, query.OrderByIsDescending, executionPlan));
}
else
{
return(res);
}
}
// if only one AndQuery, process sequentially
if (query.Elements.Count == 1)
{
var andQuery = query.Elements[0];
var set = ProcessAndQuery(andQuery, executionPlan);
if (query.OrderByProperty != null)
{
return(OrderBy(set.ToHashSet(), query.OrderByProperty, query.OrderByIsDescending, executionPlan));
}
return(set.ToList());
}
// if multiple AndQueries run in parallel
HashSet <PackedObject> orResult = null;
var results = new IList <PackedObject> [query.Elements.Count];
Parallel.For(0, query.Elements.Count, i =>
{
var andQuery = query.Elements[i];
results[i] = ProcessAndQuery(andQuery, executionPlan);
});
executionPlan.BeginMerge();
// merge the results (they may contain duplicates)
foreach (var result in results)
{
if (orResult == null)
{
orResult = new HashSet <PackedObject>(result);
}
else
{
orResult.UnionWith(result);
}
}
executionPlan.EndMerge();
if (query.OrderByProperty != null)
{
return(OrderBy(orResult, query.OrderByProperty, query.OrderByIsDescending, executionPlan));
}
return(orResult !.ToList());
}
finally
{
executionPlan.End();
ExecutionPlan = executionPlan;
_log?.LogActivity("QUERY", query.CollectionName, executionPlan.TotalTimeInMicroseconds, query.ToString(), query.Description(), executionPlan);
}
}
public void UpdateIf(PackedObject newValue, OrQuery testAsQuery)
{
var node = WhichNode(newValue);
CacheClients[node].UpdateIf(newValue, testAsQuery);
}
public IEnumerable <RankedItem> GetMany(OrQuery query, Guid sessionId = default)
{
var clientResults = new IEnumerator <RankedItem> [CacheClients.Count];
try
{
Parallel.ForEach(CacheClients, client =>
{
var resultsFromThisNode = client.GetMany(query, sessionId);
clientResults[client.ShardIndex] = resultsFromThisNode.GetEnumerator();
});
}
catch (AggregateException e)
{
if (e.InnerException != null)
{
throw e.InnerException;
}
}
HashSet <RankedItem> distinctSet = new HashSet <RankedItem>();
// for full-text queries the order is given by the result rank
// for normal queries order is either explicit (order by clause) or thy are unordered
if (!query.IsFullTextQuery)
{
// if no order-by clause simply merge the results (in a way that guarantees a stable order between calls)
if (query.OrderByProperty == null)
{
var count = 0;
while (true)
{
var allFinished = true;
foreach (var clientResult in clientResults)
{
// limit the number of returned object if Take linq extension method was used
if (query.Take > 0 && count >= query.Take)
{
yield break;
}
if (clientResult.MoveNext())
{
allFinished = false;
var current = clientResult.Current;
if (query.Distinct)
{
if (distinctSet.Add(clientResult.Current))
{
yield return(current);
}
}
else
{
yield return(current);
}
count++;
}
}
if (allFinished)
{
yield break;
}
}
}
// if ordered merge results by preserving order (either ascending or descending)
foreach (var item in OrderByHelper.MixOrderedEnumerators(query.OrderByProperty, query.OrderByIsDescending, clientResults))
{
yield return(item);
}
}
else
{
// for full text queries we have to merge all results and sort by rank
var all = new List <RankedItem>();
Parallel.ForEach(clientResults, r =>
{
List <RankedItem> resultFromClient = new List <RankedItem>();
while (r.MoveNext())
{
resultFromClient.Add(r.Current);
}
lock (all)
{
all.AddRange(resultFromClient);
}
});
foreach (var item in all.OrderByDescending(ri => ri.Rank))
{
yield return(item);
}
}
}
// [DBFlute-0.8.9.9]
// ===============================================================================
// OrScopeQuery
// ============
public void OrScopeQuery(OrQuery <Dch従業員職位履歴台帳CB> orQuery)
{
xorQ((Dch従業員職位履歴台帳CB)this, orQuery);
}
public GetDescriptionRequest(OrQuery query)
: base(DataAccessType.Read, query.TypeName)
{
Query = query;
}
public GetRequest(OrQuery query)
: base(DataAccessType.Read, query.TypeName)
{
_query = query;
}
/// <summary>
/// If an empty query is passed then all the data is available in the cache
/// If a null query is passed then no domain description is available
/// </summary>
/// <param name="descriptionAsQuery"></param>
/// <param name="isFullyLoaded">if true all data is available</param>
/// <param name="description"></param>
public DomainDescription(OrQuery descriptionAsQuery, bool isFullyLoaded = false, string description = null)
{
DescriptionAsQuery = descriptionAsQuery;
IsFullyLoaded = isFullyLoaded;
Description = description;
}
/// <summary>
/// In some cases we need the order of returned elements to be stable between calls.
/// For example if using the Take or Skip operators
/// </summary>
/// <typeparam name="TItemType"></typeparam>
/// <param name="query"></param>
/// <returns></returns>
public IEnumerable <TItemType> GetMany <TItemType>(OrQuery query)
{
var clientResults = new IEnumerator <RankedItem> [CacheClients.Count];
try
{
Parallel.ForEach(CacheClients, client =>
{
var resultsFromThisNode = client.GetManyWithRank <TItemType>(query);
clientResults[client.ShardIndex] = resultsFromThisNode.GetEnumerator();
});
}
catch (AggregateException e)
{
if (e.InnerException != null)
{
throw e.InnerException;
}
}
if (!query.IsFullTextQuery)
{
var count = 0;
while (true)
{
var allFinished = true;
foreach (var clientResult in clientResults)
{
// limit the number of returned object if Take linq extension method was used
if (query.Take > 0 && count >= query.Take)
{
yield break;
}
if (clientResult.MoveNext())
{
allFinished = false;
yield return((TItemType)clientResult.Current?.Item);
count++;
}
}
if (allFinished)
{
yield break;
}
}
}
else // for full text queries we have to merge all results and sort by rank
{
var all = new List <RankedItem>();
Parallel.ForEach(clientResults, r =>
{
List <RankedItem> resultFromClient = new List <RankedItem>();
while (r.MoveNext())
{
resultFromClient.Add(r.Current);
}
lock (all)
{
all.AddRange(resultFromClient);
}
});
foreach (var item in all.OrderByDescending(ri => ri.Rank).Select(ri => ri.Item).Cast <TItemType>())
{
yield return(item);
}
}
}
// [DBFlute-0.8.9.9]
// ===============================================================================
// OrScopeQuery
// ============
public void OrScopeQuery(OrQuery <Mst従業員マスタCB> orQuery)
{
xorQ((Mst従業員マスタCB)this, orQuery);
}
public IEnumerable <TItemType> GetMany <TItemType>(OrQuery query, bool onlyIfFullyLoaded)
{
return(InternalGetMany <TItemType>(query, onlyIfFullyLoaded));
}
// [DBFlute-0.8.9.9]
// ===============================================================================
// OrScopeQuery
// ============
public void OrScopeQuery(OrQuery<Dch客室利用台帳CB> orQuery) {
xorQ((Dch客室利用台帳CB)this, orQuery);
}
/// <summary>
/// Retrieve multiple objects using a precompiled query.<br />
/// To create a valid <see cref="OrQuery" /> use a <see cref="QueryBuilder" />.
/// </summary>
/// <typeparam name="TItemType"></typeparam>
/// <param name="query"></param>
/// <returns></returns>
public IEnumerable <TItemType> GetMany <TItemType>(OrQuery query)
{
return(InternalGetMany <TItemType>(query, false));
}