/// <summary>
/// Retrieves all the data matched by the <see cref="FetchXml"/>
/// </summary>
/// <param name="org">The <see cref="IOrganizationService"/> to execute the query against</param>
/// <param name="metadata">The metadata cache to use when executing the query</param>
/// <param name="options">The options to apply to the query execution</param>
/// <returns>The records matched by the query, with any custom filters and calculated fields applied</returns>
private IEnumerable <Entity> RetrieveSequenceInternal(IOrganizationService org, IAttributeMetadataCache metadata, IQueryExecutionOptions options)
{
if (options.Cancelled)
{
yield break;
}
var mainEntity = FetchXml.Items.OfType <FetchEntityType>().Single();
var name = mainEntity.name;
var meta = metadata[name];
options.Progress($"Retrieving {meta.DisplayCollectionName?.UserLocalizedLabel?.Label}...");
// Get the first page of results
var res = org.RetrieveMultiple(new FetchExpression(Serialize(FetchXml)));
foreach (var entity in res.Entities)
{
yield return(entity);
}
var count = res.Entities.Count;
// Aggregate queries return up to 5000 records and don't provide a method to move on to the next page
// Throw an exception to indicate the error to the caller
if (AllPages && FetchXml.aggregateSpecified && FetchXml.aggregate && count == 5000 && FetchXml.top != "5000" && !res.MoreRecords)
{
throw new ApplicationException("AggregateQueryRecordLimit");
}
// Move on to subsequent pages
while (AllPages && res.MoreRecords && !options.Cancelled && options.ContinueRetrieve(count))
{
options.Progress($"Retrieved {count:N0} {meta.DisplayCollectionName?.UserLocalizedLabel?.Label}...");
if (FetchXml.page == null)
{
FetchXml.page = "2";
}
else
{
FetchXml.page = (Int32.Parse(FetchXml.page) + 1).ToString();
}
FetchXml.pagingcookie = res.PagingCookie;
var nextPage = org.RetrieveMultiple(new FetchExpression(Serialize(FetchXml)));
foreach (var entity in nextPage.Entities)
{
yield return(entity);
}
count += nextPage.Entities.Count;
res = nextPage;
}
}
protected override IEnumerable <Entity> ExecuteInternal(IDictionary <string, DataSource> dataSources, IQueryExecutionOptions options, IDictionary <string, Type> parameterTypes, IDictionary <string, object> parameterValues)
{
PagesRetrieved = 0;
if (!dataSources.TryGetValue(DataSource, out var dataSource))
{
throw new NotSupportedQueryFragmentException("Missing datasource " + DataSource);
}
ReturnFullSchema = false;
var schema = GetSchema(dataSources, parameterTypes);
// Apply any variable conditions
if (parameterValues != null)
{
if (_parameterizedConditions == null)
{
FindParameterizedConditions();
}
foreach (var param in parameterValues)
{
if (_parameterizedConditions.TryGetValue(param.Key, out var condition))
{
condition.SetValue(param.Value, options);
}
}
}
FindEntityNameGroupings(dataSource.Metadata);
var mainEntity = FetchXml.Items.OfType <FetchEntityType>().Single();
var name = mainEntity.name;
var meta = dataSource.Metadata[name];
if (!(Parent is PartitionedAggregateNode))
{
options.Progress(0, $"Retrieving {GetDisplayName(0, meta)}...");
}
// Get the first page of results
options.RetrievingNextPage();
// Ensure we reset the page number & cookie for subsequent executions
if (_resetPage)
{
FetchXml.page = _startingPage;
FetchXml.pagingcookie = null;
}
else
{
_startingPage = FetchXml.page;
_resetPage = true;
}
var res = dataSource.Connection.RetrieveMultiple(new FetchExpression(Serialize(FetchXml)));
PagesRetrieved++;
var count = res.Entities.Count;
// Aggregate queries return up to 5000 records and don't provide a method to move on to the next page
// Throw an exception to indicate the error to the caller
if (AllPages && FetchXml.aggregateSpecified && FetchXml.aggregate && count == 5000 && FetchXml.top != "5000" && !res.MoreRecords)
{
throw new FaultException <OrganizationServiceFault>(new OrganizationServiceFault {
ErrorCode = -2147164125, Message = "AggregateQueryRecordLimitExceeded"
});
}
// Aggregate queries with grouping on lookup columns don't provide reliable paging as the sorting is done by the name of the related
// record, not the guid. Non-aggregate queries can also be sorted on the primary key as a tie-breaker.
if (res.MoreRecords && FetchXml.aggregateSpecified && FetchXml.aggregate && ContainsSortOnLookupAttribute(dataSource.Metadata, Entity.name, Entity.Items, out var lookupAttr))
{
throw new InvalidPagingException($"{lookupAttr.name} is a lookup attribute - paging with a sort order on this attribute is not reliable.");
}
foreach (var entity in res.Entities)
{
OnRetrievedEntity(entity, schema, options, dataSource.Metadata);
yield return(entity);
}
// Move on to subsequent pages
while (AllPages && res.MoreRecords && options.ContinueRetrieve(count))
{
if (!(Parent is PartitionedAggregateNode))
{
options.Progress(0, $"Retrieved {count:N0} {GetDisplayName(count, meta)}...");
}
if (FetchXml.page == null)
{
FetchXml.page = "2";
}
else
{
FetchXml.page = (Int32.Parse(FetchXml.page, CultureInfo.InvariantCulture) + 1).ToString();
}
FetchXml.pagingcookie = res.PagingCookie;
options.RetrievingNextPage();
var nextPage = dataSource.Connection.RetrieveMultiple(new FetchExpression(Serialize(FetchXml)));
PagesRetrieved++;
foreach (var entity in nextPage.Entities)
{
OnRetrievedEntity(entity, schema, options, dataSource.Metadata);
yield return(entity);
}
count += nextPage.Entities.Count;
res = nextPage;
}
}
public bool ContinueRetrieve(int count)
{
return(_options.ContinueRetrieve(count));
}