/// <summary> /// Retrieves all the data matched by the <see cref="FetchXml"/> /// </summary> /// <param name="org">The <see cref="IOrganizationService"/> to execute the query against</param> /// <param name="metadata">The metadata cache to use when executing the query</param> /// <param name="options">The options to apply to the query execution</param> /// <returns>The records matched by the query, with any custom filters and calculated fields applied</returns> private IEnumerable <Entity> RetrieveSequenceInternal(IOrganizationService org, IAttributeMetadataCache metadata, IQueryExecutionOptions options) { if (options.Cancelled) { yield break; } var mainEntity = FetchXml.Items.OfType <FetchEntityType>().Single(); var name = mainEntity.name; var meta = metadata[name]; options.Progress($"Retrieving {meta.DisplayCollectionName?.UserLocalizedLabel?.Label}..."); // Get the first page of results var res = org.RetrieveMultiple(new FetchExpression(Serialize(FetchXml))); foreach (var entity in res.Entities) { yield return(entity); } var count = res.Entities.Count; // Aggregate queries return up to 5000 records and don't provide a method to move on to the next page // Throw an exception to indicate the error to the caller if (AllPages && FetchXml.aggregateSpecified && FetchXml.aggregate && count == 5000 && FetchXml.top != "5000" && !res.MoreRecords) { throw new ApplicationException("AggregateQueryRecordLimit"); } // Move on to subsequent pages while (AllPages && res.MoreRecords && !options.Cancelled && options.ContinueRetrieve(count)) { options.Progress($"Retrieved {count:N0} {meta.DisplayCollectionName?.UserLocalizedLabel?.Label}..."); if (FetchXml.page == null) { FetchXml.page = "2"; } else { FetchXml.page = (Int32.Parse(FetchXml.page) + 1).ToString(); } FetchXml.pagingcookie = res.PagingCookie; var nextPage = org.RetrieveMultiple(new FetchExpression(Serialize(FetchXml))); foreach (var entity in nextPage.Entities) { yield return(entity); } count += nextPage.Entities.Count; res = nextPage; } }
protected override IEnumerable <Entity> ExecuteInternal(IDictionary <string, DataSource> dataSources, IQueryExecutionOptions options, IDictionary <string, Type> parameterTypes, IDictionary <string, object> parameterValues) { PagesRetrieved = 0; if (!dataSources.TryGetValue(DataSource, out var dataSource)) { throw new NotSupportedQueryFragmentException("Missing datasource " + DataSource); } ReturnFullSchema = false; var schema = GetSchema(dataSources, parameterTypes); // Apply any variable conditions if (parameterValues != null) { if (_parameterizedConditions == null) { FindParameterizedConditions(); } foreach (var param in parameterValues) { if (_parameterizedConditions.TryGetValue(param.Key, out var condition)) { condition.SetValue(param.Value, options); } } } FindEntityNameGroupings(dataSource.Metadata); var mainEntity = FetchXml.Items.OfType <FetchEntityType>().Single(); var name = mainEntity.name; var meta = dataSource.Metadata[name]; if (!(Parent is PartitionedAggregateNode)) { options.Progress(0, $"Retrieving {GetDisplayName(0, meta)}..."); } // Get the first page of results options.RetrievingNextPage(); // Ensure we reset the page number & cookie for subsequent executions if (_resetPage) { FetchXml.page = _startingPage; FetchXml.pagingcookie = null; } else { _startingPage = FetchXml.page; _resetPage = true; } var res = dataSource.Connection.RetrieveMultiple(new FetchExpression(Serialize(FetchXml))); PagesRetrieved++; var count = res.Entities.Count; // Aggregate queries return up to 5000 records and don't provide a method to move on to the next page // Throw an exception to indicate the error to the caller if (AllPages && FetchXml.aggregateSpecified && FetchXml.aggregate && count == 5000 && FetchXml.top != "5000" && !res.MoreRecords) { throw new FaultException <OrganizationServiceFault>(new OrganizationServiceFault { ErrorCode = -2147164125, Message = "AggregateQueryRecordLimitExceeded" }); } // Aggregate queries with grouping on lookup columns don't provide reliable paging as the sorting is done by the name of the related // record, not the guid. Non-aggregate queries can also be sorted on the primary key as a tie-breaker. if (res.MoreRecords && FetchXml.aggregateSpecified && FetchXml.aggregate && ContainsSortOnLookupAttribute(dataSource.Metadata, Entity.name, Entity.Items, out var lookupAttr)) { throw new InvalidPagingException($"{lookupAttr.name} is a lookup attribute - paging with a sort order on this attribute is not reliable."); } foreach (var entity in res.Entities) { OnRetrievedEntity(entity, schema, options, dataSource.Metadata); yield return(entity); } // Move on to subsequent pages while (AllPages && res.MoreRecords && options.ContinueRetrieve(count)) { if (!(Parent is PartitionedAggregateNode)) { options.Progress(0, $"Retrieved {count:N0} {GetDisplayName(count, meta)}..."); } if (FetchXml.page == null) { FetchXml.page = "2"; } else { FetchXml.page = (Int32.Parse(FetchXml.page, CultureInfo.InvariantCulture) + 1).ToString(); } FetchXml.pagingcookie = res.PagingCookie; options.RetrievingNextPage(); var nextPage = dataSource.Connection.RetrieveMultiple(new FetchExpression(Serialize(FetchXml))); PagesRetrieved++; foreach (var entity in nextPage.Entities) { OnRetrievedEntity(entity, schema, options, dataSource.Metadata); yield return(entity); } count += nextPage.Entities.Count; res = nextPage; } }
public bool ContinueRetrieve(int count) { return(_options.ContinueRetrieve(count)); }