/// <summary>
/// Performs an iterative series of RetrieveMultiple requests using FetchExpression in order to obtain all pages of results up to the provided maximum result count
/// </summary>
/// <param name="service">The current IOrganizationService instance</param>
/// <param name="fetch">The FetchExpression query to be executed</param>
/// <param name="shouldRetrieveAllPages">True = perform iterative paged query requests, otherwise return first page only</param>
/// <param name="maxResultCount">An upper limit on the maximum number of entity records that should be retrieved as the query results - useful when the total size of result set is unknown and size may cause OutOfMemoryException</param>
/// <param name="pagedOperation">An operation to perform on each page of results as it's retrieved</param>
/// <returns>An EntityCollection containing the results of the query. Details reflect the last page retrieved (e.g. MoreRecords, PagingCookie, etc.)</returns>
/// <remarks>
/// CRM limits query response to paged result sets of 5,000. This method encapsulates the logic for performing subsequent
/// query requests so that all results can be retrieved.
/// </remarks>
private static EntityCollection RetrieveMultiple(this IOrganizationService service, FetchExpression fetch, bool shouldRetrieveAllPages, long maxResultCount, Action <EntityCollection> pagedOperation)
{
XElement fetchXml = fetch.ToXml();
int pageNumber = fetchXml.GetFetchXmlPageNumber();
string pageCookie = fetchXml.GetFetchXmlPageCookie();
int pageSize = fetchXml.GetFetchXmlPageSize(QueryExtensions.DefaultPageSize);
// Establish the first page based on lesser of initial/default page size or max result count (will be skipped if top count > 0)
if (pageSize > maxResultCount)
{
pageSize = Convert.ToInt32(maxResultCount);
}
if (pageNumber <= 1 ||
String.IsNullOrWhiteSpace(pageCookie))
{
// Ensure start with first page
fetchXml.SetFetchXmlPage(null, 1, pageSize);
}
else
{
// Start with specified page
fetchXml.SetFetchXmlPage(pageCookie, pageNumber, pageSize);
}
fetch.Query = fetchXml.ToString();
// Track local long value to avoid expensive IEnumerable<T>.LongCount() method calls
long totalResultCount = 0;
var allResults = new EntityCollection();
while (true)
{
// Retrieve the page
EntityCollection page = service.RetrieveMultiple(fetch);
// Capture the page
if (totalResultCount == 0)
{
// First page
allResults = page;
}
else
{
allResults.Entities.AddRange(page.Entities);
}
// Invoke the paged operation if non-null
pagedOperation?.Invoke(page);
// Update the count of pages retrieved and processed
totalResultCount = totalResultCount + page.Entities.Count;
// Determine if we should retrieve the next page
if (shouldRetrieveAllPages &&
totalResultCount < maxResultCount &&
page.MoreRecords)
{
// Setup for next page
pageNumber++;
long remainder = maxResultCount - totalResultCount;
// If max result count is not divisible by page size, then final page may be less than the current page size and should be sized to remainder.
// No risk of coversion overflow.
if (pageSize > remainder)
{
pageSize = Convert.ToInt32(remainder);
}
fetch.SetPage(page.PagingCookie, pageNumber, pageSize);
}
else
{
allResults.CopyFrom(page);
break;
}
}
return(allResults);
}
/// <summary>
/// Performs an iterative series of RetrieveMultiple requests using QueryExpression in order to obtain all pages of results up to the provided maximum result count
/// </summary>
/// <param name="service">The current IOrganizationService instance</param>
/// <param name="query">The QueryExpression query to be executed</param>
/// <param name="shouldRetrieveAllPages">True = perform iterative paged query requests, otherwise return first page only</param>
/// <param name="maxResultCount">An upper limit on the maximum number of entity records that should be retrieved as the query results - useful when the total size of result set is unknown and size may cause OutOfMemoryException</param>
/// <param name="pagedOperation">An operation to perform on each page of results as it's retrieved</param>
/// <returns>An EntityCollection containing the results of the query. Details reflect the last page retrieved (e.g. MoreRecords, PagingCookie, etc.)</returns>
/// <remarks>
/// CRM limits query response to paged result sets of 5,000. This method encapsulates the logic for performing subsequent
/// query requests so that all results can be retrieved.
/// </remarks>
private static EntityCollection RetrieveMultiple(this IOrganizationService service, QueryExpression query, bool shouldRetrieveAllPages, long maxResultCount, Action <EntityCollection> pagedOperation)
{
// Establish page info (only if TopCount not specified)
if (query.TopCount == null)
{
if (query.PageInfo == null)
{
// Default to first page
query.PageInfo = new PagingInfo()
{
Count = QueryExtensions.DefaultPageSize,
PageNumber = 1,
PagingCookie = null,
ReturnTotalRecordCount = false
};
}
else if (query.PageInfo.PageNumber <= 1 ||
query.PageInfo.PagingCookie == null)
{
// Reset to first page
query.PageInfo.PageNumber = 1;
query.PageInfo.PagingCookie = null;
}
// Limit initial page size to max result if less than current page size. No risk of conversion overflow.
if (query.PageInfo.Count > maxResultCount)
{
query.PageInfo.Count = Convert.ToInt32(maxResultCount);
}
}
// Track local long value to avoid expensive IEnumerable<T>.LongCount() method calls
long totalResultCount = 0;
var allResults = new EntityCollection();
while (true)
{
// Retrieve the page
EntityCollection page = service.RetrieveMultiple(query);
// Capture the page
if (totalResultCount == 0)
{
// First page
allResults = page;
}
else
{
allResults.Entities.AddRange(page.Entities);
}
// Invoke the paged operation if non-null
pagedOperation?.Invoke(page);
// Update the count of pages retrieved and processed
totalResultCount = totalResultCount + page.Entities.Count;
// Determine if we should retrieve the next page
if (shouldRetrieveAllPages &&
totalResultCount < maxResultCount &&
page.MoreRecords)
{
// Setup for next page
query.PageInfo.PageNumber++;
query.PageInfo.PagingCookie = page.PagingCookie;
long remainder = maxResultCount - totalResultCount;
// If max result count is not divisible by page size, then final page may be less than the current page size and should be sized to remainder.
// No risk of coversion overflow.
if (query.PageInfo.Count > remainder)
{
query.PageInfo.Count = Convert.ToInt32(remainder);
}
}
else
{
allResults.CopyFrom(page);
break;
}
}
return(allResults);
}
