/// <summary>
/// Gets the list of updates matching the query filter from an upstream update server.
/// <para>
/// If the destinatin metadata sink also implements <see cref="IMetadataSource"/>, a delta query for changed categories is performed./>
/// </para>
/// </summary>
/// <param name="updatesFilter">Updates filter. See <see cref="QueryFilter"/> for details.</param>
/// <param name="destination">Metadata collection where to write the result. If the destination implements IMetadataSource, a delta retrieve is performed</param>
/// <remarks>
/// </remarks>
public async Task GetUpdates(QueryFilter updatesFilter, IMetadataSink destination)
{
var deltaMetadataSource = destination as IMetadataSource;
var progress = new MetadataQueryProgress();
if (updatesFilter == null || updatesFilter.ProductsFilter.Count == 0 || updatesFilter.ClassificationsFilter.Count == 0)
{
throw new Exception("The filter cannot be null of empty");
}
if (AccessToken == null || AccessToken.ExpiresIn(TimeSpan.FromMinutes(2)))
{
await RefreshAccessToken(deltaMetadataSource?.UpstreamAccountName, deltaMetadataSource?.UpstreamAccountGuid);
}
// If no configuration is known, query it now
if (ConfigData == null)
{
await RefreshServerConfigData();
}
progress.CurrentTask = MetadataQueryStage.GetRevisionIdsStart;
MetadataQueryProgress?.Invoke(this, progress);
var updatesAnchor = deltaMetadataSource?.GetAnchorForFilter(updatesFilter);
updatesFilter.Anchor = updatesAnchor;
var updatesQueryResult = await GetUpdateIds(updatesFilter);
progress.CurrentTask = MetadataQueryStage.GetRevisionIdsEnd;
MetadataQueryProgress?.Invoke(this, progress);
// Find all updates that did not change
var unchangedUpdates = GetUnchangedUpdates(deltaMetadataSource?.UpdatesIndex, updatesQueryResult.identities);
// Create the list of updates to query data for. Remove those updates that were reported as "new"
// but for which we already have metadata
var updatesToRetrieveDataFor = updatesQueryResult.identities.Where(
newUpdateId => !unchangedUpdates.ContainsKey(newUpdateId));
progress.CurrentTask = MetadataQueryStage.GetUpdateMetadataStart;
progress.Maximum = updatesToRetrieveDataFor.Count();
progress.Current = 0;
MetadataQueryProgress?.Invoke(this, progress);
GetUpdateDataForIds(
updatesToRetrieveDataFor.Select(id => id.Raw).ToList(), destination);
// Update the QueryResult filter and anchor
updatesFilter.Anchor = updatesQueryResult.anchor;
destination.SetQueryFilter(updatesFilter);
progress.CurrentTask = MetadataQueryStage.GetUpdateMetadataEnd;
MetadataQueryProgress?.Invoke(this, progress);
}
/// <summary>
/// Gets the list of categories from the upstream server and adds them to the specified metadata collection.
/// </summary>
/// <param name="destination">Metadata collection where to add the results. If the collection implements <see cref="IMetadataSource"/>, only delta changes are retrieved and added to the destination.</param>
public async Task GetCategories(IMetadataSink destination)
{
var deltaMetadataSource = destination as IMetadataSource;
var progress = new MetadataQueryProgress();
if (AccessToken == null || AccessToken.ExpiresIn(TimeSpan.FromMinutes(2)))
{
await RefreshAccessToken(deltaMetadataSource?.UpstreamAccountName, deltaMetadataSource?.UpstreamAccountGuid);
}
// If no configuration is known, query it now
if (ConfigData == null)
{
await RefreshServerConfigData();
}
// Query IDs for all categories known to the upstream server
progress.CurrentTask = MetadataQueryStage.GetRevisionIdsStart;
MetadataQueryProgress?.Invoke(this, progress);
var categoriesAnchor = deltaMetadataSource?.CategoriesAnchor;
var categoryQueryResult = await GetCategoryIds(categoriesAnchor);
progress.CurrentTask = MetadataQueryStage.GetRevisionIdsEnd;
MetadataQueryProgress?.Invoke(this, progress);
var cachedCategories = deltaMetadataSource?.CategoriesIndex;
// Find all updates that did not change
var unchangedUpdates = GetUnchangedUpdates(cachedCategories, categoryQueryResult.identities);
// Create the list of updates to query data for. Remove those updates that were reported as "new"
// but for which we already have metadata
var updatesToRetrieveDataFor = categoryQueryResult.identities.Where(
newUpdateId => !unchangedUpdates.ContainsKey(newUpdateId));
// Retrieve metadata for all new categories
progress.CurrentTask = MetadataQueryStage.GetUpdateMetadataStart;
progress.Maximum = updatesToRetrieveDataFor.Count();
progress.Current = 0;
MetadataQueryProgress?.Invoke(this, progress);
destination.SetCategoriesAnchor(categoryQueryResult.anchor);
GetUpdateDataForIds(
updatesToRetrieveDataFor.Select(id => id.Raw).ToList(), destination);
progress.CurrentTask = MetadataQueryStage.GetUpdateMetadataEnd;
MetadataQueryProgress?.Invoke(this, progress);
}
/// <summary>
/// Retrieves configuration data from the upstream server.
/// </summary>
/// <returns>Server configuration data</returns>
public async Task <ServerSyncConfigData> GetServerConfigData()
{
await RefreshAccessToken(Guid.NewGuid().ToString(), Guid.NewGuid());
var progress = new MetadataQueryProgress();
progress.CurrentTask = MetadataQueryStage.GetServerConfigStart;
MetadataQueryProgress?.Invoke(this, progress);
var result = await QueryConfigData();
progress.CurrentTask = MetadataQueryStage.GetServerConfigEnd;
MetadataQueryProgress?.Invoke(this, progress);
return(result);
}
/// <summary>
/// Retrieves update data for the list of update ids
/// </summary>
/// <param name="updateIds">The ids to retrieve data for</param>
/// <param name="destination">The metadata destination to write update metadata to</param>
private void GetUpdateDataForIds(List <UpdateIdentity> updateIds, IMetadataSink destination)
{
// Data retrieval is done is done in batches of upto MaxNumberOfUpdatesPerRequest
var retrieveBatches = CreateBatchedListFromFlatList(updateIds, ConfigData.MaxNumberOfUpdatesPerRequest);
// Progress tracking and reporting
int batchesDone = 0;
var progress = new MetadataQueryProgress()
{
CurrentTask = MetadataQueryStage.GetUpdateMetadataProgress, Maximum = updateIds.Count, Current = 0
};
MetadataQueryProgress?.Invoke(this, progress);
// Run batches in parallel
retrieveBatches.AsParallel().ForAll(batch =>
{
var updateDataRequest = new GetUpdateDataRequest();
updateDataRequest.GetUpdateData = new GetUpdateDataRequestBody();
updateDataRequest.GetUpdateData.cookie = AccessToken.AccessCookie;
updateDataRequest.GetUpdateData.updateIds = batch;
GetUpdateDataResponse updateDataReply;
int retryCount = 0;
do
{
try
{
updateDataReply = ServerSyncClient.GetUpdateDataAsync(updateDataRequest).GetAwaiter().GetResult();
}
catch (System.TimeoutException)
{
updateDataReply = null;
}
retryCount++;
} while (updateDataReply == null && retryCount < 10);
if (updateDataReply == null || updateDataReply.GetUpdateDataResponse1 == null || updateDataReply.GetUpdateDataResponse1.GetUpdateDataResult == null)
{
throw new Exception("Failed to get update metadata");
}
// Parse the list of raw files into a more usable format
var filesList = new List <UpdateFileUrl>(updateDataReply.GetUpdateDataResponse1.GetUpdateDataResult.fileUrls.Select(rawFile => new UpdateFileUrl(rawFile)));
// First add the files information to the store; it will be used to link update files with urls later
filesList.ForEach(file => destination.AddFile(file));
// Add the updates to the result, converting them to a higher level representation
destination.AddUpdates(updateDataReply.GetUpdateDataResponse1.GetUpdateDataResult.updates);
lock (destination)
{
// Track progress
batchesDone++;
progress.PercentDone = ((double)batchesDone * 100) / retrieveBatches.Count;
progress.Current += batch.Count();
MetadataQueryProgress?.Invoke(this, progress);
}
});
}