/// <summary>
/// Send the request to the server with buffer size to account for the case where there are more data
/// that we need to fetch
/// </summary>
/// <param name="query"></param>
/// <param name="request"></param>
/// <returns></returns>
public static IEnumerable<PackageBase> SendRequest(string query, NuGetRequest request)
{
const int bufferSize = 40;
// number of threads sending the requests
const int numberOfSenders = 4;
var startPoint = 0;
var tasks = new List<Task<Stream>>();
bool stopSending = false;
object stopLock = new Object();
// Send one request first
// this initial query is of the form http://www.nuget.org/api/v2/FindPackagesById()?id='jquery'&$skip={0}&$top={1}
UriBuilder initialQuery = new UriBuilder(query.InsertSkipAndTop());
PackageBase firstPackage = null;
// Send out an initial request
// we send out 1 initial request first to check for redirection and check whether repository supports odata
using (Stream stream = NuGetClient.InitialDownloadDataToStream(initialQuery, startPoint, bufferSize, request))
{
if (stream == null)
{
yield break;
}
XDocument document = XmlUtility.LoadSafe(stream, ignoreWhiteSpace: true);
var entries = document.Root.ElementsNoNamespace("entry").ToList();
// If the initial request has different number of entries than the buffer size, return it because this means the server
// does not understand odata request or there is no more data. in the former case, we have to stop to prevent infinite loop
if (entries.Count != bufferSize)
{
request.Debug(Messages.PackagesReceived, entries.Count);
stopSending = true;
}
foreach (XElement entry in entries)
{
var package = new PackageBase();
// set the first package of the request. this is used later to verify that the case when the number of packages in the repository
// is the same as the buffer size and the repository does not support odata query. in that case, we want to check whether the first package
// exists anywhere in the second call. if it is, then we cancel the request (this is to prevent infinite loop)
if (firstPackage == null)
{
firstPackage = package;
}
PackageUtility.ReadEntryElement(ref package, entry);
yield return package;
}
}
if (stopSending || request.IsCanceled)
{
yield break;
}
// To avoid more redirection (for example, if the initial query is nuget.org, it will be changed to www.nuget.org
query = initialQuery.Uri.ToString();
// Sending the initial requests
for (var i = 0; i < numberOfSenders; i++)
{
// Update the start point to fetch the packages
startPoint += bufferSize;
// Get the query
var newQuery = string.Format(query, startPoint, bufferSize);
// Send it
tasks.Add(Task.Factory.StartNew(() =>
{
Stream items = NuGetClient.DownloadDataToStream(newQuery, request);
return items;
}));
}
//Wait for the responses, parse the data, and send to the user
while (tasks.Count > 0)
{
//Cast because the compiler warning: Co-variant array conversion from Task[] to Task[] can cause run-time exception on write operation.
var index = Task.WaitAny(tasks.Cast<Task>().ToArray());
using (Stream stream = tasks[index].Result)
{
if (stream == null)
{
yield break;
}
XDocument document = XmlUtility.LoadSafe(stream, ignoreWhiteSpace: true);
var entries = document.Root.ElementsNoNamespace("entry").ToList();
if (entries.Count < bufferSize)
{
request.Debug(Messages.PackagesReceived, entries.Count);
lock (stopLock)
{
stopSending = true;
}
}
foreach (XElement entry in entries)
{
var package = new PackageBase();
PackageUtility.ReadEntryElement(ref package, entry);
if (firstPackage != null)
{
// check whether first package in the first request exists anywhere in the second request
if (string.Equals(firstPackage.GetFullName(), package.GetFullName(), StringComparison.OrdinalIgnoreCase)
&& string.Equals(firstPackage.Version, package.Version, StringComparison.OrdinalIgnoreCase))
{
lock (stopLock)
{
stopSending = true;
}
break;
}
}
yield return package;
}
// we only needs to check for the existence of the first package in the second request. don't need to do for subsequent request
if (firstPackage != null)
{
firstPackage = null;
}
}
// checks whether we should stop sending requests
if (!stopSending && !request.IsCanceled)
{
// Make sure nobody else is updating the startPoint
lock (stopLock)
{
// update the startPoint
startPoint += bufferSize;
}
// Make a new request with the new startPoint
var newQuery = string.Format(query, startPoint, bufferSize);
//Keep sending a request
tasks[index] = (Task.Factory.StartNew(searchQuery =>
{
var items = NuGetClient.DownloadDataToStream(searchQuery.ToStringSafe(), request);
return items;
}, newQuery));
}
else
{
if (request.IsCanceled)
{
request.Warning(Messages.RequestCanceled, "HttpClientPackageRepository", "SendRequest");
//stop sending request to the remote server
stopSending = true;
}
tasks.RemoveAt(index);
}
}
}
/// <summary>
/// Send the request to the server with buffer size to account for the case where there are more data
/// that we need to fetch
/// </summary>
/// <param name="query"></param>
/// <param name="request"></param>
/// <returns></returns>
public static IEnumerable <PackageBase> SendRequest(string query, NuGetRequest request)
{
const int bufferSize = 40;
// number of threads sending the requests
const int numberOfSenders = 4;
var startPoint = 0;
var tasks = new List <Task <Stream> >();
bool stopSending = false;
object stopLock = new Object();
// Send one request first
// this initial query is of the form http://www.nuget.org/api/v2/FindPackagesById()?id='jquery'&$skip={0}&$top={1}
UriBuilder initialQuery = new UriBuilder(query.InsertSkipAndTop());
PackageBase firstPackage = null;
// Send out an initial request
// we send out 1 initial request first to check for redirection and check whether repository supports odata
using (Stream stream = NuGetClient.InitialDownloadDataToStream(initialQuery, startPoint, bufferSize, request))
{
if (stream == null)
{
yield break;
}
XDocument document = XmlUtility.LoadSafe(stream, ignoreWhiteSpace: true);
var entries = document.Root.ElementsNoNamespace("entry").ToList();
// If the initial request has different number of entries than the buffer size, return it because this means the server
// does not understand odata request or there is no more data. in the former case, we have to stop to prevent infinite loop
if (entries.Count != bufferSize)
{
request.Debug(Messages.PackagesReceived, entries.Count);
stopSending = true;
}
foreach (XElement entry in entries)
{
var package = new PackageBase();
// set the first package of the request. this is used later to verify that the case when the number of packages in the repository
// is the same as the buffer size and the repository does not support odata query. in that case, we want to check whether the first package
// exists anywhere in the second call. if it is, then we cancel the request (this is to prevent infinite loop)
if (firstPackage == null)
{
firstPackage = package;
}
PackageUtility.ReadEntryElement(ref package, entry);
yield return(package);
}
}
if (stopSending || request.IsCanceled)
{
yield break;
}
// To avoid more redirection (for example, if the initial query is nuget.org, it will be changed to www.nuget.org
query = initialQuery.Uri.ToString();
// Sending the initial requests
for (var i = 0; i < numberOfSenders; i++)
{
// Update the start point to fetch the packages
startPoint += bufferSize;
// Get the query
var newQuery = string.Format(query, startPoint, bufferSize);
// Send it
tasks.Add(Task.Factory.StartNew(() =>
{
Stream items = NuGetClient.DownloadDataToStream(newQuery, request);
return(items);
}));
}
//Wait for the responses, parse the data, and send to the user
while (tasks.Count > 0)
{
//Cast because the compiler warning: Co-variant array conversion from Task[] to Task[] can cause run-time exception on write operation.
var index = Task.WaitAny(tasks.Cast <Task>().ToArray());
using (Stream stream = tasks[index].Result)
{
if (stream == null)
{
yield break;
}
XDocument document = XmlUtility.LoadSafe(stream, ignoreWhiteSpace: true);
var entries = document.Root.ElementsNoNamespace("entry").ToList();
if (entries.Count < bufferSize)
{
request.Debug(Messages.PackagesReceived, entries.Count);
lock (stopLock)
{
stopSending = true;
}
}
foreach (XElement entry in entries)
{
var package = new PackageBase();
PackageUtility.ReadEntryElement(ref package, entry);
if (firstPackage != null)
{
// check whether first package in the first request exists anywhere in the second request
if (string.Equals(firstPackage.GetFullName(), package.GetFullName(), StringComparison.OrdinalIgnoreCase) &&
string.Equals(firstPackage.Version, package.Version, StringComparison.OrdinalIgnoreCase))
{
lock (stopLock)
{
stopSending = true;
}
break;
}
}
yield return(package);
}
// we only needs to check for the existence of the first package in the second request. don't need to do for subsequent request
if (firstPackage != null)
{
firstPackage = null;
}
}
// checks whether we should stop sending requests
if (!stopSending && !request.IsCanceled)
{
// Make sure nobody else is updating the startPoint
lock (stopLock)
{
// update the startPoint
startPoint += bufferSize;
}
// Make a new request with the new startPoint
var newQuery = string.Format(query, startPoint, bufferSize);
//Keep sending a request
tasks[index] = (Task.Factory.StartNew(searchQuery =>
{
var items = NuGetClient.DownloadDataToStream(searchQuery.ToStringSafe(), request);
return(items);
}, newQuery));
}
else
{
if (request.IsCanceled)
{
request.Warning(Messages.RequestCanceled, "HttpClientPackageRepository", "SendRequest");
//stop sending request to the remote server
stopSending = true;
}
tasks.RemoveAt(index);
}
}
}
