internal static IEnumerable<PackageBase> ProcessPackagesFeed(Stream stream) { if (stream == null) { throw new ArgumentNullException("stream"); } XDocument document = XmlUtility.LoadSafe(stream, ignoreWhiteSpace: true); var entries = document.Root.ElementsNoNamespace("entry"); if (entries == null) { var message = string.Format(Messages.ManifestRequiredXmlElementMissing, "entry"); throw new InvalidDataException(message); } foreach (XElement entry in entries) { var package = new PackageBase(); ReadEntryElement(ref package, entry); yield return package; } }
/// <summary> /// Parse the 'entry' xml element. /// </summary> /// <param name="package"></param> /// <param name="xElement"></param> internal static void ReadEntryElement(ref PackageBase package, XElement xElement) { var node = xElement.FirstNode; //iterate each <entry> element from the feed, e.g. //http://www.nuget.org/api/v2/FindPackagesById()?id='Jquery' // while (node != null) { var element = node as XElement; if (element != null) { ReadEntryChildNode(ref package, element); } node = node.NextNode; } }
/// <summary> /// A helper method for processing the children of the 'metadata' tag in the .nuspec file /// </summary> /// <param name="package"></param> /// <param name="element"></param> private static void ReadPackageMetadataElement(ref PackageBase package, XElement element) { var value = element.Value.SafeTrim(); // since we put all the name to lower case, each word in case should all be in lower case switch (element.Name.LocalName.ToLowerInvariant()) { case "id": package.Id = value; break; case "version": package.Version = value; break; case "authors": package.Authors = value; break; case "owners": package.Owners = value; break; case "licenseurl": package.LicenseUrl = string.IsNullOrWhiteSpace(value) ? null : new Uri(value); break; case "projecturl": package.ProjectUrl = string.IsNullOrWhiteSpace(value) ? null : new Uri(value); break; case "iconurl": package.IconUrl = string.IsNullOrWhiteSpace(value) ? null : new Uri(value); break; case "requirelicenseacceptance": package.RequireLicenseAcceptance = XmlConvert.ToBoolean(value); break; case "developmentdependency": package.DevelopmentDependency = XmlConvert.ToBoolean(value); break; case "description": package.Description = value; break; case "summary": package.Summary = value; break; case "releasenotes": package.ReleaseNotes = value; break; case "copyright": package.Copyright = value; break; case "language": package.Language = value; break; case "title": package.Title = value; break; case "tags": package.Tags = value; break; case "dependencies": package.DependencySetList = ReadDependencyList(element); break; /* case "frameworkAssemblies": * manifestMetadata.FrameworkAssemblies = ReadFrameworkAssemblies(element); * break; * case "references": * manifestMetadata.ReferenceSets = ReadReferenceSets(element); * break;*/ } }
/// <summary> /// Parse the children of the 'entry' tag. /// </summary> /// <param name="package"></param> /// <param name="element"></param> private static void ReadEntryChildNode(ref PackageBase package, XElement element) { var value = element.Value.SafeTrim(); switch (element.Name.LocalName.ToLowerInvariant()) { case "id": //In manifest, <id>http://www.nuget.org/api/v2/Packages(Id='jQuery',Version='1.10.1')</id> //we here only need Id=Jquery package.Id = ExtractId(value); break; case "version": package.Version = value; break; case "minclientversion": package.MinClientVersion = string.IsNullOrWhiteSpace(value) ? null : new Version(value); break; case "author": package.Authors = value; break; case "owners": package.Owners = value; break; case "licenseurl": package.LicenseUrl = string.IsNullOrWhiteSpace(value) ? null : new Uri(value); break; case "projecturl": package.ProjectUrl = string.IsNullOrWhiteSpace(value) ? null : new Uri(value); break; case "iconurl": package.IconUrl = string.IsNullOrWhiteSpace(value) ? null : new Uri(value); break; case "gallerydetailsurl": package.GalleryDetailsUrl = string.IsNullOrWhiteSpace(value) ? null : new Uri(value); break; case "requirelicenseacceptance": package.RequireLicenseAcceptance = XmlConvert.ToBoolean(value); break; case "developmentdependency": package.DevelopmentDependency = XmlConvert.ToBoolean(value); break; case "description": package.Description = value; break; case "summary": package.Summary = value; break; case "content": var srcAttribute = element.Attributes(XName.Get("src")).FirstOrDefault(); if (srcAttribute != null) { package.ContentSrcUrl = srcAttribute.Value; } break; case "releasenotes": package.ReleaseNotes = value; break; case "copyright": package.Copyright = value; break; case "language": package.Language = value; break; case "title": package.Title = value; break; case "tags": package.Tags = value; break; case "islatestversion": package.IsLatestVersion = Convert.ToBoolean(value, CultureInfo.InvariantCulture); break; case "isabsolutelatestversion": package.IsAbsoluteLatestVersion = Convert.ToBoolean(value, CultureInfo.InvariantCulture); break; case "published": package.Published = GetDateTime(value); break; case "created": package.Created = GetDateTime(value); break; case "lastupdated": package.LastUpdated = GetDateTime(value); break; case "lastedited": package.LastEdited = GetDateTime(value); break; case "licensereporturl": package.LicenseReportUrl = string.IsNullOrWhiteSpace(value) ? null : new Uri(value); break; case "reportabuseurl": package.ReportAbuseUrl = string.IsNullOrWhiteSpace(value) ? null : new Uri(value); break; case "downloadcount": package.DownloadCount = XmlConvert.ToInt64(value); break; case "versiondownloadcount": package.VersionDownloadCount = XmlConvert.ToInt64(value); break; case "packagesize": package.PackageSize = XmlConvert.ToInt64(value); break; case "properties": var node = element.FirstNode; while (node != null) { var propertyElement = node as XElement; if (propertyElement != null) { ReadEntryChildNode(ref package, propertyElement); } node = node.NextNode; } break; case "dependencies": package.DependencySetList = ParseDependencySet(value); break; case "packagehashalgorithm": package.PackageHashAlgorithm = value; break; case "packagehash": package.PackageHash = value; break; /* case "frameworkAssemblies": * package.FrameworkAssemblies = ReadFrameworkAssemblies(element); * break; * case "references": * package.ReferenceSets = ReadReferenceSets(element); * break;*/ default: if (!String.IsNullOrWhiteSpace(value) && !String.IsNullOrWhiteSpace(element.Name.LocalName)) { package.AdditionalProperties.AddOrSet(element.Name.LocalName, value); } break; } }
/// <summary> /// Send the request to the server with buffer size to account for the case where there are more data /// that we need to fetch /// </summary> /// <param name="query"></param> /// <param name="request"></param> /// <returns></returns> public static IEnumerable<PackageBase> SendRequest(string query, NuGetRequest request) { const int bufferSize = 40; // number of threads sending the requests const int numberOfSenders = 4; var startPoint = 0; var tasks = new List<Task<Stream>>(); bool stopSending = false; object stopLock = new Object(); // Send one request first // this initial query is of the form http://www.nuget.org/api/v2/FindPackagesById()?id='jquery'&$skip={0}&$top={1} UriBuilder initialQuery = new UriBuilder(query.InsertSkipAndTop()); PackageBase firstPackage = null; // Send out an initial request // we send out 1 initial request first to check for redirection and check whether repository supports odata using (Stream stream = NuGetClient.InitialDownloadDataToStream(initialQuery, startPoint, bufferSize, request)) { if (stream == null) { yield break; } XDocument document = XmlUtility.LoadSafe(stream, ignoreWhiteSpace: true); var entries = document.Root.ElementsNoNamespace("entry").ToList(); // If the initial request has different number of entries than the buffer size, return it because this means the server // does not understand odata request or there is no more data. in the former case, we have to stop to prevent infinite loop if (entries.Count != bufferSize) { request.Debug(Messages.PackagesReceived, entries.Count); stopSending = true; } foreach (XElement entry in entries) { var package = new PackageBase(); // set the first package of the request. this is used later to verify that the case when the number of packages in the repository // is the same as the buffer size and the repository does not support odata query. in that case, we want to check whether the first package // exists anywhere in the second call. if it is, then we cancel the request (this is to prevent infinite loop) if (firstPackage == null) { firstPackage = package; } PackageUtility.ReadEntryElement(ref package, entry); yield return package; } } if (stopSending || request.IsCanceled) { yield break; } // To avoid more redirection (for example, if the initial query is nuget.org, it will be changed to www.nuget.org query = initialQuery.Uri.ToString(); // Sending the initial requests for (var i = 0; i < numberOfSenders; i++) { // Update the start point to fetch the packages startPoint += bufferSize; // Get the query var newQuery = string.Format(query, startPoint, bufferSize); // Send it tasks.Add(Task.Factory.StartNew(() => { Stream items = NuGetClient.DownloadDataToStream(newQuery, request); return items; })); } //Wait for the responses, parse the data, and send to the user while (tasks.Count > 0) { //Cast because the compiler warning: Co-variant array conversion from Task[] to Task[] can cause run-time exception on write operation. var index = Task.WaitAny(tasks.Cast<Task>().ToArray()); using (Stream stream = tasks[index].Result) { if (stream == null) { yield break; } XDocument document = XmlUtility.LoadSafe(stream, ignoreWhiteSpace: true); var entries = document.Root.ElementsNoNamespace("entry").ToList(); if (entries.Count < bufferSize) { request.Debug(Messages.PackagesReceived, entries.Count); lock (stopLock) { stopSending = true; } } foreach (XElement entry in entries) { var package = new PackageBase(); PackageUtility.ReadEntryElement(ref package, entry); if (firstPackage != null) { // check whether first package in the first request exists anywhere in the second request if (string.Equals(firstPackage.GetFullName(), package.GetFullName(), StringComparison.OrdinalIgnoreCase) && string.Equals(firstPackage.Version, package.Version, StringComparison.OrdinalIgnoreCase)) { lock (stopLock) { stopSending = true; } break; } } yield return package; } // we only needs to check for the existence of the first package in the second request. don't need to do for subsequent request if (firstPackage != null) { firstPackage = null; } } // checks whether we should stop sending requests if (!stopSending && !request.IsCanceled) { // Make sure nobody else is updating the startPoint lock (stopLock) { // update the startPoint startPoint += bufferSize; } // Make a new request with the new startPoint var newQuery = string.Format(query, startPoint, bufferSize); //Keep sending a request tasks[index] = (Task.Factory.StartNew(searchQuery => { var items = NuGetClient.DownloadDataToStream(searchQuery.ToStringSafe(), request); return items; }, newQuery)); } else { if (request.IsCanceled) { request.Warning(Messages.RequestCanceled, "HttpClientPackageRepository", "SendRequest"); //stop sending request to the remote server stopSending = true; } tasks.RemoveAt(index); } } }
/// <summary> /// Send the request to the server with buffer size to account for the case where there are more data /// that we need to fetch /// </summary> /// <param name="query"></param> /// <param name="request"></param> /// <returns></returns> public static IEnumerable <PackageBase> SendRequest(string query, NuGetRequest request) { const int bufferSize = 40; // number of threads sending the requests const int numberOfSenders = 4; var startPoint = 0; var tasks = new List <Task <Stream> >(); bool stopSending = false; object stopLock = new Object(); // Send one request first // this initial query is of the form http://www.nuget.org/api/v2/FindPackagesById()?id='jquery'&$skip={0}&$top={1} UriBuilder initialQuery = new UriBuilder(query.InsertSkipAndTop()); PackageBase firstPackage = null; // Send out an initial request // we send out 1 initial request first to check for redirection and check whether repository supports odata using (Stream stream = NuGetClient.InitialDownloadDataToStream(initialQuery, startPoint, bufferSize, request)) { if (stream == null) { yield break; } XDocument document = XmlUtility.LoadSafe(stream, ignoreWhiteSpace: true); var entries = document.Root.ElementsNoNamespace("entry").ToList(); // If the initial request has different number of entries than the buffer size, return it because this means the server // does not understand odata request or there is no more data. in the former case, we have to stop to prevent infinite loop if (entries.Count != bufferSize) { request.Debug(Messages.PackagesReceived, entries.Count); stopSending = true; } foreach (XElement entry in entries) { var package = new PackageBase(); // set the first package of the request. this is used later to verify that the case when the number of packages in the repository // is the same as the buffer size and the repository does not support odata query. in that case, we want to check whether the first package // exists anywhere in the second call. if it is, then we cancel the request (this is to prevent infinite loop) if (firstPackage == null) { firstPackage = package; } PackageUtility.ReadEntryElement(ref package, entry); yield return(package); } } if (stopSending || request.IsCanceled) { yield break; } // To avoid more redirection (for example, if the initial query is nuget.org, it will be changed to www.nuget.org query = initialQuery.Uri.ToString(); // Sending the initial requests for (var i = 0; i < numberOfSenders; i++) { // Update the start point to fetch the packages startPoint += bufferSize; // Get the query var newQuery = string.Format(query, startPoint, bufferSize); // Send it tasks.Add(Task.Factory.StartNew(() => { Stream items = NuGetClient.DownloadDataToStream(newQuery, request); return(items); })); } //Wait for the responses, parse the data, and send to the user while (tasks.Count > 0) { //Cast because the compiler warning: Co-variant array conversion from Task[] to Task[] can cause run-time exception on write operation. var index = Task.WaitAny(tasks.Cast <Task>().ToArray()); using (Stream stream = tasks[index].Result) { if (stream == null) { yield break; } XDocument document = XmlUtility.LoadSafe(stream, ignoreWhiteSpace: true); var entries = document.Root.ElementsNoNamespace("entry").ToList(); if (entries.Count < bufferSize) { request.Debug(Messages.PackagesReceived, entries.Count); lock (stopLock) { stopSending = true; } } foreach (XElement entry in entries) { var package = new PackageBase(); PackageUtility.ReadEntryElement(ref package, entry); if (firstPackage != null) { // check whether first package in the first request exists anywhere in the second request if (string.Equals(firstPackage.GetFullName(), package.GetFullName(), StringComparison.OrdinalIgnoreCase) && string.Equals(firstPackage.Version, package.Version, StringComparison.OrdinalIgnoreCase)) { lock (stopLock) { stopSending = true; } break; } } yield return(package); } // we only needs to check for the existence of the first package in the second request. don't need to do for subsequent request if (firstPackage != null) { firstPackage = null; } } // checks whether we should stop sending requests if (!stopSending && !request.IsCanceled) { // Make sure nobody else is updating the startPoint lock (stopLock) { // update the startPoint startPoint += bufferSize; } // Make a new request with the new startPoint var newQuery = string.Format(query, startPoint, bufferSize); //Keep sending a request tasks[index] = (Task.Factory.StartNew(searchQuery => { var items = NuGetClient.DownloadDataToStream(searchQuery.ToStringSafe(), request); return(items); }, newQuery)); } else { if (request.IsCanceled) { request.Warning(Messages.RequestCanceled, "HttpClientPackageRepository", "SendRequest"); //stop sending request to the remote server stopSending = true; } tasks.RemoveAt(index); } } }
private IEnumerable <PackageBase> GetPackagesForPackageEntry(dynamic packageEntry, NuGetSearchContext searchContext, RequestWrapper request) { PackageEntryInfo packageEntryInfo = new PackageEntryInfo(packageEntry.id); if (!PackageFilterUtility.IsValidByName(packageEntryInfo, searchContext)) { yield break; } // This will help us take packageEntryInfo.LatestVersion and packageEntryInfo.AbsoluteLatestVersion and get the matching package easily // We're not setting isLatestVersion here so we don't have to deal with "is it latest or absolute latest" Dictionary <SemanticVersion, PackageBase> versionToPackageTable = new Dictionary <SemanticVersion, PackageBase>(); NuGetSearchContext individualPackageSearchContext = new NuGetSearchContext() { PackageInfo = packageEntryInfo, AllVersions = searchContext.AllVersions, AllowPrerelease = searchContext.AllowPrerelease, RequiredVersion = searchContext.RequiredVersion, MinimumVersion = searchContext.MinimumVersion, MaximumVersion = searchContext.MaximumVersion, EnableDeepMetadataBypass = searchContext.EnableDeepMetadataBypass }; bool latestVersionRequired = !searchContext.AllVersions && searchContext.RequiredVersion == null && searchContext.MinimumVersion == null && searchContext.MaximumVersion == null; if (searchContext.EnableDeepMetadataBypass) { if (latestVersionRequired) { // Use the search result page to get the metadata for the latest version SemanticVersion individualPackageVersion = new SemanticVersion(packageEntry.version); packageEntryInfo.AddVersion(individualPackageVersion); PackageBase pb = this.ResourcesCollection.PackageConverter.Make(packageEntry); if (pb != null) { yield return(pb); } } else { // Go to the registration index of this package first. This allows us to bypass "deep" (but required) metadata in certain cases. NuGetPackageFeed3 packageFeed3 = (NuGetPackageFeed3)this.ResourcesCollection.PackagesFeed; NuGetSearchResult result = packageFeed3.Find(individualPackageSearchContext, request); foreach (PackageBase pb in result.Result.Cast <PackageBase>()) { yield return(pb); } } } else { // Either we want a specific version or we want all metadata for any packages foreach (dynamic packageVersionEntry in packageEntry.versions) { if (packageEntry.version.Equals(packageVersionEntry.version) || searchContext.AllVersions) { if (packageVersionEntry.Metadata.HasProperty("id")) { // Collect all versions from the search results so we can manually set isLatestVersion and isAbsoluteLatestVersion later SemanticVersion individualPackageVersion = new SemanticVersion(packageVersionEntry.version); packageEntryInfo.AddVersion(individualPackageVersion); // Skip prerelease versions if AllowPrereleaseVersions is not specified if (!String.IsNullOrEmpty(individualPackageVersion.SpecialVersion) && !searchContext.AllowPrerelease) { continue; } long?versionDownloadCount = null; if (packageVersionEntry.HasProperty("downloads")) { versionDownloadCount = packageVersionEntry.downloads; } string registrationUrl = packageVersionEntry.Metadata.id; // This should be PackageFeed3 // There should be a better way to reuse this function NuGetPackageFeed3 packageFeed3 = (NuGetPackageFeed3)this.ResourcesCollection.PackagesFeed; PackageBase packageVersionPackage = packageFeed3.Find(registrationUrl, individualPackageSearchContext, request, true).FirstOrDefault(); if (packageVersionPackage != null) { if (versionDownloadCount.HasValue) { packageVersionPackage.VersionDownloadCount = versionDownloadCount.Value; } // Reset these so we haven't collected all versions yet, so this is wrong packageVersionPackage.IsLatestVersion = false; packageVersionPackage.IsAbsoluteLatestVersion = false; versionToPackageTable[individualPackageVersion] = packageVersionPackage; } } } } // Now manually set the latest versions if (packageEntryInfo.LatestVersion != null && versionToPackageTable.ContainsKey(packageEntryInfo.LatestVersion)) { versionToPackageTable[packageEntryInfo.LatestVersion].IsLatestVersion = true; } if (packageEntryInfo.AbsoluteLatestVersion != null && versionToPackageTable.ContainsKey(packageEntryInfo.AbsoluteLatestVersion)) { versionToPackageTable[packageEntryInfo.AbsoluteLatestVersion].IsAbsoluteLatestVersion = true; } // I think this is the best we can do for enumeration (reads all versions of a package before yielding anything) foreach (PackageBase package in versionToPackageTable.Values) { yield return(package); } } }
public string MakeDownloadUri(PackageBase package) { // v2 doesn't really build this dynamically, so assume ContentSrcUrl is already set. This shouldn't be called for v2 as it doesn't make sense. return(package.ContentSrcUrl); }
/// <summary> /// A help for processing the metatadata tag /// </summary> /// <param name="package"></param> /// <param name="xElement"></param> private static void ReadPackageMetadata(ref PackageBase package, XElement xElement) { var node = xElement.FirstNode; if (node == null) { //Check for the tags within metadata tag throw new InvalidDataException(string.Format(CultureInfo.InvariantCulture, Messages.NuspecRequiredXmlElementMissing, "id, version, authors, or description")); } while (node != null) { var element = node as XElement; if (element != null) { ReadPackageMetadataElement(ref package, element); } node = node.NextNode; } // now check for required elements, which include <id>, <version>, <authors> and <description> EnsureRequiredXmlTags(package); }
/// <summary> /// A helper method for processing the children of the 'metadata' tag in the .nuspec file /// </summary> /// <param name="package"></param> /// <param name="element"></param> private static void ReadPackageMetadataElement(ref PackageBase package, XElement element) { var value = element.Value.SafeTrim(); // since we put all the name to lower case, each word in case should all be in lower case switch (element.Name.LocalName.ToLowerInvariant()) { case "id": package.Id = value; break; case "version": package.Version = value; break; case "authors": package.Authors = value; break; case "owners": package.Owners = value; break; case "licenseurl": package.LicenseUrl = string.IsNullOrWhiteSpace(value) ? null : new Uri(value); break; case "projecturl": package.ProjectUrl = string.IsNullOrWhiteSpace(value) ? null : new Uri(value); break; case "iconurl": package.IconUrl = string.IsNullOrWhiteSpace(value) ? null : new Uri(value); break; case "requirelicenseacceptance": package.RequireLicenseAcceptance = XmlConvert.ToBoolean(value); break; case "developmentdependency": package.DevelopmentDependency = XmlConvert.ToBoolean(value); break; case "description": package.Description = value; break; case "summary": package.Summary = value; break; case "releasenotes": package.ReleaseNotes = value; break; case "copyright": package.Copyright = value; break; case "language": package.Language = value; break; case "title": package.Title = value; break; case "tags": package.Tags = value; break; case "dependencies": package.DependencySetList = ReadDependencyList(element); break; /* case "frameworkAssemblies": manifestMetadata.FrameworkAssemblies = ReadFrameworkAssemblies(element); break; case "references": manifestMetadata.ReferenceSets = ReadReferenceSets(element); break;*/ } }
/// <summary> /// Utility to parse the nuspec file and produces the package object. /// </summary> /// <param name="rootDocument">XDocument object pointing to a .nuspec xml file </param> /// <returns></returns> private static PackageBase ProcessNuspec(XDocument rootDocument) { if (rootDocument == null) { throw new ArgumentNullException("rootDocument"); } var pkg = new PackageBase(); var metadataElement = rootDocument.Root.ElementsNoNamespace("metadata").FirstOrDefault(); if (metadataElement == null) { var message = string.Format(CultureInfo.InvariantCulture, Messages.NuspecRequiredXmlElementMissing, "metadata"); throw new InvalidDataException(message); } ReadPackageMetadata(ref pkg, metadataElement); return pkg; }
/// <summary> /// Parse the children of the 'entry' tag. /// </summary> /// <param name="package"></param> /// <param name="element"></param> private static void ReadEntryChildNode(ref PackageBase package, XElement element) { var value = element.Value.SafeTrim(); switch (element.Name.LocalName.ToLowerInvariant()) { case "id": //In manifest, <id>http://www.nuget.org/api/v2/Packages(Id='jQuery',Version='1.10.1')</id> //we here only need Id=Jquery package.Id = ExtractId(value); break; case "version": package.Version = value; break; case "minclientversion": package.MinClientVersion = string.IsNullOrWhiteSpace(value) ? null : new Version(value); break; case "author": package.Authors = value; break; case "owners": package.Owners = value; break; case "licenseurl": package.LicenseUrl = string.IsNullOrWhiteSpace(value) ? null : new Uri(value); break; case "projecturl": package.ProjectUrl = string.IsNullOrWhiteSpace(value) ? null : new Uri(value); break; case "iconurl": package.IconUrl = string.IsNullOrWhiteSpace(value) ? null : new Uri(value); break; case "gallerydetailsurl": package.GalleryDetailsUrl = string.IsNullOrWhiteSpace(value) ? null : new Uri(value); break; case "requirelicenseacceptance": package.RequireLicenseAcceptance = XmlConvert.ToBoolean(value); break; case "developmentdependency": package.DevelopmentDependency = XmlConvert.ToBoolean(value); break; case "description": package.Description = value; break; case "summary": package.Summary = value; break; case "content": var srcAttribute = element.Attributes(XName.Get("src")).FirstOrDefault(); if (srcAttribute != null) { package.ContentSrcUrl = srcAttribute.Value; } break; case "releasenotes": package.ReleaseNotes = value; break; case "copyright": package.Copyright = value; break; case "language": package.Language = value; break; case "title": package.Title = value; break; case "tags": package.Tags = value; break; case "islatestversion": package.IsLatestVersion = Convert.ToBoolean(value, CultureInfo.InvariantCulture); break; case "isabsolutelatestversion": package.IsAbsoluteLatestVersion = Convert.ToBoolean(value, CultureInfo.InvariantCulture); break; case "published": package.Published = string.IsNullOrWhiteSpace(value) ? (null as DateTimeOffset?) : Convert.ToDateTime(value, CultureInfo.InvariantCulture); break; case "created": package.Created = string.IsNullOrWhiteSpace(value) ? (null as DateTimeOffset?) : Convert.ToDateTime(value, CultureInfo.InvariantCulture); break; case "lastupdated": package.LastUpdated = string.IsNullOrWhiteSpace(value) ? (null as DateTimeOffset?) : Convert.ToDateTime(value, CultureInfo.InvariantCulture); break; case "lastedited": package.LastEdited = string.IsNullOrWhiteSpace(value) ? (null as DateTimeOffset?) : Convert.ToDateTime(value, CultureInfo.InvariantCulture); break; case "licensereporturl": package.LicenseReportUrl = string.IsNullOrWhiteSpace(value) ? null : new Uri(value); break; case "reportabuseurl": package.ReportAbuseUrl = string.IsNullOrWhiteSpace(value) ? null : new Uri(value); break; case "downloadcount": package.DownloadCount = XmlConvert.ToInt64(value); break; case "versiondownloadcount": package.VersionDownloadCount = XmlConvert.ToInt64(value); break; case "packagesize": package.PackageSize = XmlConvert.ToInt64(value); break; case "properties": var node = element.FirstNode; while (node != null) { var propertyElement = node as XElement; if (propertyElement != null) { ReadEntryChildNode(ref package, propertyElement); } node = node.NextNode; } break; case "dependencies": package.DependencySetList = ParseDependencySet(value); break; case "packagehashalgorithm": package.PackageHashAlgorithm = value; break; case "packagehash": package.PackageHash = value; break; /* case "frameworkAssemblies": package.FrameworkAssemblies = ReadFrameworkAssemblies(element); break; case "references": package.ReferenceSets = ReadReferenceSets(element); break;*/ default: if (!String.IsNullOrWhiteSpace(value) && !String.IsNullOrWhiteSpace(element.Name.LocalName)) { package.AdditionalProperties.AddOrSet(element.Name.LocalName, value); } break; } }
/// <summary> /// No download URI. /// </summary> public string MakeDownloadUri(PackageBase package) { throw new NotImplementedException(); }
/// <summary> /// Find packages when the registration URL is already known. /// </summary> /// <param name="registrationUrl"></param> /// <param name="request"></param> /// <returns></returns> internal IEnumerable <PackageBase> Find(string registrationUrl, NuGetSearchContext context, RequestWrapper request, bool finalAttempt) { request.Debug(Messages.DebugInfoCallMethod3, "NuGetPackageFeed3", "Find", registrationUrl); List <PackageBase> packages = null; PackageBase cachedPackage; if (!registrationUrl.Contains("index.json") && ConcurrentInMemoryCache.Instance.TryGet <PackageBase>(registrationUrl, out cachedPackage)) { if (cachedPackage == null) { return(packages); } else { packages = new List <PackageBase>() { cachedPackage }; return(packages); } } Stream s = NuGetClient.DownloadDataToStream(registrationUrl, request, ignoreNullResponse: true, tries: 1); if (s != null) { packages = new List <PackageBase>(); // Now we can get the catalog entry dynamic root = DynamicJsonParser.Parse(new StreamReader(s).ReadToEnd()); if (root.Metadata.HasProperty("type")) { // First check if this is a Package or PackageRegistration type // Package is from packageId + version // PackageRegistration is from packageId bool isRegistrationType = false; foreach (string t in root.Metadata.type) { if (t.Equals("PackageRegistration", StringComparison.OrdinalIgnoreCase)) { isRegistrationType = true; break; } } if (context.PackageInfo.AllVersions.Count == 0) { // Only when the version list is restricted is this method usually faster this.ResourcesCollection.FilesFeed.GetVersionInfo(context.PackageInfo, request); } HashSet <SemanticVersion> packageSemanticVersions = null; if (context.PackageInfo.AllVersions.Count > 0) { packageSemanticVersions = FilterVersionsByRequirements(context, context.PackageInfo); } if (isRegistrationType) { // This is a registration index, like: "https://api.nuget.org/v3/registration3/json/index.json" // Get all versions from files service // In addition, when DeepMetadataBypass is enabled, we MUST use the registration index to get package info // If a call to -Name -RequiredVersion is done, DeepMetadataBypass will never be enabled (for now) // If we wanted, we could enable this by checking if !isRegistrationType && context.EnableDeepMetadataBypass, then call into Find with the registration index URL if (!context.AllVersions && packageSemanticVersions != null && !context.EnableDeepMetadataBypass) { foreach (SemanticVersion packageVersion in packageSemanticVersions) { NuGetSearchResult result = this.ResourcesCollection.PackagesFeed.Find(new NuGetSearchContext() { PackageInfo = context.PackageInfo, RequiredVersion = packageVersion, EnableDeepMetadataBypass = context.EnableDeepMetadataBypass }, request); PackageBase package = result.Result == null ? null : result.Result.FirstOrDefault() as PackageBase; if (package != null) { packages.Add(package); } } } else { // Going to collect versions from the registration index in here // Map of package version -> either PackageBase (if context.EnableDeepMetadataBypass) or catalog URL Dictionary <SemanticVersion, object> catalogObjects = new Dictionary <SemanticVersion, object>(); // If the version list hasn't been built yet, we can build it from the registration page instead of using FilesFeed bool buildPackageInfoVersions = context.PackageInfo.AllVersions.Count == 0; // Fallback to catalog crawling in these cases: // - Bypass deep metadata // - Getting version list failed // - All versions required foreach (dynamic catalogPage in root.items) { dynamic actualCatalogPage = catalogPage; if (!actualCatalogPage.HasProperty("items")) { // Sometimes the catalog page on the PackageRegistration entry doesn't have the actual page contents // In this case, the ID metadata tag points to the full catalog entry Stream fullCatalogPageResponseStream = NuGetClient.DownloadDataToStream(actualCatalogPage.Metadata.id, request); if (fullCatalogPageResponseStream != null) { actualCatalogPage = DynamicJsonParser.Parse(new StreamReader(fullCatalogPageResponseStream).ReadToEnd()); } } foreach (dynamic packageEntry in actualCatalogPage.items) { SemanticVersion version = new SemanticVersion(packageEntry.catalogentry.version); if (buildPackageInfoVersions) { context.PackageInfo.AddVersion(version); } bool hasCatalogEntry = packageEntry.HasProperty("catalogentry"); if (context.EnableDeepMetadataBypass || !hasCatalogEntry) { // Bypass retrieving "deep" (but required) metadata like packageHash // Also do this if there's no catalog entry PackageBase pb = null; if (packageEntry.HasProperty("catalogentry")) { pb = this.ResourcesCollection.PackageConverter.Make(packageEntry.catalogentry, context.PackageInfo); } else { // In some implementations (lookin' at you MyGet) there's no catalogEntry object pb = this.ResourcesCollection.PackageConverter.Make(packageEntry, context.PackageInfo); } if (pb != null) { catalogObjects[version] = pb; } } else { catalogObjects[version] = this.ResourcesCollection.CatalogUrlConverter.Make(packageEntry); } } } packageSemanticVersions = FilterVersionsByRequirements(context, context.PackageInfo); foreach (SemanticVersion version in packageSemanticVersions) { if (!catalogObjects.ContainsKey(version)) { continue; } if (context.EnableDeepMetadataBypass) { packages.Add((PackageBase)catalogObjects[version]); } else { PackageBase pb = GetPackageFromCatalogUrl((string)catalogObjects[version], request, packageSemanticVersions, context); if (pb != null) { packages.Add(pb); } } } } } else { // In some implementations (lookin' at you MyGet) there's no catalogEntry object PackageBase pb = ConcurrentInMemoryCache.Instance.GetOrAdd <PackageBase>(registrationUrl, () => { if (!root.HasProperty("catalogentry")) { if ((packageSemanticVersions == null || packageSemanticVersions.Contains(new SemanticVersion(root.version)))) { return(this.ResourcesCollection.PackageConverter.Make(root)); } else { return(null); } } else { return(GetPackageFromCatalogUrl(this.ResourcesCollection.CatalogUrlConverter.Make(root), request, packageSemanticVersions, context)); } }); if (pb != null) { packages.Add(pb); } } } else { request.Warning(Messages.JsonSchemaMismatch, "type"); request.Debug(Messages.JsonObjectDump, DynamicJsonParser.Serialize(root)); } if (context.RequiredVersion == null && context.MinimumVersion == null && context.MaximumVersion == null && packages != null) { PackageBase absoluteLatestPackage = packages.Where(p => p.IsPrerelease).OrderByDescending(pb => ((IPackage)pb).Version).FirstOrDefault(); if (absoluteLatestPackage != null) { absoluteLatestPackage.IsAbsoluteLatestVersion = true; } PackageBase latestPackage = packages.Where(p => !p.IsPrerelease).OrderByDescending(pb => ((IPackage)pb).Version).FirstOrDefault(); if (latestPackage != null) { latestPackage.IsLatestVersion = true; } } } else if (finalAttempt) { // This is the last retry of this URL. It's definitely not a good one. ConcurrentInMemoryCache.Instance.GetOrAdd <PackageBase>(registrationUrl, () => null); } return(packages); }
/// <summary> /// Make sure the Nuget required xml tags exist witin the metadata tag in the nuspec file. /// </summary> /// <param name="package"></param> private static void EnsureRequiredXmlTags(PackageBase package) { //As a .nupec, id", "version", "authors", "description" tags are required. //Check if they exists if (string.IsNullOrWhiteSpace(package.Version) || string.IsNullOrWhiteSpace(package.Id) || string.IsNullOrWhiteSpace(package.Authors) || string.IsNullOrWhiteSpace(package.Description)) { //bad .nuspec xml format string message = string.Format(CultureInfo.InvariantCulture, "{0}: '{1}', '{2}, '{3}', '{4}'", Messages.NuspecRequiredXmlElementMissing, "Version", "Id", "authors", "description"); throw new InvalidDataException(message); } }
// parse the list of installed packages private IEnumerable<PackageBase> GetInstalledPackagesOptionValue() { // get possible installed packages var installedPackages = GetOptionValues("InstalledPackages") ?? new string[0]; // parse the installed package options passed in foreach (var installedPackage in installedPackages) { // we assume that the name passed in is something like jquery#1.2.5 string[] nameAndVersion = installedPackage.Split(new[] { "!#!" }, StringSplitOptions.None); var package = new PackageBase(); // only name passed in, no version if (nameAndVersion.Count() == 1) { package.Id = nameAndVersion[0]; yield return package; } else if (nameAndVersion.Count() == 2) { // this means there is version SemanticVersion semVers = null; try { // convert version to semvers semVers = new SemanticVersion(nameAndVersion[1]); } catch { // not a valid version, ignores this entry continue; } // set name and version of this installed packages package.Id = nameAndVersion[0]; package.Version = semVers.Version.ToStringSafe(); yield return package; } } }
/// <summary> /// Send the request to the server with buffer size to account for the case where there are more data /// that we need to fetch /// </summary> /// <param name="query"></param> /// <param name="request"></param> /// <returns></returns> public static IEnumerable<PackageBase> SendRequest(string query, Request request) { const int bufferSize = 40; // number of threads sending the requests const int numberOfSenders = 4; var startPoint = 0; var tasks = new List<Task<Stream>>(); bool stopSending = false; object stopLock = new Object(); // Send one request first // this initial query is of the form http://www.nuget.org/api/v2/FindPackagesById()?id='jquery'&$skip={0}&$top={1} UriBuilder initialQuery = new UriBuilder(query.InsertSkipAndTop()); // Send out an initial request using (Stream stream = NuGetClient.InitialDownloadDataToStream(initialQuery, startPoint, bufferSize, request)) { if (stream == null) { yield break; } XDocument document = XmlUtility.LoadSafe(stream, ignoreWhiteSpace: true); var entries = document.Root.ElementsNoNamespace("entry").ToList(); // If the initial request has less entries than the buffer size, return it because there won't be any more data if (entries.Count < bufferSize) { request.Debug(Messages.PackagesReceived, entries.Count); stopSending = true; } foreach (XElement entry in entries) { var package = new PackageBase(); PackageUtility.ReadEntryElement(ref package, entry); yield return package; } } if (stopSending || request.IsCanceled) { yield break; } // To avoid more redirection (for example, if the initial query is nuget.org, it will be changed to www.nuget.org query = initialQuery.Uri.ToString(); // Sending the initial requests for (var i = 0; i < numberOfSenders; i++) { // Update the start point to fetch the packages startPoint += bufferSize; // Get the query var newQuery = string.Format(query, startPoint, bufferSize); // Send it tasks.Add(Task.Factory.StartNew(() => { Stream items = NuGetClient.DownloadDataToStream(newQuery, request); return items; })); } //Wait for the responses, parse the data, and send to the user while (tasks.Count > 0) { //Cast because the compiler warning: Co-variant array conversion from Task[] to Task[] can cause run-time exception on write operation. var index = Task.WaitAny(tasks.Cast<Task>().ToArray()); using (Stream stream = tasks[index].Result) { if (stream == null) { yield break; } XDocument document = XmlUtility.LoadSafe(stream, ignoreWhiteSpace: true); var entries = document.Root.ElementsNoNamespace("entry").ToList(); if (entries.Count < bufferSize) { request.Debug(Messages.PackagesReceived, entries.Count); lock (stopLock) { stopSending = true; } } foreach (XElement entry in entries) { var package = new PackageBase(); PackageUtility.ReadEntryElement(ref package, entry); yield return package; } } // checks whether we should stop sending requests if (!stopSending && !request.IsCanceled) { // Make sure nobody else is updating the startPoint lock (stopLock) { // update the startPoint startPoint += bufferSize; } // Make a new request with the new startPoint var newQuery = string.Format(query, startPoint, bufferSize); //Keep sending a request tasks[index] = (Task.Factory.StartNew(searchQuery => { var items = NuGetClient.DownloadDataToStream(searchQuery.ToStringSafe(), request); return items; }, newQuery)); } else { if (request.IsCanceled) { request.Warning(Messages.RequestCanceled, "HttpClientPackageRepository", "SendRequest"); //stop sending request to the remote server stopSending = true; } tasks.RemoveAt(index); } } }