private IEnumerable <IPackage> ReadPackagesFromArray(JsonTextReader jsonReader) { var pkgList = new List <IPackage>(); // Inside the array, each object is an NPM package var builder = new NodeModuleBuilder(); while (jsonReader.Read()) { switch (jsonReader.TokenType) { case JsonToken.PropertyName: if (StringComparer.Ordinal.Equals(jsonReader.Value, "package")) { var token = (JProperty)JToken.ReadFrom(jsonReader); var package = ReadPackage(token.Value, builder); if (package != null) { pkgList.Add(package); } } continue; case JsonToken.EndArray: // This is the spot the function should always exit on valid data return(pkgList); default: continue; } } throw new JsonException("Unexpected end of stream reading the NPM catalog data array"); }
private void ReadPackagesFromArray(SQLiteConnection db, JsonTextReader jsonReader) { // Inside the array, each object is an NPM package var builder = new NodeModuleBuilder(); while (jsonReader.Read()) { switch (jsonReader.TokenType) { case JsonToken.StartObject: IPackage package = ReadPackage(jsonReader, builder); if (package != null) { InsertCatalogEntry(db, package); } break; case JsonToken.EndArray: // This is the spot the function should always exit on valid data return; default: throw new JsonException("Unexpected JSON token in NPM catalog data array"); } } throw new JsonException("Unexpected end of stream reading the NPM catalog data array"); }
private static void AddAuthor(NodeModuleBuilder builder, JToken author) { var name = author?["name"]; if (author != null) { builder.AddAuthor((string)name); } }
private static void AddHomepage(NodeModuleBuilder builder, JToken links) { var homepage = links?["homepage"]; if (homepage != null) { builder.AddHomepage((string)homepage); } }
private static void AddKeywords(NodeModuleBuilder builder, JToken keywords) { if (keywords != null) { foreach (var keyword in keywords.Select(v => (string)v)) { builder.AddKeyword(keyword); } } }
private void ReadPackagesFromObject(SQLiteConnection db, JsonTextReader jsonReader, string registryUrl) { var builder = new NodeModuleBuilder(); while (jsonReader.Read()) { if (jsonReader.TokenType == JsonToken.EndObject) { // Reached the end of the object literal containing the data. // This should be the normal exit point. return; } // Every property should be either the "_updated" value, or a package if (jsonReader.TokenType != JsonToken.PropertyName) { throw new JsonException("Unexpected JSON token in NPM catalog data"); } string propertyName = (string)jsonReader.Value; // If it's "_updated", update the revision info. if (propertyName.Equals("_updated", StringComparison.Ordinal)) { jsonReader.Read(); db.InsertOrReplace(new RegistryInfo() { RegistryUrl = registryUrl, Revision = (long)jsonReader.Value, UpdatedOn = DateTime.Now }); continue; } // Else the property should be an object literal representing the package jsonReader.Read(); if (jsonReader.TokenType == JsonToken.StartObject) { IPackage package = ReadPackage(jsonReader, builder); if (package != null) { InsertCatalogEntry(db, package); } } else { throw new JsonException("Unexpected JSON token reading a package from the NPM catalog data"); } } throw new JsonException("Unexpected end of stream reading the NPM catalog data object"); }
private IPackage ReadPackage(JToken package, NodeModuleBuilder builder) { builder.Reset(); try { builder.Name = (string)package["name"]; if (string.IsNullOrEmpty(builder.Name)) { // I don't believe this should ever happen if the data returned is // well formed. Could throw an exception, but just skip instead for // resiliency on the NTVS side. return(null); } builder.AppendToDescription((string)package["description"] ?? string.Empty); var date = package["date"]; if (date != null) { builder.SetDate((string)date); } var version = package["version"]; if (version != null) { var semver = SemverVersion.Parse((string)version); builder.AddVersion(semver); } AddKeywords(builder, package["keywords"]); AddAuthor(builder, package["author"]); AddHomepage(builder, package["links"]); return(builder.Build()); } catch (InvalidOperationException) { // Occurs if a JValue appears where we expect JProperty return(null); } catch (ArgumentException) { return(null); } }
private static void AddHomepage(NodeModuleBuilder builder, JToken homepage) { JArray homepageArray; string homepageString; if ((homepageArray = homepage as JArray) != null) { foreach (var subHomepage in homepageArray) { AddHomepage(builder, subHomepage); } } else if (!string.IsNullOrEmpty(homepageString = (string)homepage)) { builder.AddHomepage(homepageString); } }
private static void AddAuthor(NodeModuleBuilder builder, JToken author) { JArray authorArray; JObject authorObject; string name; if ((authorArray = author as JArray) != null) { foreach (var subAuthor in authorArray) { AddAuthor(builder, subAuthor); } } else if ((authorObject = author as JObject) != null) { AddAuthor(builder, authorObject["name"]); } else if (!string.IsNullOrEmpty(name = (string)author)) { builder.AddAuthor(name); } }
internal void ParseResultsAndAddToDatabase(TextReader reader, string dbFilename, string registryUrl) { Directory.CreateDirectory(Path.GetDirectoryName(dbFilename)); using (var db = new SQLiteConnection(dbFilename)) { db.RunInTransaction(() => { db.CreateRegistryTableIfNotExists(); using (var jsonReader = new JsonTextReader(reader)) { while (jsonReader.Read()) { if (JsonToken.PropertyName != jsonReader.TokenType) { continue; } if ((string)jsonReader.Value == "_updated") { jsonReader.Read(); db.InsertOrReplace(new RegistryInfo() { RegistryUrl = registryUrl, Revision = (long)jsonReader.Value, UpdatedOn = DateTime.Now }); continue; } var builder = new NodeModuleBuilder(); JToken token = null; #if DEV14_OR_LATER try { #endif token = JToken.ReadFrom(jsonReader); #if DEV14_OR_LATER } catch (JsonReaderException) { // Reached end of file, so continue. break; } #endif var module = token.FirstOrDefault(); while (module != null) { try { builder.Name = (string)module["name"]; if (string.IsNullOrEmpty(builder.Name)) { continue; } builder.AppendToDescription((string)module["description"] ?? string.Empty); var time = module["time"]; if (time != null) { builder.AppendToDate((string)time["modified"]); } var distTags = module["dist-tags"]; if (distTags != null) { var latestVersion = distTags .OfType <JProperty>() .Where(v => (string)v.Name == "latest") .Select(v => (string)v.Value) .FirstOrDefault(); if (!string.IsNullOrEmpty(latestVersion)) { try { builder.LatestVersion = SemverVersion.Parse(latestVersion); } catch (SemverVersionFormatException) { OnOutputLogged(String.Format(Resources.InvalidPackageSemVersion, latestVersion, builder.Name)); } } } var versions = module["versions"]; if (versions != null) { builder.AvailableVersions = GetVersions(versions); } AddKeywords(builder, module["keywords"]); AddAuthor(builder, module["author"]); AddHomepage(builder, module["homepage"]); var package = builder.Build(); InsertCatalogEntry(db, package); } catch (InvalidOperationException) { // Occurs if a JValue appears where we expect JProperty } catch (ArgumentException) { OnOutputLogged(string.Format(Resources.ParsingError, builder.Name)); if (!string.IsNullOrEmpty(builder.Name)) { var package = builder.Build(); InsertCatalogEntry(db, package); } } builder.Reset(); #if DEV14_OR_LATER try { #endif token = JToken.ReadFrom(jsonReader); #if DEV14_OR_LATER } catch (JsonReaderException) { // Reached end of file, so continue. break; } #endif module = token.FirstOrDefault(); } } } // FTS doesn't support INSERT OR REPLACE. This is the most efficient way to bypass that limitation. db.Execute("DELETE FROM CatalogEntry WHERE docid NOT IN (SELECT MAX(docid) FROM CatalogEntry GROUP BY Name)"); }); } }
private IPackage ReadPackage(JsonTextReader jsonReader, NodeModuleBuilder builder) { IPackage package = null; builder.Reset(); try { // The JsonTextReader should be positioned at the start of the // object literal token for the package var module = JToken.ReadFrom(jsonReader); builder.Name = (string)module["name"]; if (string.IsNullOrEmpty(builder.Name)) { // I don't believe this should ever happen if the data returned is // well formed. Could throw an exception, but just skip instead for // resiliency on the NTVS side. return(null); } builder.AppendToDescription((string)module["description"] ?? string.Empty); var time = module["time"]; if (time != null) { builder.AppendToDate((string)time["modified"]); } var distTags = module["dist-tags"]; if (distTags != null) { var latestVersion = distTags .OfType <JProperty>() .Where(v => (string)v.Name == "latest") .Select(v => (string)v.Value) .FirstOrDefault(); if (!string.IsNullOrEmpty(latestVersion)) { try { builder.LatestVersion = SemverVersion.Parse(latestVersion); } catch (SemverVersionFormatException) { OnOutputLogged(String.Format( Resources.InvalidPackageSemVersion, latestVersion, builder.Name)); } } } var versions = module["versions"]; if (versions != null) { builder.AvailableVersions = GetVersions(versions); } AddKeywords(builder, module["keywords"]); AddAuthor(builder, module["author"]); AddHomepage(builder, module["homepage"]); package = builder.Build(); } catch (InvalidOperationException) { // Occurs if a JValue appears where we expect JProperty return(null); } catch (ArgumentException) { OnOutputLogged(string.Format(Resources.ParsingError, builder.Name)); return(null); } return(package); }
internal void ParseResultsAndAddToDatabase(TextReader reader, string dbFilename, string registryUrl) { Directory.CreateDirectory(Path.GetDirectoryName(dbFilename)); using (var db = new SQLiteConnection(dbFilename)) { db.RunInTransaction(() => { db.CreateRegistryTableIfNotExists(); using (var jsonReader = new JsonTextReader(reader)) { while (jsonReader.Read()) { if (JsonToken.PropertyName != jsonReader.TokenType) { continue; } if ((string)jsonReader.Value == "_updated") { jsonReader.Read(); db.InsertOrReplace(new RegistryInfo() { RegistryUrl = registryUrl, Revision = (long)jsonReader.Value, UpdatedOn = DateTime.Now }); continue; } var builder = new NodeModuleBuilder(); JToken token = null; #if DEV14_OR_LATER try { #endif token = JToken.ReadFrom(jsonReader); #if DEV14_OR_LATER } catch (JsonReaderException) { // Reached end of file, so continue. break; } #endif var module = token.FirstOrDefault(); while (module != null) { try { builder.Name = (string)module["name"]; if (string.IsNullOrEmpty(builder.Name)) { continue; } builder.AppendToDescription((string)module["description"] ?? string.Empty); var time = module["time"]; if (time != null) { builder.AppendToDate((string)time["modified"]); } var distTags = module["dist-tags"]; if (distTags != null) { var latestVersion = distTags .OfType<JProperty>() .Where(v => (string)v.Name == "latest") .Select(v => (string)v.Value) .FirstOrDefault(); if (!string.IsNullOrEmpty(latestVersion)) { try { builder.LatestVersion = SemverVersion.Parse(latestVersion); } catch (SemverVersionFormatException) { OnOutputLogged(String.Format(Resources.InvalidPackageSemVersion, latestVersion, builder.Name)); } } } var versions = module["versions"]; if (versions != null) { builder.AvailableVersions = GetVersions(versions); } AddKeywords(builder, module["keywords"]); AddAuthor(builder, module["author"]); AddHomepage(builder, module["homepage"]); var package = builder.Build(); InsertCatalogEntry(db, package); } catch (InvalidOperationException) { // Occurs if a JValue appears where we expect JProperty } catch (ArgumentException) { OnOutputLogged(string.Format(Resources.ParsingError, builder.Name)); if (!string.IsNullOrEmpty(builder.Name)) { var package = builder.Build(); InsertCatalogEntry(db, package); } } builder.Reset(); #if DEV14_OR_LATER try { #endif token = JToken.ReadFrom(jsonReader); #if DEV14_OR_LATER } catch (JsonReaderException) { // Reached end of file, so continue. break; } #endif module = token.FirstOrDefault(); } } } // FTS doesn't support INSERT OR REPLACE. This is the most efficient way to bypass that limitation. db.Execute("DELETE FROM CatalogEntry WHERE docid NOT IN (SELECT MAX(docid) FROM CatalogEntry GROUP BY Name)"); }); } }
private IPackage ReadPackage(JsonTextReader jsonReader, NodeModuleBuilder builder) { IPackage package = null; builder.Reset(); try { // The JsonTextReader should be positioned at the start of the // object literal token for the package var module = JToken.ReadFrom(jsonReader); builder.Name = (string)module["name"]; if (string.IsNullOrEmpty(builder.Name)) { // I don't believe this should ever happen if the data returned is // well formed. Could throw an exception, but just skip instead for // resiliency on the NTVS side. return null; } builder.AppendToDescription((string)module["description"] ?? string.Empty); var time = module["time"]; if (time != null) { builder.AppendToDate((string)time["modified"]); } var distTags = module["dist-tags"]; if (distTags != null) { var latestVersion = distTags .OfType<JProperty>() .Where(v => (string)v.Name == "latest") .Select(v => (string)v.Value) .FirstOrDefault(); if (!string.IsNullOrEmpty(latestVersion)) { try { builder.LatestVersion = SemverVersion.Parse(latestVersion); } catch (SemverVersionFormatException) { OnOutputLogged(String.Format( Resources.InvalidPackageSemVersion, latestVersion, builder.Name)); } } } var versions = module["versions"]; if (versions != null) { builder.AvailableVersions = GetVersions(versions); } AddKeywords(builder, module["keywords"]); AddAuthor(builder, module["author"]); AddHomepage(builder, module["homepage"]); package = builder.Build(); } catch (InvalidOperationException) { // Occurs if a JValue appears where we expect JProperty return null; } catch (ArgumentException) { OnOutputLogged(string.Format(Resources.ParsingError, builder.Name)); return null; } return package; }