private IEnumerable <IPackage> ReadPackagesFromArray(JsonTextReader jsonReader)
{
var pkgList = new List <IPackage>();
// Inside the array, each object is an NPM package
var builder = new NodeModuleBuilder();
while (jsonReader.Read())
{
switch (jsonReader.TokenType)
{
case JsonToken.PropertyName:
if (StringComparer.Ordinal.Equals(jsonReader.Value, "package"))
{
var token = (JProperty)JToken.ReadFrom(jsonReader);
var package = ReadPackage(token.Value, builder);
if (package != null)
{
pkgList.Add(package);
}
}
continue;
case JsonToken.EndArray:
// This is the spot the function should always exit on valid data
return(pkgList);
default:
continue;
}
}
throw new JsonException("Unexpected end of stream reading the NPM catalog data array");
}
private void ReadPackagesFromArray(SQLiteConnection db, JsonTextReader jsonReader)
{
// Inside the array, each object is an NPM package
var builder = new NodeModuleBuilder();
while (jsonReader.Read())
{
switch (jsonReader.TokenType)
{
case JsonToken.StartObject:
IPackage package = ReadPackage(jsonReader, builder);
if (package != null)
{
InsertCatalogEntry(db, package);
}
break;
case JsonToken.EndArray:
// This is the spot the function should always exit on valid data
return;
default:
throw new JsonException("Unexpected JSON token in NPM catalog data array");
}
}
throw new JsonException("Unexpected end of stream reading the NPM catalog data array");
}
private static void AddAuthor(NodeModuleBuilder builder, JToken author)
{
var name = author?["name"];
if (author != null)
{
builder.AddAuthor((string)name);
}
}
private static void AddHomepage(NodeModuleBuilder builder, JToken links)
{
var homepage = links?["homepage"];
if (homepage != null)
{
builder.AddHomepage((string)homepage);
}
}
private static void AddKeywords(NodeModuleBuilder builder, JToken keywords)
{
if (keywords != null)
{
foreach (var keyword in keywords.Select(v => (string)v))
{
builder.AddKeyword(keyword);
}
}
}
private void ReadPackagesFromObject(SQLiteConnection db,
JsonTextReader jsonReader,
string registryUrl)
{
var builder = new NodeModuleBuilder();
while (jsonReader.Read())
{
if (jsonReader.TokenType == JsonToken.EndObject)
{
// Reached the end of the object literal containing the data.
// This should be the normal exit point.
return;
}
// Every property should be either the "_updated" value, or a package
if (jsonReader.TokenType != JsonToken.PropertyName)
{
throw new JsonException("Unexpected JSON token in NPM catalog data");
}
string propertyName = (string)jsonReader.Value;
// If it's "_updated", update the revision info.
if (propertyName.Equals("_updated", StringComparison.Ordinal))
{
jsonReader.Read();
db.InsertOrReplace(new RegistryInfo()
{
RegistryUrl = registryUrl,
Revision = (long)jsonReader.Value,
UpdatedOn = DateTime.Now
});
continue;
}
// Else the property should be an object literal representing the package
jsonReader.Read();
if (jsonReader.TokenType == JsonToken.StartObject)
{
IPackage package = ReadPackage(jsonReader, builder);
if (package != null)
{
InsertCatalogEntry(db, package);
}
}
else
{
throw new JsonException("Unexpected JSON token reading a package from the NPM catalog data");
}
}
throw new JsonException("Unexpected end of stream reading the NPM catalog data object");
}
private IPackage ReadPackage(JToken package, NodeModuleBuilder builder)
{
builder.Reset();
try
{
builder.Name = (string)package["name"];
if (string.IsNullOrEmpty(builder.Name))
{
// I don't believe this should ever happen if the data returned is
// well formed. Could throw an exception, but just skip instead for
// resiliency on the NTVS side.
return(null);
}
builder.AppendToDescription((string)package["description"] ?? string.Empty);
var date = package["date"];
if (date != null)
{
builder.SetDate((string)date);
}
var version = package["version"];
if (version != null)
{
var semver = SemverVersion.Parse((string)version);
builder.AddVersion(semver);
}
AddKeywords(builder, package["keywords"]);
AddAuthor(builder, package["author"]);
AddHomepage(builder, package["links"]);
return(builder.Build());
}
catch (InvalidOperationException)
{
// Occurs if a JValue appears where we expect JProperty
return(null);
}
catch (ArgumentException)
{
return(null);
}
}
private static void AddHomepage(NodeModuleBuilder builder, JToken homepage)
{
JArray homepageArray;
string homepageString;
if ((homepageArray = homepage as JArray) != null)
{
foreach (var subHomepage in homepageArray)
{
AddHomepage(builder, subHomepage);
}
}
else if (!string.IsNullOrEmpty(homepageString = (string)homepage))
{
builder.AddHomepage(homepageString);
}
}
private static void AddAuthor(NodeModuleBuilder builder, JToken author)
{
JArray authorArray;
JObject authorObject;
string name;
if ((authorArray = author as JArray) != null)
{
foreach (var subAuthor in authorArray)
{
AddAuthor(builder, subAuthor);
}
}
else if ((authorObject = author as JObject) != null)
{
AddAuthor(builder, authorObject["name"]);
}
else if (!string.IsNullOrEmpty(name = (string)author))
{
builder.AddAuthor(name);
}
}
internal void ParseResultsAndAddToDatabase(TextReader reader, string dbFilename, string registryUrl)
{
Directory.CreateDirectory(Path.GetDirectoryName(dbFilename));
using (var db = new SQLiteConnection(dbFilename)) {
db.RunInTransaction(() => {
db.CreateRegistryTableIfNotExists();
using (var jsonReader = new JsonTextReader(reader)) {
while (jsonReader.Read())
{
if (JsonToken.PropertyName != jsonReader.TokenType)
{
continue;
}
if ((string)jsonReader.Value == "_updated")
{
jsonReader.Read();
db.InsertOrReplace(new RegistryInfo()
{
RegistryUrl = registryUrl,
Revision = (long)jsonReader.Value,
UpdatedOn = DateTime.Now
});
continue;
}
var builder = new NodeModuleBuilder();
JToken token = null;
#if DEV14_OR_LATER
try {
#endif
token = JToken.ReadFrom(jsonReader);
#if DEV14_OR_LATER
}
catch (JsonReaderException) {
// Reached end of file, so continue.
break;
}
#endif
var module = token.FirstOrDefault();
while (module != null)
{
try {
builder.Name = (string)module["name"];
if (string.IsNullOrEmpty(builder.Name))
{
continue;
}
builder.AppendToDescription((string)module["description"] ?? string.Empty);
var time = module["time"];
if (time != null)
{
builder.AppendToDate((string)time["modified"]);
}
var distTags = module["dist-tags"];
if (distTags != null)
{
var latestVersion = distTags
.OfType <JProperty>()
.Where(v => (string)v.Name == "latest")
.Select(v => (string)v.Value)
.FirstOrDefault();
if (!string.IsNullOrEmpty(latestVersion))
{
try {
builder.LatestVersion = SemverVersion.Parse(latestVersion);
} catch (SemverVersionFormatException) {
OnOutputLogged(String.Format(Resources.InvalidPackageSemVersion, latestVersion, builder.Name));
}
}
}
var versions = module["versions"];
if (versions != null)
{
builder.AvailableVersions = GetVersions(versions);
}
AddKeywords(builder, module["keywords"]);
AddAuthor(builder, module["author"]);
AddHomepage(builder, module["homepage"]);
var package = builder.Build();
InsertCatalogEntry(db, package);
} catch (InvalidOperationException) {
// Occurs if a JValue appears where we expect JProperty
} catch (ArgumentException) {
OnOutputLogged(string.Format(Resources.ParsingError, builder.Name));
if (!string.IsNullOrEmpty(builder.Name))
{
var package = builder.Build();
InsertCatalogEntry(db, package);
}
}
builder.Reset();
#if DEV14_OR_LATER
try {
#endif
token = JToken.ReadFrom(jsonReader);
#if DEV14_OR_LATER
}
catch (JsonReaderException) {
// Reached end of file, so continue.
break;
}
#endif
module = token.FirstOrDefault();
}
}
}
// FTS doesn't support INSERT OR REPLACE. This is the most efficient way to bypass that limitation.
db.Execute("DELETE FROM CatalogEntry WHERE docid NOT IN (SELECT MAX(docid) FROM CatalogEntry GROUP BY Name)");
});
}
}
private void ReadPackagesFromArray(SQLiteConnection db, JsonTextReader jsonReader) {
// Inside the array, each object is an NPM package
var builder = new NodeModuleBuilder();
while (jsonReader.Read()) {
switch (jsonReader.TokenType) {
case JsonToken.StartObject:
IPackage package = ReadPackage(jsonReader, builder);
if (package != null) {
InsertCatalogEntry(db, package);
}
break;
case JsonToken.EndArray:
// This is the spot the function should always exit on valid data
return;
default:
throw new JsonException("Unexpected JSON token in NPM catalog data array");
}
}
throw new JsonException("Unexpected end of stream reading the NPM catalog data array");
}
private IPackage ReadPackage(JsonTextReader jsonReader, NodeModuleBuilder builder)
{
IPackage package = null;
builder.Reset();
try {
// The JsonTextReader should be positioned at the start of the
// object literal token for the package
var module = JToken.ReadFrom(jsonReader);
builder.Name = (string)module["name"];
if (string.IsNullOrEmpty(builder.Name))
{
// I don't believe this should ever happen if the data returned is
// well formed. Could throw an exception, but just skip instead for
// resiliency on the NTVS side.
return(null);
}
builder.AppendToDescription((string)module["description"] ?? string.Empty);
var time = module["time"];
if (time != null)
{
builder.AppendToDate((string)time["modified"]);
}
var distTags = module["dist-tags"];
if (distTags != null)
{
var latestVersion = distTags
.OfType <JProperty>()
.Where(v => (string)v.Name == "latest")
.Select(v => (string)v.Value)
.FirstOrDefault();
if (!string.IsNullOrEmpty(latestVersion))
{
try {
builder.LatestVersion = SemverVersion.Parse(latestVersion);
} catch (SemverVersionFormatException) {
OnOutputLogged(String.Format(
Resources.InvalidPackageSemVersion,
latestVersion,
builder.Name));
}
}
}
var versions = module["versions"];
if (versions != null)
{
builder.AvailableVersions = GetVersions(versions);
}
AddKeywords(builder, module["keywords"]);
AddAuthor(builder, module["author"]);
AddHomepage(builder, module["homepage"]);
package = builder.Build();
} catch (InvalidOperationException) {
// Occurs if a JValue appears where we expect JProperty
return(null);
} catch (ArgumentException) {
OnOutputLogged(string.Format(Resources.ParsingError, builder.Name));
return(null);
}
return(package);
}
private void ReadPackagesFromObject(SQLiteConnection db,
JsonTextReader jsonReader,
string registryUrl) {
var builder = new NodeModuleBuilder();
while (jsonReader.Read()) {
if (jsonReader.TokenType == JsonToken.EndObject) {
// Reached the end of the object literal containing the data.
// This should be the normal exit point.
return;
}
// Every property should be either the "_updated" value, or a package
if (jsonReader.TokenType != JsonToken.PropertyName) {
throw new JsonException("Unexpected JSON token in NPM catalog data");
}
string propertyName = (string)jsonReader.Value;
// If it's "_updated", update the revision info.
if (propertyName.Equals("_updated", StringComparison.Ordinal)) {
jsonReader.Read();
db.InsertOrReplace(new RegistryInfo() {
RegistryUrl = registryUrl,
Revision = (long)jsonReader.Value,
UpdatedOn = DateTime.Now
});
continue;
}
// Else the property should be an object literal representing the package
jsonReader.Read();
if (jsonReader.TokenType == JsonToken.StartObject) {
IPackage package = ReadPackage(jsonReader, builder);
if (package != null) {
InsertCatalogEntry(db, package);
}
} else {
throw new JsonException("Unexpected JSON token reading a package from the NPM catalog data");
}
}
throw new JsonException("Unexpected end of stream reading the NPM catalog data object");
}
internal void ParseResultsAndAddToDatabase(TextReader reader, string dbFilename, string registryUrl) {
Directory.CreateDirectory(Path.GetDirectoryName(dbFilename));
using (var db = new SQLiteConnection(dbFilename)) {
db.RunInTransaction(() => {
db.CreateRegistryTableIfNotExists();
using (var jsonReader = new JsonTextReader(reader)) {
while (jsonReader.Read()) {
if (JsonToken.PropertyName != jsonReader.TokenType) {
continue;
}
if ((string)jsonReader.Value == "_updated") {
jsonReader.Read();
db.InsertOrReplace(new RegistryInfo() {
RegistryUrl = registryUrl,
Revision = (long)jsonReader.Value,
UpdatedOn = DateTime.Now
});
continue;
}
var builder = new NodeModuleBuilder();
JToken token = null;
#if DEV14_OR_LATER
try {
#endif
token = JToken.ReadFrom(jsonReader);
#if DEV14_OR_LATER
} catch (JsonReaderException) {
// Reached end of file, so continue.
break;
}
#endif
var module = token.FirstOrDefault();
while (module != null) {
try {
builder.Name = (string)module["name"];
if (string.IsNullOrEmpty(builder.Name)) {
continue;
}
builder.AppendToDescription((string)module["description"] ?? string.Empty);
var time = module["time"];
if (time != null) {
builder.AppendToDate((string)time["modified"]);
}
var distTags = module["dist-tags"];
if (distTags != null) {
var latestVersion = distTags
.OfType<JProperty>()
.Where(v => (string)v.Name == "latest")
.Select(v => (string)v.Value)
.FirstOrDefault();
if (!string.IsNullOrEmpty(latestVersion)) {
try {
builder.LatestVersion = SemverVersion.Parse(latestVersion);
} catch (SemverVersionFormatException) {
OnOutputLogged(String.Format(Resources.InvalidPackageSemVersion, latestVersion, builder.Name));
}
}
}
var versions = module["versions"];
if (versions != null) {
builder.AvailableVersions = GetVersions(versions);
}
AddKeywords(builder, module["keywords"]);
AddAuthor(builder, module["author"]);
AddHomepage(builder, module["homepage"]);
var package = builder.Build();
InsertCatalogEntry(db, package);
} catch (InvalidOperationException) {
// Occurs if a JValue appears where we expect JProperty
} catch (ArgumentException) {
OnOutputLogged(string.Format(Resources.ParsingError, builder.Name));
if (!string.IsNullOrEmpty(builder.Name)) {
var package = builder.Build();
InsertCatalogEntry(db, package);
}
}
builder.Reset();
#if DEV14_OR_LATER
try {
#endif
token = JToken.ReadFrom(jsonReader);
#if DEV14_OR_LATER
} catch (JsonReaderException) {
// Reached end of file, so continue.
break;
}
#endif
module = token.FirstOrDefault();
}
}
}
// FTS doesn't support INSERT OR REPLACE. This is the most efficient way to bypass that limitation.
db.Execute("DELETE FROM CatalogEntry WHERE docid NOT IN (SELECT MAX(docid) FROM CatalogEntry GROUP BY Name)");
});
}
}
private static void AddAuthor(NodeModuleBuilder builder, JToken author) {
JArray authorArray;
JObject authorObject;
string name;
if ((authorArray = author as JArray) != null) {
foreach (var subAuthor in authorArray) {
AddAuthor(builder, subAuthor);
}
} else if ((authorObject = author as JObject) != null) {
AddAuthor(builder, authorObject["name"]);
} else if (!string.IsNullOrEmpty(name = (string)author)) {
builder.AddAuthor(name);
}
}
private static void AddHomepage(NodeModuleBuilder builder, JToken homepage) {
JArray homepageArray;
string homepageString;
if ((homepageArray = homepage as JArray) != null) {
foreach (var subHomepage in homepageArray) {
AddHomepage(builder, subHomepage);
}
} else if (!string.IsNullOrEmpty(homepageString = (string)homepage)) {
builder.AddHomepage(homepageString);
}
}
private static void AddKeywords(NodeModuleBuilder builder, JToken keywords) {
if (keywords != null) {
foreach (var keyword in keywords.Select(v => (string)v)) {
builder.AddKeyword(keyword);
}
}
}
private IPackage ReadPackage(JsonTextReader jsonReader, NodeModuleBuilder builder) {
IPackage package = null;
builder.Reset();
try {
// The JsonTextReader should be positioned at the start of the
// object literal token for the package
var module = JToken.ReadFrom(jsonReader);
builder.Name = (string)module["name"];
if (string.IsNullOrEmpty(builder.Name)) {
// I don't believe this should ever happen if the data returned is
// well formed. Could throw an exception, but just skip instead for
// resiliency on the NTVS side.
return null;
}
builder.AppendToDescription((string)module["description"] ?? string.Empty);
var time = module["time"];
if (time != null) {
builder.AppendToDate((string)time["modified"]);
}
var distTags = module["dist-tags"];
if (distTags != null) {
var latestVersion = distTags
.OfType<JProperty>()
.Where(v => (string)v.Name == "latest")
.Select(v => (string)v.Value)
.FirstOrDefault();
if (!string.IsNullOrEmpty(latestVersion)) {
try {
builder.LatestVersion = SemverVersion.Parse(latestVersion);
} catch (SemverVersionFormatException) {
OnOutputLogged(String.Format(
Resources.InvalidPackageSemVersion,
latestVersion,
builder.Name));
}
}
}
var versions = module["versions"];
if (versions != null) {
builder.AvailableVersions = GetVersions(versions);
}
AddKeywords(builder, module["keywords"]);
AddAuthor(builder, module["author"]);
AddHomepage(builder, module["homepage"]);
package = builder.Build();
} catch (InvalidOperationException) {
// Occurs if a JValue appears where we expect JProperty
return null;
} catch (ArgumentException) {
OnOutputLogged(string.Format(Resources.ParsingError, builder.Name));
return null;
}
return package;
}