private void GetExistingFrameworks(int packageKey, PackageFrameworkReport report, SqlConnection conn)
{
// Get all target frameworks in the db for this package
report.DatabaseFrameworks = new HashSet <string>(conn.Query <string>(@"
SELECT TargetFramework
FROM PackageFrameworks
WHERE Package_Key = @packageKey", new
{
packageKey = packageKey
})).ToArray();
var adds = report.PackageFrameworks.Except(report.DatabaseFrameworks).Select(targetFramework =>
new PackageFrameworkOperation()
{
Type = PackageFrameworkOperationType.Add,
Framework = targetFramework,
Applied = false,
Error = "Not Started"
});
var rems = report.DatabaseFrameworks.Except(report.PackageFrameworks).Select(targetFramework =>
new PackageFrameworkOperation()
{
Type = PackageFrameworkOperationType.Remove,
Framework = targetFramework,
Applied = false,
Error = "Not Started"
});
report.Operations = Enumerable.Concat(adds, rems).ToArray();
}
PackageFrameworkReport PopulateFrameworks(
Package package,
PackageFrameworkReport report)
{
using (var sqlConnection = new SqlConnection(ConnectionString.ConnectionString))
using (var dbExecutor = new SqlExecutor(sqlConnection))
{
sqlConnection.Open();
// Get all target frameworks in the db for this package
report.DatabaseFrameworks = new HashSet <string>(dbExecutor.Query <string>(@"
SELECT TargetFramework
FROM PackageFrameworks
WHERE Package_Key = @packageKey",
new
{
packageKey = package.Key
})).ToArray();
var adds = report.PackageFrameworks.Except(report.DatabaseFrameworks).Select(targetFramework =>
new PackageFrameworkOperation()
{
Type = PackageFrameworkOperationType.Add,
Framework = targetFramework,
Applied = false,
Error = "Not Started"
});
var rems = report.DatabaseFrameworks.Except(report.PackageFrameworks).Select(targetFramework =>
new PackageFrameworkOperation()
{
Type = PackageFrameworkOperationType.Remove,
Framework = targetFramework,
Applied = false,
Error = "Not Started"
});
report.Operations = Enumerable.Concat(adds, rems).ToArray();
}
return(report);
}
private void GetExistingFrameworks(int packageKey, PackageFrameworkReport report, SqlConnection conn)
{
// Get all target frameworks in the db for this package
report.DatabaseFrameworks = new HashSet<string>(conn.Query<string>(@"
SELECT TargetFramework
FROM PackageFrameworks
WHERE Package_Key = @packageKey", new
{
packageKey = packageKey
})).ToArray();
var adds = report.PackageFrameworks.Except(report.DatabaseFrameworks).Select(targetFramework =>
new PackageFrameworkOperation()
{
Type = PackageFrameworkOperationType.Add,
Framework = targetFramework,
Applied = false,
Error = "Not Started"
});
var rems = report.DatabaseFrameworks.Except(report.PackageFrameworks).Select(targetFramework =>
new PackageFrameworkOperation()
{
Type = PackageFrameworkOperationType.Remove,
Framework = targetFramework,
Applied = false,
Error = "Not Started"
});
report.Operations = Enumerable.Concat(adds, rems).ToArray();
}
private bool ResolveReport(PackageFrameworkReport report, SqlConnection conn)
{
report.State = PackageReportState.Unresolved;
if (report.Operations == null)
{
return false;
}
Console.WriteInfoLine(" {0}@{1} Operations to complete: {2}", report.Id, report.Version, report.Operations.Count()).Wait();
foreach (var operation in report.Operations)
{
if (operation.Type == PackageFrameworkOperationType.Add)
{
try
{
if (!WhatIf)
{
conn.Execute(@"
INSERT PackageFrameworks(TargetFramework, Package_Key)
VALUES (@targetFramework, @packageKey)", new
{
targetFramework = operation.Framework,
packageKey = report.Key
});
}
Console.WriteDataLine(" + Id={0}, Version={1}, Key={2}, Fx={3}", report.Id, report.Version, report.Key, operation.Framework).Wait();
operation.Applied = true;
}
catch (Exception ex)
{
report.State = PackageReportState.Error;
operation.Applied = false;
operation.Error = ex.ToString();
Console.WriteErrorLine(" {0}@{1} '{2}' Error: {3}", report.Id, report.Version, operation.Framework, operation.Error).Wait();
}
}
else if (operation.Type == PackageFrameworkOperationType.Remove)
{
try
{
if (!WhatIf)
{
conn.Execute(@"
DELETE PackageFrameworks
WHERE TargetFramework = @targetFramework
AND Package_Key = @packageKey", new
{
targetFramework = operation.Framework,
packageKey = report.Key
});
}
Console.WriteDataLine(" - Id={0}, Version={1}, Key={2}, Fx={3}", report.Id, report.Version, report.Key, operation.Framework).Wait();
operation.Applied = true;
}
catch (Exception ex)
{
report.State = PackageReportState.Error;
operation.Applied = false;
operation.Error = ex.ToString();
Console.WriteErrorLine(" {0}@{1} '{2}' Error: {3}", report.Id, report.Version, operation.Framework, operation.Error).Wait();
}
}
}
if (report.Operations.All(o => o.Applied))
{
report.State = PackageReportState.Resolved;
}
return report.State != PackageReportState.Error;
}
private void ProcessPackage(dynamic package, int thisPackageId, int totalCount)
{
string countPad = new string('0', totalCount.ToString().Length);
Console.WriteInfoLine("[{2}/{3} ~{4}%] Processing Package: {0}@{1} (created {5})",
(string)package.Id,
(string)package.Version,
thisPackageId.ToString(countPad),
totalCount.ToString(countPad),
(((double)thisPackageId / (double)totalCount) * 100).ToString("000.00"),
(DateTime)package.Created).Wait();
try
{
var reportPath = Path.Combine(WorkDirectory, package.Id + "_" + package.Version + ".json");
var bustedReportPath = Path.Combine(WorkDirectory, package.Id + "_" + package.Version + "_" + package.Hash + ".json");
var report = new PackageFrameworkReport()
{
Id = package.Id,
Version = package.Version,
Key = package.Key,
Hash = package.Hash,
Created = package.Created,
State = PackageReportState.Unresolved
};
if (File.Exists(bustedReportPath))
{
File.Move(bustedReportPath, reportPath);
}
using (var conn = new SqlConnection(DatabaseConnectionString))
{
bool resolved = false;
if (File.Exists(reportPath))
{
using (var reader = File.OpenText(reportPath))
{
var savedReport = (PackageFrameworkReport)_serializer.Deserialize(
reader, typeof(PackageFrameworkReport));
if (savedReport != null)
{
report = savedReport;
resolved = ResolveReport(report, conn) && report.State == PackageReportState.Resolved;
}
}
}
if (!resolved)
{
try
{
var downloadPath = DownloadPackage(package);
var nugetPackage = new ZipPackage(downloadPath);
var supportedFrameworks = GetSupportedFrameworks(nugetPackage);
report.PackageFrameworks = supportedFrameworks.ToArray();
GetExistingFrameworks((int)(package.Key), report, conn);
File.Delete(downloadPath);
ResolveReport(report, conn);
}
catch (Exception ex)
{
report.State = PackageReportState.Error;
report.Error = ex.ToString();
}
}
}
using (var writer = File.CreateText(reportPath))
{
_serializer.Serialize(writer, report);
}
Console.WriteInfoLine("[{2}/{3} ~{4}%] {6} Package: {0}@{1} (created {5})",
(string)package.Id,
(string)package.Version,
thisPackageId.ToString(countPad),
totalCount.ToString(countPad),
(((double)thisPackageId / (double)totalCount) * 100).ToString("000.00"),
(DateTime)package.Created,
report.State.ToString().PadRight(_padLength, ' ')).Wait();
if (report.State == PackageReportState.Error)
{
Console.WriteErrorLine("Recorded Error for Package {0}@{1}: {2}", report.Id, report.Version, report.Error).Wait();
}
}
catch (Exception ex)
{
Console.WriteErrorLine("[{2}/{3} ~{4}%] Error for Package: {0}@{1}: {5}",
(string)package.Id,
(string)package.Version,
thisPackageId.ToString(countPad),
totalCount.ToString(countPad),
(((double)thisPackageId / (double)totalCount) * 100).ToString("000.00"),
ex.ToString()).Wait();
}
}
public override void ExecuteCommand()
{
if (!Directory.Exists(WorkDirectory))
{
Directory.CreateDirectory(WorkDirectory);
}
Log.Info("Getting all package metadata...");
var packages = GetAllPackages();
var totalCount = packages.Count;
var processedCount = 0;
Log.Info(
"Populating frameworks for {0} packages on '{1}',",
totalCount,
ConnectionString);
packages
.AsParallel()
.AsOrdered()
.WithDegreeOfParallelism(10)
.ForAll(package =>
{
// Allocate a processed count number for this package
var thisPackageId = Interlocked.Increment(ref processedCount);
try
{
var reportPath = Path.Combine(WorkDirectory, package.Id + "_" + package.Version + ".json");
var bustedReportPath = Path.Combine(WorkDirectory, package.Id + "_" + package.Version + "_" + package.Hash + ".json");
var report = new PackageFrameworkReport()
{
Id = package.Id,
Version = package.Version,
Key = package.Key,
Hash = package.Hash,
Created = package.Created.Value,
State = PackageReportState.Unresolved
};
if (File.Exists(bustedReportPath))
{
File.Move(bustedReportPath, reportPath);
}
if (File.Exists(reportPath))
{
using (var reader = File.OpenText(reportPath))
{
report = (PackageFrameworkReport)_serializer.Deserialize(reader, typeof(PackageFrameworkReport));
}
ResolveReport(report);
}
else
{
try
{
var downloadPath = DownloadPackage(package);
using (var nugetPackage = new PackageArchiveReader(File.OpenRead(downloadPath)))
{
var supportedFrameworks = GetSupportedFrameworks(nugetPackage);
report.PackageFrameworks = supportedFrameworks.ToArray();
report = PopulateFrameworks(package, report);
}
File.Delete(downloadPath);
// Resolve the report
ResolveReport(report);
}
catch (Exception ex)
{
report.State = PackageReportState.Error;
report.Error = ex.ToString();
}
}
using (var writer = File.CreateText(reportPath))
{
_serializer.Serialize(writer, report);
}
Log.Info("[{2}/{3} {4}%] {6} Package: {0}@{1} (created {5})",
package.Id,
package.Version,
thisPackageId.ToString("000000"),
totalCount.ToString("000000"),
(((double)thisPackageId / (double)totalCount) * 100).ToString("000.00"),
package.Created.Value,
report.State.ToString().PadRight(_padLength, ' '));
}
catch (Exception ex)
{
Log.Error("[{2}/{3} {4}%] Error For Package: {0}@{1}: {5}",
package.Id,
package.Version,
thisPackageId.ToString("000000"),
totalCount.ToString("000000"),
(((double)thisPackageId / (double)totalCount) * 100).ToString("000.00"),
ex.ToString());
}
});
}
private void ResolveReport(PackageFrameworkReport report)
{
bool error = false;
using (var sqlConnection = new SqlConnection(ConnectionString.ConnectionString))
using (var dbExecutor = new SqlExecutor(sqlConnection))
{
sqlConnection.Open();
foreach (var operation in report.Operations)
{
if (!WhatIf)
{
if (operation.Type == PackageFrameworkOperationType.Add)
{
try
{
dbExecutor.Execute(@"
INSERT INTO PackageFrameworks(TargetFramework, Package_Key)
VALUES (@targetFramework, @packageKey)",
new
{
targetFramework = operation.Framework,
packageKey = report.Key
});
Log.Info(" + Id={0}, Key={1}, Fx={2}", report.Id, report.Key, operation.Framework);
operation.Applied = true;
}
catch (Exception ex)
{
error = true;
operation.Applied = false;
operation.Error = ex.ToString();
}
}
else if (operation.Type == PackageFrameworkOperationType.Remove)
{
try
{
dbExecutor.Execute(@"
DELETE FROM PackageFrameworks
WHERE TargetFramework = @targetFramework AND Package_Key = @packageKey",
new
{
targetFramework = operation.Framework,
packageKey = report.Key
});
Log.Info(" - Id={0}, Key={1}, Fx={2}", report.Id, report.Key, operation.Framework);
operation.Applied = true;
}
catch (Exception ex)
{
error = true;
operation.Applied = false;
operation.Error = ex.ToString();
}
}
}
}
}
if (error)
{
report.State = PackageReportState.Error;
}
else if (report.Operations.All(o => o.Applied))
{
report.State = PackageReportState.Resolved;
}
}
public override void ExecuteCommand()
{
if (!Directory.Exists(WorkDirectory))
{
Directory.CreateDirectory(WorkDirectory);
}
Log.Info("Getting all package metadata...");
var packages = GetAllPackages();
var totalCount = packages.Count;
var processedCount = 0;
Log.Info(
"Populating frameworks for {0} packages on '{1}',",
totalCount,
ConnectionString);
packages
.AsParallel()
.AsOrdered()
.WithDegreeOfParallelism(10)
.ForAll(package =>
{
// Allocate a processed count number for this package
var thisPackageId = Interlocked.Increment(ref processedCount);
try
{
var reportPath = Path.Combine(WorkDirectory, package.Id + "_" + package.Version + ".json");
var bustedReportPath = Path.Combine(WorkDirectory, package.Id + "_" + package.Version + "_" + package.Hash + ".json");
var report = new PackageFrameworkReport()
{
Id = package.Id,
Version = package.Version,
Key = package.Key,
Hash = package.Hash,
Created = package.Created.Value,
State = PackageReportState.Unresolved
};
if (File.Exists(bustedReportPath))
{
File.Move(bustedReportPath, reportPath);
}
if (File.Exists(reportPath))
{
using (var reader = File.OpenText(reportPath))
{
report = (PackageFrameworkReport)_serializer.Deserialize(reader, typeof(PackageFrameworkReport));
}
ResolveReport(report);
}
else
{
try
{
var downloadPath = DownloadPackage(package);
using (var nugetPackage = new PackageReader(File.OpenRead(downloadPath)))
{
var supportedFrameworks = GetSupportedFrameworks(nugetPackage);
report.PackageFrameworks = supportedFrameworks.ToArray();
report = PopulateFrameworks(package, report);
}
File.Delete(downloadPath);
// Resolve the report
ResolveReport(report);
}
catch (Exception ex)
{
report.State = PackageReportState.Error;
report.Error = ex.ToString();
}
}
using (var writer = File.CreateText(reportPath))
{
_serializer.Serialize(writer, report);
}
Log.Info("[{2}/{3} {4}%] {6} Package: {0}@{1} (created {5})",
package.Id,
package.Version,
thisPackageId.ToString("000000"),
totalCount.ToString("000000"),
(((double)thisPackageId / (double)totalCount) * 100).ToString("000.00"),
package.Created.Value,
report.State.ToString().PadRight(_padLength, ' '));
}
catch (Exception ex)
{
Log.Error("[{2}/{3} {4}%] Error For Package: {0}@{1}: {5}",
package.Id,
package.Version,
thisPackageId.ToString("000000"),
totalCount.ToString("000000"),
(((double)thisPackageId / (double)totalCount) * 100).ToString("000.00"),
ex.ToString());
}
});
}
PackageFrameworkReport PopulateFrameworks(
Package package,
PackageFrameworkReport report)
{
using (var sqlConnection = new SqlConnection(ConnectionString.ConnectionString))
using (var dbExecutor = new SqlExecutor(sqlConnection))
{
sqlConnection.Open();
// Get all target frameworks in the db for this package
report.DatabaseFrameworks = new HashSet<string>(dbExecutor.Query<string>(@"
SELECT TargetFramework
FROM PackageFrameworks
WHERE Package_Key = @packageKey",
new
{
packageKey = package.Key
})).ToArray();
var adds = report.PackageFrameworks.Except(report.DatabaseFrameworks).Select(targetFramework =>
new PackageFrameworkOperation()
{
Type = PackageFrameworkOperationType.Add,
Framework = targetFramework,
Applied = false,
Error = "Not Started"
});
var rems = report.DatabaseFrameworks.Except(report.PackageFrameworks).Select(targetFramework =>
new PackageFrameworkOperation()
{
Type = PackageFrameworkOperationType.Remove,
Framework = targetFramework,
Applied = false,
Error = "Not Started"
});
report.Operations = Enumerable.Concat(adds, rems).ToArray();
}
return report;
}
private void ResolveReport(PackageFrameworkReport report)
{
bool error = false;
using (var sqlConnection = new SqlConnection(ConnectionString.ConnectionString))
using (var dbExecutor = new SqlExecutor(sqlConnection))
{
sqlConnection.Open();
foreach (var operation in report.Operations)
{
if (!WhatIf)
{
if (operation.Type == PackageFrameworkOperationType.Add)
{
try
{
dbExecutor.Execute(@"
INSERT INTO PackageFrameworks(TargetFramework, Package_Key)
VALUES (@targetFramework, @packageKey)",
new
{
targetFramework = operation.Framework,
packageKey = report.Key
});
Log.Info(" + Id={0}, Key={1}, Fx={2}", report.Id, report.Key, operation.Framework);
operation.Applied = true;
}
catch (Exception ex)
{
error = true;
operation.Applied = false;
operation.Error = ex.ToString();
}
}
else if (operation.Type == PackageFrameworkOperationType.Remove)
{
try
{
dbExecutor.Execute(@"
DELETE FROM PackageFrameworks
WHERE TargetFramework = @targetFramework AND Package_Key = @packageKey",
new
{
targetFramework = operation.Framework,
packageKey = report.Key
});
Log.Info(" - Id={0}, Key={1}, Fx={2}", report.Id, report.Key, operation.Framework);
operation.Applied = true;
}
catch (Exception ex)
{
error = true;
operation.Applied = false;
operation.Error = ex.ToString();
}
}
}
}
}
if (error)
{
report.State = PackageReportState.Error;
}
else if (report.Operations.All(o => o.Applied))
{
report.State = PackageReportState.Resolved;
}
}
private bool ResolveReport(PackageFrameworkReport report, SqlConnection conn)
{
report.State = PackageReportState.Unresolved;
if (report.Operations == null)
{
return(false);
}
Console.WriteInfoLine(" {0}@{1} Operations to complete: {2}", report.Id, report.Version, report.Operations.Count()).Wait();
foreach (var operation in report.Operations)
{
if (operation.Type == PackageFrameworkOperationType.Add)
{
try
{
if (!WhatIf)
{
conn.Execute(@"
INSERT PackageFrameworks(TargetFramework, Package_Key)
VALUES (@targetFramework, @packageKey)", new
{
targetFramework = operation.Framework,
packageKey = report.Key
});
}
Console.WriteDataLine(" + Id={0}, Version={1}, Key={2}, Fx={3}", report.Id, report.Version, report.Key, operation.Framework).Wait();
operation.Applied = true;
}
catch (Exception ex)
{
report.State = PackageReportState.Error;
operation.Applied = false;
operation.Error = ex.ToString();
Console.WriteErrorLine(" {0}@{1} '{2}' Error: {3}", report.Id, report.Version, operation.Framework, operation.Error).Wait();
}
}
else if (operation.Type == PackageFrameworkOperationType.Remove)
{
try
{
if (!WhatIf)
{
conn.Execute(@"
DELETE PackageFrameworks
WHERE TargetFramework = @targetFramework
AND Package_Key = @packageKey", new
{
targetFramework = operation.Framework,
packageKey = report.Key
});
}
Console.WriteDataLine(" - Id={0}, Version={1}, Key={2}, Fx={3}", report.Id, report.Version, report.Key, operation.Framework).Wait();
operation.Applied = true;
}
catch (Exception ex)
{
report.State = PackageReportState.Error;
operation.Applied = false;
operation.Error = ex.ToString();
Console.WriteErrorLine(" {0}@{1} '{2}' Error: {3}", report.Id, report.Version, operation.Framework, operation.Error).Wait();
}
}
}
if (report.Operations.All(o => o.Applied))
{
report.State = PackageReportState.Resolved;
}
return(report.State != PackageReportState.Error);
}
private void ProcessPackage(dynamic package, int thisPackageId, int totalCount)
{
string countPad = new string('0', totalCount.ToString().Length);
Console.WriteInfoLine("[{2}/{3} ~{4}%] Processing Package: {0}@{1} (created {5})",
(string)package.Id,
(string)package.Version,
thisPackageId.ToString(countPad),
totalCount.ToString(countPad),
(((double)thisPackageId / (double)totalCount) * 100).ToString("000.00"),
(DateTime)package.Created).Wait();
try
{
var reportPath = Path.Combine(WorkDirectory, package.Id + "_" + package.Version + ".json");
var bustedReportPath = Path.Combine(WorkDirectory, package.Id + "_" + package.Version + "_" + package.Hash + ".json");
var report = new PackageFrameworkReport()
{
Id = package.Id,
Version = package.Version,
Key = package.Key,
Hash = package.Hash,
Created = package.Created,
State = PackageReportState.Unresolved
};
if (File.Exists(bustedReportPath))
{
File.Move(bustedReportPath, reportPath);
}
using (var conn = new SqlConnection(DatabaseConnectionString))
{
bool resolved = false;
if (File.Exists(reportPath))
{
using (var reader = File.OpenText(reportPath))
{
var savedReport = (PackageFrameworkReport)_serializer.Deserialize(
reader, typeof(PackageFrameworkReport));
if (savedReport != null)
{
report = savedReport;
resolved = ResolveReport(report, conn) && report.State == PackageReportState.Resolved;
}
}
}
if (!resolved)
{
try
{
var downloadPath = DownloadPackage(package);
var nugetPackage = new ZipPackage(downloadPath);
var supportedFrameworks = GetSupportedFrameworks(nugetPackage);
report.PackageFrameworks = supportedFrameworks.ToArray();
GetExistingFrameworks((int)(package.Key), report, conn);
File.Delete(downloadPath);
ResolveReport(report, conn);
}
catch (Exception ex)
{
report.State = PackageReportState.Error;
report.Error = ex.ToString();
}
}
}
using (var writer = File.CreateText(reportPath))
{
_serializer.Serialize(writer, report);
}
Console.WriteInfoLine("[{2}/{3} ~{4}%] {6} Package: {0}@{1} (created {5})",
(string)package.Id,
(string)package.Version,
thisPackageId.ToString(countPad),
totalCount.ToString(countPad),
(((double)thisPackageId / (double)totalCount) * 100).ToString("000.00"),
(DateTime)package.Created,
report.State.ToString().PadRight(_padLength, ' ')).Wait();
if (report.State == PackageReportState.Error)
{
Console.WriteErrorLine("Recorded Error for Package {0}@{1}: {2}", report.Id, report.Version, report.Error).Wait();
}
}
catch (Exception ex)
{
Console.WriteErrorLine("[{2}/{3} ~{4}%] Error for Package: {0}@{1}: {5}",
(string)package.Id,
(string)package.Version,
thisPackageId.ToString(countPad),
totalCount.ToString(countPad),
(((double)thisPackageId / (double)totalCount) * 100).ToString("000.00"),
ex.ToString()).Wait();
}
}
