private int StoreReport(PackageLicenseReport report, SqlConnection connection)
{
using (SqlCommand command = connection.CreateCommand())
{
command.CommandText = "AddPackageLicenseReport";
command.CommandType = CommandType.StoredProcedure;
DataTable licensesNames = new DataTable();
licensesNames.Columns.Add("Name", typeof(string));
foreach (string license in report.Licenses.Select(l => l.Trim()).Distinct(StringComparer.OrdinalIgnoreCase))
{
licensesNames.Rows.Add(license);
}
command.Parameters.AddWithValue("@licenseNames", licensesNames);
command.Parameters.AddWithValue("@sequence", report.Sequence);
command.Parameters.AddWithValue("@packageId", report.PackageId);
command.Parameters.AddWithValue("@version", report.Version);
command.Parameters.AddWithValue("@reportUrl", report.ReportUrl ?? String.Empty);
command.Parameters.AddWithValue("@comment", report.Comment);
command.Parameters.AddWithValue("@whatIf", WhatIf);
return((int)command.ExecuteScalar());
}
}
private static PackageLicenseReport CreateReport(JObject messageEvent)
{
PackageLicenseReport report = new PackageLicenseReport(messageEvent["sequence"].Value<int>());
report.PackageId = messageEvent.Value<string>("packageId");
report.Version = messageEvent.Value<string>("version");
report.ReportUrl = messageEvent.Value<string>("reportUrl");
report.Comment = messageEvent.Value<string>("comment");
foreach (JValue l in messageEvent["licenses"])
{
report.Licenses.Add(l.Value<string>());
}
return report;
}
private static PackageLicenseReport CreateReport(JObject messageEvent)
{
PackageLicenseReport report = new PackageLicenseReport(messageEvent["sequence"].Value <int>());
report.PackageId = messageEvent.Value <string>("packageId");
report.Version = messageEvent.Value <string>("version");
report.ReportUrl = messageEvent.Value <string>("reportUrl");
report.Comment = messageEvent.Value <string>("comment");
foreach (JValue l in messageEvent["licenses"])
{
report.Licenses.Add(l.Value <string>());
}
return(report);
}
private int StoreReport(PackageLicenseReport report, SqlConnection connection)
{
using (SqlCommand command = connection.CreateCommand())
{
command.CommandText = "AddPackageLicenseReport";
command.CommandType = CommandType.StoredProcedure;
DataTable licensesNames = new DataTable();
licensesNames.Columns.Add("Name", typeof(string));
foreach (string license in report.Licenses.Select(l => l.Trim()).Distinct(StringComparer.OrdinalIgnoreCase))
{
licensesNames.Rows.Add(license);
}
command.Parameters.AddWithValue("@licenseNames", licensesNames);
command.Parameters.AddWithValue("@sequence", report.Sequence);
command.Parameters.AddWithValue("@packageId", report.PackageId);
command.Parameters.AddWithValue("@version", report.Version);
command.Parameters.AddWithValue("@reportUrl", report.ReportUrl ?? String.Empty);
command.Parameters.AddWithValue("@comment", report.Comment);
command.Parameters.AddWithValue("@whatIf", WhatIf);
return (int)command.ExecuteScalar();
}
}
public override void ExecuteCommand()
{
Log.Info("Loading URL for the next license report");
Uri nextLicenseReport = null;
try
{
if (!WithConnection((connection, executor) =>
{
var nextReportUrl = executor.Query <string>(
@"SELECT NextLicenseReport FROM GallerySettings").FirstOrDefault();
if (String.IsNullOrEmpty(nextReportUrl))
{
Log.Info("No NextLicenseReport value in GallerySettings. Using default");
}
else if (!Uri.TryCreate(nextReportUrl, UriKind.Absolute, out nextLicenseReport))
{
Log.Error("Invalid NextLicenseReport value in GallerySettings: {0}", nextReportUrl);
return(false);
}
return(true);
}))
{
return;
}
}
catch (Exception e)
{
Log.Error("Database error\n\nCallstack:\n" + e.ToString());
return;
}
nextLicenseReport = nextLicenseReport ?? LicenseReportService;
while (nextLicenseReport != null)
{
HttpWebResponse response = null;
int tries = 0;
while (tries < 10 && response == null)
{
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(nextLicenseReport);
if (LicenseReportCredentials != null)
{
request.Credentials = LicenseReportCredentials;
}
Log.Http(request);
try
{
response = (HttpWebResponse)request.GetResponse();
}
catch (WebException ex)
{
response = null;
var httpResp = ex.Response as HttpWebResponse;
if (httpResp != null)
{
Log.Http(httpResp);
return;
}
else if (ex.Status == WebExceptionStatus.Timeout || ex.Status == WebExceptionStatus.ConnectFailure)
{
// Try again in 10 seconds
tries++;
if (tries < 10)
{
Log.Warn("Timeout connecting to service. Sleeping for 30 seconds and trying again ({0}/10 tries)", tries);
Thread.Sleep(10 * 1000);
}
else
{
Log.Error("Timeout connecting to service. Tried 10 times. Aborting Job");
throw;
}
}
else
{
Log.ErrorException(string.Format("WebException contacting service: {0}", ex.Status), ex);
throw;
}
}
}
using (response)
{
Log.Http(response);
if (response.StatusCode == HttpStatusCode.OK)
{
string content = (new StreamReader(response.GetResponseStream())).ReadToEnd();
JObject sonatypeMessage = JObject.Parse(content);
if (!sonatypeMessage.IsValid(sonatypeSchema))
{
Log.Error("License report is invalid");
return;
}
var events = sonatypeMessage["events"].Cast <JObject>().ToList();
for (int i = 0; i < events.Count; i++)
{
var messageEvent = events[i];
PackageLicenseReport report = CreateReport(messageEvent);
bool success = true;
try
{
WithConnection((connection) =>
{
if (StoreReport(report, connection) == -1)
{
Log.Error("[{0:000}/{1:000}] Package Not Found {2} {3}", i + 1, events.Count, report.PackageId, report.Version);
success = false;
}
});
}
catch (Exception e)
{
Log.Error("Database error\n\nCallstack:\n{0}", e.ToString());
return;
}
if (success)
{
Log.Info("[{0:000}/{1:000}] Updated {2} {3}", i + 1, events.Count, report.PackageId, report.Version);
}
}
if (sonatypeMessage["next"].Value <string>().Length > 0)
{
var nextReportUrl = sonatypeMessage["next"].Value <string>();
if (!Uri.TryCreate(nextReportUrl, UriKind.Absolute, out nextLicenseReport))
{
Log.Error("Invalid NextLicenseReport value from license report service: {0}", nextReportUrl);
return;
}
Log.Info("Found URL for the next license report: {0}", nextLicenseReport);
// Record the next report to the database so we can check it again if we get aborted before finishing.
if (!WhatIf)
{
try
{
WithConnection((connection, executor) =>
{
executor.Execute(@"
UPDATE GallerySettings
SET NextLicenseReport = @nextLicenseReport",
new { nextLicenseReport = nextLicenseReport.AbsoluteUri });
});
}
catch (Exception e)
{
Log.Error("Database error\n\nCallstack:\n{0}", e.ToString());
return;
}
}
}
else
{
nextLicenseReport = null;
}
}
else if (response.StatusCode != HttpStatusCode.NoContent)
{
Log.Info("Report is not available");
}
else
{
Log.Error("URL for the next license report caused HTTP status {0}", response.StatusCode);
return;
}
}
}
}
private async Task <bool> ProcessReports(Uri nextLicenseReport)
{
HttpWebResponse response = null;
int tries = 0;
Log.RequestingLicenseReport(nextLicenseReport.AbsoluteUri);
while (tries < RetryCount.Value && response == null)
{
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(nextLicenseReport);
if (LicenseReportCredentials != null)
{
request.Credentials = LicenseReportCredentials;
}
WebException thrown = null;
try
{
response = (HttpWebResponse)(await request.GetResponseAsync());
}
catch (WebException ex)
{
response = null;
if (ex.Status == WebExceptionStatus.Timeout || ex.Status == WebExceptionStatus.ConnectFailure)
{
// Try again in 10 seconds
tries++;
if (tries < RetryCount.Value)
{
thrown = ex;
}
else
{
throw;
}
}
else
{
throw;
}
}
if (thrown != null)
{
Log.ErrorDownloadingLicenseReport(nextLicenseReport.AbsoluteUri, thrown.ToString());
await Task.Delay(10 * 1000);
}
}
Log.RequestedLicenseReport(nextLicenseReport.AbsoluteUri);
Log.ProcessingLicenseReport(nextLicenseReport.AbsoluteUri);
using (response)
{
if (response.StatusCode == HttpStatusCode.OK)
{
Log.ReadingLicenseReport(nextLicenseReport.AbsoluteUri);
string content;
using (var reader = new StreamReader(response.GetResponseStream()))
{
content = await reader.ReadToEndAsync();
}
Log.ReadLicenseReport(nextLicenseReport.AbsoluteUri);
JObject sonatypeMessage = JObject.Parse(content);
if (!sonatypeMessage.IsValid(sonatypeSchema))
{
Log.InvalidLicenseReport(nextLicenseReport.AbsoluteUri, Strings.UpdateLicenseReportsJob_JsonDoesNotMatchSchema);
return(false);
}
var events = sonatypeMessage["events"].Cast <JObject>().ToList();
for (int i = 0; i < events.Count; i++)
{
var messageEvent = events[i];
PackageLicenseReport report = CreateReport(messageEvent);
Log.StoringLicenseReport(report.PackageId, report.Version);
if (await StoreReport(report) == -1)
{
Log.PackageNotFound(report.PackageId, report.Version);
}
else
{
Log.StoredLicenseReport(report.PackageId, report.Version);
}
}
// Store the next URL
if (sonatypeMessage["next"].Value <string>().Length > 0)
{
var nextReportUrl = sonatypeMessage["next"].Value <string>();
if (!Uri.TryCreate(nextReportUrl, UriKind.Absolute, out nextLicenseReport))
{
Log.InvalidNextReportUrl(nextReportUrl);
return(false);
}
Log.StoringNextReportUrl(nextLicenseReport.AbsoluteUri);
// Record the next report to the database so we can check it again if we get aborted before finishing.
if (!WhatIf)
{
using (var connection = await PackageDatabase.ConnectTo())
{
await connection.QueryAsync <int>(@"
UPDATE GallerySettings
SET NextLicenseReport = @nextLicenseReport",
new { nextLicenseReport = nextLicenseReport.AbsoluteUri });
}
return(true); // Continue and read the next report later
}
}
else
{
nextLicenseReport = null;
}
Log.ProcessedLicenseReport(nextLicenseReport.AbsoluteUri);
}
else if (response.StatusCode != HttpStatusCode.NoContent)
{
Log.NoReportYet(nextLicenseReport.AbsoluteUri);
}
else
{
Log.ReportHttpError(nextLicenseReport.AbsoluteUri, (int)response.StatusCode, response.StatusDescription);
}
return(false);
}
}
