private int StoreReport(PackageLicenseReport report, SqlConnection connection) { using (SqlCommand command = connection.CreateCommand()) { command.CommandText = "AddPackageLicenseReport"; command.CommandType = CommandType.StoredProcedure; DataTable licensesNames = new DataTable(); licensesNames.Columns.Add("Name", typeof(string)); foreach (string license in report.Licenses.Select(l => l.Trim()).Distinct(StringComparer.OrdinalIgnoreCase)) { licensesNames.Rows.Add(license); } command.Parameters.AddWithValue("@licenseNames", licensesNames); command.Parameters.AddWithValue("@sequence", report.Sequence); command.Parameters.AddWithValue("@packageId", report.PackageId); command.Parameters.AddWithValue("@version", report.Version); command.Parameters.AddWithValue("@reportUrl", report.ReportUrl ?? String.Empty); command.Parameters.AddWithValue("@comment", report.Comment); command.Parameters.AddWithValue("@whatIf", WhatIf); return((int)command.ExecuteScalar()); } }
private static PackageLicenseReport CreateReport(JObject messageEvent) { PackageLicenseReport report = new PackageLicenseReport(messageEvent["sequence"].Value<int>()); report.PackageId = messageEvent.Value<string>("packageId"); report.Version = messageEvent.Value<string>("version"); report.ReportUrl = messageEvent.Value<string>("reportUrl"); report.Comment = messageEvent.Value<string>("comment"); foreach (JValue l in messageEvent["licenses"]) { report.Licenses.Add(l.Value<string>()); } return report; }
private static PackageLicenseReport CreateReport(JObject messageEvent) { PackageLicenseReport report = new PackageLicenseReport(messageEvent["sequence"].Value <int>()); report.PackageId = messageEvent.Value <string>("packageId"); report.Version = messageEvent.Value <string>("version"); report.ReportUrl = messageEvent.Value <string>("reportUrl"); report.Comment = messageEvent.Value <string>("comment"); foreach (JValue l in messageEvent["licenses"]) { report.Licenses.Add(l.Value <string>()); } return(report); }
private int StoreReport(PackageLicenseReport report, SqlConnection connection) { using (SqlCommand command = connection.CreateCommand()) { command.CommandText = "AddPackageLicenseReport"; command.CommandType = CommandType.StoredProcedure; DataTable licensesNames = new DataTable(); licensesNames.Columns.Add("Name", typeof(string)); foreach (string license in report.Licenses.Select(l => l.Trim()).Distinct(StringComparer.OrdinalIgnoreCase)) { licensesNames.Rows.Add(license); } command.Parameters.AddWithValue("@licenseNames", licensesNames); command.Parameters.AddWithValue("@sequence", report.Sequence); command.Parameters.AddWithValue("@packageId", report.PackageId); command.Parameters.AddWithValue("@version", report.Version); command.Parameters.AddWithValue("@reportUrl", report.ReportUrl ?? String.Empty); command.Parameters.AddWithValue("@comment", report.Comment); command.Parameters.AddWithValue("@whatIf", WhatIf); return (int)command.ExecuteScalar(); } }
public override void ExecuteCommand() { Log.Info("Loading URL for the next license report"); Uri nextLicenseReport = null; try { if (!WithConnection((connection, executor) => { var nextReportUrl = executor.Query <string>( @"SELECT NextLicenseReport FROM GallerySettings").FirstOrDefault(); if (String.IsNullOrEmpty(nextReportUrl)) { Log.Info("No NextLicenseReport value in GallerySettings. Using default"); } else if (!Uri.TryCreate(nextReportUrl, UriKind.Absolute, out nextLicenseReport)) { Log.Error("Invalid NextLicenseReport value in GallerySettings: {0}", nextReportUrl); return(false); } return(true); })) { return; } } catch (Exception e) { Log.Error("Database error\n\nCallstack:\n" + e.ToString()); return; } nextLicenseReport = nextLicenseReport ?? LicenseReportService; while (nextLicenseReport != null) { HttpWebResponse response = null; int tries = 0; while (tries < 10 && response == null) { HttpWebRequest request = (HttpWebRequest)WebRequest.Create(nextLicenseReport); if (LicenseReportCredentials != null) { request.Credentials = LicenseReportCredentials; } Log.Http(request); try { response = (HttpWebResponse)request.GetResponse(); } catch (WebException ex) { response = null; var httpResp = ex.Response as HttpWebResponse; if (httpResp != null) { Log.Http(httpResp); return; } else if (ex.Status == WebExceptionStatus.Timeout || ex.Status == WebExceptionStatus.ConnectFailure) { // Try again in 10 seconds tries++; if (tries < 10) { Log.Warn("Timeout connecting to service. Sleeping for 30 seconds and trying again ({0}/10 tries)", tries); Thread.Sleep(10 * 1000); } else { Log.Error("Timeout connecting to service. Tried 10 times. Aborting Job"); throw; } } else { Log.ErrorException(string.Format("WebException contacting service: {0}", ex.Status), ex); throw; } } } using (response) { Log.Http(response); if (response.StatusCode == HttpStatusCode.OK) { string content = (new StreamReader(response.GetResponseStream())).ReadToEnd(); JObject sonatypeMessage = JObject.Parse(content); if (!sonatypeMessage.IsValid(sonatypeSchema)) { Log.Error("License report is invalid"); return; } var events = sonatypeMessage["events"].Cast <JObject>().ToList(); for (int i = 0; i < events.Count; i++) { var messageEvent = events[i]; PackageLicenseReport report = CreateReport(messageEvent); bool success = true; try { WithConnection((connection) => { if (StoreReport(report, connection) == -1) { Log.Error("[{0:000}/{1:000}] Package Not Found {2} {3}", i + 1, events.Count, report.PackageId, report.Version); success = false; } }); } catch (Exception e) { Log.Error("Database error\n\nCallstack:\n{0}", e.ToString()); return; } if (success) { Log.Info("[{0:000}/{1:000}] Updated {2} {3}", i + 1, events.Count, report.PackageId, report.Version); } } if (sonatypeMessage["next"].Value <string>().Length > 0) { var nextReportUrl = sonatypeMessage["next"].Value <string>(); if (!Uri.TryCreate(nextReportUrl, UriKind.Absolute, out nextLicenseReport)) { Log.Error("Invalid NextLicenseReport value from license report service: {0}", nextReportUrl); return; } Log.Info("Found URL for the next license report: {0}", nextLicenseReport); // Record the next report to the database so we can check it again if we get aborted before finishing. if (!WhatIf) { try { WithConnection((connection, executor) => { executor.Execute(@" UPDATE GallerySettings SET NextLicenseReport = @nextLicenseReport", new { nextLicenseReport = nextLicenseReport.AbsoluteUri }); }); } catch (Exception e) { Log.Error("Database error\n\nCallstack:\n{0}", e.ToString()); return; } } } else { nextLicenseReport = null; } } else if (response.StatusCode != HttpStatusCode.NoContent) { Log.Info("Report is not available"); } else { Log.Error("URL for the next license report caused HTTP status {0}", response.StatusCode); return; } } } }
private async Task <bool> ProcessReports(Uri nextLicenseReport) { HttpWebResponse response = null; int tries = 0; Log.RequestingLicenseReport(nextLicenseReport.AbsoluteUri); while (tries < RetryCount.Value && response == null) { HttpWebRequest request = (HttpWebRequest)WebRequest.Create(nextLicenseReport); if (LicenseReportCredentials != null) { request.Credentials = LicenseReportCredentials; } WebException thrown = null; try { response = (HttpWebResponse)(await request.GetResponseAsync()); } catch (WebException ex) { response = null; if (ex.Status == WebExceptionStatus.Timeout || ex.Status == WebExceptionStatus.ConnectFailure) { // Try again in 10 seconds tries++; if (tries < RetryCount.Value) { thrown = ex; } else { throw; } } else { throw; } } if (thrown != null) { Log.ErrorDownloadingLicenseReport(nextLicenseReport.AbsoluteUri, thrown.ToString()); await Task.Delay(10 * 1000); } } Log.RequestedLicenseReport(nextLicenseReport.AbsoluteUri); Log.ProcessingLicenseReport(nextLicenseReport.AbsoluteUri); using (response) { if (response.StatusCode == HttpStatusCode.OK) { Log.ReadingLicenseReport(nextLicenseReport.AbsoluteUri); string content; using (var reader = new StreamReader(response.GetResponseStream())) { content = await reader.ReadToEndAsync(); } Log.ReadLicenseReport(nextLicenseReport.AbsoluteUri); JObject sonatypeMessage = JObject.Parse(content); if (!sonatypeMessage.IsValid(sonatypeSchema)) { Log.InvalidLicenseReport(nextLicenseReport.AbsoluteUri, Strings.UpdateLicenseReportsJob_JsonDoesNotMatchSchema); return(false); } var events = sonatypeMessage["events"].Cast <JObject>().ToList(); for (int i = 0; i < events.Count; i++) { var messageEvent = events[i]; PackageLicenseReport report = CreateReport(messageEvent); Log.StoringLicenseReport(report.PackageId, report.Version); if (await StoreReport(report) == -1) { Log.PackageNotFound(report.PackageId, report.Version); } else { Log.StoredLicenseReport(report.PackageId, report.Version); } } // Store the next URL if (sonatypeMessage["next"].Value <string>().Length > 0) { var nextReportUrl = sonatypeMessage["next"].Value <string>(); if (!Uri.TryCreate(nextReportUrl, UriKind.Absolute, out nextLicenseReport)) { Log.InvalidNextReportUrl(nextReportUrl); return(false); } Log.StoringNextReportUrl(nextLicenseReport.AbsoluteUri); // Record the next report to the database so we can check it again if we get aborted before finishing. if (!WhatIf) { using (var connection = await PackageDatabase.ConnectTo()) { await connection.QueryAsync <int>(@" UPDATE GallerySettings SET NextLicenseReport = @nextLicenseReport", new { nextLicenseReport = nextLicenseReport.AbsoluteUri }); } return(true); // Continue and read the next report later } } else { nextLicenseReport = null; } Log.ProcessedLicenseReport(nextLicenseReport.AbsoluteUri); } else if (response.StatusCode != HttpStatusCode.NoContent) { Log.NoReportYet(nextLicenseReport.AbsoluteUri); } else { Log.ReportHttpError(nextLicenseReport.AbsoluteUri, (int)response.StatusCode, response.StatusDescription); } return(false); } }