private static void CopyServerEventsToStaging(CsvReader csv, SqlConnection connection) { var dt = CreateStagingDataTable(); //remove header row csv.Read(); while (csv.Read()) { if (dt == null) { dt = CreateStagingDataTable(); } var se = new ServerEvent(); //csv.GetRecords<ServerEvent>(); var id = csv.GetField <string>(0); var pod = csv.GetField <string>(1); var server = csv.GetField <string>(2); var service = csv.GetField <string>(3); var serviceTypeId = csv.GetField <string>(4); var type = csv.GetField <string>(5); var date = csv.GetField <string>(6); var details = csv.GetField <string>(7); if (!string.IsNullOrEmpty(id + pod + server + service + serviceTypeId + type + date + details)) { dt.Rows.Add(id, pod, server, service, serviceTypeId, type, details, date); System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch(); sw.Reset(); sw.Start(); if (dt.Rows.Count == 1000) { try { WriteToServer(dt, connection); dt.Rows.Clear(); } catch (Exception e) { Console.WriteLine(e.Message); } finally { sw.Stop(); Console.WriteLine($"Finished in {sw.ElapsedMilliseconds} ms."); dt.Dispose(); } } } } WriteToServer(dt, connection); }
private static void SubmitServerEventsSQL(SqlConnection connection) { var overallCount = 0; System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch(); foreach (var service in MaverickServices) { sw.Reset(); sw.Start(); var list = new List <ServerEvent>(); string tsql = BuildServerEventsSQL(service); using (var command = new SqlCommand(tsql, connection)) { using (SqlDataReader reader = command.ExecuteReader()) { if (log) { Console.WriteLine($"Reading events for {service}"); } var count = 0; while (reader.Read()) { if (list == null) { list = new List <ServerEvent>(); } count++; overallCount++; if (log) { Console.WriteLine($"Server event #{count}."); } if (log) { Console.SetCursorPosition(0, Console.CursorTop - 1); } var serverEvent = new ServerEvent(); serverEvent.Id = reader.GetString(0); serverEvent.Pod = reader.GetString(1); serverEvent.Server = reader.GetString(2); serverEvent.Service = reader.GetString(3); serverEvent.ServiceTypeId = reader.GetString(4); serverEvent.Type = reader.GetString(5); serverEvent.Details = reader.GetString(6); serverEvent.Date = GetDateTimeOffset(reader.GetString(7)); serverEvent.Url = GetUrlFromDetails(serverEvent.Details); serverEvent.TenantId = GetTenantId(serverEvent.Details); serverEvent.AuthenticationUserId = GetAuthUserId(serverEvent.Details); serverEvent.EventMessage = GetEventMessage(serverEvent.Details); serverEvent.Route = GetRouteFromUrl(serverEvent.Url); serverEvent.IsTimeout = GetTimeout(serverEvent.Details); serverEvent.IsUnhandledException = GetUnhandledException(serverEvent.Details); serverEvent.IsFailedMonitor = GetFailedMonitor(serverEvent.Details); serverEvent.IsHangfire = GetHangfire(serverEvent.Details); serverEvent.HttpStatusCode = GetHttpStatusCode(serverEvent.Details); list.Add(serverEvent); //if(list.Count == 1000) //{ // ExtractDetailsFromServerEvents(list, connection); // sw.Stop(); // Console.WriteLine($"{list.Count} record details extracted for {service} in {sw.ElapsedMilliseconds} ms."); // list = null; //} } var message = $"{count} total events for {service}"; Console.WriteLine($"----{message}----"); LogToFile(message); Console.WriteLine(); } } if (list.Any()) { ExtractDetailsFromServerEvents(list, connection); sw.Stop(); Console.WriteLine($"Finished in {sw.ElapsedMilliseconds} ms."); list.Clear(); } list = null; } var finalMessage = $"{overallCount} records processed across {MaverickServices.Count()} services.\n\n"; LogToFile(finalMessage); Console.WriteLine(finalMessage); }