static async Task Main(string[] args) { var errors = new List <string>(); var queue = new Queue <string>(GetUrls()); QuestionDataSql questionData = new QuestionDataSql(true); using (HttpClient client = new HttpClient()) { while (queue.Count > 0) { string link = queue.Dequeue(); //Trace.WriteLine(""); //Trace.TraceInformation($"Start processing {link}"); //Trace.WriteLine(""); await SendRequest(client, link, questionData); //foreach (string item in newQ) //{ // if (!queue.Contains(item)) // { // queue.Enqueue(item); // } //} //Trace.WriteLine(""); //Trace.TraceInformation($"End processing {link}"); //Trace.WriteLine(""); //Trace.WriteLine("******************************"); //Trace.WriteLine(""); } } Console.ReadLine(); }
public static async void Run([TimerTrigger("0 */15 * * * *")] TimerInfo myTimer, TraceWriter log) { log.Info($"Start parsing function executed at: {DateTime.Now}"); int limit = 100; int.TryParse(GetEnvironmentVariable("Limit"), out limit); string connStr = GetEnvironmentVariable("DefaultConnection"); IQuestionData questionData = new QuestionDataSql(connStr); using (HttpClient client = new HttpClient()) { await SendRequest(client, questionData, limit, log); } log.Info($"End parsing function executed at: {DateTime.Now}"); }
private static async Task SendRequest(HttpClient client, string url, QuestionDataSql questionData, Tournament parentTournament = null) { string xml = ""; string fullUrl = $"https://db.chgk.info{url}/xml"; Trace.WriteLine(""); Trace.WriteLine(fullUrl); Trace.WriteLine(""); try { xml = await client.GetStringAsync(fullUrl); } catch (Exception ex) { Trace.TraceError("Exception when get request", ex); } JObject token = ParseString(xml); try { FixSchemaErrors(fullUrl, ref token); string type = token.Value <string>("Type"); int id = token.Value <int>("Id"); if (type == "Г") { var tours = token["tour"]; if (!(tours is JArray)) { token["tour"] = new JArray { tours }; } var childTournaments = (token["tour"] as JArray).Select(x => $"/tour/{x["TextId"]}");//Ч token["tour"] = null; Tournament tournament = token.ToObject <Tournament>(); questionData.InsertTournament(tournament); foreach (string tournamentUrl in childTournaments) { await SendRequest(client, tournamentUrl, questionData); } } else if (type == "Ч") { var questions = token["question"]; var tours = new List <string>(); if (questions != null) { if (!(questions is JArray)) { token["question"] = new JArray { questions }; } foreach (var x in (token["question"] as JArray)) { var t = x["ParentTextId"]; if (t.HasValues) { tours.Add($"/tour/{x["ParentTextId"]}"); } else { string textId = "/tour/" + x["TextId"].ToString().Split('-')[0]; if (!tours.Contains(textId)) { tours.Add(textId); } } } } else { var tour = token["tour"]; if (!(tour is JArray)) { token["tour"] = new JArray { tour }; } tours = (token["tour"] as JArray).Select(x => $"/tour/{x["TextId"]}") .Distinct() .ToList();//Ч token["tour"] = null; } token["question"] = null; Tournament tournament = token.ToObject <Tournament>(); questionData.InsertTournament(tournament); foreach (string tourUrl in tours) { await SendRequest(client, tourUrl, questionData, tournament); } } else if (type == "Т") { JToken question = token["question"]; if (!(question is JArray)) { token["question"] = new JArray { question }; } Tour tour = token.ToObject <Tour>(); questionData.InsertTour(tour, parentTournament); } else { throw new NotSupportedException(); } } catch (Exception ex) { Trace.TraceError($"Exception when inserting to DB, {ex.Message}", ex); } }