static async Task Main(string[] args)
{
var errors = new List <string>();
var queue = new Queue <string>(GetUrls());
QuestionDataSql questionData = new QuestionDataSql(true);
using (HttpClient client = new HttpClient())
{
while (queue.Count > 0)
{
string link = queue.Dequeue();
//Trace.WriteLine("");
//Trace.TraceInformation($"Start processing {link}");
//Trace.WriteLine("");
await SendRequest(client, link, questionData);
//foreach (string item in newQ)
//{
// if (!queue.Contains(item))
// {
// queue.Enqueue(item);
// }
//}
//Trace.WriteLine("");
//Trace.TraceInformation($"End processing {link}");
//Trace.WriteLine("");
//Trace.WriteLine("******************************");
//Trace.WriteLine("");
}
}
Console.ReadLine();
}
public static async void Run([TimerTrigger("0 */15 * * * *")] TimerInfo myTimer, TraceWriter log)
{
log.Info($"Start parsing function executed at: {DateTime.Now}");
int limit = 100;
int.TryParse(GetEnvironmentVariable("Limit"), out limit);
string connStr = GetEnvironmentVariable("DefaultConnection");
IQuestionData questionData = new QuestionDataSql(connStr);
using (HttpClient client = new HttpClient())
{
await SendRequest(client, questionData, limit, log);
}
log.Info($"End parsing function executed at: {DateTime.Now}");
}
private static async Task SendRequest(HttpClient client, string url, QuestionDataSql questionData, Tournament parentTournament = null)
{
string xml = "";
string fullUrl = $"https://db.chgk.info{url}/xml";
Trace.WriteLine("");
Trace.WriteLine(fullUrl);
Trace.WriteLine("");
try
{
xml = await client.GetStringAsync(fullUrl);
}
catch (Exception ex)
{
Trace.TraceError("Exception when get request", ex);
}
JObject token = ParseString(xml);
try
{
FixSchemaErrors(fullUrl, ref token);
string type = token.Value <string>("Type");
int id = token.Value <int>("Id");
if (type == "Г")
{
var tours = token["tour"];
if (!(tours is JArray))
{
token["tour"] = new JArray {
tours
};
}
var childTournaments = (token["tour"] as JArray).Select(x => $"/tour/{x["TextId"]}");//Ч
token["tour"] = null;
Tournament tournament = token.ToObject <Tournament>();
questionData.InsertTournament(tournament);
foreach (string tournamentUrl in childTournaments)
{
await SendRequest(client, tournamentUrl, questionData);
}
}
else if (type == "Ч")
{
var questions = token["question"];
var tours = new List <string>();
if (questions != null)
{
if (!(questions is JArray))
{
token["question"] = new JArray {
questions
};
}
foreach (var x in (token["question"] as JArray))
{
var t = x["ParentTextId"];
if (t.HasValues)
{
tours.Add($"/tour/{x["ParentTextId"]}");
}
else
{
string textId = "/tour/" + x["TextId"].ToString().Split('-')[0];
if (!tours.Contains(textId))
{
tours.Add(textId);
}
}
}
}
else
{
var tour = token["tour"];
if (!(tour is JArray))
{
token["tour"] = new JArray {
tour
};
}
tours = (token["tour"] as JArray).Select(x => $"/tour/{x["TextId"]}")
.Distinct()
.ToList();//Ч
token["tour"] = null;
}
token["question"] = null;
Tournament tournament = token.ToObject <Tournament>();
questionData.InsertTournament(tournament);
foreach (string tourUrl in tours)
{
await SendRequest(client, tourUrl, questionData, tournament);
}
}
else if (type == "Т")
{
JToken question = token["question"];
if (!(question is JArray))
{
token["question"] = new JArray {
question
};
}
Tour tour = token.ToObject <Tour>();
questionData.InsertTour(tour, parentTournament);
}
else
{
throw new NotSupportedException();
}
}
catch (Exception ex)
{
Trace.TraceError($"Exception when inserting to DB, {ex.Message}", ex);
}
}