public ScrapperModule(IScrapperConfiguration scrapperConfiguration, IScrapper scrapper, IEnumerable <IParser> parsers) : base("Radio") { _scrapperConfiguration = scrapperConfiguration; _parsers = parsers; Get["/Get/{radio}"] = parameters => { if (!TryGetRadio((string)parameters.radio, out var radioConfiguration)) { throw new NotFoundErrorException($"Radio {(string) parameters.radio} not found"); } var parser = GetParser(radioConfiguration.Name); var song = scrapper.Scrap(radioConfiguration.Uri, parser); return(Response.AsJson(song)); }; Get["/GetAsync/{radio}", true] = async(parameters, ct) => { if (!TryGetRadio((string)parameters.radio, out var radioConfiguration)) { throw new NotFoundErrorException($"Radio {(string)parameters.radio} not found"); } var parser = GetParser(radioConfiguration.Name); var song = await scrapper.ScrapAsync(radioConfiguration.Uri, parser); return(Response.AsJson(song)); }; }
public async Task <bool> ScrapAsync(EventCategories category) { var todaysDay = DateTime.UtcNow.Day; var scrappedEvents = await _scrapper.Scrap(category); foreach (var scrappedEvent in scrappedEvents) { var existingEvent = await checkEventExists(scrappedEvent); await removeExistingEvent(existingEvent, todaysDay); } return(await this._eventRepository.AddRangeAsync(scrappedEvents)); }
public async Task Execute(string url) { Log.Debug("Execution started"); var state = new State() { Url = url, TableName = "vnzt0" }; Log.Debug("State: {@state}", state); var universities = await _universities.Scrap(state); var tasksSpecs = new Task <List <Specialty> > [universities.Count]; for (var i = 0; i < universities.Count; i++) { tasksSpecs[i] = _specialties.Scrap(universities[i]); } Task.WaitAll(tasksSpecs); var univerSpecs = new List <SpecialtyUniversity>(); for (var i = 0; i < universities.Count; i++) { foreach (var spec in tasksSpecs[i].Result) { univerSpecs.Add(new SpecialtyUniversity { Specialty = spec, UniversityIndex = i }); } } Log.Information("Aquired {Count} specialties data!", univerSpecs.Count); var tasksAbiturs = new Task <List <Abiturient> > [univerSpecs.Count]; for (var i = 0; i < univerSpecs.Count; i++) { tasksAbiturs[i] = _abiturients.Scrap(univerSpecs[i].Specialty); } Task.WaitAll(tasksAbiturs); var abitursSpecs = new List <AbiturientSpecialty>(); for (var i = 0; i < univerSpecs.Count; i++) { foreach (var abitur in tasksAbiturs[i].Result) { abitursSpecs.Add(new AbiturientSpecialty { Specialty = i, Abiturient = abitur }); } } Log.Information("Aquired {Count} students data!", abitursSpecs.Count); var datetime = DateTime.Now; var uFileName = $"{datetime}-univer.csv" .Replace("/", "-") .Replace(@"\", "-") .Replace(" ", ""); Log.Information("Writing universities csv"); using (TextWriter univerWriter = File.AppendText(uFileName)) { var csv = new CsvWriter(univerWriter); csv.WriteRecords(universities); csv.Dispose(); } Log.Information($"Saved at {uFileName}"); var sFileName = $"{datetime}-specialties.csv" .Replace("/", "-") .Replace(@"\", "-") .Replace(" ", ""); Log.Information("Writing specialties csv"); using (TextWriter specWriter = File.AppendText(sFileName)) { var csv = new CsvWriter(specWriter); csv.WriteRecords(univerSpecs); csv.Dispose(); } Log.Information($"Saved at {sFileName}"); var aFileName = $"{datetime}-abiturs.csv" .Replace("/", "-") .Replace(@"\", "-") .Replace(" ", ""); Log.Information("Writing abiturients csv"); using (TextWriter abitWriter = File.AppendText(aFileName)) { var csv = new CsvWriter(abitWriter); csv.WriteRecords(abitursSpecs.Select(x => new { abitur = x, univer = universities[univerSpecs[x.Specialty].UniversityIndex].Name, spec = univerSpecs[x.Specialty].Specialty.Name })); csv.Dispose(); } Log.Information($"Saved at {aFileName}"); Log.Information("Finished"); }