public static void AddDailyUpdate(DailyUpdate update) { SqlConnection con = null; try { con = new SqlConnection(System.Environment.GetEnvironmentVariable("ConnectionString")); con.Open(); using (var db = new Database(con)) { db.Insert(update); } } catch (Exception ex) { System.Diagnostics.Trace.TraceError("NosyRepo error in InsertStory. " + ex.ToString()); } finally { if (con != null) { con.Close(); } } }
public static string Update(TraceWriter log) { string result = "Daily update complete, nothing unusual to report.."; try { List <Company> companies = Utils.DataRepository.GetCompaniesFromDb(log); foreach (Company company in companies) { if (!String.IsNullOrEmpty(company.Symbol)) { DailyUpdate update = new DailyUpdate() { Name = company.Name, Symbol = company.Symbol, Date = DateTime.Now }; //update.Id = company.Name + "_" + update.Date.ToString("yyyyMMdd"); //update.Id = update.Id.Replace(" ", "_"); TweetSummary tweets = TwitterUtils.GetTweetSummary(company.Name, log); update.TweetCount = tweets.TweetCount; update.TweetLikes = tweets.TweetLikes; update.TweetRetweets = tweets.TweetRetweets; update.StockPrice = ServiceProxies.GetStockPrice(company.Symbol, "", log); GitHub ghub = ServiceProxies.GetGitHubStars(company.Name); if (ghub != null) { update.GitHubStars = ghub.Stars; update.GitHubWatchers = ghub.Watches; } List <News> news = ServiceProxies.SearchNews(company.Name, log); update.NewsStories = news.Count; DataRepository.AddDailyUpdate(update); // Sleep to not overload free APIs System.Threading.Thread.Sleep(12000); } } } catch (Exception ex) { log.Error(ex.Message, ex); result = "DailyUpdater Error: " + ex.ToString(); } return(result); }
public GitHubEntityData ParseData(GitHubDataSources sourceData) { GitHubEntityData gitHubCountriesData = new GitHubEntityData(); //Get data from sources to parse string confirmedData = sourceData.Sources.Where(s => s.Id == GitHubSeriesType.Confirmed.ToString()).FirstOrDefault().RawData; string deathsData = sourceData.Sources.Where(s => s.Id == GitHubSeriesType.Deaths.ToString()).FirstOrDefault().RawData; string recoveredData = sourceData.Sources.Where(s => s.Id == GitHubSeriesType.Recovered.ToString()).FirstOrDefault().RawData; //Get All the rows string[] countriesConfirmedRows = confirmedData.Split("\n"); string[] countriesDeathsRows = deathsData.Split("\n"); string[] countriesRecoveredRows = recoveredData.Split("\n"); string[] headerCols = countriesConfirmedRows[0].Split(","); //for each row, starting with 1, 0 stands for the header for (int i = 1; i < countriesConfirmedRows.Length - 1; i++) { string[] countriesConfirmedCols = GitHubTimeSeriesCSVParses.CleanUp(countriesConfirmedRows[i]).Split(","); Country country; if (gitHubCountriesData.CountryList.Where(c => c.Name == countriesConfirmedCols[1].Trim().ToUpper()) != null && gitHubCountriesData.CountryList.Where(c => c.Name == countriesConfirmedCols[1].Trim().ToUpper()).Count() > 0) { country = gitHubCountriesData.CountryList.Where(c => c.Name == countriesConfirmedCols[1].Trim().ToUpper()).FirstOrDefault(); } else { gitHubCountriesData.CountryList.Add( country = new Country() { Name = countriesConfirmedCols[1].Trim().ToUpper(), Latitude = Convert.ToDouble(countriesConfirmedCols[2].Trim()), Longitude = Convert.ToDouble(countriesConfirmedCols[3].Trim()) } ); } Region region = new Region(); region.Latitude = Convert.ToDouble(countriesConfirmedCols[2].Trim()); region.Longitude = Convert.ToDouble(countriesConfirmedCols[3].Trim()); if (string.IsNullOrEmpty(countriesConfirmedCols[0].Trim())) { region.Name = countriesConfirmedCols[1].Trim().ToUpper(); } else { region.Name = countriesConfirmedCols[0].Trim().ToUpper(); } country.Regions = country.Regions ?? new List <Region>(); country.Regions.Add(region); string[] countriesDeathsCols = null; foreach (var deathsRow in countriesDeathsRows) { if (!string.IsNullOrEmpty(deathsRow.Trim())) { string[] cols = GitHubTimeSeriesCSVParses.CleanUp(deathsRow).Split(","); string ctrName = cols[1].Trim().ToUpper(); string regName = cols[0].ToUpper().Trim() == string.Empty ? ctrName : cols[0].ToUpper().Trim(); if (regName + ctrName == region.Name + country.Name) { countriesDeathsCols = cols; break; } } } string[] countriesRecoveredCols = null; foreach (var recoveredRow in countriesRecoveredRows) { if (!string.IsNullOrEmpty(recoveredRow.Trim())) { string[] cols = GitHubTimeSeriesCSVParses.CleanUp(recoveredRow).Split(","); string ctrName = cols[1].Trim().ToUpper(); string regName = cols[0].ToUpper().Trim() == string.Empty ? ctrName : cols[0].ToUpper().Trim(); if (regName + ctrName == region.Name + country.Name) { countriesRecoveredCols = cols; break; } } } //for each column date; Format: MM/DD/YYYY for (int j = 4; j < countriesConfirmedCols.Length - 1; j++) { if (Convert.ToInt64(countriesConfirmedCols[j].Trim()) > 0) { DateTime dt = DateTime.Parse($"{headerCols[j].Split("/")[1]}/{DateConverter.ConvertMonthToString(headerCols[j].Split("/")[0])}/{headerCols[j].Split("/")[2]}"); DailyUpdate dailyUpdate = new DailyUpdate() { Date = dt, TotalCases = Convert.ToInt64(countriesConfirmedCols[j].Trim()), TotalDeaths = countriesDeathsCols != null?Convert.ToInt64(countriesDeathsCols[j].Trim()) : 0, TotalRecovered = countriesRecoveredCols != null?Convert.ToInt64(countriesRecoveredCols[j].Trim()) : 0 }; dailyUpdate.ActiveCases = dailyUpdate.TotalCases - (dailyUpdate.TotalRecovered + dailyUpdate.TotalDeaths); try { DailyUpdate dailyUpdateYesterday = gitHubCountriesData.CountryList.Where(c => c.Name == country.Name).FirstOrDefault().Regions.Where(r => r.Name == region.Name).FirstOrDefault().DailyUpdates.Where(d => d.Date.Date == dailyUpdate.Date.AddDays(-1).Date).FirstOrDefault(); if (dailyUpdateYesterday != null) { dailyUpdate.NewCases = dailyUpdate.TotalCases - dailyUpdateYesterday.TotalCases; dailyUpdate.NewDeaths = dailyUpdate.TotalDeaths - dailyUpdateYesterday.TotalDeaths; } } catch (Exception) { } region.DailyUpdates = region.DailyUpdates ?? new List <DailyUpdate>(); region.DailyUpdates.Add(dailyUpdate); } } } return(gitHubCountriesData); }
private void UpdateWorldOMetersData() { ApplicationDbContext _dbContext = new ApplicationDbContext(new DbContextOptionsBuilder <ApplicationDbContext>() .UseSqlServer(_configuration.GetConnectionString("DefaultConnection")) .Options); try { //Get data per country Covid19DataManager cov19DataManager = new Covid19DataManager(); Covid19Data cov19Data = (Covid19Data)cov19DataManager.GetData(new Covid19WebDataSource() { Id = "WorldOMeters" }); WorldOMetersDataProvider worldOMetersDataProvider = (WorldOMetersDataProvider)cov19DataManager.DataProviders.Where(d => d.DataSource.Id == "WorldOMeters").FirstOrDefault(); WorldOMetersCountriesData countriesData = worldOMetersDataProvider.GetCountries(); if (countriesData != null && countriesData.Countries.Count() > 0) { foreach (var countryData in countriesData.Countries) { //Add or update a country and its global region if (_dbContext.CountriesAffected.Any(c => c.Name == countryData.Name.ToUpper())) //Update Country { // _logger.LogInformation($"Updating data for {countryData.Name}", DateTimeOffset.Now); var dbCountry = _dbContext.CountriesAffected.Where(c => c.Name == countryData.Name.ToUpper()).FirstOrDefault(); var dbRegion = _dbContext.Regions.Where(d => d.CountryId == dbCountry.Id).FirstOrDefault(); var dbDailyUpdate = _dbContext.DailyUpdates.Where(update => update.RegionId == dbRegion.Id && update.Date.Date == DateTime.Now.Date).FirstOrDefault(); if (dbDailyUpdate != null) { dbDailyUpdate.Date = DateTime.Now; dbDailyUpdate.TotalCases = countryData.TotalCases < dbDailyUpdate.TotalCases ? dbDailyUpdate.TotalCases : countryData.TotalCases; //countryData.TotalCases; dbDailyUpdate.NewCases = countryData.NewCases; dbDailyUpdate.TotalDeaths = countryData.TotalDeaths < dbDailyUpdate.TotalDeaths ? dbDailyUpdate.TotalDeaths : countryData.TotalDeaths; //countryData.TotalDeaths; dbDailyUpdate.NewDeaths = countryData.NewDeaths; dbDailyUpdate.TotalRecovered = countryData.TotalRecovered < dbDailyUpdate.TotalRecovered ? dbDailyUpdate.TotalRecovered : countryData.TotalRecovered; //countryData.TotalRecovered; dbDailyUpdate.ActiveCases = countryData.ActiveCases; dbDailyUpdate.SeriouOrCritical = countryData.SeriouOrCritical; dbDailyUpdate.MildCondition = countryData.ActiveCases - countryData.SeriouOrCritical; dbDailyUpdate.TotalTests = countryData.TotalTests; dbDailyUpdate.TotalCasesPerOneMillionPopulation = countryData.TotalCasesPerOneMillionPopulation; dbDailyUpdate.TotalDeathsPerOneMillionPopulation = countryData.TotalDeathsPerOneMillionPopulation; dbDailyUpdate.TotalTestsPerOneMillionPopulation = countryData.TotalTestsPerOneMillionPopulation; //Check for missing data, and compose it //Compute recovered cases //if (dbDailyUpdate.TotalRecovered <= 0) //{ // dbDailyUpdate.TotalRecovered = dbDailyUpdate.TotalCases - dbDailyUpdate.TotalDeaths - dbDailyUpdate.ActiveCases; //} ////Compute mild condition //if (dbDailyUpdate.MildCondition <= 0 && dbDailyUpdate.ActiveCases > 0 && dbDailyUpdate.ActiveCases > dbDailyUpdate.SeriouOrCritical) //{ // dbDailyUpdate.MildCondition = dbDailyUpdate.ActiveCases - dbDailyUpdate.SeriouOrCritical; //} //if (dbDailyUpdate.ActiveCases <= 0) //{ // dbDailyUpdate.ActiveCases = 0; // dbDailyUpdate.MildCondition = 0; //} _dbContext.DailyUpdates.Update(dbDailyUpdate); } else { var dbDailyUpdateYesterday = _dbContext.DailyUpdates.Where(update => update.RegionId == dbRegion.Id && update.Date.Date == DateTime.Now.Date.AddDays(-1)).FirstOrDefault(); var dbNewDailyUpdate = new DailyUpdate() { RegionId = dbRegion.Id, Region = dbRegion, Date = DateTime.Now, TotalCases = dbDailyUpdateYesterday == null ? countryData.TotalCases : countryData.TotalCases >= dbDailyUpdateYesterday.TotalCases ? countryData.TotalCases : dbDailyUpdateYesterday.TotalCases, // countryData.TotalCases NewCases = countryData.NewCases, TotalDeaths = dbDailyUpdateYesterday == null ? countryData.TotalDeaths : countryData.TotalDeaths >= dbDailyUpdateYesterday.TotalDeaths ? countryData.TotalDeaths : dbDailyUpdateYesterday.TotalDeaths, //countryData.TotalDeaths, NewDeaths = countryData.NewDeaths, TotalRecovered = dbDailyUpdateYesterday == null ? countryData.TotalRecovered : countryData.TotalRecovered >= dbDailyUpdateYesterday.TotalRecovered ? countryData.TotalRecovered : dbDailyUpdateYesterday.TotalRecovered, //countryData.TotalRecovered, ActiveCases = countryData.ActiveCases, SeriouOrCritical = countryData.SeriouOrCritical, MildCondition = countryData.ActiveCases - countryData.SeriouOrCritical, TotalTests = countryData.TotalTests, TotalCasesPerOneMillionPopulation = countryData.TotalCasesPerOneMillionPopulation, TotalDeathsPerOneMillionPopulation = countryData.TotalDeathsPerOneMillionPopulation, TotalTestsPerOneMillionPopulation = countryData.TotalTestsPerOneMillionPopulation }; //Check for missing data, and compose it //Compute recovered cases //if (dbNewDailyUpdate.TotalRecovered <= 0) //{ // dbNewDailyUpdate.TotalRecovered = dbNewDailyUpdate.TotalCases - dbNewDailyUpdate.TotalDeaths - dbNewDailyUpdate.ActiveCases; //} ////Compute mild condition //if (dbNewDailyUpdate.MildCondition <= 0 && dbNewDailyUpdate.ActiveCases > 0 && dbNewDailyUpdate.ActiveCases > dbNewDailyUpdate.SeriouOrCritical) //{ // dbNewDailyUpdate.MildCondition = dbNewDailyUpdate.ActiveCases - dbNewDailyUpdate.SeriouOrCritical; //} //if (dbNewDailyUpdate.ActiveCases <= 0) //{ // dbNewDailyUpdate.ActiveCases = 0; // dbNewDailyUpdate.MildCondition = 0; //} _dbContext.DailyUpdates.Add(dbNewDailyUpdate); } _dbContext.Regions.Update(dbRegion); _dbContext.CountriesAffected.Update(dbCountry); } else //Add Country and its global region { //_logger.LogInformation($"Adding data for {countryData.Name}", DateTimeOffset.Now); var dbCountry = new Country() { Name = countryData.Name.ToUpper() }; var dbRegion = new Region() { Name = countryData.Name.ToUpper(), CountryId = dbCountry.Id, Country = dbCountry }; _dbContext.CountriesAffected.Add(dbCountry); _dbContext.Regions.Add(dbRegion); _dbContext.DailyUpdates.Add(new DailyUpdate() { RegionId = dbRegion.Id, Region = dbRegion, Date = DateTime.Now, TotalCases = countryData.TotalCases, NewCases = countryData.NewCases, TotalDeaths = countryData.TotalDeaths, NewDeaths = countryData.NewDeaths, TotalRecovered = countryData.TotalRecovered, ActiveCases = countryData.ActiveCases, SeriouOrCritical = countryData.SeriouOrCritical, MildCondition = countryData.ActiveCases - countryData.SeriouOrCritical, TotalCasesPerOneMillionPopulation = countryData.TotalCasesPerOneMillionPopulation }); } } } } catch (Exception e) { throw e; } try { //dataDB _dbContext.SaveChanges(); _dbContext.Dispose(); } catch (Exception e) { _logger.LogCritical(e, "Unable to save data", DateTimeOffset.Now); } }
private void UpdateGitHubTimeSeries() { ApplicationDbContext _dbContext = new ApplicationDbContext(new DbContextOptionsBuilder <ApplicationDbContext>() .UseSqlServer(_configuration.GetConnectionString("DefaultConnection")) .Options); try { DataExtractionHistory dataExtractionHistory = new DataExtractionHistory(); if ((DateTime.Now - dataExtractionHistory.DataHistoryExtractionDate).TotalMinutes > 60) { _logger.LogInformation("Extracting history from GitHub TimeSeries", DateTimeOffset.Now); dataExtractionHistory.WasDataExtracted = false; if (dataExtractionHistory.WasDataExtracted == false) { GitHubTimeSeriesProvider gitHubTimeSeriesProvider = new GitHubTimeSeriesProvider(); GitHubEntityData gitHubData = gitHubTimeSeriesProvider.GetData(); foreach (var gitHubCountryData in gitHubData.CountryList) { bool hasMainCountryAsARegion = false; List <DailyUpdate> dailyUpdatePerAllCountryList = new List <DailyUpdate>(); Country dbCountry = _dbContext.CountriesAffected.Where(c => c.Name == gitHubCountryData.Name)?.FirstOrDefault(); if (dbCountry != null) { foreach (var gitHubRegion in gitHubCountryData.Regions) { Region dbRegion = _dbContext.Regions.Where(r => r.Name == gitHubRegion.Name)?.FirstOrDefault(); if (dbRegion == null) //add missing region { dbRegion = _dbContext.Regions.Add(new Region() { Country = dbCountry, CountryId = dbCountry.Id, Name = gitHubRegion.Name, Longitude = gitHubRegion.Longitude, Latitude = gitHubRegion.Latitude, DailyUpdates = new List <DailyUpdate>() }).Entity; } if (dbRegion.Name == dbCountry.Name) { hasMainCountryAsARegion = true; } if (gitHubRegion.DailyUpdates != null) { foreach (DailyUpdate gitHubDailyUpdate in gitHubRegion.DailyUpdates) { DailyUpdate dbDailyUpdate = _dbContext.DailyUpdates.Where(d => d.Date.Date == gitHubDailyUpdate.Date.Date)?.FirstOrDefault(); if (dbDailyUpdate == null) //add missing DailyUpate { dbDailyUpdate = _dbContext.DailyUpdates.Add(new DailyUpdate() { Region = dbRegion, RegionId = dbRegion.Id, Date = gitHubDailyUpdate.Date, TotalCases = gitHubDailyUpdate.TotalCases, TotalDeaths = gitHubDailyUpdate.TotalDeaths, TotalRecovered = gitHubDailyUpdate.TotalRecovered, ActiveCases = gitHubDailyUpdate.ActiveCases, NewCases = gitHubDailyUpdate.NewCases, NewDeaths = gitHubDailyUpdate.NewDeaths }).Entity; } DailyUpdate dailyUpdatePerAllCountry = dailyUpdatePerAllCountryList.Where(d => d.Date.Date == dbDailyUpdate.Date.Date)?.FirstOrDefault(); if (dailyUpdatePerAllCountry == null) { dailyUpdatePerAllCountryList.Add(dailyUpdatePerAllCountry = new DailyUpdate() { TotalCases = dbDailyUpdate.TotalCases, TotalDeaths = dbDailyUpdate.TotalDeaths, TotalRecovered = dbDailyUpdate.TotalRecovered, ActiveCases = dbDailyUpdate.ActiveCases, NewCases = dbDailyUpdate.NewCases, NewDeaths = dbDailyUpdate.NewDeaths, Date = dbDailyUpdate.Date }); } else { dailyUpdatePerAllCountry.TotalCases += dbDailyUpdate.TotalCases; dailyUpdatePerAllCountry.TotalDeaths += dbDailyUpdate.TotalDeaths; dailyUpdatePerAllCountry.TotalRecovered += dbDailyUpdate.TotalRecovered; dailyUpdatePerAllCountry.ActiveCases += dbDailyUpdate.ActiveCases; dailyUpdatePerAllCountry.NewCases += dbDailyUpdate.NewCases; dailyUpdatePerAllCountry.NewDeaths += dbDailyUpdate.NewDeaths; } } } } if (hasMainCountryAsARegion == false) { Console.WriteLine(dbCountry.Id); //Ad the country region Region countryRegion = _dbContext.Regions.Where(r => r.CountryId == dbCountry.Id)?.FirstOrDefault(); if (countryRegion == null) { countryRegion = _dbContext.Regions.Add(new Region() { Country = dbCountry, CountryId = dbCountry.Id, Latitude = dbCountry.Latitude, Longitude = dbCountry.Longitude, Name = dbCountry.Name }).Entity; } foreach (var dailyUpdatePerAllCountry in dailyUpdatePerAllCountryList) { dailyUpdatePerAllCountry.Region = countryRegion; dailyUpdatePerAllCountry.RegionId = countryRegion.Id; if (_dbContext.DailyUpdates.Where(d => d.RegionId == countryRegion.Id && d.Date.Date == dailyUpdatePerAllCountry.Date.Date)?.FirstOrDefault() == null) { _dbContext.DailyUpdates.Add(dailyUpdatePerAllCountry); } } } _dbContext.CountriesAffected.Update(dbCountry); } } dataExtractionHistory.WasDataExtracted = true; } dataExtractionHistory.DataHistoryExtractionDate = DateTime.Now; } foreach (var country in _dbContext.CountriesAffected) { try { if (country.Flag == null) { using (FileStream img = new FileStream($"{_configuration.GetValue<string>("Flags:RootFolder")}{country.Name}.png", FileMode.Open, FileAccess.Read)) { using (var ms = new MemoryStream()) { img.CopyTo(ms); country.Flag = ms.ToArray(); } } } } catch (Exception e) { //_logger.LogWarning($"Unable to update flag for: {country.Flag} using path: {_configuration.GetValue<string>("Flags:RootFolder")}{country.Name}.png", DateTimeOffset.Now); } } } catch (Exception e) { throw e; } try { //dataDB _dbContext.SaveChanges(); _dbContext.Dispose(); } catch (Exception e) { _logger.LogCritical(e, "Unable to save data", DateTimeOffset.Now); } }
public IActionResult Index(string country, string region) { DailyUpdate updateNow = new DailyUpdate(); DailyUpdate updateYesterday = new DailyUpdate(); DateTime yesterdaysDate = DateTime.Now.Subtract(new TimeSpan(1, 0, 0, 0)).Date; var dbContext = _context.Regions.Include(r => r.DailyUpdates); if (!string.IsNullOrEmpty(region)) { updateNow = dbContext.Where(r => r.Name == region.ToUpper())?.FirstOrDefault()?.DailyUpdates.Where(d => d.Date.Date == DateTime.Now.Date)?.FirstOrDefault(); updateYesterday = dbContext.Where(r => r.Name == region.ToUpper())?.FirstOrDefault()?.DailyUpdates.Where(d => d.Date.Date == yesterdaysDate)?.FirstOrDefault(); } else if (!string.IsNullOrEmpty(country)) { updateNow = dbContext.Where(r => r.Name == country.ToUpper())?.FirstOrDefault()?.DailyUpdates.Where(d => d.Date.Date == DateTime.Now.Date)?.FirstOrDefault(); updateYesterday = dbContext.Where(r => r.Name == country.ToUpper())?.FirstOrDefault()?.DailyUpdates.Where(d => d.Date.Date == yesterdaysDate)?.FirstOrDefault(); } else { updateNow = new DailyUpdate() { Date = DateTime.Now, Region = new Region() { Name = "GLOBAL" } }; foreach (var u in _context.DailyUpdates.Where(d => d.Date.Date == DateTime.Now.Date)) { updateNow.ActiveCases += u.ActiveCases; updateNow.MildCondition += u.MildCondition; updateNow.NewCases += u.NewCases; updateNow.NewDeaths += u.NewDeaths; updateNow.SeriouOrCritical += u.SeriouOrCritical; updateNow.TotalCases += u.TotalCases; updateNow.TotalCasesPerOneMillionPopulation += u.TotalCasesPerOneMillionPopulation; updateNow.TotalDeaths += u.TotalDeaths; updateNow.TotalRecovered += u.TotalRecovered; updateNow.Date = DateTime.Now; } foreach (var u in _context.DailyUpdates.Where(d => d.Date.Date == yesterdaysDate)) { updateYesterday.ActiveCases += u.ActiveCases; updateYesterday.MildCondition += u.MildCondition; updateYesterday.NewCases += u.NewCases; updateYesterday.NewDeaths += u.NewDeaths; updateYesterday.SeriouOrCritical += u.SeriouOrCritical; updateYesterday.TotalCases += u.TotalCases; updateYesterday.TotalCasesPerOneMillionPopulation += u.TotalCasesPerOneMillionPopulation; updateYesterday.TotalDeaths += u.TotalDeaths; updateYesterday.TotalRecovered += u.TotalRecovered; updateYesterday.Date = yesterdaysDate; } } updateNow = updateNow ?? new DailyUpdate(); updateYesterday = updateYesterday ?? updateNow ?? new DailyUpdate(); updateYesterday.MildCondition = updateYesterday.MildCondition > updateNow.MildCondition ? updateNow.MildCondition : updateYesterday.MildCondition; updateYesterday.SeriouOrCritical = updateYesterday.SeriouOrCritical > updateNow.SeriouOrCritical ? updateNow.SeriouOrCritical : updateYesterday.SeriouOrCritical; // dbContext.OrderBy(r => r.DailyUpdates.Where(d => d.Date.Date == DateTime.Now.Date).FirstOrDefault().TotalCases); return(View(Tuple.Create(updateNow, updateYesterday))); }
} // Dummy constructor public NEventDailyUpdate(DailyUpdate update, BaseDailyUpdater updater, DateTime time) { }