public List <AnnualStatisticData> GetDataFromGitHubWebLink3(List <ProgrammingLanguage> RequestedLanguages, List <AnnualStatisticData> ExistingStats)
{
var statData = new List <AnnualStatisticData>();
//Request URL must be changed based on current json link - if throws 404 then update the URL
WebRequest request = WebRequest.Create(
"https://madnight.github.io/githut/gh-issue-event_374d40.json");
// If required by the server, set the credentials.
request.Credentials = CredentialCache.DefaultCredentials;
// Get the response.
WebResponse response = request.GetResponse();
// Display the status.
//Console.WriteLine(((HttpWebResponse)response).StatusDescription);
// Get the stream containing content returned by the server.
// The using block ensures the stream is automatically closed.
using (Stream dataStream = response.GetResponseStream())
{
// Open the stream using a StreamReader for easy access.
StreamReader reader = new StreamReader(dataStream);
// Read the content.
var responseFromServer = reader.ReadToEnd().Replace("}\n{", "},\n{").Replace("}\r\n{", "},\n{");
var jsonResponseFormat = "{\"items\": [\n" + responseFromServer + "\n]}";
var jsons = JObject.Parse(jsonResponseFormat).SelectToken("items");
var statsGH = new List <AnnualStatisticData>();
foreach (var outerLoopItem in jsons)
{
var name = outerLoopItem.SelectToken("name").ToString();
var year = outerLoopItem.SelectToken("year").ToString();
var allQuaters = jsons.Where(innerLoopItem =>
string.Equals(innerLoopItem.SelectToken("name").ToString(), name, StringComparison.CurrentCultureIgnoreCase) &&
string.Equals(innerLoopItem.SelectToken("year").ToString(), year, StringComparison.CurrentCultureIgnoreCase));
var annualSum = allQuaters.Sum(item => int.Parse(item.SelectToken("count").ToString()));
var newRecord = new AnnualStatisticData(name, int.Parse(year));
newRecord.PullRequestsAmount = annualSum;
if (statsGH.Any(rec => rec.LanguageName == newRecord.LanguageName && rec.Year == newRecord.Year))
{
continue;
}
statsGH.Add(newRecord);
}
//selecting only requested languages
var requestedStats = statsGH.Where(record =>
RequestedLanguages.Any(language =>
string.Equals(language.Name, record.LanguageName, StringComparison.CurrentCultureIgnoreCase)))
.ToList();
//extract stats consistent with existing stats
var existingStatsDuplicates = requestedStats.Where(record => ExistingStats.Any(item =>
string.Equals(item.LanguageName, record.LanguageName, StringComparison.CurrentCultureIgnoreCase) &&
item.Year == record.Year))
.ToList();
//and those non consistent
var existingStatsNonDuplicates = ExistingStats.Where(record =>
existingStatsDuplicates.Any(item =>
item == record)).ToList();
//complete duplicated stats
foreach (var stat in existingStatsDuplicates)
{
var missingData = ExistingStats.FirstOrDefault(record =>
string.Equals(stat.LanguageName, record.LanguageName, StringComparison.CurrentCultureIgnoreCase) &&
stat.Year == record.Year);
stat.PopularitySurvey = missingData.PopularitySurvey;
}
//records from GH only:
var ghOnly = requestedStats.Where(stat1 => !existingStatsDuplicates.Any(stat2 => stat1 == stat2));
//add to the returned result
statData.AddRange(existingStatsDuplicates);
statData.AddRange(existingStatsNonDuplicates);
statData.AddRange(ghOnly);
}
// Close the response.
response.Close();
return(statData);
}
public List<AnnualStatisticData> GetDataFromBG()
{
var data_list = new List<AnnualStatisticData>();
int amount_of_books;
string[] tab_of_languages = new string[] {"java","python","c","c%2B%2B","visualBasicNET","c%23","javaScript","php","sql","objectiveC","matlab","assembly","perl","ruby","groovy","swift","go","objectPascal","visualBasic"};
for(int i=0; i < 19; i++)
{
var name_to_obj = tab_of_languages[i];
for(var year = 2015; year <= DateTime.Today.Year; year++)
{
var html = GetRawSourceCode("https://katalogagh.cyfronet.pl/search/query?match_1=PHRASE&field_1&term_1="+ tab_of_languages[i] +"&facet_date=1.201." + year + "&sort=dateNewest&theme=bgagh");
var htmlDoc = new HtmlDocument();
htmlDoc.LoadHtml(html);
var year_to_obj = year;
var checks = from check in htmlDoc.DocumentNode.Descendants("ul")
where check.Attributes["class"] != null && check.Attributes["class"].Value == "page-messages"
select check;
if(!checks.Any())
{
var inputs = from input in htmlDoc.DocumentNode.Descendants("div")
where input.Attributes["class"] != null && input.Attributes["class"].Value == "resultCount"
select input;
if (inputs.Any())
{
string s1 = (inputs.Last().InnerText);
amount_of_books = Int32.Parse(s1.Remove(0,23).Trim().Trim( new Char[] { '.' } ));
}
else amount_of_books = 0;
}
else amount_of_books = 0;
AnnualStatisticData obj = new AnnualStatisticData(name_to_obj, year_to_obj);
data_list.Add(obj);
obj.PublicationsAmount = amount_of_books;
/* foreach (AnnualStatisticData obj in data_list)
{
Console.WriteLine("Year: " + obj.Year +
"\nLanguage: " + obj.LanguageName +
"\nPublicationsAmount: " + obj.PublicationsAmount);
}*/
return data_list;
}
string GetRawSourceCode(string urlAddress)
{
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(urlAddress);
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
if (response.StatusCode == HttpStatusCode.OK)
{
Stream receiveStream = response.GetResponseStream();
StreamReader readStream = null;
if (response.CharacterSet == null)
{
readStream = new StreamReader(receiveStream);
}
else
{
readStream = new StreamReader(receiveStream, Encoding.GetEncoding(response.CharacterSet));
}
string data = readStream.ReadToEnd();
response.Close();
readStream.Close();
return data;
}
return new string("");
}
}
}