private async Task <List <GitHubFile> > GetGitHubFiles(string gitHubRepository, bool navigateSubFolders = true)
{
List <GitHubFile> gitHubFiles = new List <GitHubFile>();
var files = await this.ProcessHTML(gitHubRepository, navigateSubFolders);
IEnumerable <Task> ProcessFiles()
{
foreach (var file in files)
{
yield return(Task.Run(async() =>
{
FileInformations fileInformations = await GetFileInformations(file.UrlAttribute);
file.SetLines(fileInformations.Lines);
file.SetSize(fileInformations.Size);
}));
}
}
await Task.WhenAll(ProcessFiles());
gitHubFiles = files;
return(gitHubFiles);
}
private async Task <FileInformations> GetFileInformations(string urlAttribute)
{
int lines = 0;
decimal size = 0;
try
{
var url = $"https://github.com/{urlAttribute}";
HttpWebRequest request = null;
HttpWebResponse response = null;
Request(url, ref request, ref response);
if (response.StatusCode == HttpStatusCode.OK)
{
Stream receiveStream = response.GetResponseStream();
StreamReader readStream = null;
if (response.CharacterSet == null)
{
readStream = new StreamReader(receiveStream);
}
else
{
readStream = new StreamReader(receiveStream, Encoding.GetEncoding(response.CharacterSet));
}
string data = readStream.ReadToEnd();
var doc = CQ.CreateDocument(data);
var boxHeader = doc[".Box-header"];
if (boxHeader != null)
{
var element = boxHeader
.Elements
.LastOrDefault()
.ChildElements?
.FirstOrDefault(c =>
c.Classes.Any(c2 => c2.ToUpper() == "TEXT-MONO") &&
c.Classes.Any(c2 => c2.ToUpper() == "F6"));
var text = element?.InnerText;
text = text?.Replace("\n", "")?.Trim();
var split = text?.Split(" ");
if (split != null)
{
var linesItemArray = split.FirstOrDefault(s => s == "lines" || s == "line");
var indexItemArray = split.IndexOf(linesItemArray);
if (indexItemArray > 0)
{
lines = Convert.ToInt32(split[indexItemArray - 1]);
}
var units = split[split.Length - 1];
decimal sizeFile = Convert.ToDecimal(split[split.Length - 2].Replace(".", ","));
size = ConvertToBytes(sizeFile, units);
}
else
{
}
}
else
{
}
response.Close();
readStream.Close();
}
}
catch (Exception error)
{
throw error;
}
FileInformations fileInformations = new FileInformations(lines, size);
return(fileInformations);
}
