public HashSet <Item> GetRootDirectories(string content)
{
var rootUrls = GetRootUrls(content);
var result = new HashSet <Item>();
for (var i = 0; i < rootUrls.Count; i++)
{
var element = rootUrls.ElementAt(i);
var url = UrlNormalizer.NormalizeUrl(element);
var name = UrlNormalizer.GetNameFromUrl(url);
var breadcrumbs = new List <string> {
name
};
var breadcrumbsString = breadcrumbs.Join("|");
var item = new Item
{
Url = url,
Type = ItemType.Dir,
Name = name,
Breadcrumbs = breadcrumbs,
BreadcrumbsString = breadcrumbsString,
Timestamp = DateTime.Today,
};
result.Add(item);
}
return(result);
}
public HashSet <Item> GetDirectories(List <ParsedDirectoryItemDto> tempItemsArray, Item item)
{
var result = new HashSet <Item>();
for (var i = 0; i < tempItemsArray.Count; i++)
{
var tempItem = tempItemsArray[i];
var type = GetItemType(tempItem.Type, tempItem.Content);
var name = UrlNormalizer.GetNameFromUrl(tempItem.Url);
name = NormalizeItemName(name);
var breadcrumbs = new List <string>(item.Breadcrumbs)
{
name
};
var breadcrumbsString = breadcrumbs.Join("|");
result.Add(new Item
{
Name = name,
Url = tempItem.Url,
Type = type,
Parent = item,
RootParent = item.Parent ?? item,
Breadcrumbs = breadcrumbs,
BreadcrumbsString = breadcrumbsString,
Timestamp = DateTime.Today,
});
}
return(result);
}
public Parser(UrlNormalizer urlNormalizer, IConfiguration configuration)
{
UrlNormalizer = urlNormalizer;
Configuration = configuration;
MdnRepositoryBranchNameForRootUrlsRegexPattern = Configuration["MdnRepositoryBranchNameForRootUrlsRegexPattern"];
RootUrlsRegexPattern = $"https://github.com/mdn/browser-compat-data/tree/{MdnRepositoryBranchNameForRootUrlsRegexPattern}/[a-zA-Z]+";
}