コード例 #1
0
 public void TestSizing()
 {
     Assert.AreEqual(FileSizeUtilities.ParseFromSuffixedString("1MB"), (ulong)1024 * 1024);
     Assert.AreEqual(FileSizeUtilities.ParseFromSuffixedString("4k"), (ulong)1024 * 4);
     Assert.AreEqual(FileSizeUtilities.ParseFromSuffixedString("23gb"), (ulong)1024 * 1024 * 1024 * 23);
     Assert.AreEqual(FileSizeUtilities.ParseFromSuffixedString("7.2MB"), (ulong)Math.Floor(1024 * 1024 * 7.2));
     Assert.AreEqual(FileSizeUtilities.ParseFromSuffixedString("14b"), (ulong)1 * 14);
 }
コード例 #2
0
        protected override async IAsyncEnumerable <IContentItem> ListDirectoryAsync(Uri dir)
        {
            var httpResponse = await httpClient.GetAsync(dir);

            var responseBodyText = httpResponse.Content.ReadAsStringAsync();
            var document         = new HtmlDocument();

            document.LoadHtml(await responseBodyText);

            var tables = document.DocumentNode.Descendants("table");

            foreach (var table in tables)
            {
                foreach (var tableRow in table.Descendants("tr").Skip(1)) // Skip header
                {
                    var linkNode = tableRow.ChildNodes.Descendants("a").FirstOrDefault();
                    if (linkNode == null)
                    {
                        continue;
                    }
                    var relativeLink = linkNode.Attributes["href"].Value;
                    var completeUri  = new Uri(dir, relativeLink);
                    var dateNode     = linkNode.ParentNode.NextSibling;
                    var sizeNode     = dateNode?.NextSibling;

                    IContentItem item = relativeLink.EndsWith("/")
                        ? (IContentItem) new FolderContentItem()
                        : ContentItemStore.GetOrCreateByDownloadUrl <ContentItem>(completeUri);

                    item.Source = dir;
                    item.Title  = relativeLink;
                    item.DownloadLinks.Add(completeUri);
                    item.ReportedSizeInBytes = FileSizeUtilities.ParseFromSuffixedString(sizeNode?.InnerText);

                    if (DateTime.TryParse(dateNode?.InnerText, out DateTime dateTimeParsed))
                    {
                        item.DatePosted = dateTimeParsed;
                    }

                    yield return(item);
                }
            }
        }
コード例 #3
0
        private async Task <IContentItem> ParsePageContentsAsync(Uri url)
        {
            // The timeout here is 3 hours, why? Based on my monitoring of the website,
            // it doesn't update too often - the individual pages even less often - therefore a 3 hour timeout
            // Should be a good balance. Also reduces hammering the poor, useful website.
            var contentItem = ContentItemStore.GetOrCreateBySourceUrl <ContentItem>(url, timeout: TimeSpan.FromHours(3));

            var httpResponse = await httpClient.GetAsync(url);

            var document = new HtmlDocument();

            document.LoadHtml(await httpResponse.Content.ReadAsStringAsync());

            contentItem.Title      = HttpUtility.HtmlDecode(document.DocumentNode.Descendants().Where(node => node.HasClass("post-title")).FirstOrDefault().InnerText);
            contentItem.DatePosted = DateTime.Parse(document.DocumentNode.Descendants()
                                                    .Where(node => node.HasClass("post-date")).FirstOrDefault()
                                                    .Descendants("time").FirstOrDefault()
                                                    .GetAttributeValue("datetime", ""));
            contentItem.Source = url;
            contentItem.ReportedSizeInBytes = FileSizeUtilities.ParseFromSuffixedString(document.DocumentNode.Descendants()
                                                                                        .Where(node => node.InnerText.StartsWith("Size", StringComparison.OrdinalIgnoreCase))
                                                                                        .FirstOrDefault().NextSibling.InnerText);

            // Filter down to relevant links based on what I deduced from looking at the page source
            var linkNodes = document.DocumentNode.Descendants("section")
                            .Where(child => child.HasClass("post-contents"))
                            .SelectMany(node => node.Descendants("div"))
                            .Where(node => node.HasClass("aio-pulse"))
                            .SelectMany(node => node.Descendants("a"));

            foreach (var linkNode in linkNodes)
            {
                var link        = linkNode.GetAttributeValue("href", "<could not scrape link>");
                var linkUri     = new Uri(link);
                var resolvedUri = linkUri;
                contentItem.DownloadLinks.Add(resolvedUri);
            }
            return(contentItem);
        }