コード例 #1
0
        public RestMSFeed Get(string name)
        {
            //TODO: Get needs Last Modified and ETag
            var feedRetriever = new FeedRetriever(feedRepository);

            return(feedRetriever.Retrieve(new Name(name)));
        }
コード例 #2
0
ファイル: Program.cs プロジェクト: pranavnegandhi/Scraper
        private async static Task Handler(string mode, Uri uri)
        {
            IFileLayout docLayout;

            if (!_modeMap.TryGetValue(mode, out docLayout))
            {
                throw new ArgumentException($"The value of {nameof(mode)} should be single or package.");
            }

            var mapper = new FeedRetriever(uri);
            await mapper.ReadAsync();

            Trace.TraceInformation($"Retrieved {mapper.Count} links.");

            var    retriever = new HttpRetriever();
            var    article   = new ArticleExtractor();
            var    images    = new ImageExtractor();
            var    context   = BrowsingContext.New(Configuration.Default);
            var    inserter  = new ArticleInserter();
            var    imgWriter = new FileWriter(_modeMap["nested"]);
            var    docWriter = new HtmlWriter(docLayout);
            string fileName;

            foreach (var i in mapper)
            {
                Trace.TraceInformation($"{i.Title}\t{i.Link}");

                var itemUri = new Uri(i.Link);
                var input   = await retriever.StartAsync(itemUri);

                var content = await article.ExtractAsync(input, "type-post");

                var imgs = await images.ExtractAllAsync(input);

                foreach (var j in imgs)
                {
                    var source = ((IHtmlImageElement)j)?.Source ?? "http://127.0.0.1";
                    var imgUri = new Uri(source);
                    var file   = await retriever.StartAsync(imgUri);

                    await imgWriter.WriteAsync(file, imgUri.LocalPath);

                    Trace.TraceInformation($"Path: {imgUri.AbsoluteUri} Length: {file.Length}");
                }

                var document = await context.CreateDocumentAsync("templates\\post.html");

                var placeholder = document.QuerySelector("div#article-placeholder");
                inserter.Replace(placeholder, content);

                fileName = itemUri.Segments[1];
                fileName = fileName.Replace("/", "");

                await docWriter.WriteAsync(document, fileName);
            }
        }
コード例 #3
0
        public RestMSFeed Get(string name)
        {
            var feedRetriever = new FeedRetriever(feedRepository);

            return(feedRetriever.Retrieve(new Name(name)));
        }