public async Task Parse(UrlSource twitterSource) { var result = await client.GetStringAsync(twitterSource.Url); JArray jsonDat = JArray.Parse(result); foreach (var tweet in jsonDat.Children()) { string user = "******" + tweet["user"]["screen_name"].ToString(); string createdTime = tweet["created_at"].ToString(); string tweettext = tweet["text"].ToString(); string profileBanner = tweet["user"]["profile_banner_url"].ToString(); const string format = "ddd MMM dd HH:mm:ss zzzz yyyy"; DateTime dateTimeResult = DateTime.ParseExact(createdTime, format, CultureInfo.InvariantCulture); createdTime = dateTimeResult.ToString("ddd, d MMM yyyy"); TwitterData.Add(new Item { Title = user, Subtitle = createdTime, Description = tweettext, Image = profileBanner, Group = twitterSource.Group, UrlSource = twitterSource }); } }
public async Task <List <Item> > GetItems() { if (AppSettings.YoutubePublicAPIKey.Length < 30 && !AppSettings.EnableRemoteUrlSourceService) { ServiceLocator.MessageService.ShowErrorAsync("YoutubeService is enabled but YoutubePublicAPIKey appears to be invalid please check this value in AppSettings.cs", "Application Error"); } try { YoutubeData = new List <Item>(); foreach (var youtubeSource in AppSettings.YoutubeAddressCollection) { youtubeSource.Type = "YoutubeSource"; currentYoutubeSource = youtubeSource; await Parse(youtubeSource); } } catch { ServiceLocator.MessageService.ShowErrorAsync("Error when retrieving items from YoutubeService", "Application Error"); } return(YoutubeData); }
public void TestUrlSourceCheckExisting() { AsserIgnoreIfNoInternetConnection(); UrlSource source = new UrlSource("https://www.google.com/robots.txt"); Assert.IsTrue(source.Exists()); }
public void TestUrlSourceCheckNonExisting() { AsserIgnoreIfNoInternetConnection(); UrlSource source = new UrlSource("http://thisisnotavalidwebsiteisitnoitisnt.noitisnt/notexistingfile.dat"); Assert.IsFalse(source.Exists()); }
public void TestUrlSourceAccessNonExisting() { AsserIgnoreIfNoInternetConnection(); UrlSource source = new UrlSource("http://thisisnotavalidwebsiteisitnoitisnt.noitisnt/notexistingfile.dat"); Assert.Throws <InvalidOperationException>(() => source.Prepare()); Assert.Throws <InvalidOperationException>(() => source.Retrieve()); }
public async Task Parse(UrlSource youtubeSource) { var _UserAgent = "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; WOW64; Trident/6.0)"; httpClient.DefaultRequestHeaders.Add("user-agent", _UserAgent); var result = await httpClient.GetStringAsync(youtubeSource.Url); JObject jsonDat = JObject.Parse(result); string nextPageToken = null; try { nextPageToken = jsonDat["nextPageToken"].ToString(); } catch { nextPageToken = null; } JArray jsonItems = JArray.Parse(jsonDat["items"].ToString()); string youtubeHtmlTemplate = "<p><a href=\"{0}\"><img src=\"{1}\" alt=\"\" width=300></a></p><p><a style=\"font-size: 15px; font-weight: bold; font-decoration: none;\" href=\"{0}\">{2}</a></p><p>{3}</p>"; string youtubePrefix; youtubePrefix = AppSettings.ForceYoutubeVideosToLoadFullScreen ? "/watch_popup?v=" : "/watch?v="; foreach (var youtubeItem in jsonItems) { var description = youtubeItem["snippet"]["description"].ToString(); if (description != "This video is private." && description != "This video is unavailable.") { YoutubeData.Add(new Item { Title = youtubeItem["snippet"]["title"].ToString(), Subtitle = youtubeItem["snippet"]["publishedAt"].ToString(), Description = string.Format(youtubeHtmlTemplate, "https://www.youtube.com" + youtubePrefix + youtubeItem["snippet"]["resourceId"]["videoId"], youtubeItem["snippet"]["thumbnails"]["high"]["url"].ToString(), youtubeItem["snippet"]["title"].ToString(), description), Image = youtubeItem["snippet"]["thumbnails"]["medium"]["url"].ToString(), Group = youtubeSource.Group, UrlSource = currentYoutubeSource }); } } if (nextPageToken != null) { await Parse(new UrlSource { Url = currentYoutubeSource.Url + "&pageToken=" + nextPageToken, Group = currentYoutubeSource.Group }); } }
public JsonValue ToJson(JsonSerializer serializer) { var json = new JsonObject(); Id.Serialize(json, serializer, "id"); Board.SerializeId(json, "idBoard"); Closed.Serialize(json, serializer, "closed"); Desc.Serialize(json, serializer, "desc"); Due.Serialize(json, serializer, "due", ForceDueDate); List.SerializeId(json, "idList"); Name.Serialize(json, serializer, "name"); Pos.Serialize(json, serializer, "pos"); Subscribed.Serialize(json, serializer, "subscribed"); CardSource.SerializeId(json, "idCardSource"); UrlSource.Serialize(json, serializer, "urlSource"); return(json); }
public void TestUrlSourceAccessExisting() { AsserIgnoreIfNoInternetConnection(); UrlSource source = new UrlSource("https://www.google.com/robots.txt", Path.GetTempPath() + ".unittestfileurltest1"); Assert.Throws <InvalidOperationException>(() => source.Retrieve()); source.Prepare(); Stream stream = source.Retrieve(); Assert.IsNotNull(stream); stream.Dispose(); File.Delete(Path.GetTempPath() + ".unittestfileurltest1"); }
public static async Task GetRemoteUrlSources() { string remoteUrlSourceFile = await httpClient.GetStringAsync(AppSettings.RemoteUrlSourceUrl); if (!String.IsNullOrWhiteSpace(remoteUrlSourceFile)) { // Parse the file into a list of strings. Split by either carraige return, or // line feed to account for platform differences in the platform that created // the RemoteUrlSource feeds list file. string[] arrayCRLF = { "\r", "\n" }; string[] RemoteUrlSources = remoteUrlSourceFile.Split(arrayCRLF, StringSplitOptions.RemoveEmptyEntries); // Now parse each additional feed and add it to the master collection. foreach (string RemoteUrlSourceAsStr in RemoteUrlSources) { UrlSource source = stringToUrlSource(RemoteUrlSourceAsStr); switch (source.Type) { case "RssService": ArrayUtilities.Add(ref AppSettings.RssAddressCollection, source); AppSettings.EnableRssService = true; break; case "YoutubeService": ArrayUtilities.Add(ref AppSettings.YoutubeAddressCollection, source); AppSettings.EnableYoutubeService = true; break; case "TwitterService": ArrayUtilities.Add(ref AppSettings.TwitterAddressCollection, source); //Twitter should be enabled prior to app submission on account of Oauth requirements break; default: ServiceLocator.MessageService.ShowErrorAsync("Unable to add UrlSource of type " + source.Type + " to a valid collection, source will not be retrieved", "Application Error"); break; } } } }
public async Task DoLike(string url, string token) { string email = _tokenService.DecodeToken(token); if (string.IsNullOrEmpty(email)) { throw new UnauthorizedAccessException("invalid token"); } var liker = await _likerReository.GetByEmailAsync(email) ?? new Liker { Email = email }; if (liker.HasLiked(url)) { throw new UnauthorizedAccessException("user already liked this url"); } var urlSource = await _urlRepository.GetByUrlAsync(url); if (urlSource != null) { urlSource.AddUserWhoLiked(liker); await _urlRepository.UpdateAsync(urlSource); } else { var newUrl = new UrlSource { Url = url }; newUrl.AddUserWhoLiked(liker); await _urlRepository.InsertAsync(newUrl); } await _cache.RemoveAsync(url); }
public JsonValue ToJson(JsonSerializer serializer) { var json = new JsonObject(); Id.Serialize(json, serializer, "id"); Board.SerializeId(json, "idBoard"); Closed.Serialize(json, serializer, "closed"); Desc.Serialize(json, serializer, "desc"); Due.Serialize(json, serializer, "due", ForceDueDate); DueComplete.Serialize(json, serializer, "dueComplete"); List.SerializeId(json, "idList"); Name.Serialize(json, serializer, "name"); Pos.Serialize(json, serializer, "pos"); Subscribed.Serialize(json, serializer, "subscribed"); CardSource.SerializeId(json, "idCardSource"); UrlSource.Serialize(json, serializer, "urlSource"); IdMembers.Serialize(json, serializer, "idMembers"); IdLabels.Serialize(json, serializer, "idLabels"); if (KeepFromSource != CardCopyKeepFromSourceOptions.None) { KeepFromSource.Serialize(json, serializer, "keepFromSource"); } return(json); }
private async Task AddSubscribeAsync() { var cancellationToken = LoadSnapshotCancellationSource.Token; var config = new HostedConfig(); // Create source switch (sourcePivot.SelectedIndex) { case 0: // URL if (string.IsNullOrWhiteSpace(urlText.Text)) { throw new OperationCanceledException("Empty input"); } if (!Uri.IsWellFormedUriString(urlText.Text, UriKind.Absolute)) { throw new InvalidDataException("The URL is invalid"); } var urlSource = new UrlSource() { Url = urlText.Text }; config.Source = urlSource; break; case 1: // File if (!(previewFileNameText.DataContext is StorageFile file)) { file = await FileOpenPicker.PickSingleFileAsync(); } if (file == null) { throw new InvalidDataException("No file is chosen"); } config.Source = new FileSource(file); break; default: throw new NotSupportedException("The source type is not supported yet"); } // Create format Snapshot snapshot; cancellationToken.ThrowIfCancellationRequested(); switch (((NewHostedConfigType)hostedTypeGridView.SelectedItem).Id) { case "ssd": config.Format = new Ssd(); break; case "clash": config.Format = new Clash(); break; default: throw new NotSupportedException("The hosted config type is not supported yet"); } using (var source = await config.Source.FetchAsync().AsTask(cancellationToken)) { snapshot = await source.DecodeAsync(config.Format).AsTask(cancellationToken); } config.Name = config.Source.GetFileName(); // Save cancellationToken.ThrowIfCancellationRequested(); var configFile = await HostedUtils.SaveHostedConfigAsync(config); try { _ = CachedFileManager.CompleteUpdatesAsync(configFile); cancellationToken.ThrowIfCancellationRequested(); // No need to update local file _ = await HostedUtils.SaveSnapshotAsync(snapshot, config); } catch (Exception) { // Rollback if snapshot is not saved await configFile.DeleteAsync(); throw; } if (Frame.CanGoBack) { Frame.GoBack(); } var newItem = new HostedConfigListItem(config, snapshot); await Task.Delay(300); HostedConfigListItems?.Add(newItem); await Task.Delay(800); HostedConfigListPage.itemForBackNavigation = newItem; Frame.Navigate(typeof(HostedConfigPage), newItem); }
public async Task Parse(UrlSource rssSource) { try { var httpClient = new HttpClient(); var _UserAgent = "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; WOW64; Trident/6.0)"; httpClient.DefaultRequestHeaders.Add("user-agent", _UserAgent); var response = await httpClient.GetStringAsync(rssSource.Url); XNamespace xmlns = "http://www.w3.org/2005/Atom"; XNamespace media = "http://search.yahoo.com/mrss/"; XNamespace content = "http://purl.org/rss/1.0/modules/content/"; XDocument Feed = XDocument.Parse(response); string group = rssSource.Group.Length > 1 ? rssSource.Group : Feed.Descendants("channel").Select(e => (string)e.Element("title").Value).First(); IEnumerable <Item> items = new List <Item>(); if (rssSource.Url.StartsWith("http://gdata.youtube.com/feeds/api/playlists/")) //parse Youtube Playlist RSS { //0 is link, 1 is image, 2 is title, 3 is description string youtubeHtmlTemplate = "<p><iframe back width=\"315\" height=\"177\" src=\"{0}\" frameBorder=\"0\"> </iframe></p><p><a style=\"font-size: 15px; font-weight: bold; font-decoration: none;\" href=\"{0}\">{2}</a></p><p>{3}</p>"; //string youtubeHtmlTemplate = "<iframe back width=\"315\" height=\"177\" src=\"{0}\" frameBorder=\"0\"> </iframe>"; items = from item in Feed.Descendants("item") select new Item() { Title = item.Element("title").Value, Subtitle = item.Element("pubDate").Value, Description = item.Descendants(media + "thumbnail").Count() > 0 ? string.Format(youtubeHtmlTemplate, item.Element("link").Value, item.Descendants(media + "thumbnail").Select(e => (string)e.Attribute("url")).FirstOrDefault(), item.Element("title").Value, item.Element("description").Value.Substring(0, Math.Min(580, item.Element("description").Value.Length))) : string.Empty, Image = item.Descendants(media + "thumbnail") != null?item.Descendants(media + "thumbnail").Select(e => (string)e.Attribute("url")).FirstOrDefault() : string.Empty, Group = @group, UrlSource = rssSource }; items = items.Where(x => x.Description != string.Empty); } else { string audio_template = "<audio src=\"{0}\" controls autoplay>Your browser does not support the <code>audio</code> element.<br/><a href=\"{0}\">Link to file</a>.</audio><br/>"; var feeditems = AppSettings.RssMaxItemsPerFeed < 0 ? Feed.Descendants("item") : Feed.Descendants("item").Take(AppSettings.RssMaxItemsPerFeed); items = from item in feeditems let body = item.Descendants(content + "encoded").FirstOrDefault() // TODO: perhaps this needs to use the url's MIME type to determine the tag for audio, video, PDFs, etc.? let parsed = (item.Element("enclosure") != null ? string.Format(audio_template, (string)(item.Element("enclosure").Attribute("url"))) : string.Empty) + (item.Element("description") != null ? (string)(item.Element("description").Value) : string.Empty) + (item.Element("link") != null ? " <a href=" + (string)(item.Element("link").Value) + ">Link</a>" : string.Empty) select new Item() { Title = item.Element("title") != null?item.Element("title").Value : string.Empty, Subtitle = item.Element("pubDate") != null?item.Element("pubDate").Value : DateTime.Now.ToString(), Description = body != null ? body.Value : parsed, Image = item.Descendants(media + "thumbnail") != null?item.Descendants(media + "thumbnail").Select(e => (string)e.Attribute("url")).FirstOrDefault() : "", Group = @group, UrlSource = rssSource }; } if (items.ToList().Count > 0) { foreach (var item in items) { if (item.Image == null) //Attempt to parse an image out of the description if one is not returned in the RSS { item.Image = Regex.Match(item.Description, @"(https?:)?//?[^'""<>]+?\.(jpg|jpeg|gif|png)").Value; } if (item.Image == string.Empty) //Unable to locate any image, so fallback to logo { item.Image = "/Assets/Logo.png"; } //Format dates to look cleaner DateTime dateTimeResult = new DateTime(); if (DateTime.TryParse(item.Subtitle, out dateTimeResult)) { item.Subtitle = dateTimeResult.ToString("ddd, d MMM yyyy"); } if (AppSettings.ForceYoutubeVideosToLoadFullScreen) { item.Description = item.Description.Replace("/watch?v=", "/watch_popup?v="); } // Fix "shortcut" urls item.Description = item.Description.Replace("src=\"//", "src=\"http://"); item.Description = item.Description.Replace("src='//", "src='http://"); // Facebook Data if (item.UrlSource.Url.StartsWith("https://www.facebook.com/feeds/page.php?")) { //Payload comes back HTML encoded and begins with a space item.Title = XPlatformCloudKit.Helpers.HttpUtility.HtmlDecode(item.Title).TrimStart(' '); //Use normal image instead of small item.Image = item.Image.Replace("_s.", "_n."); } RssData.Add(item); } ; } else { await ServiceLocator.MessageService.ShowErrorAsync("Zero items retrieved from " + rssSource.Url, "Application Error"); } } catch (Exception e) { ServiceLocator.MessageService.ShowErrorAsync("Error when retrieving items from RssService: " + e.Message + "\nUrl: " + rssSource.Url, "Application Error"); } }
private void Hao123MoviesCrawler(List <string> urlList, bool isDetial = false) { HtmlParser htmlParser = new HtmlParser(); string resource = Const.SourcesType.Hao123; for (var i = 0; i < urlList.Count; i++) { var crawler = new SimpleCrawler(); crawler.OnStart += (s, e) => { Console.WriteLine("爬虫开始抓取地址:" + e.Uri.ToString()); }; crawler.OnError += (s, e) => { Console.WriteLine("爬虫抓取出现错误:" + e.Uri.ToString() + ",异常消息:" + e.Exception.Message); }; crawler.OnCompleted += (s, e) => { if (isDetial) { var dom = htmlParser.ParseDocument(e.PageSource); var moviesInfo = new MoviesInfo(); var urlSourceList = new List <UrlSource>(); moviesInfo.Id = GuidExtend.NewGuid(); moviesInfo.Resource = resource; moviesInfo.CreateTime = DateTime.Now; var a = dom.QuerySelectorAll("div.poster>a"); if (a.Any()) { moviesInfo.Name = a[0].GetAttribute("title"); //--电影名称 } else { return; } var stars = dom.All.Where(sl => sl.GetAttribute("monkey") == "actor").ToList(); if (stars.Any()) { moviesInfo.Stars = string.Join(",", stars[0].QuerySelectorAll("a").Select(X => X.InnerHtml).ToList().Distinct()); } var type = dom.All.Where(sl => sl.GetAttribute("monkey") == "category").ToList(); if (type.Any()) { moviesInfo.Type = string.Join(",", type[0].QuerySelectorAll("a").Select(X => X.InnerHtml).ToList().Distinct()); } var area = dom.All.Where(sl => sl.GetAttribute("monkey") == "area").ToList(); if (area.Any()) { moviesInfo.Area = string.Join(",", area[0].QuerySelectorAll("a").Select(X => X.InnerHtml).ToList().Distinct()); } var year = dom.All.Where(sl => sl.GetAttribute("monkey") == "decade").ToList(); if (year.Any()) { moviesInfo.Year = string.Join(",", year[0].QuerySelectorAll("a").Select(X => X.InnerHtml).ToList().Distinct()); } var img = dom.QuerySelectorAll("div.poster>a>img"); if (img.Any()) { moviesInfo.ImageUrl = img[0].GetAttribute("src"); //--图片 } var des = dom.QuerySelectorAll("p.abstract>em"); if (des.Any()) { moviesInfo.Description = des[0].InnerHtml; } var url = dom.QuerySelectorAll("div.source>a.play-btn"); if (url.Any()) { var urlSource = new UrlSource(); urlSource.Url = url[0].GetAttribute("href"); urlSource.VideoSource = url[0].GetAttribute("alog-text"); urlSource.Id = GuidExtend.NewGuid(); urlSource.MovieId = moviesInfo.Id; urlSource.Resource = resource; urlSourceList.Add(urlSource); } var urls = dom.QuerySelectorAll("div.source")[0].QuerySelectorAll("ul>li>a"). Select(x => new UrlSource { Id = GuidExtend.NewGuid(), MovieId = moviesInfo.Id, Url = x.GetAttribute("href"), VideoSource = x.TextContent, Resource = resource }); if (urls.Any()) { urlSourceList.AddRange(urls); } if (!string.IsNullOrEmpty(moviesInfo.Name) && urlSourceList.Count > 0) { var oldData = _repository.All <MoviesInfo>(sl => sl.Name == moviesInfo.Name && sl.ImageUrl == moviesInfo.ImageUrl); oldData.DeleteFromQuery(); _repository.DeleteByExpression <UrlSource>(sl => oldData.Select(m => m.Id).Contains(sl.MovieId)); _repository.Insert(moviesInfo, true); _repository.BulkInsert <UrlSource>(urlSourceList); } } else { var dom = htmlParser.ParseDocument(e.PageSource); var MovieUrlList = dom.QuerySelectorAll("li.card>a").Select(a => a.GetAttribute("href")).ToList(); Hao123MoviesCrawler(MovieUrlList, true); } }; crawler.Start(new Uri(urlList[i])).Wait(); } }