public async Task Parse(UrlSource twitterSource)
{
var result = await client.GetStringAsync(twitterSource.Url);
JArray jsonDat = JArray.Parse(result);
foreach (var tweet in jsonDat.Children())
{
string user = "******" + tweet["user"]["screen_name"].ToString();
string createdTime = tweet["created_at"].ToString();
string tweettext = tweet["text"].ToString();
string profileBanner = tweet["user"]["profile_banner_url"].ToString();
const string format = "ddd MMM dd HH:mm:ss zzzz yyyy";
DateTime dateTimeResult = DateTime.ParseExact(createdTime, format, CultureInfo.InvariantCulture);
createdTime = dateTimeResult.ToString("ddd, d MMM yyyy");
TwitterData.Add(new Item
{
Title = user,
Subtitle = createdTime,
Description = tweettext,
Image = profileBanner,
Group = twitterSource.Group,
UrlSource = twitterSource
});
}
}
public async Task <List <Item> > GetItems()
{
if (AppSettings.YoutubePublicAPIKey.Length < 30 && !AppSettings.EnableRemoteUrlSourceService)
{
ServiceLocator.MessageService.ShowErrorAsync("YoutubeService is enabled but YoutubePublicAPIKey appears to be invalid please check this value in AppSettings.cs", "Application Error");
}
try
{
YoutubeData = new List <Item>();
foreach (var youtubeSource in AppSettings.YoutubeAddressCollection)
{
youtubeSource.Type = "YoutubeSource";
currentYoutubeSource = youtubeSource;
await Parse(youtubeSource);
}
}
catch
{
ServiceLocator.MessageService.ShowErrorAsync("Error when retrieving items from YoutubeService", "Application Error");
}
return(YoutubeData);
}
public void TestUrlSourceCheckExisting()
{
AsserIgnoreIfNoInternetConnection();
UrlSource source = new UrlSource("https://www.google.com/robots.txt");
Assert.IsTrue(source.Exists());
}
public void TestUrlSourceCheckNonExisting()
{
AsserIgnoreIfNoInternetConnection();
UrlSource source = new UrlSource("http://thisisnotavalidwebsiteisitnoitisnt.noitisnt/notexistingfile.dat");
Assert.IsFalse(source.Exists());
}
public void TestUrlSourceAccessNonExisting()
{
AsserIgnoreIfNoInternetConnection();
UrlSource source = new UrlSource("http://thisisnotavalidwebsiteisitnoitisnt.noitisnt/notexistingfile.dat");
Assert.Throws <InvalidOperationException>(() => source.Prepare());
Assert.Throws <InvalidOperationException>(() => source.Retrieve());
}
public async Task Parse(UrlSource youtubeSource)
{
var _UserAgent = "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; WOW64; Trident/6.0)";
httpClient.DefaultRequestHeaders.Add("user-agent", _UserAgent);
var result = await httpClient.GetStringAsync(youtubeSource.Url);
JObject jsonDat = JObject.Parse(result);
string nextPageToken = null;
try
{
nextPageToken = jsonDat["nextPageToken"].ToString();
}
catch
{
nextPageToken = null;
}
JArray jsonItems = JArray.Parse(jsonDat["items"].ToString());
string youtubeHtmlTemplate = "<p><a href=\"{0}\"><img src=\"{1}\" alt=\"\" width=300></a></p><p><a style=\"font-size: 15px; font-weight: bold; font-decoration: none;\" href=\"{0}\">{2}</a></p><p>{3}</p>";
string youtubePrefix;
youtubePrefix = AppSettings.ForceYoutubeVideosToLoadFullScreen ? "/watch_popup?v=" : "/watch?v=";
foreach (var youtubeItem in jsonItems)
{
var description = youtubeItem["snippet"]["description"].ToString();
if (description != "This video is private." && description != "This video is unavailable.")
{
YoutubeData.Add(new Item
{
Title = youtubeItem["snippet"]["title"].ToString(),
Subtitle = youtubeItem["snippet"]["publishedAt"].ToString(),
Description = string.Format(youtubeHtmlTemplate, "https://www.youtube.com" + youtubePrefix + youtubeItem["snippet"]["resourceId"]["videoId"], youtubeItem["snippet"]["thumbnails"]["high"]["url"].ToString(), youtubeItem["snippet"]["title"].ToString(), description),
Image = youtubeItem["snippet"]["thumbnails"]["medium"]["url"].ToString(),
Group = youtubeSource.Group,
UrlSource = currentYoutubeSource
});
}
}
if (nextPageToken != null)
{
await Parse(new UrlSource { Url = currentYoutubeSource.Url + "&pageToken=" + nextPageToken, Group = currentYoutubeSource.Group });
}
}
public JsonValue ToJson(JsonSerializer serializer)
{
var json = new JsonObject();
Id.Serialize(json, serializer, "id");
Board.SerializeId(json, "idBoard");
Closed.Serialize(json, serializer, "closed");
Desc.Serialize(json, serializer, "desc");
Due.Serialize(json, serializer, "due", ForceDueDate);
List.SerializeId(json, "idList");
Name.Serialize(json, serializer, "name");
Pos.Serialize(json, serializer, "pos");
Subscribed.Serialize(json, serializer, "subscribed");
CardSource.SerializeId(json, "idCardSource");
UrlSource.Serialize(json, serializer, "urlSource");
return(json);
}
public void TestUrlSourceAccessExisting()
{
AsserIgnoreIfNoInternetConnection();
UrlSource source = new UrlSource("https://www.google.com/robots.txt", Path.GetTempPath() + ".unittestfileurltest1");
Assert.Throws <InvalidOperationException>(() => source.Retrieve());
source.Prepare();
Stream stream = source.Retrieve();
Assert.IsNotNull(stream);
stream.Dispose();
File.Delete(Path.GetTempPath() + ".unittestfileurltest1");
}
public static async Task GetRemoteUrlSources()
{
string remoteUrlSourceFile = await httpClient.GetStringAsync(AppSettings.RemoteUrlSourceUrl);
if (!String.IsNullOrWhiteSpace(remoteUrlSourceFile))
{
// Parse the file into a list of strings. Split by either carraige return, or
// line feed to account for platform differences in the platform that created
// the RemoteUrlSource feeds list file.
string[] arrayCRLF = { "\r", "\n" };
string[] RemoteUrlSources = remoteUrlSourceFile.Split(arrayCRLF, StringSplitOptions.RemoveEmptyEntries);
// Now parse each additional feed and add it to the master collection.
foreach (string RemoteUrlSourceAsStr in RemoteUrlSources)
{
UrlSource source = stringToUrlSource(RemoteUrlSourceAsStr);
switch (source.Type)
{
case "RssService":
ArrayUtilities.Add(ref AppSettings.RssAddressCollection, source);
AppSettings.EnableRssService = true;
break;
case "YoutubeService":
ArrayUtilities.Add(ref AppSettings.YoutubeAddressCollection, source);
AppSettings.EnableYoutubeService = true;
break;
case "TwitterService":
ArrayUtilities.Add(ref AppSettings.TwitterAddressCollection, source);
//Twitter should be enabled prior to app submission on account of Oauth requirements
break;
default:
ServiceLocator.MessageService.ShowErrorAsync("Unable to add UrlSource of type " + source.Type + " to a valid collection, source will not be retrieved", "Application Error");
break;
}
}
}
}
public async Task DoLike(string url, string token)
{
string email = _tokenService.DecodeToken(token);
if (string.IsNullOrEmpty(email))
{
throw new UnauthorizedAccessException("invalid token");
}
var liker = await _likerReository.GetByEmailAsync(email) ?? new Liker
{
Email = email
};
if (liker.HasLiked(url))
{
throw new UnauthorizedAccessException("user already liked this url");
}
var urlSource = await _urlRepository.GetByUrlAsync(url);
if (urlSource != null)
{
urlSource.AddUserWhoLiked(liker);
await _urlRepository.UpdateAsync(urlSource);
}
else
{
var newUrl = new UrlSource {
Url = url
};
newUrl.AddUserWhoLiked(liker);
await _urlRepository.InsertAsync(newUrl);
}
await _cache.RemoveAsync(url);
}
public JsonValue ToJson(JsonSerializer serializer)
{
var json = new JsonObject();
Id.Serialize(json, serializer, "id");
Board.SerializeId(json, "idBoard");
Closed.Serialize(json, serializer, "closed");
Desc.Serialize(json, serializer, "desc");
Due.Serialize(json, serializer, "due", ForceDueDate);
DueComplete.Serialize(json, serializer, "dueComplete");
List.SerializeId(json, "idList");
Name.Serialize(json, serializer, "name");
Pos.Serialize(json, serializer, "pos");
Subscribed.Serialize(json, serializer, "subscribed");
CardSource.SerializeId(json, "idCardSource");
UrlSource.Serialize(json, serializer, "urlSource");
IdMembers.Serialize(json, serializer, "idMembers");
IdLabels.Serialize(json, serializer, "idLabels");
if (KeepFromSource != CardCopyKeepFromSourceOptions.None)
{
KeepFromSource.Serialize(json, serializer, "keepFromSource");
}
return(json);
}
private async Task AddSubscribeAsync()
{
var cancellationToken = LoadSnapshotCancellationSource.Token;
var config = new HostedConfig();
// Create source
switch (sourcePivot.SelectedIndex)
{
case 0:
// URL
if (string.IsNullOrWhiteSpace(urlText.Text))
{
throw new OperationCanceledException("Empty input");
}
if (!Uri.IsWellFormedUriString(urlText.Text, UriKind.Absolute))
{
throw new InvalidDataException("The URL is invalid");
}
var urlSource = new UrlSource()
{
Url = urlText.Text
};
config.Source = urlSource;
break;
case 1:
// File
if (!(previewFileNameText.DataContext is StorageFile file))
{
file = await FileOpenPicker.PickSingleFileAsync();
}
if (file == null)
{
throw new InvalidDataException("No file is chosen");
}
config.Source = new FileSource(file);
break;
default:
throw new NotSupportedException("The source type is not supported yet");
}
// Create format
Snapshot snapshot;
cancellationToken.ThrowIfCancellationRequested();
switch (((NewHostedConfigType)hostedTypeGridView.SelectedItem).Id)
{
case "ssd":
config.Format = new Ssd();
break;
case "clash":
config.Format = new Clash();
break;
default:
throw new NotSupportedException("The hosted config type is not supported yet");
}
using (var source = await config.Source.FetchAsync().AsTask(cancellationToken))
{
snapshot = await source.DecodeAsync(config.Format).AsTask(cancellationToken);
}
config.Name = config.Source.GetFileName();
// Save
cancellationToken.ThrowIfCancellationRequested();
var configFile = await HostedUtils.SaveHostedConfigAsync(config);
try
{
_ = CachedFileManager.CompleteUpdatesAsync(configFile);
cancellationToken.ThrowIfCancellationRequested();
// No need to update local file
_ = await HostedUtils.SaveSnapshotAsync(snapshot, config);
}
catch (Exception)
{
// Rollback if snapshot is not saved
await configFile.DeleteAsync();
throw;
}
if (Frame.CanGoBack)
{
Frame.GoBack();
}
var newItem = new HostedConfigListItem(config, snapshot);
await Task.Delay(300);
HostedConfigListItems?.Add(newItem);
await Task.Delay(800);
HostedConfigListPage.itemForBackNavigation = newItem;
Frame.Navigate(typeof(HostedConfigPage), newItem);
}
public async Task Parse(UrlSource rssSource)
{
try
{
var httpClient = new HttpClient();
var _UserAgent = "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; WOW64; Trident/6.0)";
httpClient.DefaultRequestHeaders.Add("user-agent", _UserAgent);
var response = await httpClient.GetStringAsync(rssSource.Url);
XNamespace xmlns = "http://www.w3.org/2005/Atom";
XNamespace media = "http://search.yahoo.com/mrss/";
XNamespace content = "http://purl.org/rss/1.0/modules/content/";
XDocument Feed = XDocument.Parse(response);
string group = rssSource.Group.Length > 1 ? rssSource.Group : Feed.Descendants("channel").Select(e => (string)e.Element("title").Value).First();
IEnumerable <Item> items = new List <Item>();
if (rssSource.Url.StartsWith("http://gdata.youtube.com/feeds/api/playlists/")) //parse Youtube Playlist RSS
{
//0 is link, 1 is image, 2 is title, 3 is description
string youtubeHtmlTemplate = "<p><iframe back width=\"315\" height=\"177\" src=\"{0}\" frameBorder=\"0\"> </iframe></p><p><a style=\"font-size: 15px; font-weight: bold; font-decoration: none;\" href=\"{0}\">{2}</a></p><p>{3}</p>";
//string youtubeHtmlTemplate = "<iframe back width=\"315\" height=\"177\" src=\"{0}\" frameBorder=\"0\"> </iframe>";
items = from item in Feed.Descendants("item")
select new Item()
{
Title = item.Element("title").Value,
Subtitle = item.Element("pubDate").Value,
Description = item.Descendants(media + "thumbnail").Count() > 0 ? string.Format(youtubeHtmlTemplate, item.Element("link").Value, item.Descendants(media + "thumbnail").Select(e => (string)e.Attribute("url")).FirstOrDefault(), item.Element("title").Value, item.Element("description").Value.Substring(0, Math.Min(580, item.Element("description").Value.Length))) : string.Empty,
Image = item.Descendants(media + "thumbnail") != null?item.Descendants(media + "thumbnail").Select(e => (string)e.Attribute("url")).FirstOrDefault() : string.Empty,
Group = @group,
UrlSource = rssSource
};
items = items.Where(x => x.Description != string.Empty);
}
else
{
string audio_template = "<audio src=\"{0}\" controls autoplay>Your browser does not support the <code>audio</code> element.<br/><a href=\"{0}\">Link to file</a>.</audio><br/>";
var feeditems = AppSettings.RssMaxItemsPerFeed < 0
? Feed.Descendants("item")
: Feed.Descendants("item").Take(AppSettings.RssMaxItemsPerFeed);
items = from item in feeditems
let body = item.Descendants(content + "encoded").FirstOrDefault()
// TODO: perhaps this needs to use the url's MIME type to determine the tag for audio, video, PDFs, etc.?
let parsed = (item.Element("enclosure") != null
? string.Format(audio_template, (string)(item.Element("enclosure").Attribute("url")))
: string.Empty)
+ (item.Element("description") != null
? (string)(item.Element("description").Value)
: string.Empty)
+ (item.Element("link") != null
? " <a href=" + (string)(item.Element("link").Value) + ">Link</a>"
: string.Empty)
select new Item()
{
Title = item.Element("title") != null?item.Element("title").Value : string.Empty,
Subtitle = item.Element("pubDate") != null?item.Element("pubDate").Value : DateTime.Now.ToString(),
Description = body != null ? body.Value : parsed,
Image = item.Descendants(media + "thumbnail") != null?item.Descendants(media + "thumbnail").Select(e => (string)e.Attribute("url")).FirstOrDefault() : "",
Group = @group,
UrlSource = rssSource
};
}
if (items.ToList().Count > 0)
{
foreach (var item in items)
{
if (item.Image == null) //Attempt to parse an image out of the description if one is not returned in the RSS
{
item.Image = Regex.Match(item.Description, @"(https?:)?//?[^'""<>]+?\.(jpg|jpeg|gif|png)").Value;
}
if (item.Image == string.Empty) //Unable to locate any image, so fallback to logo
{
item.Image = "/Assets/Logo.png";
}
//Format dates to look cleaner
DateTime dateTimeResult = new DateTime();
if (DateTime.TryParse(item.Subtitle, out dateTimeResult))
{
item.Subtitle = dateTimeResult.ToString("ddd, d MMM yyyy");
}
if (AppSettings.ForceYoutubeVideosToLoadFullScreen)
{
item.Description = item.Description.Replace("/watch?v=", "/watch_popup?v=");
}
// Fix "shortcut" urls
item.Description = item.Description.Replace("src=\"//", "src=\"http://");
item.Description = item.Description.Replace("src='//", "src='http://");
// Facebook Data
if (item.UrlSource.Url.StartsWith("https://www.facebook.com/feeds/page.php?"))
{
//Payload comes back HTML encoded and begins with a space
item.Title = XPlatformCloudKit.Helpers.HttpUtility.HtmlDecode(item.Title).TrimStart(' ');
//Use normal image instead of small
item.Image = item.Image.Replace("_s.", "_n.");
}
RssData.Add(item);
}
;
}
else
{
await ServiceLocator.MessageService.ShowErrorAsync("Zero items retrieved from " + rssSource.Url, "Application Error");
}
}
catch (Exception e)
{
ServiceLocator.MessageService.ShowErrorAsync("Error when retrieving items from RssService: " + e.Message + "\nUrl: " + rssSource.Url, "Application Error");
}
}
private void Hao123MoviesCrawler(List <string> urlList, bool isDetial = false)
{
HtmlParser htmlParser = new HtmlParser();
string resource = Const.SourcesType.Hao123;
for (var i = 0; i < urlList.Count; i++)
{
var crawler = new SimpleCrawler();
crawler.OnStart += (s, e) =>
{
Console.WriteLine("爬虫开始抓取地址：" + e.Uri.ToString());
};
crawler.OnError += (s, e) =>
{
Console.WriteLine("爬虫抓取出现错误：" + e.Uri.ToString() + "，异常消息：" + e.Exception.Message);
};
crawler.OnCompleted += (s, e) =>
{
if (isDetial)
{
var dom = htmlParser.ParseDocument(e.PageSource);
var moviesInfo = new MoviesInfo();
var urlSourceList = new List <UrlSource>();
moviesInfo.Id = GuidExtend.NewGuid();
moviesInfo.Resource = resource;
moviesInfo.CreateTime = DateTime.Now;
var a = dom.QuerySelectorAll("div.poster>a");
if (a.Any())
{
moviesInfo.Name = a[0].GetAttribute("title"); //--电影名称
}
else
{
return;
}
var stars = dom.All.Where(sl => sl.GetAttribute("monkey") == "actor").ToList();
if (stars.Any())
{
moviesInfo.Stars = string.Join(",", stars[0].QuerySelectorAll("a").Select(X => X.InnerHtml).ToList().Distinct());
}
var type = dom.All.Where(sl => sl.GetAttribute("monkey") == "category").ToList();
if (type.Any())
{
moviesInfo.Type = string.Join(",", type[0].QuerySelectorAll("a").Select(X => X.InnerHtml).ToList().Distinct());
}
var area = dom.All.Where(sl => sl.GetAttribute("monkey") == "area").ToList();
if (area.Any())
{
moviesInfo.Area = string.Join(",", area[0].QuerySelectorAll("a").Select(X => X.InnerHtml).ToList().Distinct());
}
var year = dom.All.Where(sl => sl.GetAttribute("monkey") == "decade").ToList();
if (year.Any())
{
moviesInfo.Year = string.Join(",", year[0].QuerySelectorAll("a").Select(X => X.InnerHtml).ToList().Distinct());
}
var img = dom.QuerySelectorAll("div.poster>a>img");
if (img.Any())
{
moviesInfo.ImageUrl = img[0].GetAttribute("src"); //--图片
}
var des = dom.QuerySelectorAll("p.abstract>em");
if (des.Any())
{
moviesInfo.Description = des[0].InnerHtml;
}
var url = dom.QuerySelectorAll("div.source>a.play-btn");
if (url.Any())
{
var urlSource = new UrlSource();
urlSource.Url = url[0].GetAttribute("href");
urlSource.VideoSource = url[0].GetAttribute("alog-text");
urlSource.Id = GuidExtend.NewGuid();
urlSource.MovieId = moviesInfo.Id;
urlSource.Resource = resource;
urlSourceList.Add(urlSource);
}
var urls = dom.QuerySelectorAll("div.source")[0].QuerySelectorAll("ul>li>a").
Select(x => new UrlSource
{
Id = GuidExtend.NewGuid(),
MovieId = moviesInfo.Id,
Url = x.GetAttribute("href"),
VideoSource = x.TextContent,
Resource = resource
});
if (urls.Any())
{
urlSourceList.AddRange(urls);
}
if (!string.IsNullOrEmpty(moviesInfo.Name) && urlSourceList.Count > 0)
{
var oldData = _repository.All <MoviesInfo>(sl => sl.Name == moviesInfo.Name && sl.ImageUrl == moviesInfo.ImageUrl);
oldData.DeleteFromQuery();
_repository.DeleteByExpression <UrlSource>(sl => oldData.Select(m => m.Id).Contains(sl.MovieId));
_repository.Insert(moviesInfo, true);
_repository.BulkInsert <UrlSource>(urlSourceList);
}
}
else
{
var dom = htmlParser.ParseDocument(e.PageSource);
var MovieUrlList = dom.QuerySelectorAll("li.card>a").Select(a => a.GetAttribute("href")).ToList();
Hao123MoviesCrawler(MovieUrlList, true);
}
};
crawler.Start(new Uri(urlList[i])).Wait();
}
}
