private ArticleModel FeedToArticleModel(ChildNode item, FeedModel fcm)
{
if (item == null || item.klub != null)
return null;
return ExecuteSafe(() =>
{
var a = ConstructArticleModel(fcm);
a.Title = item.title;
a.SubTitle = item.kicker;
a.Teaser = item.text;
a.LogicUri = item.linkURL;
a.PublicUri = fcm.Source.PublicBaseUrl + item.docpath;
if (item.teaserImageURL != null)
{
var url = item.teaserImageURL;
if (url.Contains("w=320"))
url = url.Replace("w=320", "w=400");
a.LeadImage = new ImageContentModel() { Url = url };
}
return a;
});
}
public static void AddFeed(FeedModel fm, SourceModel source, bool isActive = false)
{
fm.Source = source;
if (isActive)
fm.Source.ActiveFeeds.Add(fm);
fm.IsActive = isActive;
}
public override Task<List<ArticleModel>> EvaluateFeed(FeedModel feedModel)
{
return ExecuteSafe(async () =>
{
var feed = await DownloadAsync(feedModel);
var articlelist = new List<ArticleModel>();
if (feed == null) return articlelist;
var rootObj = JsonConvert.DeserializeObject<FeedRoot>(feed);
if (rootObj == null)
LogHelper.Instance.Log(LogLevel.Error,
"BildHelper.EvaluateFeed failed: rootObj is null after deserialisation", this);
else
{
foreach (var children in rootObj.__childNodes__)
{
if (children.__childNodes__ != null)
{
foreach (var childNode in children.__childNodes__.Where(a => a.targetType == "article"))
{
var article = FeedToArticleModel(childNode, feedModel);
if (article != null)
articlelist.Add(article);
}
}
}
}
return articlelist;
});
}
public ArticleModel FeedToArticleModel(NzzFeedArticle nfa, FeedModel scm)
{
if (nfa == null || !nfa.path.Contains("/api/")) return null;
try
{
var a = ConstructArticleModel(scm);
a.PublishDateTime = nfa.publicationDateTime;
a.Title = nfa.title;
a.SubTitle = nfa.subTitle;
if (string.IsNullOrWhiteSpace(a.SubTitle) && nfa.title == "Was heute wichtig ist")
a.SubTitle = "Dieser Artikel wird laufend aktualisiert";
a.LeadImage = LeadImageToImage(nfa.leadImage, scm);
a.LogicUri = scm.Source.LogicBaseUrl + nfa.path.Substring("/api/".Length);
var guid = nfa.path.Substring(nfa.path.LastIndexOf("/", StringComparison.Ordinal) + 1);
a.PublicUri = scm.Source.PublicBaseUrl + guid;
return a;
}
catch (Exception ex)
{
LogHelper.Instance.Log(LogLevel.Error, "NzzHelper.FeedToArticleModel failed", this, ex);
return null;
}
}
public override Task<List<ArticleModel>> EvaluateFeed(FeedModel feedModel)
{
return ExecuteSafe(async () =>
{
var articlelist = new List<ArticleModel>();
var feed = await DownloadAsync(feedModel);
if (feed == null) return articlelist;
feed = feed.Substring(feed.IndexOf(">", StringComparison.Ordinal));
feed = feed.Substring(feed.IndexOf("<", StringComparison.Ordinal));
feed = XmlHelper.RemoveXmlLvl(feed);
feed = feed.Replace(" & ", " & ");
var channel = XmlHelper.Deserialize<Channel>(feed);
if (channel == null)
LogHelper.Instance.Log(LogLevel.Error,
"BildHelper.EvaluateFeed failed: rootObj is null after deserialisation", this);
else
{
foreach (var children in channel.Item)
{
var article = FeedToArticleModel(children, feedModel);
if (article != null)
articlelist.Add(article);
}
}
return articlelist;
});
}
public override Task<List<ArticleModel>> EvaluateFeed(FeedModel feedModel)
{
return ExecuteSafe(async () =>
{
var feed = await DownloadAsync(feedModel);
var articlelist = new List<ArticleModel>();
if (feed == null) return articlelist;
feed = feed.Substring(feed.IndexOf(">", StringComparison.Ordinal));
feed = feed.Substring(feed.IndexOf("<", StringComparison.Ordinal));
if (!RepairFeedXml(ref feed))
return articlelist;
var zeitFeed = XmlHelper.Deserialize<Feed>(feed);
if (zeitFeed == null)
LogHelper.Instance.Log(LogLevel.Error,
"ZeitHelper.EvaluateFeed failed: Feed is null after deserialisation", this);
else
{
foreach (var item in zeitFeed.References)
{
var article = FeedReferenceToArticleModel(item, feedModel);
if (article != null)
articlelist.Add(article);
}
}
return articlelist;
});
}
protected ArticleModel ConstructArticleModel(FeedModel feed)
{
return new ArticleModel()
{
DownloadDateTime = DateTime.Now,
Feed = feed
};
}
public void Download(FeedModel model)
{
for (int index = 0; index < model.AllArticles.Count; index++)
{
var articleModel = model.AllArticles[index];
if (articleModel.LeadImage != null)
Download(articleModel.LeadImage, index < 15);
}
}
public ArticleModel FeedToArticleModel(HtmlNode hn, FeedModel feed)
{
if (hn == null) return null;
try
{
var a = new ArticleModel();
var linknode = hn.ChildNodes.Descendants("a").FirstOrDefault();
if (linknode != null)
{
a.PublicUri = linknode.GetAttributeValue("href", null);
a.LogicUri = a.PublicUri;
if (linknode.ChildNodes != null && linknode.ChildNodes.Any())
{
var img = linknode.ChildNodes[0];
a.LeadImage = new ImageContentModel() { Url = img.GetAttributeValue("src", null) };
}
}
else
{
//newsticker
return null;
}
var titlenode = hn.ChildNodes.FirstOrDefault(atr => atr.GetAttributeValue("title", null) != null);
if (titlenode != null)
a.Title = titlenode.GetAttributeValue("title", null);
else
return null;
if (a.Title.Contains(":"))
{
a.SubTitle = a.Title.Substring(a.Title.IndexOf(":", StringComparison.Ordinal) + 1).Trim();
a.Title = a.Title.Substring(0, a.Title.IndexOf(":", StringComparison.Ordinal));
}
a.DownloadDateTime = DateTime.Now;
a.Author = "Chefredaktor";
return a;
}
catch (Exception ex)
{
LogHelper.Instance.Log(LogLevel.Error, "PostillonHelper.FeedToArticleModel failed", this, ex);
return null;
}
}
public static void SetFeedActiveState(FeedModel fm, bool isActive)
{
if (isActive && !fm.Source.ActiveFeeds.Contains(fm))
{
fm.Source.ActiveFeeds.Add(fm);
SetSourceActiveState(fm.Source, true);
}
else if (!isActive && fm.Source.ActiveFeeds.Contains(fm))
{
fm.Source.ActiveFeeds.Remove(fm);
if (fm.Source.ActiveFeeds.Count == 0)
SetSourceActiveState(fm.Source, false);
}
fm.IsActive = isActive;
}
private ArticleModel FeedModelToArticleModel(feeditem item, FeedModel feedModel)
{
return ExecuteSafe(() =>
{
var am = ConstructArticleModel(feedModel);
am.Title = item.title;
am.LogicUri = item.targetUrl;
am.PublicUri = item.targetUrl.Substring(0, item.targetUrl.IndexOf(".json", StringComparison.Ordinal)) + ".html";
am.PublishDateTime = item.publicationDate;
am.SubTitle = item.catchword;
am.Teaser = item.lead;
am.LeadImage = new ImageContentModel() {Url = item.img.src};
return am;
});
}
private ArticleModel FeedToArticleModel(Item children, FeedModel fcm)
{
if (children == null)
return null;
return ExecuteSafe(() =>
{
var title = children.Title.Substring(0, children.Title.IndexOf(":", StringComparison.Ordinal));
var subTitle = children.Title.Substring(children.Title.IndexOf(":", StringComparison.Ordinal) + 2);
var bannedSubTitles = new[]
{
"alle artikel",
"die wichtigsten artikel"
};
var bannedTitles = new[]
{
"Newsblog",
};
var lowerSub = subTitle.ToLower();
if (bannedSubTitles.Any(s => lowerSub.Contains(s)))
return null;
if (bannedTitles.Any(s => title.Contains(s)))
return null;
var a = ConstructArticleModel(fcm);
a.Title = title;
a.SubTitle = subTitle;
a.Teaser = children.Description;
a.PublishDateTime = DateTime.Parse(children.PubDate);
a.PublicUri = children.Guid;
a.LogicUri = children.Link;
if (children.Enclosure != null)
a.LeadImage = new ImageContentModel() { Url = children.Enclosure.Url };
a.Themes.Clear();
a.AfterSaveFunc = () => AddThemesAsync(a, new[] { children.Category });
return a;
});
}
public ArticleModel FeedToArticleModel(Entry2 nfa, FeedModel scm)
{
if (nfa == null) return null;
return ExecuteSafe(() =>
{
if (nfa.articleType == "standard-article")
{
var lowkicker = nfa.kicker.ToLower().Trim();
if (lowkicker.EndsWith("quiz") || nfa.kicker.Contains("test"))
return null;
var blockedIds = new[]
{
"6979480", //persönlichkeitstest
"6974764", //Quiz
"6948896", //Europa Quiz
"6979480", //persönlichkeitstest
"6979480", //persönlichkeitstest
};
if (blockedIds.Any(i => i == nfa.contentId))
return null;
var a = ConstructArticleModel(scm);
a.PublishDateTime = DateTime.Parse(nfa.timestamp);
a.Title = nfa.kicker;
a.SubTitle = nfa.headline;
a.Teaser = nfa.teaser;
if (nfa.images != null && nfa.images.Count > 3)
a.LeadImage = new ImageContentModel()
{
Url = nfa.images[3].src
};
a.LogicUri = scm.Source.LogicBaseUrl + nfa.contentId + ".json";
a.PublicUri = scm.Source.PublicBaseUrl + nfa.contentId;
return a;
}
return null;
});
}
public static async Task<ArticleModel> LoadForFeed(int id, FeedModel feed, ISqliteService sqliteService, IImageDownloadService imageDownloadService)
{
var arRepo = new GenericRepository<ArticleModel, ArticleEntity>(sqliteService);
var imgRepo = new GenericRepository<ImageContentModel, ImageContentEntity>(sqliteService);
var art = await arRepo.GetByIdAsync(id);
var contents = await sqliteService.GetByCondition<ContentEntity>(s => s.ParentId == id && s.ContentType == (int)ContentType.LeadImage, s => s.Index, false, 1, 0);
if (contents?.FirstOrDefault() != null)
{
var image = await imgRepo.GetByIdAsync(contents.FirstOrDefault().ContentId);
art.LeadImage = image;
if (art.LeadImage?.LoadingState < LoadingState.Loaded)
imageDownloadService.Download(art);
}
art.Feed = feed;
return art;
}
private void button1_Click(object sender, EventArgs e)
{
var sourceModel = JsonConvert.DeserializeObject<List<SourceModel>>(jsonInput.Text);
foreach (var sourceConfigurationModel in sourceModel)
{
var doc = new HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(htmlInput.Text);
List<HtmlNode> menu = doc.DocumentNode
.DescendantsAndSelf("aside")
.Where(
o => o.GetAttributeValue("id", null) == "leftMenu"
)
.ToList();
if (menu.Count == 1)
{
var menuItems = menu.First().Descendants("li")
.Where(
o => o.GetAttributeValue("class", null) == "category"
)
.ToList();
foreach (var menuItem in menuItems)
{
var a = menuItem.Descendants("a").FirstOrDefault();
if (a != null)
{
var feedModel = new FeedModel()
{
Url = sourceConfigurationModel.LogicBaseUrl + "api" + a.GetAttributeValue("href", null),
Name = a.InnerText,
Guid = Guid.NewGuid()
};
sourceConfigurationModel.Feeds.Add(feedModel);
}
}
}
}
jsonOutput.Text = JsonConvert.SerializeObject(sourceModel, Formatting.Indented);
}
public async Task<bool> SetFeedActiveStateAsync(FeedModel feedModel, bool isActive)
{
return true;
}
public async Task<bool> SwitchFeedActiveStateAsync(FeedModel feedModel)
{
return true;
}
public override Task<List<ArticleModel>> EvaluateFeed(FeedModel feedModel)
{
return ExecuteSafe(async () =>
{
var articlelist = new List<ArticleModel>();
var feed = await DownloadAsync(feedModel);
var f = JsonConvert.DeserializeObject<NzzFeed>(feed);
articlelist.AddRange(f.articles.Select(am => FeedToArticleModel(am, feedModel)).Where(am => am != null));
return articlelist;
});
}
public ArticleRepositoryMock()
{
var article = GetSampleArticle();
var article2 = GetSampleArticle();
var sm = new SourceModel()
{
Abbreviation = "NZZ",
Guid = Guid.NewGuid(),
Source = Sources.Nzz,
Name = "Neue Zürcher Zeitung",
LogicBaseUrl = "http://api.nzz.ch/",
PublicBaseUrl = "http://nzz.ch/",
IsActive = true
};
var sm2 = new SourceModel()
{
Abbreviation = "BAZ",
Guid = Guid.NewGuid(),
Source = Sources.Nzz,
Name = "Basler Zeitung",
LogicBaseUrl = "http://api.nzz.ch/",
PublicBaseUrl = "http://nzz.ch/",
IsActive = false
};
var fm = new FeedModel()
{
Name = "Home",
Guid = Guid.NewGuid(),
Source = sm,
Url = "home",
AllArticles =
{
article,
article2,
article
}
};
var fm2 = new FeedModel()
{
Name = "Ausland",
Guid = Guid.NewGuid(),
Source = sm,
Url = "ausland",
AllArticles =
{
article,
article2,
article
}
};
sm.ActiveFeeds.Add(fm);
sm.Feeds.Add(fm);
sm.Feeds.Add(fm2);
sm.Feeds.Add(fm2);
_sources.Add(sm);
_sources.Add(sm);
_sources.Add(sm2);
_sources.Add(sm2);
_sources.Add(sm);
}
private ImageContentModel LeadImageToImage(NzzLeadImage li, FeedModel feedModel)
{
if (li != null)
{
try
{
var uri = ParseImageUri(li.path);
if (uri != null)
{
return new ImageContentModel()
{
Text = new TextContentModel()
{
Content = HtmlConverter.CreateOnce(feedModel.Source.PublicBaseUrl).HtmlToParagraph(li.caption)
},
Url = uri
};
}
}
catch (Exception ex)
{
LogHelper.Instance.Log(LogLevel.Error, "NzzHelper.LeadImageToImage deserialization failed", this, ex);
return null;
}
}
return null;
}
private void SelectFeed(FeedModel model)
{
SimpleIoc.Default.GetInstance<FeedPageViewModel>().SelectFeed(model);
_historyNavigationService.NavigateTo(PageKeys.Feed.ToString());
}
public Task<bool> LoadFullFeedAsync(FeedModel fm)
{
return ExecuteSafe(async () =>
{
await LoadArticlesIntoFeed(fm);
return true;
});
}
public async Task<bool> LoadFullFeedAsync(FeedModel fm)
{
var article = GetSampleArticle();
article.Feed = fm;
fm.AllArticles.Add(article);
fm.AllArticles.Add(article);
fm.AllArticles.Add(article);
fm.AllArticles.Add(article);
fm.AllArticles.Add(article);
fm.AllArticles.Add(article);
fm.AllArticles.Add(article);
return true;
}
public override Task<List<ArticleModel>> EvaluateFeed(FeedModel feedModel)
{
return ExecuteSafe(async () =>
{
var articlelist = new List<ArticleModel>();
var feed = await DownloadAsync(feedModel);
if (feed == null) return articlelist;
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(feed);
List<HtmlNode> articles = doc.DocumentNode
.DescendantsAndSelf("div")
.Where(
o => o.GetAttributeValue("class", null) != null &&
o.GetAttributeValue("class", null).Contains("post-body")
)
.ToList();
if (articles != null && articles.Any())
{
foreach (var article in articles)
{
var res = FeedToArticleModel(article, feedModel);
if (res != null)
articlelist.Add(res);
}
}
return articlelist;
});
}
private ArticleModel FeedToArticleModel(Item nfa, FeedModel feedModel)
{
if (nfa == null || nfa.Link.Contains("/video/") || !nfa.Link.StartsWith("http://www.spiegel.de") || nfa.Link.Contains("/fotostrecke/"))
return null;
var bannedSubTitles = new[]
{
"alle artikel",
"die wichtigsten artikel"
};
var bannedTitles = new[]
{
"Newsblog",
};
var title = "";
var subTitle = "";
string author = null;
if (title.Contains(":"))
{
title = nfa.Title.Substring(0, nfa.Title.IndexOf(":", StringComparison.Ordinal));
subTitle = nfa.Title.Substring(nfa.Title.IndexOf(":", StringComparison.Ordinal) + 2);
if (subTitle.Contains("<li>"))
{
author = subTitle.Substring(subTitle.IndexOf("<li>", StringComparison.Ordinal));
author = Regex.Replace(author, "<(\\/)*([a-zA-Z])+>", "");
author = author.Replace("Von", "");
author = author.Replace("von", "").Trim();
subTitle = subTitle.Substring(0, subTitle.IndexOf("<li>", StringComparison.Ordinal));
}
}
else
{
title = nfa.Title;
}
var lowerSub = subTitle.ToLower();
if (bannedSubTitles.Any(s => lowerSub.Contains(s)))
return null;
if (bannedTitles.Any(s => title.Contains(s)))
return null;
var link = nfa.Link;
if (link.Contains("#ref=rss"))
link = link.Replace("#ref=rss", "");
var a = ConstructArticleModel(feedModel);
a.Title = title;
a.SubTitle = subTitle.Trim();
a.Teaser = nfa.Description;
a.Author = author;
a.PublishDateTime = DateTime.Parse(nfa.PubDate);
a.PublicUri = nfa.Link;
a.LogicUri = link;
a.Themes.Clear();
a.AfterSaveFunc = async () => await AddThemesAsync(a, new[] {nfa.Category});
var imagEndings = new[] {"img", "png", "giv", "jpg", "jpeg"};
if (nfa.Enclosure != null && ((nfa.Enclosure.Type != null && nfa.Enclosure.Type.Contains("image")) || imagEndings.Any(s => nfa.Enclosure.Url.EndsWith(s))))
{
var url = nfa.Enclosure.Url;
if (url.Contains("thumbsmall"))
{
url = url.Replace("thumbsmall", "hppano");
}
a.LeadImage = new ImageContentModel { Url = url };
}
return a;
}
public override Task<List<ArticleModel>> EvaluateFeed(FeedModel feedModel)
{
return ExecuteSafe(async () =>
{
var feed = await DownloadAsync(feedModel);
var articlelist = new List<ArticleModel>();
if (feed == null) return articlelist;
var f = JsonConvert.DeserializeObject<feed[]>(feed);
feed articlefeed = f.FirstOrDefault(i => i.type == "teaser");
if (articlefeed != null)
{
articlelist.AddRange(articlefeed.items.Where(i => i.type == "teaser")
.Select(i => FeedModelToArticleModel(i, feedModel))
.Where(am => am != null));
}
return articlelist;
});
}
public Task<bool> SwitchFeedActiveStateAsync(FeedModel feedModel)
{
return SetFeedActiveStateAsync(feedModel, !SourceManager.GetFeedActiveState(feedModel));
}
public async void SelectFeed(FeedModel obj)
{
Feed = obj;
await _articleRepository.LoadFullFeedAsync(obj);
}
public Task<bool> SetFeedActiveStateAsync(FeedModel feedModel, bool isActive)
{
SourceManager.SetFeedActiveState(feedModel, isActive);
_sourceCacheEntity.IsEnabledDictionary[feedModel.Guid] = isActive;
return SaveCache();
}
private async Task LoadArticlesIntoFeed(FeedModel feed, int max = -1)
{
if (max == -1)
max = 100;
for (int i = 0; i < max; i++)
{
if (feed.AllArticles.Count <= i)
{
var stringGuid = feed.Guid.ToString();
var realmax = max == 0 ? 0 : max - i;
var relations = await _sqliteService.GetByCondition<FeedArticleRelationEntity>(s => s.FeedGuid == stringGuid, s => s.Index, false, realmax, i);
foreach (var feedArticleRelationEntity in relations)
{
var article = await LoadHelper.LoadForFeed(feedArticleRelationEntity.ArticleId, feed, _sqliteService, _imageDownloadService);
feed.AllArticles.Add(article);
}
}
_imageDownloadService.Download(feed);
//no more entries
if (feed.AllArticles.Count <= i)
return;
}
}
