static void ReadArticles() { string xmlContent; DateTime now = DateTime.Now; DateTime prevMinute = now.AddMinutes(-60); MongoRepo <CatalogueSource> repoSource = new MongoRepo <CatalogueSource>(MongoDbCollections.CatalogueSource); Tuple <HttpWebResponse, List <CatalogueSource> > result = repoSource.GetAll(); if (result.Item1.StatusCode == HttpStatusCode.OK) { using (var wc = new WebClient()) { wc.Encoding = Encoding.UTF8; var xmlDoc = new XmlDocument(); int index = 0; foreach (var source in result.Item2) { Console.WriteLine(string.Format("{0} -> {1} fetching...", index, source.Name)); if (index > 20) { break; } try { xmlContent = wc.DownloadString(string.Format(urlPattern, source.Id)); xmlDoc.LoadXml(xmlContent); XmlArticle response = null; XmlSerializer serializer = new XmlSerializer(typeof(XmlArticle)); using (XmlReader reader = new XmlNodeReader(xmlDoc)) { try { response = (XmlArticle)serializer.Deserialize(reader); List <Model.Article> newArticles = new List <Model.Article>(); Model.Article newArticle = null; foreach (var item in response.Articles.Article) { if (!item.ArticleId.HasValue) { continue; } DateTime pubDate = Convert.ToDateTime(item.ArticlePubDate); try { newArticle = new Model.Article(); newArticle.Active = true; newArticle.Approved = false; newArticle.Body = item.ArticleBody; newArticle.DetailedDate = item.ArticleDetailedDate; newArticle.Id = item.ArticleId.Value; newArticle.ImageUrl = item.ArticleImageUrl; newArticle.InsertedAt = now; newArticle.PubDate = pubDate; newArticle.SharingTitle = item.ArticleSharingTitle; newArticle.SourceId = source.Id; newArticle.SourceUrl = item.SourceImageUrl; newArticle.ArticleUrl = item.ArticleUrl; newArticle.Title = item.ArticleTitle; newArticle.TweetId = item.TwitterId; newArticle.VideoUrl = item.ArticleVideoUrl; newArticle.Approved = false; newArticle.CategoryId = 1; newArticle.FavoriteHits = 0; newArticle.Hits = 0; newArticle.LikeHits = 0; newArticle.UnlikeHits = 0; newArticles.Add(newArticle); } catch (Exception ex) { logger.addLog(item.ArticleId + logger.s + "Not Converted Local Format" + logger.s + ex.ToString()); } } try { MongoRepo <Model.Article> repoArticle = new MongoRepo <Model.Article>(MongoDbCollections.Article); Tuple <HttpWebResponse, string> resultArticle = repoArticle.Put(newArticles); logger.addLog("SUCCESSFULLY" + logger.s + resultArticle.Item2 + logger.s + source.Name, true); Console.WriteLine(string.Format("\t{0} completed. Items: {1} Result: {2}", source.Name, newArticles.Count, resultArticle.Item2)); } catch (Exception ex) { logger.addLog(source.Name + logger.s + "Not Saved to Database" + logger.s + ex.ToString()); } } catch (Exception ex) { logger.addLog(source.Name + logger.s + "Not Deserialize" + logger.s + ex.ToString()); } } } catch (Exception ex) { logger.addLog(source.Name + logger.s + "Not Downloaded" + logger.s + ex.ToString()); } index++; } } } logger.addLog("Excuted comleted"); logger.addLog("Agent closed"); }
static void InsertSources() { List <CatalogueSource> catalogueSources = new List <CatalogueSource>(); string xmlStr; using (var wc = new WebClient()) { wc.Encoding = Encoding.UTF8; xmlStr = wc.DownloadString(sourcesUrl); } var xmlDoc = new XmlDocument(); xmlDoc.LoadXml(xmlStr); XmlSource response = null; XmlSerializer serializer = new XmlSerializer(typeof(XmlSource)); using (XmlReader reader = new XmlNodeReader(xmlDoc)) { response = (XmlSource)serializer.Deserialize(reader); } if (response != null) { CatalogueSource catalogueSource = null; foreach (var categories in response.Categories) { foreach (var category in categories.Category) { foreach (var subCategories in category.SubCategories) { foreach (var subCategory in subCategories.SubCategory) { foreach (var sources in subCategory.Sources) { foreach (var source in sources.Source) { catalogueSource = new CatalogueSource(); catalogueSource.Active = true; catalogueSource.Description = source.SourceDescription.Count > 1 ? source.SourceDescription[0] : ""; catalogueSource.Followers = source.Followers; catalogueSource.Id = source.SourceId; catalogueSource.ImageUrl = source.SourceImageUrl; catalogueSource.Name = source.SourceName; catalogueSource.SourceUrl = source.SourceImageUrl; catalogueSources.Add(catalogueSource); } } } } } } MongoRepo <CatalogueSource> repo = new MongoRepo <CatalogueSource>(MongoDbCollections.CatalogueSource); Tuple <HttpWebResponse, string> result = repo.Insert(catalogueSources); } }