private static void BuildOneItem(DetailResult item, OutputFormatBuilder builder) { builder.Build("ItemID", item.ItemID); builder.Build("ParentItemID", item.ParentItemID); builder.Build("CrawlID", item.CrawlID); builder.Build("CleanTitle", item.CleanTitle); builder.Build("CleanText", item.CleanText); builder.Build("Url", item.Url); builder.Build("PubDate", item.PubDate); builder.Build("FetchTime", item.PubDate); builder.Build("ReplyCount", item.ReplyCount); builder.Build("ForwardCount", item.ForwardCount); builder.Build("ViewCount", item.ViewCount); builder.Build("CurrentHistoryFetchTime", item.CurrentHistoryFetchTime); builder.Build("KeywordQuery", item.KeywordQuery); builder.Build("MediaID", item.MediaID); builder.Build("ParentMediaID", item.ParentMediaID); builder.Build("MediaName", item.MediaName); builder.Build("Channel", item.Channel); builder.Build("MediaType", item.MediaType); builder.Build("MediaTendency", item.MediaTendency); builder.Build("MediaOrganType", item.MediaOrganType); builder.Build("MediaElitismType", item.MediaElitismType); builder.Build("MediaWeight", item.MediaWeight); builder.Build("MediaStyle", item.MediaStyle); builder.Build("RegionType", item.RegionType); builder.Build("AuthorName", item.AuthorName); builder.Build("AuthorID", item.AuthorID); builder.Build("AuthorCertificated", item.AuthorCertificated); builder.Build("Source", item.Source); builder.Build("IsPublicLeader", item.IsPublicLeader); LeaderInfo leaderInfo = item.LeaderInfo ?? new LeaderInfo(); builder.Build("Rank", leaderInfo.Rank); builder.Build("Gender", leaderInfo.Gender); builder.Build("Job", leaderInfo.Job); builder.Build("Age", leaderInfo.Age); builder.Build("Follower", leaderInfo.Follower); builder.Build("Following", leaderInfo.Following); builder.Build("TweetCount", ""); //目前没有对应的微博数 builder.Build("AvgFoward", leaderInfo.AvgFoward); builder.Build("AvgComment", leaderInfo.AvgComment); builder.Build("AvgFollowersFollower", leaderInfo.AvgFollowersFollower); builder.Build("HasAttachUrl", item.HasAttachUrl); builder.Build("AttachUrl", item.AttachUrl); builder.Build("GovProcess", item.GovProcess); builder.Build("ArticleHot", item.ArticleHot); builder.Build("CrawlName", item.CrawlName); builder.NewLine(); }
private DetailResult[] GetHistoryData(DateTime startDate, DateTime endDate, Dictionary <string, LeaderInfo> leaderInfoMapping, Dictionary <string, FocusEvent> focusEventMapping, int startCnt = 10, int endCnt = 30) { List <DetailResult> list = new List <DetailResult>(); string[] crawlIDs; using (PalasDB db = new PalasDB()) { crawlIDs = db.Crawl.Where(model => model.CrawlID.StartsWith("Weibo")).Select(model => model.CrawlID).ToArray(); } crawlIDs = crawlIDs.Where(model => model != "WeiboSubscribe").ToArray(); foreach (var crawlID in crawlIDs) { for (int i = startCnt; i < endCnt; i++) { //var crawlQuery = Query.Matches("CrawlID", "/Weibo/"); var crawlQuery = Query.EQ("CrawlID", crawlID); var query = Query.And(crawlQuery, Query.GTE("FetchTime", startDate), Query.LTE("FetchTime", endDate), Query.Size("CountHistory", i) ); //var sort = SortBy.Ascending("PubDate"); Item[] result = null; try { result = MongoItemAccess.Items.Find(query).SetFields("ItemID", "CountHistory", "CrawlID", "Url", "CleanTitle", "MediaID", "PubDate", "ParentItemID", "CleanText", "AuthorName", "AuthorID", "AuthorCertificated", "Source", "AttachUrl") //.SetSortOrder(sort) .ToArray(); } catch (Exception) { continue; } foreach (var item in result) { DetailResult dest = new DetailResult(); item.CopyTo(ref dest); //赋值Item中字段 //取Media字段 using (PalasDB db = new PalasDB()) { var media = db.Media.FirstOrDefault(model => model.MediaID == item.MediaID); if (media != null) { media.CopyTo(ref dest); } var crawl = db.Crawl.FirstOrDefault(model => model.CrawlID == item.CrawlID); if (crawl != null) { dest.KeywordQuery = crawl.KeywordQuery; dest.CrawlName = crawl.Name; } } //匹配意见领袖 LeaderInfo info; bool isPublicLeader = leaderInfoMapping.TryGetValue(dest.AuthorName ?? "", out info); dest.IsPublicLeader = isPublicLeader ?1:0; if (isPublicLeader) { dest.LeaderInfo = info; } //匹配事件 FocusEvent focus; bool isGovProcess = focusEventMapping.TryGetValue(dest.CrawlName ?? "", out focus); dest.GovProcess = isGovProcess? 1 : 0; dest.HasAttachUrl = !string.IsNullOrEmpty(dest.AttachUrl) ? 1 : 0; //赋值不同的嵌套字段 foreach (var itemCountData in item.CountHistory) { var clonedDest = dest.SwallowClone(); clonedDest.CurrentHistoryFetchTime = itemCountData.FetchTime; clonedDest.ViewCount = itemCountData.ViewCount; clonedDest.ReplyCount = itemCountData.ReplyCount; clonedDest.ForwardCount = itemCountData.ForwardCount; list.Add(clonedDest); } } } } return(list.ToArray()); }