private static void BuildOneItem(DetailResult item, OutputFormatBuilder builder)
{
builder.Build("ItemID", item.ItemID);
builder.Build("ParentItemID", item.ParentItemID);
builder.Build("CrawlID", item.CrawlID);
builder.Build("CleanTitle", item.CleanTitle);
builder.Build("CleanText", item.CleanText);
builder.Build("Url", item.Url);
builder.Build("PubDate", item.PubDate);
builder.Build("FetchTime", item.PubDate);
builder.Build("ReplyCount", item.ReplyCount);
builder.Build("ForwardCount", item.ForwardCount);
builder.Build("ViewCount", item.ViewCount);
builder.Build("CurrentHistoryFetchTime", item.CurrentHistoryFetchTime);
builder.Build("KeywordQuery", item.KeywordQuery);
builder.Build("MediaID", item.MediaID);
builder.Build("ParentMediaID", item.ParentMediaID);
builder.Build("MediaName", item.MediaName);
builder.Build("Channel", item.Channel);
builder.Build("MediaType", item.MediaType);
builder.Build("MediaTendency", item.MediaTendency);
builder.Build("MediaOrganType", item.MediaOrganType);
builder.Build("MediaElitismType", item.MediaElitismType);
builder.Build("MediaWeight", item.MediaWeight);
builder.Build("MediaStyle", item.MediaStyle);
builder.Build("RegionType", item.RegionType);
builder.Build("AuthorName", item.AuthorName);
builder.Build("AuthorID", item.AuthorID);
builder.Build("AuthorCertificated", item.AuthorCertificated);
builder.Build("Source", item.Source);
builder.Build("IsPublicLeader", item.IsPublicLeader);
LeaderInfo leaderInfo = item.LeaderInfo ?? new LeaderInfo();
builder.Build("Rank", leaderInfo.Rank);
builder.Build("Gender", leaderInfo.Gender);
builder.Build("Job", leaderInfo.Job);
builder.Build("Age", leaderInfo.Age);
builder.Build("Follower", leaderInfo.Follower);
builder.Build("Following", leaderInfo.Following);
builder.Build("TweetCount", ""); //目前没有对应的微博数
builder.Build("AvgFoward", leaderInfo.AvgFoward);
builder.Build("AvgComment", leaderInfo.AvgComment);
builder.Build("AvgFollowersFollower", leaderInfo.AvgFollowersFollower);
builder.Build("HasAttachUrl", item.HasAttachUrl);
builder.Build("AttachUrl", item.AttachUrl);
builder.Build("GovProcess", item.GovProcess);
builder.Build("ArticleHot", item.ArticleHot);
builder.Build("CrawlName", item.CrawlName);
builder.NewLine();
}
private DetailResult[] GetHistoryData(DateTime startDate, DateTime endDate, Dictionary <string, LeaderInfo> leaderInfoMapping, Dictionary <string, FocusEvent> focusEventMapping, int startCnt = 10, int endCnt = 30)
{
List <DetailResult> list = new List <DetailResult>();
string[] crawlIDs;
using (PalasDB db = new PalasDB())
{
crawlIDs = db.Crawl.Where(model => model.CrawlID.StartsWith("Weibo")).Select(model => model.CrawlID).ToArray();
}
crawlIDs = crawlIDs.Where(model => model != "WeiboSubscribe").ToArray();
foreach (var crawlID in crawlIDs)
{
for (int i = startCnt; i < endCnt; i++)
{
//var crawlQuery = Query.Matches("CrawlID", "/Weibo/");
var crawlQuery = Query.EQ("CrawlID", crawlID);
var query = Query.And(crawlQuery, Query.GTE("FetchTime", startDate), Query.LTE("FetchTime", endDate), Query.Size("CountHistory", i)
);
//var sort = SortBy.Ascending("PubDate");
Item[] result = null;
try
{
result =
MongoItemAccess.Items.Find(query).SetFields("ItemID", "CountHistory", "CrawlID", "Url", "CleanTitle", "MediaID",
"PubDate", "ParentItemID", "CleanText", "AuthorName", "AuthorID", "AuthorCertificated", "Source", "AttachUrl")
//.SetSortOrder(sort)
.ToArray();
}
catch (Exception)
{
continue;
}
foreach (var item in result)
{
DetailResult dest = new DetailResult();
item.CopyTo(ref dest); //赋值Item中字段
//取Media字段
using (PalasDB db = new PalasDB())
{
var media = db.Media.FirstOrDefault(model => model.MediaID == item.MediaID);
if (media != null)
{
media.CopyTo(ref dest);
}
var crawl = db.Crawl.FirstOrDefault(model => model.CrawlID == item.CrawlID);
if (crawl != null)
{
dest.KeywordQuery = crawl.KeywordQuery;
dest.CrawlName = crawl.Name;
}
}
//匹配意见领袖
LeaderInfo info;
bool isPublicLeader = leaderInfoMapping.TryGetValue(dest.AuthorName ?? "", out info);
dest.IsPublicLeader = isPublicLeader ?1:0;
if (isPublicLeader)
{
dest.LeaderInfo = info;
}
//匹配事件
FocusEvent focus;
bool isGovProcess = focusEventMapping.TryGetValue(dest.CrawlName ?? "", out focus);
dest.GovProcess = isGovProcess? 1 : 0;
dest.HasAttachUrl = !string.IsNullOrEmpty(dest.AttachUrl) ? 1 : 0;
//赋值不同的嵌套字段
foreach (var itemCountData in item.CountHistory)
{
var clonedDest = dest.SwallowClone();
clonedDest.CurrentHistoryFetchTime = itemCountData.FetchTime;
clonedDest.ViewCount = itemCountData.ViewCount;
clonedDest.ReplyCount = itemCountData.ReplyCount;
clonedDest.ForwardCount = itemCountData.ForwardCount;
list.Add(clonedDest);
}
}
}
}
return(list.ToArray());
}
