public (User, bool, string) TryCreateOrUpdateUserByWeibo(WeiboUser wuser, string access_token)
{
var usrr = this.users.Find(i => i.WeiboID == wuser.idstr).FirstOrDefault();
try {
if (usrr != null)
{
var wusr = this.WEIBOS.Find(i => i.UID == wuser.idstr).FirstOrDefault();
if (wusr == null)
{
this.WEIBOS.InsertOne(WeiboUserProvider.CreateUser(wuser, access_token));
}
else
{
this.WEIBOS.UpdateOne(i => i.UID == wuser.idstr, this.CreateUpdate(wusr.Update(wuser)));
}
}
else
{
this.users.InsertOne(IdentityProvider.CreateUserByWeibo(wuser));
this.WEIBOS.InsertOne(WeiboUserProvider.CreateUser(wuser, access_token));
}
return(usrr, true, null);
} catch (Exception e) {
return(null, false, Errors.SaveUserInfosFailed + $" : ${e.Message}");
}
}
bool GetRssNews(int id)
{
WeiboUser user = null;
try
{
if (RssFeedsQueue.Count == 0)
{
return(false);
}
user = RssFeedsQueue.Dequeue();
if (user == null)
{
return(false);
}
}
catch (Exception ex)
{
_logger.LogError($"Thread[{id}] error.", ex);
return(false);
}
try
{
IList <News> newsDatas = RssParser.Parse(user.RssUrl);
foreach (var news in newsDatas)
{
try
{
if (_rssDataService.GetByLink(news.Link) != null)
{
continue;
}
news.City = user.City;
news.Country = user.Country;
news.Province = user.Province;
news.Publisher = user.Name;
news.Source = "微博";
news.Status = NewsStatus.Unconfirmed;
_rssDataService.Create(news);
}
catch (Exception ex)
{
_logger.LogError($"Thread[{id}] mongo error.", ex);
continue;
}
}
}
catch (Exception ex)
{
_logger.LogError($"Thread[{id}] rss error.", ex);
}
return(true);
}
public static User CreateUserByWeibo(WeiboUser wuser)
{
var uuid = CreateGuid(GuidType.N);
return(new User {
UUID = uuid,
UserName = "******" + uuid,
Gender = wuser.gender == "m" ? Gender.Male : Gender.Female,
WeiboID = wuser.idstr,
Role = UserRoleConstants.User,
IsThirdPart = true
});
}
public static WeiboDBUser CreateUser(WeiboUser wuser, string access_token)
{
if (wuser == null)
{
return(null);
}
return(new WeiboDBUser {
UID = wuser.idstr,
ScreenName = wuser.screen_name,
Description = wuser.description,
Domain = wuser.domain,
AvatarUrl = wuser.avatar_large,
Cover = wuser.cover_image,
CoverMobile = wuser.cover_image_phone,
AccessToken = access_token
});
}
public static WeiboDBUser Update(this WeiboDBUser wuser, WeiboUser newUser, string access_token = null)
{
if (wuser == null || newUser == null)
{
return(wuser);
}
wuser.AvatarUrl = newUser.avatar_large;
wuser.Cover = newUser.cover_image;
wuser.CoverMobile = newUser.cover_image_phone;
wuser.Description = newUser.description;
wuser.Domain = newUser.domain;
wuser.ScreenName = newUser.screen_name;
wuser.UpdateTime = DateTime.UtcNow.Ticks;
if (!string.IsNullOrEmpty(access_token))
{
wuser.AccessToken = access_token;
}
return(wuser);
}
static void ImportWeiboUserRss()
{
var userService = new WeiboUserService(ConnectionString, DatabaseName);
DataTable tb = ExcelHelper.ReadExcelToDataTable(@"F:\Development\wuhan2020\544WeiboUsersRSS.xlsx");
foreach (DataRow row in tb.Rows)
{
string name = row["名称"]?.ToString();
WeiboUser user = userService.GetByName(name);
if (user != null)
{
user.UserId = row["userId"]?.ToString();
user.UserIdUrl = row["UserId地址"]?.ToString();
user.RssUrl = row["RSS地址"]?.ToString();
userService.Update(user.Id, user);
}
}
}
static void ImportWeiboUserData()
{
DataTable tb = ExcelHelper.ReadExcelToDataTable(@"D:\临时\WeiboUsers.xlsx");
var service = new RssFeedService(ConnectionString, DatabaseName);
var cityService = new CityService(ConnectionString, DatabaseName);
var userService = new WeiboUserService(ConnectionString, DatabaseName);
List <DataRow> rows = new List <DataRow>();
DataTable filtedDt = tb.Clone();
filtedDt.Clear();
foreach (DataRow row in tb.Rows)
{
string vipStyle = row["官方认证样式"]?.ToString();
if (string.IsNullOrEmpty(vipStyle) || !vipStyle.Contains("icon-vip-b"))
{
continue;
}
string name = row["名称"]?.ToString();
if (string.IsNullOrEmpty(name))
{
continue;
}
if (WeiboNameKeywords.Any(k => name.IndexOf(k, StringComparison.OrdinalIgnoreCase) >= 0))
{
filtedDt.ImportRow(row);
}
else
{
continue;
}
WeiboUser user = new WeiboUser()
{
Name = name,
Url = row["微博地址"]?.ToString(),
Address = row["省"]?.ToString(),
Description = row["描述"]?.ToString(),
Followers = row["粉丝"]?.ToString(),
PublishCount = row["微博数"]?.ToString(),
Tags = row["标签"]?.ToString(),
Summary = row["简介"]?.ToString(),
UpdateTime = DateTime.Now
};
if (string.IsNullOrEmpty(user.Address) && string.IsNullOrEmpty(user.Description))
{
continue;
}
string province = Provinces.FirstOrDefault(k => user.Description.IndexOf(k, StringComparison.OrdinalIgnoreCase) >= 0);
string city = "";
if (!string.IsNullOrEmpty(user.Address))
{
var cityData = cityService.GetLikeCityName(user.Address);
if (cityData != null)
{
city = cityData.City;
if (province == null)
{
province = cityData.Province;
}
}
}
if (string.IsNullOrEmpty(province) && string.IsNullOrEmpty(city))
{
continue;
}
user.Province = province;
user.City = city;
userService.Create(user);
}
}
// 获取指定目标中的所有微博评论，如果指定了目标ID，则只获取指定目标ID的评论
public WeiboData getTotalComments( string url, string selfId, string friendId)
{
cc = cc0;
WeiboData weiboData = new WeiboData();
List<string> commentEntities = getTotalCommentEntities(cc, url, startPage, endPage); // 得到微博评论入口
// 遍历入口得到所有评论
foreach (var entity in commentEntities)
{
Console.Write("[" + threadName + "]评论入口:" + entity + " ");
int n = 0;
int maxCommentPage = 0;
for (int i = 1; i <= (maxCommentPage == 0 ? 2 : maxCommentPage); i++)
{
string content = "";
try
{
content = SpiderHelper.RequestHttpGetWithCookie(entity + "&page=" + i, cc);
}
catch (Exception e)
{
Console.Write(e.Message);
continue;
}
// 查找评论的最大页码
if (maxCommentPage == 0)
{
Match regMaxCommentPage = Regex.Match(content, "/([0-9]+)页</div>");
if (regMaxCommentPage.Length > 0)
{
maxCommentPage = int.Parse(regMaxCommentPage.Groups[1].Value);
}
maxCommentPage = maxCommentPage > 10 ? 10 : maxCommentPage; // 评论超过 10 页，应该也是大V没错，惹不起
}
//
// 评论示例：<a href="/u/1990224584?st=9771">目标用户</a>:<span class="ctt">哈哈，眼熟吧</span>
// .*? 这种写法是因为c#的正则表达式匹配默认是贪婪模式，默认匹配最多字符，用这个可以禁止贪婪模式
string regstr = "<a href=\"/u/([0-9]+).*?\">(.*?)</a>.*?<span class=\"ctt\">.*?</span>";
MatchCollection mc = Regex.Matches(content, regstr, RegexOptions.Compiled | RegexOptions.IgnoreCase);
if (mc.Count == 0)
{
cc = (cc == cc0) ? cc1 : cc0; // COOKIE已经被检测到异常，换一个COOKIE继续
}
foreach (Match m in mc)
{
WeiboComment comment = new WeiboComment();
comment.userid = m.Groups[1].Value;
string s = m.Value.Replace("</span>", "");
int k = s.LastIndexOf('>');
comment.comment = s.Substring(k + 1, s.Length - k - 1);
comment.username = m.Groups[2].Value;
n++;
if (friendId == null)
{ // 此时是获取博主自身的
weiboData.comments.Add(comment);
if (!comment.userid.Equals(selfId))
{ // 有新的好友
WeiboUser friend;
weiboData.friends.TryGetValue(comment.userid, out friend);
if (friend == null)
{
friend = new WeiboUser();
friend.userid = comment.userid;
friend.url = "http://weibo.cn/u/" + friend.userid + "?page=";
friend.username = comment.username;
weiboData.friends.Add(friend.userid, friend);
}
friend.count++;
}
continue;
}
else
{ // 此时是获取博主好友的，就只保存博主和好友说过的
if (comment.userid.Equals(selfId)) // || comment.userid.Equals(friendId))
{
weiboData.comments.Add(comment);
}
}
}
}
Console.WriteLine(n + " 条评论");
}
return weiboData;
}
