/// <summary>
/// 生成一个 One-time Token
/// </summary>
/// <param name="payload">该 Token 对应的负载，可以在之后 Consume 时取出</param>
/// <param name="expiry">Token 有效期</param>
/// <param name="purpose">该 Token 用途</param>
/// <param name="tokenFactory">Token 生成方法，如果为 null 则使用默认的生成器</param>
/// <typeparam name="T">负载类型</typeparam>
/// <returns>生成的 One-time Token</returns>
public async Task <string> Generate <T>(T payload, TimeSpan expiry, string purpose,
Func <Task <string> > tokenFactory = null)
{
if (tokenFactory == null)
{
tokenFactory = () =>
{
var guid = Guid.NewGuid();
return(Task.FromResult(Helpers.Md5(guid.ToByteArray())));
}
}
;
var redisDb = _redis.GetDatabase();
string token, cacheKey;
do
{
token = await tokenFactory.Invoke();
cacheKey = CacheKey(purpose, token);
} while (!(await redisDb.StringGetAsync(cacheKey)).IsNull);
await redisDb.StringSetAsync(cacheKey, RedisProvider.Serialize(payload), expiry);
return(token);
}
private static async Task <bool> UpdateUserSteamFrinedsAsync([NotNull] string userId, KeylolDbContext dbContext,
KeylolUserManager userManager, RedisProvider redis)
{
var cacheKey = UserSteamFriendRecordsCrawlerStampCacheKey(userId);
var redisDb = redis.GetDatabase();
var cacheResult = await redisDb.StringGetAsync(cacheKey);
if (cacheResult.HasValue)
{
return(false);
}
await redisDb.StringSetAsync(cacheKey, DateTime.Now.ToTimestamp(), UserSteamFriendsUpdatePeriod);
try
{
var steamId = new SteamID();
steamId.SetFromSteam3String(await userManager.GetSteamIdAsync(userId));
var result = JObject.Parse(await HttpClient.GetStringAsync(
$"http://api.steampowered.com/ISteamUser/GetFriendList/v1/?key={ApiKey}&format=json&steamid={steamId.ConvertToUInt64()}&relationship=friend"));
var oldRecords = (await dbContext.UserSteamFriendRecords.Where(r => r.UserId == userId).ToListAsync())
.ToDictionary(r => r.FriendSteamId, r => r);
foreach (var friend in result["friendslist"]["friends"])
{
var friendSteamId = (new SteamID(ulong.Parse((string)friend["steamid"]))).Render(true);
UserSteamFriendRecord record;
if (oldRecords.TryGetValue(friendSteamId, out record))
{
oldRecords.Remove(friendSteamId);
}
else
{
record = new UserSteamFriendRecord
{
UserId = userId,
FriendSteamId = friendSteamId
};
dbContext.UserSteamFriendRecords.Add(record);
if (friend["friend_since"] != null)
{
record.FriendSince = Helpers.DateTimeFromTimeStamp((ulong)friend["friend_since"]);
}
}
}
dbContext.UserSteamFriendRecords.RemoveRange(oldRecords.Values);
await dbContext.SaveChangesAsync();
return(true);
}
catch (Exception)
{
await redisDb.KeyExpireAsync(cacheKey, SilenceTime);
return(false);
}
}
private static async Task <bool> UpdateSteamSpyData(string pointId, KeylolDbContext dbContext,
RedisProvider redis)
{
var cacheKey = PointSteamSpyCrawlerStampCacheKey(pointId);
var redisDb = redis.GetDatabase();
var cacheResult = await redisDb.StringGetAsync(cacheKey);
if (cacheResult.HasValue)
{
return(false);
}
await redisDb.StringSetAsync(cacheKey, DateTime.Now.ToTimestamp(), SteamSpyDataUpdatePeriod);
try
{
var point = await dbContext.Points.FindAsync(pointId);
if (point.SteamAppId == null)
{
return(true);
}
var result = JToken.Parse(await HttpClient.GetStringAsync(
$"http://steamspy.com/api.php?request=appdetails&appid={point.SteamAppId}"));
if (result["name"] == null || (string)result["name"] == "Results are hidden")
{
return(true);
}
point.OwnerCount = (int)result["owners"];
point.OwnerCountVariance = (int)result["owners_variance"];
point.TotalPlayerCount = (int)result["players_forever"];
point.TotalPlayerCountVariance = (int)result["players_forever_variance"];
point.TwoWeekPlayerCount = (int)result["players_2weeks"];
point.TwoWeekPlayerCountVariance = (int)result["players_2weeks_variance"];
point.AveragePlayedTime = (int)result["average_forever"];
point.TwoWeekAveragePlayedTime = (int)result["average_2weeks"];
point.MedianPlayedTime = (int)result["median_forever"];
point.TwoWeekMedianPlayedTime = (int)result["median_2weeks"];
point.Ccu = (int)result["ccu"];
await dbContext.SaveChangesAsync(KeylolDbContext.ConcurrencyStrategy.ClientWin);
return(true);
}
catch (Exception)
{
await redisDb.KeyExpireAsync(cacheKey, SilenceTime);
return(false);
}
}
private static async Task <bool> UpdateOnSalePoints(KeylolDbContext dbContext, RedisProvider redis)
{
var cacheKey = OnSalePointsCrawlerStampCacheKey();
var redisDb = redis.GetDatabase();
var cacheResult = await redisDb.StringGetAsync(cacheKey);
if (cacheResult.HasValue)
{
return(false);
}
var expiry = OnSalePointsUpdateTime - DateTime.Now.TimeOfDay;
if (expiry <= TimeSpan.Zero)
{
expiry += TimeSpan.FromHours(24);
}
await redisDb.StringSetAsync(cacheKey, DateTime.Now.ToTimestamp(), expiry);
try
{
var points = new HashSet <string>();
var currentPage = 1;
var continueNext = true;
do
{
var dom = CQ.Create(await HttpClient.GetStreamAsync(
$"http://store.steampowered.com/search/results?specials=1&os=win&page={currentPage}&cc=cn"));
var anchors = dom[".search_result_row"];
if (anchors.Length < 25)
{
continueNext = false;
}
foreach (var anchor in anchors)
{
int appId;
if (!int.TryParse(anchor.Attributes["data-ds-appid"], out appId))
{
continue;
}
var point = await dbContext.Points.Where(p => p.SteamAppId == appId).SingleOrDefaultAsync();
if (point == null)
{
continue;
}
if (!points.Add(point.Id))
{
continue;
}
var matches = Regex.Matches(anchor.Cq().Find(".search_price").Text(), @"¥ (\d+)");
if (matches.Count != 2)
{
continue;
}
point.SteamPrice = double.Parse(matches[0].Groups[1].Value);
point.SteamDiscountedPrice = double.Parse(matches[1].Groups[1].Value);
}
currentPage++;
} while (continueNext && points.Count < OnSalePointMinCount);
var oldPoints = await dbContext.Feeds.Where(f => f.StreamName == OnSalePointStream.Name).ToListAsync();
dbContext.Feeds.RemoveRange(oldPoints);
dbContext.Feeds.AddRange(points.Select(id => new Feed
{
StreamName = OnSalePointStream.Name,
EntryType = FeedEntryType.PointId,
Entry = id
}).Reverse());
foreach (var pointId in points)
{
await redisDb.StringSetAsync(PointPriceCrawlerStampCacheKey(pointId), DateTime.Now.ToTimestamp(),
PointPriceUpdatePeriod);
}
await dbContext.SaveChangesAsync(KeylolDbContext.ConcurrencyStrategy.DatabaseWin);
return(true);
}
catch (Exception)
{
await redisDb.KeyExpireAsync(cacheKey, SilenceTime);
return(false);
}
}
private static async Task <bool> UpdatePointPriceAsync([NotNull] string pointId, KeylolDbContext dbContext,
RedisProvider redis)
{
var cacheKey = PointPriceCrawlerStampCacheKey(pointId);
var redisDb = redis.GetDatabase();
var cacheResult = await redisDb.StringGetAsync(cacheKey);
if (cacheResult.HasValue)
{
return(false);
}
await redisDb.StringSetAsync(cacheKey, DateTime.Now.ToTimestamp(), PointPriceUpdatePeriod);
try
{
var point = await dbContext.Points.FindAsync(pointId);
if (point.SteamAppId != null) // Steam
{
var steamResult = JToken.Parse(await HttpClient.GetStringAsync(
$"http://store.steampowered.com/api/appdetails/?appids={point.SteamAppId}&cc=cn&l=english"));
steamResult = steamResult[point.SteamAppId.ToString()];
if ((bool)steamResult["success"])
{
steamResult = steamResult["data"];
if ((bool)steamResult["is_free"])
{
point.SteamPrice = 0;
point.SteamDiscountedPrice = null;
}
else
{
point.SteamPrice = (double)steamResult["price_overview"]["initial"] / 100;
point.SteamDiscountedPrice = (int)steamResult["price_overview"]["discount_percent"] > 0
? (double)steamResult["price_overview"]["final"] / 100
: (double?)null;
}
await dbContext.SaveChangesAsync(KeylolDbContext.ConcurrencyStrategy.ClientWin);
}
}
if (point.SonkwoProductId != null) // 杉果
{
var sonkwoHtml =
await HttpClient.GetStringAsync($"http://www.sonkwo.com/products/{point.SonkwoProductId}");
var sonkwoDom = CQ.Create(sonkwoHtml);
var sonkwoPriceText = sonkwoDom[".sale-price, .list-price"].Text()
.Replace("￥", string.Empty);
double sonkwoPrice;
if (double.TryParse(sonkwoPriceText, out sonkwoPrice))
{
point.SonkwoPrice = sonkwoPrice;
var sonkwoDiscountedPriceText = sonkwoDom[".discounted-sale-price"].Text()
.Replace("￥", string.Empty);
double sonkwoDiscountedPrice;
if (double.TryParse(sonkwoDiscountedPriceText, out sonkwoDiscountedPrice))
{
point.SonkwoDiscountedPrice = sonkwoDiscountedPrice;
}
else
{
point.SonkwoDiscountedPrice = null;
}
await dbContext.SaveChangesAsync(KeylolDbContext.ConcurrencyStrategy.ClientWin);
}
}
return(true);
}
catch (Exception)
{
await redisDb.KeyExpireAsync(cacheKey, SilenceTime);
return(false);
}
}
/// <summary>
/// 抓取指定用户的 Steam App 库
/// </summary>
/// <param name="userId">用户 ID</param>
/// <param name="dbContext"><see cref="KeylolDbContext"/></param>
/// <param name="userManager"><see cref="KeylolUserManager"/></param>
/// <param name="redis"><see cref="RedisProvider"/></param>
/// <param name="cachedData"><see cref="CachedDataProvider.CachedDataProvider"/></param>
/// <returns>如果抓取成功，返回 <c>true</c></returns>
public static async Task <bool> UpdateUserSteamGameRecordsAsync([NotNull] string userId,
KeylolDbContext dbContext, KeylolUserManager userManager, RedisProvider redis,
CachedDataProvider.CachedDataProvider cachedData)
{
var cacheKey = UserSteamGameRecordsCrawlerStampCacheKey(userId);
var redisDb = redis.GetDatabase();
var cacheResult = await redisDb.StringGetAsync(cacheKey);
if (cacheResult.HasValue)
{
return(false);
}
await redisDb.StringSetAsync(cacheKey, DateTime.Now.ToTimestamp(), UserSteamGameRecordsUpdatePeriod);
try
{
var user = await userManager.FindByIdAsync(userId);
var steamId = new SteamID();
steamId.SetFromSteam3String(await userManager.GetSteamIdAsync(user.Id));
string allGamesHtml;
if (user.SteamBotId != null && user.SteamBot.IsOnline())
{
var botCoordinator = SteamBotCoordinator.Sessions[user.SteamBot.SessionId];
allGamesHtml = await botCoordinator.Client.Curl(user.SteamBotId,
$"http://steamcommunity.com/profiles/{steamId.ConvertToUInt64()}/games/?tab=all&l=english");
}
else
{
allGamesHtml = await HttpClient.GetStringAsync(
$"http://steamcommunity.com/profiles/{steamId.ConvertToUInt64()}/games/?tab=all&l=english");
}
if (string.IsNullOrWhiteSpace(allGamesHtml))
{
throw new Exception();
}
var match = Regex.Match(allGamesHtml, @"<script language=""javascript"">\s*var rgGames = (.*)");
if (!match.Success)
{
throw new Exception();
}
var trimed = match.Groups[1].Value.Trim();
var games = JArray.Parse(trimed.Substring(0, trimed.Length - 1));
var oldRecords = (await dbContext.UserSteamGameRecords.Where(r => r.UserId == user.Id).ToListAsync())
.ToDictionary(r => r.SteamAppId, r => r);
foreach (var game in games)
{
var appId = (int)game["appid"];
UserSteamGameRecord record;
if (oldRecords.TryGetValue(appId, out record))
{
oldRecords.Remove(appId);
}
else
{
record = new UserSteamGameRecord
{
UserId = user.Id,
SteamAppId = appId
};
dbContext.UserSteamGameRecords.Add(record);
}
record.TwoWeekPlayedTime = game["hours"] != null ? (double)game["hours"] : 0;
record.TotalPlayedTime = game["hours_forever"] != null ? (double)game["hours_forever"] : 0;
if (game["last_played"] != null)
{
record.LastPlayTime = Helpers.DateTimeFromTimeStamp((ulong)game["last_played"]);
}
}
dbContext.UserSteamGameRecords.RemoveRange(oldRecords.Values);
await dbContext.SaveChangesAsync();
await cachedData.Users.PurgeSteamAppLibraryCacheAsync(userId);
return(true);
}
catch (Exception)
{
await redisDb.KeyExpireAsync(cacheKey, SilenceTime);
return(false);
}
}
