private async Task DownloadDepots(uint appID, List <ManifestJob> depots)
{
Log.WriteDebug("Depot Downloader", "Will process {0} depots ({1} depot locks left)", depots.Count, DepotLocks.Count);
var processTasks = new List <Task <EResult> >();
var anyFilesDownloaded = false;
var willDownloadFiles = false;
foreach (var depot in depots)
{
if (depot.DepotKey == null)
{
await GetDepotDecryptionKey(Steam.Instance.Apps, depot, appID);
if (depot.DepotKey == null && depot.LastManifestID == depot.ManifestID)
{
RemoveLock(depot.DepotID);
continue;
}
}
depot.Server = GetContentServer();
DepotManifest depotManifest = null;
var lastError = string.Empty;
for (var i = 0; i <= 5; i++)
{
try
{
depotManifest = await CDNClient.DownloadManifestAsync(depot.DepotID, depot.ManifestID, depot.Server, string.Empty, depot.DepotKey);
break;
}
catch (Exception e)
{
lastError = e.Message;
Log.WriteError("Depot Processor", "Failed to download depot manifest for app {0} depot {1} ({2}: {3}) (#{4})", appID, depot.DepotID, depot.Server, lastError, i);
}
if (depot.DepotKey != null)
{
RemoveErroredServer(depot.Server);
}
depot.Server = GetContentServer();
if (depotManifest == null)
{
await Task.Delay(Utils.ExponentionalBackoff(i));
}
}
if (depotManifest == null)
{
RemoveLock(depot.DepotID);
if (FileDownloader.IsImportantDepot(depot.DepotID))
{
IRC.Instance.SendOps("{0}[{1}]{2} Failed to download manifest ({3})",
Colors.OLIVE, depot.DepotName, Colors.NORMAL, lastError);
}
if (!Settings.IsFullRun && depot.DepotKey != null)
{
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null));
}
continue;
}
var task = ProcessDepotAfterDownload(depot, depotManifest);
processTasks.Add(task);
if (!FileDownloader.IsImportantDepot(depot.DepotID) || depot.DepotKey == null)
{
continue;
}
willDownloadFiles = true;
task = TaskManager.Run(async() =>
{
var result = EResult.Fail;
try
{
result = await FileDownloader.DownloadFilesFromDepot(depot, depotManifest);
if (result == EResult.OK)
{
anyFilesDownloaded = true;
}
}
catch (Exception e)
{
ErrorReporter.Notify($"Depot Processor {depot.DepotID}", e);
}
return(result);
}).Unwrap();
processTasks.Add(task);
}
if (SaveLocalConfig)
{
SaveLocalConfig = false;
LocalConfig.Save();
}
await Task.WhenAll(processTasks).ConfigureAwait(false);
Log.WriteDebug("Depot Downloader", $"{depots.Count} depot downloads finished for app {appID}");
// TODO: use ContinueWith on tasks
if (!anyFilesDownloaded && !willDownloadFiles)
{
foreach (var depot in depots)
{
RemoveLock(depot.DepotID);
}
return;
}
lock (UpdateScriptLock)
{
foreach (var depot in depots)
{
if (depot.Result == EResult.OK)
{
RunUpdateScript(UpdateScript, string.Format("{0} no-git", depot.DepotID));
}
else if (depot.Result != EResult.Ignored)
{
Log.WriteWarn("Depot Processor", $"Download failed for {depot.DepotID}");
// Mark this depot for redownload
var db = Database.Get();
db.Execute("UPDATE `Depots` SET `LastManifestID` = 0 WHERE `DepotID` = @DepotID", new { depot.DepotID });
}
RemoveLock(depot.DepotID);
}
// Only commit changes if all depots downloaded
if (processTasks.All(x => x.Result == EResult.OK || x.Result == EResult.Ignored))
{
if (!RunUpdateScriptForApp(appID, depots[0].BuildID))
{
RunUpdateScript(UpdateScript, "0");
}
}
else
{
Log.WriteDebug("Depot Processor", "Reprocessing the app {0} because some files failed to download", appID);
IRC.Instance.SendOps("{0}[{1}]{2} Reprocessing the app due to download failures",
Colors.OLIVE, Steam.GetAppName(appID), Colors.NORMAL
);
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null));
}
}
}
protected override async Task ProcessData()
{
await LoadData();
ChangeNumber = ProductInfo.ChangeNumber;
if (Settings.IsFullRun)
{
await DbConnection.ExecuteAsync("INSERT INTO `Changelists` (`ChangeID`) VALUES (@ChangeNumber) ON DUPLICATE KEY UPDATE `Date` = `Date`", new { ProductInfo.ChangeNumber });
await DbConnection.ExecuteAsync("INSERT INTO `ChangelistsSubs` (`ChangeID`, `SubID`) VALUES (@ChangeNumber, @SubID) ON DUPLICATE KEY UPDATE `SubID` = `SubID`", new { SubID, ProductInfo.ChangeNumber });
}
await ProcessKey("root_changenumber", "changenumber", ChangeNumber.ToString());
var appAddedToThisPackage = false;
var packageOwned = LicenseList.OwnedSubs.ContainsKey(SubID);
var newPackageName = ProductInfo.KeyValues["name"].AsString();
var apps = (await DbConnection.QueryAsync <PackageApp>("SELECT `AppID`, `Type` FROM `SubsApps` WHERE `SubID` = @SubID", new { SubID })).ToDictionary(x => x.AppID, x => x.Type);
// TODO: Ideally this should be SteamDB Unknown Package and proper checks like app processor does
if (newPackageName == null)
{
newPackageName = string.Concat("Steam Sub ", SubID);
}
if (newPackageName != null)
{
if (string.IsNullOrEmpty(PackageName))
{
await DbConnection.ExecuteAsync("INSERT INTO `Subs` (`SubID`, `Name`, `LastKnownName`) VALUES (@SubID, @Name, @Name)", new { SubID, Name = newPackageName });
await MakeHistory("created_sub");
await MakeHistory("created_info", SteamDB.DATABASE_NAME_TYPE, string.Empty, newPackageName);
}
else if (!PackageName.Equals(newPackageName))
{
if (newPackageName.StartsWith("Steam Sub ", StringComparison.Ordinal))
{
await DbConnection.ExecuteAsync("UPDATE `Subs` SET `Name` = @Name WHERE `SubID` = @SubID", new { SubID, Name = newPackageName });
}
else
{
await DbConnection.ExecuteAsync("UPDATE `Subs` SET `Name` = @Name, `LastKnownName` = @Name WHERE `SubID` = @SubID", new { SubID, Name = newPackageName });
}
await MakeHistory("modified_info", SteamDB.DATABASE_NAME_TYPE, PackageName, newPackageName);
}
}
foreach (var section in ProductInfo.KeyValues.Children)
{
var sectionName = section.Name.ToLower();
if (string.IsNullOrEmpty(sectionName) || sectionName.Equals("packageid") || sectionName.Equals("changenumber") || sectionName.Equals("name"))
{
// Ignore common keys
continue;
}
if (sectionName.Equals("appids") || sectionName.Equals("depotids"))
{
// Remove "ids", so we get "app" from appids and "depot" from depotids
var type = sectionName.Replace("ids", string.Empty);
var isAppSection = type.Equals("app");
var typeID = (uint)(isAppSection ? 0 : 1); // 0 = app, 1 = depot; can't store as string because it's in the `key` field
foreach (var childrenApp in section.Children)
{
var appID = uint.Parse(childrenApp.Value);
// Is this appid already in this package?
if (apps.ContainsKey(appID))
{
// Is this appid's type the same?
if (apps[appID] != type)
{
await DbConnection.ExecuteAsync("UPDATE `SubsApps` SET `Type` = @Type WHERE `SubID` = @SubID AND `AppID` = @AppID", new { SubID, AppID = appID, Type = type });
await MakeHistory("added_to_sub", typeID, apps[appID].Equals("app")? "0" : "1", childrenApp.Value);
appAddedToThisPackage = true;
// TODO: Log relevant add/remove history for depot/app?
}
apps.Remove(appID);
}
else
{
await DbConnection.ExecuteAsync("INSERT INTO `SubsApps` (`SubID`, `AppID`, `Type`) VALUES(@SubID, @AppID, @Type)", new { SubID, AppID = appID, Type = type });
await MakeHistory("added_to_sub", typeID, string.Empty, childrenApp.Value);
if (isAppSection)
{
await DbConnection.ExecuteAsync(AppProcessor.HistoryQuery,
new PICSHistory
{
ID = appID,
ChangeID = ChangeNumber,
NewValue = SubID.ToString(),
Action = "added_to_sub"
}
);
}
else
{
await DbConnection.ExecuteAsync(DepotProcessor.HistoryQuery,
new DepotHistory
{
DepotID = appID,
ChangeID = ChangeNumber,
NewValue = SubID,
Action = "added_to_sub"
}
);
}
appAddedToThisPackage = true;
if (packageOwned && !LicenseList.OwnedApps.ContainsKey(appID))
{
LicenseList.OwnedApps.Add(appID, 1);
}
}
}
}
else if (sectionName.Equals("extended"))
{
foreach (var children in section.Children)
{
var keyName = string.Format("{0}_{1}", sectionName, children.Name);
if (children.Children.Count > 0)
{
await ProcessKey(keyName, children.Name, Utils.JsonifyKeyValue(children), true);
}
else
{
await ProcessKey(keyName, children.Name, children.Value);
}
}
}
else if (section.Children.Any())
{
sectionName = string.Format("root_{0}", sectionName);
await ProcessKey(sectionName, sectionName, Utils.JsonifyKeyValue(section), true);
}
else if (!string.IsNullOrEmpty(section.Value))
{
var keyName = string.Format("root_{0}", sectionName);
await ProcessKey(keyName, sectionName, section.Value);
}
}
foreach (var data in CurrentData.Values.Where(data => !data.Processed && !data.KeyName.StartsWith("website", StringComparison.Ordinal)))
{
await DbConnection.ExecuteAsync("DELETE FROM `SubsInfo` WHERE `SubID` = @SubID AND `Key` = @Key", new { SubID, data.Key });
await MakeHistory("removed_key", data.Key, data.Value);
}
var appsRemoved = apps.Any();
foreach (var app in apps)
{
await DbConnection.ExecuteAsync("DELETE FROM `SubsApps` WHERE `SubID` = @SubID AND `AppID` = @AppID AND `Type` = @Type", new { SubID, AppID = app.Key, Type = app.Value });
var isAppSection = app.Value.Equals("app");
var typeID = (uint)(isAppSection ? 0 : 1); // 0 = app, 1 = depot; can't store as string because it's in the `key` field
await MakeHistory("removed_from_sub", typeID, app.Key.ToString());
if (isAppSection)
{
await DbConnection.ExecuteAsync(AppProcessor.HistoryQuery,
new PICSHistory
{
ID = app.Key,
ChangeID = ChangeNumber,
OldValue = SubID.ToString(),
Action = "removed_from_sub"
}
);
}
else
{
await DbConnection.ExecuteAsync(DepotProcessor.HistoryQuery,
new DepotHistory
{
DepotID = app.Key,
ChangeID = ChangeNumber,
OldValue = SubID,
Action = "removed_from_sub"
}
);
}
}
if (appsRemoved)
{
LicenseList.RefreshApps();
}
if (!packageOwned && SubID != 17906)
{
FreeLicense.RequestFromPackage(SubID, ProductInfo.KeyValues);
}
// Re-queue apps in this package so we can update depots and whatnot
if (appAddedToThisPackage && !Settings.IsFullRun && !string.IsNullOrEmpty(PackageName))
{
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(ProductInfo.KeyValues["appids"].Children.Select(x => (uint)x.AsInteger()), Enumerable.Empty <uint>()));
}
// Maintain a list of anonymous content
if (appAddedToThisPackage && SubID == 17906)
{
LicenseList.RefreshAnonymous();
}
}
public override async Task OnCommand(CommandArguments command)
{
if (command.Message.Length == 0)
{
command.Reply("Usage:{0} app <appid or partial game name>", Colors.OLIVE);
return;
}
string name;
if (!uint.TryParse(command.Message, out var appID))
{
name = command.Message;
if (!Utils.ConvertUserInputToSQLSearch(ref name))
{
command.Reply("Your request is invalid or too short.");
return;
}
using (var db = Database.GetConnection())
{
appID = db.ExecuteScalar <uint>("SELECT `AppID` FROM `Apps` WHERE `Apps`.`StoreName` LIKE @Name OR `Apps`.`Name` LIKE @Name OR `Apps`.`LastKnownName` LIKE @Name ORDER BY `LastUpdated` DESC LIMIT 1", new { Name = name });
}
if (appID == 0)
{
command.Reply("Nothing was found matching your request.");
return;
}
}
var tokenCallback = await Steam.Instance.Apps.PICSGetAccessTokens(appID, null);
SteamApps.PICSRequest request;
if (tokenCallback.AppTokens.ContainsKey(appID))
{
request = Utils.NewPICSRequest(appID, tokenCallback.AppTokens[appID]);
}
else
{
request = Utils.NewPICSRequest(appID);
}
var job = await Steam.Instance.Apps.PICSGetProductInfo(new List <SteamApps.PICSRequest> {
request
}, Enumerable.Empty <SteamApps.PICSRequest>());
var callback = job.Results.FirstOrDefault(x => x.Apps.ContainsKey(appID));
if (callback == null)
{
command.Reply("Unknown AppID: {0}{1}{2}", Colors.BLUE, appID, LicenseList.OwnedApps.ContainsKey(appID) ? SteamDB.StringCheckmark : string.Empty);
return;
}
var info = callback.Apps[appID];
if (info.KeyValues["common"]["name"].Value != null)
{
name = Utils.RemoveControlCharacters(info.KeyValues["common"]["name"].AsString());
}
else
{
name = Steam.GetAppName(info.ID);
}
info.KeyValues.SaveToFile(Path.Combine(Application.Path, "app", string.Format("{0}.vdf", info.ID)), false);
command.Reply("{0}{1}{2} -{3} {4}{5} - Dump:{6} {7}{8}{9}{10}",
Colors.BLUE, name, Colors.NORMAL,
Colors.DARKBLUE, SteamDB.GetAppURL(info.ID), Colors.NORMAL,
Colors.DARKBLUE, SteamDB.GetRawAppURL(info.ID), Colors.NORMAL,
info.MissingToken ? SteamDB.StringNeedToken : string.Empty,
LicenseList.OwnedApps.ContainsKey(info.ID) ? SteamDB.StringCheckmark : string.Empty
);
if (command.IsUserAdmin && !LicenseList.OwnedApps.ContainsKey(info.ID))
{
JobManager.AddJob(() => Steam.Instance.Apps.RequestFreeLicense(info.ID));
}
}
private static void OnFreeLicenseCallback(SteamApps.FreeLicenseCallback callback)
{
JobManager.TryRemoveJob(callback.JobID);
var packageIDs = callback.GrantedPackages;
var appIDs = callback.GrantedApps;
Log.WriteDebug("FreeLicense", "Received free license: {0} ({1} apps: {2}, {3} packages: {4})",
callback.Result, appIDs.Count, string.Join(", ", appIDs), packageIDs.Count, string.Join(", ", packageIDs));
if (appIDs.Count > 0)
{
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appIDs, Enumerable.Empty <uint>()));
}
if (packageIDs.Count > 0)
{
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetProductInfo(Enumerable.Empty <uint>(), packageIDs));
// We don't want to block our main thread with web requests
TaskManager.Run(() =>
{
string data = null;
try
{
var response = WebAuth.PerformRequest("GET", "https://store.steampowered.com/account/licenses/");
using (var responseStream = response.GetResponseStream())
{
using (var reader = new StreamReader(responseStream))
{
data = reader.ReadToEnd();
}
}
}
catch (WebException e)
{
Log.WriteError("FreeLicense", "Failed to fetch account details page: {0}", e.Message);
}
using (var db = Database.GetConnection())
{
foreach (var package in packageIDs)
{
var packageData = db.Query <Package>("SELECT `SubID`, `Name`, `LastKnownName` FROM `Subs` WHERE `SubID` = @SubID", new { SubID = package }).FirstOrDefault();
if (!string.IsNullOrEmpty(data))
{
// Tell me all about using regex
var match = Regex.Match(data, string.Format("RemoveFreeLicense\\( ?{0}, ?'(.+)' ?\\)", package));
if (match.Success)
{
var grantedName = Encoding.UTF8.GetString(Convert.FromBase64String(match.Groups[1].Value));
// Update last known name if we can
if (packageData.SubID > 0 && (string.IsNullOrEmpty(packageData.LastKnownName) || packageData.LastKnownName.StartsWith("Steam Sub ", StringComparison.Ordinal)))
{
db.Execute("UPDATE `Subs` SET `LastKnownName` = @Name WHERE `SubID` = @SubID", new { SubID = package, Name = grantedName });
db.Execute(SubProcessor.GetHistoryQuery(),
new PICSHistory
{
ID = package,
Key = SteamDB.DATABASE_NAME_TYPE,
OldValue = "free on demand; account page",
NewValue = grantedName,
Action = "created_info"
}
);
// Add a app comment on each app in this package
var comment = string.Format("This app is in a free on demand package called <b>{0}</b>", SecurityElement.Escape(grantedName));
var apps = db.Query <PackageApp>("SELECT `AppID` FROM `SubsApps` WHERE `SubID` = @SubID", new { SubID = package }).ToList();
var types = db.Query <App>("SELECT `AppID` FROM `Apps` WHERE `AppType` > 0 AND `AppID` IN @Ids", new { Ids = apps.Select(x => x.AppID) }).ToDictionary(x => x.AppID, x => true);
var key = db.ExecuteScalar <uint>("SELECT `ID` FROM `KeyNames` WHERE `Name` = 'website_comment'");
foreach (var app in apps)
{
if (types.ContainsKey(app.AppID))
{
continue;
}
db.Execute("INSERT INTO `AppsInfo` VALUES (@AppID, @Key, @Value) ON DUPLICATE KEY UPDATE `Key` = `Key`", new { app.AppID, Key = key, value = comment });
}
}
packageData.LastKnownName = grantedName;
}
}
IRC.Instance.SendMain("New free license granted: {0}{1}{2} -{3} {4}",
Colors.BLUE, Steam.FormatPackageName(package, packageData), Colors.NORMAL,
Colors.DARKBLUE, SteamDB.GetPackageURL(package)
);
}
}
});
}
}
private async Task <EPurchaseResultDetail> ActivateKey(string key)
{
var msg = new ClientMsgProtobuf <CMsgClientRegisterKey>(EMsg.ClientRegisterKey)
{
SourceJobID = Steam.Instance.Client.GetNextJobID(),
Body =
{
key = key
}
};
Steam.Instance.Client.Send(msg);
PurchaseResponseCallback job;
try
{
job = await new AsyncJob <PurchaseResponseCallback>(Steam.Instance.Client, msg.SourceJobID);
}
catch (Exception)
{
return(EPurchaseResultDetail.Timeout);
}
await using (var db = await Database.GetConnectionAsync())
{
using var sha = new SHA1CryptoServiceProvider();
await db.ExecuteAsync("UPDATE `SteamKeys` SET `SteamKey` = @HashedKey, `SubID` = @SubID, `Result` = @PurchaseResultDetail WHERE `SteamKey` = @SteamKey OR `SteamKey` = @HashedKey",
new
{
job.PurchaseResultDetail,
SubID = job.Packages.Count > 0 ? (int)job.Packages.First().Key : -1,
SteamKey = key,
HashedKey = Utils.ByteArrayToString(sha.ComputeHash(Encoding.ASCII.GetBytes(key)))
});
}
if (job.Packages.Count == 0)
{
if (job.PurchaseResultDetail != EPurchaseResultDetail.BadActivationCode)
{
IRC.Instance.SendOps($"{Colors.GREEN}[Keys]{Colors.NORMAL} Key not activated:{Colors.OLIVE} {job.Result} - {job.PurchaseResultDetail}");
}
return(job.PurchaseResultDetail);
}
if (job.PurchaseResultDetail != EPurchaseResultDetail.AlreadyPurchased &&
job.PurchaseResultDetail != EPurchaseResultDetail.DuplicateActivationCode &&
job.PurchaseResultDetail != EPurchaseResultDetail.DoesNotOwnRequiredApp)
{
var response = job.PurchaseResultDetail == EPurchaseResultDetail.NoDetail ?
$"{Colors.GREEN}Key activated" : $"{Colors.BLUE}{job.PurchaseResultDetail}";
IRC.Instance.SendOps($"{Colors.GREEN}[Keys]{Colors.NORMAL} {response}{Colors.NORMAL}. Packages:{Colors.OLIVE} {string.Join(", ", job.Packages.Select(x => $"{x.Key}: {x.Value}"))}");
}
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(Enumerable.Empty <uint>(), job.Packages.Keys));
await using (var db = await Database.GetConnectionAsync())
{
var apps = await db.QueryAsync <uint>("SELECT `AppID` FROM `SubsApps` WHERE `Type` = \"app\" AND `SubID` IN @Ids", new { Ids = job.Packages.Keys });
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(apps, Enumerable.Empty <uint>()));
foreach (var package in job.Packages)
{
var databaseName = (await db.QueryAsync <string>("SELECT `LastKnownName` FROM `Subs` WHERE `SubID` = @SubID", new { SubID = package.Key })).FirstOrDefault() ?? string.Empty;
if (databaseName.Equals(package.Value, StringComparison.CurrentCultureIgnoreCase))
{
continue;
}
await db.ExecuteAsync("UPDATE `Subs` SET `LastKnownName` = @Name WHERE `SubID` = @SubID", new { SubID = package.Key, Name = package.Value });
await db.ExecuteAsync(SubProcessor.HistoryQuery,
new PICSHistory
{
ID = package.Key,
Key = SteamDB.DATABASE_NAME_TYPE,
OldValue = "key activation",
NewValue = package.Value,
Action = "created_info"
}
);
}
}
return(job.PurchaseResultDetail);
}
private static void OnLicenseListCallback(SteamApps.LicenseListCallback licenseList)
{
if (licenseList.Result != EResult.OK)
{
Log.WriteError("LicenseList", "Failed: {0}", licenseList.Result);
return;
}
Log.WriteInfo("LicenseList", "Received {0} licenses from Steam", licenseList.LicenseList.Count);
if (licenseList.LicenseList.Count == 0)
{
return;
}
var ownedSubs = new Dictionary <uint, byte>();
var newSubs = new List <uint>();
var hasAnyLicense = OwnedSubs.Count > 0;
foreach (var license in licenseList.LicenseList)
{
// Expired licenses block access to depots, so we have no use in these
if (license.LicenseFlags.HasFlag(ELicenseFlags.Expired))
{
continue;
}
// For some obscure reason license list can contain duplicates
if (ownedSubs.ContainsKey(license.PackageID))
{
Log.WriteWarn("LicenseList", "Already contains {0} ({1})", license.PackageID, license.PaymentMethod);
continue;
}
if (hasAnyLicense && !OwnedSubs.ContainsKey(license.PackageID))
{
Log.WriteInfo("LicenseList", $"New license granted: {license.PackageID} ({license.PaymentMethod}, {license.LicenseFlags})");
newSubs.Add(license.PackageID);
}
ownedSubs.Add(license.PackageID, (byte)license.PaymentMethod);
}
OwnedSubs = ownedSubs;
RefreshApps();
if (newSubs.Count <= 0)
{
return;
}
using (var db = Database.Get())
{
var apps = db.Query <uint>("SELECT `AppID` FROM `SubsApps` WHERE `Type` = \"app\" AND `SubID` IN @Ids", new { Ids = newSubs });
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(apps, Enumerable.Empty <uint>()));
}
}
private void OnPICSChanges(SteamApps.PICSChangesCallback callback)
{
if (PreviousChangeNumber == callback.CurrentChangeNumber)
{
return;
}
Log.WriteInfo(nameof(PICSChanges), $"Changelist {callback.LastChangeNumber} -> {callback.CurrentChangeNumber} ({callback.AppChanges.Count} apps, {callback.PackageChanges.Count} packages)");
PreviousChangeNumber = callback.CurrentChangeNumber;
TaskManager.Run(async() => await HandleChangeNumbers(callback));
if (callback.RequiresFullAppUpdate || callback.RequiresFullPackageUpdate)
{
TaskManager.Run(async() =>
{
if (callback.RequiresFullAppUpdate)
{
IRC.Instance.SendOps($"Changelist {callback.CurrentChangeNumber} has forced a full app update");
// When full update flag is set, presumably Steam client start hammering the servers
// and the PICS service just does not return any data for a while until it clears up
await FullUpdateProcessor.FullUpdateAppsMetadata(true);
}
if (callback.RequiresFullPackageUpdate)
{
IRC.Instance.SendOps($"Changelist {callback.CurrentChangeNumber} has forced a full package update");
await FullUpdateProcessor.FullUpdatePackagesMetadata();
}
IRC.Instance.SendOps($"Changelist {callback.CurrentChangeNumber} full update has finished");
});
}
if (callback.AppChanges.Count == 0 && callback.PackageChanges.Count == 0)
{
return;
}
const int appsPerJob = 50;
if (callback.AppChanges.Count > appsPerJob)
{
foreach (var list in callback.AppChanges.Keys.Split(appsPerJob))
{
JobManager.AddJob(
() => Steam.Instance.Apps.PICSGetAccessTokens(list, Enumerable.Empty <uint>()),
new PICSTokens.RequestedTokens
{
Apps = list.ToList()
});
}
}
else if (callback.AppChanges.Count > 0)
{
JobManager.AddJob(
() => Steam.Instance.Apps.PICSGetAccessTokens(callback.AppChanges.Keys, Enumerable.Empty <uint>()),
new PICSTokens.RequestedTokens
{
Apps = callback.AppChanges.Keys.ToList()
});
}
if (callback.PackageChanges.Count > appsPerJob)
{
foreach (var list in callback.PackageChanges.Keys.Split(appsPerJob))
{
JobManager.AddJob(
() => Steam.Instance.Apps.PICSGetAccessTokens(Enumerable.Empty <uint>(), list),
new PICSTokens.RequestedTokens
{
Packages = list.ToList()
});
}
}
else if (callback.PackageChanges.Count > 0)
{
JobManager.AddJob(
() => Steam.Instance.Apps.PICSGetAccessTokens(Enumerable.Empty <uint>(), callback.PackageChanges.Keys),
new PICSTokens.RequestedTokens
{
Packages = callback.PackageChanges.Keys.ToList()
});
}
if (callback.AppChanges.Count > 0)
{
_ = TaskManager.Run(async() => await HandleApps(callback));
}
if (callback.PackageChanges.Count > 0)
{
_ = TaskManager.Run(async() => await HandlePackages(callback));
_ = TaskManager.Run(async() => await HandlePackagesChangelists(callback));
}
if (PreviousChangeNumber - LastStoredChangeNumber >= 1000)
{
LastStoredChangeNumber = PreviousChangeNumber;
_ = TaskManager.Run(async() => await LocalConfig.Update("backend.changenumber", LastStoredChangeNumber.ToString()));
}
PrintImportants(callback);
}
private void OnPICSChanges(SteamApps.PICSChangesCallback callback)
{
if (PreviousChangeNumber == callback.CurrentChangeNumber)
{
return;
}
Log.WriteInfo(nameof(PICSChanges), $"Changelist {PreviousChangeNumber} -> {callback.CurrentChangeNumber} ({callback.AppChanges.Count} apps, {callback.PackageChanges.Count} packages)");
LocalConfig.Current.ChangeNumber = PreviousChangeNumber = callback.CurrentChangeNumber;
TaskManager.Run(async() => await HandleChangeNumbers(callback));
if (callback.RequiresFullAppUpdate || callback.RequiresFullPackageUpdate)
{
TaskManager.Run(async() =>
{
if (callback.RequiresFullAppUpdate)
{
IRC.Instance.SendOps($"Changelist {callback.CurrentChangeNumber} has forced a full app update");
await FullUpdateProcessor.FullUpdateAppsMetadata();
}
if (callback.RequiresFullPackageUpdate)
{
IRC.Instance.SendOps($"Changelist {callback.CurrentChangeNumber} has forced a full package update");
await FullUpdateProcessor.FullUpdatePackagesMetadata();
}
});
}
if (callback.AppChanges.Count == 0 && callback.PackageChanges.Count == 0)
{
IRC.Instance.SendAnnounce($"{Colors.RED}»{Colors.NORMAL} Changelist {Colors.BLUE}{PreviousChangeNumber}{Colors.DARKGRAY} (empty)");
return;
}
const int appsPerJob = 50;
if (callback.AppChanges.Count > appsPerJob)
{
foreach (var list in callback.AppChanges.Keys.Split(appsPerJob))
{
JobManager.AddJob(
() => Steam.Instance.Apps.PICSGetAccessTokens(list, Enumerable.Empty <uint>()),
new PICSTokens.RequestedTokens
{
Apps = list.ToList()
});
}
}
else if (callback.AppChanges.Count > 0)
{
JobManager.AddJob(
() => Steam.Instance.Apps.PICSGetAccessTokens(callback.AppChanges.Keys, Enumerable.Empty <uint>()),
new PICSTokens.RequestedTokens
{
Apps = callback.AppChanges.Keys.ToList()
});
}
if (callback.PackageChanges.Count > appsPerJob)
{
foreach (var list in callback.PackageChanges.Keys.Split(appsPerJob))
{
JobManager.AddJob(
() => Steam.Instance.Apps.PICSGetAccessTokens(Enumerable.Empty <uint>(), list),
new PICSTokens.RequestedTokens
{
Packages = list.ToList()
});
}
}
else if (callback.PackageChanges.Count > 0)
{
JobManager.AddJob(
() => Steam.Instance.Apps.PICSGetAccessTokens(Enumerable.Empty <uint>(), callback.PackageChanges.Keys),
new PICSTokens.RequestedTokens
{
Packages = callback.PackageChanges.Keys.ToList()
});
}
if (callback.AppChanges.Count > 0)
{
_ = TaskManager.Run(async() => await HandleApps(callback));
}
if (callback.PackageChanges.Count > 0)
{
_ = TaskManager.Run(async() => await HandlePackages(callback));
_ = TaskManager.Run(async() => await HandlePackagesChangelists(callback));
}
_ = TaskManager.Run(async() => await SendChangelistsToIRC(callback));
PrintImportants(callback);
}
public void Process(uint appID, uint changeNumber, KeyValue depots)
{
var requests = new List <ManifestJob>();
// Get data in format we want first
foreach (var depot in depots.Children)
{
// Ignore these for now, parent app should be updated too anyway
if (depot["depotfromapp"].Value != null)
{
continue;
}
var request = new ManifestJob
{
ChangeNumber = changeNumber,
ParentAppID = appID,
DepotName = depot["name"].AsString()
};
// Ignore keys that aren't integers, for example "branches"
if (!uint.TryParse(depot.Name, out request.DepotID))
{
continue;
}
// TODO: instead of locking we could wait for current process to finish
if (DepotLocks.ContainsKey(request.DepotID))
{
continue;
}
// If there is no public manifest for this depot, it still could have some sort of open beta
if (depot["manifests"]["public"].Value == null || !ulong.TryParse(depot["manifests"]["public"].Value, out request.ManifestID))
{
var branch = depot["manifests"].Children.FirstOrDefault(x => x.Name != "local");
if (branch == null || !ulong.TryParse(branch.Value, out request.ManifestID))
{
using (var db = Database.GetConnection())
{
db.Execute("INSERT INTO `Depots` (`DepotID`, `Name`) VALUES (@DepotID, @DepotName) ON DUPLICATE KEY UPDATE `Name` = @DepotName", new { request.DepotID, request.DepotName });
}
continue;
}
request.BuildID = branch["build"].AsInteger();
}
else
{
request.BuildID = depots["branches"]["public"]["buildid"].AsInteger();
}
requests.Add(request);
}
if (!requests.Any())
{
return;
}
using (var db = Database.GetConnection())
{
var dbDepots = db.Query <Depot>("SELECT `DepotID`, `Name`, `BuildID`, `ManifestID`, `LastManifestID` FROM `Depots` WHERE `DepotID` IN @Depots", new { Depots = requests.Select(x => x.DepotID) })
.ToDictionary(x => x.DepotID, x => x);
foreach (var request in requests)
{
Depot dbDepot;
if (dbDepots.ContainsKey(request.DepotID))
{
dbDepot = dbDepots[request.DepotID];
if (dbDepot.BuildID > request.BuildID)
{
// buildid went back in time? this either means a rollback, or a shared depot that isn't synced properly
Log.WriteDebug("Depot Processor", "Skipping depot {0} due to old buildid: {1} > {2}", request.DepotID, dbDepot.BuildID, request.BuildID);
continue;
}
if (dbDepot.LastManifestID == request.ManifestID && dbDepot.ManifestID == request.ManifestID && Settings.Current.FullRun < 2)
{
// Update depot name if changed
if (!request.DepotName.Equals(dbDepot.Name))
{
db.Execute("UPDATE `Depots` SET `Name` = @DepotName WHERE `DepotID` = @DepotID", new { request.DepotID, request.DepotName });
}
continue;
}
}
else
{
dbDepot = new Depot();
}
if (dbDepot.BuildID != request.BuildID || dbDepot.ManifestID != request.ManifestID || !request.DepotName.Equals(dbDepot.Name))
{
db.Execute(@"INSERT INTO `Depots` (`DepotID`, `Name`, `BuildID`, `ManifestID`) VALUES (@DepotID, @DepotName, @BuildID, @ManifestID)
ON DUPLICATE KEY UPDATE `LastUpdated` = CURRENT_TIMESTAMP(), `Name` = @DepotName, `BuildID` = @BuildID, `ManifestID` = @ManifestID",
new {
request.DepotID,
request.DepotName,
request.BuildID,
request.ManifestID
});
}
if (dbDepot.ManifestID != request.ManifestID)
{
MakeHistory(db, request, string.Empty, "manifest_change", dbDepot.ManifestID, request.ManifestID);
}
if (LicenseList.OwnedApps.ContainsKey(request.DepotID) || Settings.Current.FullRun > 1)
{
DepotLocks.TryAdd(request.DepotID, 1);
JobManager.AddJob(() => Steam.Instance.Apps.GetDepotDecryptionKey(request.DepotID, request.ParentAppID), request);
}
#if DEBUG
else
{
Log.WriteDebug("Depot Processor", "Skipping depot {0} from app {1} because we don't own it", request.DepotID, request.ParentAppID);
}
#endif
}
}
}
private void OnTimer(object sender, ElapsedEventArgs e)
{
if (!Steam.Instance.IsLoggedOn)
{
lock (FreeLicenseTimer)
{
FreeLicenseTimer.Start();
}
return;
}
if (FreeLicensesToRequest.IsEmpty)
{
TaskManager.Run(RequestBetas);
return;
}
var list = FreeLicensesToRequest.Take(REQUEST_RATE_LIMIT).ToList();
var now = DateUtils.DateTimeToUnixTime(DateTime.UtcNow) - 60;
Dictionary <uint, ulong> startTimes;
using (var db = Database.Get())
{
startTimes = db.Query(
"SELECT `SubID`, `Value` FROM `SubsInfo` WHERE `Key` = @Key AND `SubID` IN @Ids",
new
{
Key = KeyNameCache.GetSubKeyID("extended_starttime"),
Ids = list.Select(x => x.Key)
}
).ToDictionary(x => (uint)x.SubID, x => Convert.ToUInt64((string)x.Value));
}
foreach (var(subId, _) in list)
{
if (startTimes.TryGetValue(subId, out var startTime) && startTime > now)
{
// If start time has not been reached yet, don't remove this app from the list and keep trying to activate it
continue;
}
FreeLicensesToRequest.TryRemove(subId, out _);
}
TaskManager.Run(Save);
TaskManager.Run(RequestBetas);
var appids = list.Select(x => x.Value).Distinct();
AppsRequestedInHour = appids.Count();
Log.WriteDebug(nameof(FreeLicense), $"Requesting {AppsRequestedInHour} free apps as the rate limit timer ran: {string.Join(", ", appids)}");
JobManager.AddJob(() => Steam.Instance.Apps.RequestFreeLicense(appids));
if (!FreeLicensesToRequest.IsEmpty)
{
lock (FreeLicenseTimer)
{
FreeLicenseTimer.Start();
}
}
}
private void DownloadManifest(ManifestJob request)
{
Log.WriteInfo("Depot Processor", "DepotID: {0}", request.DepotID);
DepotManifest depotManifest = null;
string lastError = string.Empty;
// CDN is very random, just keep trying
for (var i = 0; i <= 5; i++)
{
try
{
depotManifest = CDNClient.DownloadManifest(request.DepotID, request.ManifestID, request.Server, request.CDNToken, request.DepotKey);
break;
}
catch (Exception e)
{
lastError = e.Message;
}
}
if (depotManifest == null)
{
Log.WriteError("Depot Processor", "Failed to download depot manifest for depot {0} ({1}: {2}) (#{3})", request.DepotID, request.Server, lastError, request.Tries);
if (--request.Tries >= 0)
{
request.Server = GetContentServer(request.Tries);
JobManager.AddJob(() => Steam.Instance.Apps.GetCDNAuthToken(request.DepotID, request.Server), request);
return;
}
RemoveLock(request.DepotID); // TODO: Remove this once task in OnCDNAuthTokenCallback is used
if (FileDownloader.IsImportantDepot(request.DepotID))
{
IRC.Instance.SendOps("{0}[{1}]{2} Failed to download depot {3} manifest ({4}: {5})",
Colors.OLIVE, Steam.GetAppName(request.ParentAppID), Colors.NORMAL, request.DepotID, request.Server, lastError);
}
return;
}
if (FileDownloader.IsImportantDepot(request.DepotID))
{
TaskManager.Run(() => FileDownloader.DownloadFilesFromDepot(request, depotManifest));
}
// TODO: Task here instead of in OnCDNAuthTokenCallback due to mono's silly threadpool
TaskManager.Run(() =>
{
using (var db = Database.GetConnection())
{
ProcessDepotAfterDownload(db, request, depotManifest);
}
}).ContinueWith(task =>
{
RemoveLock(request.DepotID);
Log.WriteDebug("Depot Processor", "Processed depot {0} ({1} depot locks left)", request.DepotID, DepotLocks.Count);
});
}
private async Task DownloadDepots(uint appID, List <ManifestJob> depots)
{
Log.WriteDebug(nameof(DepotProcessor), $"Will process {depots.Count} depots from app {appID} ({DepotLocks.Count} depot locks left)");
var processTasks = new List <Task <(uint DepotID, EResult Result)> >();
var anyFilesDownloaded = false;
var willDownloadFiles = false;
foreach (var depot in depots)
{
if (depot.DepotKey == null)
{
await GetDepotDecryptionKey(Steam.Instance.Apps, depot, appID);
if (depot.DepotKey == null &&
depot.LastManifestID == depot.ManifestID &&
Settings.FullRun != FullRunState.WithForcedDepots)
{
RemoveLock(depot.DepotID);
continue;
}
}
depot.Server = GetContentServer();
DepotManifest depotManifest = null;
var lastError = string.Empty;
for (var i = 0; i <= 5; i++)
{
try
{
await ManifestDownloadSemaphore.WaitAsync(TaskManager.TaskCancellationToken.Token).ConfigureAwait(false);
depotManifest = await CDNClient.DownloadManifestAsync(depot.DepotID, depot.ManifestID, depot.Server, string.Empty, depot.DepotKey);
break;
}
catch (Exception e)
{
lastError = e.Message;
Log.WriteError(nameof(DepotProcessor), $"Failed to download depot manifest for app {appID} depot {depot.DepotID} ({depot.Server}: {lastError}) (#{i})");
}
finally
{
ManifestDownloadSemaphore.Release();
}
if (depot.DepotKey != null)
{
RemoveErroredServer(depot.Server);
}
if (depotManifest == null && i < 5)
{
await Task.Delay(Utils.ExponentionalBackoff(i + 1));
depot.Server = GetContentServer();
}
}
if (depotManifest == null)
{
RemoveLock(depot.DepotID);
if (FileDownloader.IsImportantDepot(depot.DepotID))
{
IRC.Instance.SendOps($"{Colors.OLIVE}[{depot.DepotName}]{Colors.NORMAL} Failed to download manifest ({lastError})");
}
if (!Settings.IsFullRun && depot.DepotKey != null)
{
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null));
}
continue;
}
var task = ProcessDepotAfterDownload(depot, depotManifest);
processTasks.Add(task);
if (!FileDownloader.IsImportantDepot(depot.DepotID) || depot.DepotKey == null)
{
depot.Result = EResult.Ignored;
continue;
}
willDownloadFiles = true;
task = TaskManager.Run(async() =>
{
var result = EResult.Fail;
try
{
result = await FileDownloader.DownloadFilesFromDepot(depot, depotManifest);
if (result == EResult.OK)
{
anyFilesDownloaded = true;
}
}
catch (Exception e)
{
ErrorReporter.Notify($"Depot Processor {depot.DepotID}", e);
}
return(depot.DepotID, result);
});
processTasks.Add(task);
}
if (!anyFilesDownloaded && !willDownloadFiles)
{
foreach (var task in processTasks)
{
_ = task.ContinueWith(result =>
{
RemoveLock(result.Result.DepotID);
}, TaskManager.TaskCancellationToken.Token);
}
await Task.WhenAll(processTasks).ConfigureAwait(false);
return;
}
await Task.WhenAll(processTasks).ConfigureAwait(false);
Log.WriteDebug(nameof(DepotProcessor), $"{depots.Count} depot downloads finished for app {appID}");
lock (UpdateScriptLock)
{
foreach (var depot in depots)
{
if (depot.Result == EResult.OK)
{
RunUpdateScript(UpdateScript, $"{depot.DepotID} no-git");
}
else if (depot.Result != EResult.Ignored)
{
Log.WriteWarn(nameof(DepotProcessor), $"Download failed for {depot.DepotID}: {depot.Result}");
RemoveErroredServer(depot.Server);
// Mark this depot for redownload
using var db = Database.Get();
db.Execute("UPDATE `Depots` SET `LastManifestID` = 0 WHERE `DepotID` = @DepotID", new { depot.DepotID });
}
RemoveLock(depot.DepotID);
}
// Only commit changes if all depots downloaded
if (processTasks.All(x => x.Result.Result == EResult.OK || x.Result.Result == EResult.Ignored))
{
if (!RunUpdateScriptForApp(appID, depots[0].BuildID))
{
RunUpdateScript(UpdateScript, "0");
}
}
else
{
Log.WriteDebug(nameof(DepotProcessor), $"Reprocessing the app {appID} because some files failed to download");
IRC.Instance.SendOps($"{Colors.OLIVE}[{Steam.GetAppName(appID)}]{Colors.NORMAL} Reprocessing the app due to download failures");
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null));
}
}
}
private static void OnPICSProductInfo(SteamApps.PICSProductInfoCallback callback)
{
JobManager.TryRemoveJob(callback.JobID);
var apps = callback.Apps.Concat(callback.UnknownApps.ToDictionary(x => x, x => (SteamApps.PICSProductInfoCallback.PICSProductInfo)null));
var packages = callback.Packages.Concat(callback.UnknownPackages.ToDictionary(x => x, x => (SteamApps.PICSProductInfoCallback.PICSProductInfo)null));
foreach (var workaround in apps)
{
var app = workaround;
Log.WriteInfo("PICSProductInfo", "{0}AppID: {1}", app.Value == null ? "Unknown " : "", app.Key);
IWorkItemResult mostRecentItem;
lock (ProcessedApps)
{
ProcessedApps.TryGetValue(app.Key, out mostRecentItem);
}
var workerItem = ProcessorThreadPool.QueueWorkItem(delegate
{
try
{
if (mostRecentItem != null && !mostRecentItem.IsCompleted)
{
Log.WriteDebug("PICSProductInfo", "Waiting for app {0} to finish processing", app.Key);
SmartThreadPool.WaitAll(new IWaitableResult[] { mostRecentItem });
}
using (var processor = new AppProcessor(app.Key))
{
if (app.Value == null)
{
processor.ProcessUnknown();
}
else
{
processor.Process(app.Value);
}
}
}
catch (MySqlException e)
{
Log.WriteError("PICSProductInfo", "App {0} faulted: {1}", app.Key, e);
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(app.Key, null));
}
catch (Exception e)
{
Log.WriteError("PICSProductInfo", "App {0} faulted: {1}", app.Key, e);
}
finally
{
lock (ProcessedApps)
{
if (ProcessedApps.TryGetValue(app.Key, out mostRecentItem) && mostRecentItem.IsCompleted)
{
ProcessedApps.Remove(app.Key);
}
}
}
});
if (Settings.IsFullRun)
{
continue;
}
lock (ProcessedApps)
{
ProcessedApps[app.Key] = workerItem;
}
}
foreach (var workaround in packages)
{
var package = workaround;
Log.WriteInfo("PICSProductInfo", "{0}SubID: {1}", package.Value == null ? "Unknown " : "", package.Key);
IWorkItemResult mostRecentItem;
lock (ProcessedSubs)
{
ProcessedSubs.TryGetValue(package.Key, out mostRecentItem);
}
var workerItem = ProcessorThreadPool.QueueWorkItem(delegate
{
try
{
if (mostRecentItem != null && !mostRecentItem.IsCompleted)
{
Log.WriteDebug("PICSProductInfo", "Waiting for package {0} to finish processing", package.Key);
SmartThreadPool.WaitAll(new IWaitableResult[] { mostRecentItem });
}
using (var processor = new SubProcessor(package.Key))
{
if (package.Value == null)
{
processor.ProcessUnknown();
}
else
{
processor.Process(package.Value);
}
}
}
catch (MySqlException e)
{
Log.WriteError("PICSProductInfo", "Package {0} faulted: {1}", package.Key, e);
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetProductInfo(null, package.Key, false, false));
}
catch (Exception e)
{
Log.WriteError("PICSProductInfo", "Package {0} faulted: {1}", package.Key, e);
}
finally
{
lock (ProcessedSubs)
{
if (ProcessedSubs.TryGetValue(package.Key, out mostRecentItem) && mostRecentItem.IsCompleted)
{
ProcessedSubs.Remove(package.Key);
}
}
}
});
if (Settings.IsFullRun)
{
continue;
}
lock (ProcessedSubs)
{
ProcessedSubs[package.Key] = workerItem;
}
}
}
public static async Task HandleMetadataInfo(SteamApps.PICSProductInfoCallback callback)
{
var apps = new List <uint>();
var subs = new List <uint>();
await using var db = await Database.GetConnectionAsync();
if (callback.Apps.Any())
{
Log.WriteDebug(nameof(FullUpdateProcessor), $"Received metadata only product info for {callback.Apps.Count} apps ({callback.Apps.First().Key}...{callback.Apps.Last().Key}), job: {callback.JobID}");
var currentChangeNumbers = (await db.QueryAsync <(uint, uint)>(
"SELECT `AppID`, `Value` FROM `AppsInfo` WHERE `Key` = @ChangeNumberKey AND `AppID` IN @Apps",
new
{
ChangeNumberKey = KeyNameCache.GetAppKeyID("root_changenumber"),
Apps = callback.Apps.Keys
}
)).ToDictionary(x => x.Item1, x => x.Item2);
foreach (var app in callback.Apps.Values)
{
currentChangeNumbers.TryGetValue(app.ID, out var currentChangeNumber);
if (currentChangeNumber == app.ChangeNumber)
{
continue;
}
Log.WriteInfo(nameof(FullUpdateProcessor), $"App {app.ID} - Change: {currentChangeNumber} -> {app.ChangeNumber}");
apps.Add(app.ID);
if (!Settings.IsFullRun)
{
await db.ExecuteAsync("INSERT INTO `Changelists` (`ChangeID`) VALUES (@ChangeNumber) ON DUPLICATE KEY UPDATE `Date` = `Date`", new { app.ChangeNumber });
await db.ExecuteAsync("INSERT INTO `ChangelistsApps` (`ChangeID`, `AppID`) VALUES (@ChangeNumber, @AppID) ON DUPLICATE KEY UPDATE `AppID` = `AppID`", new { AppID = app.ID, app.ChangeNumber });
}
}
}
if (callback.Packages.Any())
{
Log.WriteDebug(nameof(FullUpdateProcessor), $"Received metadata only product info for {callback.Packages.Count} packages ({callback.Packages.First().Key}...{callback.Packages.Last().Key}), job: {callback.JobID}");
var currentChangeNumbers = (await db.QueryAsync <(uint, uint)>(
"SELECT `SubID`, `Value` FROM `SubsInfo` WHERE `Key` = @ChangeNumberKey AND `SubID` IN @Subs",
new
{
ChangeNumberKey = KeyNameCache.GetSubKeyID("root_changenumber"),
Subs = callback.Packages.Keys
}
)).ToDictionary(x => x.Item1, x => x.Item2);
foreach (var sub in callback.Packages.Values)
{
currentChangeNumbers.TryGetValue(sub.ID, out var currentChangeNumber);
if (currentChangeNumber == sub.ChangeNumber)
{
continue;
}
Log.WriteInfo(nameof(FullUpdateProcessor), $"Package {sub.ID} - Change: {currentChangeNumber} -> {sub.ChangeNumber}");
subs.Add(sub.ID);
if (!Settings.IsFullRun)
{
await db.ExecuteAsync("INSERT INTO `Changelists` (`ChangeID`) VALUES (@ChangeNumber) ON DUPLICATE KEY UPDATE `Date` = `Date`", new { sub.ChangeNumber });
await db.ExecuteAsync("INSERT INTO `ChangelistsSubs` (`ChangeID`, `SubID`) VALUES (@ChangeNumber, @SubID) ON DUPLICATE KEY UPDATE `SubID` = `SubID`", new { SubID = sub.ID, sub.ChangeNumber });
}
}
}
if (apps.Any() || subs.Any())
{
JobManager.AddJob(
() => Steam.Instance.Apps.PICSGetAccessTokens(apps, subs),
new PICSTokens.RequestedTokens
{
Apps = apps,
Packages = subs,
});
}
}
public override async Task OnCommand(CommandArguments command)
{
if (string.IsNullOrWhiteSpace(command.Message))
{
command.Reply("Usage:{0} app <appid or partial game name>", Colors.OLIVE);
return;
}
string name;
if (!uint.TryParse(command.Message, out var appID))
{
appID = await TrySearchAppId(command);
if (appID == 0)
{
return;
}
}
var tokenCallback = await Steam.Instance.Apps.PICSGetAccessTokens(appID, null);
SteamApps.PICSRequest request;
if (tokenCallback.AppTokens.ContainsKey(appID))
{
request = Utils.NewPICSRequest(appID, tokenCallback.AppTokens[appID]);
}
else
{
request = Utils.NewPICSRequest(appID);
}
var job = await Steam.Instance.Apps.PICSGetProductInfo(new List <SteamApps.PICSRequest> {
request
}, Enumerable.Empty <SteamApps.PICSRequest>());
var callback = job.Results.FirstOrDefault(x => x.Apps.ContainsKey(appID));
if (callback == null)
{
command.Reply("Unknown AppID: {0}{1}{2}", Colors.BLUE, appID, LicenseList.OwnedApps.ContainsKey(appID) ? SteamDB.StringCheckmark : string.Empty);
return;
}
var info = callback.Apps[appID];
if (info.KeyValues["common"]["name"].Value != null)
{
name = Utils.RemoveControlCharacters(info.KeyValues["common"]["name"].AsString());
}
else
{
name = Steam.GetAppName(info.ID);
}
info.KeyValues.SaveToFile(Path.Combine(Application.Path, "app", string.Format("{0}.vdf", info.ID)), false);
command.Reply("{0}{1}{2} -{3} {4}{5} - Dump:{6} {7}{8}{9}{10}",
Colors.BLUE, name, Colors.NORMAL,
Colors.DARKBLUE, SteamDB.GetAppURL(info.ID), Colors.NORMAL,
Colors.DARKBLUE, SteamDB.GetRawAppURL(info.ID), Colors.NORMAL,
info.MissingToken ? SteamDB.StringNeedToken : string.Empty,
LicenseList.OwnedApps.ContainsKey(info.ID) ? SteamDB.StringCheckmark : string.Empty
);
if (command.IsUserAdmin && !LicenseList.OwnedApps.ContainsKey(info.ID))
{
JobManager.AddJob(() => Steam.Instance.Apps.RequestFreeLicense(info.ID));
}
}
protected override async Task ProcessData()
{
await LoadData();
ChangeNumber = ProductInfo.ChangeNumber;
if (Settings.IsFullRun)
{
await DbConnection.ExecuteAsync("INSERT INTO `Changelists` (`ChangeID`) VALUES (@ChangeNumber) ON DUPLICATE KEY UPDATE `Date` = `Date`", new { ProductInfo.ChangeNumber });
await DbConnection.ExecuteAsync("INSERT INTO `ChangelistsSubs` (`ChangeID`, `SubID`) VALUES (@ChangeNumber, @SubID) ON DUPLICATE KEY UPDATE `SubID` = `SubID`", new { SubID, ProductInfo.ChangeNumber });
}
await ProcessKey("root_changenumber", "changenumber", ChangeNumber.ToString());
var appAddedToThisPackage = false;
var hasPackageInfo = ProductInfo.KeyValues.Children.Count > 0;
var packageOwned = LicenseList.OwnedSubs.ContainsKey(SubID);
var newPackageName = ProductInfo.KeyValues["name"].AsString() ?? string.Concat("Steam Sub ", SubID);
var apps = (await DbConnection.QueryAsync <PackageApp>("SELECT `AppID`, `Type` FROM `SubsApps` WHERE `SubID` = @SubID", new { SubID })).ToDictionary(x => x.AppID, x => x.Type);
var alreadySeenAppIds = new HashSet <uint>();
if (!hasPackageInfo)
{
ProductInfo.KeyValues.Children.Add(new KeyValue("steamdb_requires_token", "1"));
}
if (string.IsNullOrEmpty(PackageName))
{
await DbConnection.ExecuteAsync("INSERT INTO `Subs` (`SubID`, `Name`, `LastKnownName`) VALUES (@SubID, @Name, @Name)", new { SubID, Name = newPackageName });
await MakeHistory("created_sub");
await MakeHistory("created_info", SteamDB.DatabaseNameType, string.Empty, newPackageName);
}
else if (PackageName != newPackageName)
{
if (newPackageName.StartsWith("Steam Sub ", StringComparison.Ordinal))
{
await DbConnection.ExecuteAsync("UPDATE `Subs` SET `Name` = @Name WHERE `SubID` = @SubID", new { SubID, Name = newPackageName });
}
else
{
await DbConnection.ExecuteAsync("UPDATE `Subs` SET `Name` = @Name, `LastKnownName` = @Name WHERE `SubID` = @SubID", new { SubID, Name = newPackageName });
}
await MakeHistory("modified_info", SteamDB.DatabaseNameType, PackageName, newPackageName);
}
foreach (var section in ProductInfo.KeyValues.Children)
{
var sectionName = section.Name.ToLowerInvariant();
if (string.IsNullOrEmpty(sectionName) || sectionName == "packageid" || sectionName == "changenumber" || sectionName == "name")
{
// Ignore common keys
continue;
}
if (sectionName == "appids" || sectionName == "depotids")
{
// Remove "ids", so we get "app" from appids and "depot" from depotids
var type = sectionName.Replace("ids", string.Empty);
var isAppSection = type == "app";
var typeID = (uint)(isAppSection ? 0 : 1); // 0 = app, 1 = depot; can't store as string because it's in the `key` field
foreach (var childrenApp in section.Children)
{
if (!uint.TryParse(childrenApp.Value, out var appID))
{
Log.WriteWarn(nameof(SubProcessor), $"Package {SubID} has an invalid uint: {childrenApp.Value}");
continue;
}
if (alreadySeenAppIds.Contains(appID))
{
Log.WriteWarn(nameof(SubProcessor), $"Package {SubID} has a duplicate app: {appID}");
continue;
}
alreadySeenAppIds.Add(appID);
// Is this appid already in this package?
if (apps.ContainsKey(appID))
{
// Is this appid's type the same?
if (apps[appID] != type)
{
await DbConnection.ExecuteAsync("UPDATE `SubsApps` SET `Type` = @Type WHERE `SubID` = @SubID AND `AppID` = @AppID", new { SubID, AppID = appID, Type = type });
await MakeHistory("added_to_sub", typeID, apps[appID] == "app"? "0" : "1", childrenApp.Value);
appAddedToThisPackage = true;
// Log relevant add/remove history events for depot and app
var appHistory = new PICSHistory
{
ID = appID,
ChangeID = ChangeNumber,
};
if (isAppSection)
{
appHistory.NewValue = SubID.ToString();
appHistory.Action = "added_to_sub";
}
else
{
appHistory.OldValue = SubID.ToString();
appHistory.Action = "removed_from_sub";
}
await DbConnection.ExecuteAsync(AppProcessor.HistoryQuery, appHistory);
var depotHistory = new DepotHistory
{
DepotID = appID,
ManifestID = 0,
ChangeID = ChangeNumber,
OldValue = SubID,
Action = isAppSection ? "removed_from_sub" : "added_to_sub"
};
if (isAppSection)
{
depotHistory.OldValue = SubID;
depotHistory.Action = "removed_from_sub";
}
else
{
depotHistory.NewValue = SubID;
depotHistory.Action = "added_to_sub";
}
await DbConnection.ExecuteAsync(DepotProcessor.HistoryQuery, depotHistory);
}
apps.Remove(appID);
}
else
{
await DbConnection.ExecuteAsync("INSERT INTO `SubsApps` (`SubID`, `AppID`, `Type`) VALUES(@SubID, @AppID, @Type)", new { SubID, AppID = appID, Type = type });
await MakeHistory("added_to_sub", typeID, string.Empty, childrenApp.Value);
if (isAppSection)
{
await DbConnection.ExecuteAsync(AppProcessor.HistoryQuery,
new PICSHistory
{
ID = appID,
ChangeID = ChangeNumber,
NewValue = SubID.ToString(),
Action = "added_to_sub"
}
);
}
else
{
await DbConnection.ExecuteAsync(DepotProcessor.HistoryQuery,
new DepotHistory
{
DepotID = appID,
ManifestID = 0,
ChangeID = ChangeNumber,
NewValue = SubID,
Action = "added_to_sub"
}
);
}
appAddedToThisPackage = true;
if (packageOwned && !LicenseList.OwnedApps.ContainsKey(appID))
{
LicenseList.OwnedApps.Add(appID, 1);
}
}
}
}
else if (sectionName == "extended")
{
foreach (var children in section.Children)
{
var keyName = $"{sectionName}_{children.Name}";
if (children.Children.Count > 0)
{
await ProcessKey(keyName, children.Name, Utils.JsonifyKeyValue(children), true);
}
else
{
await ProcessKey(keyName, children.Name, children.Value);
}
}
}
else if (sectionName == "appitems" && section.Children.Count > 1)
{
sectionName = $"root_{sectionName}";
var fixedAppItems = new KeyValue(section.Name);
// Valve for some reason creates a new children for each item,
// instead of actually making it an array.
// This causes json_decode in php override the key, thus lose data.
foreach (var item in section.Children)
{
var appItem = fixedAppItems.Children.Find(s => s.Name == item.Name);
if (appItem == default)
{
appItem = new KeyValue(item.Name);
fixedAppItems.Children.Add(appItem);
}
foreach (var itemId in item.Children)
{
appItem.Children.Add(new KeyValue(itemId.Name, itemId.Value));
}
}
await ProcessKey(sectionName, sectionName, Utils.JsonifyKeyValue(fixedAppItems), true);
}
else if (section.Children.Count > 0)
{
sectionName = $"root_{sectionName}";
await ProcessKey(sectionName, sectionName, Utils.JsonifyKeyValue(section), true);
}
else if (!string.IsNullOrEmpty(section.Value))
{
var keyName = $"root_{sectionName}";
await ProcessKey(keyName, sectionName, section.Value);
}
}
// If this package no longer returns any package info, keep the existing info we have
if (hasPackageInfo)
{
foreach (var data in CurrentData.Values.Where(data => !data.Processed && !data.KeyName.StartsWith("website", StringComparison.Ordinal)))
{
await DbConnection.ExecuteAsync("DELETE FROM `SubsInfo` WHERE `SubID` = @SubID AND `Key` = @Key", new { SubID, data.Key });
await MakeHistory("removed_key", data.Key, data.Value);
}
var appsRemoved = apps.Count > 0;
foreach (var(appid, type) in apps)
{
await DbConnection.ExecuteAsync("DELETE FROM `SubsApps` WHERE `SubID` = @SubID AND `AppID` = @AppID AND `Type` = @Type", new { SubID, AppID = appid, Type = type });
var isAppSection = type == "app";
var typeID = (uint)(isAppSection ? 0 : 1); // 0 = app, 1 = depot; can't store as string because it's in the `key` field
await MakeHistory("removed_from_sub", typeID, appid.ToString());
if (isAppSection)
{
await DbConnection.ExecuteAsync(AppProcessor.HistoryQuery,
new PICSHistory
{
ID = appid,
ChangeID = ChangeNumber,
OldValue = SubID.ToString(),
Action = "removed_from_sub"
}
);
}
else
{
await DbConnection.ExecuteAsync(DepotProcessor.HistoryQuery,
new DepotHistory
{
DepotID = appid,
ManifestID = 0,
ChangeID = ChangeNumber,
OldValue = SubID,
Action = "removed_from_sub"
}
);
}
}
if (appsRemoved)
{
LicenseList.RefreshApps();
}
if (!packageOwned && SubID != 17906 && Settings.IsMillhaven)
{
Steam.Instance.FreeLicense.RequestFromPackage(SubID, ProductInfo.KeyValues);
}
}
// Re-queue apps in this package so we can update depots and whatnot
if (appAddedToThisPackage && !Settings.IsFullRun && !string.IsNullOrEmpty(PackageName))
{
var appsToRequest = ProductInfo.KeyValues["appids"].Children.Select(x => (uint)x.AsInteger()).ToList();
JobManager.AddJob(
() => Steam.Instance.Apps.PICSGetAccessTokens(appsToRequest, Enumerable.Empty <uint>()),
new PICSTokens.RequestedTokens
{
Apps = appsToRequest
});
}
if (ProductInfo.MissingToken && PICSTokens.HasPackageToken(SubID))
{
Log.WriteError(nameof(PICSTokens), $"Overridden token for subid {SubID} is invalid?");
IRC.Instance.SendOps($"[Tokens] Looks like the overridden token for subid {SubID} ({newPackageName}) is invalid");
}
}
private async Task DownloadDepots(uint appID, List <ManifestJob> depots)
{
Log.WriteDebug("Depot Downloader", "Will process {0} depots ({1} depot locks left)", depots.Count(), DepotLocks.Count);
var processTasks = new List <Task <EResult> >();
bool anyFilesDownloaded = false;
foreach (var depot in depots)
{
var instance = depot.Anonymous ? Steam.Anonymous.Apps : Steam.Instance.Apps;
depot.DepotKey = await GetDepotDecryptionKey(instance, depot.DepotID, appID);
if (depot.DepotKey == null)
{
RemoveLock(depot.DepotID);
continue;
}
var cdnToken = await GetCDNAuthToken(instance, appID, depot.DepotID);
if (cdnToken == null)
{
RemoveLock(depot.DepotID);
Log.WriteDebug("Depot Downloader", "Got a depot key for depot {0} but no cdn auth token", depot.DepotID);
continue;
}
depot.CDNToken = cdnToken.Token;
depot.Server = cdnToken.Server;
DepotManifest depotManifest = null;
string lastError = string.Empty;
for (var i = 0; i <= 5; i++)
{
try
{
depotManifest = CDNClient.DownloadManifest(depot.DepotID, depot.ManifestID, depot.Server, depot.CDNToken, depot.DepotKey);
break;
}
catch (Exception e)
{
Log.WriteWarn("Depot Downloader", "[{0}] Manifest download failed: {1} - {2}", depot.DepotID, e.GetType(), e.Message);
lastError = e.Message;
}
// TODO: get new auth key if auth fails
if (depotManifest == null)
{
await Task.Delay(Utils.ExponentionalBackoff(i));
}
}
if (depotManifest == null)
{
RemoveLock(depot.DepotID);
Log.WriteError("Depot Processor", "Failed to download depot manifest for app {0} depot {1} ({2}: {3})", appID, depot.DepotID, depot.Server, lastError);
if (FileDownloader.IsImportantDepot(depot.DepotID))
{
IRC.Instance.SendOps("{0}[{1}]{2} Failed to download app {3} depot {4} manifest ({5}: {6})",
Colors.OLIVE, Steam.GetAppName(appID), Colors.NORMAL, appID, depot.DepotID, depot.Server, lastError);
}
continue;
}
var task = TaskManager.Run(() =>
{
using (var db = Database.GetConnection())
{
using (var transaction = db.BeginTransaction())
{
var result = ProcessDepotAfterDownload(db, depot, depotManifest);
transaction.Commit();
return(result);
}
}
});
processTasks.Add(task);
if (FileDownloader.IsImportantDepot(depot.DepotID))
{
task = TaskManager.Run(() =>
{
var result = FileDownloader.DownloadFilesFromDepot(appID, depot, depotManifest);
if (result == EResult.OK)
{
anyFilesDownloaded = true;
}
return(result);
}, TaskCreationOptions.LongRunning);
TaskManager.RegisterErrorHandler(task);
processTasks.Add(task);
}
}
if (SaveLocalConfig)
{
SaveLocalConfig = false;
LocalConfig.Save();
}
await Task.WhenAll(processTasks);
Log.WriteDebug("Depot Downloader", "{0} depot downloads finished", depots.Count());
// TODO: use ContinueWith on tasks
if (!anyFilesDownloaded)
{
foreach (var depot in depots)
{
RemoveLock(depot.DepotID);
}
return;
}
if (!File.Exists(UpdateScript))
{
return;
}
bool lockTaken = false;
try
{
UpdateScriptLock.Enter(ref lockTaken);
foreach (var depot in depots)
{
if (depot.Result == EResult.OK)
{
RunUpdateScript(string.Format("{0} no-git", depot.DepotID));
}
RemoveLock(depot.DepotID);
}
// Only commit changes if all depots downloaded
if (processTasks.All(x => x.Result == EResult.OK || x.Result == EResult.Ignored))
{
if (!RunUpdateScript(appID))
{
RunUpdateScript("0");
}
}
else
{
Log.WriteDebug("Depot Processor", "Reprocessing the app {0} because some files failed to download", appID);
IRC.Instance.SendOps("{0}[{1}]{2} Reprocessing the app due to download failures",
Colors.OLIVE, Steam.GetAppName(appID), Colors.NORMAL
);
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null));
}
}
finally
{
if (lockTaken)
{
UpdateScriptLock.Exit();
}
}
}
private void OnLoggedOn(SteamUser.LoggedOnCallback callback)
{
GameCoordinator.UpdateStatus(0, callback.Result.ToString());
if (callback.Result == EResult.AccountLogonDenied)
{
Console.Write("STEAM GUARD! Please enter the auth code sent to the email at {0}: ", callback.EmailDomain);
AuthCode = Console.ReadLine();
if (AuthCode != null)
{
AuthCode = AuthCode.Trim();
}
return;
}
if (callback.Result != EResult.OK)
{
Log.WriteInfo("Steam", "Failed to login: {0}", callback.Result);
IRC.Instance.SendEmoteAnnounce("failed to log in: {0}", callback.Result);
return;
}
var cellId = callback.CellID;
if (LocalConfig.CellID != cellId)
{
Log.WriteDebug("Local Config", "CellID differs, {0} != {1}, forcing server refetch", LocalConfig.CellID, cellId);
LocalConfig.CellID = cellId;
// TODO: is this really needed?
LocalConfig.LoadServers();
LocalConfig.Save();
}
LastSuccessfulLogin = DateTime.Now;
Log.WriteInfo("Steam", "Logged in, current Valve time is {0}", callback.ServerTime.ToString("R"));
IRC.Instance.SendEmoteAnnounce("logged in. Valve time: {0}", callback.ServerTime.ToString("R"));
if (Settings.IsFullRun)
{
if (Settings.Current.FullRun == FullRunState.ImportantOnly)
{
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(Application.ImportantApps.Keys, Enumerable.Empty <uint>()));
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetProductInfo(Enumerable.Empty <SteamApps.PICSRequest>(), Application.ImportantSubs.Keys.Select(package => Utils.NewPICSRequest(package))));
}
else if (Steam.Instance.PICSChanges.PreviousChangeNumber == 1)
{
Steam.Instance.PICSChanges.PerformSync();
//JobManager.AddJob(() => Steam.Instance.Apps.PICSGetChangesSince(1, true, true));
}
}
else
{
JobManager.RestartJobsIfAny();
Application.ChangelistTimer.Start();
}
}
private async void OnPICSChanges(SteamApps.PICSChangesCallback callback)
{
if (PreviousChangeNumber == callback.CurrentChangeNumber)
{
return;
}
Log.WriteInfo("PICSChanges", $"Changelist {PreviousChangeNumber} -> {callback.CurrentChangeNumber} ({callback.AppChanges.Count} apps, {callback.PackageChanges.Count} packages)");
LocalConfig.Current.ChangeNumber = PreviousChangeNumber = callback.CurrentChangeNumber;
await HandleChangeNumbers(callback);
if (callback.AppChanges.Count == 0 && callback.PackageChanges.Count == 0)
{
IRC.Instance.SendAnnounce("{0}»{1} Changelist {2}{3}{4} (empty)", Colors.RED, Colors.NORMAL, Colors.BLUE, PreviousChangeNumber, Colors.DARKGRAY);
return;
}
const int appsPerJob = 50;
if (callback.AppChanges.Count > appsPerJob)
{
foreach (var list in callback.AppChanges.Keys.Split(appsPerJob))
{
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(list, Enumerable.Empty <uint>()));
}
}
else if (callback.AppChanges.Count > 0)
{
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(callback.AppChanges.Keys, Enumerable.Empty <uint>()));
}
if (callback.PackageChanges.Count > appsPerJob)
{
foreach (var list in callback.PackageChanges.Keys.Select(Utils.NewPICSRequest).Split(appsPerJob))
{
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetProductInfo(Enumerable.Empty <SteamApps.PICSRequest>(), list));
}
}
else if (callback.PackageChanges.Count > 0)
{
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetProductInfo(Enumerable.Empty <SteamApps.PICSRequest>(), callback.PackageChanges.Keys.Select(Utils.NewPICSRequest)));
}
// We don't care about waiting for separate database queries to finish, so don't await them
#pragma warning disable 4014
if (callback.AppChanges.Count > 0)
{
TaskManager.RunAsync(async() => await HandleApps(callback));
}
if (callback.PackageChanges.Count > 0)
{
TaskManager.RunAsync(async() => await HandlePackages(callback));
TaskManager.RunAsync(async() => await HandlePackagesChangelists(callback));
}
TaskManager.RunAsync(async() => await SendChangelistsToIRC(callback));
#pragma warning restore 4014
PrintImportants(callback);
}
private static void OnLicenseListCallback(SteamApps.LicenseListCallback licenseList)
{
if (licenseList.Result != EResult.OK)
{
Log.WriteError(nameof(LicenseList), $"Failed: {licenseList.Result}");
return;
}
Log.WriteInfo(nameof(LicenseList), $"Received {licenseList.LicenseList.Count} licenses from Steam");
if (licenseList.LicenseList.Count == 0)
{
return;
}
var ownedSubs = new Dictionary <uint, byte>();
var newSubs = new List <uint>();
var hasAnyLicense = OwnedSubs.Count > 0;
foreach (var license in licenseList.LicenseList)
{
if (license.AccessToken > 0)
{
PICSTokens.NewPackageRequest(license.PackageID, license.AccessToken);
}
// Expired licenses block access to depots, so we have no use in these
if ((license.LicenseFlags & ELicenseFlags.Expired) != 0)
{
continue;
}
// For some obscure reason license list can contain duplicates
if (ownedSubs.ContainsKey(license.PackageID))
{
#if DEBUG
Log.WriteWarn(nameof(LicenseList), $"Already contains {license.PackageID} ({license.PaymentMethod})");
#endif
continue;
}
if (hasAnyLicense && !OwnedSubs.ContainsKey(license.PackageID))
{
Log.WriteInfo(nameof(LicenseList), $"New license granted: {license.PackageID} ({license.PaymentMethod}, {license.LicenseFlags})");
newSubs.Add(license.PackageID);
}
if (Steam.Instance.FreeLicense.FreeLicensesToRequest.ContainsKey(license.PackageID))
{
Log.WriteInfo(nameof(FreeLicense), $"Package {license.PackageID} was granted, removed from free request");
Steam.Instance.FreeLicense.FreeLicensesToRequest.Remove(license.PackageID);
}
ownedSubs.Add(license.PackageID, (byte)license.PaymentMethod);
}
OwnedSubs = ownedSubs;
RefreshApps();
if (newSubs.Count <= 0)
{
return;
}
using var db = Database.Get();
var apps = db.Query <uint>("SELECT `AppID` FROM `SubsApps` WHERE `Type` = \"app\" AND `SubID` IN @Ids", new { Ids = newSubs });
JobManager.AddJob(
() => Steam.Instance.Apps.PICSGetAccessTokens(apps, newSubs),
new PICSTokens.RequestedTokens
{
Apps = apps.ToList(),
Packages = newSubs,
});
}
public override async Task OnCommand(CommandArguments command)
{
await Task.Yield();
if (command.Message.Length == 0)
{
command.Reply("Usage:{0} steamid <steamid> [individual/group/gamegroup]", Colors.OLIVE);
return;
}
var args = command.Message.Split(' ');
var urlType = EVanityURLType.Default;
if (args.Length > 1)
{
switch (args[1])
{
case "individual":
urlType = EVanityURLType.Individual;
break;
case "group":
urlType = EVanityURLType.Group;
break;
case "game":
case "gamegroup":
urlType = EVanityURLType.OfficialGameGroup;
break;
default:
command.Reply("Invalid vanity url type.");
return;
}
}
SteamID steamID;
if (urlType != EVanityURLType.Default || !TrySetSteamID(args[0], out steamID))
{
if (urlType == EVanityURLType.Default)
{
urlType = EVanityURLType.Individual;
}
var eResult = ResolveVanityURL(args[0], urlType, out steamID);
if (eResult != EResult.OK)
{
command.Reply("Failed to resolve vanity url: {0}{1}", Colors.RED, eResult.ToString());
return;
}
}
command.Reply(ExpandSteamID(steamID));
if (!steamID.IsValid || (!steamID.IsIndividualAccount && !steamID.IsClanAccount))
{
return;
}
JobManager.TryRemoveJob(new JobID(steamID)); // Remove previous "job" if any
JobManager.AddJob(
() => FakePersonaStateJob(steamID),
command
);
}
public override async Task OnCommand(CommandArguments command)
{
if (string.IsNullOrWhiteSpace(command.Message))
{
command.Reply($"Usage:{Colors.OLIVE} app <appid or partial game name>");
return;
}
string name;
if (!uint.TryParse(command.Message, out var appID))
{
appID = await TrySearchAppId(command);
if (appID == 0)
{
return;
}
}
var tokenTask = Steam.Instance.Apps.PICSGetAccessTokens(appID, null);
tokenTask.Timeout = TimeSpan.FromSeconds(10);
var tokenCallback = await tokenTask;
SteamApps.PICSRequest request;
if (tokenCallback.AppTokens.ContainsKey(appID))
{
request = PICSTokens.NewAppRequest(appID, tokenCallback.AppTokens[appID]);
}
else
{
request = PICSTokens.NewAppRequest(appID);
}
var infoTask = Steam.Instance.Apps.PICSGetProductInfo(new List <SteamApps.PICSRequest> {
request
}, Enumerable.Empty <SteamApps.PICSRequest>());
infoTask.Timeout = TimeSpan.FromSeconds(10);
var job = await infoTask;
var callback = job.Results?.FirstOrDefault(x => x.Apps.ContainsKey(appID));
if (callback == null)
{
command.Reply($"Unknown AppID: {Colors.BLUE}{appID}{(LicenseList.OwnedApps.ContainsKey(appID) ? SteamDB.StringCheckmark : string.Empty)}");
return;
}
var info = callback.Apps[appID];
if (info.KeyValues["common"]["name"].Value != null)
{
name = Utils.LimitStringLength(Utils.RemoveControlCharacters(info.KeyValues["common"]["name"].AsString()));
}
else
{
name = Steam.GetAppName(info.ID);
}
info.KeyValues.SaveToFile(Path.Combine(Application.Path, "app", $"{info.ID}.vdf"), false);
command.Reply($"{Colors.BLUE}{name}{Colors.NORMAL} -{Colors.DARKBLUE} {SteamDB.GetAppUrl(info.ID)}{Colors.NORMAL} - Dump:{Colors.DARKBLUE} {SteamDB.GetRawAppUrl(info.ID)}{Colors.NORMAL}{(info.MissingToken ? SteamDB.StringNeedToken : string.Empty)}{(LicenseList.OwnedApps.ContainsKey(info.ID) ? SteamDB.StringCheckmark : string.Empty)}");
if (command.IsUserAdmin && !LicenseList.OwnedApps.ContainsKey(info.ID))
{
JobManager.AddJob(() => Steam.Instance.Apps.RequestFreeLicense(info.ID));
}
}
public void Process(SteamApps.PICSProductInfoCallback.PICSProductInfo productInfo)
{
ChangeNumber = productInfo.ChangeNumber;
#if !DEBUG
if (Settings.IsFullRun)
#endif
{
Log.WriteDebug("Sub Processor", "SubID: {0}", SubID);
}
var appAddedToThisPackage = false;
var packageOwned = LicenseList.OwnedSubs.ContainsKey(SubID);
var kv = productInfo.KeyValues.Children.FirstOrDefault();
var newPackageName = kv["name"].AsString();
var apps = DbConnection.Query <PackageApp>("SELECT `AppID`, `Type` FROM `SubsApps` WHERE `SubID` = @SubID", new { SubID }).ToDictionary(x => x.AppID, x => x.Type);
// TODO: Ideally this should be SteamDB Unknown Package and proper checks like app processor does
if (newPackageName == null)
{
newPackageName = string.Concat("Steam Sub ", SubID);
}
if (newPackageName != null)
{
if (string.IsNullOrEmpty(PackageName))
{
DbConnection.Execute("INSERT INTO `Subs` (`SubID`, `Name`, `LastKnownName`) VALUES (@SubID, @Name, @Name) ON DUPLICATE KEY UPDATE `Name` = @Name", new { SubID, Name = newPackageName });
MakeHistory("created_sub");
MakeHistory("created_info", SteamDB.DATABASE_NAME_TYPE, string.Empty, newPackageName);
}
else if (!PackageName.Equals(newPackageName))
{
if (newPackageName.StartsWith("Steam Sub ", StringComparison.Ordinal))
{
DbConnection.Execute("UPDATE `Subs` SET `Name` = @Name WHERE `SubID` = @SubID", new { SubID, Name = newPackageName });
}
else
{
DbConnection.Execute("UPDATE `Subs` SET `Name` = @Name, `LastKnownName` = @Name WHERE `SubID` = @SubID", new { SubID, Name = newPackageName });
}
MakeHistory("modified_info", SteamDB.DATABASE_NAME_TYPE, PackageName, newPackageName);
}
}
foreach (var section in kv.Children)
{
string sectionName = section.Name.ToLower();
if (string.IsNullOrEmpty(sectionName) || sectionName.Equals("packageid") || sectionName.Equals("name"))
{
// Ignore common keys
continue;
}
if (sectionName.Equals("appids") || sectionName.Equals("depotids"))
{
// Remove "ids", so we get "app" from appids and "depot" from depotids
string type = sectionName.Replace("ids", string.Empty);
var isAppSection = type.Equals("app");
var typeID = (uint)(isAppSection ? 0 : 1); // 0 = app, 1 = depot; can't store as string because it's in the `key` field
foreach (var childrenApp in section.Children)
{
uint appID = uint.Parse(childrenApp.Value);
// Is this appid already in this package?
if (apps.ContainsKey(appID))
{
// Is this appid's type the same?
if (apps[appID] != type)
{
DbConnection.Execute("UPDATE `SubsApps` SET `Type` = @Type WHERE `SubID` = @SubID AND `AppID` = @AppID", new { SubID, AppID = appID, Type = type });
MakeHistory("added_to_sub", typeID, apps[appID].Equals("app") ? "0" : "1", childrenApp.Value);
appAddedToThisPackage = true;
// TODO: Log relevant add/remove history for depot/app?
}
apps.Remove(appID);
}
else
{
DbConnection.Execute("INSERT INTO `SubsApps` (`SubID`, `AppID`, `Type`) VALUES(@SubID, @AppID, @Type) ON DUPLICATE KEY UPDATE `Type` = @Type", new { SubID, AppID = appID, Type = type });
MakeHistory("added_to_sub", typeID, string.Empty, childrenApp.Value);
if (isAppSection)
{
DbConnection.Execute(AppProcessor.GetHistoryQuery(),
new PICSHistory
{
ID = appID,
ChangeID = ChangeNumber,
NewValue = SubID.ToString(),
Action = "added_to_sub"
}
);
}
else
{
DbConnection.Execute(DepotProcessor.GetHistoryQuery(),
new DepotHistory
{
DepotID = appID,
ChangeID = ChangeNumber,
NewValue = SubID,
Action = "added_to_sub"
}
);
}
appAddedToThisPackage = true;
if (packageOwned && !LicenseList.OwnedApps.ContainsKey(appID))
{
LicenseList.OwnedApps.Add(appID, (byte)1);
}
}
}
}
else if (sectionName.Equals("extended"))
{
string keyName;
foreach (var children in section.Children)
{
keyName = string.Format("{0}_{1}", sectionName, children.Name);
if (children.Children.Count > 0)
{
ProcessKey(keyName, children.Name, Utils.JsonifyKeyValue(children), true);
}
else
{
ProcessKey(keyName, children.Name, children.Value);
}
}
}
else if (section.Children.Any())
{
sectionName = string.Format("root_{0}", sectionName);
ProcessKey(sectionName, sectionName, Utils.JsonifyKeyValue(section), true);
}
else if (!string.IsNullOrEmpty(section.Value))
{
string keyName = string.Format("root_{0}", sectionName);
ProcessKey(keyName, sectionName, section.Value);
}
}
foreach (var data in CurrentData.Values)
{
if (!data.Processed && !data.KeyName.StartsWith("website", StringComparison.Ordinal))
{
DbConnection.Execute("DELETE FROM `SubsInfo` WHERE `SubID` = @SubID AND `Key` = @Key", new { SubID, data.Key });
MakeHistory("removed_key", data.Key, data.Value);
}
}
var appsRemoved = apps.Any();
foreach (var app in apps)
{
DbConnection.Execute("DELETE FROM `SubsApps` WHERE `SubID` = @SubID AND `AppID` = @AppID AND `Type` = @Type", new { SubID, AppID = app.Key, Type = app.Value });
var isAppSection = app.Value.Equals("app");
var typeID = (uint)(isAppSection ? 0 : 1); // 0 = app, 1 = depot; can't store as string because it's in the `key` field
MakeHistory("removed_from_sub", typeID, app.Key.ToString());
if (isAppSection)
{
DbConnection.Execute(AppProcessor.GetHistoryQuery(),
new PICSHistory
{
ID = app.Key,
ChangeID = ChangeNumber,
OldValue = SubID.ToString(),
Action = "removed_from_sub"
}
);
}
else
{
DbConnection.Execute(DepotProcessor.GetHistoryQuery(),
new DepotHistory
{
DepotID = app.Key,
ChangeID = ChangeNumber,
OldValue = SubID,
Action = "removed_from_sub"
}
);
}
}
if (appsRemoved)
{
LicenseList.RefreshApps();
}
if (kv["billingtype"].AsInteger() == 12 && !packageOwned) // 12 == free on demand
{
Log.WriteDebug("Sub Processor", "Requesting apps in SubID {0} as a free license", SubID);
JobManager.AddJob(() => SteamDB.RequestFreeLicense(kv["appids"].Children.Select(appid => (uint)appid.AsInteger()).ToList()));
}
// Re-queue apps in this package so we can update depots and whatnot
if (appAddedToThisPackage && !Settings.IsFullRun && !string.IsNullOrEmpty(PackageName))
{
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(kv["appids"].Children.Select(x => (uint)x.AsInteger()), Enumerable.Empty <uint>()));
}
}
public override void OnCommand(CommandArguments command)
{
if (command.Message.Length == 0)
{
CommandHandler.ReplyToCommand(command, "Usage:{0} steamid <steamid> [individual/group/gamegroup]", Colors.OLIVE);
return;
}
var args = command.Message.Split(' ');
var urlType = EVanityURLType.Default;
if (args.Length > 1)
{
switch (args[1])
{
case "individual":
urlType = EVanityURLType.Individual;
break;
case "group":
urlType = EVanityURLType.Group;
break;
case "game":
case "gamegroup":
urlType = EVanityURLType.OfficialGameGroup;
break;
default:
CommandHandler.ReplyToCommand(command, "Invalid vanity url type.");
return;
}
}
SteamID steamID;
if (urlType != EVanityURLType.Default || !TrySetSteamID(args[0], out steamID))
{
if (urlType == EVanityURLType.Default)
{
urlType = EVanityURLType.Individual;
}
var eResult = ResolveVanityURL(args[0], urlType, out steamID);
if (eResult != EResult.OK)
{
CommandHandler.ReplyToCommand(command, "Failed to resolve vanity url: {0}{1}", Colors.RED, eResult.ToString());
return;
}
}
CommandHandler.ReplyToCommand(command, ExpandSteamID(steamID));
if (!steamID.IsValid || (!steamID.IsIndividualAccount && !steamID.IsClanAccount))
{
return;
}
JobAction job;
if (JobManager.TryRemoveJob(new JobID(steamID), out job) && job.IsCommand)
{
CommandHandler.ReplyToCommand(job.CommandRequest.Command, true, "Your !steamid request was lost in space.");
}
JobManager.AddJob(
() => FakePersonaStateJob(steamID),
new JobManager.IRCRequest
{
Command = command
}
);
}
private async void OnPICSChanges(SteamApps.PICSChangesCallback callback)
{
if (PreviousChangeNumber == callback.CurrentChangeNumber)
{
return;
}
Log.WriteInfo("PICSChanges", $"Changelist {PreviousChangeNumber} -> {callback.CurrentChangeNumber} ({callback.AppChanges.Count} apps, {callback.PackageChanges.Count} packages)");
LocalConfig.Current.ChangeNumber = PreviousChangeNumber = callback.CurrentChangeNumber;
await HandleChangeNumbers(callback);
if (callback.AppChanges.Count == 0 && callback.PackageChanges.Count == 0)
{
IRC.Instance.SendAnnounce("{0}»{1} Changelist {2}{3}{4} (empty)", Colors.RED, Colors.NORMAL, Colors.BLUE, PreviousChangeNumber, Colors.DARKGRAY);
return;
}
const int appsPerJob = 50;
if (callback.AppChanges.Count > appsPerJob)
{
foreach (var list in callback.AppChanges.Keys.Split(appsPerJob))
{
JobManager.AddJob(
() => Steam.Instance.Apps.PICSGetAccessTokens(list, Enumerable.Empty <uint>()),
new PICSTokens.RequestedTokens
{
Apps = list.ToList()
});
}
}
else if (callback.AppChanges.Count > 0)
{
JobManager.AddJob(
() => Steam.Instance.Apps.PICSGetAccessTokens(callback.AppChanges.Keys, Enumerable.Empty <uint>()),
new PICSTokens.RequestedTokens
{
Apps = callback.AppChanges.Keys.ToList()
});
}
if (callback.PackageChanges.Count > appsPerJob)
{
foreach (var list in callback.PackageChanges.Keys.Split(appsPerJob))
{
JobManager.AddJob(
() => Steam.Instance.Apps.PICSGetAccessTokens(Enumerable.Empty <uint>(), list),
new PICSTokens.RequestedTokens
{
Packages = list.ToList()
});
}
}
else if (callback.PackageChanges.Count > 0)
{
JobManager.AddJob(
() => Steam.Instance.Apps.PICSGetAccessTokens(Enumerable.Empty <uint>(), callback.PackageChanges.Keys),
new PICSTokens.RequestedTokens
{
Packages = callback.PackageChanges.Keys.ToList()
});
}
if (callback.AppChanges.Count > 0)
{
_ = TaskManager.RunAsync(async() => await HandleApps(callback));
}
if (callback.PackageChanges.Count > 0)
{
_ = TaskManager.RunAsync(async() => await HandlePackages(callback));
_ = TaskManager.RunAsync(async() => await HandlePackagesChangelists(callback));
}
_ = TaskManager.RunAsync(async() => await SendChangelistsToIRC(callback));
PrintImportants(callback);
}
private async Task DownloadDepots(uint appID, List <ManifestJob> depots)
{
Log.WriteDebug("Depot Downloader", "Will process {0} depots ({1} depot locks left)", depots.Count(), DepotLocks.Count);
var processTasks = new List <Task <EResult> >();
bool anyFilesDownloaded = false;
foreach (var depot in depots)
{
var instance = depot.Anonymous ? Steam.Anonymous.Apps : Steam.Instance.Apps;
depot.DepotKey = await GetDepotDecryptionKey(instance, depot.DepotID, appID);
if (depot.DepotKey == null)
{
RemoveLock(depot.DepotID);
continue;
}
var cdnToken = await GetCDNAuthToken(instance, appID, depot.DepotID);
if (cdnToken == null)
{
RemoveLock(depot.DepotID);
Log.WriteDebug("Depot Downloader", "Got a depot key for depot {0} but no cdn auth token", depot.DepotID);
continue;
}
depot.CDNToken = cdnToken.Token;
depot.Server = GetContentServer();
DepotManifest depotManifest = null;
string lastError = string.Empty;
for (var i = 0; i <= 5; i++)
{
try
{
depotManifest = await CDNClient.DownloadManifestAsync(depot.DepotID, depot.ManifestID, depot.Server, depot.CDNToken, depot.DepotKey);
break;
}
catch (Exception e)
{
lastError = e.Message;
Log.WriteError("Depot Processor", "Failed to download depot manifest for app {0} depot {1} ({2}: {3}) (#{4})", appID, depot.DepotID, depot.Server, lastError, i);
}
// TODO: get new auth key if auth fails
depot.Server = GetContentServer();
if (depotManifest == null)
{
await Task.Delay(Utils.ExponentionalBackoff(i));
}
}
if (depotManifest == null)
{
LocalConfig.CDNAuthTokens.TryRemove(depot.DepotID, out _);
RemoveLock(depot.DepotID);
if (FileDownloader.IsImportantDepot(depot.DepotID))
{
IRC.Instance.SendOps("{0}[{1}]{2} Failed to download manifest ({3})",
Colors.OLIVE, depot.DepotName, Colors.NORMAL, lastError);
}
if (!Settings.IsFullRun)
{
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null));
}
continue;
}
var task = ProcessDepotAfterDownload(depot, depotManifest);
processTasks.Add(task);
if (!FileDownloader.IsImportantDepot(depot.DepotID))
{
continue;
}
task = TaskManager.Run(async() =>
{
var result = EResult.Fail;
try
{
result = await FileDownloader.DownloadFilesFromDepot(depot, depotManifest);
if (result == EResult.OK)
{
anyFilesDownloaded = true;
}
}
catch (Exception e)
{
ErrorReporter.Notify($"Depot Processor {depot.DepotID}", e);
}
return(result);
}).Unwrap();
processTasks.Add(task);
}
if (SaveLocalConfig)
{
SaveLocalConfig = false;
LocalConfig.Save();
}
await Task.WhenAll(processTasks).ConfigureAwait(false);
Log.WriteDebug("Depot Downloader", "{0} depot downloads finished", depots.Count());
// TODO: use ContinueWith on tasks
if (!anyFilesDownloaded)
{
foreach (var depot in depots)
{
RemoveLock(depot.DepotID);
}
return;
}
if (!File.Exists(UpdateScript))
{
return;
}
lock (UpdateScriptLock)
{
foreach (var depot in depots)
{
if (depot.Result == EResult.OK)
{
RunUpdateScript(string.Format("{0} no-git", depot.DepotID));
}
else if (depot.Result != EResult.Ignored)
{
Log.WriteWarn("Depot Processor", "Dropping stored token for {0} due to download failures", depot.DepotID);
LocalConfig.CDNAuthTokens.TryRemove(depot.DepotID, out _);
using (var db = Database.Get())
{
// Mark this depot for redownload
db.Execute("UPDATE `Depots` SET `LastManifestID` = 0 WHERE `DepotID` = @DepotID", new { depot.DepotID });
}
}
RemoveLock(depot.DepotID);
}
// Only commit changes if all depots downloaded
if (processTasks.All(x => x.Result == EResult.OK || x.Result == EResult.Ignored))
{
if (!RunUpdateScript(appID, depots.First().BuildID))
{
RunUpdateScript("0");
}
}
else
{
Log.WriteDebug("Depot Processor", "Reprocessing the app {0} because some files failed to download", appID);
IRC.Instance.SendOps("{0}[{1}]{2} Reprocessing the app due to download failures",
Colors.OLIVE, Steam.GetAppName(appID), Colors.NORMAL
);
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(appID, null));
}
}
}
private static void OnPICSProductInfo(SteamApps.PICSProductInfoCallback callback)
{
JobManager.TryRemoveJob(callback.JobID);
var apps = callback.Apps.Concat(callback.UnknownApps.ToDictionary(x => x, x => (SteamApps.PICSProductInfoCallback.PICSProductInfo)null));
var packages = callback.Packages.Concat(callback.UnknownPackages.ToDictionary(x => x, x => (SteamApps.PICSProductInfoCallback.PICSProductInfo)null));
foreach (var workaround in apps)
{
var app = workaround;
Log.WriteInfo("PICSProductInfo", "{0}AppID: {1}", app.Value == null ? "Unknown " : "", app.Key);
Task mostRecentItem;
lock (ProcessedApps)
{
ProcessedApps.TryGetValue(app.Key, out mostRecentItem);
}
var workerItem = TaskManager.Run(async() =>
{
try
{
await ProcessorSemaphore.WaitAsync().ConfigureAwait(false);
if (mostRecentItem != null && !mostRecentItem.IsCompleted)
{
Log.WriteDebug("PICSProductInfo", "Waiting for app {0} to finish processing", app.Key);
await mostRecentItem.ConfigureAwait(false);
}
using (var processor = new AppProcessor(app.Key))
{
if (app.Value == null)
{
processor.ProcessUnknown();
}
else
{
processor.Process(app.Value);
}
}
}
catch (MySqlException e)
{
ErrorReporter.Notify($"App {app.Key}", e);
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(app.Key, null));
}
catch (Exception e)
{
ErrorReporter.Notify($"App {app.Key}", e);
}
finally
{
lock (ProcessedApps)
{
if (ProcessedApps.TryGetValue(app.Key, out mostRecentItem) && mostRecentItem.IsCompleted)
{
ProcessedApps.Remove(app.Key);
}
}
ProcessorSemaphore.Release();
}
});
if (Settings.IsFullRun)
{
continue;
}
lock (ProcessedApps)
{
ProcessedApps[app.Key] = workerItem;
}
}
foreach (var workaround in packages)
{
var package = workaround;
Log.WriteInfo("PICSProductInfo", "{0}SubID: {1}", package.Value == null ? "Unknown " : "", package.Key);
Task mostRecentItem;
lock (ProcessedSubs)
{
ProcessedSubs.TryGetValue(package.Key, out mostRecentItem);
}
var workerItem = TaskManager.Run(async() =>
{
try
{
await ProcessorSemaphore.WaitAsync().ConfigureAwait(false);
if (mostRecentItem != null && !mostRecentItem.IsCompleted)
{
Log.WriteDebug("PICSProductInfo", "Waiting for package {0} to finish processing", package.Key);
await mostRecentItem.ConfigureAwait(false);
}
using (var processor = new SubProcessor(package.Key))
{
if (package.Value == null)
{
processor.ProcessUnknown();
}
else
{
processor.Process(package.Value);
}
}
}
catch (MySqlException e)
{
ErrorReporter.Notify($"Package {package.Key}", e);
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetProductInfo(null, package.Key, false, false));
}
catch (Exception e)
{
ErrorReporter.Notify($"Package {package.Key}", e);
}
finally
{
lock (ProcessedSubs)
{
if (ProcessedSubs.TryGetValue(package.Key, out mostRecentItem) && mostRecentItem.IsCompleted)
{
ProcessedSubs.Remove(package.Key);
}
}
ProcessorSemaphore.Release();
}
});
if (Settings.IsFullRun)
{
continue;
}
lock (ProcessedSubs)
{
ProcessedSubs[package.Key] = workerItem;
}
}
}