public void ReloadImportant(CommandHandler.CommandArguments command = null)
{
using (MySqlDataReader Reader = DbWorker.ExecuteReader("SELECT `AppID` FROM `ImportantApps` WHERE `Announce` = 1"))
{
ImportantApps.Clear();
while (Reader.Read())
{
ImportantApps.Add(Reader.GetUInt32("AppID"));
}
}
using (MySqlDataReader Reader = DbWorker.ExecuteReader("SELECT `SubID` FROM `ImportantSubs`"))
{
ImportantSubs.Clear();
while (Reader.Read())
{
ImportantSubs.Add(Reader.GetUInt32("SubID"));
}
}
if (command == null)
{
Log.WriteInfo("IRC Proxy", "Loaded {0} important apps and {1} packages", ImportantApps.Count, ImportantSubs.Count);
}
else
{
CommandHandler.ReplyToCommand(command, "{0}{1}{2}: Reloaded {3} important apps and {4} packages", Colors.OLIVE, command.Nickname, Colors.NORMAL, ImportantApps.Count, ImportantSubs.Count);
}
}
private static void Cleanup()
{
Log.WriteInfo("Main", "Exiting...");
foreach (var idler in GCIdlers)
{
try
{
idler.IsRunning = false;
idler.Client.Disconnect();
}
catch { }
}
Steam.Instance.IsRunning = false;
try { Steam.Instance.Timer.Stop(); } catch { }
try { DepotProcessor.ThreadPool.Shutdown(true, 1000); } catch { }
try { Steam.Instance.SecondaryPool.Shutdown(true, 1000); } catch { }
try { Steam.Instance.ProcessorPool.Shutdown(true, 1000); } catch { }
try { Steam.Instance.Client.Disconnect(); } catch { }
if (Settings.Current.IRC.Enabled)
{
IRC.Instance.Kill();
}
DbWorker.ExecuteNonQuery("DELETE FROM `GC`");
}
public static void UpdateStatus(uint appID, string status)
{
DbWorker.ExecuteNonQuery("INSERT INTO `GC` (`AppID`, `Status`) VALUES(@AppID, @Status) ON DUPLICATE KEY UPDATE `Status` = @Status",
new MySqlParameter("@AppID", appID),
new MySqlParameter("@Status", status)
);
}
private void GetLastChangeNumber()
{
// If we're in a full run, request all changes from #1
if (Settings.IsFullRun)
{
PreviousChange = 1;
return;
}
using (MySqlDataReader Reader = DbWorker.ExecuteReader("SELECT `ChangeID` FROM `Changelists` ORDER BY `ChangeID` DESC LIMIT 1"))
{
if (Reader.Read())
{
PreviousChange = Reader.GetUInt32("ChangeID");
Log.WriteInfo("Steam", "Previous changelist was {0}", PreviousChange);
}
}
if (PreviousChange == 0)
{
Log.WriteWarn("Steam", "Looks like there are no changelists in the database.");
Log.WriteWarn("Steam", "If you want to fill up your database first, restart with \"FullRun\" setting set to 1.");
}
}
private void TryProcessUnknown()
{
string name;
using (MySqlDataReader MainReader = DbWorker.ExecuteReader("SELECT `Name` FROM `Subs` WHERE `SubID` = @SubID LIMIT 1", new MySqlParameter("SubID", SubID)))
{
if (!MainReader.Read())
{
return;
}
name = DbWorker.GetString("Name", MainReader);
}
using (MySqlDataReader Reader = DbWorker.ExecuteReader("SELECT `Name`, `Key`, `Value` FROM `SubsInfo` INNER JOIN `KeyNamesSubs` ON `SubsInfo`.`Key` = `KeyNamesSubs`.`ID` WHERE `SubID` = @SubID", new MySqlParameter("SubID", SubID)))
{
while (Reader.Read())
{
if (!DbWorker.GetString("Name", Reader).StartsWith("website", StringComparison.Ordinal))
{
MakeHistory("removed_key", Reader.GetUInt32("Key"), DbWorker.GetString("Value", Reader));
}
}
}
DbWorker.ExecuteNonQuery("DELETE FROM `Subs` WHERE `SubID` = @SubID", new MySqlParameter("@SubID", SubID));
DbWorker.ExecuteNonQuery("DELETE FROM `SubsInfo` WHERE `SubID` = @SubID", new MySqlParameter("@SubID", SubID));
DbWorker.ExecuteNonQuery("DELETE FROM `StoreSubs` WHERE `SubID` = @SubID", new MySqlParameter("@SubID", SubID));
// TODO
MakeHistory("deleted_sub", 0, name);
}
public static void OnMarketingMessage(SteamUser.MarketingMessageCallback callback)
{
foreach (var message in callback.Messages)
{
// TODO: Move this query outside this loop
using (MySqlDataReader Reader = DbWorker.ExecuteReader("SELECT `ID` FROM `MarketingMessages` WHERE `ID` = @ID", new MySqlParameter("ID", message.ID)))
{
if (Reader.Read())
{
continue;
}
}
if (message.Flags == EMarketingMessageFlags.None)
{
IRC.SendMain("New marketing message:{0} {1}", Colors.DARK_BLUE, message.URL);
}
else
{
IRC.SendMain("New marketing message:{0} {1} {2}({3})", Colors.DARK_BLUE, message.URL, Colors.DARK_GRAY, message.Flags.ToString().Replace("Platform", string.Empty));
}
DbWorker.ExecuteNonQuery("INSERT INTO `MarketingMessages` (`ID`, `Flags`) VALUES (@ID, @Flags)",
new MySqlParameter("@ID", message.ID),
new MySqlParameter("@Flags", message.Flags)
);
}
}
private void InsertInfo(uint id, string value)
{
DbWorker.ExecuteNonQuery("INSERT INTO `SubsInfo` VALUES (@SubID, @KeyNameID, @Value) ON DUPLICATE KEY UPDATE `Value` = @Value",
new MySqlParameter("@SubID", SubID),
new MySqlParameter("@KeyNameID", id),
new MySqlParameter("@Value", value)
);
}
private static void OnCommandApp(CommandArguments command)
{
if (command.Message.Length == 0)
{
ReplyToCommand(command, "Usage:{0} !app <appid or partial game name>", Colors.OLIVE);
return;
}
uint appID;
if (uint.TryParse(command.Message, out appID))
{
var apps = new List <uint>();
apps.Add(appID);
var jobID = Steam.Instance.Apps.PICSGetAccessTokens(apps, Enumerable.Empty <uint>());
SteamProxy.Instance.IRCRequests.Add(new SteamProxy.IRCRequest
{
JobID = jobID,
Target = appID,
Type = SteamProxy.IRCRequestType.TYPE_APP,
Command = command
});
}
else
{
string name = string.Format("%{0}%", command.Message);
using (MySqlDataReader Reader = DbWorker.ExecuteReader("SELECT `AppID` FROM `Apps` WHERE `Apps`.`StoreName` LIKE @Name OR `Apps`.`Name` LIKE @Name ORDER BY `LastUpdated` DESC LIMIT 1", new MySqlParameter("Name", name)))
{
if (Reader.Read())
{
appID = Reader.GetUInt32("AppID");
var apps = new List <uint>();
apps.Add(appID);
var jobID = Steam.Instance.Apps.PICSGetAccessTokens(apps, Enumerable.Empty <uint>());
SteamProxy.Instance.IRCRequests.Add(new SteamProxy.IRCRequest
{
JobID = jobID,
Target = appID,
Type = SteamProxy.IRCRequestType.TYPE_APP,
Command = command
});
}
else
{
ReplyToCommand(command, "{0}{1}{2}: Nothing was found matching your request", Colors.OLIVE, command.Nickname, Colors.NORMAL);
}
}
}
}
private void MakeHistory(string action, uint keyNameID = 0, string oldValue = "", string newValue = "")
{
DbWorker.ExecuteNonQuery("INSERT INTO `SubsHistory` (`ChangeID`, `SubID`, `Action`, `Key`, `OldValue`, `NewValue`) VALUES (@ChangeID, @SubID, @Action, @KeyNameID, @OldValue, @NewValue)",
new MySqlParameter("@SubID", SubID),
new MySqlParameter("@ChangeID", ChangeNumber),
new MySqlParameter("@Action", action),
new MySqlParameter("@KeyNameID", keyNameID),
new MySqlParameter("@OldValue", oldValue),
new MySqlParameter("@NewValue", newValue)
);
}
public static void MakeHistory(uint appID, uint changeNumber, string action, uint keyNameID = 0, string oldValue = "", string newValue = "")
{
DbWorker.ExecuteNonQuery("INSERT INTO `AppsHistory` (`ChangeID`, `AppID`, `Action`, `Key`, `OldValue`, `NewValue`) VALUES (@ChangeID, @AppID, @Action, @KeyNameID, @OldValue, @NewValue)",
new MySqlParameter("@AppID", appID),
new MySqlParameter("@ChangeID", changeNumber),
new MySqlParameter("@Action", action),
new MySqlParameter("@KeyNameID", keyNameID),
new MySqlParameter("@OldValue", oldValue),
new MySqlParameter("@NewValue", newValue)
);
}
private static void MakeHistory(ManifestJob request, string file, string action, ulong oldValue = 0, ulong newValue = 0)
{
DbWorker.ExecuteNonQuery("INSERT INTO `DepotsHistory` (`ChangeID`, `DepotID`, `File`, `Action`, `OldValue`, `NewValue`) VALUES (@ChangeID, @DepotID, @File, @Action, @OldValue, @NewValue)",
new MySqlParameter("@DepotID", request.DepotID),
new MySqlParameter("@ChangeID", request.ChangeNumber),
new MySqlParameter("@File", file),
new MySqlParameter("@Action", action),
new MySqlParameter("@OldValue", oldValue),
new MySqlParameter("@NewValue", newValue)
);
}
private static uint GetKeyNameID(string keyName)
{
using (MySqlDataReader Reader = DbWorker.ExecuteReader("SELECT `ID` FROM `KeyNamesSubs` WHERE `Name` = @KeyName LIMIT 1", new MySqlParameter("KeyName", keyName)))
{
if (Reader.Read())
{
return(Reader.GetUInt32("ID"));
}
}
return(0);
}
private static void OnCommandPlayers(CommandArguments command)
{
if (command.Message.Length == 0)
{
ReplyToCommand(command, "Usage:{0} !players <appid or partial game name>", Colors.OLIVE);
return;
}
uint appID;
if (uint.TryParse(command.Message, out appID))
{
var jobID = Steam.Instance.UserStats.GetNumberOfCurrentPlayers(appID);
SteamProxy.Instance.IRCRequests.Add(new SteamProxy.IRCRequest
{
JobID = jobID,
Target = appID,
Command = command
});
}
else
{
string name = string.Format("%{0}%", command.Message);
using (MySqlDataReader Reader = DbWorker.ExecuteReader("SELECT `AppID` FROM `Apps` LEFT JOIN `AppsTypes` ON `Apps`.`AppType` = `AppsTypes`.`AppType` WHERE `AppsTypes`.`Name` IN ('game', 'application') AND (`Apps`.`StoreName` LIKE @Name OR `Apps`.`Name` LIKE @Name) ORDER BY `LastUpdated` DESC LIMIT 1", new MySqlParameter("Name", name)))
{
if (Reader.Read())
{
appID = Reader.GetUInt32("AppID");
var jobID = Steam.Instance.UserStats.GetNumberOfCurrentPlayers(appID);
SteamProxy.Instance.IRCRequests.Add(new SteamProxy.IRCRequest
{
JobID = jobID,
Target = appID,
Command = command
});
}
else
{
ReplyToCommand(command, "{0}{1}{2}: Nothing was found matching your request", Colors.OLIVE, command.Nickname, Colors.NORMAL);
}
}
}
}
public static string JsonifyKeyValue(KeyValue keys)
{
string value = string.Empty;
using (var sw = new StringWriter(new StringBuilder()))
{
using (JsonWriter w = new JsonTextWriter(sw))
{
DbWorker.JsonifyKeyValue(w, keys.Children);
}
value = sw.ToString();
}
return(value);
}
private static void OnDepotKeyCallback(SteamApps.DepotKeyCallback callback, JobID jobID)
{
ManifestJob request;
lock (ManifestJobs)
{
request = ManifestJobs.Find(r => r.JobID == jobID);
}
if (request == null)
{
return;
}
if (callback.Result != EResult.OK)
{
lock (ManifestJobs)
{
ManifestJobs.Remove(request);
}
if (callback.Result != EResult.Blocked)
{
Log.WriteError("Depot Processor", "Failed to get depot key for depot {0} (parent {1}) - {2}", callback.DepotID, request.ParentAppID, callback.Result);
}
return;
}
Log.WriteInfo("Depot Processor", "DepotID: {0}", request.DepotID);
if (request.PreviousManifestID != request.ManifestID)
{
// Update manifestid here because actually downloading the manifest has chances of failing
DbWorker.ExecuteNonQuery("INSERT INTO `Depots` (`DepotID`, `Name`, `ManifestID`) VALUES (@DepotID, @Name, @ManifestID) ON DUPLICATE KEY UPDATE `LastUpdated` = CURRENT_TIMESTAMP(), `Name` = @Name, `ManifestID` = @ManifestID",
new MySqlParameter("@DepotID", request.DepotID),
new MySqlParameter("@ManifestID", request.ManifestID),
new MySqlParameter("@Name", request.DepotName)
);
MakeHistory(request, string.Empty, "manifest_change", request.PreviousManifestID, request.ManifestID);
}
request.DepotKey = callback.DepotKey;
ThreadPool.QueueWorkItem(TryDownloadManifest, request);
}
private void TryProcessUnknown()
{
string AppName;
using (MySqlDataReader MainReader = DbWorker.ExecuteReader("SELECT `Name` FROM `Apps` WHERE `AppID` = @AppID LIMIT 1", new MySqlParameter("AppID", AppID)))
{
if (!MainReader.Read())
{
return;
}
AppName = DbWorker.GetString("Name", MainReader);
}
bool historyChanged = false;
using (MySqlDataReader Reader = DbWorker.ExecuteReader("SELECT `Name`, `Key`, `Value` FROM `AppsInfo` INNER JOIN `KeyNames` ON `AppsInfo`.`Key` = `KeyNames`.`ID` WHERE `AppID` = @AppID", new MySqlParameter("AppID", AppID)))
{
while (Reader.Read())
{
if (!DbWorker.GetString("Name", Reader).StartsWith("website", StringComparison.Ordinal))
{
MakeHistory("removed_key", Reader.GetUInt32("Key"), DbWorker.GetString("Value", Reader));
historyChanged = true;
}
}
}
DbWorker.ExecuteNonQuery("DELETE FROM `Apps` WHERE `AppID` = @AppID", new MySqlParameter("@AppID", AppID));
DbWorker.ExecuteNonQuery("DELETE FROM `AppsInfo` WHERE `AppID` = @AppID", new MySqlParameter("@AppID", AppID));
DbWorker.ExecuteNonQuery("DELETE FROM `Store` WHERE `AppID` = @AppID", new MySqlParameter("@AppID", AppID));
if (!AppName.StartsWith(SteamDB.UNKNOWN_APP, StringComparison.Ordinal))
{
MakeHistory("deleted_app", 0, AppName);
historyChanged = true;
}
// TODO: This is a dirty hack so we somehow track these app changes
if (!historyChanged && !Settings.IsFullRun)
{
MakeHistory("removed_key", GetKeyNameID("root_change_number"), "0", "0");
}
}
public static string GetAppName(uint appID, bool returnEmptyOnFailure = false)
{
using (MySqlDataReader reader = DbWorker.ExecuteReader("SELECT `Name`, `StoreName`, `LastKnownName` FROM `Apps` WHERE `AppID` = @AppID", new MySqlParameter("AppID", appID)))
{
if (reader.Read())
{
string name = DbWorker.GetString("Name", reader);
//string nameStore = DbWorker.GetString("StoreName", reader);
string nameLast = DbWorker.GetString("LastKnownName", reader);
if (!string.IsNullOrEmpty(nameLast) && !name.Equals(nameLast))
{
return(string.Format("{0} {1}({2}){3}", name, Colors.DARK_GRAY, nameLast, Colors.NORMAL));
}
return(name);
}
}
return(returnEmptyOnFailure ? string.Empty : string.Format("AppID {0}", appID));
}
public void OnNumberOfPlayers(SteamUserStats.NumberOfPlayersCallback callback, JobID jobID)
{
var request = IRCRequests.Find(r => r.JobID == jobID);
if (request == null)
{
return;
}
IRCRequests.Remove(request);
if (callback.Result != EResult.OK)
{
CommandHandler.ReplyToCommand(request.Command, "{0}{1}{2}: Unable to request player count: {3}", Colors.OLIVE, request.Command.Nickname, Colors.NORMAL, callback.Result);
}
else if (request.Target == 0)
{
CommandHandler.ReplyToCommand(request.Command, "{0}{1}{2}: {3}{4:N0}{5} people praising lord Gaben right now, influence:{6} {7}", Colors.OLIVE, request.Command.Nickname, Colors.NORMAL, Colors.OLIVE, callback.NumPlayers, Colors.NORMAL, Colors.DARK_BLUE, SteamDB.GetGraphURL(0));
}
else
{
string graphUrl = string.Empty;
string name = GetAppName(request.Target);
using (MySqlDataReader Reader = DbWorker.ExecuteReader("SELECT `AppID` FROM `ImportantApps` WHERE (`Graph` = 1 OR `MaxPlayers` > 0) AND `AppID` = @AppID", new MySqlParameter("AppID", request.Target)))
{
if (Reader.Read())
{
graphUrl = string.Format("{0} -{1} {2}", Colors.NORMAL, Colors.DARK_BLUE, SteamDB.GetGraphURL(request.Target));
}
}
CommandHandler.ReplyToCommand(request.Command, "{0}{1}{2}: People playing {3}{4}{5} right now: {6}{7:N0}{8} -{9} {10}{11}",
Colors.OLIVE, request.Command.Nickname, Colors.NORMAL,
Colors.OLIVE, name, Colors.NORMAL,
Colors.GREEN, callback.NumPlayers, Colors.NORMAL,
Colors.DARK_BLUE, SteamDB.GetAppURL(request.Target), graphUrl);
}
}
public static string GetPackageName(uint subID, bool returnEmptyOnFailure = false)
{
using (MySqlDataReader Reader = DbWorker.ExecuteReader("SELECT `Name`, `StoreName` FROM `Subs` WHERE `SubID` = @SubID", new MySqlParameter("SubID", subID)))
{
if (Reader.Read())
{
string name = DbWorker.GetString("Name", Reader);
if (name.StartsWith("Steam Sub", StringComparison.Ordinal))
{
string nameStore = DbWorker.GetString("StoreName", Reader);
if (!string.IsNullOrEmpty(nameStore))
{
name = string.Format("{0} {1}({2}){3}", name, Colors.DARK_GRAY, nameStore, Colors.NORMAL);
}
}
return(name);
}
}
return(returnEmptyOnFailure ? string.Empty : string.Format("SubID {0}", subID));
}
private void OnPICSChanges(SteamApps.PICSChangesCallback callback, JobID job)
{
if (PreviousChange == callback.CurrentChangeNumber)
{
return;
}
if (ProcessorPool.IsIdle)
{
Log.WriteDebug("Steam", "Cleaning processed apps and subs");
ProcessedApps.Clear();
ProcessedSubs.Clear();
}
var packageChangesCount = callback.PackageChanges.Count;
var appChangesCount = callback.AppChanges.Count;
Log.WriteInfo("Steam", "Changelist {0} -> {1} ({2} apps, {3} packages)", PreviousChange, callback.CurrentChangeNumber, appChangesCount, packageChangesCount);
PreviousChange = callback.CurrentChangeNumber;
DbWorker.ExecuteNonQuery("INSERT INTO `Changelists` (`ChangeID`) VALUES (@ChangeID) ON DUPLICATE KEY UPDATE `Date` = CURRENT_TIMESTAMP()", new MySqlParameter("@ChangeID", callback.CurrentChangeNumber));
if (appChangesCount == 0 && packageChangesCount == 0)
{
IRC.SendAnnounce("{0}»{1} Changelist {2}{3}{4} (empty)", Colors.RED, Colors.NORMAL, Colors.OLIVE, PreviousChange, Colors.DARK_GRAY);
return;
}
SecondaryPool.QueueWorkItem(SteamProxy.Instance.OnPICSChanges, callback);
// Packages have no tokens so we request info for them right away
if (packageChangesCount > 0)
{
Apps.PICSGetProductInfo(Enumerable.Empty <SteamApps.PICSRequest>(), callback.PackageChanges.Keys.Select(package => NewPICSRequest(package)));
}
if (appChangesCount > 0)
{
// Get all app tokens
Apps.PICSGetAccessTokens(callback.AppChanges.Keys, Enumerable.Empty <uint>());
SecondaryPool.QueueWorkItem(delegate
{
string changes = string.Empty;
foreach (var app in callback.AppChanges.Values)
{
if (callback.CurrentChangeNumber != app.ChangeNumber)
{
DbWorker.ExecuteNonQuery("INSERT INTO `Changelists` (`ChangeID`) VALUES (@ChangeID) ON DUPLICATE KEY UPDATE `Date` = `Date`", new MySqlParameter("@ChangeID", app.ChangeNumber));
}
DbWorker.ExecuteNonQuery("UPDATE `Apps` SET `LastUpdated` = CURRENT_TIMESTAMP() WHERE `AppID` = @AppID", new MySqlParameter("@AppID", app.ID));
changes += string.Format("({0}, {1}),", app.ChangeNumber, app.ID);
}
if (!changes.Equals(string.Empty))
{
changes = string.Format("INSERT INTO `ChangelistsApps` (`ChangeID`, `AppID`) VALUES {0} ON DUPLICATE KEY UPDATE `AppID` = `AppID`", changes.Remove(changes.Length - 1));
DbWorker.ExecuteNonQuery(changes);
}
});
}
if (packageChangesCount > 0)
{
SecondaryPool.QueueWorkItem(delegate
{
string changes = string.Empty;
foreach (var package in callback.PackageChanges.Values)
{
if (callback.CurrentChangeNumber != package.ChangeNumber)
{
DbWorker.ExecuteNonQuery("INSERT INTO `Changelists` (`ChangeID`) VALUES (@ChangeID) ON DUPLICATE KEY UPDATE `Date` = `Date`", new MySqlParameter("@ChangeID", package.ChangeNumber));
}
DbWorker.ExecuteNonQuery("UPDATE `Subs` SET `LastUpdated` = CURRENT_TIMESTAMP() WHERE `SubID` = @SubID", new MySqlParameter("@SubID", package.ID));
changes += string.Format("({0}, {1}),", package.ChangeNumber, package.ID);
}
if (!changes.Equals(string.Empty))
{
changes = string.Format("INSERT INTO `ChangelistsSubs` (`ChangeID`, `SubID`) VALUES {0} ON DUPLICATE KEY UPDATE `SubID` = `SubID`", changes.Remove(changes.Length - 1));
DbWorker.ExecuteNonQuery(changes);
}
});
}
}
public static void Process(uint appID, uint changeNumber, KeyValue depots)
{
foreach (KeyValue depot in depots.Children)
{
// Ignore these for now, parent app should be updated too anyway
if (depot["depotfromapp"].Value != null)
{
////Log.WriteDebug("Depot Processor", "Ignoring depot {0} with depotfromapp value {1} (parent {2})", depot.Name, depot["depotfromapp"].AsString(), AppID);
continue;
}
uint depotID;
if (!uint.TryParse(depot.Name, out depotID))
{
// Ignore keys that aren't integers, for example "branches"
continue;
}
lock (ManifestJobs)
{
if (ManifestJobs.Find(r => r.DepotID == depotID) != null)
{
// If we already have this depot in our job list, ignore it
continue;
}
}
ulong manifestID;
if (depot["manifests"]["public"].Value == null || !ulong.TryParse(depot["manifests"]["public"].Value, out manifestID))
{
#if false
Log.WriteDebug("Depot Processor", "Failed to public branch for depot {0} (parent {1}) - {2}", DepotID, AppID);
// If there is no public manifest for this depot, it still could have some sort of open beta
var branch = depot["manifests"].Children.SingleOrDefault(x => x.Name != "local");
if (branch == null || !ulong.TryParse(branch.Value, out ManifestID))
{
continue;
}
#endif
continue;
}
var request = new ManifestJob
{
ChangeNumber = changeNumber,
ParentAppID = appID,
DepotID = depotID,
ManifestID = manifestID,
DepotName = depot["name"].AsString()
};
// Check if manifestid in our database is equal
using (MySqlDataReader Reader = DbWorker.ExecuteReader("SELECT `ManifestID` FROM `Depots` WHERE `DepotID` = @DepotID AND `Files` != '' LIMIT 1", new MySqlParameter("DepotID", depotID)))
{
if (Reader.Read())
{
request.PreviousManifestID = Reader.GetUInt64("ManifestID");
if (request.PreviousManifestID == manifestID && Settings.Current.FullRun < 2)
{
continue;
}
}
}
lock (ManifestJobs)
{
ManifestJobs.Add(request);
}
request.Server = CDNServers[new Random().Next(CDNServers.Count)];
request.JobID = Steam.Instance.Apps.GetCDNAuthToken(depotID, request.Server);
}
}
private static void DownloadManifest(ManifestJob request)
{
DepotManifest depotManifest = null;
string lastError = string.Empty;
// CDN is very random, just keep trying
for (var i = 0; i <= 5; i++)
{
try
{
depotManifest = CDNClient.DownloadManifest(request.DepotID, request.ManifestID, request.Server, request.DepotKey, request.CDNToken);
break;
}
catch (Exception e)
{
lastError = e.Message;
}
}
if (depotManifest == null)
{
Log.WriteError("Depot Processor", "Failed to download depot manifest for depot {0} (jobs still in queue: {1}) ({2}: {3})", request.DepotID, ManifestJobs.Count - 1, request.Server, lastError);
if (SteamProxy.Instance.ImportantApps.Contains(request.ParentAppID))
{
IRC.SendMain("Important depot update: {0}{1}{2} -{3} failed to download depot manifest", Colors.OLIVE, request.DepotName, Colors.NORMAL, Colors.RED);
}
return;
}
if (SteamProxy.Instance.ImportantApps.Contains(request.ParentAppID))
{
IRC.SendMain("Important depot update: {0}{1}{2} -{3} {4}", Colors.OLIVE, request.DepotName, Colors.NORMAL, Colors.DARK_BLUE, SteamDB.GetDepotURL(request.DepotID, "history"));
}
var sortedFiles = depotManifest.Files.OrderBy(f => f.FileName, StringComparer.OrdinalIgnoreCase);
bool shouldHistorize = false;
var filesNew = new List <DepotFile>();
var filesOld = new Dictionary <string, DepotFile>();
foreach (var file in sortedFiles)
{
System.Text.Encoding.UTF8.GetString(file.FileHash);
var depotFile = new DepotFile
{
Name = file.FileName.Replace('\\', '/'),
Size = file.TotalSize,
Chunks = file.Chunks.Count,
Flags = (int)file.Flags
};
// TODO: Ideally we would check if filehash is not empty
if (!file.Flags.HasFlag(EDepotFileFlag.Directory))
{
depotFile.Hash = string.Concat(Array.ConvertAll(file.FileHash, x => x.ToString("X2")));
}
filesNew.Add(depotFile);
}
using (MySqlDataReader Reader = DbWorker.ExecuteReader("SELECT `Files` FROM `Depots` WHERE `DepotID` = @DepotID LIMIT 1", new MySqlParameter("DepotID", request.DepotID)))
{
if (Reader.Read())
{
string files = Reader.GetString("Files");
if (!string.IsNullOrEmpty(files))
{
shouldHistorize = true;
var _filesOld = JsonConvert.DeserializeObject <List <DepotFile> >(files);
filesOld = _filesOld.ToDictionary(x => x.Name);
}
}
}
DbWorker.ExecuteNonQuery("UPDATE `Depots` SET `Files` = @Files WHERE `DepotID` = @DepotID",
new MySqlParameter("@DepotID", request.DepotID),
new MySqlParameter("@Files", JsonConvert.SerializeObject(filesNew, new JsonSerializerSettings {
DefaultValueHandling = DefaultValueHandling.Ignore
}))
);
if (shouldHistorize)
{
var filesAdded = new List <string>();
foreach (var file in filesNew)
{
if (filesOld.ContainsKey(file.Name))
{
var oldFile = filesOld[file.Name];
if (oldFile.Size != file.Size)
{
MakeHistory(request, file.Name, "modified", oldFile.Size, file.Size);
}
else if (file.Hash != null && oldFile.Hash != null && !file.Hash.Equals(oldFile.Hash))
{
MakeHistory(request, file.Name, "modified", oldFile.Size, file.Size);
}
filesOld.Remove(file.Name);
}
else
{
// We want to historize modifications first, and only then deletions and additions
filesAdded.Add(file.Name);
}
}
foreach (var file in filesOld)
{
MakeHistory(request, file.Value.Name, "removed");
}
foreach (string file in filesAdded)
{
MakeHistory(request, file, "added");
}
}
lock (ManifestJobs)
{
Log.WriteDebug("Depot Processor", "DepotID: Processed {0} (jobs still in queue: {1})", request.DepotID, ManifestJobs.Count - 1);
}
}
private void TryProcess(SteamApps.PICSProductInfoCallback.PICSProductInfo productInfo)
{
using (MySqlDataReader Reader = DbWorker.ExecuteReader("SELECT `Name`, `Value` FROM `AppsInfo` INNER JOIN `KeyNames` ON `AppsInfo`.`Key` = `KeyNames`.`ID` WHERE `AppID` = @AppID", new MySqlParameter("AppID", AppID)))
{
while (Reader.Read())
{
CurrentData.Add(DbWorker.GetString("Name", Reader), DbWorker.GetString("Value", Reader));
}
}
string appName = string.Empty;
string appType = "0";
using (MySqlDataReader Reader = DbWorker.ExecuteReader("SELECT `Name`, `AppType` FROM `Apps` WHERE `AppID` = @AppID LIMIT 1", new MySqlParameter("AppID", AppID)))
{
if (Reader.Read())
{
appName = DbWorker.GetString("Name", Reader);
appType = DbWorker.GetString("AppType", Reader);
}
}
if (productInfo.KeyValues["common"]["name"].Value != null)
{
string newAppType = "0";
string currentType = productInfo.KeyValues["common"]["type"].AsString().ToLower();
using (MySqlDataReader Reader = DbWorker.ExecuteReader("SELECT `AppType` FROM `AppsTypes` WHERE `Name` = @Type LIMIT 1", new MySqlParameter("Type", currentType)))
{
if (Reader.Read())
{
newAppType = DbWorker.GetString("AppType", Reader);
}
else
{
// TODO: Create it?
Log.WriteError("App Processor", "AppID {0} - unknown app type: {1}", AppID, currentType);
// TODO: This is debuggy just so we are aware of new app types
IRC.SendAnnounce("Unknown app type \"{0}\" for appid {1}, cc Alram and xPaw", currentType, AppID);
}
}
if (string.IsNullOrEmpty(appName) || appName.StartsWith(SteamDB.UNKNOWN_APP, StringComparison.Ordinal))
{
DbWorker.ExecuteNonQuery("INSERT INTO `Apps` (`AppID`, `AppType`, `Name`) VALUES (@AppID, @Type, @AppName) ON DUPLICATE KEY UPDATE `Name` = @AppName, `AppType` = @Type",
new MySqlParameter("@AppID", AppID),
new MySqlParameter("@Type", newAppType),
new MySqlParameter("@AppName", productInfo.KeyValues["common"]["name"].Value)
);
MakeHistory("created_app");
MakeHistory("created_info", DATABASE_NAME_TYPE, string.Empty, productInfo.KeyValues["common"]["name"].Value);
// TODO: Testy testy
if (!Settings.IsFullRun &&
Settings.Current.ChatRooms.Count > 0 &&
!appName.StartsWith("SteamApp", StringComparison.Ordinal) &&
!appName.StartsWith("ValveTest", StringComparison.Ordinal))
{
Steam.Instance.Friends.SendChatRoomMessage(Settings.Current.ChatRooms[0], EChatEntryType.ChatMsg,
string.Format(
":retreat: New {0} was published: {1}\nSteamDB: {2}\nSteam: http://store.steampowered.com/app/{3}/",
currentType,
productInfo.KeyValues["common"]["name"].AsString(),
SteamDB.GetAppURL(AppID),
AppID
)
);
}
}
else if (!appName.Equals(productInfo.KeyValues["common"]["name"].Value))
{
string newAppName = productInfo.KeyValues["common"]["name"].AsString();
DbWorker.ExecuteNonQuery("UPDATE `Apps` SET `Name` = @AppName WHERE `AppID` = @AppID",
new MySqlParameter("@AppID", AppID),
new MySqlParameter("@AppName", newAppName)
);
MakeHistory("modified_info", DATABASE_NAME_TYPE, appName, newAppName);
// TODO: Testy testy
if (!Settings.IsFullRun &&
Settings.Current.ChatRooms.Count > 0 &&
!string.Equals(appName, newAppName, StringComparison.OrdinalIgnoreCase) &&
!newAppName.StartsWith("SteamApp", StringComparison.Ordinal) &&
!newAppName.StartsWith("ValveTest", StringComparison.Ordinal))
{
Steam.Instance.Friends.SendChatRoomMessage(Settings.Current.ChatRooms[0], EChatEntryType.ChatMsg,
string.Format(
":retreat: {0} name was changed - {1}\n« {2}\n» {3}",
currentType,
SteamDB.GetAppURL(AppID, "history"),
appName,
newAppName
)
);
}
}
if (appType.Equals("0"))
{
DbWorker.ExecuteNonQuery("UPDATE `Apps` SET `AppType` = @Type WHERE `AppID` = @AppID",
new MySqlParameter("@AppID", AppID),
new MySqlParameter("@Type", newAppType)
);
MakeHistory("created_info", DATABASE_APPTYPE, string.Empty, newAppType);
}
else if (!appType.Equals(newAppType))
{
DbWorker.ExecuteNonQuery("UPDATE `Apps` SET `AppType` = @Type WHERE `AppID` = @AppID",
new MySqlParameter("@AppID", AppID),
new MySqlParameter("@Type", newAppType)
);
MakeHistory("modified_info", DATABASE_APPTYPE, appType, newAppType);
}
}
// If we are full running, process depots too
bool depotsSectionModified = Settings.IsFullRun && productInfo.KeyValues["depots"].Children.Count > 0;
foreach (KeyValue section in productInfo.KeyValues.Children)
{
string sectionName = section.Name.ToLower();
if (sectionName == "appid" || sectionName == "public_only")
{
continue;
}
if (sectionName == "change_number") // Carefully handle change_number
{
sectionName = "root_change_number";
// TODO: Remove this key, move it to Apps table itself
ProcessKey(sectionName, "change_number", productInfo.ChangeNumber.ToString()); //section.AsString());
}
else if (sectionName == "common" || sectionName == "extended")
{
string keyName;
foreach (KeyValue keyvalue in section.Children)
{
keyName = string.Format("{0}_{1}", sectionName, keyvalue.Name);
if (keyName.Equals("common_type") || keyName.Equals("common_gameid") || keyName.Equals("common_name") || keyName.Equals("extended_order"))
{
// Ignore common keys that are either duplicated or serve no real purpose
continue;
}
if (keyvalue.Children.Count > 0)
{
if (keyName.Equals("common_languages"))
{
ProcessKey(keyName, keyvalue.Name, string.Join(",", keyvalue.Children.Select(x => x.Name)));
}
else
{
ProcessKey(keyName, keyvalue.Name, DbWorker.JsonifyKeyValue(keyvalue), true);
}
}
else if (!string.IsNullOrEmpty(keyvalue.Value))
{
ProcessKey(keyName, keyvalue.Name, keyvalue.Value);
}
}
}
else
{
sectionName = string.Format("root_{0}", sectionName);
if (ProcessKey(sectionName, sectionName, DbWorker.JsonifyKeyValue(section), true) && sectionName.Equals("root_depots"))
{
depotsSectionModified = true;
}
}
}
foreach (string keyName in CurrentData.Keys)
{
if (!keyName.StartsWith("website", StringComparison.Ordinal))
{
uint ID = GetKeyNameID(keyName);
DbWorker.ExecuteNonQuery("DELETE FROM AppsInfo WHERE `AppID` = @AppID AND `Key` = @KeyNameID",
new MySqlParameter("@AppID", AppID),
new MySqlParameter("@KeyNameID", ID)
);
MakeHistory("removed_key", ID, CurrentData[keyName]);
}
}
if (productInfo.KeyValues["common"]["name"].Value == null)
{
if (string.IsNullOrEmpty(appName)) // We don't have the app in our database yet
{
DbWorker.ExecuteNonQuery("INSERT INTO `Apps` (`AppID`, `Name`) VALUES (@AppID, @AppName) ON DUPLICATE KEY UPDATE `AppType` = `AppType`",
new MySqlParameter("@AppID", AppID),
new MySqlParameter("@AppName", string.Format("{0} {1}", SteamDB.UNKNOWN_APP, AppID))
);
}
else if (!appName.StartsWith(SteamDB.UNKNOWN_APP, StringComparison.Ordinal)) // We do have the app, replace it with default name
{
DbWorker.ExecuteNonQuery("UPDATE `Apps` SET `Name` = @AppName, `AppType` = 0 WHERE `AppID` = @AppID",
new MySqlParameter("@AppID", AppID),
new MySqlParameter("@AppName", string.Format("{0} {1}", SteamDB.UNKNOWN_APP, AppID))
);
MakeHistory("deleted_app", 0, appName);
}
}
if (depotsSectionModified)
{
DepotProcessor.Process(AppID, ChangeNumber, productInfo.KeyValues["depots"]);
}
}
private void TryProcess(SteamApps.PICSProductInfoCallback.PICSProductInfo productInfo)
{
string packageName = string.Empty;
var apps = new Dictionary <uint, string>();
using (MySqlDataReader Reader = DbWorker.ExecuteReader("SELECT `Name`, `Value` FROM `SubsInfo` INNER JOIN `KeyNamesSubs` ON `SubsInfo`.`Key` = `KeyNamesSubs`.`ID` WHERE `SubID` = @SubID", new MySqlParameter("@SubID", SubID)))
{
while (Reader.Read())
{
CurrentData.Add(DbWorker.GetString("Name", Reader), DbWorker.GetString("Value", Reader));
}
}
using (MySqlDataReader Reader = DbWorker.ExecuteReader("SELECT `Name` FROM `Subs` WHERE `SubID` = @SubID LIMIT 1", new MySqlParameter("@SubID", SubID)))
{
if (Reader.Read())
{
packageName = DbWorker.GetString("Name", Reader);
}
}
using (MySqlDataReader Reader = DbWorker.ExecuteReader("SELECT `AppID`, `Type` FROM `SubsApps` WHERE `SubID` = @SubID", new MySqlParameter("@SubID", SubID)))
{
while (Reader.Read())
{
apps.Add(Reader.GetUInt32("AppID"), Reader.GetString("Type"));
}
}
var kv = productInfo.KeyValues.Children.FirstOrDefault();
if (kv["name"].Value != null)
{
if (string.IsNullOrEmpty(packageName))
{
DbWorker.ExecuteNonQuery("INSERT INTO `Subs` (`SubID`, `Name`) VALUES (@SubID, @Name) ON DUPLICATE KEY UPDATE `Name` = @Name",
new MySqlParameter("@SubID", SubID),
new MySqlParameter("@Name", kv["name"].Value)
);
MakeHistory("created_sub");
MakeHistory("created_info", DATABASE_NAME_TYPE, string.Empty, kv["name"].Value);
}
else if (!packageName.Equals(kv["name"].Value))
{
DbWorker.ExecuteNonQuery("UPDATE `Subs` SET `Name` = @Name WHERE `SubID` = @SubID",
new MySqlParameter("@SubID", SubID),
new MySqlParameter("@Name", kv["name"].Value)
);
MakeHistory("modified_info", DATABASE_NAME_TYPE, packageName, kv["name"].Value);
}
}
foreach (KeyValue section in kv.Children)
{
string sectionName = section.Name.ToLower();
if (string.IsNullOrEmpty(sectionName) || sectionName.Equals("packageid") || sectionName.Equals("name"))
{
// Ignore common keys
continue;
}
if (sectionName.Equals("appids") || sectionName.Equals("depotids"))
{
string type = sectionName.Replace("ids", string.Empty); // Remove "ids", so we get app from appids and depot from depotids
uint typeID = (uint)(type.Equals("app") ? 0 : 1); // TODO: Remove legacy 0/1 and replace with type
foreach (KeyValue childrenApp in section.Children)
{
uint appID = uint.Parse(childrenApp.Value);
// Is this appid already in this package?
if (apps.ContainsKey(appID))
{
// Is this appid's type the same?
if (apps[appID] != type)
{
DbWorker.ExecuteNonQuery("UPDATE `SubsApps` SET `Type` = @Type WHERE `SubID` = @SubID AND `AppID` = @AppID",
new MySqlParameter("@SubID", SubID),
new MySqlParameter("@AppID", appID),
new MySqlParameter("@Type", type)
);
MakeHistory("added_to_sub", typeID, apps[appID].Equals("app") ? "0" : "1", childrenApp.Value);
}
apps.Remove(appID);
}
else
{
DbWorker.ExecuteNonQuery("INSERT INTO `SubsApps` (`SubID`, `AppID`, `Type`) VALUES(@SubID, @AppID, @Type) ON DUPLICATE KEY UPDATE `Type` = @Type",
new MySqlParameter("@SubID", SubID),
new MySqlParameter("@AppID", appID),
new MySqlParameter("@Type", type)
);
MakeHistory("added_to_sub", typeID, string.Empty, childrenApp.Value);
AppProcessor.MakeHistory(appID, ChangeNumber, "added_to_sub", typeID, string.Empty, SubID.ToString());
if (SteamProxy.Instance.ImportantApps.Contains(appID))
{
IRC.SendMain("Important app {0}{1}{2} was added to package {3}{4}{5} -{6} {7}",
Colors.OLIVE, SteamProxy.GetAppName(appID), Colors.NORMAL,
Colors.OLIVE, packageName, Colors.NORMAL,
Colors.DARK_BLUE, SteamDB.GetPackageURL(SubID, "history")
);
}
}
}
}
else if (sectionName.Equals("extended"))
{
string keyName;
foreach (KeyValue children in section.Children)
{
if (children.Children.Count > 0)
{
Log.WriteError("Sub Processor", "SubID {0} has childen in extended section", SubID);
}
keyName = string.Format("{0}_{1}", sectionName, children.Name);
ProcessKey(keyName, children.Name, children.Value);
}
}
else if (section.Children.Count > 0)
{
sectionName = string.Format("root_{0}", sectionName);
ProcessKey(sectionName, sectionName, DbWorker.JsonifyKeyValue(section), true);
}
else if (!string.IsNullOrEmpty(section.Value))
{
string keyName = string.Format("root_{0}", sectionName);
ProcessKey(keyName, sectionName, section.Value);
}
}
foreach (string keyName in CurrentData.Keys)
{
if (!keyName.StartsWith("website", StringComparison.Ordinal))
{
uint ID = GetKeyNameID(keyName);
DbWorker.ExecuteNonQuery("DELETE FROM `SubsInfo` WHERE `SubID` = @SubID AND `Key` = @KeyNameID",
new MySqlParameter("@SubID", SubID),
new MySqlParameter("@KeyNameID", ID)
);
MakeHistory("removed_key", ID, CurrentData[keyName]);
}
}
foreach (var app in apps)
{
DbWorker.ExecuteNonQuery("DELETE FROM `SubsApps` WHERE `SubID` = @SubID AND `AppID` = @AppID AND `Type` = @Type",
new MySqlParameter("@SubID", SubID),
new MySqlParameter("@AppID", app.Key),
new MySqlParameter("@Type", app.Value)
);
uint typeID = (uint)(app.Value.Equals("app") ? 0 : 1); // TODO: Remove legacy 0/1 and replace with type
MakeHistory("removed_from_sub", typeID, app.Key.ToString());
AppProcessor.MakeHistory(app.Key, ChangeNumber, "removed_from_sub", typeID, SubID.ToString());
if (SteamProxy.Instance.ImportantApps.Contains(app.Key))
{
IRC.SendMain("Important app {0}{1}{2} was removed from package {3}{4}{5} -{6} {7}",
Colors.OLIVE, SteamProxy.GetAppName(app.Key), Colors.NORMAL,
Colors.OLIVE, packageName, Colors.NORMAL,
Colors.DARK_BLUE, SteamDB.GetPackageURL(SubID, "history")
);
}
}
#if DEBUG
if (kv["name"].Value == null)
{
if (string.IsNullOrEmpty(packageName)) // We don't have the package in our database yet
{
// Don't do anything then
Log.WriteError("Sub Processor", "Got a package without a name, and we don't have it in our database: {0}", SubID);
}
else
{
////MakeHistory("deleted_sub", "0", packageName, "", true);
Log.WriteError("Sub Processor", "Got a package without a name, but we have it in our database: {0}", SubID);
}
}
#endif
}
private bool ProcessKey(string keyName, string displayName, string value, bool isJSON = false)
{
// All keys in PICS are supposed to be lower case.
// But currently some keys in packages are not lowercased,
// this lowercases everything to make sure nothing breaks in future
keyName = keyName.ToLower();
if (!CurrentData.ContainsKey(keyName))
{
uint ID = GetKeyNameID(keyName);
if (ID == 0)
{
if (isJSON)
{
const uint DB_TYPE_JSON = 86;
DbWorker.ExecuteNonQuery("INSERT INTO `KeyNamesSubs` (`Name`, `Type`, `DisplayName`) VALUES(@Name, @Type, @DisplayName) ON DUPLICATE KEY UPDATE `Type` = `Type`",
new MySqlParameter("@Name", keyName),
new MySqlParameter("@DisplayName", displayName),
new MySqlParameter("@Type", DB_TYPE_JSON)
);
}
else
{
DbWorker.ExecuteNonQuery("INSERT INTO `KeyNamesSubs` (`Name`, `DisplayName`) VALUES(@Name, @DisplayName) ON DUPLICATE KEY UPDATE `Type` = `Type`",
new MySqlParameter("@Name", keyName),
new MySqlParameter("@DisplayName", displayName)
);
}
ID = GetKeyNameID(keyName);
if (ID == 0)
{
// We can't insert anything because key wasn't created
Log.WriteError("Sub Processor", "Failed to create key {0} for SubID {1}, not inserting info.", keyName, SubID);
return(false);
}
}
InsertInfo(ID, value);
MakeHistory("created_key", ID, string.Empty, value);
return(true);
}
string currentValue = CurrentData[keyName];
CurrentData.Remove(keyName);
if (!currentValue.Equals(value))
{
uint ID = GetKeyNameID(keyName);
InsertInfo(ID, value);
MakeHistory("modified_key", ID, currentValue, value);
return(true);
}
return(false);
}