public void Dispose()
{
if (DepotProcessor != null)
{
DepotProcessor.Dispose();
DepotProcessor = null;
}
if (WatchdogHandle != null)
{
WatchdogHandle.Dispose();
WatchdogHandle = null;
}
if (FreeLicense != null)
{
FreeLicense.Dispose();
FreeLicense = null;
}
if (KeyActivatorHandle != null)
{
KeyActivatorHandle.Dispose();
KeyActivatorHandle = null;
}
}
public Steam()
{
Client = new SteamClient();
User = Client.GetHandler <SteamUser>();
Apps = Client.GetHandler <SteamApps>();
Friends = Client.GetHandler <SteamFriends>();
UserStats = Client.GetHandler <SteamUserStats>();
CallbackManager = new CallbackManager(Client);
Client.AddHandler(new ReadMachineAuth());
new Connection(CallbackManager);
new PICSProductInfo(CallbackManager);
new PICSTokens(CallbackManager);
new LicenseList(CallbackManager);
new WebAuth(CallbackManager);
new FreeLicense(CallbackManager);
if (!Settings.IsFullRun)
{
new AccountInfo(CallbackManager);
new ClanState(CallbackManager);
new ChatMemberInfo(CallbackManager);
new GameCoordinator(Client, CallbackManager);
new Watchdog();
}
PICSChanges = new PICSChanges(CallbackManager);
DepotProcessor = new DepotProcessor(Client);
IsRunning = true;
}
public Steam()
{
Client = new SteamClient();
User = Client.GetHandler<SteamUser>();
Apps = Client.GetHandler<SteamApps>();
Friends = Client.GetHandler<SteamFriends>();
UserStats = Client.GetHandler<SteamUserStats>();
CallbackManager = new CallbackManager(Client);
Client.AddHandler(new ReadMachineAuth());
new Connection(CallbackManager);
new PICSProductInfo(CallbackManager);
new PICSTokens(CallbackManager);
new LicenseList(CallbackManager);
new WebAuth(CallbackManager);
new FreeLicense(CallbackManager);
if (!Settings.IsFullRun)
{
new AccountInfo(CallbackManager);
new ClanState(CallbackManager);
new ChatMemberInfo(CallbackManager);
new GameCoordinator(Client, CallbackManager);
new Watchdog();
}
PICSChanges = new PICSChanges(CallbackManager);
DepotProcessor = new DepotProcessor(Client, CallbackManager);
IsRunning = true;
}
public void Dispose()
{
if (DepotProcessor != null)
{
DepotProcessor.Dispose();
DepotProcessor = null;
}
}
private Steam()
{
Configuration = SteamConfiguration.Create(b => b
.WithServerListProvider(new FileStorageServerListProvider(Path.Combine(Application.Path, "files", ".support", "servers.bin")))
.WithCellID(LocalConfig.Current.CellID)
.WithProtocolTypes(ProtocolTypes.Tcp)
.WithWebAPIKey(Settings.Current.Steam.WebAPIKey)
);
Client = new SteamClient(Configuration, "SteamDB");
#if DEBUG_NETHOOK
Client.DebugNetworkListener = new NetHookNetworkListener();
#endif
User = Client.GetHandler <SteamUser>();
Apps = Client.GetHandler <SteamApps>();
Friends = Client.GetHandler <SteamFriends>();
UserStats = Client.GetHandler <SteamUserStats>();
UnifiedMessages = Client.GetHandler <SteamUnifiedMessages>();
CallbackManager = new CallbackManager(Client);
Client.AddHandler(new PurchaseResponse());
Handlers = new List <SteamHandler>
{
new Connection(CallbackManager),
new AccountInfo(CallbackManager),
new PICSProductInfo(CallbackManager),
new PICSTokens(CallbackManager),
new LicenseList(CallbackManager),
new WebAuth(CallbackManager)
};
if (Settings.Current.CanQueryStore)
{
Handlers.Add(new FreeLicense(CallbackManager));
}
if (!Settings.IsFullRun)
{
Handlers.Add(new ClanState(CallbackManager));
WatchdogHandle = new Watchdog();
}
PICSChanges = new PICSChanges(CallbackManager);
DepotProcessor = new DepotProcessor(Client, CallbackManager);
IsRunning = true;
}
public static void AddJob(Func<JobID> action, DepotProcessor.ManifestJob manifestJob)
{
var jobID = action();
var job = new JobAction
{
Action = action,
ManifestJob = manifestJob
};
Log.WriteDebug("Job Manager", "New depot job: {0} ({1} - {2})", jobID, manifestJob.DepotID, manifestJob.ManifestID);
Jobs.TryAdd(jobID, job);
}
public void Dispose()
{
if (DepotProcessor != null)
{
DepotProcessor.Dispose();
DepotProcessor = null;
}
if (WatchdogHandle != null)
{
WatchdogHandle.Dispose();
WatchdogHandle = null;
}
}
private Steam()
{
Configuration = SteamConfiguration.Create(b => b
.WithServerListProvider(new FileStorageServerListProvider(Path.Combine(Application.Path, "files", ".support", "servers.bin")))
.WithCellID(LocalConfig.CellID)
.WithProtocolTypes(ProtocolTypes.Tcp)
.WithWebAPIBaseAddress(Settings.Current.Steam.WebAPIUrl)
.WithWebAPIKey(Settings.Current.Steam.WebAPIKey)
);
Client = new SteamClient(Configuration);
User = Client.GetHandler <SteamUser>();
Apps = Client.GetHandler <SteamApps>();
Friends = Client.GetHandler <SteamFriends>();
UserStats = Client.GetHandler <SteamUserStats>();
CallbackManager = new CallbackManager(Client);
Client.AddHandler(new PurchaseResponse());
new Connection(CallbackManager);
new AccountInfo(CallbackManager);
new PICSProductInfo(CallbackManager);
new PICSTokens(CallbackManager);
new LicenseList(CallbackManager);
new WebAuth(CallbackManager);
if (Settings.Current.CanQueryStore)
{
new FreeLicense(CallbackManager);
}
if (!Settings.IsFullRun)
{
new MarketingMessage(CallbackManager);
new ClanState(CallbackManager);
new ChatMemberInfo(CallbackManager);
new GameCoordinator(Client, CallbackManager);
new Watchdog();
}
PICSChanges = new PICSChanges(CallbackManager);
DepotProcessor = new DepotProcessor(Client, CallbackManager);
IsRunning = true;
}
private Steam()
{
Configuration = new SteamConfiguration
{
ServerListProvider = new FileStorageServerListProvider(Path.Combine(Application.Path, "files", ".support", "servers.bin")),
WebAPIBaseAddress = Settings.Current.Steam.WebAPIUrl,
CellID = LocalConfig.CellID,
ProtocolTypes = ProtocolTypes.Tcp
};
Client = new SteamClient(Configuration);
User = Client.GetHandler <SteamUser>();
Apps = Client.GetHandler <SteamApps>();
Friends = Client.GetHandler <SteamFriends>();
UserStats = Client.GetHandler <SteamUserStats>();
CallbackManager = new CallbackManager(Client);
Client.AddHandler(new ReadMachineAuth());
new Connection(CallbackManager);
new PICSProductInfo(CallbackManager);
new PICSTokens(CallbackManager);
new LicenseList(CallbackManager);
new WebAuth(CallbackManager);
new FreeLicense(CallbackManager);
if (!Settings.IsFullRun)
{
new AccountInfo(CallbackManager);
new ClanState(CallbackManager);
new ChatMemberInfo(CallbackManager);
new GameCoordinator(Client, CallbackManager);
new Watchdog();
}
PICSChanges = new PICSChanges(CallbackManager);
DepotProcessor = new DepotProcessor(Client, CallbackManager);
IsRunning = true;
}
public void Process(SteamApps.PICSProductInfoCallback.PICSProductInfo productInfo)
{
ChangeNumber = productInfo.ChangeNumber;
if (Settings.IsFullRun)
{
Log.WriteDebug("Sub Processor", "SubID: {0}", SubID);
DbConnection.Execute("INSERT INTO `Changelists` (`ChangeID`) VALUES (@ChangeNumber) ON DUPLICATE KEY UPDATE `Date` = `Date`", new { productInfo.ChangeNumber });
DbConnection.Execute("INSERT INTO `ChangelistsSubs` (`ChangeID`, `SubID`) VALUES (@ChangeNumber, @SubID) ON DUPLICATE KEY UPDATE `SubID` = `SubID`", new { SubID, productInfo.ChangeNumber });
}
ProcessKey("root_changenumber", "changenumber", ChangeNumber.ToString());
var appAddedToThisPackage = false;
var packageOwned = LicenseList.OwnedSubs.ContainsKey(SubID);
var newPackageName = productInfo.KeyValues["name"].AsString();
var apps = DbConnection.Query <PackageApp>("SELECT `AppID`, `Type` FROM `SubsApps` WHERE `SubID` = @SubID", new { SubID }).ToDictionary(x => x.AppID, x => x.Type);
// TODO: Ideally this should be SteamDB Unknown Package and proper checks like app processor does
if (newPackageName == null)
{
newPackageName = string.Concat("Steam Sub ", SubID);
}
if (newPackageName != null)
{
if (string.IsNullOrEmpty(PackageName))
{
DbConnection.Execute("INSERT INTO `Subs` (`SubID`, `Name`, `LastKnownName`) VALUES (@SubID, @Name, @Name)", new { SubID, Name = newPackageName });
MakeHistory("created_sub");
MakeHistory("created_info", SteamDB.DATABASE_NAME_TYPE, string.Empty, newPackageName);
}
else if (!PackageName.Equals(newPackageName))
{
if (newPackageName.StartsWith("Steam Sub ", StringComparison.Ordinal))
{
DbConnection.Execute("UPDATE `Subs` SET `Name` = @Name WHERE `SubID` = @SubID", new { SubID, Name = newPackageName });
}
else
{
DbConnection.Execute("UPDATE `Subs` SET `Name` = @Name, `LastKnownName` = @Name WHERE `SubID` = @SubID", new { SubID, Name = newPackageName });
}
MakeHistory("modified_info", SteamDB.DATABASE_NAME_TYPE, PackageName, newPackageName);
}
}
foreach (var section in productInfo.KeyValues.Children)
{
string sectionName = section.Name.ToLower();
if (string.IsNullOrEmpty(sectionName) || sectionName.Equals("packageid") || sectionName.Equals("changenumber") || sectionName.Equals("name"))
{
// Ignore common keys
continue;
}
if (sectionName.Equals("appids") || sectionName.Equals("depotids"))
{
// Remove "ids", so we get "app" from appids and "depot" from depotids
string type = sectionName.Replace("ids", string.Empty);
var isAppSection = type.Equals("app");
var typeID = (uint)(isAppSection ? 0 : 1); // 0 = app, 1 = depot; can't store as string because it's in the `key` field
foreach (var childrenApp in section.Children)
{
uint appID = uint.Parse(childrenApp.Value);
// Is this appid already in this package?
if (apps.ContainsKey(appID))
{
// Is this appid's type the same?
if (apps[appID] != type)
{
DbConnection.Execute("UPDATE `SubsApps` SET `Type` = @Type WHERE `SubID` = @SubID AND `AppID` = @AppID", new { SubID, AppID = appID, Type = type });
MakeHistory("added_to_sub", typeID, apps[appID].Equals("app") ? "0" : "1", childrenApp.Value);
appAddedToThisPackage = true;
// TODO: Log relevant add/remove history for depot/app?
}
apps.Remove(appID);
}
else
{
DbConnection.Execute("INSERT INTO `SubsApps` (`SubID`, `AppID`, `Type`) VALUES(@SubID, @AppID, @Type)", new { SubID, AppID = appID, Type = type });
MakeHistory("added_to_sub", typeID, string.Empty, childrenApp.Value);
if (isAppSection)
{
DbConnection.Execute(AppProcessor.HistoryQuery,
new PICSHistory
{
ID = appID,
ChangeID = ChangeNumber,
NewValue = SubID.ToString(),
Action = "added_to_sub"
}
);
}
else
{
DbConnection.Execute(DepotProcessor.GetHistoryQuery(),
new DepotHistory
{
DepotID = appID,
ChangeID = ChangeNumber,
NewValue = SubID,
Action = "added_to_sub"
}
);
}
appAddedToThisPackage = true;
if (packageOwned && !LicenseList.OwnedApps.ContainsKey(appID))
{
LicenseList.OwnedApps.Add(appID, 1);
}
}
}
}
else if (sectionName.Equals("extended"))
{
foreach (var children in section.Children)
{
var keyName = string.Format("{0}_{1}", sectionName, children.Name);
if (children.Children.Count > 0)
{
ProcessKey(keyName, children.Name, Utils.JsonifyKeyValue(children), true);
}
else
{
ProcessKey(keyName, children.Name, children.Value);
}
}
}
else if (section.Children.Any())
{
sectionName = string.Format("root_{0}", sectionName);
ProcessKey(sectionName, sectionName, Utils.JsonifyKeyValue(section), true);
}
else if (!string.IsNullOrEmpty(section.Value))
{
string keyName = string.Format("root_{0}", sectionName);
ProcessKey(keyName, sectionName, section.Value);
}
}
foreach (var data in CurrentData.Values.Where(data => !data.Processed && !data.KeyName.StartsWith("website", StringComparison.Ordinal)))
{
DbConnection.Execute("DELETE FROM `SubsInfo` WHERE `SubID` = @SubID AND `Key` = @Key", new { SubID, data.Key });
MakeHistory("removed_key", data.Key, data.Value);
}
var appsRemoved = apps.Any();
foreach (var app in apps)
{
DbConnection.Execute("DELETE FROM `SubsApps` WHERE `SubID` = @SubID AND `AppID` = @AppID AND `Type` = @Type", new { SubID, AppID = app.Key, Type = app.Value });
var isAppSection = app.Value.Equals("app");
var typeID = (uint)(isAppSection ? 0 : 1); // 0 = app, 1 = depot; can't store as string because it's in the `key` field
MakeHistory("removed_from_sub", typeID, app.Key.ToString());
if (isAppSection)
{
DbConnection.Execute(AppProcessor.HistoryQuery,
new PICSHistory
{
ID = app.Key,
ChangeID = ChangeNumber,
OldValue = SubID.ToString(),
Action = "removed_from_sub"
}
);
}
else
{
DbConnection.Execute(DepotProcessor.GetHistoryQuery(),
new DepotHistory
{
DepotID = app.Key,
ChangeID = ChangeNumber,
OldValue = SubID,
Action = "removed_from_sub"
}
);
}
}
if (appsRemoved)
{
LicenseList.RefreshApps();
}
var billingType = (EBillingType)productInfo.KeyValues["billingtype"].AsInteger();
if (!packageOwned && (billingType == EBillingType.FreeOnDemand || (billingType == EBillingType.NoCost && SubID != 17906)))
{
Log.WriteDebug("Sub Processor", "Requesting apps in SubID {0} as a free license", SubID);
JobManager.AddJob(() => Steam.Instance.Apps.RequestFreeLicense(productInfo.KeyValues["appids"].Children.Select(appid => (uint)appid.AsInteger()).ToList()));
}
// Re-queue apps in this package so we can update depots and whatnot
if (appAddedToThisPackage && !Settings.IsFullRun && !string.IsNullOrEmpty(PackageName))
{
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(productInfo.KeyValues["appids"].Children.Select(x => (uint)x.AsInteger()), Enumerable.Empty <uint>()));
}
// Maintain a list of anonymous content
if (appAddedToThisPackage && SubID == 17906)
{
LicenseList.RefreshAnonymous();
}
}
/*
* Here be dragons.
*/
public static void DownloadFilesFromDepot(DepotProcessor.ManifestJob job, DepotManifest depotManifest)
{
var randomGenerator = new Random();
var files = depotManifest.Files.Where(x => IsFileNameMatching(job.DepotID, x.FileName)).ToList();
var filesUpdated = false;
var filesAnyFailed = false;
var hashesFile = Path.Combine(Application.Path, "files", ".support", "hashes", string.Format("{0}.json", job.DepotID));
var hashes = new Dictionary<string, byte[]>();
if (File.Exists(hashesFile))
{
hashes = JsonConvert.DeserializeObject<Dictionary<string, byte[]>>(File.ReadAllText(hashesFile));
}
Log.WriteDebug("FileDownloader", "Will download {0} files from depot {1}", files.Count, job.DepotID);
Parallel.ForEach(files, new ParallelOptions { MaxDegreeOfParallelism = 2 }, (file, state2) =>
{
string directory = Path.Combine(Application.Path, "files", job.DepotID.ToString(), Path.GetDirectoryName(file.FileName));
string finalPath = Path.Combine(directory, Path.GetFileName(file.FileName));
string downloadPath = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Path.GetRandomFileName(), ".steamdb_tmp"));
if (!Directory.Exists(directory))
{
Directory.CreateDirectory(directory);
}
else if (file.TotalSize == 0)
{
if (File.Exists(finalPath))
{
var f = new FileInfo(finalPath);
if(f.Length == 0)
{
#if DEBUG
Log.WriteDebug("FileDownloader", "{0} is already empty", file.FileName);
#endif
return;
}
}
else
{
File.Create(finalPath);
Log.WriteInfo("FileDownloader", "{0} created an empty file", file.FileName);
return;
}
}
else if(hashes.ContainsKey(file.FileName) && file.FileHash.SequenceEqual(hashes[file.FileName]))
{
#if DEBUG
Log.WriteDebug("FileDownloader", "{0} already matches the file we have", file.FileName);
#endif
return;
}
var chunks = file.Chunks.OrderBy(x => x.Offset).ToList();
Log.WriteInfo("FileDownloader", "Downloading {0} ({1} bytes, {2} chunks)", file.FileName, file.TotalSize, chunks.Count);
uint count = 0;
byte[] checksum;
string lastError = "or checksum failed";
string oldChunksFile;
using (var sha = new SHA1Managed())
{
oldChunksFile = Path.Combine(Application.Path, "files", ".support", "chunks",
string.Format("{0}-{1}.json", job.DepotID, BitConverter.ToString(sha.ComputeHash(Encoding.UTF8.GetBytes(file.FileName))))
);
}
using (var fs = File.Open(downloadPath, FileMode.OpenOrCreate, FileAccess.ReadWrite))
{
fs.SetLength((long)file.TotalSize);
var lockObject = new object();
var neededChunks = new List<DepotManifest.ChunkData>();
if (File.Exists(oldChunksFile) && File.Exists(finalPath))
{
var oldChunks = JsonConvert.DeserializeObject<List<DepotManifest.ChunkData>>(
File.ReadAllText(oldChunksFile),
new JsonSerializerSettings { PreserveReferencesHandling = PreserveReferencesHandling.All }
);
using (var fsOld = File.Open(finalPath, FileMode.Open, FileAccess.Read))
{
foreach (var chunk in chunks)
{
var oldChunk = oldChunks.FirstOrDefault(c => c.ChunkID.SequenceEqual(chunk.ChunkID));
if (oldChunk != null)
{
var oldData = new byte[oldChunk.UncompressedLength];
fsOld.Seek((long)oldChunk.Offset, SeekOrigin.Begin);
fsOld.Read(oldData, 0, oldData.Length);
var existingChecksum = Utils.AdlerHash(oldData);
if (existingChecksum.SequenceEqual(chunk.Checksum))
{
fs.Seek((long)chunk.Offset, SeekOrigin.Begin);
fs.Write(oldData, 0, oldData.Length);
#if DEBUG
Log.WriteDebug("FileDownloader", "{0} Found chunk ({1}), not downloading ({2}/{3})", file.FileName, chunk.Offset, ++count, chunks.Count);
#endif
}
else
{
neededChunks.Add(chunk);
#if DEBUG
Log.WriteDebug("FileDownloader", "{0} Found chunk ({1}), but checksum differs", file.FileName, chunk.Offset);
#endif
}
}
else
{
neededChunks.Add(chunk);
}
}
}
}
else
{
neededChunks = chunks;
}
Parallel.ForEach(neededChunks, new ParallelOptions { MaxDegreeOfParallelism = 3 }, (chunk, state) =>
{
var downloaded = false;
for (var i = 0; i <= 5; i++)
{
try
{
var chunkData = CDNClient.DownloadDepotChunk(job.DepotID, chunk, job.Server, job.CDNToken, job.DepotKey);
lock (lockObject)
{
fs.Seek((long)chunk.Offset, SeekOrigin.Begin);
fs.Write(chunkData.Data, 0, chunkData.Data.Length);
Log.WriteDebug("FileDownloader", "Downloaded {0} ({1}/{2})", file.FileName, ++count, chunks.Count);
}
downloaded = true;
break;
}
catch (Exception e)
{
lastError = e.Message;
}
// See https://developers.google.com/drive/web/handle-errors
Task.Delay((1 << i) * 1000 + randomGenerator.Next(1001)).Wait();
}
if (!downloaded)
{
state.Stop();
}
});
fs.Seek(0, SeekOrigin.Begin);
using (var sha = new SHA1Managed())
{
checksum = sha.ComputeHash(fs);
}
}
if (file.FileHash.SequenceEqual(checksum))
{
Log.WriteInfo("FileDownloader", "Downloaded {0} from {1}", file.FileName, Steam.GetAppName(job.ParentAppID));
hashes[file.FileName] = checksum;
if (File.Exists(finalPath))
{
File.Delete(finalPath);
}
File.Move(downloadPath, finalPath);
if (chunks.Count > 1)
{
File.WriteAllText(oldChunksFile,
JsonConvert.SerializeObject(
chunks,
Formatting.None,
new JsonSerializerSettings { PreserveReferencesHandling = PreserveReferencesHandling.All }
)
);
}
else if (File.Exists(oldChunksFile))
{
File.Delete(oldChunksFile);
}
filesUpdated = true;
}
else
{
filesAnyFailed = true;
IRC.Instance.SendOps("{0}[{1}]{2} Failed to download {3}: Only {4} out of {5} chunks downloaded ({6})",
Colors.OLIVE, Steam.GetAppName(job.ParentAppID), Colors.NORMAL, file.FileName, count, chunks.Count, lastError);
Log.WriteError("FileDownloader", "Failed to download {0}: Only {1} out of {2} chunks downloaded from {3} ({4})",
file.FileName, count, chunks.Count, job.Server, lastError);
File.Delete(downloadPath);
}
});
if (filesUpdated)
{
if (filesAnyFailed)
{
IRC.Instance.SendOps("{0}[{1}]{2} Failed to download some files, not running update script to prevent broken diffs.",
Colors.OLIVE, Steam.GetAppName(job.ParentAppID), Colors.NORMAL);
return;
}
File.WriteAllText(hashesFile, JsonConvert.SerializeObject(hashes));
var updateScript = Path.Combine(Application.Path, "files", "update.sh");
if (File.Exists(updateScript))
{
lock (updateLock)
{
// YOLO
Process.Start(updateScript, job.DepotID.ToString());
}
}
}
}
private void TryProcess(SteamApps.PICSProductInfoCallback.PICSProductInfo productInfo)
{
using (MySqlDataReader Reader = DbWorker.ExecuteReader("SELECT `Name`, `Value` FROM `AppsInfo` INNER JOIN `KeyNames` ON `AppsInfo`.`Key` = `KeyNames`.`ID` WHERE `AppID` = @AppID", new MySqlParameter("AppID", AppID)))
{
while (Reader.Read())
{
CurrentData.Add(DbWorker.GetString("Name", Reader), DbWorker.GetString("Value", Reader));
}
}
string appName = string.Empty;
string appType = "0";
using (MySqlDataReader Reader = DbWorker.ExecuteReader("SELECT `Name`, `AppType` FROM `Apps` WHERE `AppID` = @AppID LIMIT 1", new MySqlParameter("AppID", AppID)))
{
if (Reader.Read())
{
appName = DbWorker.GetString("Name", Reader);
appType = DbWorker.GetString("AppType", Reader);
}
}
if (productInfo.KeyValues["common"]["name"].Value != null)
{
string newAppType = "0";
string currentType = productInfo.KeyValues["common"]["type"].AsString().ToLower();
using (MySqlDataReader Reader = DbWorker.ExecuteReader("SELECT `AppType` FROM `AppsTypes` WHERE `Name` = @Type LIMIT 1", new MySqlParameter("Type", currentType)))
{
if (Reader.Read())
{
newAppType = DbWorker.GetString("AppType", Reader);
}
else
{
// TODO: Create it?
Log.WriteError("App Processor", "AppID {0} - unknown app type: {1}", AppID, currentType);
// TODO: This is debuggy just so we are aware of new app types
IRC.SendAnnounce("Unknown app type \"{0}\" for appid {1}, cc Alram and xPaw", currentType, AppID);
}
}
if (string.IsNullOrEmpty(appName) || appName.StartsWith(SteamDB.UNKNOWN_APP, StringComparison.Ordinal))
{
DbWorker.ExecuteNonQuery("INSERT INTO `Apps` (`AppID`, `AppType`, `Name`) VALUES (@AppID, @Type, @AppName) ON DUPLICATE KEY UPDATE `Name` = @AppName, `AppType` = @Type",
new MySqlParameter("@AppID", AppID),
new MySqlParameter("@Type", newAppType),
new MySqlParameter("@AppName", productInfo.KeyValues["common"]["name"].Value)
);
MakeHistory("created_app");
MakeHistory("created_info", DATABASE_NAME_TYPE, string.Empty, productInfo.KeyValues["common"]["name"].Value);
// TODO: Testy testy
if (!Settings.IsFullRun &&
Settings.Current.ChatRooms.Count > 0 &&
!appName.StartsWith("SteamApp", StringComparison.Ordinal) &&
!appName.StartsWith("ValveTest", StringComparison.Ordinal))
{
Steam.Instance.Friends.SendChatRoomMessage(Settings.Current.ChatRooms[0], EChatEntryType.ChatMsg,
string.Format(
":retreat: New {0} was published: {1}\nSteamDB: {2}\nSteam: http://store.steampowered.com/app/{3}/",
currentType,
productInfo.KeyValues["common"]["name"].AsString(),
SteamDB.GetAppURL(AppID),
AppID
)
);
}
}
else if (!appName.Equals(productInfo.KeyValues["common"]["name"].Value))
{
string newAppName = productInfo.KeyValues["common"]["name"].AsString();
DbWorker.ExecuteNonQuery("UPDATE `Apps` SET `Name` = @AppName WHERE `AppID` = @AppID",
new MySqlParameter("@AppID", AppID),
new MySqlParameter("@AppName", newAppName)
);
MakeHistory("modified_info", DATABASE_NAME_TYPE, appName, newAppName);
// TODO: Testy testy
if (!Settings.IsFullRun &&
Settings.Current.ChatRooms.Count > 0 &&
!string.Equals(appName, newAppName, StringComparison.OrdinalIgnoreCase) &&
!newAppName.StartsWith("SteamApp", StringComparison.Ordinal) &&
!newAppName.StartsWith("ValveTest", StringComparison.Ordinal))
{
Steam.Instance.Friends.SendChatRoomMessage(Settings.Current.ChatRooms[0], EChatEntryType.ChatMsg,
string.Format(
":retreat: {0} name was changed - {1}\n« {2}\n» {3}",
currentType,
SteamDB.GetAppURL(AppID, "history"),
appName,
newAppName
)
);
}
}
if (appType.Equals("0"))
{
DbWorker.ExecuteNonQuery("UPDATE `Apps` SET `AppType` = @Type WHERE `AppID` = @AppID",
new MySqlParameter("@AppID", AppID),
new MySqlParameter("@Type", newAppType)
);
MakeHistory("created_info", DATABASE_APPTYPE, string.Empty, newAppType);
}
else if (!appType.Equals(newAppType))
{
DbWorker.ExecuteNonQuery("UPDATE `Apps` SET `AppType` = @Type WHERE `AppID` = @AppID",
new MySqlParameter("@AppID", AppID),
new MySqlParameter("@Type", newAppType)
);
MakeHistory("modified_info", DATABASE_APPTYPE, appType, newAppType);
}
}
// If we are full running, process depots too
bool depotsSectionModified = Settings.IsFullRun && productInfo.KeyValues["depots"].Children.Count > 0;
foreach (KeyValue section in productInfo.KeyValues.Children)
{
string sectionName = section.Name.ToLower();
if (sectionName == "appid" || sectionName == "public_only")
{
continue;
}
if (sectionName == "change_number") // Carefully handle change_number
{
sectionName = "root_change_number";
// TODO: Remove this key, move it to Apps table itself
ProcessKey(sectionName, "change_number", productInfo.ChangeNumber.ToString()); //section.AsString());
}
else if (sectionName == "common" || sectionName == "extended")
{
string keyName;
foreach (KeyValue keyvalue in section.Children)
{
keyName = string.Format("{0}_{1}", sectionName, keyvalue.Name);
if (keyName.Equals("common_type") || keyName.Equals("common_gameid") || keyName.Equals("common_name") || keyName.Equals("extended_order"))
{
// Ignore common keys that are either duplicated or serve no real purpose
continue;
}
if (keyvalue.Children.Count > 0)
{
if (keyName.Equals("common_languages"))
{
ProcessKey(keyName, keyvalue.Name, string.Join(",", keyvalue.Children.Select(x => x.Name)));
}
else
{
ProcessKey(keyName, keyvalue.Name, DbWorker.JsonifyKeyValue(keyvalue), true);
}
}
else if (!string.IsNullOrEmpty(keyvalue.Value))
{
ProcessKey(keyName, keyvalue.Name, keyvalue.Value);
}
}
}
else
{
sectionName = string.Format("root_{0}", sectionName);
if (ProcessKey(sectionName, sectionName, DbWorker.JsonifyKeyValue(section), true) && sectionName.Equals("root_depots"))
{
depotsSectionModified = true;
}
}
}
foreach (string keyName in CurrentData.Keys)
{
if (!keyName.StartsWith("website", StringComparison.Ordinal))
{
uint ID = GetKeyNameID(keyName);
DbWorker.ExecuteNonQuery("DELETE FROM AppsInfo WHERE `AppID` = @AppID AND `Key` = @KeyNameID",
new MySqlParameter("@AppID", AppID),
new MySqlParameter("@KeyNameID", ID)
);
MakeHistory("removed_key", ID, CurrentData[keyName]);
}
}
if (productInfo.KeyValues["common"]["name"].Value == null)
{
if (string.IsNullOrEmpty(appName)) // We don't have the app in our database yet
{
DbWorker.ExecuteNonQuery("INSERT INTO `Apps` (`AppID`, `Name`) VALUES (@AppID, @AppName) ON DUPLICATE KEY UPDATE `AppType` = `AppType`",
new MySqlParameter("@AppID", AppID),
new MySqlParameter("@AppName", string.Format("{0} {1}", SteamDB.UNKNOWN_APP, AppID))
);
}
else if (!appName.StartsWith(SteamDB.UNKNOWN_APP, StringComparison.Ordinal)) // We do have the app, replace it with default name
{
DbWorker.ExecuteNonQuery("UPDATE `Apps` SET `Name` = @AppName, `AppType` = 0 WHERE `AppID` = @AppID",
new MySqlParameter("@AppID", AppID),
new MySqlParameter("@AppName", string.Format("{0} {1}", SteamDB.UNKNOWN_APP, AppID))
);
MakeHistory("deleted_app", 0, appName);
}
}
if (depotsSectionModified)
{
DepotProcessor.Process(AppID, ChangeNumber, productInfo.KeyValues["depots"]);
}
}
public void Init()
{
ProcessorPool = new SmartThreadPool(new STPStartInfo {
WorkItemPriority = WorkItemPriority.Highest, MaxWorkerThreads = 50
});
SecondaryPool = new SmartThreadPool();
ProcessorPool.Name = "Processor Pool";
SecondaryPool.Name = "Secondary Pool";
OwnedPackages = new List <uint>();
ProcessedApps = new ConcurrentDictionary <uint, IWorkItemResult>();
ProcessedSubs = new ConcurrentDictionary <uint, IWorkItemResult>();
Timer = new System.Timers.Timer();
Timer.Elapsed += OnTimer;
Timer.Interval = TimeSpan.FromSeconds(1).TotalMilliseconds;
Client = new SteamClient();
User = Client.GetHandler <SteamUser>();
Apps = Client.GetHandler <SteamApps>();
Friends = Client.GetHandler <SteamFriends>();
UserStats = Client.GetHandler <SteamUserStats>();
CallbackManager = new CallbackManager(Client);
CallbackManager.Register(new Callback <SteamClient.ConnectedCallback>(OnConnected));
CallbackManager.Register(new Callback <SteamClient.DisconnectedCallback>(OnDisconnected));
CallbackManager.Register(new Callback <SteamUser.LoggedOnCallback>(OnLoggedOn));
CallbackManager.Register(new Callback <SteamUser.LoggedOffCallback>(OnLoggedOff));
CallbackManager.Register(new Callback <SteamApps.LicenseListCallback>(OnLicenseListCallback));
CallbackManager.Register(new JobCallback <SteamUser.UpdateMachineAuthCallback>(OnMachineAuth));
CallbackManager.Register(new JobCallback <SteamApps.PICSProductInfoCallback>(OnPICSProductInfo));
CallbackManager.Register(new JobCallback <SteamApps.PICSTokensCallback>(OnPICSTokens));
if (Settings.IsFullRun)
{
CallbackManager.Register(new JobCallback <SteamApps.PICSChangesCallback>(OnPICSChangesFullRun));
}
else
{
CallbackManager.Register(new JobCallback <SteamApps.PICSChangesCallback>(OnPICSChanges));
CallbackManager.Register(new JobCallback <SteamUserStats.NumberOfPlayersCallback>(SteamProxy.Instance.OnNumberOfPlayers));
CallbackManager.Register(new Callback <SteamFriends.ClanStateCallback>(SteamProxy.Instance.OnClanState));
CallbackManager.Register(new Callback <SteamFriends.ChatMsgCallback>(SteamProxy.Instance.OnChatMessage));
CallbackManager.Register(new Callback <SteamFriends.ChatMemberInfoCallback>(SteamProxy.Instance.OnChatMemberInfo));
CallbackManager.Register(new Callback <SteamUser.MarketingMessageCallback>(MarketingHandler.OnMarketingMessage));
CallbackManager.Register(new Callback <SteamUser.AccountInfoCallback>(OnAccountInfo));
}
DepotProcessor.Init();
GetLastChangeNumber();
IsRunning = true;
Client.Connect();
while (IsRunning)
{
CallbackManager.RunWaitCallbacks(TimeSpan.FromSeconds(1));
}
}
public static void DownloadFilesFromDepot(DepotProcessor.ManifestJob job, DepotManifest depotManifest)
{
var files = depotManifest.Files.Where(x => IsFileNameMatching(job.DepotID, x.FileName)).ToList();
var filesUpdated = false;
Log.WriteDebug("FileDownloader", "Will download {0} files from depot {1}", files.Count(), job.DepotID);
foreach (var file in files)
{
string directory = Path.Combine(Application.Path, FILES_DIRECTORY, job.DepotID.ToString(), Path.GetDirectoryName(file.FileName));
string finalPath = Path.Combine(directory, Path.GetFileName(file.FileName));
string downloadPath = Path.GetTempFileName();
if (!Directory.Exists(directory))
{
Directory.CreateDirectory(directory);
}
else if (File.Exists(finalPath))
{
using (var fs = File.Open(finalPath, FileMode.Open))
{
using (var sha = new SHA1Managed())
{
if (file.FileHash.SequenceEqual(sha.ComputeHash(fs)))
{
Log.WriteDebug("FileDownloader", "{0} already matches the file we have", file.FileName);
continue;
}
}
}
}
Log.WriteInfo("FileDownloader", "Downloading {0} ({1} bytes, {2} chunks)", file.FileName, file.TotalSize, file.Chunks.Count);
uint count = 0;
byte[] checksum;
string lastError = "or checksum failed";
using (var fs = File.Open(downloadPath, FileMode.OpenOrCreate))
{
fs.SetLength((long)file.TotalSize);
var lockObject = new object();
// TODO: We *could* verify each chunk and only download needed ones
Parallel.ForEach(file.Chunks, (chunk, state) =>
{
var downloaded = false;
for (var i = 0; i <= 5; i++)
{
try
{
var chunkData = CDNClient.DownloadDepotChunk(job.DepotID, chunk, job.Server, job.CDNToken, job.DepotKey);
lock (lockObject)
{
fs.Seek((long)chunk.Offset, SeekOrigin.Begin);
fs.Write(chunkData.Data, 0, chunkData.Data.Length);
Log.WriteDebug("FileDownloader", "Downloaded {0} ({1}/{2})", file.FileName, ++count, file.Chunks.Count);
}
downloaded = true;
break;
}
catch (Exception e)
{
lastError = e.Message;
}
}
if (!downloaded)
{
state.Stop();
}
});
fs.Seek(0, SeekOrigin.Begin);
using (var sha = new SHA1Managed())
{
checksum = sha.ComputeHash(fs);
}
}
if (file.Chunks.Count == 0 || file.FileHash.SequenceEqual(checksum))
{
Log.WriteInfo("FileDownloader", "Downloaded {0} from {1}", file.FileName, Steam.GetAppName(job.ParentAppID));
if (File.Exists(finalPath))
{
File.Delete(finalPath);
}
File.Move(downloadPath, finalPath);
filesUpdated = true;
}
else
{
IRC.Instance.SendOps("{0}[{1}]{2} Failed to download {3}: Only {4} out of {5} chunks downloaded ({6})",
Colors.OLIVE, Steam.GetAppName(job.ParentAppID), Colors.NORMAL, file.FileName, count, file.Chunks.Count, lastError);
Log.WriteError("FileDownloader", "Failed to download {0}: Only {1} out of {2} chunks downloaded from {3} ({4})",
file.FileName, count, file.Chunks.Count, job.Server, lastError);
File.Delete(downloadPath);
}
}
if (filesUpdated)
{
var updateScript = Path.Combine(Application.Path, "files", "update.sh");
if (File.Exists(updateScript))
{
// YOLO
Process.Start(updateScript, job.DepotID.ToString());
}
}
}
