public void Dispose() { if (DepotProcessor != null) { DepotProcessor.Dispose(); DepotProcessor = null; } if (WatchdogHandle != null) { WatchdogHandle.Dispose(); WatchdogHandle = null; } if (FreeLicense != null) { FreeLicense.Dispose(); FreeLicense = null; } if (KeyActivatorHandle != null) { KeyActivatorHandle.Dispose(); KeyActivatorHandle = null; } }
public Steam() { Client = new SteamClient(); User = Client.GetHandler <SteamUser>(); Apps = Client.GetHandler <SteamApps>(); Friends = Client.GetHandler <SteamFriends>(); UserStats = Client.GetHandler <SteamUserStats>(); CallbackManager = new CallbackManager(Client); Client.AddHandler(new ReadMachineAuth()); new Connection(CallbackManager); new PICSProductInfo(CallbackManager); new PICSTokens(CallbackManager); new LicenseList(CallbackManager); new WebAuth(CallbackManager); new FreeLicense(CallbackManager); if (!Settings.IsFullRun) { new AccountInfo(CallbackManager); new ClanState(CallbackManager); new ChatMemberInfo(CallbackManager); new GameCoordinator(Client, CallbackManager); new Watchdog(); } PICSChanges = new PICSChanges(CallbackManager); DepotProcessor = new DepotProcessor(Client); IsRunning = true; }
public Steam() { Client = new SteamClient(); User = Client.GetHandler<SteamUser>(); Apps = Client.GetHandler<SteamApps>(); Friends = Client.GetHandler<SteamFriends>(); UserStats = Client.GetHandler<SteamUserStats>(); CallbackManager = new CallbackManager(Client); Client.AddHandler(new ReadMachineAuth()); new Connection(CallbackManager); new PICSProductInfo(CallbackManager); new PICSTokens(CallbackManager); new LicenseList(CallbackManager); new WebAuth(CallbackManager); new FreeLicense(CallbackManager); if (!Settings.IsFullRun) { new AccountInfo(CallbackManager); new ClanState(CallbackManager); new ChatMemberInfo(CallbackManager); new GameCoordinator(Client, CallbackManager); new Watchdog(); } PICSChanges = new PICSChanges(CallbackManager); DepotProcessor = new DepotProcessor(Client, CallbackManager); IsRunning = true; }
public void Dispose() { if (DepotProcessor != null) { DepotProcessor.Dispose(); DepotProcessor = null; } }
private Steam() { Configuration = SteamConfiguration.Create(b => b .WithServerListProvider(new FileStorageServerListProvider(Path.Combine(Application.Path, "files", ".support", "servers.bin"))) .WithCellID(LocalConfig.Current.CellID) .WithProtocolTypes(ProtocolTypes.Tcp) .WithWebAPIKey(Settings.Current.Steam.WebAPIKey) ); Client = new SteamClient(Configuration, "SteamDB"); #if DEBUG_NETHOOK Client.DebugNetworkListener = new NetHookNetworkListener(); #endif User = Client.GetHandler <SteamUser>(); Apps = Client.GetHandler <SteamApps>(); Friends = Client.GetHandler <SteamFriends>(); UserStats = Client.GetHandler <SteamUserStats>(); UnifiedMessages = Client.GetHandler <SteamUnifiedMessages>(); CallbackManager = new CallbackManager(Client); Client.AddHandler(new PurchaseResponse()); Handlers = new List <SteamHandler> { new Connection(CallbackManager), new AccountInfo(CallbackManager), new PICSProductInfo(CallbackManager), new PICSTokens(CallbackManager), new LicenseList(CallbackManager), new WebAuth(CallbackManager) }; if (Settings.Current.CanQueryStore) { Handlers.Add(new FreeLicense(CallbackManager)); } if (!Settings.IsFullRun) { Handlers.Add(new ClanState(CallbackManager)); WatchdogHandle = new Watchdog(); } PICSChanges = new PICSChanges(CallbackManager); DepotProcessor = new DepotProcessor(Client, CallbackManager); IsRunning = true; }
public static void AddJob(Func<JobID> action, DepotProcessor.ManifestJob manifestJob) { var jobID = action(); var job = new JobAction { Action = action, ManifestJob = manifestJob }; Log.WriteDebug("Job Manager", "New depot job: {0} ({1} - {2})", jobID, manifestJob.DepotID, manifestJob.ManifestID); Jobs.TryAdd(jobID, job); }
public void Dispose() { if (DepotProcessor != null) { DepotProcessor.Dispose(); DepotProcessor = null; } if (WatchdogHandle != null) { WatchdogHandle.Dispose(); WatchdogHandle = null; } }
private Steam() { Configuration = SteamConfiguration.Create(b => b .WithServerListProvider(new FileStorageServerListProvider(Path.Combine(Application.Path, "files", ".support", "servers.bin"))) .WithCellID(LocalConfig.CellID) .WithProtocolTypes(ProtocolTypes.Tcp) .WithWebAPIBaseAddress(Settings.Current.Steam.WebAPIUrl) .WithWebAPIKey(Settings.Current.Steam.WebAPIKey) ); Client = new SteamClient(Configuration); User = Client.GetHandler <SteamUser>(); Apps = Client.GetHandler <SteamApps>(); Friends = Client.GetHandler <SteamFriends>(); UserStats = Client.GetHandler <SteamUserStats>(); CallbackManager = new CallbackManager(Client); Client.AddHandler(new PurchaseResponse()); new Connection(CallbackManager); new AccountInfo(CallbackManager); new PICSProductInfo(CallbackManager); new PICSTokens(CallbackManager); new LicenseList(CallbackManager); new WebAuth(CallbackManager); if (Settings.Current.CanQueryStore) { new FreeLicense(CallbackManager); } if (!Settings.IsFullRun) { new MarketingMessage(CallbackManager); new ClanState(CallbackManager); new ChatMemberInfo(CallbackManager); new GameCoordinator(Client, CallbackManager); new Watchdog(); } PICSChanges = new PICSChanges(CallbackManager); DepotProcessor = new DepotProcessor(Client, CallbackManager); IsRunning = true; }
private Steam() { Configuration = new SteamConfiguration { ServerListProvider = new FileStorageServerListProvider(Path.Combine(Application.Path, "files", ".support", "servers.bin")), WebAPIBaseAddress = Settings.Current.Steam.WebAPIUrl, CellID = LocalConfig.CellID, ProtocolTypes = ProtocolTypes.Tcp }; Client = new SteamClient(Configuration); User = Client.GetHandler <SteamUser>(); Apps = Client.GetHandler <SteamApps>(); Friends = Client.GetHandler <SteamFriends>(); UserStats = Client.GetHandler <SteamUserStats>(); CallbackManager = new CallbackManager(Client); Client.AddHandler(new ReadMachineAuth()); new Connection(CallbackManager); new PICSProductInfo(CallbackManager); new PICSTokens(CallbackManager); new LicenseList(CallbackManager); new WebAuth(CallbackManager); new FreeLicense(CallbackManager); if (!Settings.IsFullRun) { new AccountInfo(CallbackManager); new ClanState(CallbackManager); new ChatMemberInfo(CallbackManager); new GameCoordinator(Client, CallbackManager); new Watchdog(); } PICSChanges = new PICSChanges(CallbackManager); DepotProcessor = new DepotProcessor(Client, CallbackManager); IsRunning = true; }
public void Process(SteamApps.PICSProductInfoCallback.PICSProductInfo productInfo) { ChangeNumber = productInfo.ChangeNumber; if (Settings.IsFullRun) { Log.WriteDebug("Sub Processor", "SubID: {0}", SubID); DbConnection.Execute("INSERT INTO `Changelists` (`ChangeID`) VALUES (@ChangeNumber) ON DUPLICATE KEY UPDATE `Date` = `Date`", new { productInfo.ChangeNumber }); DbConnection.Execute("INSERT INTO `ChangelistsSubs` (`ChangeID`, `SubID`) VALUES (@ChangeNumber, @SubID) ON DUPLICATE KEY UPDATE `SubID` = `SubID`", new { SubID, productInfo.ChangeNumber }); } ProcessKey("root_changenumber", "changenumber", ChangeNumber.ToString()); var appAddedToThisPackage = false; var packageOwned = LicenseList.OwnedSubs.ContainsKey(SubID); var newPackageName = productInfo.KeyValues["name"].AsString(); var apps = DbConnection.Query <PackageApp>("SELECT `AppID`, `Type` FROM `SubsApps` WHERE `SubID` = @SubID", new { SubID }).ToDictionary(x => x.AppID, x => x.Type); // TODO: Ideally this should be SteamDB Unknown Package and proper checks like app processor does if (newPackageName == null) { newPackageName = string.Concat("Steam Sub ", SubID); } if (newPackageName != null) { if (string.IsNullOrEmpty(PackageName)) { DbConnection.Execute("INSERT INTO `Subs` (`SubID`, `Name`, `LastKnownName`) VALUES (@SubID, @Name, @Name)", new { SubID, Name = newPackageName }); MakeHistory("created_sub"); MakeHistory("created_info", SteamDB.DATABASE_NAME_TYPE, string.Empty, newPackageName); } else if (!PackageName.Equals(newPackageName)) { if (newPackageName.StartsWith("Steam Sub ", StringComparison.Ordinal)) { DbConnection.Execute("UPDATE `Subs` SET `Name` = @Name WHERE `SubID` = @SubID", new { SubID, Name = newPackageName }); } else { DbConnection.Execute("UPDATE `Subs` SET `Name` = @Name, `LastKnownName` = @Name WHERE `SubID` = @SubID", new { SubID, Name = newPackageName }); } MakeHistory("modified_info", SteamDB.DATABASE_NAME_TYPE, PackageName, newPackageName); } } foreach (var section in productInfo.KeyValues.Children) { string sectionName = section.Name.ToLower(); if (string.IsNullOrEmpty(sectionName) || sectionName.Equals("packageid") || sectionName.Equals("changenumber") || sectionName.Equals("name")) { // Ignore common keys continue; } if (sectionName.Equals("appids") || sectionName.Equals("depotids")) { // Remove "ids", so we get "app" from appids and "depot" from depotids string type = sectionName.Replace("ids", string.Empty); var isAppSection = type.Equals("app"); var typeID = (uint)(isAppSection ? 0 : 1); // 0 = app, 1 = depot; can't store as string because it's in the `key` field foreach (var childrenApp in section.Children) { uint appID = uint.Parse(childrenApp.Value); // Is this appid already in this package? if (apps.ContainsKey(appID)) { // Is this appid's type the same? if (apps[appID] != type) { DbConnection.Execute("UPDATE `SubsApps` SET `Type` = @Type WHERE `SubID` = @SubID AND `AppID` = @AppID", new { SubID, AppID = appID, Type = type }); MakeHistory("added_to_sub", typeID, apps[appID].Equals("app") ? "0" : "1", childrenApp.Value); appAddedToThisPackage = true; // TODO: Log relevant add/remove history for depot/app? } apps.Remove(appID); } else { DbConnection.Execute("INSERT INTO `SubsApps` (`SubID`, `AppID`, `Type`) VALUES(@SubID, @AppID, @Type)", new { SubID, AppID = appID, Type = type }); MakeHistory("added_to_sub", typeID, string.Empty, childrenApp.Value); if (isAppSection) { DbConnection.Execute(AppProcessor.HistoryQuery, new PICSHistory { ID = appID, ChangeID = ChangeNumber, NewValue = SubID.ToString(), Action = "added_to_sub" } ); } else { DbConnection.Execute(DepotProcessor.GetHistoryQuery(), new DepotHistory { DepotID = appID, ChangeID = ChangeNumber, NewValue = SubID, Action = "added_to_sub" } ); } appAddedToThisPackage = true; if (packageOwned && !LicenseList.OwnedApps.ContainsKey(appID)) { LicenseList.OwnedApps.Add(appID, 1); } } } } else if (sectionName.Equals("extended")) { foreach (var children in section.Children) { var keyName = string.Format("{0}_{1}", sectionName, children.Name); if (children.Children.Count > 0) { ProcessKey(keyName, children.Name, Utils.JsonifyKeyValue(children), true); } else { ProcessKey(keyName, children.Name, children.Value); } } } else if (section.Children.Any()) { sectionName = string.Format("root_{0}", sectionName); ProcessKey(sectionName, sectionName, Utils.JsonifyKeyValue(section), true); } else if (!string.IsNullOrEmpty(section.Value)) { string keyName = string.Format("root_{0}", sectionName); ProcessKey(keyName, sectionName, section.Value); } } foreach (var data in CurrentData.Values.Where(data => !data.Processed && !data.KeyName.StartsWith("website", StringComparison.Ordinal))) { DbConnection.Execute("DELETE FROM `SubsInfo` WHERE `SubID` = @SubID AND `Key` = @Key", new { SubID, data.Key }); MakeHistory("removed_key", data.Key, data.Value); } var appsRemoved = apps.Any(); foreach (var app in apps) { DbConnection.Execute("DELETE FROM `SubsApps` WHERE `SubID` = @SubID AND `AppID` = @AppID AND `Type` = @Type", new { SubID, AppID = app.Key, Type = app.Value }); var isAppSection = app.Value.Equals("app"); var typeID = (uint)(isAppSection ? 0 : 1); // 0 = app, 1 = depot; can't store as string because it's in the `key` field MakeHistory("removed_from_sub", typeID, app.Key.ToString()); if (isAppSection) { DbConnection.Execute(AppProcessor.HistoryQuery, new PICSHistory { ID = app.Key, ChangeID = ChangeNumber, OldValue = SubID.ToString(), Action = "removed_from_sub" } ); } else { DbConnection.Execute(DepotProcessor.GetHistoryQuery(), new DepotHistory { DepotID = app.Key, ChangeID = ChangeNumber, OldValue = SubID, Action = "removed_from_sub" } ); } } if (appsRemoved) { LicenseList.RefreshApps(); } var billingType = (EBillingType)productInfo.KeyValues["billingtype"].AsInteger(); if (!packageOwned && (billingType == EBillingType.FreeOnDemand || (billingType == EBillingType.NoCost && SubID != 17906))) { Log.WriteDebug("Sub Processor", "Requesting apps in SubID {0} as a free license", SubID); JobManager.AddJob(() => Steam.Instance.Apps.RequestFreeLicense(productInfo.KeyValues["appids"].Children.Select(appid => (uint)appid.AsInteger()).ToList())); } // Re-queue apps in this package so we can update depots and whatnot if (appAddedToThisPackage && !Settings.IsFullRun && !string.IsNullOrEmpty(PackageName)) { JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(productInfo.KeyValues["appids"].Children.Select(x => (uint)x.AsInteger()), Enumerable.Empty <uint>())); } // Maintain a list of anonymous content if (appAddedToThisPackage && SubID == 17906) { LicenseList.RefreshAnonymous(); } }
/* * Here be dragons. */ public static void DownloadFilesFromDepot(DepotProcessor.ManifestJob job, DepotManifest depotManifest) { var randomGenerator = new Random(); var files = depotManifest.Files.Where(x => IsFileNameMatching(job.DepotID, x.FileName)).ToList(); var filesUpdated = false; var filesAnyFailed = false; var hashesFile = Path.Combine(Application.Path, "files", ".support", "hashes", string.Format("{0}.json", job.DepotID)); var hashes = new Dictionary<string, byte[]>(); if (File.Exists(hashesFile)) { hashes = JsonConvert.DeserializeObject<Dictionary<string, byte[]>>(File.ReadAllText(hashesFile)); } Log.WriteDebug("FileDownloader", "Will download {0} files from depot {1}", files.Count, job.DepotID); Parallel.ForEach(files, new ParallelOptions { MaxDegreeOfParallelism = 2 }, (file, state2) => { string directory = Path.Combine(Application.Path, "files", job.DepotID.ToString(), Path.GetDirectoryName(file.FileName)); string finalPath = Path.Combine(directory, Path.GetFileName(file.FileName)); string downloadPath = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Path.GetRandomFileName(), ".steamdb_tmp")); if (!Directory.Exists(directory)) { Directory.CreateDirectory(directory); } else if (file.TotalSize == 0) { if (File.Exists(finalPath)) { var f = new FileInfo(finalPath); if(f.Length == 0) { #if DEBUG Log.WriteDebug("FileDownloader", "{0} is already empty", file.FileName); #endif return; } } else { File.Create(finalPath); Log.WriteInfo("FileDownloader", "{0} created an empty file", file.FileName); return; } } else if(hashes.ContainsKey(file.FileName) && file.FileHash.SequenceEqual(hashes[file.FileName])) { #if DEBUG Log.WriteDebug("FileDownloader", "{0} already matches the file we have", file.FileName); #endif return; } var chunks = file.Chunks.OrderBy(x => x.Offset).ToList(); Log.WriteInfo("FileDownloader", "Downloading {0} ({1} bytes, {2} chunks)", file.FileName, file.TotalSize, chunks.Count); uint count = 0; byte[] checksum; string lastError = "or checksum failed"; string oldChunksFile; using (var sha = new SHA1Managed()) { oldChunksFile = Path.Combine(Application.Path, "files", ".support", "chunks", string.Format("{0}-{1}.json", job.DepotID, BitConverter.ToString(sha.ComputeHash(Encoding.UTF8.GetBytes(file.FileName)))) ); } using (var fs = File.Open(downloadPath, FileMode.OpenOrCreate, FileAccess.ReadWrite)) { fs.SetLength((long)file.TotalSize); var lockObject = new object(); var neededChunks = new List<DepotManifest.ChunkData>(); if (File.Exists(oldChunksFile) && File.Exists(finalPath)) { var oldChunks = JsonConvert.DeserializeObject<List<DepotManifest.ChunkData>>( File.ReadAllText(oldChunksFile), new JsonSerializerSettings { PreserveReferencesHandling = PreserveReferencesHandling.All } ); using (var fsOld = File.Open(finalPath, FileMode.Open, FileAccess.Read)) { foreach (var chunk in chunks) { var oldChunk = oldChunks.FirstOrDefault(c => c.ChunkID.SequenceEqual(chunk.ChunkID)); if (oldChunk != null) { var oldData = new byte[oldChunk.UncompressedLength]; fsOld.Seek((long)oldChunk.Offset, SeekOrigin.Begin); fsOld.Read(oldData, 0, oldData.Length); var existingChecksum = Utils.AdlerHash(oldData); if (existingChecksum.SequenceEqual(chunk.Checksum)) { fs.Seek((long)chunk.Offset, SeekOrigin.Begin); fs.Write(oldData, 0, oldData.Length); #if DEBUG Log.WriteDebug("FileDownloader", "{0} Found chunk ({1}), not downloading ({2}/{3})", file.FileName, chunk.Offset, ++count, chunks.Count); #endif } else { neededChunks.Add(chunk); #if DEBUG Log.WriteDebug("FileDownloader", "{0} Found chunk ({1}), but checksum differs", file.FileName, chunk.Offset); #endif } } else { neededChunks.Add(chunk); } } } } else { neededChunks = chunks; } Parallel.ForEach(neededChunks, new ParallelOptions { MaxDegreeOfParallelism = 3 }, (chunk, state) => { var downloaded = false; for (var i = 0; i <= 5; i++) { try { var chunkData = CDNClient.DownloadDepotChunk(job.DepotID, chunk, job.Server, job.CDNToken, job.DepotKey); lock (lockObject) { fs.Seek((long)chunk.Offset, SeekOrigin.Begin); fs.Write(chunkData.Data, 0, chunkData.Data.Length); Log.WriteDebug("FileDownloader", "Downloaded {0} ({1}/{2})", file.FileName, ++count, chunks.Count); } downloaded = true; break; } catch (Exception e) { lastError = e.Message; } // See https://developers.google.com/drive/web/handle-errors Task.Delay((1 << i) * 1000 + randomGenerator.Next(1001)).Wait(); } if (!downloaded) { state.Stop(); } }); fs.Seek(0, SeekOrigin.Begin); using (var sha = new SHA1Managed()) { checksum = sha.ComputeHash(fs); } } if (file.FileHash.SequenceEqual(checksum)) { Log.WriteInfo("FileDownloader", "Downloaded {0} from {1}", file.FileName, Steam.GetAppName(job.ParentAppID)); hashes[file.FileName] = checksum; if (File.Exists(finalPath)) { File.Delete(finalPath); } File.Move(downloadPath, finalPath); if (chunks.Count > 1) { File.WriteAllText(oldChunksFile, JsonConvert.SerializeObject( chunks, Formatting.None, new JsonSerializerSettings { PreserveReferencesHandling = PreserveReferencesHandling.All } ) ); } else if (File.Exists(oldChunksFile)) { File.Delete(oldChunksFile); } filesUpdated = true; } else { filesAnyFailed = true; IRC.Instance.SendOps("{0}[{1}]{2} Failed to download {3}: Only {4} out of {5} chunks downloaded ({6})", Colors.OLIVE, Steam.GetAppName(job.ParentAppID), Colors.NORMAL, file.FileName, count, chunks.Count, lastError); Log.WriteError("FileDownloader", "Failed to download {0}: Only {1} out of {2} chunks downloaded from {3} ({4})", file.FileName, count, chunks.Count, job.Server, lastError); File.Delete(downloadPath); } }); if (filesUpdated) { if (filesAnyFailed) { IRC.Instance.SendOps("{0}[{1}]{2} Failed to download some files, not running update script to prevent broken diffs.", Colors.OLIVE, Steam.GetAppName(job.ParentAppID), Colors.NORMAL); return; } File.WriteAllText(hashesFile, JsonConvert.SerializeObject(hashes)); var updateScript = Path.Combine(Application.Path, "files", "update.sh"); if (File.Exists(updateScript)) { lock (updateLock) { // YOLO Process.Start(updateScript, job.DepotID.ToString()); } } } }
private void TryProcess(SteamApps.PICSProductInfoCallback.PICSProductInfo productInfo) { using (MySqlDataReader Reader = DbWorker.ExecuteReader("SELECT `Name`, `Value` FROM `AppsInfo` INNER JOIN `KeyNames` ON `AppsInfo`.`Key` = `KeyNames`.`ID` WHERE `AppID` = @AppID", new MySqlParameter("AppID", AppID))) { while (Reader.Read()) { CurrentData.Add(DbWorker.GetString("Name", Reader), DbWorker.GetString("Value", Reader)); } } string appName = string.Empty; string appType = "0"; using (MySqlDataReader Reader = DbWorker.ExecuteReader("SELECT `Name`, `AppType` FROM `Apps` WHERE `AppID` = @AppID LIMIT 1", new MySqlParameter("AppID", AppID))) { if (Reader.Read()) { appName = DbWorker.GetString("Name", Reader); appType = DbWorker.GetString("AppType", Reader); } } if (productInfo.KeyValues["common"]["name"].Value != null) { string newAppType = "0"; string currentType = productInfo.KeyValues["common"]["type"].AsString().ToLower(); using (MySqlDataReader Reader = DbWorker.ExecuteReader("SELECT `AppType` FROM `AppsTypes` WHERE `Name` = @Type LIMIT 1", new MySqlParameter("Type", currentType))) { if (Reader.Read()) { newAppType = DbWorker.GetString("AppType", Reader); } else { // TODO: Create it? Log.WriteError("App Processor", "AppID {0} - unknown app type: {1}", AppID, currentType); // TODO: This is debuggy just so we are aware of new app types IRC.SendAnnounce("Unknown app type \"{0}\" for appid {1}, cc Alram and xPaw", currentType, AppID); } } if (string.IsNullOrEmpty(appName) || appName.StartsWith(SteamDB.UNKNOWN_APP, StringComparison.Ordinal)) { DbWorker.ExecuteNonQuery("INSERT INTO `Apps` (`AppID`, `AppType`, `Name`) VALUES (@AppID, @Type, @AppName) ON DUPLICATE KEY UPDATE `Name` = @AppName, `AppType` = @Type", new MySqlParameter("@AppID", AppID), new MySqlParameter("@Type", newAppType), new MySqlParameter("@AppName", productInfo.KeyValues["common"]["name"].Value) ); MakeHistory("created_app"); MakeHistory("created_info", DATABASE_NAME_TYPE, string.Empty, productInfo.KeyValues["common"]["name"].Value); // TODO: Testy testy if (!Settings.IsFullRun && Settings.Current.ChatRooms.Count > 0 && !appName.StartsWith("SteamApp", StringComparison.Ordinal) && !appName.StartsWith("ValveTest", StringComparison.Ordinal)) { Steam.Instance.Friends.SendChatRoomMessage(Settings.Current.ChatRooms[0], EChatEntryType.ChatMsg, string.Format( ":retreat: New {0} was published: {1}\nSteamDB: {2}\nSteam: http://store.steampowered.com/app/{3}/", currentType, productInfo.KeyValues["common"]["name"].AsString(), SteamDB.GetAppURL(AppID), AppID ) ); } } else if (!appName.Equals(productInfo.KeyValues["common"]["name"].Value)) { string newAppName = productInfo.KeyValues["common"]["name"].AsString(); DbWorker.ExecuteNonQuery("UPDATE `Apps` SET `Name` = @AppName WHERE `AppID` = @AppID", new MySqlParameter("@AppID", AppID), new MySqlParameter("@AppName", newAppName) ); MakeHistory("modified_info", DATABASE_NAME_TYPE, appName, newAppName); // TODO: Testy testy if (!Settings.IsFullRun && Settings.Current.ChatRooms.Count > 0 && !string.Equals(appName, newAppName, StringComparison.OrdinalIgnoreCase) && !newAppName.StartsWith("SteamApp", StringComparison.Ordinal) && !newAppName.StartsWith("ValveTest", StringComparison.Ordinal)) { Steam.Instance.Friends.SendChatRoomMessage(Settings.Current.ChatRooms[0], EChatEntryType.ChatMsg, string.Format( ":retreat: {0} name was changed - {1}\n« {2}\n» {3}", currentType, SteamDB.GetAppURL(AppID, "history"), appName, newAppName ) ); } } if (appType.Equals("0")) { DbWorker.ExecuteNonQuery("UPDATE `Apps` SET `AppType` = @Type WHERE `AppID` = @AppID", new MySqlParameter("@AppID", AppID), new MySqlParameter("@Type", newAppType) ); MakeHistory("created_info", DATABASE_APPTYPE, string.Empty, newAppType); } else if (!appType.Equals(newAppType)) { DbWorker.ExecuteNonQuery("UPDATE `Apps` SET `AppType` = @Type WHERE `AppID` = @AppID", new MySqlParameter("@AppID", AppID), new MySqlParameter("@Type", newAppType) ); MakeHistory("modified_info", DATABASE_APPTYPE, appType, newAppType); } } // If we are full running, process depots too bool depotsSectionModified = Settings.IsFullRun && productInfo.KeyValues["depots"].Children.Count > 0; foreach (KeyValue section in productInfo.KeyValues.Children) { string sectionName = section.Name.ToLower(); if (sectionName == "appid" || sectionName == "public_only") { continue; } if (sectionName == "change_number") // Carefully handle change_number { sectionName = "root_change_number"; // TODO: Remove this key, move it to Apps table itself ProcessKey(sectionName, "change_number", productInfo.ChangeNumber.ToString()); //section.AsString()); } else if (sectionName == "common" || sectionName == "extended") { string keyName; foreach (KeyValue keyvalue in section.Children) { keyName = string.Format("{0}_{1}", sectionName, keyvalue.Name); if (keyName.Equals("common_type") || keyName.Equals("common_gameid") || keyName.Equals("common_name") || keyName.Equals("extended_order")) { // Ignore common keys that are either duplicated or serve no real purpose continue; } if (keyvalue.Children.Count > 0) { if (keyName.Equals("common_languages")) { ProcessKey(keyName, keyvalue.Name, string.Join(",", keyvalue.Children.Select(x => x.Name))); } else { ProcessKey(keyName, keyvalue.Name, DbWorker.JsonifyKeyValue(keyvalue), true); } } else if (!string.IsNullOrEmpty(keyvalue.Value)) { ProcessKey(keyName, keyvalue.Name, keyvalue.Value); } } } else { sectionName = string.Format("root_{0}", sectionName); if (ProcessKey(sectionName, sectionName, DbWorker.JsonifyKeyValue(section), true) && sectionName.Equals("root_depots")) { depotsSectionModified = true; } } } foreach (string keyName in CurrentData.Keys) { if (!keyName.StartsWith("website", StringComparison.Ordinal)) { uint ID = GetKeyNameID(keyName); DbWorker.ExecuteNonQuery("DELETE FROM AppsInfo WHERE `AppID` = @AppID AND `Key` = @KeyNameID", new MySqlParameter("@AppID", AppID), new MySqlParameter("@KeyNameID", ID) ); MakeHistory("removed_key", ID, CurrentData[keyName]); } } if (productInfo.KeyValues["common"]["name"].Value == null) { if (string.IsNullOrEmpty(appName)) // We don't have the app in our database yet { DbWorker.ExecuteNonQuery("INSERT INTO `Apps` (`AppID`, `Name`) VALUES (@AppID, @AppName) ON DUPLICATE KEY UPDATE `AppType` = `AppType`", new MySqlParameter("@AppID", AppID), new MySqlParameter("@AppName", string.Format("{0} {1}", SteamDB.UNKNOWN_APP, AppID)) ); } else if (!appName.StartsWith(SteamDB.UNKNOWN_APP, StringComparison.Ordinal)) // We do have the app, replace it with default name { DbWorker.ExecuteNonQuery("UPDATE `Apps` SET `Name` = @AppName, `AppType` = 0 WHERE `AppID` = @AppID", new MySqlParameter("@AppID", AppID), new MySqlParameter("@AppName", string.Format("{0} {1}", SteamDB.UNKNOWN_APP, AppID)) ); MakeHistory("deleted_app", 0, appName); } } if (depotsSectionModified) { DepotProcessor.Process(AppID, ChangeNumber, productInfo.KeyValues["depots"]); } }
public void Init() { ProcessorPool = new SmartThreadPool(new STPStartInfo { WorkItemPriority = WorkItemPriority.Highest, MaxWorkerThreads = 50 }); SecondaryPool = new SmartThreadPool(); ProcessorPool.Name = "Processor Pool"; SecondaryPool.Name = "Secondary Pool"; OwnedPackages = new List <uint>(); ProcessedApps = new ConcurrentDictionary <uint, IWorkItemResult>(); ProcessedSubs = new ConcurrentDictionary <uint, IWorkItemResult>(); Timer = new System.Timers.Timer(); Timer.Elapsed += OnTimer; Timer.Interval = TimeSpan.FromSeconds(1).TotalMilliseconds; Client = new SteamClient(); User = Client.GetHandler <SteamUser>(); Apps = Client.GetHandler <SteamApps>(); Friends = Client.GetHandler <SteamFriends>(); UserStats = Client.GetHandler <SteamUserStats>(); CallbackManager = new CallbackManager(Client); CallbackManager.Register(new Callback <SteamClient.ConnectedCallback>(OnConnected)); CallbackManager.Register(new Callback <SteamClient.DisconnectedCallback>(OnDisconnected)); CallbackManager.Register(new Callback <SteamUser.LoggedOnCallback>(OnLoggedOn)); CallbackManager.Register(new Callback <SteamUser.LoggedOffCallback>(OnLoggedOff)); CallbackManager.Register(new Callback <SteamApps.LicenseListCallback>(OnLicenseListCallback)); CallbackManager.Register(new JobCallback <SteamUser.UpdateMachineAuthCallback>(OnMachineAuth)); CallbackManager.Register(new JobCallback <SteamApps.PICSProductInfoCallback>(OnPICSProductInfo)); CallbackManager.Register(new JobCallback <SteamApps.PICSTokensCallback>(OnPICSTokens)); if (Settings.IsFullRun) { CallbackManager.Register(new JobCallback <SteamApps.PICSChangesCallback>(OnPICSChangesFullRun)); } else { CallbackManager.Register(new JobCallback <SteamApps.PICSChangesCallback>(OnPICSChanges)); CallbackManager.Register(new JobCallback <SteamUserStats.NumberOfPlayersCallback>(SteamProxy.Instance.OnNumberOfPlayers)); CallbackManager.Register(new Callback <SteamFriends.ClanStateCallback>(SteamProxy.Instance.OnClanState)); CallbackManager.Register(new Callback <SteamFriends.ChatMsgCallback>(SteamProxy.Instance.OnChatMessage)); CallbackManager.Register(new Callback <SteamFriends.ChatMemberInfoCallback>(SteamProxy.Instance.OnChatMemberInfo)); CallbackManager.Register(new Callback <SteamUser.MarketingMessageCallback>(MarketingHandler.OnMarketingMessage)); CallbackManager.Register(new Callback <SteamUser.AccountInfoCallback>(OnAccountInfo)); } DepotProcessor.Init(); GetLastChangeNumber(); IsRunning = true; Client.Connect(); while (IsRunning) { CallbackManager.RunWaitCallbacks(TimeSpan.FromSeconds(1)); } }
public static void DownloadFilesFromDepot(DepotProcessor.ManifestJob job, DepotManifest depotManifest) { var files = depotManifest.Files.Where(x => IsFileNameMatching(job.DepotID, x.FileName)).ToList(); var filesUpdated = false; Log.WriteDebug("FileDownloader", "Will download {0} files from depot {1}", files.Count(), job.DepotID); foreach (var file in files) { string directory = Path.Combine(Application.Path, FILES_DIRECTORY, job.DepotID.ToString(), Path.GetDirectoryName(file.FileName)); string finalPath = Path.Combine(directory, Path.GetFileName(file.FileName)); string downloadPath = Path.GetTempFileName(); if (!Directory.Exists(directory)) { Directory.CreateDirectory(directory); } else if (File.Exists(finalPath)) { using (var fs = File.Open(finalPath, FileMode.Open)) { using (var sha = new SHA1Managed()) { if (file.FileHash.SequenceEqual(sha.ComputeHash(fs))) { Log.WriteDebug("FileDownloader", "{0} already matches the file we have", file.FileName); continue; } } } } Log.WriteInfo("FileDownloader", "Downloading {0} ({1} bytes, {2} chunks)", file.FileName, file.TotalSize, file.Chunks.Count); uint count = 0; byte[] checksum; string lastError = "or checksum failed"; using (var fs = File.Open(downloadPath, FileMode.OpenOrCreate)) { fs.SetLength((long)file.TotalSize); var lockObject = new object(); // TODO: We *could* verify each chunk and only download needed ones Parallel.ForEach(file.Chunks, (chunk, state) => { var downloaded = false; for (var i = 0; i <= 5; i++) { try { var chunkData = CDNClient.DownloadDepotChunk(job.DepotID, chunk, job.Server, job.CDNToken, job.DepotKey); lock (lockObject) { fs.Seek((long)chunk.Offset, SeekOrigin.Begin); fs.Write(chunkData.Data, 0, chunkData.Data.Length); Log.WriteDebug("FileDownloader", "Downloaded {0} ({1}/{2})", file.FileName, ++count, file.Chunks.Count); } downloaded = true; break; } catch (Exception e) { lastError = e.Message; } } if (!downloaded) { state.Stop(); } }); fs.Seek(0, SeekOrigin.Begin); using (var sha = new SHA1Managed()) { checksum = sha.ComputeHash(fs); } } if (file.Chunks.Count == 0 || file.FileHash.SequenceEqual(checksum)) { Log.WriteInfo("FileDownloader", "Downloaded {0} from {1}", file.FileName, Steam.GetAppName(job.ParentAppID)); if (File.Exists(finalPath)) { File.Delete(finalPath); } File.Move(downloadPath, finalPath); filesUpdated = true; } else { IRC.Instance.SendOps("{0}[{1}]{2} Failed to download {3}: Only {4} out of {5} chunks downloaded ({6})", Colors.OLIVE, Steam.GetAppName(job.ParentAppID), Colors.NORMAL, file.FileName, count, file.Chunks.Count, lastError); Log.WriteError("FileDownloader", "Failed to download {0}: Only {1} out of {2} chunks downloaded from {3} ({4})", file.FileName, count, file.Chunks.Count, job.Server, lastError); File.Delete(downloadPath); } } if (filesUpdated) { var updateScript = Path.Combine(Application.Path, "files", "update.sh"); if (File.Exists(updateScript)) { // YOLO Process.Start(updateScript, job.DepotID.ToString()); } } }