static async Task <DepotDownloadInfo> GetDepotInfoAsync(uint depotId, uint appId, ulong manifestId, string branch) { string contentName = await GetAppOrDepotName(depotId, appId).ConfigureAwait(false); // Skip requesting an app ticket //SteamSession.AppTickets[depotId] = null; uint uVersion = await GetSteam3AppBuildNumber(appId, branch).ConfigureAwait(false); string installDir; if (!CreateDirectories(depotId, uVersion, out installDir)) { throw new ContentDownloaderException("Could not create install directories."); } byte[] depotKey = await SteamSession.RequestDepotKey(depotId, appId).ConfigureAwait(false); if (depotKey == null) { throw new ContentDownloaderException($"No valid depot key for {depotId}, unable to download."); } var info = new DepotDownloadInfo(depotId, manifestId, installDir, contentName); info.DepotKey = depotKey; return(info); }
static DepotDownloadInfo GetDepotInfo(uint depotId, uint appId, string branch) { if (steam3 != null && appId != INVALID_APP_ID) { steam3.RequestAppInfo(( uint )appId); } string contentName = GetAppOrDepotName(depotId, appId); if (!AccountHasAccess(depotId)) { Console.WriteLine("Depot {0} ({1}) is not available from this account.", depotId, contentName); return(null); } if (steam3 != null) { steam3.RequestAppTicket(( uint )depotId); } ulong manifestID = GetSteam3DepotManifest(depotId, appId, branch); if (manifestID == INVALID_MANIFEST_ID && branch != "public") { Console.WriteLine("Warning: Depot {0} does not have branch named \"{1}\". Trying public branch.", depotId, branch); branch = "public"; manifestID = GetSteam3DepotManifest(depotId, appId, branch); } if (manifestID == INVALID_MANIFEST_ID) { Console.WriteLine("Depot {0} ({1}) missing public subsection or manifest section.", depotId, contentName); return(null); } uint uVersion = GetSteam3AppBuildNumber(appId, branch); string installDir; if (!CreateDirectories(depotId, uVersion, out installDir)) { Console.WriteLine("Error: Unable to create install directories!"); return(null); } steam3.RequestDepotKey(depotId, appId); if (!steam3.DepotKeys.ContainsKey(depotId)) { Console.WriteLine("No valid depot key for {0}, unable to download.", depotId); return(null); } byte[] depotKey = steam3.DepotKeys[depotId]; var info = new DepotDownloadInfo(depotId, manifestID, installDir, contentName); info.depotKey = depotKey; return(info); }
private static List <CDNClient> CollectCDNClientsForDepot(DepotDownloadInfo depot) { var cdnClients = new List <CDNClient>(); CDNClient initialClient = new CDNClient(steam3.steamClient, depot.id, steam3.AppTickets[depot.id], depot.depotKey); var cdnServers = initialClient.FetchServerList(cellId: (uint)Config.CellID); // Grab up to the first eight server in the allegedly best-to-worst order from Steam var limit = cdnServers.Take(Config.MaxServers); int tries = 0; foreach (var s in limit) { CDNClient c; if (tries == 0) { c = initialClient; } else { c = new CDNClient(steam3.steamClient, depot.id, steam3.AppTickets[depot.id], depot.depotKey); } try { c.Connect(s); cdnClients.Add(c); } catch { Console.WriteLine("\nFailed to connect to content server {0}. Remaining content servers for depot: {1}.", s, Config.MaxServers - tries - 1); } tries++; } if (cdnClients.Count == 0) { Console.WriteLine("\nUnable to find any content servers for depot {0} - {1}", depot.id, depot.contentName); } return(cdnClients); }
public static void DownloadApp(uint appId, uint depotId, string branch, bool forceDepot = false) { if (steam3 != null) { steam3.RequestAppInfo(appId); } if (!AccountHasAccess(appId)) { if (steam3.RequestFreeAppLicense(appId)) { Console.WriteLine("Obtained FreeOnDemand license for app {0}", appId); } else { string contentName = GetAppOrDepotName(INVALID_DEPOT_ID, appId); Console.WriteLine("App {0} ({1}) is not available from this account.", appId, contentName); return; } } Console.WriteLine("Using app branch: '{0}'.", branch); var depotIDs = new List <uint>(); KeyValue depots = GetSteam3AppSection(appId, EAppInfoSection.Depots); if (forceDepot) { depotIDs.Add(depotId); } else { if (depots != null) { foreach (var depotSection in depots.Children) { uint id = INVALID_DEPOT_ID; if (depotSection.Children.Count == 0) { continue; } if (!uint.TryParse(depotSection.Name, out id)) { continue; } if (depotId != INVALID_DEPOT_ID && id != depotId) { continue; } if (!Config.DownloadAllPlatforms) { var depotConfig = depotSection["config"]; if (depotConfig != KeyValue.Invalid && depotConfig["oslist"] != KeyValue.Invalid && !string.IsNullOrWhiteSpace(depotConfig["oslist"].Value)) { var oslist = depotConfig["oslist"].Value.Split(','); if (Array.IndexOf(oslist, Util.GetSteamOS()) == -1) { continue; } } } depotIDs.Add(id); } } if (depotIDs == null || (depotIDs.Count == 0 && depotId == INVALID_DEPOT_ID)) { Console.WriteLine("Couldn't find any depots to download for app {0}", appId); return; } else if (depotIDs.Count == 0) { Console.Write("Depot {0} not listed for app {1}", depotId, appId); if (!Config.DownloadAllPlatforms) { Console.Write(" or not available on this platform"); } Console.WriteLine(); return; } } var infos = new List <DepotDownloadInfo>(); foreach (var depot in depotIDs) { DepotDownloadInfo info = GetDepotInfo(depot, appId, branch); if (info != null) { infos.Add(info); } } try { DownloadSteam3(appId, infos); } catch (OperationCanceledException) { Console.WriteLine("App {0} was not completely downloaded.", appId); } }
static DepotDownloadInfo GetDepotInfo(uint depotId, uint appId, ulong manifestId, string branch) { if (steam3 != null && appId != INVALID_APP_ID) { steam3.RequestAppInfo(( uint )appId); } string contentName = GetAppOrDepotName(depotId, appId); // Skip requesting an app ticket steam3.AppTickets[depotId] = null; if (manifestId == INVALID_MANIFEST_ID) { manifestId = GetSteam3DepotManifest(depotId, appId, branch); if (manifestId == INVALID_MANIFEST_ID && branch != "public") { Console.WriteLine("Warning: Depot {0} does not have branch named \"{1}\". Trying public branch.", depotId, branch); branch = "public"; manifestId = GetSteam3DepotManifest(depotId, appId, branch); } if (manifestId == INVALID_MANIFEST_ID) { Console.WriteLine("Depot {0} ({1}) missing public subsection or manifest section.", depotId, contentName); return(null); } } uint uVersion = GetSteam3AppBuildNumber(appId, branch); string installDir; if (!CreateDirectories(appId, appName, depotId, uVersion, contentName, out installDir)) { Console.WriteLine("Error: Unable to create install directories!"); return(null); } if (!DepotKeyStore.ContainsKey(depotId) && !AccountHasAccess(depotId)) { Console.WriteLine("Depot {0} ({1}) is not available from this account and no key found in depot key store.", depotId, contentName); return(null); } byte[] depotKey; if (DepotKeyStore.ContainsKey(depotId)) { depotKey = DepotKeyStore.Get(depotId); } else { steam3.RequestDepotKey(depotId, appId); if (!steam3.DepotKeys.ContainsKey(depotId)) { Console.WriteLine("No valid depot key for {0}, unable to download.", depotId); return(null); } depotKey = steam3.DepotKeys[depotId]; } var info = new DepotDownloadInfo(depotId, manifestId, installDir, contentName); info.depotKey = depotKey; File.WriteAllText($"depots\\{depotId}.key", BitConverter.ToString(depotKey).Replace("-", "")); return(info); }
static DepotDownloadInfo GetDepotInfo(int depotId, int appId, string branch) { if(steam3 != null && appId > 0) steam3.RequestAppInfo((uint)appId); string contentName = GetAppOrDepotName(depotId, appId); if (!AccountHasAccess(depotId, false)) { Console.WriteLine("Depot {0} ({1}) is not available from this account.", depotId, contentName); return null; } uint uVersion = GetSteam3AppBuildNumber(appId, branch); string installDir; if (!CreateDirectories(depotId, uVersion, out installDir)) { Console.WriteLine("Error: Unable to create install directories!"); return null; } if(steam3 != null) steam3.RequestAppTicket((uint)depotId); ulong manifestID = GetSteam3DepotManifest(depotId, appId, branch); if (manifestID == 0) { Console.WriteLine("Depot {0} ({1}) missing public subsection or manifest section.", depotId, contentName); return null; } steam3.RequestDepotKey( ( uint )depotId, ( uint )appId ); if (!steam3.DepotKeys.ContainsKey((uint)depotId)) { Console.WriteLine("No valid depot key for {0}, unable to download.", depotId); return null; } byte[] depotKey = steam3.DepotKeys[(uint)depotId]; var info = new DepotDownloadInfo( depotId, manifestID, installDir, contentName ); info.depotKey = depotKey; return info; }
static async Task <DepotDownloadInfo> GetDepotInfo(Steam3Session steam3, uint depotId, uint appId, ulong manifestId, string branch) { if (steam3 != null && appId != INVALID_APP_ID) { await steam3.RequestAppInfo(appId); } string contentName = GetAppOrDepotName(steam3, depotId, appId); bool hasAccess = await AccountHasAccess(steam3, depotId); if (!hasAccess) { DebugLog.WriteLine("ContentDownloader", "Depot " + depotId + " (" + contentName + ") is not available from this account."); return(null); } // Skip requesting an app ticket steam3.AppTickets[depotId] = null; if (manifestId == INVALID_MANIFEST_ID) { manifestId = await GetSteam3DepotManifest(steam3, depotId, appId, branch); if (manifestId == INVALID_MANIFEST_ID && branch != "public") { DebugLog.WriteLine("ContentDownloader", "Warning: Depot " + depotId + " does not have branch named \"" + branch + "\". Trying public branch."); branch = "public"; manifestId = await GetSteam3DepotManifest(steam3, depotId, appId, branch); } if (manifestId == INVALID_MANIFEST_ID) { DebugLog.WriteLine("ContentDownloader", "Depot " + depotId + " (" + contentName + ") missing public subsection or manifest section."); return(null); } } uint uVersion = GetSteam3AppBuildNumber(steam3, appId, branch); string installDir; if (!CreateDirectories(depotId, uVersion, out installDir)) { DebugLog.WriteLine("ContentDownloader", "Error: Unable to create install directories!"); return(null); } await steam3.RequestDepotKey(depotId, appId); if (!steam3.DepotKeys.ContainsKey(depotId)) { DebugLog.WriteLine("ContentDownloader", "No valid depot key for " + depotId + ", unable to download."); return(null); } byte[] depotKey = steam3.DepotKeys[depotId]; var info = new DepotDownloadInfo(depotId, manifestId, installDir, contentName); info.depotKey = depotKey; return(info); }
private static BlockingCollection <CDNClient> CollectCDNClientsForDepot(DepotDownloadInfo depot) { Console.WriteLine("Finding content servers..."); var cdnClients = new BlockingCollection <CDNClient>(); CDNClient initialClient = new CDNClient(steam3.steamClient, steam3.AppTickets[depot.id]); List <CDNClient.Server> cdnServers = null; while (true) { try { cdnServers = initialClient.FetchServerList(cellId: (uint)Config.CellID); if (cdnServers != null) { break; } } catch (WebException) { Console.WriteLine("\nFailed to retrieve content server list."); Thread.Sleep(500); } } if (cdnServers == null) { Console.WriteLine("\nUnable to query any content servers for depot {0} - {1}", depot.id, depot.contentName); return(cdnClients); } var weightedCdnServers = cdnServers.Select(x => { int penalty = 0; ConfigStore.TheConfig.ContentServerPenalty.TryGetValue(x.Host, out penalty); return(Tuple.Create(x, penalty)); }).OrderBy(x => x.Item2).ThenBy(x => x.Item1.WeightedLoad); // Grab up to the first eight server in the allegedly best-to-worst order from Steam Parallel.ForEach(weightedCdnServers, new ParallelOptions { MaxDegreeOfParallelism = 2 }, (serverTuple, parallelLoop) => { var server = serverTuple.Item1; CDNClient c = new CDNClient(steam3.steamClient, steam3.AppTickets[depot.id]); try { for (int i = 0; i < server.NumEntries; i++) { c.Connect(server); string cdnAuthToken = null; if (server.Type == "CDN") { steam3.RequestCDNAuthToken(depot.id, server.Host); cdnAuthToken = steam3.CDNAuthTokens[Tuple.Create(depot.id, server.Host)].Token; } c.AuthenticateDepot(depot.id, depot.depotKey, cdnAuthToken); cdnClients.Add(c); } if (cdnClients.Count >= Config.MaxServers) { parallelLoop.Stop(); } } catch (Exception ex) { int penalty = 0; ConfigStore.TheConfig.ContentServerPenalty.TryGetValue(server.Host, out penalty); ConfigStore.TheConfig.ContentServerPenalty[server.Host] = penalty + 1; Console.WriteLine("\nFailed to connect to content server {0}: {1}", server, ex.Message); } }); if (cdnClients.Count == 0) { Console.WriteLine("\nUnable to find any content servers for depot {0} - {1}", depot.id, depot.contentName); } Config.MaxDownloads = Math.Min(Config.MaxDownloads, cdnClients.Count); return(cdnClients); }
private static List<CDNClient> CollectCDNClientsForDepot(DepotDownloadInfo depot) { var cdnClients = new List<CDNClient>(); CDNClient initialClient = new CDNClient(steam3.steamClient, steam3.AppTickets[depot.id]); var cdnServers = initialClient.FetchServerList(cellId: (uint)Config.CellID); // Grab up to the first eight server in the allegedly best-to-worst order from Steam var limit = cdnServers.Take(Config.MaxServers); int tries = 0; foreach (var s in limit) { CDNClient c; if (tries == 0) { c = initialClient; } else { c = new CDNClient(steam3.steamClient, steam3.AppTickets[depot.id]); } try { c.Connect(s); cdnClients.Add(c); } catch { Console.WriteLine("\nFailed to connect to content server {0}. Remaining content servers for depot: {1}.", s, Config.MaxServers - tries - 1); } tries++; } if (cdnClients.Count == 0) { Console.WriteLine("\nUnable to find any content servers for depot {0} - {1}", depot.id, depot.contentName); } return cdnClients; }
private static async Task <DepotFilesData> ProcessDepotManifestAndFiles(CancellationTokenSource cts, uint appId, DepotDownloadInfo depot) { DepotDownloadCounter depotCounter = new DepotDownloadCounter(); Console.WriteLine("Processing depot {0} - {1}", depot.id, depot.contentName); ProtoManifest oldProtoManifest = null; ProtoManifest newProtoManifest = null; string configDir = Path.Combine(depot.installDir, CONFIG_DIR); ulong lastManifestId = INVALID_MANIFEST_ID; DepotConfigStore.Instance.InstalledManifestIDs.TryGetValue(depot.id, out lastManifestId); // In case we have an early exit, this will force equiv of verifyall next run. DepotConfigStore.Instance.InstalledManifestIDs[depot.id] = INVALID_MANIFEST_ID; DepotConfigStore.Save(); if (lastManifestId != INVALID_MANIFEST_ID) { var oldManifestFileName = Path.Combine(configDir, string.Format("{0}_{1}.bin", depot.id, lastManifestId)); if (File.Exists(oldManifestFileName)) { byte[] expectedChecksum, currentChecksum; try { expectedChecksum = File.ReadAllBytes(oldManifestFileName + ".sha"); } catch (IOException) { expectedChecksum = null; } oldProtoManifest = ProtoManifest.LoadFromFile(oldManifestFileName, out currentChecksum); if (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum)) { // We only have to show this warning if the old manifest ID was different if (lastManifestId != depot.manifestId) { Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", lastManifestId); } oldProtoManifest = null; } } } if (lastManifestId == depot.manifestId && oldProtoManifest != null) { newProtoManifest = oldProtoManifest; Console.WriteLine("Already have manifest {0} for depot {1}.", depot.manifestId, depot.id); } else { var newManifestFileName = Path.Combine(configDir, string.Format("{0}_{1}.bin", depot.id, depot.manifestId)); if (newManifestFileName != null) { byte[] expectedChecksum, currentChecksum; try { expectedChecksum = File.ReadAllBytes(newManifestFileName + ".sha"); } catch (IOException) { expectedChecksum = null; } newProtoManifest = ProtoManifest.LoadFromFile(newManifestFileName, out currentChecksum); if (newProtoManifest != null && (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum))) { Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", depot.manifestId); newProtoManifest = null; } } if (newProtoManifest != null) { Console.WriteLine("Already have manifest {0} for depot {1}.", depot.manifestId, depot.id); } else { Console.Write("Downloading depot manifest..."); DepotManifest depotManifest = null; do { cts.Token.ThrowIfCancellationRequested(); CDNClient.Server connection = null; try { connection = cdnPool.GetConnection(cts.Token); var cdnToken = await cdnPool.AuthenticateConnection(appId, depot.id, connection); depotManifest = await cdnPool.CDNClient.DownloadManifestAsync(depot.id, depot.manifestId, connection, cdnToken, depot.depotKey).ConfigureAwait(false); cdnPool.ReturnConnection(connection); } catch (TaskCanceledException) { Console.WriteLine("Connection timeout downloading depot manifest {0} {1}", depot.id, depot.manifestId); } catch (SteamKitWebRequestException e) { cdnPool.ReturnBrokenConnection(connection); if (e.StatusCode == HttpStatusCode.Unauthorized || e.StatusCode == HttpStatusCode.Forbidden) { Console.WriteLine("Encountered 401 for depot manifest {0} {1}. Aborting.", depot.id, depot.manifestId); break; } else { Console.WriteLine("Encountered error downloading depot manifest {0} {1}: {2}", depot.id, depot.manifestId, e.StatusCode); } } catch (OperationCanceledException) { break; } catch (Exception e) { cdnPool.ReturnBrokenConnection(connection); Console.WriteLine("Encountered error downloading manifest for depot {0} {1}: {2}", depot.id, depot.manifestId, e.Message); } }while (depotManifest == null); if (depotManifest == null) { Console.WriteLine("\nUnable to download manifest {0} for depot {1}", depot.manifestId, depot.id); cts.Cancel(); } // Throw the cancellation exception if requested so that this task is marked failed cts.Token.ThrowIfCancellationRequested(); byte[] checksum; newProtoManifest = new ProtoManifest(depotManifest, depot.manifestId); newProtoManifest.SaveToFile(newManifestFileName, out checksum); File.WriteAllBytes(newManifestFileName + ".sha", checksum); Console.WriteLine(" Done!"); } } newProtoManifest.Files.Sort((x, y) => string.Compare(x.FileName, y.FileName, StringComparison.Ordinal)); Console.WriteLine("Manifest {0} ({1})", depot.manifestId, newProtoManifest.CreationTime); if (Config.DownloadManifestOnly) { StringBuilder manifestBuilder = new StringBuilder(); string txtManifest = Path.Combine(depot.installDir, string.Format("manifest_{0}_{1}.txt", depot.id, depot.manifestId)); manifestBuilder.Append(string.Format("{0}\n\n", newProtoManifest.CreationTime)); foreach (var file in newProtoManifest.Files) { if (file.Flags.HasFlag(EDepotFileFlag.Directory)) { continue; } manifestBuilder.Append(string.Format("{0}\n", file.FileName)); manifestBuilder.Append(string.Format("\t{0}\n", file.TotalSize)); manifestBuilder.Append(string.Format("\t{0}\n", BitConverter.ToString(file.FileHash).Replace("-", ""))); } File.WriteAllText(txtManifest, manifestBuilder.ToString()); return(null); } string stagingDir = Path.Combine(depot.installDir, STAGING_DIR); var filesAfterExclusions = newProtoManifest.Files.AsParallel().Where(f => TestIsFileIncluded(f.FileName)).ToList(); var allFileNames = new HashSet <string>(filesAfterExclusions.Count); // Pre-process filesAfterExclusions.ForEach(file => { allFileNames.Add(file.FileName); var fileFinalPath = Path.Combine(depot.installDir, file.FileName); var fileStagingPath = Path.Combine(stagingDir, file.FileName); if (file.Flags.HasFlag(EDepotFileFlag.Directory)) { Directory.CreateDirectory(fileFinalPath); Directory.CreateDirectory(fileStagingPath); } else { // Some manifests don't explicitly include all necessary directories Directory.CreateDirectory(Path.GetDirectoryName(fileFinalPath)); Directory.CreateDirectory(Path.GetDirectoryName(fileStagingPath)); depotCounter.CompleteDownloadSize += file.TotalSize; } }); return(new DepotFilesData { depotDownloadInfo = depot, depotCounter = depotCounter, stagingDir = stagingDir, manifest = newProtoManifest, previousManifest = oldProtoManifest, filteredFiles = filesAfterExclusions, allFileNames = allFileNames }); }
private static BlockingCollection<CDNClient> CollectCDNClientsForDepot(DepotDownloadInfo depot) { Console.WriteLine("Finding content servers..."); var cdnClients = new BlockingCollection<CDNClient>(); CDNClient initialClient = new CDNClient( steam3.steamClient, steam3.AppTickets[depot.id] ); List<CDNClient.Server> cdnServers = null; while(true) { try { cdnServers = initialClient.FetchServerList( cellId: (uint)Config.CellID ); if (cdnServers != null) break; } catch (WebException) { Console.WriteLine("\nFailed to retrieve content server list."); Thread.Sleep(500); } } if (cdnServers == null) { Console.WriteLine("\nUnable to query any content servers for depot {0} - {1}", depot.id, depot.contentName); return cdnClients; } var weightedCdnServers = cdnServers.Select(x => { int penalty = 0; ConfigStore.TheConfig.ContentServerPenalty.TryGetValue(x.Host, out penalty); return Tuple.Create(x, penalty); }).OrderBy(x => x.Item2).ThenBy(x => x.Item1.WeightedLoad); // Grab up to the first eight server in the allegedly best-to-worst order from Steam Parallel.ForEach(weightedCdnServers, new ParallelOptions { MaxDegreeOfParallelism = 2 }, (serverTuple, parallelLoop) => { var server = serverTuple.Item1; CDNClient c = new CDNClient( steam3.steamClient, steam3.AppTickets[ depot.id ] ); try { for (int i = 0; i < server.NumEntries; i++) { c.Connect(server); string cdnAuthToken = null; if (server.Type == "CDN") { steam3.RequestCDNAuthToken(depot.id, server.Host); cdnAuthToken = steam3.CDNAuthTokens[Tuple.Create(depot.id, server.Host)].Token; } c.AuthenticateDepot(depot.id, depot.depotKey, cdnAuthToken); cdnClients.Add(c); } if (cdnClients.Count >= Config.MaxServers) parallelLoop.Stop(); } catch (Exception ex) { int penalty = 0; ConfigStore.TheConfig.ContentServerPenalty.TryGetValue(server.Host, out penalty); ConfigStore.TheConfig.ContentServerPenalty[server.Host] = penalty + 1; Console.WriteLine("\nFailed to connect to content server {0}: {1}", server, ex.Message); } }); if (cdnClients.Count == 0) { Console.WriteLine("\nUnable to find any content servers for depot {0} - {1}", depot.id, depot.contentName); } Config.MaxDownloads = Math.Min(Config.MaxDownloads, cdnClients.Count); return cdnClients; }