/// <summary> /// Loads an existing PatchFile from a remote CDN /// </summary> /// <param name="client"></param> /// <param name="ekey"></param> public PatchFile(CDNClient client, MD5Hash ekey) : this() { string url = Helpers.GetCDNUrl(ekey.ToString(), "patch"); using (var stream = client.OpenStream(url).Result) Read(stream); }
private bool ReauthConnection(CDNClient client, CDNClient.Server server, uint depotId, byte[] depotKey) { DebugLog.Assert(server.Type == "CDN" || steamSession.AppTickets[depotId] == null, "CDNClientPool", "Re-authing a CDN or anonymous connection"); String cdnAuthToken = null; if (server.Type == "CDN") { steamSession.RequestCDNAuthToken(depotId, server.Host); cdnAuthToken = steamSession.CDNAuthTokens[Tuple.Create(depotId, server.Host)].Token; } try { client.AuthenticateDepot(depotId, depotKey, cdnAuthToken); activeClientAuthed[client] = Tuple.Create(depotId, server); return(true); } catch (Exception ex) { Console.WriteLine("Failed to reauth to content server {0}: {1}", server, ex.Message); } return(false); }
public async Task <CDNClient> GetConnectionForDepotAsync(uint appId, uint depotId, byte[] depotKey, CancellationToken token) { CDNClient client = null; Tuple <uint, CDNClient.Server> authData; activeClientPool.TryTake(out client); // if we couldn't find a connection, make one now if (client == null) { client = await BuildConnectionAsync(appId, depotId, depotKey, null, token).ConfigureAwait(false); } // if we couldn't find the authorization data or it's not authed to this depotid, re-initialize if (!activeClientAuthed.TryGetValue(client, out authData) || authData.Item1 != depotId) { if (authData.Item2.Type == "CDN" && await ReauthConnectionAsync(client, authData.Item2, appId, depotId, depotKey).ConfigureAwait(false)) { Logger.Info("Re-authed CDN connection to content server {0} from {1} to {2}", authData.Item2, authData.Item1, depotId); } else if (authData.Item2.Type == "CS" && steamSession.AppTickets[depotId] == null && await ReauthConnectionAsync(client, authData.Item2, appId, depotId, depotKey).ConfigureAwait(false)) { Logger.Info("Re-authed anonymous connection to content server {0} from {1} to {2}", authData.Item2, authData.Item1, depotId); } else { ReleaseConnection(client); client = await BuildConnectionAsync(appId, depotId, depotKey, authData.Item2, token).ConfigureAwait(false); } } return(client); }
/// <summary> /// Opens the CDNs, Versions from Ribbit and the config files from Blizzard's CDN /// </summary> public void OpenRemote(ManifestContainer manifestContainer) { if (manifestContainer?.VersionsFile == null || manifestContainer?.CDNsFile == null) { throw new Exception("Versions and CDNs files must be loaded first"); } if (!manifestContainer.VersionsFile.HasLocale(manifestContainer.Locale)) { throw new Exception($"Versions missing {manifestContainer.Locale} locale"); } var cdnClient = new CDNClient(manifestContainer); if (manifestContainer.BuildConfigMD5.Value != null) { string configUrl = Helpers.GetCDNUrl(manifestContainer.BuildConfigMD5.ToString(), "config"); BuildConfig = new KeyValueConfig(cdnClient.OpenStream(configUrl).Result, ConfigType.BuildConfig); } if (manifestContainer.CDNConfigMD5.Value != null) { string configUrl = Helpers.GetCDNUrl(manifestContainer.CDNConfigMD5.ToString(), "config"); CDNConfig = new KeyValueConfig(cdnClient.OpenStream(configUrl).Result, ConfigType.CDNConfig); } if (PatchConfigMD5.Value != null) { string configUrl = Helpers.GetCDNUrl(PatchConfigMD5.ToString(), "config"); PatchConfig = new KeyValueConfig(cdnClient.OpenStream(configUrl).Result, ConfigType.PatchConfig); } }
public async Task <CDNClient> GetConnectionForDepotAsync(uint appId, uint depotId, byte[] depotKey, CancellationToken token) { CDNClient client = null; Tuple <uint, CDNClient.Server> authData; _activeClientPool.TryTake(out client); if (client == null) { client = await BuildConnection(appId, depotId, depotKey, null, token); } if (!_activeClientAuthed.TryGetValue(client, out authData) || authData.Item1 != depotId) { if (authData?.Item2.Type == "CDN" || authData?.Item2.Type == "SteamCache") { Console.WriteLine("Re-authed CDN connection to content server {0} from {1} to {2}", authData.Item2, authData.Item1, depotId); await AuthenticateConnection(client, authData.Item2, appId, depotId, depotKey); } else if (authData?.Item2.Type == "CS") { Console.WriteLine("Re-authed anonymous connection to content server {0} from {1} to {2}", authData.Item2, authData.Item1, depotId); await AuthenticateConnection(client, authData.Item2, appId, depotId, depotKey); } else { client = await BuildConnection(appId, depotId, depotKey, authData?.Item2, token); } } return(client); }
public CDNClient GetConnectionForDepot(uint depotId, byte[] depotKey, CancellationToken token) { CDNClient client = null; Tuple <uint, CDNClient.Server> authData; activeClientPool.TryTake(out client); // if we couldn't find a connection, make one now if (client == null) { client = BuildConnection(depotId, depotKey, null, token); } // if we couldn't find the authorization data or it's not authed to this depotid, re-initialize if (!activeClientAuthed.TryGetValue(client, out authData) || authData.Item1 != depotId) { if (authData.Item2.Type == "CDN" && ReauthConnection(client, authData.Item2, depotId, depotKey)) { Console.WriteLine("Re-authed CDN connection to content server {0} from {1} to {2}", authData.Item2, authData.Item1, depotId); } else if (authData.Item2.Type == "CS" && steamSession.AppTickets[depotId] == null && ReauthConnection(client, authData.Item2, depotId, depotKey)) { Console.WriteLine("Re-authed anonymous connection to content server {0} from {1} to {2}", authData.Item2, authData.Item1, depotId); } else { ReleaseConnection(client); client = BuildConnection(depotId, depotKey, authData.Item2, token); } } return(client); }
/// <summary> /// Download and open an remote TACT container /// <para>Note: This will download the entire CDN so will take a while</para> /// </summary> /// <param name="url"></param> /// <param name="tprDirectory"></param> /// <param name="product"></param> /// <param name="locale"></param> public void DownloadRemote(string tprDirectory, string product, Locale locale, bool systemFileOnly = false) { ManifestContainer = new Configs.ManifestContainer(product, locale); ManifestContainer.DownloadRemote(BaseDirectory); ConfigContainer = new Configs.ConfigContainer(); ConfigContainer.DownloadRemote(tprDirectory, ManifestContainer); var cdnClient = new CDNClient(ManifestContainer); var queuedDownload = new QueuedDownloader(tprDirectory, cdnClient); if (ConfigContainer.EncodingEKey.Value != null) { // Download encoding file var encodingEKey = DownloadSystemFile(ConfigContainer.EncodingEKey, cdnClient, tprDirectory); if (encodingEKey.Value != null) { EncodingFile = new Encoding.EncodingFile(tprDirectory, encodingEKey, true); } // Download PatchFile DownloadSystemFile(ConfigContainer.PatchEKey, cdnClient, tprDirectory, "patch"); // Download RootFile if (EncodingFile.TryGetCKeyEntry(ConfigContainer.RootCKey, out var ekeyEntry)) { ekeyEntry.EKeys.ForEach(x => queuedDownload.Enqueue(x.Value.ToString())); } // Download InstallFile if (EncodingFile.TryGetCKeyEntry(ConfigContainer.InstallCKey, out ekeyEntry)) { ekeyEntry.EKeys.ForEach(x => queuedDownload.Enqueue(x.Value.ToString())); } // Download DownloadFile if (EncodingFile.TryGetCKeyEntry(ConfigContainer.DownloadCKey, out ekeyEntry)) { ekeyEntry.EKeys.ForEach(x => queuedDownload.Enqueue(x.Value.ToString())); } // Download DownloadSizeFile if (EncodingFile.TryGetCKeyEntry(ConfigContainer.DownloadSizeCKey, out ekeyEntry)) { ekeyEntry.EKeys.ForEach(x => queuedDownload.Enqueue(x.Value.ToString())); } queuedDownload.Download("data"); } // Download Indices and archives if (!systemFileOnly) { IndexContainer = new Indices.IndexContainer(); IndexContainer.DownloadRemote(tprDirectory, ConfigContainer, ManifestContainer); } Open(tprDirectory); }
public void Dispose() { if (CDNClient != null) { CDNClient.Dispose(); CDNClient = null; } }
public void ReturnConnection(CDNClient client) { if (client == null) { return; } _activeClientPool.Add(client); }
/// <summary> /// Loads an existing DownloadFile from a remote CDN /// </summary> /// <param name="client"></param> /// <param name="ekey"></param> public DownloadFile(CDNClient client, MD5Hash ekey) : this() { string url = Helpers.GetCDNUrl(ekey.ToString(), "data"); using (var stream = client.OpenStream(url).Result) using (var bt = new BlockTableStreamReader(stream)) Read(bt); }
public void ReturnBrokenConnection(CDNClient client) { if (client == null) { return; } ReleaseConnection(client); }
/// <summary> /// Downloads all Index and Archive files from a remote CDN /// </summary> /// <param name="directory"></param> /// <param name="configContainer"></param> public void DownloadRemote(string directory, Configs.ConfigContainer configContainer, Configs.ManifestContainer manifestContainer) { _client = new CDNClient(manifestContainer); var queuedDownloader = new QueuedDownloader(directory, _client); // download data archives var archives = configContainer.CDNConfig.GetValues("archives"); if (archives != null && archives.Count > 0) { queuedDownloader.Enqueue(archives); queuedDownloader.Enqueue(archives, (x) => x + ".index"); queuedDownloader.Download("data"); } // download patch archives var patcharchives = configContainer.CDNConfig.GetValues("patch-archives"); if (patcharchives != null && patcharchives.Count > 0) { queuedDownloader.Enqueue(patcharchives); queuedDownloader.Enqueue(patcharchives, (x) => x + ".index"); queuedDownloader.Download("patch"); } // download loose file index var fileIndex = configContainer.CDNConfig.GetValue("file-index"); if (fileIndex != null) { string url = Helpers.GetCDNUrl(fileIndex, "data"); string path = Helpers.GetCDNPath(fileIndex, "data", directory, true); _client.DownloadFile(url, path).Wait(); // download loose files var index = new IndexFile(path); queuedDownloader.Enqueue(index.Entries, (x) => x.Key.ToString()); queuedDownloader.Download("data"); } // download loose patch file index var patchIndex = configContainer.CDNConfig.GetValue("patch-file-index"); if (patchIndex != null) { string url = Helpers.GetCDNUrl(patchIndex, "patch"); string path = Helpers.GetCDNPath(patchIndex, "patch", directory, true); _client.DownloadFile(url, path).Wait(); // download loose patches var index = new IndexFile(path); queuedDownloader.Enqueue(index.Entries, (x) => x.Key.ToString()); queuedDownloader.Download("patch"); } Open(directory); }
public static void SetCDNClient(CDNClient cdnClient) { CDNClient = cdnClient; ReloadFileList(); string filesDir = Path.Combine(Application.Path, "files", ".support", "hashes"); Directory.CreateDirectory(filesDir); }
/// <summary> /// Loads an existing EncodingFile from a remote CDN /// </summary> /// <param name="client"></param> /// <param name="ekey"></param> /// <param name="partial">Only reads the mandatory information. Prevents write support</param> public EncodingFile(CDNClient client, MD5Hash ekey, bool partial = false) : this() { Partial = partial; string url = Helpers.GetCDNUrl(ekey.ToString(), "data"); using (var stream = client.OpenStream(url).Result) using (var bt = new BlockTableStreamReader(stream)) Read(bt); }
private CDNClient BuildConnection(uint depotId, byte[] depotKey, CDNClient.Server serverSeed, CancellationToken token) { CDNClient.Server server = null; CDNClient client = null; while (client == null) { // if we want to re-initialize a specific content server, try that one first if (serverSeed != null) { server = serverSeed; serverSeed = null; } else { if (availableServerEndpoints.Count < ServerEndpointMinimumSize) { populatePoolEvent.Set(); } server = availableServerEndpoints.Take(token); } client = new CDNClient(steamSession.steamClient, steamSession.AppTickets[depotId]); string cdnAuthToken = null; if (server.Type == "CDN") { steamSession.RequestCDNAuthToken(depotId, server.Host); cdnAuthToken = steamSession.CDNAuthTokens[Tuple.Create(depotId, server.Host)].Token; } try { client.Connect(server); client.AuthenticateDepot(depotId, depotKey, cdnAuthToken); } catch (Exception ex) { client = null; Console.WriteLine("Failed to connect to content server {0}: {1}", server, ex.Message); int penalty = 0; ConfigStore.TheConfig.ContentServerPenalty.TryGetValue(server.Host, out penalty); ConfigStore.TheConfig.ContentServerPenalty[server.Host] = penalty + 1; } } Console.WriteLine("Initialized connection to content server {0} with depot id {1}", server, depotId); activeClientAuthed[client] = Tuple.Create(depotId, server); return(client); }
public DepotProcessor(SteamClient client, CallbackManager manager) { UpdateScript = Path.Combine(Application.Path, "files", "update.sh"); DepotLocks = new Dictionary <uint, byte>(); CDNClient = new CDNClient(client); CDNServers = new List <string>(); FileDownloader.SetCDNClient(CDNClient); manager.Subscribe <SteamClient.ServerListCallback>(OnServerList); }
/// <summary> /// Download and open an remote TACT container /// <para>Note: This will download the entire CDN so will take a while</para> /// </summary> /// <param name="url"></param> /// <param name="directory"></param> /// <param name="product"></param> /// <param name="locale"></param> public void DownloadRemote(string directory, string product, Locale locale) { ConfigContainer = new Configs.ConfigContainer(product, locale); ConfigContainer.DownloadRemote(directory); var cdnClient = new CDNClient(ConfigContainer); var queuedDownload = new QueuedDownloader(directory, cdnClient); if (ConfigContainer.EncodingEKey.Value != null) { // Download encoding file var encodingEKey = DownloadSystemFile(ConfigContainer.EncodingEKey, cdnClient, directory); if (encodingEKey.Value != null) { EncodingFile = new Encoding.EncodingFile(BaseDirectory, encodingEKey, true); } // Download PatchFile DownloadSystemFile(ConfigContainer.PatchEKey, cdnClient, directory, "patch"); // Download RootFile if (EncodingFile.TryGetCKeyEntry(ConfigContainer.RootCKey, out var ekeyEntry)) { queuedDownload.Enqueue(ekeyEntry.EKey.ToString()); } // Download InstallFile if (EncodingFile.TryGetCKeyEntry(ConfigContainer.InstallCKey, out ekeyEntry)) { queuedDownload.Enqueue(ekeyEntry.EKey.ToString()); } // Download DownloadFile if (EncodingFile.TryGetCKeyEntry(ConfigContainer.DownloadCKey, out ekeyEntry)) { queuedDownload.Enqueue(ekeyEntry.EKey.ToString()); } // Download DownloadSizeFile if (EncodingFile.TryGetCKeyEntry(ConfigContainer.DownloadSizeCKey, out ekeyEntry)) { queuedDownload.Enqueue(ekeyEntry.EKey.ToString()); } queuedDownload.Download("data"); } // Download Indices and archives IndexContainer = new Indices.IndexContainer(); IndexContainer.DownloadRemote(directory, ConfigContainer); Open(directory, product, locale); }
public async Task <KeyValuePair <CDNClient, CDNClient.Server> > GetConnectionForAppAsync(uint appId) { CDNClient client = new CDNClient(Client, await RequestAppTicket(appId)); var serverList = await client.FetchServerListAsync(); var server = serverList.Where(x => x.Type == "CDN").OrderBy(x => x.WeightedLoad).First(); await client.ConnectAsync(server); return(new KeyValuePair <CDNClient, CDNClient.Server>(client, server)); }
private async Task <CDNClient> BuildConnection(uint appId, uint depotId, byte[] depotKey, CDNClient.Server serverSeed, CancellationToken token) { CDNClient.Server server = null; CDNClient client = null; while (client == null) { if (serverSeed != null) { server = serverSeed; serverSeed = null; } else { if (_availableServerEndpoints.Count < ServerEndpointMinimumSize) { _populateEvent.Set(); } server = _availableServerEndpoints.Take(token); } client = new CDNClient(_client); try { var cdnToken = ""; if (server.Type == "CDN" || server.Type == "SteamCache") { cdnToken = (await RequestCDNAuthToken(appId, depotId, server.Host))?.Token; } await client.ConnectAsync(server); await client.AuthenticateDepotAsync(depotId, depotKey, cdnToken); } catch (Exception ex) { client = null; Console.WriteLine("Failed to connect to content server {0}: {1}", server, ex.Message); _contentServerPenalty.TryGetValue(server.Host, out var penalty); _contentServerPenalty[server.Host] = ++penalty; } } Console.WriteLine("Initialized connection to content server {0} with depot id {1}", server, depotId); _activeClientAuthed[client] = Tuple.Create(depotId, server); return(client); }
public SteamCdnServerPool(SteamContentClient steamContentClient, AppId appId, CancellationToken cancellationToken = default) { _steamContentClient = steamContentClient; _appId = appId; _cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, new CancellationTokenSource().Token); _availableServerEndpoints = new BlockingCollection <CDNClient.Server>(); _activeServerEndpoints = new ConcurrentStack <CDNClient.Server>(); _populatePoolEvent = new AutoResetEvent(true); _populatedEvent = new AsyncManualResetEvent(false); _populatorTask = Task.Factory.StartNew(MonitorAsync).Unwrap(); CdnClient = new CDNClient(_steamContentClient.SteamClient.InternalClient); }
public CDNClientPool(SteamSession steamSession) { this.steamSession = steamSession ?? throw new ArgumentNullException(nameof(steamSession)); CDNClient = new CDNClient(steamSession.Client); activeConnectionPool = new ConcurrentBag <CDNClient.Server>(); availableServerEndpoints = new BlockingCollection <CDNClient.Server>(); populatePoolEvent = new AutoResetEvent(true); shutdownToken = new CancellationTokenSource(); monitorTask = Task.Factory.StartNew(ConnectionPoolMonitorAsync).Unwrap(); }
public DepotProcessor(SteamClient client) { UpdateScript = Path.Combine(Application.Path, "files", "update.sh"); CDNClient = new CDNClient(client); CDNServers = new List <CDNClient.Server> { new DnsEndPoint("valve500.steamcontent.com", 80) }; CDNClient.RequestTimeout = TimeSpan.FromSeconds(30); FileDownloader.SetCDNClient(CDNClient); }
public CDNClientPool(UInt32 appId) { this.appId = appId; CDNClient = new CDNClient(steamSession.Client); activeConnectionPool = new ConcurrentStack <CDNClient.Server>(); availableServerEndpoints = new BlockingCollection <CDNClient.Server>(); populatePoolEvent = new AutoResetEvent(true); shutdownToken = new CancellationTokenSource(); monitorTask = Task.Factory.StartNew(ConnectionPoolMonitorAsync).Unwrap(); }
public void Dispose() { if (CDNClient != null) { CDNClient.Dispose(); CDNClient = null; } if (ManifestDownloadSemaphore != null) { ManifestDownloadSemaphore.Dispose(); ManifestDownloadSemaphore = null; } }
/// <summary> /// Loads an IndexFile from a remote CDN /// </summary> /// <param name="stream"></param> public IndexFile(CDNClient client, string path, IndexType type) : this(IndexType.Unknown) { if (!path.EndsWith(".index", StringComparison.OrdinalIgnoreCase)) { path += ".index"; } string endpoint = type.HasFlag(IndexType.Data) ? "data" : "patch"; string url = Helpers.GetCDNUrl(path, endpoint); using (var stream = client.OpenStream(url).Result) Read(stream); Type = DetermineType(type); }
public DepotProcessor(SteamClient client, CallbackManager manager) { UpdateScript = Path.Combine(Application.Path, "files", "update.sh"); DepotLocks = new Dictionary <uint, byte>(); CDNClient = new CDNClient(client); CDNServers = new List <string> { "valve500.steamcontent.com" }; CDNClient.RequestTimeout = TimeSpan.FromSeconds(30); FileDownloader.SetCDNClient(CDNClient); manager.Subscribe <SteamClient.ServerListCallback>(OnServerList); }
public static void SetCDNClient(CDNClient cdnClient) { CDNClient = cdnClient; try { string filesDir = Path.Combine(Application.Path, FILES_DIRECTORY); Directory.CreateDirectory(filesDir); } catch (Exception ex) { Log.WriteError("FileDownloader", "Unable to create files directory: {0}", ex.Message); } ReloadFileList(); }
public DepotProcessor(SteamClient client) { UpdateScript = Path.Combine(Application.Path, "files", "update.sh"); CDNClient = new CDNClient(client); CDNServers = new List <CDNClient.Server> { new DnsEndPoint(DefaultServer, 80) }; CDNClient.RequestTimeout = TimeSpan.FromSeconds(30); FileDownloader.SetCDNClient(CDNClient); DepotNameStart = new Regex("^(\u00dalo\u017ei\u0161t\u011b \u2013|\u0425\u0440\u0430\u043d\u0438\u043b\u0438\u0449\u0435|D\u00e9p\u00f4t :|Depot|Depot:|Repositorio) ", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.ExplicitCapture); DepotNameEnd = new Regex(" (- magazyn zawarto\u015bci|\u03bd\u03c4\u03b5\u03c0\u03cc|\u0441\u0445\u043e\u0432\u0438\u0449\u0435|\u0e14\u0e35\u0e42\u0e1b|\u2013 depot|\u30c7\u30dd|\u4e2a Depot|\uac1c\uc758 \ub514\ud3ec|dep\u00e5|dep\u00f3|Depo|Depot|depot|depotti)$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.ExplicitCapture); }
public DepotProcessor(SteamClient client, CallbackManager manager) { UpdateScript = Path.Combine(Application.Path, "files", "update.sh"); CDNClient = new CDNClient(client); CDNServers = new List <CDNClient.Server> { new DnsEndPoint("valve500.steamcontent.com", 80) }; CDNClient.RequestTimeout = TimeSpan.FromSeconds(30); FileDownloader.SetCDNClient(CDNClient); manager.Subscribe <SteamUser.LoggedOnCallback>(OnLoggedOn); TaskManager.RunAsync(UpdateContentServerList); }
public async Task ThrowsSteamKitWebExceptionOnUnsuccessfulWebResponse() { var configuration = SteamConfiguration.Create(x => x.WithHttpClientFactory(() => new HttpClient(new TeapotHttpMessageHandler()))); var steam = new SteamClient(configuration); var client = new CDNClient(steam); try { await client.DownloadManifestAsync(0, 0, "localhost", "12345"); throw new InvalidOperationException("This should be unreachable."); } catch (SteamKitWebRequestException ex) { Assert.Equal((HttpStatusCode)418, ex.StatusCode); } }