/// <summary>
/// Loads an existing PatchFile from a remote CDN
/// </summary>
/// <param name="client"></param>
/// <param name="ekey"></param>
public PatchFile(CDNClient client, MD5Hash ekey) : this()
{
string url = Helpers.GetCDNUrl(ekey.ToString(), "patch");
using (var stream = client.OpenStream(url).Result)
Read(stream);
}
private bool ReauthConnection(CDNClient client, CDNClient.Server server, uint depotId, byte[] depotKey)
{
DebugLog.Assert(server.Type == "CDN" || steamSession.AppTickets[depotId] == null, "CDNClientPool", "Re-authing a CDN or anonymous connection");
String cdnAuthToken = null;
if (server.Type == "CDN")
{
steamSession.RequestCDNAuthToken(depotId, server.Host);
cdnAuthToken = steamSession.CDNAuthTokens[Tuple.Create(depotId, server.Host)].Token;
}
try
{
client.AuthenticateDepot(depotId, depotKey, cdnAuthToken);
activeClientAuthed[client] = Tuple.Create(depotId, server);
return(true);
}
catch (Exception ex)
{
Console.WriteLine("Failed to reauth to content server {0}: {1}", server, ex.Message);
}
return(false);
}
public async Task <CDNClient> GetConnectionForDepotAsync(uint appId, uint depotId, byte[] depotKey, CancellationToken token)
{
CDNClient client = null;
Tuple <uint, CDNClient.Server> authData;
activeClientPool.TryTake(out client);
// if we couldn't find a connection, make one now
if (client == null)
{
client = await BuildConnectionAsync(appId, depotId, depotKey, null, token).ConfigureAwait(false);
}
// if we couldn't find the authorization data or it's not authed to this depotid, re-initialize
if (!activeClientAuthed.TryGetValue(client, out authData) || authData.Item1 != depotId)
{
if (authData.Item2.Type == "CDN" && await ReauthConnectionAsync(client, authData.Item2, appId, depotId, depotKey).ConfigureAwait(false))
{
Logger.Info("Re-authed CDN connection to content server {0} from {1} to {2}", authData.Item2, authData.Item1, depotId);
}
else if (authData.Item2.Type == "CS" && steamSession.AppTickets[depotId] == null && await ReauthConnectionAsync(client, authData.Item2, appId, depotId, depotKey).ConfigureAwait(false))
{
Logger.Info("Re-authed anonymous connection to content server {0} from {1} to {2}", authData.Item2, authData.Item1, depotId);
}
else
{
ReleaseConnection(client);
client = await BuildConnectionAsync(appId, depotId, depotKey, authData.Item2, token).ConfigureAwait(false);
}
}
return(client);
}
/// <summary>
/// Opens the CDNs, Versions from Ribbit and the config files from Blizzard's CDN
/// </summary>
public void OpenRemote(ManifestContainer manifestContainer)
{
if (manifestContainer?.VersionsFile == null || manifestContainer?.CDNsFile == null)
{
throw new Exception("Versions and CDNs files must be loaded first");
}
if (!manifestContainer.VersionsFile.HasLocale(manifestContainer.Locale))
{
throw new Exception($"Versions missing {manifestContainer.Locale} locale");
}
var cdnClient = new CDNClient(manifestContainer);
if (manifestContainer.BuildConfigMD5.Value != null)
{
string configUrl = Helpers.GetCDNUrl(manifestContainer.BuildConfigMD5.ToString(), "config");
BuildConfig = new KeyValueConfig(cdnClient.OpenStream(configUrl).Result, ConfigType.BuildConfig);
}
if (manifestContainer.CDNConfigMD5.Value != null)
{
string configUrl = Helpers.GetCDNUrl(manifestContainer.CDNConfigMD5.ToString(), "config");
CDNConfig = new KeyValueConfig(cdnClient.OpenStream(configUrl).Result, ConfigType.CDNConfig);
}
if (PatchConfigMD5.Value != null)
{
string configUrl = Helpers.GetCDNUrl(PatchConfigMD5.ToString(), "config");
PatchConfig = new KeyValueConfig(cdnClient.OpenStream(configUrl).Result, ConfigType.PatchConfig);
}
}
public async Task <CDNClient> GetConnectionForDepotAsync(uint appId, uint depotId, byte[] depotKey,
CancellationToken token)
{
CDNClient client = null;
Tuple <uint, CDNClient.Server> authData;
_activeClientPool.TryTake(out client);
if (client == null)
{
client = await BuildConnection(appId, depotId, depotKey, null, token);
}
if (!_activeClientAuthed.TryGetValue(client, out authData) || authData.Item1 != depotId)
{
if (authData?.Item2.Type == "CDN" || authData?.Item2.Type == "SteamCache")
{
Console.WriteLine("Re-authed CDN connection to content server {0} from {1} to {2}", authData.Item2, authData.Item1, depotId);
await AuthenticateConnection(client, authData.Item2, appId, depotId, depotKey);
}
else if (authData?.Item2.Type == "CS")
{
Console.WriteLine("Re-authed anonymous connection to content server {0} from {1} to {2}", authData.Item2, authData.Item1, depotId);
await AuthenticateConnection(client, authData.Item2, appId, depotId, depotKey);
}
else
{
client = await BuildConnection(appId, depotId, depotKey, authData?.Item2, token);
}
}
return(client);
}
public CDNClient GetConnectionForDepot(uint depotId, byte[] depotKey, CancellationToken token)
{
CDNClient client = null;
Tuple <uint, CDNClient.Server> authData;
activeClientPool.TryTake(out client);
// if we couldn't find a connection, make one now
if (client == null)
{
client = BuildConnection(depotId, depotKey, null, token);
}
// if we couldn't find the authorization data or it's not authed to this depotid, re-initialize
if (!activeClientAuthed.TryGetValue(client, out authData) || authData.Item1 != depotId)
{
if (authData.Item2.Type == "CDN" && ReauthConnection(client, authData.Item2, depotId, depotKey))
{
Console.WriteLine("Re-authed CDN connection to content server {0} from {1} to {2}", authData.Item2, authData.Item1, depotId);
}
else if (authData.Item2.Type == "CS" && steamSession.AppTickets[depotId] == null && ReauthConnection(client, authData.Item2, depotId, depotKey))
{
Console.WriteLine("Re-authed anonymous connection to content server {0} from {1} to {2}", authData.Item2, authData.Item1, depotId);
}
else
{
ReleaseConnection(client);
client = BuildConnection(depotId, depotKey, authData.Item2, token);
}
}
return(client);
}
/// <summary>
/// Download and open an remote TACT container
/// <para>Note: This will download the entire CDN so will take a while</para>
/// </summary>
/// <param name="url"></param>
/// <param name="tprDirectory"></param>
/// <param name="product"></param>
/// <param name="locale"></param>
public void DownloadRemote(string tprDirectory, string product, Locale locale, bool systemFileOnly = false)
{
ManifestContainer = new Configs.ManifestContainer(product, locale);
ManifestContainer.DownloadRemote(BaseDirectory);
ConfigContainer = new Configs.ConfigContainer();
ConfigContainer.DownloadRemote(tprDirectory, ManifestContainer);
var cdnClient = new CDNClient(ManifestContainer);
var queuedDownload = new QueuedDownloader(tprDirectory, cdnClient);
if (ConfigContainer.EncodingEKey.Value != null)
{
// Download encoding file
var encodingEKey = DownloadSystemFile(ConfigContainer.EncodingEKey, cdnClient, tprDirectory);
if (encodingEKey.Value != null)
{
EncodingFile = new Encoding.EncodingFile(tprDirectory, encodingEKey, true);
}
// Download PatchFile
DownloadSystemFile(ConfigContainer.PatchEKey, cdnClient, tprDirectory, "patch");
// Download RootFile
if (EncodingFile.TryGetCKeyEntry(ConfigContainer.RootCKey, out var ekeyEntry))
{
ekeyEntry.EKeys.ForEach(x => queuedDownload.Enqueue(x.Value.ToString()));
}
// Download InstallFile
if (EncodingFile.TryGetCKeyEntry(ConfigContainer.InstallCKey, out ekeyEntry))
{
ekeyEntry.EKeys.ForEach(x => queuedDownload.Enqueue(x.Value.ToString()));
}
// Download DownloadFile
if (EncodingFile.TryGetCKeyEntry(ConfigContainer.DownloadCKey, out ekeyEntry))
{
ekeyEntry.EKeys.ForEach(x => queuedDownload.Enqueue(x.Value.ToString()));
}
// Download DownloadSizeFile
if (EncodingFile.TryGetCKeyEntry(ConfigContainer.DownloadSizeCKey, out ekeyEntry))
{
ekeyEntry.EKeys.ForEach(x => queuedDownload.Enqueue(x.Value.ToString()));
}
queuedDownload.Download("data");
}
// Download Indices and archives
if (!systemFileOnly)
{
IndexContainer = new Indices.IndexContainer();
IndexContainer.DownloadRemote(tprDirectory, ConfigContainer, ManifestContainer);
}
Open(tprDirectory);
}
public void Dispose()
{
if (CDNClient != null)
{
CDNClient.Dispose();
CDNClient = null;
}
}
public void ReturnConnection(CDNClient client)
{
if (client == null)
{
return;
}
_activeClientPool.Add(client);
}
/// <summary>
/// Loads an existing DownloadFile from a remote CDN
/// </summary>
/// <param name="client"></param>
/// <param name="ekey"></param>
public DownloadFile(CDNClient client, MD5Hash ekey) : this()
{
string url = Helpers.GetCDNUrl(ekey.ToString(), "data");
using (var stream = client.OpenStream(url).Result)
using (var bt = new BlockTableStreamReader(stream))
Read(bt);
}
public void ReturnBrokenConnection(CDNClient client)
{
if (client == null)
{
return;
}
ReleaseConnection(client);
}
/// <summary>
/// Downloads all Index and Archive files from a remote CDN
/// </summary>
/// <param name="directory"></param>
/// <param name="configContainer"></param>
public void DownloadRemote(string directory, Configs.ConfigContainer configContainer, Configs.ManifestContainer manifestContainer)
{
_client = new CDNClient(manifestContainer);
var queuedDownloader = new QueuedDownloader(directory, _client);
// download data archives
var archives = configContainer.CDNConfig.GetValues("archives");
if (archives != null && archives.Count > 0)
{
queuedDownloader.Enqueue(archives);
queuedDownloader.Enqueue(archives, (x) => x + ".index");
queuedDownloader.Download("data");
}
// download patch archives
var patcharchives = configContainer.CDNConfig.GetValues("patch-archives");
if (patcharchives != null && patcharchives.Count > 0)
{
queuedDownloader.Enqueue(patcharchives);
queuedDownloader.Enqueue(patcharchives, (x) => x + ".index");
queuedDownloader.Download("patch");
}
// download loose file index
var fileIndex = configContainer.CDNConfig.GetValue("file-index");
if (fileIndex != null)
{
string url = Helpers.GetCDNUrl(fileIndex, "data");
string path = Helpers.GetCDNPath(fileIndex, "data", directory, true);
_client.DownloadFile(url, path).Wait();
// download loose files
var index = new IndexFile(path);
queuedDownloader.Enqueue(index.Entries, (x) => x.Key.ToString());
queuedDownloader.Download("data");
}
// download loose patch file index
var patchIndex = configContainer.CDNConfig.GetValue("patch-file-index");
if (patchIndex != null)
{
string url = Helpers.GetCDNUrl(patchIndex, "patch");
string path = Helpers.GetCDNPath(patchIndex, "patch", directory, true);
_client.DownloadFile(url, path).Wait();
// download loose patches
var index = new IndexFile(path);
queuedDownloader.Enqueue(index.Entries, (x) => x.Key.ToString());
queuedDownloader.Download("patch");
}
Open(directory);
}
public static void SetCDNClient(CDNClient cdnClient)
{
CDNClient = cdnClient;
ReloadFileList();
string filesDir = Path.Combine(Application.Path, "files", ".support", "hashes");
Directory.CreateDirectory(filesDir);
}
/// <summary>
/// Loads an existing EncodingFile from a remote CDN
/// </summary>
/// <param name="client"></param>
/// <param name="ekey"></param>
/// <param name="partial">Only reads the mandatory information. Prevents write support</param>
public EncodingFile(CDNClient client, MD5Hash ekey, bool partial = false) : this()
{
Partial = partial;
string url = Helpers.GetCDNUrl(ekey.ToString(), "data");
using (var stream = client.OpenStream(url).Result)
using (var bt = new BlockTableStreamReader(stream))
Read(bt);
}
private CDNClient BuildConnection(uint depotId, byte[] depotKey, CDNClient.Server serverSeed, CancellationToken token)
{
CDNClient.Server server = null;
CDNClient client = null;
while (client == null)
{
// if we want to re-initialize a specific content server, try that one first
if (serverSeed != null)
{
server = serverSeed;
serverSeed = null;
}
else
{
if (availableServerEndpoints.Count < ServerEndpointMinimumSize)
{
populatePoolEvent.Set();
}
server = availableServerEndpoints.Take(token);
}
client = new CDNClient(steamSession.steamClient, steamSession.AppTickets[depotId]);
string cdnAuthToken = null;
if (server.Type == "CDN")
{
steamSession.RequestCDNAuthToken(depotId, server.Host);
cdnAuthToken = steamSession.CDNAuthTokens[Tuple.Create(depotId, server.Host)].Token;
}
try
{
client.Connect(server);
client.AuthenticateDepot(depotId, depotKey, cdnAuthToken);
}
catch (Exception ex)
{
client = null;
Console.WriteLine("Failed to connect to content server {0}: {1}", server, ex.Message);
int penalty = 0;
ConfigStore.TheConfig.ContentServerPenalty.TryGetValue(server.Host, out penalty);
ConfigStore.TheConfig.ContentServerPenalty[server.Host] = penalty + 1;
}
}
Console.WriteLine("Initialized connection to content server {0} with depot id {1}", server, depotId);
activeClientAuthed[client] = Tuple.Create(depotId, server);
return(client);
}
public DepotProcessor(SteamClient client, CallbackManager manager)
{
UpdateScript = Path.Combine(Application.Path, "files", "update.sh");
DepotLocks = new Dictionary <uint, byte>();
CDNClient = new CDNClient(client);
CDNServers = new List <string>();
FileDownloader.SetCDNClient(CDNClient);
manager.Subscribe <SteamClient.ServerListCallback>(OnServerList);
}
/// <summary>
/// Download and open an remote TACT container
/// <para>Note: This will download the entire CDN so will take a while</para>
/// </summary>
/// <param name="url"></param>
/// <param name="directory"></param>
/// <param name="product"></param>
/// <param name="locale"></param>
public void DownloadRemote(string directory, string product, Locale locale)
{
ConfigContainer = new Configs.ConfigContainer(product, locale);
ConfigContainer.DownloadRemote(directory);
var cdnClient = new CDNClient(ConfigContainer);
var queuedDownload = new QueuedDownloader(directory, cdnClient);
if (ConfigContainer.EncodingEKey.Value != null)
{
// Download encoding file
var encodingEKey = DownloadSystemFile(ConfigContainer.EncodingEKey, cdnClient, directory);
if (encodingEKey.Value != null)
{
EncodingFile = new Encoding.EncodingFile(BaseDirectory, encodingEKey, true);
}
// Download PatchFile
DownloadSystemFile(ConfigContainer.PatchEKey, cdnClient, directory, "patch");
// Download RootFile
if (EncodingFile.TryGetCKeyEntry(ConfigContainer.RootCKey, out var ekeyEntry))
{
queuedDownload.Enqueue(ekeyEntry.EKey.ToString());
}
// Download InstallFile
if (EncodingFile.TryGetCKeyEntry(ConfigContainer.InstallCKey, out ekeyEntry))
{
queuedDownload.Enqueue(ekeyEntry.EKey.ToString());
}
// Download DownloadFile
if (EncodingFile.TryGetCKeyEntry(ConfigContainer.DownloadCKey, out ekeyEntry))
{
queuedDownload.Enqueue(ekeyEntry.EKey.ToString());
}
// Download DownloadSizeFile
if (EncodingFile.TryGetCKeyEntry(ConfigContainer.DownloadSizeCKey, out ekeyEntry))
{
queuedDownload.Enqueue(ekeyEntry.EKey.ToString());
}
queuedDownload.Download("data");
}
// Download Indices and archives
IndexContainer = new Indices.IndexContainer();
IndexContainer.DownloadRemote(directory, ConfigContainer);
Open(directory, product, locale);
}
public async Task <KeyValuePair <CDNClient, CDNClient.Server> > GetConnectionForAppAsync(uint appId)
{
CDNClient client = new CDNClient(Client, await RequestAppTicket(appId));
var serverList = await client.FetchServerListAsync();
var server = serverList.Where(x => x.Type == "CDN").OrderBy(x => x.WeightedLoad).First();
await client.ConnectAsync(server);
return(new KeyValuePair <CDNClient, CDNClient.Server>(client, server));
}
private async Task <CDNClient> BuildConnection(uint appId, uint depotId, byte[] depotKey,
CDNClient.Server serverSeed, CancellationToken token)
{
CDNClient.Server server = null;
CDNClient client = null;
while (client == null)
{
if (serverSeed != null)
{
server = serverSeed;
serverSeed = null;
}
else
{
if (_availableServerEndpoints.Count < ServerEndpointMinimumSize)
{
_populateEvent.Set();
}
server = _availableServerEndpoints.Take(token);
}
client = new CDNClient(_client);
try
{
var cdnToken = "";
if (server.Type == "CDN" || server.Type == "SteamCache")
{
cdnToken = (await RequestCDNAuthToken(appId, depotId, server.Host))?.Token;
}
await client.ConnectAsync(server);
await client.AuthenticateDepotAsync(depotId, depotKey, cdnToken);
}
catch (Exception ex)
{
client = null;
Console.WriteLine("Failed to connect to content server {0}: {1}", server, ex.Message);
_contentServerPenalty.TryGetValue(server.Host, out var penalty);
_contentServerPenalty[server.Host] = ++penalty;
}
}
Console.WriteLine("Initialized connection to content server {0} with depot id {1}", server, depotId);
_activeClientAuthed[client] = Tuple.Create(depotId, server);
return(client);
}
public SteamCdnServerPool(SteamContentClient steamContentClient, AppId appId, CancellationToken cancellationToken = default)
{
_steamContentClient = steamContentClient;
_appId = appId;
_cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, new CancellationTokenSource().Token);
_availableServerEndpoints = new BlockingCollection <CDNClient.Server>();
_activeServerEndpoints = new ConcurrentStack <CDNClient.Server>();
_populatePoolEvent = new AutoResetEvent(true);
_populatedEvent = new AsyncManualResetEvent(false);
_populatorTask = Task.Factory.StartNew(MonitorAsync).Unwrap();
CdnClient = new CDNClient(_steamContentClient.SteamClient.InternalClient);
}
public CDNClientPool(SteamSession steamSession)
{
this.steamSession = steamSession ?? throw new ArgumentNullException(nameof(steamSession));
CDNClient = new CDNClient(steamSession.Client);
activeConnectionPool = new ConcurrentBag <CDNClient.Server>();
availableServerEndpoints = new BlockingCollection <CDNClient.Server>();
populatePoolEvent = new AutoResetEvent(true);
shutdownToken = new CancellationTokenSource();
monitorTask = Task.Factory.StartNew(ConnectionPoolMonitorAsync).Unwrap();
}
public DepotProcessor(SteamClient client)
{
UpdateScript = Path.Combine(Application.Path, "files", "update.sh");
CDNClient = new CDNClient(client);
CDNServers = new List <CDNClient.Server>
{
new DnsEndPoint("valve500.steamcontent.com", 80)
};
CDNClient.RequestTimeout = TimeSpan.FromSeconds(30);
FileDownloader.SetCDNClient(CDNClient);
}
public CDNClientPool(UInt32 appId)
{
this.appId = appId;
CDNClient = new CDNClient(steamSession.Client);
activeConnectionPool = new ConcurrentStack <CDNClient.Server>();
availableServerEndpoints = new BlockingCollection <CDNClient.Server>();
populatePoolEvent = new AutoResetEvent(true);
shutdownToken = new CancellationTokenSource();
monitorTask = Task.Factory.StartNew(ConnectionPoolMonitorAsync).Unwrap();
}
public void Dispose()
{
if (CDNClient != null)
{
CDNClient.Dispose();
CDNClient = null;
}
if (ManifestDownloadSemaphore != null)
{
ManifestDownloadSemaphore.Dispose();
ManifestDownloadSemaphore = null;
}
}
/// <summary>
/// Loads an IndexFile from a remote CDN
/// </summary>
/// <param name="stream"></param>
public IndexFile(CDNClient client, string path, IndexType type) : this(IndexType.Unknown)
{
if (!path.EndsWith(".index", StringComparison.OrdinalIgnoreCase))
{
path += ".index";
}
string endpoint = type.HasFlag(IndexType.Data) ? "data" : "patch";
string url = Helpers.GetCDNUrl(path, endpoint);
using (var stream = client.OpenStream(url).Result)
Read(stream);
Type = DetermineType(type);
}
public DepotProcessor(SteamClient client, CallbackManager manager)
{
UpdateScript = Path.Combine(Application.Path, "files", "update.sh");
DepotLocks = new Dictionary <uint, byte>();
CDNClient = new CDNClient(client);
CDNServers = new List <string> {
"valve500.steamcontent.com"
};
CDNClient.RequestTimeout = TimeSpan.FromSeconds(30);
FileDownloader.SetCDNClient(CDNClient);
manager.Subscribe <SteamClient.ServerListCallback>(OnServerList);
}
public static void SetCDNClient(CDNClient cdnClient)
{
CDNClient = cdnClient;
try
{
string filesDir = Path.Combine(Application.Path, FILES_DIRECTORY);
Directory.CreateDirectory(filesDir);
}
catch (Exception ex)
{
Log.WriteError("FileDownloader", "Unable to create files directory: {0}", ex.Message);
}
ReloadFileList();
}
public DepotProcessor(SteamClient client)
{
UpdateScript = Path.Combine(Application.Path, "files", "update.sh");
CDNClient = new CDNClient(client);
CDNServers = new List <CDNClient.Server>
{
new DnsEndPoint(DefaultServer, 80)
};
CDNClient.RequestTimeout = TimeSpan.FromSeconds(30);
FileDownloader.SetCDNClient(CDNClient);
DepotNameStart = new Regex("^(\u00dalo\u017ei\u0161t\u011b \u2013|\u0425\u0440\u0430\u043d\u0438\u043b\u0438\u0449\u0435|D\u00e9p\u00f4t :|Depot|Depot:|Repositorio) ", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.ExplicitCapture);
DepotNameEnd = new Regex(" (- magazyn zawarto\u015bci|\u03bd\u03c4\u03b5\u03c0\u03cc|\u0441\u0445\u043e\u0432\u0438\u0449\u0435|\u0e14\u0e35\u0e42\u0e1b|\u2013 depot|\u30c7\u30dd|\u4e2a Depot|\uac1c\uc758 \ub514\ud3ec|dep\u00e5|dep\u00f3|Depo|Depot|depot|depotti)$", RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.ExplicitCapture);
}
public DepotProcessor(SteamClient client, CallbackManager manager)
{
UpdateScript = Path.Combine(Application.Path, "files", "update.sh");
CDNClient = new CDNClient(client);
CDNServers = new List <CDNClient.Server>
{
new DnsEndPoint("valve500.steamcontent.com", 80)
};
CDNClient.RequestTimeout = TimeSpan.FromSeconds(30);
FileDownloader.SetCDNClient(CDNClient);
manager.Subscribe <SteamUser.LoggedOnCallback>(OnLoggedOn);
TaskManager.RunAsync(UpdateContentServerList);
}
public async Task ThrowsSteamKitWebExceptionOnUnsuccessfulWebResponse()
{
var configuration = SteamConfiguration.Create(x => x.WithHttpClientFactory(() => new HttpClient(new TeapotHttpMessageHandler())));
var steam = new SteamClient(configuration);
var client = new CDNClient(steam);
try
{
await client.DownloadManifestAsync(0, 0, "localhost", "12345");
throw new InvalidOperationException("This should be unreachable.");
}
catch (SteamKitWebRequestException ex)
{
Assert.Equal((HttpStatusCode)418, ex.StatusCode);
}
}
