|
public static FetchServerList ( ClientEndPoint csServer, int cellID ) : List |
||
| csServer | ClientEndPoint | The server to request a server list from. |
| cellID | int | The CellID. |
| return | List |
|
private static void DownloadManifest(ManifestJob request)
{
CDNClient cdnClient = new CDNClient(Steam.Instance.Client, request.DepotID, request.Ticket, request.DepotKey);
List<CDNClient.Server> cdnServers;
try
{
cdnServers = cdnClient.FetchServerList();
if (cdnServers.Count == 0)
{
throw new InvalidOperationException("No servers returned"); // Great programming!
}
}
catch
{
Log.WriteError("Depot Processor", "Failed to get server list for depot {0}", request.DepotID);
lock (ManifestJobs)
{
ManifestJobs.Remove(request);
}
return;
}
DepotManifest depotManifest = null;
foreach (var server in cdnServers)
{
try
{
cdnClient.Connect(server);
depotManifest = cdnClient.DownloadManifest(request.ManifestID);
break;
}
catch { }
}
lock (ManifestJobs)
{
ManifestJobs.Remove(request);
}
if (depotManifest == null)
{
Log.WriteError("Depot Processor", "Failed to download depot manifest for depot {0} (parent {1}) (jobs still in queue: {2})", request.DepotID, request.ParentAppID, ManifestJobs.Count);
if (SteamProxy.Instance.ImportantApps.Contains(request.ParentAppID))
{
IRC.SendMain("Important manifest update: {0}{1}{2} {3}(parent {4}){5} -{6} manifest download failed from {7} servers", Colors.OLIVE, request.DepotName, Colors.NORMAL, Colors.DARK_GRAY, request.ParentAppID, Colors.NORMAL, Colors.RED, cdnServers.Count);
}
return;
}
if (SteamProxy.Instance.ImportantApps.Contains(request.ParentAppID))
{
IRC.SendMain("Important manifest update: {0}{1}{2} {3}(parent {4}){5} -{6} {7}", Colors.OLIVE, request.DepotName, Colors.NORMAL, Colors.DARK_GRAY, request.ParentAppID, Colors.NORMAL, Colors.DARK_BLUE, SteamDB.GetDepotURL(request.DepotID, "history"));
}
var sortedFiles = depotManifest.Files.OrderBy(f => f.FileName, StringComparer.OrdinalIgnoreCase);
bool shouldHistorize = false;
List<DepotFile> filesNew = new List<DepotFile>();
List<DepotFile> filesOld = new List<DepotFile>();
foreach (var file in sortedFiles)
{
System.Text.Encoding.UTF8.GetString(file.FileHash);
filesNew.Add(new DepotFile
{
Name = file.FileName.Replace('\\', '/'),
Size = file.TotalSize,
Chunks = file.Chunks.Count,
Flags = (int)file.Flags
});
}
using (MySqlDataReader Reader = DbWorker.ExecuteReader("SELECT `Files` FROM `Depots` WHERE `DepotID` = @DepotID LIMIT 1", new MySqlParameter("DepotID", request.DepotID)))
{
if (Reader.Read())
{
string files = Reader.GetString("Files");
if (!string.IsNullOrEmpty(files))
{
shouldHistorize = true;
filesOld = JsonConvert.DeserializeObject<List<DepotFile>>(files);
}
}
}
DbWorker.ExecuteNonQuery("UPDATE `Depots` SET `Files` = @Files WHERE `DepotID` = @DepotID",
new MySqlParameter("@DepotID", request.DepotID),
new MySqlParameter("@Files", JsonConvert.SerializeObject(filesNew, new JsonSerializerSettings { DefaultValueHandling = DefaultValueHandling.Ignore }))
);
if (shouldHistorize)
{
List<string> filesAdded = new List<string>();
foreach (var file in filesNew)
{
var oldFile = filesOld.Find(x => x.Name == file.Name);
if (oldFile == null)
{
// We want to historize modifications first, and only then deletions and additions
filesAdded.Add(file.Name);
}
else
{
if (oldFile.Size != file.Size)
{
MakeHistory(request, file.Name, "modified", oldFile.Size, file.Size);
}
filesOld.Remove(oldFile);
}
}
foreach (var file in filesOld)
{
MakeHistory(request, file.Name, "removed");
}
foreach (string file in filesAdded)
{
MakeHistory(request, file, "added");
}
}
lock (ManifestJobs)
{
Log.WriteDebug("Depot Processor", "DepotID: Processed {0} (jobs still in queue: {1})", request.DepotID, ManifestJobs.Count);
}
// Our job is done here, now download important files if any
if (Settings.Current.ImportantFiles.ContainsKey(request.DepotID))
{
// TODO: Horribly inefficient
var importantFiles = sortedFiles.Where(x => Settings.Current.ImportantFiles[request.DepotID].Contains(x.FileName.Replace('\\', '/')));
ThreadPool.QueueWorkItem(DownloadFiles, importantFiles, request.DepotID, cdnClient);
}
}
private static List<CDNClient> CollectCDNClientsForDepot(DepotDownloadInfo depot)
{
var cdnClients = new List<CDNClient>();
CDNClient initialClient = new CDNClient(steam3.steamClient, steam3.AppTickets[depot.id]);
var cdnServers = initialClient.FetchServerList(cellId: (uint)Config.CellID);
// Grab up to the first eight server in the allegedly best-to-worst order from Steam
var limit = cdnServers.Take(Config.MaxServers);
int tries = 0;
foreach (var s in limit)
{
CDNClient c;
if (tries == 0)
{
c = initialClient;
}
else
{
c = new CDNClient(steam3.steamClient, steam3.AppTickets[depot.id]);
}
try
{
c.Connect(s);
cdnClients.Add(c);
}
catch
{
Console.WriteLine("\nFailed to connect to content server {0}. Remaining content servers for depot: {1}.", s, Config.MaxServers - tries - 1);
}
tries++;
}
if (cdnClients.Count == 0)
{
Console.WriteLine("\nUnable to find any content servers for depot {0} - {1}", depot.id, depot.contentName);
}
return cdnClients;
}
private static void DownloadManifest(ManifestJob request)
{
var cdnClient = new CDNClient(Steam.Instance.Client, request.DepotID, request.Ticket, request.DepotKey);
List<CDNClient.Server> cdnServers = null;
for (var i = 0; i <= 5; i++)
{
try
{
cdnServers = cdnClient.FetchServerList();
if (cdnServers.Count > 0)
{
break;
}
}
catch { }
}
if (cdnServers == null || cdnServers.Count == 0)
{
Log.WriteError("Depot Processor", "Failed to get server list for depot {0}", request.DepotID);
if (SteamProxy.Instance.ImportantApps.Contains(request.ParentAppID))
{
IRC.SendMain("Important manifest update: {0}{1}{2} {3}(parent {4}){5} -{6} failed to fetch server list", Colors.OLIVE, request.DepotName, Colors.NORMAL, Colors.DARK_GRAY, request.ParentAppID, Colors.NORMAL, Colors.RED);
}
lock (ManifestJobs)
{
ManifestJobs.Remove(request);
}
return;
}
DepotManifest depotManifest = null;
foreach (var server in cdnServers)
{
try
{
cdnClient.Connect(server);
depotManifest = cdnClient.DownloadManifest(request.ManifestID);
break;
}
catch { }
}
lock (ManifestJobs)
{
ManifestJobs.Remove(request);
}
if (depotManifest == null)
{
Log.WriteError("Depot Processor", "Failed to download depot manifest for depot {0} (parent {1}) (jobs still in queue: {2})", request.DepotID, request.ParentAppID, ManifestJobs.Count);
if (SteamProxy.Instance.ImportantApps.Contains(request.ParentAppID))
{
IRC.SendMain("Important manifest update: {0}{1}{2} {3}(parent {4}){5} -{6} failed to download depot manifest from {7} servers", Colors.OLIVE, request.DepotName, Colors.NORMAL, Colors.DARK_GRAY, request.ParentAppID, Colors.NORMAL, Colors.RED, cdnServers.Count);
}
return;
}
if (SteamProxy.Instance.ImportantApps.Contains(request.ParentAppID))
{
IRC.SendMain("Important manifest update: {0}{1}{2} {3}(parent {4}){5} -{6} {7}", Colors.OLIVE, request.DepotName, Colors.NORMAL, Colors.DARK_GRAY, request.ParentAppID, Colors.NORMAL, Colors.DARK_BLUE, SteamDB.GetDepotURL(request.DepotID, "history"));
}
var sortedFiles = depotManifest.Files.OrderBy(f => f.FileName, StringComparer.OrdinalIgnoreCase);
bool shouldHistorize = false;
var filesNew = new List<DepotFile>();
var filesOld = new List<DepotFile>();
foreach (var file in sortedFiles)
{
System.Text.Encoding.UTF8.GetString(file.FileHash);
var depotFile = new DepotFile
{
Name = file.FileName.Replace('\\', '/'),
Size = file.TotalSize,
Chunks = file.Chunks.Count,
Flags = (int)file.Flags
};
// TODO: Ideally we would check if filehash is not empty
if (!file.Flags.HasFlag(EDepotFileFlag.Directory))
{
depotFile.Hash = string.Concat(Array.ConvertAll(file.FileHash, x => x.ToString("X2")));
}
filesNew.Add(depotFile);
}
using (MySqlDataReader Reader = DbWorker.ExecuteReader("SELECT `Files` FROM `Depots` WHERE `DepotID` = @DepotID LIMIT 1", new MySqlParameter("DepotID", request.DepotID)))
{
if (Reader.Read())
{
string files = Reader.GetString("Files");
if (!string.IsNullOrEmpty(files))
{
shouldHistorize = true;
filesOld = JsonConvert.DeserializeObject<List<DepotFile>>(files);
}
}
}
DbWorker.ExecuteNonQuery("UPDATE `Depots` SET `Files` = @Files WHERE `DepotID` = @DepotID",
new MySqlParameter("@DepotID", request.DepotID),
new MySqlParameter("@Files", JsonConvert.SerializeObject(filesNew, new JsonSerializerSettings { DefaultValueHandling = DefaultValueHandling.Ignore }))
);
if (shouldHistorize)
{
var filesAdded = new List<string>();
foreach (var file in filesNew)
{
var oldFile = filesOld.Find(x => x.Name == file.Name);
if (oldFile == null)
{
// We want to historize modifications first, and only then deletions and additions
filesAdded.Add(file.Name);
}
else
{
if (oldFile.Size != file.Size)
{
MakeHistory(request, file.Name, "modified", oldFile.Size, file.Size);
}
else if (file.Hash != null && oldFile.Hash != null && !file.Hash.Equals(oldFile.Hash))
{
MakeHistory(request, file.Name, "modified", oldFile.Size, file.Size);
}
filesOld.Remove(oldFile);
}
}
foreach (var file in filesOld)
{
MakeHistory(request, file.Name, "removed");
}
foreach (string file in filesAdded)
{
MakeHistory(request, file, "added");
}
}
lock (ManifestJobs)
{
Log.WriteDebug("Depot Processor", "DepotID: Processed {0} (jobs still in queue: {1})", request.DepotID, ManifestJobs.Count);
}
}
private static BlockingCollection<CDNClient> CollectCDNClientsForDepot(DepotDownloadInfo depot)
{
Console.WriteLine("Finding content servers...");
var cdnClients = new BlockingCollection<CDNClient>();
CDNClient initialClient = new CDNClient( steam3.steamClient, steam3.AppTickets[depot.id] );
List<CDNClient.Server> cdnServers = null;
while(true)
{
try
{
cdnServers = initialClient.FetchServerList( cellId: (uint)Config.CellID );
if (cdnServers != null) break;
}
catch (WebException)
{
Console.WriteLine("\nFailed to retrieve content server list.");
Thread.Sleep(500);
}
}
if (cdnServers == null)
{
Console.WriteLine("\nUnable to query any content servers for depot {0} - {1}", depot.id, depot.contentName);
return cdnClients;
}
var weightedCdnServers = cdnServers.Select(x =>
{
int penalty = 0;
ConfigStore.TheConfig.ContentServerPenalty.TryGetValue(x.Host, out penalty);
return Tuple.Create(x, penalty);
}).OrderBy(x => x.Item2).ThenBy(x => x.Item1.WeightedLoad);
// Grab up to the first eight server in the allegedly best-to-worst order from Steam
Parallel.ForEach(weightedCdnServers, new ParallelOptions { MaxDegreeOfParallelism = 2 }, (serverTuple, parallelLoop) =>
{
var server = serverTuple.Item1;
CDNClient c = new CDNClient( steam3.steamClient, steam3.AppTickets[ depot.id ] );
try
{
for (int i = 0; i < server.NumEntries; i++)
{
c.Connect(server);
string cdnAuthToken = null;
if (server.Type == "CDN")
{
steam3.RequestCDNAuthToken(depot.id, server.Host);
cdnAuthToken = steam3.CDNAuthTokens[Tuple.Create(depot.id, server.Host)].Token;
}
c.AuthenticateDepot(depot.id, depot.depotKey, cdnAuthToken);
cdnClients.Add(c);
}
if (cdnClients.Count >= Config.MaxServers) parallelLoop.Stop();
}
catch (Exception ex)
{
int penalty = 0;
ConfigStore.TheConfig.ContentServerPenalty.TryGetValue(server.Host, out penalty);
ConfigStore.TheConfig.ContentServerPenalty[server.Host] = penalty + 1;
Console.WriteLine("\nFailed to connect to content server {0}: {1}", server, ex.Message);
}
});
if (cdnClients.Count == 0)
{
Console.WriteLine("\nUnable to find any content servers for depot {0} - {1}", depot.id, depot.contentName);
}
Config.MaxDownloads = Math.Min(Config.MaxDownloads, cdnClients.Count);
return cdnClients;
}
