private static void ProcessDepotAfterDownload(IDbConnection db, ManifestJob request, DepotManifest depotManifest)
{
var filesOld = db.Query<DepotFile>("SELECT `ID`, `File`, `Hash`, `Size`, `Flags` FROM `DepotsFiles` WHERE `DepotID` = @DepotID", new { request.DepotID }).ToDictionary(x => x.File, x => x);
var filesNew = new List<DepotFile>();
var filesAdded = new List<DepotFile>();
var shouldHistorize = filesOld.Any(); // Don't historize file additions if we didn't have any data before
foreach (var file in depotManifest.Files)
{
var name = file.FileName.Replace('\\', '/');
// safe guard
if (name.Length > 255)
{
ErrorReporter.Notify(new OverflowException(string.Format("File \"{0}\" in depot {1} is too long", name, request.DepotID)));
continue;
}
var depotFile = new DepotFile
{
DepotID = request.DepotID,
File = name,
Size = file.TotalSize,
Flags = file.Flags
};
if (file.FileHash.Length > 0 && !file.Flags.HasFlag(EDepotFileFlag.Directory))
{
depotFile.Hash = Utils.ByteArrayToString(file.FileHash);
}
else
{
depotFile.Hash = "0000000000000000000000000000000000000000";
}
filesNew.Add(depotFile);
}
foreach (var file in filesNew)
{
if (filesOld.ContainsKey(file.File))
{
var oldFile = filesOld[file.File];
var updateFile = false;
if (oldFile.Size != file.Size || !file.Hash.Equals(oldFile.Hash))
{
MakeHistory(db, request, file.File, "modified", oldFile.Size, file.Size);
updateFile = true;
}
if (oldFile.Flags != file.Flags)
{
MakeHistory(db, request, file.File, "modified_flags", (ulong)oldFile.Flags, (ulong)file.Flags);
updateFile = true;
}
if (updateFile)
{
file.ID = oldFile.ID;
db.Execute("UPDATE `DepotsFiles` SET `Hash` = @Hash, `Size` = @Size, `Flags` = @Flags WHERE `DepotID` = @DepotID AND `ID` = @ID", file);
}
filesOld.Remove(file.File);
}
else
{
// We want to historize modifications first, and only then deletions and additions
filesAdded.Add(file);
}
}
if (filesOld.Any())
{
db.Execute("DELETE FROM `DepotsFiles` WHERE `DepotID` = @DepotID AND `ID` IN @Files", new { request.DepotID, Files = filesOld.Select(x => x.Value.ID) });
db.Execute(GetHistoryQuery(), filesOld.Select(x => new DepotHistory
{
DepotID = request.DepotID,
ChangeID = request.ChangeNumber,
Action = "removed",
File = x.Value.File
}));
}
if (filesAdded.Any())
{
db.Execute("INSERT INTO `DepotsFiles` (`DepotID`, `File`, `Hash`, `Size`, `Flags`) VALUES (@DepotID, @File, @Hash, @Size, @Flags)", filesAdded);
if (shouldHistorize)
{
db.Execute(GetHistoryQuery(), filesAdded.Select(x => new DepotHistory
{
DepotID = request.DepotID,
ChangeID = request.ChangeNumber,
Action = "added",
File = x.File
}));
}
}
db.Execute("UPDATE `Depots` SET `LastManifestID` = @ManifestID WHERE `DepotID` = @DepotID", new { request.DepotID, request.ManifestID });
}
private void DownloadManifest(ManifestJob request)
{
DepotManifest depotManifest = null;
string lastError = string.Empty;
// CDN is very random, just keep trying
for (var i = 0; i <= 5; i++)
{
try
{
depotManifest = CDNClient.DownloadManifest(request.DepotID, request.ManifestID, request.Server, request.CDNToken, request.DepotKey);
break;
}
catch (Exception e)
{
lastError = e.Message;
}
}
if (depotManifest == null)
{
Log.WriteError("Depot Processor", "Failed to download depot manifest for depot {0} ({1}: {2})", request.DepotID, request.Server, lastError);
if (Application.ImportantApps.ContainsKey(request.ParentAppID))
{
IRC.Instance.SendMain("Important depot update: {0}{1}{2} -{3} failed to download depot manifest", Colors.OLIVE, request.DepotName, Colors.NORMAL, Colors.RED);
}
return;
}
if (Application.ImportantApps.ContainsKey(request.ParentAppID))
{
IRC.Instance.SendMain("Important depot update: {0}{1}{2} -{3} {4}", Colors.OLIVE, request.DepotName, Colors.NORMAL, Colors.DARKBLUE, SteamDB.GetDepotURL(request.DepotID, "history"));
}
var sortedFiles = depotManifest.Files.OrderBy(f => f.FileName, StringComparer.OrdinalIgnoreCase);
bool shouldHistorize = false;
var filesNew = new List<DepotFile>();
var filesOld = new Dictionary<string, DepotFile>();
foreach (var file in sortedFiles)
{
System.Text.Encoding.UTF8.GetString(file.FileHash);
var depotFile = new DepotFile
{
Name = file.FileName.Replace('\\', '/'),
Size = file.TotalSize,
Chunks = file.Chunks.Count,
Flags = (int)file.Flags
};
// TODO: Ideally we would check if filehash is not empty
if (!file.Flags.HasFlag(EDepotFileFlag.Directory))
{
depotFile.Hash = string.Concat(Array.ConvertAll(file.FileHash, x => x.ToString("X2")));
}
filesNew.Add(depotFile);
}
using (MySqlDataReader Reader = DbWorker.ExecuteReader("SELECT `Files` FROM `Depots` WHERE `DepotID` = @DepotID LIMIT 1", new MySqlParameter("DepotID", request.DepotID)))
{
if (Reader.Read())
{
string files = Reader.GetString("Files");
if (!string.IsNullOrEmpty(files))
{
shouldHistorize = true;
filesOld = JsonConvert.DeserializeObject<List<DepotFile>>(files).ToDictionary(x => x.Name);
}
}
}
DbWorker.ExecuteNonQuery(
"UPDATE `Depots` SET `Files` = @Files WHERE `DepotID` = @DepotID",
new MySqlParameter("@DepotID", request.DepotID),
new MySqlParameter("@Files", JsonConvert.SerializeObject(filesNew, new JsonSerializerSettings { DefaultValueHandling = DefaultValueHandling.Ignore }))
);
if (shouldHistorize)
{
var filesAdded = new List<string>();
foreach (var file in filesNew)
{
if (filesOld.ContainsKey(file.Name))
{
var oldFile = filesOld[file.Name];
if (oldFile.Size != file.Size)
{
MakeHistory(request, file.Name, "modified", oldFile.Size, file.Size);
}
else if (file.Hash != null && oldFile.Hash != null && !file.Hash.Equals(oldFile.Hash))
{
MakeHistory(request, file.Name, "modified", oldFile.Size, file.Size);
}
filesOld.Remove(file.Name);
}
else
{
// We want to historize modifications first, and only then deletions and additions
filesAdded.Add(file.Name);
}
}
foreach (var file in filesOld)
{
MakeHistory(request, file.Value.Name, "removed");
}
foreach (string file in filesAdded)
{
MakeHistory(request, file, "added");
}
}
}
private async Task <EResult> ProcessDepotAfterDownload(IDbConnection db, IDbTransaction transaction, ManifestJob request, DepotManifest depotManifest)
{
var filesOld = (await db.QueryAsync <DepotFile>("SELECT `ID`, `File`, `Hash`, `Size`, `Flags` FROM `DepotsFiles` WHERE `DepotID` = @DepotID", new { request.DepotID }, transaction: transaction)).ToDictionary(x => x.File, x => x);
var filesNew = new List <DepotFile>();
var filesAdded = new List <DepotFile>();
var shouldHistorize = filesOld.Any(); // Don't historize file additions if we didn't have any data before
foreach (var file in depotManifest.Files)
{
var name = file.FileName.Replace('\\', '/');
// safe guard
if (name.Length > 255)
{
ErrorReporter.Notify("Depot Processor", new OverflowException(string.Format("File \"{0}\" in depot {1} is too long", name, request.DepotID)));
continue;
}
var depotFile = new DepotFile
{
DepotID = request.DepotID,
File = name,
Size = file.TotalSize,
Flags = file.Flags
};
if (file.FileHash.Length > 0 && !file.Flags.HasFlag(EDepotFileFlag.Directory))
{
depotFile.Hash = Utils.ByteArrayToString(file.FileHash);
}
else
{
depotFile.Hash = "0000000000000000000000000000000000000000";
}
filesNew.Add(depotFile);
}
foreach (var file in filesNew)
{
if (filesOld.ContainsKey(file.File))
{
var oldFile = filesOld[file.File];
var updateFile = false;
if (oldFile.Size != file.Size || !file.Hash.Equals(oldFile.Hash))
{
await MakeHistory(db, transaction, request, file.File, "modified", oldFile.Size, file.Size);
updateFile = true;
}
if (oldFile.Flags != file.Flags)
{
await MakeHistory(db, transaction, request, file.File, "modified_flags", (ulong)oldFile.Flags, (ulong)file.Flags);
updateFile = true;
}
if (updateFile)
{
file.ID = oldFile.ID;
await db.ExecuteAsync("UPDATE `DepotsFiles` SET `Hash` = @Hash, `Size` = @Size, `Flags` = @Flags WHERE `DepotID` = @DepotID AND `ID` = @ID", file, transaction : transaction);
}
filesOld.Remove(file.File);
}
else
{
// We want to historize modifications first, and only then deletions and additions
filesAdded.Add(file);
}
}
if (filesOld.Any())
{
await db.ExecuteAsync("DELETE FROM `DepotsFiles` WHERE `DepotID` = @DepotID AND `ID` IN @Files", new { request.DepotID, Files = filesOld.Select(x => x.Value.ID) }, transaction : transaction);
await db.ExecuteAsync(HistoryQuery, filesOld.Select(x => new DepotHistory
{
DepotID = request.DepotID,
ChangeID = request.ChangeNumber,
Action = "removed",
File = x.Value.File
}), transaction : transaction);
}
if (filesAdded.Any())
{
await db.ExecuteAsync("INSERT INTO `DepotsFiles` (`DepotID`, `File`, `Hash`, `Size`, `Flags`) VALUES (@DepotID, @File, @Hash, @Size, @Flags)", filesAdded, transaction : transaction);
if (shouldHistorize)
{
await db.ExecuteAsync(HistoryQuery, filesAdded.Select(x => new DepotHistory
{
DepotID = request.DepotID,
ChangeID = request.ChangeNumber,
Action = "added",
File = x.File
}), transaction : transaction);
}
}
await db.ExecuteAsync("UPDATE `Depots` SET `LastManifestID` = @ManifestID, `LastUpdated` = CURRENT_TIMESTAMP() WHERE `DepotID` = @DepotID", new { request.DepotID, request.ManifestID }, transaction : transaction);
return(EResult.OK);
}
private static void ProcessDepotAfterDownload(IDbConnection db, ManifestJob request, DepotManifest depotManifest)
{
var filesOld = db.Query <DepotFile>("SELECT `ID`, `File`, `Hash`, `Size`, `Flags` FROM `DepotsFiles` WHERE `DepotID` = @DepotID", new { request.DepotID }).ToDictionary(x => x.File, x => x);
var filesNew = new List <DepotFile>();
var filesAdded = new List <DepotFile>();
var shouldHistorize = filesOld.Any(); // Don't historize file additions if we didn't have any data before
foreach (var file in depotManifest.Files)
{
var name = file.FileName.Replace('\\', '/');
// safe guard
if (name.Length > 255)
{
ErrorReporter.Notify(new OverflowException(string.Format("File \"{0}\" in depot {1} is too long", name, request.DepotID)));
continue;
}
var depotFile = new DepotFile
{
DepotID = request.DepotID,
File = name,
Size = file.TotalSize,
Flags = file.Flags
};
if (file.FileHash.Length > 0 && !file.Flags.HasFlag(EDepotFileFlag.Directory))
{
depotFile.Hash = string.Concat(Array.ConvertAll(file.FileHash, x => x.ToString("X2")));
}
else
{
depotFile.Hash = "0000000000000000000000000000000000000000";
}
filesNew.Add(depotFile);
}
foreach (var file in filesNew)
{
if (filesOld.ContainsKey(file.File))
{
var oldFile = filesOld[file.File];
var updateFile = false;
if (oldFile.Size != file.Size || !file.Hash.Equals(oldFile.Hash))
{
MakeHistory(db, request, file.File, "modified", oldFile.Size, file.Size);
updateFile = true;
}
if (oldFile.Flags != file.Flags)
{
MakeHistory(db, request, file.File, "modified_flags", (ulong)oldFile.Flags, (ulong)file.Flags);
updateFile = true;
}
if (updateFile)
{
db.Execute("UPDATE `DepotsFiles` SET `Hash` = @Hash, `Size` = @Size, `Flags` = @Flags WHERE `DepotID` = @DepotID AND `ID` = @ID", file);
}
filesOld.Remove(file.File);
}
else
{
// We want to historize modifications first, and only then deletions and additions
filesAdded.Add(file);
}
}
if (filesOld.Any())
{
db.Execute("DELETE FROM `DepotsFiles` WHERE `DepotID` = @DepotID AND `ID` IN @Files", new { request.DepotID, Files = filesOld.Select(x => x.Value.ID) });
db.Execute(GetHistoryQuery(), filesOld.Select(x => new DepotHistory
{
DepotID = request.DepotID,
ChangeID = request.ChangeNumber,
Action = "removed",
File = x.Value.File
}));
}
if (filesAdded.Any())
{
db.Execute("INSERT INTO `DepotsFiles` (`DepotID`, `File`, `Hash`, `Size`, `Flags`) VALUES (@DepotID, @File, @Hash, @Size, @Flags)", filesAdded);
if (shouldHistorize)
{
db.Execute(GetHistoryQuery(), filesAdded.Select(x => new DepotHistory
{
DepotID = request.DepotID,
ChangeID = request.ChangeNumber,
Action = "added",
File = x.File
}));
}
}
db.Execute("UPDATE `Depots` SET `LastManifestID` = @ManifestID WHERE `DepotID` = @DepotID", new { request.DepotID, request.ManifestID });
}
private static void DownloadManifest(ManifestJob request)
{
var cdnClient = new CDNClient(Steam.Instance.Client, request.DepotID, request.Ticket, request.DepotKey);
List<CDNClient.Server> cdnServers = null;
for (var i = 0; i <= 5; i++)
{
try
{
cdnServers = cdnClient.FetchServerList();
if (cdnServers.Count > 0)
{
break;
}
}
catch { }
}
if (cdnServers == null || cdnServers.Count == 0)
{
Log.WriteError("Depot Processor", "Failed to get server list for depot {0}", request.DepotID);
if (SteamProxy.Instance.ImportantApps.Contains(request.ParentAppID))
{
IRC.SendMain("Important manifest update: {0}{1}{2} {3}(parent {4}){5} -{6} failed to fetch server list", Colors.OLIVE, request.DepotName, Colors.NORMAL, Colors.DARK_GRAY, request.ParentAppID, Colors.NORMAL, Colors.RED);
}
lock (ManifestJobs)
{
ManifestJobs.Remove(request);
}
return;
}
DepotManifest depotManifest = null;
foreach (var server in cdnServers)
{
try
{
cdnClient.Connect(server);
depotManifest = cdnClient.DownloadManifest(request.ManifestID);
break;
}
catch { }
}
lock (ManifestJobs)
{
ManifestJobs.Remove(request);
}
if (depotManifest == null)
{
Log.WriteError("Depot Processor", "Failed to download depot manifest for depot {0} (parent {1}) (jobs still in queue: {2})", request.DepotID, request.ParentAppID, ManifestJobs.Count);
if (SteamProxy.Instance.ImportantApps.Contains(request.ParentAppID))
{
IRC.SendMain("Important manifest update: {0}{1}{2} {3}(parent {4}){5} -{6} failed to download depot manifest from {7} servers", Colors.OLIVE, request.DepotName, Colors.NORMAL, Colors.DARK_GRAY, request.ParentAppID, Colors.NORMAL, Colors.RED, cdnServers.Count);
}
return;
}
if (SteamProxy.Instance.ImportantApps.Contains(request.ParentAppID))
{
IRC.SendMain("Important manifest update: {0}{1}{2} {3}(parent {4}){5} -{6} {7}", Colors.OLIVE, request.DepotName, Colors.NORMAL, Colors.DARK_GRAY, request.ParentAppID, Colors.NORMAL, Colors.DARK_BLUE, SteamDB.GetDepotURL(request.DepotID, "history"));
}
var sortedFiles = depotManifest.Files.OrderBy(f => f.FileName, StringComparer.OrdinalIgnoreCase);
bool shouldHistorize = false;
var filesNew = new List<DepotFile>();
var filesOld = new List<DepotFile>();
foreach (var file in sortedFiles)
{
System.Text.Encoding.UTF8.GetString(file.FileHash);
var depotFile = new DepotFile
{
Name = file.FileName.Replace('\\', '/'),
Size = file.TotalSize,
Chunks = file.Chunks.Count,
Flags = (int)file.Flags
};
// TODO: Ideally we would check if filehash is not empty
if (!file.Flags.HasFlag(EDepotFileFlag.Directory))
{
depotFile.Hash = string.Concat(Array.ConvertAll(file.FileHash, x => x.ToString("X2")));
}
filesNew.Add(depotFile);
}
using (MySqlDataReader Reader = DbWorker.ExecuteReader("SELECT `Files` FROM `Depots` WHERE `DepotID` = @DepotID LIMIT 1", new MySqlParameter("DepotID", request.DepotID)))
{
if (Reader.Read())
{
string files = Reader.GetString("Files");
if (!string.IsNullOrEmpty(files))
{
shouldHistorize = true;
filesOld = JsonConvert.DeserializeObject<List<DepotFile>>(files);
}
}
}
DbWorker.ExecuteNonQuery("UPDATE `Depots` SET `Files` = @Files WHERE `DepotID` = @DepotID",
new MySqlParameter("@DepotID", request.DepotID),
new MySqlParameter("@Files", JsonConvert.SerializeObject(filesNew, new JsonSerializerSettings { DefaultValueHandling = DefaultValueHandling.Ignore }))
);
if (shouldHistorize)
{
var filesAdded = new List<string>();
foreach (var file in filesNew)
{
var oldFile = filesOld.Find(x => x.Name == file.Name);
if (oldFile == null)
{
// We want to historize modifications first, and only then deletions and additions
filesAdded.Add(file.Name);
}
else
{
if (oldFile.Size != file.Size)
{
MakeHistory(request, file.Name, "modified", oldFile.Size, file.Size);
}
else if (file.Hash != null && oldFile.Hash != null && !file.Hash.Equals(oldFile.Hash))
{
MakeHistory(request, file.Name, "modified", oldFile.Size, file.Size);
}
filesOld.Remove(oldFile);
}
}
foreach (var file in filesOld)
{
MakeHistory(request, file.Name, "removed");
}
foreach (string file in filesAdded)
{
MakeHistory(request, file, "added");
}
}
lock (ManifestJobs)
{
Log.WriteDebug("Depot Processor", "DepotID: Processed {0} (jobs still in queue: {1})", request.DepotID, ManifestJobs.Count);
}
}
private static void DownloadManifest(ManifestJob request)
{
DepotManifest depotManifest = null;
string lastError = string.Empty;
// CDN is very random, just keep trying
for (var i = 0; i <= 5; i++)
{
try
{
depotManifest = CDNClient.DownloadManifest(request.DepotID, request.ManifestID, request.Server, request.DepotKey, request.CDNToken);
break;
}
catch (Exception e)
{
lastError = e.Message;
}
}
if (depotManifest == null)
{
Log.WriteError("Depot Processor", "Failed to download depot manifest for depot {0} (jobs still in queue: {1}) ({2}: {3})", request.DepotID, ManifestJobs.Count - 1, request.Server, lastError);
if (SteamProxy.Instance.ImportantApps.Contains(request.ParentAppID))
{
IRC.SendMain("Important depot update: {0}{1}{2} -{3} failed to download depot manifest", Colors.OLIVE, request.DepotName, Colors.NORMAL, Colors.RED);
}
return;
}
if (SteamProxy.Instance.ImportantApps.Contains(request.ParentAppID))
{
IRC.SendMain("Important depot update: {0}{1}{2} -{3} {4}", Colors.OLIVE, request.DepotName, Colors.NORMAL, Colors.DARK_BLUE, SteamDB.GetDepotURL(request.DepotID, "history"));
}
var sortedFiles = depotManifest.Files.OrderBy(f => f.FileName, StringComparer.OrdinalIgnoreCase);
bool shouldHistorize = false;
var filesNew = new List <DepotFile>();
var filesOld = new Dictionary <string, DepotFile>();
foreach (var file in sortedFiles)
{
System.Text.Encoding.UTF8.GetString(file.FileHash);
var depotFile = new DepotFile
{
Name = file.FileName.Replace('\\', '/'),
Size = file.TotalSize,
Chunks = file.Chunks.Count,
Flags = (int)file.Flags
};
// TODO: Ideally we would check if filehash is not empty
if (!file.Flags.HasFlag(EDepotFileFlag.Directory))
{
depotFile.Hash = string.Concat(Array.ConvertAll(file.FileHash, x => x.ToString("X2")));
}
filesNew.Add(depotFile);
}
using (MySqlDataReader Reader = DbWorker.ExecuteReader("SELECT `Files` FROM `Depots` WHERE `DepotID` = @DepotID LIMIT 1", new MySqlParameter("DepotID", request.DepotID)))
{
if (Reader.Read())
{
string files = Reader.GetString("Files");
if (!string.IsNullOrEmpty(files))
{
shouldHistorize = true;
var _filesOld = JsonConvert.DeserializeObject <List <DepotFile> >(files);
filesOld = _filesOld.ToDictionary(x => x.Name);
}
}
}
DbWorker.ExecuteNonQuery("UPDATE `Depots` SET `Files` = @Files WHERE `DepotID` = @DepotID",
new MySqlParameter("@DepotID", request.DepotID),
new MySqlParameter("@Files", JsonConvert.SerializeObject(filesNew, new JsonSerializerSettings {
DefaultValueHandling = DefaultValueHandling.Ignore
}))
);
if (shouldHistorize)
{
var filesAdded = new List <string>();
foreach (var file in filesNew)
{
if (filesOld.ContainsKey(file.Name))
{
var oldFile = filesOld[file.Name];
if (oldFile.Size != file.Size)
{
MakeHistory(request, file.Name, "modified", oldFile.Size, file.Size);
}
else if (file.Hash != null && oldFile.Hash != null && !file.Hash.Equals(oldFile.Hash))
{
MakeHistory(request, file.Name, "modified", oldFile.Size, file.Size);
}
filesOld.Remove(file.Name);
}
else
{
// We want to historize modifications first, and only then deletions and additions
filesAdded.Add(file.Name);
}
}
foreach (var file in filesOld)
{
MakeHistory(request, file.Value.Name, "removed");
}
foreach (string file in filesAdded)
{
MakeHistory(request, file, "added");
}
}
lock (ManifestJobs)
{
Log.WriteDebug("Depot Processor", "DepotID: Processed {0} (jobs still in queue: {1})", request.DepotID, ManifestJobs.Count - 1);
}
}