private static async Task GetPackage(HttpListenerContext context)
{
var subid = context.Request.QueryString.Get("subid");
if (subid == null)
{
throw new MissingFieldException("subid parameter is missing");
}
var result = await PackageCommand.GetPackageData(uint.Parse(subid));
if (result == null)
{
context.Response.StatusCode = (int)HttpStatusCode.NotFound;
await WriteJsonResponse("App not found", context.Response);
return;
}
context.Response.ContentType = "text/vdf; charset=utf-8";
context.Response.Headers[HttpResponseHeader.ETag] = $"\"{Utils.ByteArrayToString(result.SHAHash)}\"";
await using var kvMemory = new MemoryStream();
result.KeyValues.SaveToStream(kvMemory, false);
kvMemory.Position = 0;
await kvMemory.CopyToAsync(context.Response.OutputStream);
}
public override async Task OnCommand(CommandArguments command)
{
if (command.Message.Length == 0 || !uint.TryParse(command.Message, out var subID))
{
command.Reply($"Usage:{Colors.OLIVE} sub <subid>");
return;
}
var info = await GetPackageData(subID);
if (info == null)
{
command.Reply($"Unknown SubID: {Colors.BLUE}{subID}{(LicenseList.OwnedSubs.ContainsKey(subID) ? SteamDB.StringCheckmark : string.Empty)}");
return;
}
if (!info.KeyValues.Children.Any())
{
command.Reply($"No package info returned for SubID: {Colors.BLUE}{subID}{(info.MissingToken ? SteamDB.StringNeedToken : string.Empty)}{(LicenseList.OwnedSubs.ContainsKey(subID) ? SteamDB.StringCheckmark : string.Empty)}");
return;
}
var filename = $"{Utils.ByteArrayToString(info.SHAHash)}.vdf";
info.KeyValues.SaveToFile(Path.Combine(Application.Path, "sub", filename), false);
command.Reply($"{Colors.BLUE}{Steam.GetPackageName(info.ID)}{Colors.NORMAL} -{Colors.DARKBLUE} <{SteamDB.GetPackageUrl(info.ID)}>{Colors.NORMAL} - Dump:{Colors.DARKBLUE} <{SteamDB.GetRawPackageUrl(filename)}>{Colors.NORMAL}{(info.MissingToken ? SteamDB.StringNeedToken : string.Empty)}{(LicenseList.OwnedSubs.ContainsKey(info.ID) ? SteamDB.StringCheckmark : string.Empty)}");
}
private void OnDepotKeyCallback(SteamApps.DepotKeyCallback callback)
{
JobAction job;
if (!JobManager.TryRemoveJob(callback.JobID, out job))
{
RemoveLock(callback.DepotID);
return;
}
var request = job.ManifestJob;
if (callback.Result != EResult.OK)
{
if (callback.Result != EResult.AccessDenied || FileDownloader.IsImportantDepot(request.DepotID))
{
Log.WriteError("Depot Processor", "Failed to get depot key for depot {0} (parent {1}) - {2}", callback.DepotID, request.ParentAppID, callback.Result);
}
RemoveLock(request.DepotID);
return;
}
request.DepotKey = callback.DepotKey;
request.Tries = CDNServers.Count;
request.Server = GetContentServer();
JobManager.AddJob(() => Steam.Instance.Apps.GetCDNAuthToken(request.DepotID, request.Server), request);
var decryptionKey = Utils.ByteArrayToString(callback.DepotKey);
using (var db = Database.GetConnection())
{
var currentDecryptionKey = db.ExecuteScalar <string>("SELECT `Key` FROM `DepotsKeys` WHERE `DepotID` = @DepotID", new { callback.DepotID });
if (decryptionKey != currentDecryptionKey)
{
if (currentDecryptionKey != null)
{
Log.WriteInfo("Depot Processor", "Decryption key for {0} changed: {1} -> {2}", callback.DepotID, currentDecryptionKey, decryptionKey);
IRC.Instance.SendOps("Decryption key for {0} changed: {1} -> {2}", callback.DepotID, currentDecryptionKey, decryptionKey);
}
db.Execute("INSERT INTO `DepotsKeys` (`DepotID`, `Key`) VALUES (@DepotID, @Key) ON DUPLICATE KEY UPDATE `Key` = VALUES(`Key`)", new { callback.DepotID, Key = decryptionKey });
}
}
}
public override async Task OnCommand(CommandArguments command)
{
if (string.IsNullOrWhiteSpace(command.Message))
{
command.Reply($"Usage:{Colors.OLIVE} app <appid or partial game name>");
return;
}
if (!uint.TryParse(command.Message, out var appID))
{
appID = await TrySearchAppId(command);
if (appID == 0)
{
return;
}
}
var info = await GetAppData(appID);
if (info == null)
{
command.Reply($"Unknown AppID: {Colors.BLUE}{appID}{(LicenseList.OwnedApps.ContainsKey(appID) ? SteamDB.StringCheckmark : string.Empty)}");
return;
}
string name;
if (info.KeyValues["common"]["name"].Value != null)
{
name = Utils.LimitStringLength(Utils.RemoveControlCharacters(info.KeyValues["common"]["name"].AsString()));
}
else
{
name = Steam.GetAppName(info.ID);
}
var filename = $"{Utils.ByteArrayToString(info.SHAHash)}.vdf";
info.KeyValues.SaveToFile(Path.Combine(Application.Path, "app", filename), false);
command.Reply($"{Colors.BLUE}{name}{Colors.NORMAL} -{Colors.DARKBLUE} <{SteamDB.GetAppUrl(info.ID)}>{Colors.NORMAL} - Dump:{Colors.DARKBLUE} <{SteamDB.GetRawAppUrl(filename)}>{Colors.NORMAL}{(info.MissingToken ? SteamDB.StringNeedToken : string.Empty)}{(LicenseList.OwnedApps.ContainsKey(info.ID) ? SteamDB.StringCheckmark : string.Empty)}");
if (command.Recipient == Settings.Current.IRC.Channel.Ops && !LicenseList.OwnedApps.ContainsKey(info.ID))
{
JobManager.AddJob(() => Steam.Instance.Apps.RequestFreeLicense(info.ID));
}
}
private async Task <byte[]> GetDepotDecryptionKey(SteamApps instance, uint depotID, uint appID)
{
using (var db = Database.Get())
{
var currentDecryptionKey = await db.ExecuteScalarAsync <string>("SELECT `Key` FROM `DepotsKeys` WHERE `DepotID` = @DepotID", new { depotID });
if (currentDecryptionKey != null)
{
return(Utils.StringToByteArray(currentDecryptionKey));
}
}
var task = instance.GetDepotDecryptionKey(depotID, appID);
task.Timeout = TimeSpan.FromMinutes(15);
SteamApps.DepotKeyCallback callback;
try
{
callback = await task;
}
catch (TaskCanceledException)
{
Log.WriteError("Depot Processor", "Decryption key timed out for {0}", depotID);
return(null);
}
if (callback.Result != EResult.OK)
{
if (callback.Result != EResult.AccessDenied)
{
Log.WriteError("Depot Processor", "No access to depot {0} ({1})", depotID, callback.Result);
}
return(null);
}
Log.WriteDebug("Depot Downloader", "Got a new depot key for depot {0}", depotID);
using (var db = Database.Get())
{
await db.ExecuteAsync("INSERT INTO `DepotsKeys` (`DepotID`, `Key`) VALUES (@DepotID, @Key) ON DUPLICATE KEY UPDATE `Key` = VALUES(`Key`)", new { depotID, Key = Utils.ByteArrayToString(callback.DepotKey) });
}
return(callback.DepotKey);
}
private async Task <EPurchaseResultDetail> ActivateKey(string key)
{
var msg = new ClientMsgProtobuf <CMsgClientRegisterKey>(EMsg.ClientRegisterKey)
{
SourceJobID = Steam.Instance.Client.GetNextJobID(),
Body =
{
key = key
}
};
Steam.Instance.Client.Send(msg);
PurchaseResponseCallback job;
try
{
job = await new AsyncJob <PurchaseResponseCallback>(Steam.Instance.Client, msg.SourceJobID);
}
catch (Exception)
{
return(EPurchaseResultDetail.Timeout);
}
await using (var db = await Database.GetConnectionAsync())
{
using var sha = new SHA1CryptoServiceProvider();
await db.ExecuteAsync("UPDATE `SteamKeys` SET `SteamKey` = @HashedKey, `SubID` = @SubID, `Result` = @PurchaseResultDetail WHERE `SteamKey` = @SteamKey OR `SteamKey` = @HashedKey",
new
{
job.PurchaseResultDetail,
SubID = job.Packages.Count > 0 ? (int)job.Packages.First().Key : -1,
SteamKey = key,
HashedKey = Utils.ByteArrayToString(sha.ComputeHash(Encoding.ASCII.GetBytes(key)))
});
}
if (job.Packages.Count == 0)
{
if (job.PurchaseResultDetail != EPurchaseResultDetail.BadActivationCode)
{
IRC.Instance.SendOps($"{Colors.GREEN}[Keys]{Colors.NORMAL} Key not activated:{Colors.OLIVE} {job.Result} - {job.PurchaseResultDetail}");
}
return(job.PurchaseResultDetail);
}
if (job.PurchaseResultDetail != EPurchaseResultDetail.AlreadyPurchased &&
job.PurchaseResultDetail != EPurchaseResultDetail.DuplicateActivationCode &&
job.PurchaseResultDetail != EPurchaseResultDetail.DoesNotOwnRequiredApp)
{
var response = job.PurchaseResultDetail == EPurchaseResultDetail.NoDetail ?
$"{Colors.GREEN}Key activated" : $"{Colors.BLUE}{job.PurchaseResultDetail}";
IRC.Instance.SendOps($"{Colors.GREEN}[Keys]{Colors.NORMAL} {response}{Colors.NORMAL}. Packages:{Colors.OLIVE} {string.Join(", ", job.Packages.Select(x => $"{x.Key}: {x.Value}"))}");
}
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(Enumerable.Empty <uint>(), job.Packages.Keys));
await using (var db = await Database.GetConnectionAsync())
{
var apps = await db.QueryAsync <uint>("SELECT `AppID` FROM `SubsApps` WHERE `Type` = \"app\" AND `SubID` IN @Ids", new { Ids = job.Packages.Keys });
JobManager.AddJob(() => Steam.Instance.Apps.PICSGetAccessTokens(apps, Enumerable.Empty <uint>()));
foreach (var package in job.Packages)
{
var databaseName = (await db.QueryAsync <string>("SELECT `LastKnownName` FROM `Subs` WHERE `SubID` = @SubID", new { SubID = package.Key })).FirstOrDefault() ?? string.Empty;
if (databaseName.Equals(package.Value, StringComparison.CurrentCultureIgnoreCase))
{
continue;
}
await db.ExecuteAsync("UPDATE `Subs` SET `LastKnownName` = @Name WHERE `SubID` = @SubID", new { SubID = package.Key, Name = package.Value });
await db.ExecuteAsync(SubProcessor.HistoryQuery,
new PICSHistory
{
ID = package.Key,
Key = SteamDB.DATABASE_NAME_TYPE,
OldValue = "key activation",
NewValue = package.Value,
Action = "created_info"
}
);
}
}
return(job.PurchaseResultDetail);
}
private async Task <EResult> ProcessDepotAfterDownload(IDbConnection db, IDbTransaction transaction, ManifestJob request, DepotManifest depotManifest)
{
var filesOld = (await db.QueryAsync <DepotFile>("SELECT `ID`, `File`, `Hash`, `Size`, `Flags` FROM `DepotsFiles` WHERE `DepotID` = @DepotID", new { request.DepotID }, transaction: transaction)).ToDictionary(x => x.File, x => x);
var filesNew = new List <DepotFile>();
var filesAdded = new List <DepotFile>();
var shouldHistorize = filesOld.Any(); // Don't historize file additions if we didn't have any data before
foreach (var file in depotManifest.Files)
{
var name = file.FileName.Replace('\\', '/');
// safe guard
if (name.Length > 255)
{
ErrorReporter.Notify("Depot Processor", new OverflowException(string.Format("File \"{0}\" in depot {1} is too long", name, request.DepotID)));
continue;
}
var depotFile = new DepotFile
{
DepotID = request.DepotID,
File = name,
Size = file.TotalSize,
Flags = file.Flags
};
if (file.FileHash.Length > 0 && !file.Flags.HasFlag(EDepotFileFlag.Directory))
{
depotFile.Hash = Utils.ByteArrayToString(file.FileHash);
}
else
{
depotFile.Hash = "0000000000000000000000000000000000000000";
}
filesNew.Add(depotFile);
}
foreach (var file in filesNew)
{
if (filesOld.ContainsKey(file.File))
{
var oldFile = filesOld[file.File];
var updateFile = false;
if (oldFile.Size != file.Size || !file.Hash.Equals(oldFile.Hash))
{
await MakeHistory(db, transaction, request, file.File, "modified", oldFile.Size, file.Size);
updateFile = true;
}
if (oldFile.Flags != file.Flags)
{
await MakeHistory(db, transaction, request, file.File, "modified_flags", (ulong)oldFile.Flags, (ulong)file.Flags);
updateFile = true;
}
if (updateFile)
{
file.ID = oldFile.ID;
await db.ExecuteAsync("UPDATE `DepotsFiles` SET `Hash` = @Hash, `Size` = @Size, `Flags` = @Flags WHERE `DepotID` = @DepotID AND `ID` = @ID", file, transaction : transaction);
}
filesOld.Remove(file.File);
}
else
{
// We want to historize modifications first, and only then deletions and additions
filesAdded.Add(file);
}
}
if (filesOld.Any())
{
await db.ExecuteAsync("DELETE FROM `DepotsFiles` WHERE `DepotID` = @DepotID AND `ID` IN @Files", new { request.DepotID, Files = filesOld.Select(x => x.Value.ID) }, transaction : transaction);
await db.ExecuteAsync(HistoryQuery, filesOld.Select(x => new DepotHistory
{
DepotID = request.DepotID,
ChangeID = request.ChangeNumber,
Action = "removed",
File = x.Value.File
}), transaction : transaction);
}
if (filesAdded.Any())
{
await db.ExecuteAsync("INSERT INTO `DepotsFiles` (`DepotID`, `File`, `Hash`, `Size`, `Flags`) VALUES (@DepotID, @File, @Hash, @Size, @Flags)", filesAdded, transaction : transaction);
if (shouldHistorize)
{
await db.ExecuteAsync(HistoryQuery, filesAdded.Select(x => new DepotHistory
{
DepotID = request.DepotID,
ChangeID = request.ChangeNumber,
Action = "added",
File = x.File
}), transaction : transaction);
}
}
await db.ExecuteAsync("UPDATE `Depots` SET `LastManifestID` = @ManifestID, `LastUpdated` = CURRENT_TIMESTAMP() WHERE `DepotID` = @DepotID", new { request.DepotID, request.ManifestID }, transaction : transaction);
return(EResult.OK);
}
private async Task <EPurchaseResultDetail> ActivateKey(string key, uint id = 0)
{
var msg = new ClientMsgProtobuf <CMsgClientRegisterKey>(EMsg.ClientRegisterKey)
{
SourceJobID = Steam.Instance.Client.GetNextJobID(),
Body =
{
key = key
}
};
Steam.Instance.Client.Send(msg);
PurchaseResponseCallback job;
try
{
job = await new AsyncJob <PurchaseResponseCallback>(Steam.Instance.Client, msg.SourceJobID);
}
catch (Exception)
{
return(EPurchaseResultDetail.Timeout);
}
await using var db = await Database.GetConnectionAsync();
if (id > 0)
{
using var sha = SHA1.Create();
await db.ExecuteAsync(
"UPDATE `SteamKeys` SET `SteamKey` = @HashedKey, `SubID` = @SubID, `Result` = @Result WHERE `ID` = @ID",
new
{
ID = id,
Result = job.PurchaseResultDetail,
SubID = job.Packages.Count > 0 ? (int)job.Packages.First().Key : -1,
HashedKey = Utils.ByteArrayToString(sha.ComputeHash(Encoding.ASCII.GetBytes(key)))
});
}
if (job.Packages.Count == 0)
{
if (job.PurchaseResultDetail != EPurchaseResultDetail.BadActivationCode &&
job.PurchaseResultDetail != EPurchaseResultDetail.DuplicateActivationCode &&
job.PurchaseResultDetail != EPurchaseResultDetail.RestrictedCountry)
{
IRC.Instance.SendOps($"{Colors.GREEN}[Keys]{Colors.NORMAL} Key not activated:{Colors.OLIVE} {job.Result} - {job.PurchaseResultDetail}");
}
return(job.PurchaseResultDetail);
}
if (job.PurchaseResultDetail == EPurchaseResultDetail.NoDetail)
{
_ = TaskManager.Run(async() => await Utils.SendWebhook(new
{
Type = "KeyActivated",
Key = id,
job.Packages,
}));
}
foreach (var(subid, name) in job.Packages)
{
var databaseName = (await db.QueryAsync <string>("SELECT `LastKnownName` FROM `Subs` WHERE `SubID` = @SubID", new { SubID = subid })).FirstOrDefault() ?? string.Empty;
if (databaseName.Equals(name, StringComparison.CurrentCultureIgnoreCase))
{
continue;
}
await db.ExecuteAsync("UPDATE `Subs` SET `LastKnownName` = @Name WHERE `SubID` = @SubID", new { SubID = subid, Name = name });
await db.ExecuteAsync(SubProcessor.HistoryQuery,
new PICSHistory
{
ID = subid,
Key = SteamDB.DatabaseNameType,
OldValue = "key activation",
NewValue = name,
Action = "created_info"
}
);
}
return(job.PurchaseResultDetail);
}
private static async void OnClanState(SteamFriends.ClanStateCallback callback)
{
if (callback.Events.Count == 0 && callback.Announcements.Count == 0)
{
return;
}
var groupName = callback.ClanName ?? Steam.Instance.Friends.GetClanName(callback.ClanID);
var groupAvatar = Utils.ByteArrayToString(callback.AvatarHash ?? Steam.Instance.Friends.GetClanAvatar(callback.ClanID) ?? System.Array.Empty <byte>()).ToLowerInvariant();
if (string.IsNullOrEmpty(groupName))
{
groupName = "Group";
}
foreach (var announcement in callback.Announcements)
{
var url = $"https://steamcommunity.com/gid/{callback.ClanID.AccountID}/announcements/detail/{announcement.ID}";
IRC.Instance.SendAnnounce($"{Colors.BLUE}{groupName}{Colors.NORMAL} announcement: {Colors.OLIVE}{announcement.Headline}{Colors.NORMAL} -{Colors.DARKBLUE} {url}");
_ = TaskManager.Run(async() => await Utils.SendWebhook(new
{
Type = "GroupAnnouncement",
Title = announcement.Headline,
Group = groupName,
Avatar = groupAvatar,
Url = url,
GroupID = callback.ClanID.AccountID,
}));
Log.WriteInfo(nameof(ClanState), $"{groupName} \"{announcement.Headline}\"");
}
await using var db = await Database.GetConnectionAsync();
foreach (var groupEvent in callback.Events)
{
var link = $"https://steamcommunity.com/gid/{callback.ClanID.AccountID}/events/{groupEvent.ID}";
var id = await db.ExecuteScalarAsync <int>("SELECT `ID` FROM `RSS` WHERE `Link` = @Link", new { Link = link });
if (id > 0)
{
continue;
}
IRC.Instance.SendAnnounce(
$"{Colors.BLUE}{groupName}{Colors.NORMAL} event: {Colors.OLIVE}{groupEvent.Headline}{Colors.NORMAL} -{Colors.DARKBLUE} {link} {Colors.DARKGRAY}({groupEvent.EventTime.ToString("s", CultureInfo.InvariantCulture).Replace("T", " ")})"
);
Log.WriteInfo(nameof(ClanState), $"{groupName} Event \"{groupEvent.Headline}\" {link}");
await db.ExecuteAsync("INSERT INTO `RSS` (`Link`, `Title`, `Date`) VALUES(@Link, @Title, @EventTime)", new
{
Link = link,
Title = groupEvent.Headline,
groupEvent.EventTime,
});
_ = TaskManager.Run(async() => await Utils.SendWebhook(new
{
Type = "GroupAnnouncement",
Title = groupEvent.Headline,
Group = groupName,
Avatar = groupAvatar,
Url = link,
GroupID = callback.ClanID.AccountID,
}));
}
}
private static async Task <EResult> ProcessDepotAfterDownload(IDbConnection db, IDbTransaction transaction, ManifestJob request, DepotManifest depotManifest)
{
var filesOld = (await db.QueryAsync <DepotFile>("SELECT `File`, `Hash`, `Size`, `Flags` FROM `DepotsFiles` WHERE `DepotID` = @DepotID", new { request.DepotID }, transaction)).ToDictionary(x => x.File, x => x);
var filesAdded = new List <DepotFile>();
var shouldHistorize = filesOld.Count > 0 && !depotManifest.FilenamesEncrypted; // Don't historize file additions if we didn't have any data before
if (request.StoredFilenamesEncrypted && !depotManifest.FilenamesEncrypted)
{
Log.WriteInfo(nameof(DepotProcessor), $"Depot {request.DepotID} will decrypt stored filenames");
var decryptedFilesOld = new Dictionary <string, DepotFile>();
foreach (var file in filesOld.Values)
{
var oldFile = file.File;
file.File = DecryptFilename(oldFile, request.DepotKey);
decryptedFilesOld.Add(file.File, file);
await db.ExecuteAsync("UPDATE `DepotsFiles` SET `File` = @File WHERE `DepotID` = @DepotID AND `File` = @OldFile", new
{
request.DepotID,
file.File,
OldFile = oldFile
}, transaction);
}
await MakeHistory(db, transaction, request, string.Empty, "files_decrypted");
filesOld = decryptedFilesOld;
}
foreach (var file in depotManifest.Files.OrderByDescending(x => x.FileName))
{
var name = depotManifest.FilenamesEncrypted ? file.FileName.Replace("\n", "") : file.FileName.Replace('\\', '/');
byte[] hash = null;
// Store empty hashes as NULL (e.g. an empty file)
if ((file.Flags & EDepotFileFlag.Directory) == 0 && file.FileHash.Length > 0 && file.FileHash.Any(c => c != 0))
{
hash = file.FileHash;
}
// Limit path names to 260 characters (default windows max length)
// File column is varchar(260) and not higher to prevent reducing performance
// See https://stackoverflow.com/questions/1962310/importance-of-varchar-length-in-mysql-table/1962329#1962329
// Until 2019 there hasn't been a single file that went over this limit, so far there has been only one
// game with a big node_modules path, so we're safeguarding by limiting it.
if (name.Length > 260)
{
if (depotManifest.FilenamesEncrypted)
{
continue;
}
using var sha = SHA1.Create();
var nameHash = Utils.ByteArrayToString(sha.ComputeHash(Encoding.UTF8.GetBytes(name)));
name = $"{{SteamDB file name is too long}}/{nameHash}/...{name.Substring(name.Length - 150)}";
}
if (filesOld.ContainsKey(name))
{
var oldFile = filesOld[name];
var updateFile = false;
if (oldFile.Size != file.TotalSize || !Utils.ByteArrayEquals(hash, oldFile.Hash))
{
await MakeHistory(db, transaction, request, name, "modified", oldFile.Size, file.TotalSize);
updateFile = true;
}
if (oldFile.Flags != file.Flags)
{
await MakeHistory(db, transaction, request, name, "modified_flags", (ulong)oldFile.Flags, (ulong)file.Flags);
updateFile = true;
}
if (updateFile)
{
await db.ExecuteAsync("UPDATE `DepotsFiles` SET `Hash` = @Hash, `Size` = @Size, `Flags` = @Flags WHERE `DepotID` = @DepotID AND `File` = @File", new DepotFile
{
DepotID = request.DepotID,
File = name,
Hash = hash,
Size = file.TotalSize,
Flags = file.Flags
}, transaction);
}
filesOld.Remove(name);
}
else
{
// We want to historize modifications first, and only then deletions and additions
filesAdded.Add(new DepotFile
{
DepotID = request.DepotID,
Hash = hash,
File = name,
Size = file.TotalSize,
Flags = file.Flags
});
}
}
if (filesOld.Count > 0)
{
// Chunk file deletion queries so it doesn't go over max_allowed_packet
var filesOldChunks = filesOld.Select(x => x.Value.File).Split(1000);
foreach (var filesOldChunk in filesOldChunks)
{
await db.ExecuteAsync("DELETE FROM `DepotsFiles` WHERE `DepotID` = @DepotID AND `File` IN @Files",
new
{
request.DepotID,
Files = filesOldChunk,
}, transaction);
}
if (shouldHistorize)
{
await db.ExecuteAsync(HistoryQuery, filesOld.Select(x => new DepotHistory
{
DepotID = request.DepotID,
ManifestID = request.ManifestID,
ChangeID = request.ChangeNumber,
Action = "removed",
File = x.Value.File,
OldValue = x.Value.Size
}), transaction);
}
}
if (filesAdded.Count > 0)
{
await db.ExecuteAsync("INSERT INTO `DepotsFiles` (`DepotID`, `File`, `Hash`, `Size`, `Flags`) VALUES (@DepotID, @File, @Hash, @Size, @Flags)", filesAdded, transaction);
if (shouldHistorize)
{
await db.ExecuteAsync(HistoryQuery, filesAdded.Select(x => new DepotHistory
{
DepotID = request.DepotID,
ManifestID = request.ManifestID,
ChangeID = request.ChangeNumber,
Action = "added",
File = x.File,
NewValue = x.Size
}), transaction);
}
}
await db.ExecuteAsync(
request.LastManifestID == request.ManifestID?
"UPDATE `Depots` SET `LastManifestID` = @ManifestID, `ManifestDate` = @ManifestDate, `FilenamesEncrypted` = @FilenamesEncrypted, `SizeOriginal` = @SizeOriginal, `SizeCompressed` = @SizeCompressed WHERE `DepotID` = @DepotID" :
"UPDATE `Depots` SET `LastManifestID` = @ManifestID, `ManifestDate` = @ManifestDate, `FilenamesEncrypted` = @FilenamesEncrypted, `SizeOriginal` = @SizeOriginal, `SizeCompressed` = @SizeCompressed, `LastUpdated` = CURRENT_TIMESTAMP() WHERE `DepotID` = @DepotID",
new
{
request.DepotID,
request.ManifestID,
depotManifest.FilenamesEncrypted,
ManifestDate = depotManifest.CreationTime,
SizeOriginal = depotManifest.TotalUncompressedSize,
SizeCompressed = depotManifest.TotalCompressedSize,
}, transaction);
return(EResult.OK);
}
private static async Task GetDepotDecryptionKey(SteamApps instance, ManifestJob depot, uint appID)
{
var task = instance.GetDepotDecryptionKey(depot.DepotID, appID);
task.Timeout = TimeSpan.FromMinutes(15);
SteamApps.DepotKeyCallback callback;
try
{
callback = await task;
}
catch (TaskCanceledException)
{
Log.WriteWarn(nameof(DepotProcessor), $"Decryption key timed out for {depot.DepotID}");
return;
}
if (callback.Result != EResult.OK)
{
if (callback.Result != EResult.AccessDenied)
{
Log.WriteWarn(nameof(DepotProcessor), $"No access to depot {depot.DepotID} ({callback.Result})");
}
return;
}
Log.WriteDebug(nameof(DepotProcessor), $"Got a new depot key for depot {depot.DepotID}");
await using (var db = await Database.GetConnectionAsync())
{
await db.ExecuteAsync("INSERT INTO `DepotsKeys` (`DepotID`, `Key`) VALUES (@DepotID, @Key) ON DUPLICATE KEY UPDATE `Key` = VALUES(`Key`)", new { depot.DepotID, Key = Utils.ByteArrayToString(callback.DepotKey) });
}
depot.DepotKey = callback.DepotKey;
}
private static async Task <EResult> ProcessDepotAfterDownload(IDbConnection db, IDbTransaction transaction, ManifestJob request, DepotManifest depotManifest)
{
var filesOld = (await db.QueryAsync <DepotFile>("SELECT `ID`, `File`, `Hash`, `Size`, `Flags` FROM `DepotsFiles` WHERE `DepotID` = @DepotID", new { request.DepotID }, transaction: transaction)).ToDictionary(x => x.File, x => x);
var filesAdded = new List <DepotFile>();
var shouldHistorize = filesOld.Count > 0; // Don't historize file additions if we didn't have any data before
foreach (var file in depotManifest.Files.OrderByDescending(x => x.FileName))
{
var name = file.FileName.Replace('\\', '/');
byte[] hash = null;
// Store empty hashes as NULL (e.g. an empty file)
if (file.FileHash.Length > 0 && (file.Flags & EDepotFileFlag.Directory) == 0)
{
for (var i = 0; i < file.FileHash.Length; ++i)
{
if (file.FileHash[i] != 0)
{
hash = file.FileHash;
break;
}
}
}
// Limit path names to 260 characters (default windows max length)
// File column is varchar(260) and not higher to prevent reducing performance
// See https://stackoverflow.com/questions/1962310/importance-of-varchar-length-in-mysql-table/1962329#1962329
// Until 2019 there hasn't been a single file that went over this limit, so far there has been only one
// game with a big node_modules path, so we're safeguarding by limiting it.
if (name.Length > 260)
{
using var sha = new System.Security.Cryptography.SHA1Managed();
var nameHash = Utils.ByteArrayToString(sha.ComputeHash(System.Text.Encoding.UTF8.GetBytes(name)));
name = $"{{SteamDB file name is too long}}/{nameHash}/...{name.Substring(name.Length - 150)}";
}
if (filesOld.ContainsKey(name))
{
var oldFile = filesOld[name];
var updateFile = false;
if (oldFile.Size != file.TotalSize || !Utils.IsEqualSHA1(hash, oldFile.Hash))
{
await MakeHistory(db, transaction, request, name, "modified", oldFile.Size, file.TotalSize);
updateFile = true;
}
if (oldFile.Flags != file.Flags)
{
await MakeHistory(db, transaction, request, name, "modified_flags", (ulong)oldFile.Flags, (ulong)file.Flags);
updateFile = true;
}
if (updateFile)
{
await db.ExecuteAsync("UPDATE `DepotsFiles` SET `Hash` = @Hash, `Size` = @Size, `Flags` = @Flags WHERE `DepotID` = @DepotID AND `ID` = @ID", new DepotFile
{
ID = oldFile.ID,
DepotID = request.DepotID,
Hash = hash,
Size = file.TotalSize,
Flags = file.Flags
}, transaction : transaction);
}
filesOld.Remove(name);
}
else
{
// We want to historize modifications first, and only then deletions and additions
filesAdded.Add(new DepotFile
{
DepotID = request.DepotID,
Hash = hash,
File = name,
Size = file.TotalSize,
Flags = file.Flags
});
}
}
if (filesOld.Count > 0)
{
await db.ExecuteAsync("DELETE FROM `DepotsFiles` WHERE `DepotID` = @DepotID AND `ID` IN @Files", new { request.DepotID, Files = filesOld.Select(x => x.Value.ID) }, transaction : transaction);
await db.ExecuteAsync(HistoryQuery, filesOld.Select(x => new DepotHistory
{
DepotID = request.DepotID,
ManifestID = request.ManifestID,
ChangeID = request.ChangeNumber,
Action = "removed",
File = x.Value.File,
OldValue = x.Value.Size
}), transaction : transaction);
}
if (filesAdded.Count > 0)
{
await db.ExecuteAsync("INSERT INTO `DepotsFiles` (`DepotID`, `File`, `Hash`, `Size`, `Flags`) VALUES (@DepotID, @File, @Hash, @Size, @Flags)", filesAdded, transaction : transaction);
if (shouldHistorize)
{
await db.ExecuteAsync(HistoryQuery, filesAdded.Select(x => new DepotHistory
{
DepotID = request.DepotID,
ManifestID = request.ManifestID,
ChangeID = request.ChangeNumber,
Action = "added",
File = x.File,
NewValue = x.Size
}), transaction : transaction);
}
}
await db.ExecuteAsync("UPDATE `Depots` SET `LastManifestID` = @ManifestID, `LastUpdated` = CURRENT_TIMESTAMP() WHERE `DepotID` = @DepotID", new { request.DepotID, request.ManifestID }, transaction : transaction);
return(EResult.OK);
}
