private static async Task <EResult> ProcessDepotAfterDownload(IDbConnection db, IDbTransaction transaction, ManifestJob request, DepotManifest depotManifest) { var filesOld = (await db.QueryAsync <DepotFile>("SELECT `File`, `Hash`, `Size`, `Flags` FROM `DepotsFiles` WHERE `DepotID` = @DepotID", new { request.DepotID }, transaction)).ToDictionary(x => x.File, x => x); var filesAdded = new List <DepotFile>(); var shouldHistorize = filesOld.Count > 0 && !depotManifest.FilenamesEncrypted; // Don't historize file additions if we didn't have any data before if (request.StoredFilenamesEncrypted && !depotManifest.FilenamesEncrypted) { Log.WriteInfo(nameof(DepotProcessor), $"Depot {request.DepotID} will decrypt stored filenames"); var decryptedFilesOld = new Dictionary <string, DepotFile>(); foreach (var file in filesOld.Values) { var oldFile = file.File; file.File = DecryptFilename(oldFile, request.DepotKey); decryptedFilesOld.Add(file.File, file); await db.ExecuteAsync("UPDATE `DepotsFiles` SET `File` = @File WHERE `DepotID` = @DepotID AND `File` = @OldFile", new { request.DepotID, file.File, OldFile = oldFile }, transaction); } await MakeHistory(db, transaction, request, string.Empty, "files_decrypted"); filesOld = decryptedFilesOld; } foreach (var file in depotManifest.Files.OrderByDescending(x => x.FileName)) { var name = depotManifest.FilenamesEncrypted ? file.FileName.Replace("\n", "") : file.FileName.Replace('\\', '/'); byte[] hash = null; // Store empty hashes as NULL (e.g. an empty file) if ((file.Flags & EDepotFileFlag.Directory) == 0 && file.FileHash.Length > 0 && file.FileHash.Any(c => c != 0)) { hash = file.FileHash; } // Limit path names to 260 characters (default windows max length) // File column is varchar(260) and not higher to prevent reducing performance // See https://stackoverflow.com/questions/1962310/importance-of-varchar-length-in-mysql-table/1962329#1962329 // Until 2019 there hasn't been a single file that went over this limit, so far there has been only one // game with a big node_modules path, so we're safeguarding by limiting it. if (name.Length > 260) { if (depotManifest.FilenamesEncrypted) { continue; } using var sha = SHA1.Create(); var nameHash = Utils.ByteArrayToString(sha.ComputeHash(Encoding.UTF8.GetBytes(name))); name = $"{{SteamDB file name is too long}}/{nameHash}/...{name.Substring(name.Length - 150)}"; } if (filesOld.ContainsKey(name)) { var oldFile = filesOld[name]; var updateFile = false; if (oldFile.Size != file.TotalSize || !Utils.ByteArrayEquals(hash, oldFile.Hash)) { await MakeHistory(db, transaction, request, name, "modified", oldFile.Size, file.TotalSize); updateFile = true; } if (oldFile.Flags != file.Flags) { await MakeHistory(db, transaction, request, name, "modified_flags", (ulong)oldFile.Flags, (ulong)file.Flags); updateFile = true; } if (updateFile) { await db.ExecuteAsync("UPDATE `DepotsFiles` SET `Hash` = @Hash, `Size` = @Size, `Flags` = @Flags WHERE `DepotID` = @DepotID AND `File` = @File", new DepotFile { DepotID = request.DepotID, File = name, Hash = hash, Size = file.TotalSize, Flags = file.Flags }, transaction); } filesOld.Remove(name); } else { // We want to historize modifications first, and only then deletions and additions filesAdded.Add(new DepotFile { DepotID = request.DepotID, Hash = hash, File = name, Size = file.TotalSize, Flags = file.Flags }); } } if (filesOld.Count > 0) { // Chunk file deletion queries so it doesn't go over max_allowed_packet var filesOldChunks = filesOld.Select(x => x.Value.File).Split(1000); foreach (var filesOldChunk in filesOldChunks) { await db.ExecuteAsync("DELETE FROM `DepotsFiles` WHERE `DepotID` = @DepotID AND `File` IN @Files", new { request.DepotID, Files = filesOldChunk, }, transaction); } if (shouldHistorize) { await db.ExecuteAsync(HistoryQuery, filesOld.Select(x => new DepotHistory { DepotID = request.DepotID, ManifestID = request.ManifestID, ChangeID = request.ChangeNumber, Action = "removed", File = x.Value.File, OldValue = x.Value.Size }), transaction); } } if (filesAdded.Count > 0) { await db.ExecuteAsync("INSERT INTO `DepotsFiles` (`DepotID`, `File`, `Hash`, `Size`, `Flags`) VALUES (@DepotID, @File, @Hash, @Size, @Flags)", filesAdded, transaction); if (shouldHistorize) { await db.ExecuteAsync(HistoryQuery, filesAdded.Select(x => new DepotHistory { DepotID = request.DepotID, ManifestID = request.ManifestID, ChangeID = request.ChangeNumber, Action = "added", File = x.File, NewValue = x.Size }), transaction); } } await db.ExecuteAsync( request.LastManifestID == request.ManifestID? "UPDATE `Depots` SET `LastManifestID` = @ManifestID, `ManifestDate` = @ManifestDate, `FilenamesEncrypted` = @FilenamesEncrypted, `SizeOriginal` = @SizeOriginal, `SizeCompressed` = @SizeCompressed WHERE `DepotID` = @DepotID" : "UPDATE `Depots` SET `LastManifestID` = @ManifestID, `ManifestDate` = @ManifestDate, `FilenamesEncrypted` = @FilenamesEncrypted, `SizeOriginal` = @SizeOriginal, `SizeCompressed` = @SizeCompressed, `LastUpdated` = CURRENT_TIMESTAMP() WHERE `DepotID` = @DepotID", new { request.DepotID, request.ManifestID, depotManifest.FilenamesEncrypted, ManifestDate = depotManifest.CreationTime, SizeOriginal = depotManifest.TotalUncompressedSize, SizeCompressed = depotManifest.TotalCompressedSize, }, transaction); return(EResult.OK); }