private async Task <EResult> ProcessDepotAfterDownload(IDbConnection db, IDbTransaction transaction, ManifestJob request, DepotManifest depotManifest)
{
var filesOld = (await db.QueryAsync <DepotFile>("SELECT `ID`, `File`, `Hash`, `Size`, `Flags` FROM `DepotsFiles` WHERE `DepotID` = @DepotID", new { request.DepotID }, transaction: transaction)).ToDictionary(x => x.File, x => x);
var filesAdded = new List <DepotFile>();
var shouldHistorize = filesOld.Any(); // Don't historize file additions if we didn't have any data before
foreach (var file in depotManifest.Files)
{
var name = file.FileName.Replace('\\', '/');
byte[] hash = null;
// Store empty hashes as NULL (e.g. an empty file)
if (file.FileHash.Length > 0 && !file.Flags.HasFlag(EDepotFileFlag.Directory))
{
for (int i = 0; i < file.FileHash.Length; ++i)
{
if (file.FileHash[i] != 0)
{
hash = file.FileHash;
break;
}
}
}
// safe guard
if (name.Length > 255)
{
ErrorReporter.Notify("Depot Processor", new OverflowException(string.Format("File \"{0}\" in depot {1} is too long", name, request.DepotID)));
continue;
}
if (filesOld.ContainsKey(name))
{
var oldFile = filesOld[name];
var updateFile = false;
if (oldFile.Size != file.TotalSize || !Utils.IsEqualSHA1(hash, oldFile.Hash))
{
await MakeHistory(db, transaction, request, name, "modified", oldFile.Size, file.TotalSize);
updateFile = true;
}
if (oldFile.Flags != file.Flags)
{
await MakeHistory(db, transaction, request, name, "modified_flags", (ulong)oldFile.Flags, (ulong)file.Flags);
updateFile = true;
}
if (updateFile)
{
await db.ExecuteAsync("UPDATE `DepotsFiles` SET `Hash` = @Hash, `Size` = @Size, `Flags` = @Flags WHERE `DepotID` = @DepotID AND `ID` = @ID", new DepotFile
{
ID = oldFile.ID,
DepotID = request.DepotID,
Hash = hash,
Size = file.TotalSize,
Flags = file.Flags
}, transaction : transaction);
}
filesOld.Remove(name);
}
else
{
// We want to historize modifications first, and only then deletions and additions
filesAdded.Add(new DepotFile
{
DepotID = request.DepotID,
Hash = hash,
File = name,
Size = file.TotalSize,
Flags = file.Flags
});
}
}
if (filesOld.Any())
{
await db.ExecuteAsync("DELETE FROM `DepotsFiles` WHERE `DepotID` = @DepotID AND `ID` IN @Files", new { request.DepotID, Files = filesOld.Select(x => x.Value.ID) }, transaction : transaction);
await db.ExecuteAsync(HistoryQuery, filesOld.Select(x => new DepotHistory
{
DepotID = request.DepotID,
ChangeID = request.ChangeNumber,
Action = "removed",
File = x.Value.File
}), transaction : transaction);
}
if (filesAdded.Any())
{
await db.ExecuteAsync("INSERT INTO `DepotsFiles` (`DepotID`, `File`, `Hash`, `Size`, `Flags`) VALUES (@DepotID, @File, @Hash, @Size, @Flags)", filesAdded, transaction : transaction);
if (shouldHistorize)
{
await db.ExecuteAsync(HistoryQuery, filesAdded.Select(x => new DepotHistory
{
DepotID = request.DepotID,
ChangeID = request.ChangeNumber,
Action = "added",
File = x.File
}), transaction : transaction);
}
}
await db.ExecuteAsync("UPDATE `Depots` SET `LastManifestID` = @ManifestID, `LastUpdated` = CURRENT_TIMESTAMP() WHERE `DepotID` = @DepotID", new { request.DepotID, request.ManifestID }, transaction : transaction);
return(EResult.OK);
}
private static async Task <EResult> ProcessDepotAfterDownload(IDbConnection db, IDbTransaction transaction, ManifestJob request, DepotManifest depotManifest)
{
var filesOld = (await db.QueryAsync <DepotFile>("SELECT `File`, `Hash`, `Size`, `Flags` FROM `DepotsFiles` WHERE `DepotID` = @DepotID", new { request.DepotID }, transaction: transaction)).ToDictionary(x => x.File, x => x);
var filesAdded = new List <DepotFile>();
var shouldHistorize = filesOld.Count > 0 && !depotManifest.FilenamesEncrypted; // Don't historize file additions if we didn't have any data before
if (request.StoredFilenamesEncrypted && !depotManifest.FilenamesEncrypted)
{
Log.WriteInfo(nameof(DepotProcessor), $"Depot {request.DepotID} will decrypt stored filenames");
var decryptedFilesOld = new Dictionary <string, DepotFile>();
foreach (var file in filesOld.Values)
{
var oldFile = file.File;
file.File = DecryptFilename(oldFile, request.DepotKey);
decryptedFilesOld.Add(file.File, file);
await db.ExecuteAsync("UPDATE `DepotsFiles` SET `File` = @File WHERE `DepotID` = @DepotID AND `File` = @OldFile", new
{
request.DepotID,
file.File,
OldFile = oldFile
}, transaction);
}
filesOld = decryptedFilesOld;
}
foreach (var file in depotManifest.Files.OrderByDescending(x => x.FileName))
{
var name = depotManifest.FilenamesEncrypted ? file.FileName.Replace("\n", "") : file.FileName.Replace('\\', '/');
byte[] hash = null;
// Store empty hashes as NULL (e.g. an empty file)
if ((file.Flags & EDepotFileFlag.Directory) == 0 && file.FileHash.Length > 0 && file.FileHash.Any(c => c != 0))
{
hash = file.FileHash;
}
// Limit path names to 260 characters (default windows max length)
// File column is varchar(260) and not higher to prevent reducing performance
// See https://stackoverflow.com/questions/1962310/importance-of-varchar-length-in-mysql-table/1962329#1962329
// Until 2019 there hasn't been a single file that went over this limit, so far there has been only one
// game with a big node_modules path, so we're safeguarding by limiting it.
if (name.Length > 260)
{
if (depotManifest.FilenamesEncrypted)
{
continue;
}
using var sha = new System.Security.Cryptography.SHA1Managed();
var nameHash = Utils.ByteArrayToString(sha.ComputeHash(Encoding.UTF8.GetBytes(name)));
name = $"{{SteamDB file name is too long}}/{nameHash}/...{name.Substring(name.Length - 150)}";
}
if (filesOld.ContainsKey(name))
{
var oldFile = filesOld[name];
var updateFile = false;
if (oldFile.Size != file.TotalSize || !Utils.IsEqualSHA1(hash, oldFile.Hash))
{
await MakeHistory(db, transaction, request, name, "modified", oldFile.Size, file.TotalSize);
updateFile = true;
}
if (oldFile.Flags != file.Flags)
{
await MakeHistory(db, transaction, request, name, "modified_flags", (ulong)oldFile.Flags, (ulong)file.Flags);
updateFile = true;
}
if (updateFile)
{
await db.ExecuteAsync("UPDATE `DepotsFiles` SET `Hash` = @Hash, `Size` = @Size, `Flags` = @Flags WHERE `DepotID` = @DepotID AND `File` = @File", new DepotFile
{
DepotID = request.DepotID,
File = name,
Hash = hash,
Size = file.TotalSize,
Flags = file.Flags
}, transaction);
}
filesOld.Remove(name);
}
else
{
// We want to historize modifications first, and only then deletions and additions
filesAdded.Add(new DepotFile
{
DepotID = request.DepotID,
Hash = hash,
File = name,
Size = file.TotalSize,
Flags = file.Flags
});
}
}
if (filesOld.Count > 0)
{
// Chunk file deletion queries so it doesn't go over max_allowed_packet
var filesOldChunks = filesOld.Select(x => x.Value.File).Split(1000);
foreach (var filesOldChunk in filesOldChunks)
{
await db.ExecuteAsync("DELETE FROM `DepotsFiles` WHERE `DepotID` = @DepotID AND `File` IN @Files",
new
{
request.DepotID,
Files = filesOldChunk,
}, transaction);
}
if (shouldHistorize)
{
await db.ExecuteAsync(HistoryQuery, filesOld.Select(x => new DepotHistory
{
DepotID = request.DepotID,
ManifestID = request.ManifestID,
ChangeID = request.ChangeNumber,
Action = "removed",
File = x.Value.File,
OldValue = x.Value.Size
}), transaction);
}
}
if (filesAdded.Count > 0)
{
await db.ExecuteAsync("INSERT INTO `DepotsFiles` (`DepotID`, `File`, `Hash`, `Size`, `Flags`) VALUES (@DepotID, @File, @Hash, @Size, @Flags)", filesAdded, transaction : transaction);
if (shouldHistorize)
{
await db.ExecuteAsync(HistoryQuery, filesAdded.Select(x => new DepotHistory
{
DepotID = request.DepotID,
ManifestID = request.ManifestID,
ChangeID = request.ChangeNumber,
Action = "added",
File = x.File,
NewValue = x.Size
}), transaction);
}
}
await db.ExecuteAsync(
request.LastManifestID == request.ManifestID?
"UPDATE `Depots` SET `LastManifestID` = @ManifestID, `FilenamesEncrypted` = @FilenamesEncrypted WHERE `DepotID` = @DepotID" :
"UPDATE `Depots` SET `LastManifestID` = @ManifestID, `FilenamesEncrypted` = @FilenamesEncrypted, `LastUpdated` = CURRENT_TIMESTAMP() WHERE `DepotID` = @DepotID",
new
{
request.DepotID,
request.ManifestID,
depotManifest.FilenamesEncrypted,
}, transaction);
return(EResult.OK);
}
