// Validate a file against Steam3 Chunk data
public static DepotManifest.ChunkData[] ValidateSteam3FileChecksums(FileStream fs, DepotManifest.ChunkData[] chunkdata)
{
var neededChunks = new List<DepotManifest.ChunkData>();
int read;
foreach (DepotManifest.ChunkData data in chunkdata)
{
byte[] chunk = new byte[data.UncompressedLength];
fs.Seek((long)data.Offset, SeekOrigin.Begin);
read = fs.Read(chunk, 0, (int)data.UncompressedLength);
byte[] tempchunk;
if (read < data.UncompressedLength)
{
tempchunk = new byte[read];
Array.Copy(chunk, 0, tempchunk, 0, read);
}
else
{
tempchunk = chunk;
}
byte[] adler = AdlerHash(tempchunk);
if (!adler.SequenceEqual(data.Checksum))
{
neededChunks.Add(data);
}
}
return neededChunks.ToArray();
}
public FileData(DepotManifest.FileData sourceData) : this()
{
FileName = sourceData.FileName;
sourceData.Chunks.ForEach(c => Chunks.Add(new ChunkData(c)));
Flags = sourceData.Flags;
TotalSize = sourceData.TotalSize;
FileHash = sourceData.FileHash;
}
public ChunkData(DepotManifest.ChunkData sourceChunk)
{
ChunkID = sourceChunk.ChunkID;
Checksum = sourceChunk.Checksum;
Offset = sourceChunk.Offset;
CompressedLength = sourceChunk.CompressedLength;
UncompressedLength = sourceChunk.UncompressedLength;
}
/// <summary>
/// Downloads the depot manifest specified by the given manifest ID, and optionally decrypts the manifest's filenames if the depot decryption key has been provided.
/// </summary>
/// <param name="manifestId">The unique identifier of the manifest to be downloaded.</param>
/// <returns>A <see cref="DepotManifest"/> instance that contains information about the files present within a depot.</returns>
public DepotManifest DownloadManifest(ulong manifestId)
{
byte[] compressedManifest = DoRawCommand(connectedServer, "depot", doAuth: true, args: string.Format("{0}/manifest/{1}", depotId, manifestId));
byte[] manifestData = ZipUtil.Decompress(compressedManifest);
var depotManifest = new DepotManifest(manifestData);
if (depotKey != null)
{
// if we have the depot key, decrypt the manifest filenames
depotManifest.DecryptFilenames(depotKey);
}
return(depotManifest);
}
async Task <DepotManifest> DownloadManifestCoreAsync(uint depotId, ulong manifestId, Server server, string cdnAuthToken, byte[] depotKey)
{
var manifestData = await DoRawCommandAsync(server, HttpMethod.Get, "depot", doAuth : true, args : string.Format("{0}/manifest/{1}/5", depotId, manifestId), authtoken : cdnAuthToken).ConfigureAwait(false);
manifestData = ZipUtil.Decompress(manifestData);
var depotManifest = new DepotManifest(manifestData);
if (depotKey != null)
{
// if we have the depot key, decrypt the manifest filenames
depotManifest.DecryptFilenames(depotKey);
}
return(depotManifest);
}
DepotManifest DownloadManifestCore(uint depotId, ulong manifestId, Server server, string cdnAuthToken, byte[] depotKey)
{
byte[] manifestData = DoRawCommand(server, "depot", doAuth: true, args: string.Format("{0}/manifest/{1}/5", depotId, manifestId), authtoken: cdnAuthToken);
manifestData = ZipUtil.Decompress(manifestData);
var depotManifest = new DepotManifest(manifestData);
if (depotKey != null)
{
// if we have the depot key, decrypt the manifest filenames
depotManifest.DecryptFilenames(depotKey);
}
return(depotManifest);
}
/// <summary>
/// Downloads the depot manifest specified by the given manifest ID, and optionally decrypts the manifest's filenames if the depot decryption key has been provided.
/// </summary>
/// <param name="depotId">The id of the depot being accessed.</param>
/// <param name="manifestId">The unique identifier of the manifest to be downloaded.</param>
/// <returns>A <see cref="DepotManifest"/> instance that contains information about the files present within a depot.</returns>
public DepotManifest DownloadManifest(uint depotId, ulong manifestId)
{
string cdnToken = null;
depotCdnAuthKeys.TryGetValue(depotId, out cdnToken);
byte[] compressedManifest = DoRawCommand(connectedServer, "depot", doAuth: true, args: string.Format("{0}/manifest/{1}/5", depotId, manifestId), authtoken: cdnToken);
byte[] manifestData = ZipUtil.Decompress(compressedManifest);
var depotManifest = new DepotManifest(manifestData);
byte[] depotKey;
if (depotKeys.TryGetValue(depotId, out depotKey))
{
// if we have the depot key, decrypt the manifest filenames
depotManifest.DecryptFilenames(depotKey);
}
return(depotManifest);
}
/// <summary>
/// Downloads the depot manifest specified by the given manifest ID, and optionally decrypts the manifest's filenames if the depot decryption key has been provided.
/// </summary>
/// <param name="depotId">The id of the depot being accessed.</param>
/// <param name="manifestId">The unique identifier of the manifest to be downloaded.</param>
/// <param name="server">The content server to connect to.</param>
/// <param name="cdnAuthToken">CDN auth token for CDN content server endpoints.</param>
/// <param name="depotKey">
/// The depot decryption key for the depot that will be downloaded.
/// This is used for decrypting filenames (if needed) in depot manifests, and processing depot chunks.
/// </param>
/// <param name="proxyServer">Optional content server marked as UseAsProxy which transforms the request.</param>
/// <returns>A <see cref="DepotManifest"/> instance that contains information about the files present within a depot.</returns>
/// <exception cref="System.ArgumentNullException"><see ref="server"/> was null.</exception>
/// <exception cref="HttpRequestException">An network error occurred when performing the request.</exception>
/// <exception cref="SteamKitWebRequestException">A network error occurred when performing the request.</exception>
public async Task <DepotManifest> DownloadManifestAsync(uint depotId, ulong manifestId, Server server, string?cdnAuthToken, byte[]?depotKey, Server?proxyServer = null)
{
if (server == null)
{
throw new ArgumentNullException(nameof(server));
}
var manifestData = await DoRawCommandAsync(server, string.Format("depot/{0}/manifest/{1}/5", depotId, manifestId), cdnAuthToken, proxyServer).ConfigureAwait(false);
manifestData = ZipUtil.Decompress(manifestData);
var depotManifest = new DepotManifest(manifestData);
if (depotKey != null)
{
// if we have the depot key, decrypt the manifest filenames
depotManifest.DecryptFilenames(depotKey);
}
return(depotManifest);
}
/*
* Here be dragons.
*/
public static void DownloadFilesFromDepot(DepotProcessor.ManifestJob job, DepotManifest depotManifest)
{
var randomGenerator = new Random();
var files = depotManifest.Files.Where(x => IsFileNameMatching(job.DepotID, x.FileName)).ToList();
var filesUpdated = false;
var filesAnyFailed = false;
var hashesFile = Path.Combine(Application.Path, "files", ".support", "hashes", string.Format("{0}.json", job.DepotID));
var hashes = new Dictionary<string, byte[]>();
if (File.Exists(hashesFile))
{
hashes = JsonConvert.DeserializeObject<Dictionary<string, byte[]>>(File.ReadAllText(hashesFile));
}
Log.WriteDebug("FileDownloader", "Will download {0} files from depot {1}", files.Count, job.DepotID);
Parallel.ForEach(files, new ParallelOptions { MaxDegreeOfParallelism = 2 }, (file, state2) =>
{
string directory = Path.Combine(Application.Path, "files", job.DepotID.ToString(), Path.GetDirectoryName(file.FileName));
string finalPath = Path.Combine(directory, Path.GetFileName(file.FileName));
string downloadPath = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Path.GetRandomFileName(), ".steamdb_tmp"));
if (!Directory.Exists(directory))
{
Directory.CreateDirectory(directory);
}
else if (file.TotalSize == 0)
{
if (File.Exists(finalPath))
{
var f = new FileInfo(finalPath);
if(f.Length == 0)
{
#if DEBUG
Log.WriteDebug("FileDownloader", "{0} is already empty", file.FileName);
#endif
return;
}
}
else
{
File.Create(finalPath);
Log.WriteInfo("FileDownloader", "{0} created an empty file", file.FileName);
return;
}
}
else if(hashes.ContainsKey(file.FileName) && file.FileHash.SequenceEqual(hashes[file.FileName]))
{
#if DEBUG
Log.WriteDebug("FileDownloader", "{0} already matches the file we have", file.FileName);
#endif
return;
}
var chunks = file.Chunks.OrderBy(x => x.Offset).ToList();
Log.WriteInfo("FileDownloader", "Downloading {0} ({1} bytes, {2} chunks)", file.FileName, file.TotalSize, chunks.Count);
uint count = 0;
byte[] checksum;
string lastError = "or checksum failed";
string oldChunksFile;
using (var sha = new SHA1Managed())
{
oldChunksFile = Path.Combine(Application.Path, "files", ".support", "chunks",
string.Format("{0}-{1}.json", job.DepotID, BitConverter.ToString(sha.ComputeHash(Encoding.UTF8.GetBytes(file.FileName))))
);
}
using (var fs = File.Open(downloadPath, FileMode.OpenOrCreate, FileAccess.ReadWrite))
{
fs.SetLength((long)file.TotalSize);
var lockObject = new object();
var neededChunks = new List<DepotManifest.ChunkData>();
if (File.Exists(oldChunksFile) && File.Exists(finalPath))
{
var oldChunks = JsonConvert.DeserializeObject<List<DepotManifest.ChunkData>>(
File.ReadAllText(oldChunksFile),
new JsonSerializerSettings { PreserveReferencesHandling = PreserveReferencesHandling.All }
);
using (var fsOld = File.Open(finalPath, FileMode.Open, FileAccess.Read))
{
foreach (var chunk in chunks)
{
var oldChunk = oldChunks.FirstOrDefault(c => c.ChunkID.SequenceEqual(chunk.ChunkID));
if (oldChunk != null)
{
var oldData = new byte[oldChunk.UncompressedLength];
fsOld.Seek((long)oldChunk.Offset, SeekOrigin.Begin);
fsOld.Read(oldData, 0, oldData.Length);
var existingChecksum = Utils.AdlerHash(oldData);
if (existingChecksum.SequenceEqual(chunk.Checksum))
{
fs.Seek((long)chunk.Offset, SeekOrigin.Begin);
fs.Write(oldData, 0, oldData.Length);
#if DEBUG
Log.WriteDebug("FileDownloader", "{0} Found chunk ({1}), not downloading ({2}/{3})", file.FileName, chunk.Offset, ++count, chunks.Count);
#endif
}
else
{
neededChunks.Add(chunk);
#if DEBUG
Log.WriteDebug("FileDownloader", "{0} Found chunk ({1}), but checksum differs", file.FileName, chunk.Offset);
#endif
}
}
else
{
neededChunks.Add(chunk);
}
}
}
}
else
{
neededChunks = chunks;
}
Parallel.ForEach(neededChunks, new ParallelOptions { MaxDegreeOfParallelism = 3 }, (chunk, state) =>
{
var downloaded = false;
for (var i = 0; i <= 5; i++)
{
try
{
var chunkData = CDNClient.DownloadDepotChunk(job.DepotID, chunk, job.Server, job.CDNToken, job.DepotKey);
lock (lockObject)
{
fs.Seek((long)chunk.Offset, SeekOrigin.Begin);
fs.Write(chunkData.Data, 0, chunkData.Data.Length);
Log.WriteDebug("FileDownloader", "Downloaded {0} ({1}/{2})", file.FileName, ++count, chunks.Count);
}
downloaded = true;
break;
}
catch (Exception e)
{
lastError = e.Message;
}
// See https://developers.google.com/drive/web/handle-errors
Task.Delay((1 << i) * 1000 + randomGenerator.Next(1001)).Wait();
}
if (!downloaded)
{
state.Stop();
}
});
fs.Seek(0, SeekOrigin.Begin);
using (var sha = new SHA1Managed())
{
checksum = sha.ComputeHash(fs);
}
}
if (file.FileHash.SequenceEqual(checksum))
{
Log.WriteInfo("FileDownloader", "Downloaded {0} from {1}", file.FileName, Steam.GetAppName(job.ParentAppID));
hashes[file.FileName] = checksum;
if (File.Exists(finalPath))
{
File.Delete(finalPath);
}
File.Move(downloadPath, finalPath);
if (chunks.Count > 1)
{
File.WriteAllText(oldChunksFile,
JsonConvert.SerializeObject(
chunks,
Formatting.None,
new JsonSerializerSettings { PreserveReferencesHandling = PreserveReferencesHandling.All }
)
);
}
else if (File.Exists(oldChunksFile))
{
File.Delete(oldChunksFile);
}
filesUpdated = true;
}
else
{
filesAnyFailed = true;
IRC.Instance.SendOps("{0}[{1}]{2} Failed to download {3}: Only {4} out of {5} chunks downloaded ({6})",
Colors.OLIVE, Steam.GetAppName(job.ParentAppID), Colors.NORMAL, file.FileName, count, chunks.Count, lastError);
Log.WriteError("FileDownloader", "Failed to download {0}: Only {1} out of {2} chunks downloaded from {3} ({4})",
file.FileName, count, chunks.Count, job.Server, lastError);
File.Delete(downloadPath);
}
});
if (filesUpdated)
{
if (filesAnyFailed)
{
IRC.Instance.SendOps("{0}[{1}]{2} Failed to download some files, not running update script to prevent broken diffs.",
Colors.OLIVE, Steam.GetAppName(job.ParentAppID), Colors.NORMAL);
return;
}
File.WriteAllText(hashesFile, JsonConvert.SerializeObject(hashes));
var updateScript = Path.Combine(Application.Path, "files", "update.sh");
if (File.Exists(updateScript))
{
lock (updateLock)
{
// YOLO
Process.Start(updateScript, job.DepotID.ToString());
}
}
}
}
private static void ProcessDepotAfterDownload(IDbConnection db, ManifestJob request, DepotManifest depotManifest)
{
var filesOld = db.Query<DepotFile>("SELECT `ID`, `File`, `Hash`, `Size`, `Flags` FROM `DepotsFiles` WHERE `DepotID` = @DepotID", new { request.DepotID }).ToDictionary(x => x.File, x => x);
var filesNew = new List<DepotFile>();
var filesAdded = new List<DepotFile>();
var shouldHistorize = filesOld.Any(); // Don't historize file additions if we didn't have any data before
foreach (var file in depotManifest.Files)
{
var name = file.FileName.Replace('\\', '/');
// safe guard
if (name.Length > 255)
{
ErrorReporter.Notify(new OverflowException(string.Format("File \"{0}\" in depot {1} is too long", name, request.DepotID)));
continue;
}
var depotFile = new DepotFile
{
DepotID = request.DepotID,
File = name,
Size = file.TotalSize,
Flags = file.Flags
};
if (file.FileHash.Length > 0 && !file.Flags.HasFlag(EDepotFileFlag.Directory))
{
depotFile.Hash = Utils.ByteArrayToString(file.FileHash);
}
else
{
depotFile.Hash = "0000000000000000000000000000000000000000";
}
filesNew.Add(depotFile);
}
foreach (var file in filesNew)
{
if (filesOld.ContainsKey(file.File))
{
var oldFile = filesOld[file.File];
var updateFile = false;
if (oldFile.Size != file.Size || !file.Hash.Equals(oldFile.Hash))
{
MakeHistory(db, request, file.File, "modified", oldFile.Size, file.Size);
updateFile = true;
}
if (oldFile.Flags != file.Flags)
{
MakeHistory(db, request, file.File, "modified_flags", (ulong)oldFile.Flags, (ulong)file.Flags);
updateFile = true;
}
if (updateFile)
{
file.ID = oldFile.ID;
db.Execute("UPDATE `DepotsFiles` SET `Hash` = @Hash, `Size` = @Size, `Flags` = @Flags WHERE `DepotID` = @DepotID AND `ID` = @ID", file);
}
filesOld.Remove(file.File);
}
else
{
// We want to historize modifications first, and only then deletions and additions
filesAdded.Add(file);
}
}
if (filesOld.Any())
{
db.Execute("DELETE FROM `DepotsFiles` WHERE `DepotID` = @DepotID AND `ID` IN @Files", new { request.DepotID, Files = filesOld.Select(x => x.Value.ID) });
db.Execute(GetHistoryQuery(), filesOld.Select(x => new DepotHistory
{
DepotID = request.DepotID,
ChangeID = request.ChangeNumber,
Action = "removed",
File = x.Value.File
}));
}
if (filesAdded.Any())
{
db.Execute("INSERT INTO `DepotsFiles` (`DepotID`, `File`, `Hash`, `Size`, `Flags`) VALUES (@DepotID, @File, @Hash, @Size, @Flags)", filesAdded);
if (shouldHistorize)
{
db.Execute(GetHistoryQuery(), filesAdded.Select(x => new DepotHistory
{
DepotID = request.DepotID,
ChangeID = request.ChangeNumber,
Action = "added",
File = x.File
}));
}
}
db.Execute("UPDATE `Depots` SET `LastManifestID` = @ManifestID WHERE `DepotID` = @DepotID", new { request.DepotID, request.ManifestID });
}
// Ambiguous reference in cref attribute: 'CDNClient.DownloadManifest'. Assuming 'SteamKit2.CDNClient.DownloadManifest(uint, ulong)',
// but could have also matched other overloads including 'SteamKit2.CDNClient.DownloadManifest(uint, ulong, string, string, byte[])'.
#pragma warning disable 0419
/// <summary>
/// Downloads the specified depot chunk, and optionally processes the chunk and verifies the checksum if the depot decryption key has been provided.
/// </summary>
/// <remarks>
/// This function will also validate the length of the downloaded chunk with the value of <see cref="DepotManifest.ChunkData.CompressedLength"/>,
/// if it has been assigned a value.
/// </remarks>
/// <param name="depotId">The id of the depot being accessed.</param>
/// <param name="chunk">
/// A <see cref="DepotManifest.ChunkData"/> instance that represents the chunk to download.
/// This value should come from a manifest downloaded with <see cref="CDNClient.DownloadManifest"/>.
/// </param>
/// <returns>A <see cref="DepotChunk"/> instance that contains the data for the given chunk.</returns>
/// <param name="host">CDN hostname.</param>
/// <param name="cdnAuthToken">CDN auth token for CDN content server endpoints.</param>
/// <param name="depotKey">
/// The depot decryption key for the depot that will be downloaded.
/// This is used for decrypting filenames (if needed) in depot manifests, and processing depot chunks.
/// </param>
/// <exception cref="System.ArgumentNullException">chunk's <see cref="DepotManifest.ChunkData.ChunkID"/> was null.</exception>
public DepotChunk DownloadDepotChunk( uint depotId, DepotManifest.ChunkData chunk, string host, string cdnAuthToken, byte[] depotKey = null )
#pragma warning restore 0419
{
if (chunk.ChunkID == null)
throw new ArgumentNullException( "chunk.ChunkID" );
var server = new Server
{
Host = host,
Port = 80
};
return DownloadDepotChunkCore( depotId, chunk, server, cdnAuthToken, depotKey );
}
// Ambiguous reference in cref attribute: 'CDNClient.DownloadManifest'. Assuming 'SteamKit2.CDNClient.DownloadManifest(uint, ulong)',
// but could have also matched other overloads including 'SteamKit2.CDNClient.DownloadManifest(uint, ulong, string, string, byte[])'.
#pragma warning disable 0419
/// <summary>
/// Downloads the specified depot chunk, and optionally processes the chunk and verifies the checksum if the depot decryption key has been provided.
/// </summary>
/// <remarks>
/// This function will also validate the length of the downloaded chunk with the value of <see cref="DepotManifest.ChunkData.CompressedLength"/>,
/// if it has been assigned a value.
/// </remarks>
/// <param name="depotId">The id of the depot being accessed.</param>
/// <param name="chunk">
/// A <see cref="DepotManifest.ChunkData"/> instance that represents the chunk to download.
/// This value should come from a manifest downloaded with <see cref="CDNClient.DownloadManifest"/>.
/// </param>
/// <returns>A <see cref="DepotChunk"/> instance that contains the data for the given chunk.</returns>
/// <exception cref="System.ArgumentNullException">chunk's <see cref="DepotManifest.ChunkData.ChunkID"/> was null.</exception>
public DepotChunk DownloadDepotChunk( uint depotId, DepotManifest.ChunkData chunk )
#pragma warning restore 0419
{
if ( chunk.ChunkID == null )
throw new ArgumentNullException( "chunk.ChunkID" );
string cdnToken = null;
depotCdnAuthKeys.TryGetValue( depotId, out cdnToken );
byte[] depotKey = null;
depotKeys.TryGetValue( depotId, out depotKey );
return DownloadDepotChunkCore( depotId, chunk, connectedServer, cdnToken, depotKey );
}
public ProtoManifest(DepotManifest sourceManifest, ulong id) : this()
{
sourceManifest.Files.ForEach(f => Files.Add(new FileData(f)));
ID = id;
}
/// <summary>
/// Downloads the specified depot chunk, and optionally processes the chunk and verifies the checksum if the depot decryption key has been provided.
/// </summary>
/// <remarks>
/// This function will also validate the length of the downloaded chunk with the value of <see cref="DepotManifest.ChunkData.CompressedLength"/>,
/// if it has been assigned a value.
/// </remarks>
/// <param name="depotId">The id of the depot being accessed.</param>
/// <param name="chunk">
/// A <see cref="DepotManifest.ChunkData"/> instance that represents the chunk to download.
/// This value should come from a manifest downloaded with <see cref="CDNClient.DownloadManifest"/>.
/// </param>
/// <returns>A <see cref="DepotChunk"/> instance that contains the data for the given chunk.</returns>
/// <exception cref="System.ArgumentNullException">chunk's <see cref="DepotManifest.ChunkData.ChunkID"/> was null.</exception>
public DepotChunk DownloadDepotChunk( uint depotId, DepotManifest.ChunkData chunk )
{
if ( chunk.ChunkID == null )
throw new ArgumentNullException( "chunk.ChunkID" );
string chunkId = Utils.EncodeHexString( chunk.ChunkID );
string cdnToken = null;
depotCdnAuthKeys.TryGetValue(depotId, out cdnToken);
byte[] chunkData = DoRawCommand( connectedServer, "depot", doAuth: true, args: string.Format( "{0}/chunk/{1}", depotId, chunkId ), authtoken: cdnToken );
if ( chunk.CompressedLength != default( uint ) )
{
// assert that lengths match only if the chunk has a length assigned.
DebugLog.Assert( chunkData.Length == chunk.CompressedLength, "CDNClient", "Length mismatch after downloading depot chunk!" );
}
var depotChunk = new DepotChunk
{
ChunkInfo = chunk,
Data = chunkData,
};
byte[] depotKey;
if ( depotKeys.TryGetValue( depotId, out depotKey ) )
{
// if we have the depot key, we can process the chunk immediately
depotChunk.Process( depotKey );
}
return depotChunk;
}
/// <summary>
/// Downloads the depot manifest specified by the given manifest ID, and optionally decrypts the manifest's filenames if the depot decryption key has been provided.
/// </summary>
/// <param name="depotId">The id of the depot being accessed.</param>
/// <param name="manifestId">The unique identifier of the manifest to be downloaded.</param>
/// <returns>A <see cref="DepotManifest"/> instance that contains information about the files present within a depot.</returns>
public DepotManifest DownloadManifest( uint depotId, ulong manifestId )
{
string cdnToken = null;
depotCdnAuthKeys.TryGetValue( depotId, out cdnToken );
byte[] compressedManifest = DoRawCommand( connectedServer, "depot", doAuth: true, args: string.Format( "{0}/manifest/{1}/5", depotId, manifestId ), authtoken: cdnToken );
byte[] manifestData = ZipUtil.Decompress( compressedManifest );
var depotManifest = new DepotManifest( manifestData );
byte[] depotKey;
if ( depotKeys.TryGetValue( depotId, out depotKey ) )
{
// if we have the depot key, decrypt the manifest filenames
depotManifest.DecryptFilenames( depotKey );
}
return depotManifest;
}
/// <summary>
/// Downloads the depot manifest specified by the given manifest ID, and optionally decrypts the manifest's filenames if the depot decryption key has been provided.
/// </summary>
/// <param name="manifestId">The unique identifier of the manifest to be downloaded.</param>
/// <returns>A <see cref="DepotManifest"/> instance that contains information about the files present within a depot.</returns>
public DepotManifest DownloadManifest( ulong manifestId )
{
byte[] compressedManifest = DoRawCommand( connectedServer, "depot", doAuth: true, args: string.Format( "{0}/manifest/{1}", depotId, manifestId ) );
byte[] manifestData = ZipUtil.Decompress( compressedManifest );
var depotManifest = new DepotManifest( manifestData );
if ( depotKey != null )
{
// if we have the depot key, decrypt the manifest filenames
depotManifest.DecryptFilenames( depotKey );
}
return depotManifest;
}
/// <summary>
/// Downloads the specified depot chunk, and optionally processes the chunk and verifies the checksum if the depot decryption key has been provided.
/// </summary>
/// <param name="chunk">
/// A <see cref="DepotManifest.ChunkData"/> instance that represents the chunk to download.
/// This value should come from a manifest downloaded with <see cref="CDNClient.DownloadManifest"/>.
/// </param>
/// <returns>A <see cref="DepotChunk"/> instance that contains the data for the given chunk.</returns>
public DepotChunk DownloadDepotChunk( DepotManifest.ChunkData chunk )
{
string chunkId = Utils.EncodeHexString( chunk.ChunkID );
byte[] chunkData = DoRawCommand( connectedServer, "depot", doAuth: true, args: string.Format( "{0}/chunk/{1}", depotId, chunkId ) );
DebugLog.Assert( chunkData.Length == chunk.CompressedLength, "CDNClient", "Length mismatch after downloading depot chunk!" );
var depotChunk = new DepotChunk
{
ChunkInfo = chunk,
Data = chunkData,
};
if ( depotKey != null )
{
// if we have the depot key, we can process the chunk immediately
depotChunk.Process( depotKey );
}
return depotChunk;
}
DepotManifest DownloadManifestCore( uint depotId, ulong manifestId, Server server, string cdnAuthToken, byte[] depotKey )
{
byte[] manifestData = DoRawCommand( server, "depot", doAuth: true, args: string.Format( "{0}/manifest/{1}/5", depotId, manifestId ), authtoken: cdnAuthToken );
manifestData = ZipUtil.Decompress( manifestData );
var depotManifest = new DepotManifest( manifestData );
if ( depotKey != null )
{
// if we have the depot key, decrypt the manifest filenames
depotManifest.DecryptFilenames( depotKey );
}
return depotManifest;
}
DepotChunk DownloadDepotChunkCore( uint depotId, DepotManifest.ChunkData chunk, Server server, string cdnAuthToken, byte[] depotKey )
{
var chunkID = Utils.EncodeHexString( chunk.ChunkID );
byte[] chunkData = DoRawCommand( server, "depot", doAuth: true, args: string.Format( "{0}/chunk/{1}", depotId, chunkID ), authtoken: cdnAuthToken );
if ( chunk.CompressedLength != default( uint ) )
{
// assert that lengths match only if the chunk has a length assigned.
DebugLog.Assert( chunkData.Length == chunk.CompressedLength, "CDNClient", "Length mismatch after downloading depot chunk!" );
}
var depotChunk = new DepotChunk
{
ChunkInfo = chunk,
Data = chunkData,
};
if ( depotKey != null )
{
// if we have the depot key, we can process the chunk immediately
depotChunk.Process( depotKey );
}
return depotChunk;
}
public static void DownloadFilesFromDepot(DepotProcessor.ManifestJob job, DepotManifest depotManifest)
{
var files = depotManifest.Files.Where(x => IsFileNameMatching(job.DepotID, x.FileName)).ToList();
var filesUpdated = false;
Log.WriteDebug("FileDownloader", "Will download {0} files from depot {1}", files.Count(), job.DepotID);
foreach (var file in files)
{
string directory = Path.Combine(Application.Path, FILES_DIRECTORY, job.DepotID.ToString(), Path.GetDirectoryName(file.FileName));
string finalPath = Path.Combine(directory, Path.GetFileName(file.FileName));
string downloadPath = Path.GetTempFileName();
if (!Directory.Exists(directory))
{
Directory.CreateDirectory(directory);
}
else if (File.Exists(finalPath))
{
using (var fs = File.Open(finalPath, FileMode.Open))
{
using (var sha = new SHA1Managed())
{
if (file.FileHash.SequenceEqual(sha.ComputeHash(fs)))
{
Log.WriteDebug("FileDownloader", "{0} already matches the file we have", file.FileName);
continue;
}
}
}
}
Log.WriteInfo("FileDownloader", "Downloading {0} ({1} bytes, {2} chunks)", file.FileName, file.TotalSize, file.Chunks.Count);
uint count = 0;
byte[] checksum;
string lastError = "or checksum failed";
using (var fs = File.Open(downloadPath, FileMode.OpenOrCreate))
{
fs.SetLength((long)file.TotalSize);
var lockObject = new object();
// TODO: We *could* verify each chunk and only download needed ones
Parallel.ForEach(file.Chunks, (chunk, state) =>
{
var downloaded = false;
for (var i = 0; i <= 5; i++)
{
try
{
var chunkData = CDNClient.DownloadDepotChunk(job.DepotID, chunk, job.Server, job.CDNToken, job.DepotKey);
lock (lockObject)
{
fs.Seek((long)chunk.Offset, SeekOrigin.Begin);
fs.Write(chunkData.Data, 0, chunkData.Data.Length);
Log.WriteDebug("FileDownloader", "Downloaded {0} ({1}/{2})", file.FileName, ++count, file.Chunks.Count);
}
downloaded = true;
break;
}
catch (Exception e)
{
lastError = e.Message;
}
}
if (!downloaded)
{
state.Stop();
}
});
fs.Seek(0, SeekOrigin.Begin);
using (var sha = new SHA1Managed())
{
checksum = sha.ComputeHash(fs);
}
}
if (file.Chunks.Count == 0 || file.FileHash.SequenceEqual(checksum))
{
Log.WriteInfo("FileDownloader", "Downloaded {0} from {1}", file.FileName, Steam.GetAppName(job.ParentAppID));
if (File.Exists(finalPath))
{
File.Delete(finalPath);
}
File.Move(downloadPath, finalPath);
filesUpdated = true;
}
else
{
IRC.Instance.SendOps("{0}[{1}]{2} Failed to download {3}: Only {4} out of {5} chunks downloaded ({6})",
Colors.OLIVE, Steam.GetAppName(job.ParentAppID), Colors.NORMAL, file.FileName, count, file.Chunks.Count, lastError);
Log.WriteError("FileDownloader", "Failed to download {0}: Only {1} out of {2} chunks downloaded from {3} ({4})",
file.FileName, count, file.Chunks.Count, job.Server, lastError);
File.Delete(downloadPath);
}
}
if (filesUpdated)
{
var updateScript = Path.Combine(Application.Path, "files", "update.sh");
if (File.Exists(updateScript))
{
// YOLO
Process.Start(updateScript, job.DepotID.ToString());
}
}
}