public static void DownloadFilesFromDepot(DepotProcessor.ManifestJob job, DepotManifest depotManifest)
{
var files = depotManifest.Files.Where(x => IsFileNameMatching(job.DepotID, x.FileName)).ToList();
var filesUpdated = false;
Log.WriteDebug("FileDownloader", "Will download {0} files from depot {1}", files.Count(), job.DepotID);
foreach (var file in files)
{
string directory = Path.Combine(Application.Path, FILES_DIRECTORY, job.DepotID.ToString(), Path.GetDirectoryName(file.FileName));
string finalPath = Path.Combine(directory, Path.GetFileName(file.FileName));
string downloadPath = Path.GetTempFileName();
if (!Directory.Exists(directory))
{
Directory.CreateDirectory(directory);
}
else if (File.Exists(finalPath))
{
using (var fs = File.Open(finalPath, FileMode.Open))
{
using (var sha = new SHA1Managed())
{
if (file.FileHash.SequenceEqual(sha.ComputeHash(fs)))
{
Log.WriteDebug("FileDownloader", "{0} already matches the file we have", file.FileName);
continue;
}
}
}
}
Log.WriteInfo("FileDownloader", "Downloading {0} ({1} bytes, {2} chunks)", file.FileName, file.TotalSize, file.Chunks.Count);
uint count = 0;
byte[] checksum;
string lastError = "or checksum failed";
using (var fs = File.Open(downloadPath, FileMode.OpenOrCreate))
{
fs.SetLength((long)file.TotalSize);
var lockObject = new object();
// TODO: We *could* verify each chunk and only download needed ones
Parallel.ForEach(file.Chunks, (chunk, state) =>
{
var downloaded = false;
for (var i = 0; i <= 5; i++)
{
try
{
var chunkData = CDNClient.DownloadDepotChunk(job.DepotID, chunk, job.Server, job.CDNToken, job.DepotKey);
lock (lockObject)
{
fs.Seek((long)chunk.Offset, SeekOrigin.Begin);
fs.Write(chunkData.Data, 0, chunkData.Data.Length);
Log.WriteDebug("FileDownloader", "Downloaded {0} ({1}/{2})", file.FileName, ++count, file.Chunks.Count);
}
downloaded = true;
break;
}
catch (Exception e)
{
lastError = e.Message;
}
}
if (!downloaded)
{
state.Stop();
}
});
fs.Seek(0, SeekOrigin.Begin);
using (var sha = new SHA1Managed())
{
checksum = sha.ComputeHash(fs);
}
}
if (file.Chunks.Count == 0 || file.FileHash.SequenceEqual(checksum))
{
Log.WriteInfo("FileDownloader", "Downloaded {0} from {1}", file.FileName, Steam.GetAppName(job.ParentAppID));
if (File.Exists(finalPath))
{
File.Delete(finalPath);
}
File.Move(downloadPath, finalPath);
filesUpdated = true;
}
else
{
IRC.Instance.SendOps("{0}[{1}]{2} Failed to download {3}: Only {4} out of {5} chunks downloaded ({6})",
Colors.OLIVE, Steam.GetAppName(job.ParentAppID), Colors.NORMAL, file.FileName, count, file.Chunks.Count, lastError);
Log.WriteError("FileDownloader", "Failed to download {0}: Only {1} out of {2} chunks downloaded from {3} ({4})",
file.FileName, count, file.Chunks.Count, job.Server, lastError);
File.Delete(downloadPath);
}
}
if (filesUpdated)
{
var updateScript = Path.Combine(Application.Path, "files", "update.sh");
if (File.Exists(updateScript))
{
// YOLO
Process.Start(updateScript, job.DepotID.ToString());
}
}
}
/*
* Here be dragons.
*/
public static EResult DownloadFilesFromDepot(uint appID, DepotProcessor.ManifestJob job, DepotManifest depotManifest)
{
var files = depotManifest.Files.Where(x => IsFileNameMatching(job.DepotID, x.FileName)).ToList();
var filesUpdated = false;
var filesAnyFailed = false;
var hashesFile = Path.Combine(Application.Path, "files", ".support", "hashes", string.Format("{0}.json", job.DepotID));
var hashes = new Dictionary <string, byte[]>();
if (File.Exists(hashesFile))
{
hashes = JsonConvert.DeserializeObject <Dictionary <string, byte[]> >(File.ReadAllText(hashesFile));
}
Log.WriteDebug("FileDownloader", "Will download {0} files from depot {1}", files.Count, job.DepotID);
Parallel.ForEach(files, new ParallelOptions {
MaxDegreeOfParallelism = 2
}, (file, state2) =>
{
string directory = Path.Combine(Application.Path, "files", DownloadFolders[job.DepotID], Path.GetDirectoryName(file.FileName));
string finalPath = Path.Combine(directory, Path.GetFileName(file.FileName));
string downloadPath = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Path.GetRandomFileName(), ".steamdb_tmp"));
if (!Directory.Exists(directory))
{
Directory.CreateDirectory(directory);
}
else if (file.TotalSize == 0)
{
if (File.Exists(finalPath))
{
var f = new FileInfo(finalPath);
if (f.Length == 0)
{
#if DEBUG
Log.WriteDebug("FileDownloader", "{0} is already empty", file.FileName);
#endif
return;
}
}
else
{
File.Create(finalPath);
Log.WriteInfo("FileDownloader", "{0} created an empty file", file.FileName);
return;
}
}
else if (hashes.ContainsKey(file.FileName) && file.FileHash.SequenceEqual(hashes[file.FileName]))
{
#if DEBUG
Log.WriteDebug("FileDownloader", "{0} already matches the file we have", file.FileName);
#endif
return;
}
var chunks = file.Chunks.OrderBy(x => x.Offset).ToList();
Log.WriteInfo("FileDownloader", "Downloading {0} ({1} bytes, {2} chunks)", file.FileName, file.TotalSize, chunks.Count);
uint count = 0;
byte[] checksum;
string lastError = "or checksum failed";
string oldChunksFile;
using (var sha = new SHA1Managed())
{
oldChunksFile = Path.Combine(Application.Path, "files", ".support", "chunks",
string.Format("{0}-{1}.json", job.DepotID, BitConverter.ToString(sha.ComputeHash(Encoding.UTF8.GetBytes(file.FileName))))
);
}
using (var fs = File.Open(downloadPath, FileMode.OpenOrCreate, FileAccess.ReadWrite))
{
fs.SetLength((long)file.TotalSize);
var lockObject = new object();
var neededChunks = new List <DepotManifest.ChunkData>();
if (File.Exists(oldChunksFile) && File.Exists(finalPath))
{
var oldChunks = JsonConvert.DeserializeObject <List <DepotManifest.ChunkData> >(
File.ReadAllText(oldChunksFile),
new JsonSerializerSettings {
PreserveReferencesHandling = PreserveReferencesHandling.All
}
);
using (var fsOld = File.Open(finalPath, FileMode.Open, FileAccess.Read))
{
foreach (var chunk in chunks)
{
var oldChunk = oldChunks.FirstOrDefault(c => c.ChunkID.SequenceEqual(chunk.ChunkID));
if (oldChunk != null)
{
var oldData = new byte[oldChunk.UncompressedLength];
fsOld.Seek((long)oldChunk.Offset, SeekOrigin.Begin);
fsOld.Read(oldData, 0, oldData.Length);
var existingChecksum = Utils.AdlerHash(oldData);
if (existingChecksum.SequenceEqual(chunk.Checksum))
{
fs.Seek((long)chunk.Offset, SeekOrigin.Begin);
fs.Write(oldData, 0, oldData.Length);
#if DEBUG
Log.WriteDebug("FileDownloader", "{0} Found chunk ({1}), not downloading ({2}/{3})", file.FileName, chunk.Offset, ++count, chunks.Count);
#endif
}
else
{
neededChunks.Add(chunk);
#if DEBUG
Log.WriteDebug("FileDownloader", "{0} Found chunk ({1}), but checksum differs", file.FileName, chunk.Offset);
#endif
}
}
else
{
neededChunks.Add(chunk);
}
}
}
}
else
{
neededChunks = chunks;
}
Parallel.ForEach(neededChunks, new ParallelOptions {
MaxDegreeOfParallelism = 3
}, (chunk, state) =>
{
var downloaded = false;
for (var i = 0; i <= 5; i++)
{
try
{
var chunkData = CDNClient.DownloadDepotChunk(job.DepotID, chunk, job.Server, job.CDNToken, job.DepotKey);
lock (lockObject)
{
fs.Seek((long)chunk.Offset, SeekOrigin.Begin);
fs.Write(chunkData.Data, 0, chunkData.Data.Length);
Log.WriteDebug("FileDownloader", "Downloaded {0} ({1}/{2})", file.FileName, ++count, chunks.Count);
}
downloaded = true;
break;
}
catch (Exception e)
{
lastError = e.Message;
}
Task.Delay(Utils.ExponentionalBackoff(i)).Wait();
}
if (!downloaded)
{
state.Stop();
}
});
fs.Seek(0, SeekOrigin.Begin);
using (var sha = new SHA1Managed())
{
checksum = sha.ComputeHash(fs);
}
}
if (file.FileHash.SequenceEqual(checksum))
{
Log.WriteInfo("FileDownloader", "Downloaded {0} from {1}", file.FileName, Steam.GetAppName(appID));
hashes[file.FileName] = checksum;
if (File.Exists(finalPath))
{
File.Delete(finalPath);
}
File.Move(downloadPath, finalPath);
if (chunks.Count > 1)
{
File.WriteAllText(oldChunksFile,
JsonConvert.SerializeObject(
chunks,
Formatting.None,
new JsonSerializerSettings {
PreserveReferencesHandling = PreserveReferencesHandling.All
}
)
);
}
else if (File.Exists(oldChunksFile))
{
File.Delete(oldChunksFile);
}
filesUpdated = true;
}
else
{
filesAnyFailed = true;
IRC.Instance.SendOps("{0}[{1}]{2} Failed to download {3}: Only {4} out of {5} chunks downloaded ({6})",
Colors.OLIVE, Steam.GetAppName(appID), Colors.NORMAL, file.FileName, count, chunks.Count, lastError);
Log.WriteError("FileDownloader", "Failed to download {0}: Only {1} out of {2} chunks downloaded from {3} ({4})",
file.FileName, count, chunks.Count, job.Server, lastError);
File.Delete(downloadPath);
}
});
if (filesAnyFailed)
{
using (var db = Database.GetConnection())
{
// Mark this depot for redownload
db.Execute("UPDATE `Depots` SET `LastManifestID` = 0 WHERE `DepotID` = @DepotID", new { job.DepotID });
}
IRC.Instance.SendOps("{0}[{1}]{2} Failed to download some files, not running update script to prevent broken diffs.",
Colors.OLIVE, Steam.GetAppName(appID), Colors.NORMAL);
}
else if (filesUpdated)
{
File.WriteAllText(hashesFile, JsonConvert.SerializeObject(hashes));
job.Result = EResult.OK;
}
else
{
job.Result = EResult.Ignored;
}
return(job.Result);
}
async private void dataGridView1_CellDoubleClick(object sender, DataGridViewCellEventArgs e)
{
if (e.RowIndex < 0)
{
return;
}
try
{
ConfigItem item = datagridConfigs.Rows[e.RowIndex].DataBoundItem as ConfigItem;
if (item != null)
{
if (item.URL == "")
{
var callback = await steamWorkshop.RequestInfo(241100, item.Details.publishedfileid);
var itemInfo = callback.Items.FirstOrDefault();
var ticket = await steamClient.GetHandler <SteamApps>().GetAppOwnershipTicket(241100);
var decryptKey = await steamClient.GetHandler <SteamApps>().GetDepotDecryptionKey(241100, 241100);
var cdn = new CDNClient(steamClient, ticket.Ticket);
var servers = cdn.FetchServerList();
cdn.Connect(servers.First());
cdn.AuthenticateDepot(241100, decryptKey.DepotKey);
var manifest = cdn.DownloadManifest(241100, itemInfo.ManifestID);
manifest.DecryptFilenames(decryptKey.DepotKey);
if (manifest.Files.First().TotalSize == 0)
{
MessageBox.Show("Steam Refused Download Request");
return;
}
var chunk = cdn.DownloadDepotChunk(241100, manifest.Files.First().Chunks.First());
if (saveFileDialog1.ShowDialog() == DialogResult.OK)
{
using (var wc = new WebClient())
{
using (var io = saveFileDialog1.OpenFile())
{
io.Write(chunk.Data, 0, chunk.Data.Length);
MessageBox.Show("Download Done!");
}
}
}
}
else
{
if (saveFileDialog1.ShowDialog() == DialogResult.OK)
{
using (var wc = new WebClient())
{
wc.DownloadFile(new Uri(item.URL), saveFileDialog1.FileName);
MessageBox.Show("Download Done!");
}
}
}
}
}
catch (TaskCanceledException)
{
MessageBox.Show("Timeout! This can happen if you're using anonymous account and trying to download.");
}
catch (Exception ex)
{
MessageBox.Show($"Error Downloading Config: {ex.ToString()}");
}
}
private static void DownloadSteam3(int depotId, ulong depot_manifest, byte[] depotKey, string installDir)
{
Console.Write("Finding content servers...");
List <IPEndPoint> serverList = steam3.steamClient.GetServersOfType(EServerType.CS);
List <CDNClient.ClientEndPoint> cdnServers = null;
int tries = 0, counterDeferred = 0;
for (int i = 0; ; i++)
{
IPEndPoint endpoint = serverList[i % serverList.Count];
cdnServers = CDNClient.FetchServerList(new CDNClient.ClientEndPoint(endpoint.Address.ToString(), endpoint.Port), Config.CellID);
if (cdnServers == null)
{
counterDeferred++;
}
if (cdnServers != null && cdnServers.Count((ep) => { return(ep.Type == "CS"); }) > 0)
{
break;
}
if (((i + 1) % serverList.Count) == 0)
{
if (++tries > MAX_CONNECT_RETRIES)
{
Console.WriteLine("\nGiving up finding Steam3 content server.");
return;
}
Console.Write("\nSearching for content servers... (deferred: {0})", counterDeferred);
counterDeferred = 0;
Thread.Sleep(1000);
}
}
if (cdnServers == null || cdnServers.Count == 0)
{
Console.WriteLine("Unable to find any Steam3 content servers");
return;
}
Console.WriteLine(" Done!");
Console.Write("Downloading depot manifest...");
List <CDNClient.ClientEndPoint> cdnEndpoints = cdnServers.Where((ep) => { return(ep.Type == "CDN"); }).ToList();
List <CDNClient.ClientEndPoint> csEndpoints = cdnServers.Where((ep) => { return(ep.Type == "CS"); }).ToList();
CDNClient cdnClient = null;
foreach (var server in csEndpoints)
{
CDNClient client = new CDNClient(server, steam3.AppTickets[(uint)depotId]);
if (client.Connect())
{
cdnClient = client;
break;
}
}
if (cdnClient == null)
{
Console.WriteLine("\nCould not initialize connection with CDN.");
return;
}
DepotManifest depotManifest = cdnClient.DownloadDepotManifest(depotId, depot_manifest);
if (depotManifest == null)
{
Console.WriteLine("\nUnable to download manifest {0} for depot {1}", depot_manifest, depotId);
return;
}
if (!depotManifest.DecryptFilenames(depotKey))
{
Console.WriteLine("\nUnable to decrypt manifest for depot {0}", depotId);
return;
}
Console.WriteLine(" Done!");
ulong complete_download_size = 0;
ulong size_downloaded = 0;
depotManifest.Files.RemoveAll((x) => !TestIsFileIncluded(x.FileName));
depotManifest.Files.Sort((x, y) => { return(x.FileName.CompareTo(y.FileName)); });
foreach (var file in depotManifest.Files)
{
complete_download_size += file.TotalSize;
}
foreach (var file in depotManifest.Files)
{
string download_path = Path.Combine(installDir, file.FileName);
if (file.Flags.HasFlag(EDepotFileFlag.Directory))
{
if (!Directory.Exists(download_path))
{
Directory.CreateDirectory(download_path);
}
continue;
}
string dir_path = Path.GetDirectoryName(download_path);
if (!Directory.Exists(dir_path))
{
Directory.CreateDirectory(dir_path);
}
// TODO: non-checksum validation
FileInfo fi = new FileInfo(download_path);
if (fi.Exists && (ulong)fi.Length == file.TotalSize)
{
size_downloaded += file.TotalSize;
Console.WriteLine("{0,6:#00.00}% {1}", ((float)size_downloaded / (float)complete_download_size) * 100.0f, download_path);
continue;
}
Console.Write("{0,6:#00.00}% {1}", ((float)size_downloaded / (float)complete_download_size) * 100.0f, download_path);
FileStream fs = File.Create(download_path);
fs.SetLength((long)file.TotalSize);
foreach (var chunk in file.Chunks)
{
string chunkID = EncodeHexString(chunk.ChunkID);
byte[] encrypted_chunk = cdnClient.DownloadDepotChunk(depotId, chunkID);
byte[] chunk_data = CDNClient.ProcessChunk(encrypted_chunk, depotKey);
fs.Seek((long)chunk.Offset, SeekOrigin.Begin);
fs.Write(chunk_data, 0, chunk_data.Length);
size_downloaded += chunk.UncompressedLength;
Console.CursorLeft = 0;
Console.Write("{0,6:#00.00}%", ((float)size_downloaded / (float)complete_download_size) * 100.0f);
}
Console.WriteLine();
}
}