private static async Task <bool> DownloadChunk(DepotProcessor.ManifestJob job, DepotManifest.ChunkData chunk, FileInfo downloadPath) { for (var i = 0; i <= 5; i++) { try { var chunkData = await CDNClient.DownloadDepotChunkAsync(job.DepotID, chunk, job.Server, job.CDNToken, job.DepotKey); using (var fs = downloadPath.Open(FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite)) { fs.Seek((long)chunk.Offset, SeekOrigin.Begin); await fs.WriteAsync(chunkData.Data, 0, chunkData.Data.Length); } return(true); } catch (Exception e) { Log.WriteWarn("FileDownloader", "{0} exception: {1}", job.DepotID, e.Message); } await Task.Delay(Utils.ExponentionalBackoff(i)); } return(false); }
private static async Task <bool> DownloadChunk(DepotProcessor.ManifestJob job, DepotManifest.ChunkData chunk, FileInfo downloadPath) { for (var i = 0; i <= 5; i++) { try { var chunkData = await CDNClient.DownloadDepotChunkAsync(job.DepotID, chunk, job.Server, string.Empty, job.DepotKey); await using var fs = downloadPath.Open(FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite); fs.Seek((long)chunk.Offset, SeekOrigin.Begin); await fs.WriteAsync(chunkData.Data, 0, chunkData.Data.Length); return(true); } catch (Exception e) { Log.WriteWarn($"FileDownloader {job.DepotID}", $"Exception: {e}"); } if (i < 5) { await Task.Delay(Utils.ExponentionalBackoff(i + 1)); } } return(false); }
private static async Task <bool> DownloadChunk(DepotProcessor.ManifestJob job, DepotManifest.ChunkData chunk, FileInfo downloadPath, CancellationTokenSource chunkCancellation) { const int TRIES = 3; for (var i = 0; i <= TRIES; i++) { chunkCancellation.Token.ThrowIfCancellationRequested(); try { var chunkData = await CDNClient.DownloadDepotChunkAsync(job.DepotID, chunk, job.Server, string.Empty, job.DepotKey); await using var fs = downloadPath.Open(FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite); fs.Seek((long)chunk.Offset, SeekOrigin.Begin); await fs.WriteAsync(chunkData.Data); return(true); } catch (Exception e) { Log.WriteWarn($"FileDownloader {job.DepotID}", $"Exception: {e}"); } if (i < TRIES) { await Task.Delay(Utils.ExponentionalBackoff(i + 1)); } } return(false); }
public static void AddJob(Func <JobID> action, DepotProcessor.ManifestJob manifestJob) { var jobID = action(); var job = new JobAction { Action = action, ManifestJob = manifestJob }; Log.WriteDebug("Job Manager", "New depot job: {0} ({1} - {2})", jobID, manifestJob.DepotID, manifestJob.ManifestID); Jobs.TryAdd(jobID, job); }
/* * Here be dragons. */ public static async Task <EResult> DownloadFilesFromDepot(DepotProcessor.ManifestJob job, DepotManifest depotManifest) { var files = depotManifest.Files.Where(x => IsFileNameMatching(job.DepotID, x.FileName)).ToList(); var downloadState = EResult.Fail; var hashesFile = Path.Combine(Application.Path, "files", ".support", "hashes", string.Format("{0}.json", job.DepotID)); ConcurrentDictionary <string, byte[]> hashes; if (File.Exists(hashesFile)) { hashes = JsonConvert.DeserializeObject <ConcurrentDictionary <string, byte[]> >(File.ReadAllText(hashesFile)); } else { hashes = new ConcurrentDictionary <string, byte[]>(); } foreach (var file in hashes.Keys.Except(files.Select(x => x.FileName))) { Log.WriteWarn(nameof(FileDownloader), $"\"{file}\" no longer exists in manifest"); } Log.WriteInfo("FileDownloader", "Will download {0} files from depot {1}", files.Count, job.DepotID); var downloadedFiles = 0; var fileTasks = new Task[files.Count]; for (var i = 0; i < fileTasks.Length; i++) { var file = files[i]; fileTasks[i] = TaskManager.Run(async() => { hashes.TryGetValue(file.FileName, out var hash); var fileState = await DownloadFile(job, file, hash); if (fileState == EResult.OK || fileState == EResult.SameAsPreviousValue) { hashes[file.FileName] = file.FileHash; downloadedFiles++; } if (fileState != EResult.SameAsPreviousValue) { // Do not write progress info to log file Console.WriteLine("{1} [{0,6:#00.00}%] {2} files left to download", downloadedFiles / (float)files.Count * 100.0f, job.DepotName, files.Count - downloadedFiles); } if (downloadState == EResult.DataCorruption) { return; } if (fileState == EResult.OK || fileState == EResult.DataCorruption) { downloadState = fileState; } }).Unwrap(); // Register error handler on inner task TaskManager.RegisterErrorHandler(fileTasks[i]); } await Task.WhenAll(fileTasks).ConfigureAwait(false); if (downloadState == EResult.OK) { File.WriteAllText(hashesFile, JsonConvert.SerializeObject(hashes)); job.Result = EResult.OK; } else { job.Result = EResult.Ignored; } return(job.Result); }
private static async Task <EResult> DownloadFile(DepotProcessor.ManifestJob job, DepotManifest.FileData file, byte[] hash) { var directory = Path.Combine(Application.Path, "files", DownloadFolders[job.DepotID], Path.GetDirectoryName(file.FileName)); var finalPath = new FileInfo(Path.Combine(directory, Path.GetFileName(file.FileName))); var downloadPath = new FileInfo(Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Path.GetRandomFileName(), ".steamdb_tmp"))); if (!Directory.Exists(directory)) { Directory.CreateDirectory(directory); } else if (file.TotalSize == 0) { if (!finalPath.Exists) { using (var _ = finalPath.Create()) { // FileInfo.Create returns a stream but we don't need it } Log.WriteInfo("FileDownloader", "{0} created an empty file", file.FileName); return(EResult.SameAsPreviousValue); } else if (finalPath.Length == 0) { #if DEBUG Log.WriteDebug("FileDownloader", "{0} is already empty", file.FileName); #endif return(EResult.SameAsPreviousValue); } } else if (hash != null && file.FileHash.SequenceEqual(hash)) { #if DEBUG Log.WriteDebug("FileDownloader", "{0} already matches the file we have", file.FileName); #endif return(EResult.SameAsPreviousValue); } byte[] checksum; using (var sha = SHA1.Create()) { checksum = sha.ComputeHash(Encoding.UTF8.GetBytes(file.FileName)); } var neededChunks = new List <DepotManifest.ChunkData>(); var chunks = file.Chunks.OrderBy(x => x.Offset).ToList(); var oldChunksFile = Path.Combine(Application.Path, "files", ".support", "chunks", string.Format("{0}-{1}.json", job.DepotID, BitConverter.ToString(checksum))); using (var fs = downloadPath.Open(FileMode.OpenOrCreate, FileAccess.ReadWrite)) { fs.SetLength((long)file.TotalSize); if (finalPath.Exists && File.Exists(oldChunksFile)) { var oldChunks = JsonConvert.DeserializeObject <List <DepotManifest.ChunkData> >(File.ReadAllText(oldChunksFile), JsonHandleAllReferences); using (var fsOld = finalPath.Open(FileMode.Open, FileAccess.Read)) { foreach (var chunk in chunks) { var oldChunk = oldChunks.Find(c => c.ChunkID.SequenceEqual(chunk.ChunkID)); if (oldChunk != null) { var oldData = new byte[oldChunk.UncompressedLength]; fsOld.Seek((long)oldChunk.Offset, SeekOrigin.Begin); fsOld.Read(oldData, 0, oldData.Length); var existingChecksum = Utils.AdlerHash(oldData); if (existingChecksum.SequenceEqual(chunk.Checksum)) { fs.Seek((long)chunk.Offset, SeekOrigin.Begin); fs.Write(oldData, 0, oldData.Length); #if DEBUG Log.WriteDebug("FileDownloader", "{0} Found chunk ({1}), not downloading", file.FileName, chunk.Offset); #endif } else { neededChunks.Add(chunk); #if DEBUG Log.WriteDebug("FileDownloader", "{0} Found chunk ({1}), but checksum differs", file.FileName, chunk.Offset); #endif } } else { neededChunks.Add(chunk); } } } } else { neededChunks = chunks; } } var downloadedSize = file.TotalSize - (ulong)neededChunks.Sum(x => x.UncompressedLength); var chunkCancellation = new CancellationTokenSource(); var chunkTasks = new Task[neededChunks.Count]; Log.WriteInfo("FileDownloader", "Downloading {0} ({1} bytes, {2} out of {3} chunks)", file.FileName, downloadedSize, neededChunks.Count, chunks.Count); for (var i = 0; i < chunkTasks.Length; i++) { var chunk = neededChunks[i]; chunkTasks[i] = TaskManager.Run(async() => { try { chunkCancellation.Token.ThrowIfCancellationRequested(); await ChunkDownloadingSemaphore.WaitAsync(chunkCancellation.Token).ConfigureAwait(false); var result = await DownloadChunk(job, chunk, downloadPath); if (!result) { Log.WriteWarn("FileDownloader", "Failed to download chunk for {0}", file.FileName); chunkCancellation.Cancel(); } else { downloadedSize += chunk.UncompressedLength; // Do not write progress info to log file Console.WriteLine("{2} [{0,6:#00.00}%] {1}", downloadedSize / (float)file.TotalSize * 100.0f, file.FileName, job.DepotName); } } finally { ChunkDownloadingSemaphore.Release(); } }).Unwrap(); // Register error handler on inner task TaskManager.RegisterErrorHandler(chunkTasks[i]); } await Task.WhenAll(chunkTasks).ConfigureAwait(false); using (var fs = downloadPath.Open(FileMode.Open, FileAccess.ReadWrite)) { fs.Seek(0, SeekOrigin.Begin); using (var sha = SHA1.Create()) { checksum = sha.ComputeHash(fs); } } if (!file.FileHash.SequenceEqual(checksum)) { IRC.Instance.SendOps("{0}[{1}]{2} Failed to correctly download {3}{4}", Colors.OLIVE, job.DepotName, Colors.NORMAL, Colors.BLUE, file.FileName); Log.WriteWarn("FileDownloader", "Failed to download file {0} ({1})", file.FileName, job.Server); downloadPath.Delete(); return(EResult.DataCorruption); } Log.WriteInfo("FileDownloader", "Downloaded {0} from {1}", file.FileName, job.DepotName); finalPath.Delete(); downloadPath.MoveTo(finalPath.FullName); if (chunks.Count > 1) { File.WriteAllText(oldChunksFile, JsonConvert.SerializeObject(chunks, Formatting.None, JsonHandleAllReferences)); } else if (File.Exists(oldChunksFile)) { File.Delete(oldChunksFile); } return(EResult.OK); }
public static void DownloadFilesFromDepot(DepotProcessor.ManifestJob job, DepotManifest depotManifest) { var files = depotManifest.Files.Where(x => IsFileNameMatching(job.DepotID, x.FileName)).ToList(); var filesUpdated = false; Log.WriteDebug("FileDownloader", "Will download {0} files from depot {1}", files.Count(), job.DepotID); foreach (var file in files) { string directory = Path.Combine(Application.Path, FILES_DIRECTORY, job.DepotID.ToString(), Path.GetDirectoryName(file.FileName)); string finalPath = Path.Combine(directory, Path.GetFileName(file.FileName)); string downloadPath = Path.GetTempFileName(); if (!Directory.Exists(directory)) { Directory.CreateDirectory(directory); } else if (File.Exists(finalPath)) { using (var fs = File.Open(finalPath, FileMode.Open)) { using (var sha = new SHA1Managed()) { if (file.FileHash.SequenceEqual(sha.ComputeHash(fs))) { Log.WriteDebug("FileDownloader", "{0} already matches the file we have", file.FileName); continue; } } } } Log.WriteInfo("FileDownloader", "Downloading {0} ({1} bytes, {2} chunks)", file.FileName, file.TotalSize, file.Chunks.Count); uint count = 0; byte[] checksum; string lastError = "or checksum failed"; using (var fs = File.Open(downloadPath, FileMode.OpenOrCreate)) { fs.SetLength((long)file.TotalSize); var lockObject = new object(); // TODO: We *could* verify each chunk and only download needed ones Parallel.ForEach(file.Chunks, (chunk, state) => { var downloaded = false; for (var i = 0; i <= 5; i++) { try { var chunkData = CDNClient.DownloadDepotChunk(job.DepotID, chunk, job.Server, job.CDNToken, job.DepotKey); lock (lockObject) { fs.Seek((long)chunk.Offset, SeekOrigin.Begin); fs.Write(chunkData.Data, 0, chunkData.Data.Length); Log.WriteDebug("FileDownloader", "Downloaded {0} ({1}/{2})", file.FileName, ++count, file.Chunks.Count); } downloaded = true; break; } catch (Exception e) { lastError = e.Message; } } if (!downloaded) { state.Stop(); } }); fs.Seek(0, SeekOrigin.Begin); using (var sha = new SHA1Managed()) { checksum = sha.ComputeHash(fs); } } if (file.Chunks.Count == 0 || file.FileHash.SequenceEqual(checksum)) { Log.WriteInfo("FileDownloader", "Downloaded {0} from {1}", file.FileName, Steam.GetAppName(job.ParentAppID)); if (File.Exists(finalPath)) { File.Delete(finalPath); } File.Move(downloadPath, finalPath); filesUpdated = true; } else { IRC.Instance.SendOps("{0}[{1}]{2} Failed to download {3}: Only {4} out of {5} chunks downloaded ({6})", Colors.OLIVE, Steam.GetAppName(job.ParentAppID), Colors.NORMAL, file.FileName, count, file.Chunks.Count, lastError); Log.WriteError("FileDownloader", "Failed to download {0}: Only {1} out of {2} chunks downloaded from {3} ({4})", file.FileName, count, file.Chunks.Count, job.Server, lastError); File.Delete(downloadPath); } } if (filesUpdated) { var updateScript = Path.Combine(Application.Path, "files", "update.sh"); if (File.Exists(updateScript)) { // YOLO Process.Start(updateScript, job.DepotID.ToString()); } } }
/* * Here be dragons. */ public static async Task <EResult> DownloadFilesFromDepot(DepotProcessor.ManifestJob job, DepotManifest depotManifest) { var filesRegex = Files[job.DepotID]; var files = depotManifest.Files.Where(x => filesRegex.IsMatch(x.FileName.Replace('\\', '/'))).ToList(); var downloadState = EResult.Fail; ConcurrentDictionary <string, ExistingFileData> existingFileData; await using (var db = await Database.GetConnectionAsync()) { var data = db.ExecuteScalar <string>("SELECT `Value` FROM `LocalConfig` WHERE `ConfigKey` = @Key", new { Key = $"depot.{job.DepotID}" }); if (data != null) { existingFileData = JsonConvert.DeserializeObject <ConcurrentDictionary <string, ExistingFileData> >(data); } else { existingFileData = new ConcurrentDictionary <string, ExistingFileData>(); } } foreach (var file in existingFileData.Keys.Except(files.Select(x => x.FileName))) { Log.WriteWarn(nameof(FileDownloader), $"\"{file}\" no longer exists in manifest"); } Log.WriteInfo($"FileDownloader {job.DepotID}", $"Will download {files.Count} files"); var downloadedFiles = 0; var fileTasks = new Task[files.Count]; for (var i = 0; i < fileTasks.Length; i++) { var file = files[i]; fileTasks[i] = TaskManager.Run(async() => { var existingFile = existingFileData.GetOrAdd(file.FileName, _ => new ExistingFileData()); EResult fileState; try { await ChunkDownloadingSemaphore.WaitAsync().ConfigureAwait(false); fileState = await DownloadFile(job, file, existingFile); } finally { ChunkDownloadingSemaphore.Release(); } if (fileState == EResult.OK || fileState == EResult.SameAsPreviousValue) { existingFile.FileHash = file.FileHash; downloadedFiles++; } if (fileState != EResult.SameAsPreviousValue) { // Do not write progress info to log file Console.WriteLine($"{job.DepotName} [{downloadedFiles / (float) files.Count * 100.0f,6:#00.00}%] {files.Count - downloadedFiles} files left to download"); } if (downloadState == EResult.DataCorruption) { return; } if (fileState == EResult.OK || fileState == EResult.DataCorruption) { downloadState = fileState; } }); } await Task.WhenAll(fileTasks).ConfigureAwait(false); await LocalConfig.Update($"depot.{job.DepotID}", JsonConvert.SerializeObject(existingFileData)); job.Result = downloadState switch { EResult.OK => EResult.OK, EResult.DataCorruption => EResult.DataCorruption, _ => EResult.Ignored }; return(job.Result); }
private static async Task <EResult> DownloadFile(DepotProcessor.ManifestJob job, DepotManifest.FileData file, ExistingFileData existingFile) { var directory = Path.Combine(Application.Path, "files", DownloadFolders[job.DepotID], Path.GetDirectoryName(file.FileName)); var finalPath = new FileInfo(Path.Combine(directory, Path.GetFileName(file.FileName))); var downloadPath = new FileInfo(Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Path.GetRandomFileName(), ".steamdb_tmp"))); if (!Directory.Exists(directory)) { Directory.CreateDirectory(directory); } else if (file.TotalSize == 0) { if (!finalPath.Exists) { await using (var _ = finalPath.Create()) { // FileInfo.Create returns a stream but we don't need it } Log.WriteInfo($"FileDownloader {job.DepotID}", $"{file.FileName} created an empty file"); return(EResult.SameAsPreviousValue); } else if (finalPath.Length == 0) { #if DEBUG Log.WriteDebug($"FileDownloader {job.DepotID}", $"{file.FileName} is already empty"); #endif return(EResult.SameAsPreviousValue); } } else if (existingFile.FileHash != null && file.FileHash.SequenceEqual(existingFile.FileHash)) { #if DEBUG Log.WriteDebug($"FileDownloader {job.DepotID}", $"{file.FileName} already matches the file we have"); #endif return(EResult.SameAsPreviousValue); } using var sha = SHA1.Create(); var neededChunks = new List <DepotManifest.ChunkData>(); var chunks = file.Chunks.OrderBy(x => x.Offset).ToList(); await using (var fs = downloadPath.Open(FileMode.OpenOrCreate, FileAccess.ReadWrite)) { fs.SetLength((long)file.TotalSize); if (finalPath.Exists) { await using var fsOld = finalPath.Open(FileMode.Open, FileAccess.Read); foreach (var chunk in chunks) { var oldChunk = existingFile.Chunks.FirstOrDefault(c => c.Value.SequenceEqual(chunk.ChunkID)); if (oldChunk.Value != null) { var oldData = new byte[chunk.UncompressedLength]; fsOld.Seek((long)oldChunk.Key, SeekOrigin.Begin); fsOld.Read(oldData, 0, oldData.Length); var existingChecksum = sha.ComputeHash(oldData); if (existingChecksum.SequenceEqual(chunk.ChunkID)) { fs.Seek((long)chunk.Offset, SeekOrigin.Begin); fs.Write(oldData, 0, oldData.Length); #if DEBUG Log.WriteDebug($"FileDownloader {job.DepotID}", $"{file.FileName} Found chunk ({chunk.Offset}), not downloading"); #endif } else { neededChunks.Add(chunk); #if DEBUG Log.WriteDebug($"FileDownloader {job.DepotID}", $"{file.FileName} Found chunk ({chunk.Offset}), but checksum differs"); #endif } } else { neededChunks.Add(chunk); } } } else { neededChunks = chunks; } } using var chunkCancellation = new CancellationTokenSource(); var downloadedSize = file.TotalSize - (ulong)neededChunks.Sum(x => x.UncompressedLength); var chunkTasks = new Task[neededChunks.Count]; Log.WriteInfo($"FileDownloader {job.DepotID}", $"Downloading {file.FileName} ({neededChunks.Count} out of {chunks.Count} chunks to download)"); for (var i = 0; i < chunkTasks.Length; i++) { var chunk = neededChunks[i]; chunkTasks[i] = TaskManager.Run(async() => { var result = await DownloadChunk(job, chunk, downloadPath, chunkCancellation); if (!result) { Log.WriteWarn($"FileDownloader {job.DepotID}", $"Failed to download chunk for {file.FileName} ({chunk.Offset})"); chunkCancellation.Cancel(); } else { downloadedSize += chunk.UncompressedLength; // Do not write progress info to log file Console.WriteLine($"{job.DepotName} [{downloadedSize / (float) file.TotalSize * 100.0f,6:#00.00}%] {file.FileName}"); } }); } await Task.WhenAll(chunkTasks).ConfigureAwait(false); byte[] checksum; await using (var fs = downloadPath.Open(FileMode.Open, FileAccess.ReadWrite)) { checksum = await sha.ComputeHashAsync(fs, chunkCancellation.Token); } if (!file.FileHash.SequenceEqual(checksum)) { if (!job.DownloadCorrupted) { job.DownloadCorrupted = true; IRC.Instance.SendOps($"{Colors.OLIVE}[{job.DepotName}]{Colors.NORMAL} Failed to correctly download {Colors.BLUE}{file.FileName}"); } Log.WriteWarn($"FileDownloader {job.DepotID}", $"Hash check failed for {file.FileName} ({job.Server})"); downloadPath.Delete(); existingFile.FileHash = null; existingFile.Chunks.Clear(); return(EResult.DataCorruption); } Log.WriteInfo($"FileDownloader {job.DepotID}", $"Downloaded {file.FileName}"); finalPath.Delete(); downloadPath.MoveTo(finalPath.FullName); if (chunks.Count > 0) { existingFile.Chunks = chunks.ToDictionary(chunk => chunk.Offset, chunk => chunk.ChunkID); } else { existingFile.Chunks.Clear(); } return(EResult.OK); }
/* * Here be dragons. */ public static EResult DownloadFilesFromDepot(uint appID, DepotProcessor.ManifestJob job, DepotManifest depotManifest) { var files = depotManifest.Files.Where(x => IsFileNameMatching(job.DepotID, x.FileName)).ToList(); var filesUpdated = false; var filesAnyFailed = false; var hashesFile = Path.Combine(Application.Path, "files", ".support", "hashes", string.Format("{0}.json", job.DepotID)); var hashes = new Dictionary <string, byte[]>(); if (File.Exists(hashesFile)) { hashes = JsonConvert.DeserializeObject <Dictionary <string, byte[]> >(File.ReadAllText(hashesFile)); } Log.WriteDebug("FileDownloader", "Will download {0} files from depot {1}", files.Count, job.DepotID); Parallel.ForEach(files, new ParallelOptions { MaxDegreeOfParallelism = 2 }, (file, state2) => { string directory = Path.Combine(Application.Path, "files", DownloadFolders[job.DepotID], Path.GetDirectoryName(file.FileName)); string finalPath = Path.Combine(directory, Path.GetFileName(file.FileName)); string downloadPath = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Path.GetRandomFileName(), ".steamdb_tmp")); if (!Directory.Exists(directory)) { Directory.CreateDirectory(directory); } else if (file.TotalSize == 0) { if (File.Exists(finalPath)) { var f = new FileInfo(finalPath); if (f.Length == 0) { #if DEBUG Log.WriteDebug("FileDownloader", "{0} is already empty", file.FileName); #endif return; } } else { File.Create(finalPath); Log.WriteInfo("FileDownloader", "{0} created an empty file", file.FileName); return; } } else if (hashes.ContainsKey(file.FileName) && file.FileHash.SequenceEqual(hashes[file.FileName])) { #if DEBUG Log.WriteDebug("FileDownloader", "{0} already matches the file we have", file.FileName); #endif return; } var chunks = file.Chunks.OrderBy(x => x.Offset).ToList(); Log.WriteInfo("FileDownloader", "Downloading {0} ({1} bytes, {2} chunks)", file.FileName, file.TotalSize, chunks.Count); uint count = 0; byte[] checksum; string lastError = "or checksum failed"; string oldChunksFile; using (var sha = new SHA1Managed()) { oldChunksFile = Path.Combine(Application.Path, "files", ".support", "chunks", string.Format("{0}-{1}.json", job.DepotID, BitConverter.ToString(sha.ComputeHash(Encoding.UTF8.GetBytes(file.FileName)))) ); } using (var fs = File.Open(downloadPath, FileMode.OpenOrCreate, FileAccess.ReadWrite)) { fs.SetLength((long)file.TotalSize); var lockObject = new object(); var neededChunks = new List <DepotManifest.ChunkData>(); if (File.Exists(oldChunksFile) && File.Exists(finalPath)) { var oldChunks = JsonConvert.DeserializeObject <List <DepotManifest.ChunkData> >( File.ReadAllText(oldChunksFile), new JsonSerializerSettings { PreserveReferencesHandling = PreserveReferencesHandling.All } ); using (var fsOld = File.Open(finalPath, FileMode.Open, FileAccess.Read)) { foreach (var chunk in chunks) { var oldChunk = oldChunks.FirstOrDefault(c => c.ChunkID.SequenceEqual(chunk.ChunkID)); if (oldChunk != null) { var oldData = new byte[oldChunk.UncompressedLength]; fsOld.Seek((long)oldChunk.Offset, SeekOrigin.Begin); fsOld.Read(oldData, 0, oldData.Length); var existingChecksum = Utils.AdlerHash(oldData); if (existingChecksum.SequenceEqual(chunk.Checksum)) { fs.Seek((long)chunk.Offset, SeekOrigin.Begin); fs.Write(oldData, 0, oldData.Length); #if DEBUG Log.WriteDebug("FileDownloader", "{0} Found chunk ({1}), not downloading ({2}/{3})", file.FileName, chunk.Offset, ++count, chunks.Count); #endif } else { neededChunks.Add(chunk); #if DEBUG Log.WriteDebug("FileDownloader", "{0} Found chunk ({1}), but checksum differs", file.FileName, chunk.Offset); #endif } } else { neededChunks.Add(chunk); } } } } else { neededChunks = chunks; } Parallel.ForEach(neededChunks, new ParallelOptions { MaxDegreeOfParallelism = 3 }, (chunk, state) => { var downloaded = false; for (var i = 0; i <= 5; i++) { try { var chunkData = CDNClient.DownloadDepotChunk(job.DepotID, chunk, job.Server, job.CDNToken, job.DepotKey); lock (lockObject) { fs.Seek((long)chunk.Offset, SeekOrigin.Begin); fs.Write(chunkData.Data, 0, chunkData.Data.Length); Log.WriteDebug("FileDownloader", "Downloaded {0} ({1}/{2})", file.FileName, ++count, chunks.Count); } downloaded = true; break; } catch (Exception e) { lastError = e.Message; } Task.Delay(Utils.ExponentionalBackoff(i)).Wait(); } if (!downloaded) { state.Stop(); } }); fs.Seek(0, SeekOrigin.Begin); using (var sha = new SHA1Managed()) { checksum = sha.ComputeHash(fs); } } if (file.FileHash.SequenceEqual(checksum)) { Log.WriteInfo("FileDownloader", "Downloaded {0} from {1}", file.FileName, Steam.GetAppName(appID)); hashes[file.FileName] = checksum; if (File.Exists(finalPath)) { File.Delete(finalPath); } File.Move(downloadPath, finalPath); if (chunks.Count > 1) { File.WriteAllText(oldChunksFile, JsonConvert.SerializeObject( chunks, Formatting.None, new JsonSerializerSettings { PreserveReferencesHandling = PreserveReferencesHandling.All } ) ); } else if (File.Exists(oldChunksFile)) { File.Delete(oldChunksFile); } filesUpdated = true; } else { filesAnyFailed = true; IRC.Instance.SendOps("{0}[{1}]{2} Failed to download {3}: Only {4} out of {5} chunks downloaded ({6})", Colors.OLIVE, Steam.GetAppName(appID), Colors.NORMAL, file.FileName, count, chunks.Count, lastError); Log.WriteError("FileDownloader", "Failed to download {0}: Only {1} out of {2} chunks downloaded from {3} ({4})", file.FileName, count, chunks.Count, job.Server, lastError); File.Delete(downloadPath); } }); if (filesAnyFailed) { using (var db = Database.GetConnection()) { // Mark this depot for redownload db.Execute("UPDATE `Depots` SET `LastManifestID` = 0 WHERE `DepotID` = @DepotID", new { job.DepotID }); } IRC.Instance.SendOps("{0}[{1}]{2} Failed to download some files, not running update script to prevent broken diffs.", Colors.OLIVE, Steam.GetAppName(appID), Colors.NORMAL); } else if (filesUpdated) { File.WriteAllText(hashesFile, JsonConvert.SerializeObject(hashes)); job.Result = EResult.OK; } else { job.Result = EResult.Ignored; } return(job.Result); }