private static async Task <bool> DownloadChunk(DepotProcessor.ManifestJob job, DepotManifest.ChunkData chunk, FileInfo downloadPath)
{
for (var i = 0; i <= 5; i++)
{
try
{
var chunkData = await CDNClient.DownloadDepotChunkAsync(job.DepotID, chunk, job.Server, job.CDNToken, job.DepotKey);
using (var fs = downloadPath.Open(FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite))
{
fs.Seek((long)chunk.Offset, SeekOrigin.Begin);
await fs.WriteAsync(chunkData.Data, 0, chunkData.Data.Length);
}
return(true);
}
catch (Exception e)
{
Log.WriteWarn("FileDownloader", "{0} exception: {1}", job.DepotID, e.Message);
}
await Task.Delay(Utils.ExponentionalBackoff(i));
}
return(false);
}
private static async Task <bool> DownloadChunk(DepotProcessor.ManifestJob job, DepotManifest.ChunkData chunk, FileInfo downloadPath)
{
for (var i = 0; i <= 5; i++)
{
try
{
var chunkData = await CDNClient.DownloadDepotChunkAsync(job.DepotID, chunk, job.Server, string.Empty, job.DepotKey);
await using var fs = downloadPath.Open(FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite);
fs.Seek((long)chunk.Offset, SeekOrigin.Begin);
await fs.WriteAsync(chunkData.Data, 0, chunkData.Data.Length);
return(true);
}
catch (Exception e)
{
Log.WriteWarn($"FileDownloader {job.DepotID}", $"Exception: {e}");
}
if (i < 5)
{
await Task.Delay(Utils.ExponentionalBackoff(i + 1));
}
}
return(false);
}
private static async Task <bool> DownloadChunk(DepotProcessor.ManifestJob job, DepotManifest.ChunkData chunk, FileInfo downloadPath, CancellationTokenSource chunkCancellation)
{
const int TRIES = 3;
for (var i = 0; i <= TRIES; i++)
{
chunkCancellation.Token.ThrowIfCancellationRequested();
try
{
var chunkData = await CDNClient.DownloadDepotChunkAsync(job.DepotID, chunk, job.Server, string.Empty, job.DepotKey);
await using var fs = downloadPath.Open(FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite);
fs.Seek((long)chunk.Offset, SeekOrigin.Begin);
await fs.WriteAsync(chunkData.Data);
return(true);
}
catch (Exception e)
{
Log.WriteWarn($"FileDownloader {job.DepotID}", $"Exception: {e}");
}
if (i < TRIES)
{
await Task.Delay(Utils.ExponentionalBackoff(i + 1));
}
}
return(false);
}
public static void AddJob(Func <JobID> action, DepotProcessor.ManifestJob manifestJob)
{
var jobID = action();
var job = new JobAction
{
Action = action,
ManifestJob = manifestJob
};
Log.WriteDebug("Job Manager", "New depot job: {0} ({1} - {2})", jobID, manifestJob.DepotID, manifestJob.ManifestID);
Jobs.TryAdd(jobID, job);
}
/*
* Here be dragons.
*/
public static async Task <EResult> DownloadFilesFromDepot(DepotProcessor.ManifestJob job, DepotManifest depotManifest)
{
var files = depotManifest.Files.Where(x => IsFileNameMatching(job.DepotID, x.FileName)).ToList();
var downloadState = EResult.Fail;
var hashesFile = Path.Combine(Application.Path, "files", ".support", "hashes", string.Format("{0}.json", job.DepotID));
ConcurrentDictionary <string, byte[]> hashes;
if (File.Exists(hashesFile))
{
hashes = JsonConvert.DeserializeObject <ConcurrentDictionary <string, byte[]> >(File.ReadAllText(hashesFile));
}
else
{
hashes = new ConcurrentDictionary <string, byte[]>();
}
foreach (var file in hashes.Keys.Except(files.Select(x => x.FileName)))
{
Log.WriteWarn(nameof(FileDownloader), $"\"{file}\" no longer exists in manifest");
}
Log.WriteInfo("FileDownloader", "Will download {0} files from depot {1}", files.Count, job.DepotID);
var downloadedFiles = 0;
var fileTasks = new Task[files.Count];
for (var i = 0; i < fileTasks.Length; i++)
{
var file = files[i];
fileTasks[i] = TaskManager.Run(async() =>
{
hashes.TryGetValue(file.FileName, out var hash);
var fileState = await DownloadFile(job, file, hash);
if (fileState == EResult.OK || fileState == EResult.SameAsPreviousValue)
{
hashes[file.FileName] = file.FileHash;
downloadedFiles++;
}
if (fileState != EResult.SameAsPreviousValue)
{
// Do not write progress info to log file
Console.WriteLine("{1} [{0,6:#00.00}%] {2} files left to download", downloadedFiles / (float)files.Count * 100.0f, job.DepotName, files.Count - downloadedFiles);
}
if (downloadState == EResult.DataCorruption)
{
return;
}
if (fileState == EResult.OK || fileState == EResult.DataCorruption)
{
downloadState = fileState;
}
}).Unwrap();
// Register error handler on inner task
TaskManager.RegisterErrorHandler(fileTasks[i]);
}
await Task.WhenAll(fileTasks).ConfigureAwait(false);
if (downloadState == EResult.OK)
{
File.WriteAllText(hashesFile, JsonConvert.SerializeObject(hashes));
job.Result = EResult.OK;
}
else
{
job.Result = EResult.Ignored;
}
return(job.Result);
}
private static async Task <EResult> DownloadFile(DepotProcessor.ManifestJob job, DepotManifest.FileData file, byte[] hash)
{
var directory = Path.Combine(Application.Path, "files", DownloadFolders[job.DepotID], Path.GetDirectoryName(file.FileName));
var finalPath = new FileInfo(Path.Combine(directory, Path.GetFileName(file.FileName)));
var downloadPath = new FileInfo(Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Path.GetRandomFileName(), ".steamdb_tmp")));
if (!Directory.Exists(directory))
{
Directory.CreateDirectory(directory);
}
else if (file.TotalSize == 0)
{
if (!finalPath.Exists)
{
using (var _ = finalPath.Create())
{
// FileInfo.Create returns a stream but we don't need it
}
Log.WriteInfo("FileDownloader", "{0} created an empty file", file.FileName);
return(EResult.SameAsPreviousValue);
}
else if (finalPath.Length == 0)
{
#if DEBUG
Log.WriteDebug("FileDownloader", "{0} is already empty", file.FileName);
#endif
return(EResult.SameAsPreviousValue);
}
}
else if (hash != null && file.FileHash.SequenceEqual(hash))
{
#if DEBUG
Log.WriteDebug("FileDownloader", "{0} already matches the file we have", file.FileName);
#endif
return(EResult.SameAsPreviousValue);
}
byte[] checksum;
using (var sha = SHA1.Create())
{
checksum = sha.ComputeHash(Encoding.UTF8.GetBytes(file.FileName));
}
var neededChunks = new List <DepotManifest.ChunkData>();
var chunks = file.Chunks.OrderBy(x => x.Offset).ToList();
var oldChunksFile = Path.Combine(Application.Path, "files", ".support", "chunks", string.Format("{0}-{1}.json", job.DepotID, BitConverter.ToString(checksum)));
using (var fs = downloadPath.Open(FileMode.OpenOrCreate, FileAccess.ReadWrite))
{
fs.SetLength((long)file.TotalSize);
if (finalPath.Exists && File.Exists(oldChunksFile))
{
var oldChunks = JsonConvert.DeserializeObject <List <DepotManifest.ChunkData> >(File.ReadAllText(oldChunksFile), JsonHandleAllReferences);
using (var fsOld = finalPath.Open(FileMode.Open, FileAccess.Read))
{
foreach (var chunk in chunks)
{
var oldChunk = oldChunks.Find(c => c.ChunkID.SequenceEqual(chunk.ChunkID));
if (oldChunk != null)
{
var oldData = new byte[oldChunk.UncompressedLength];
fsOld.Seek((long)oldChunk.Offset, SeekOrigin.Begin);
fsOld.Read(oldData, 0, oldData.Length);
var existingChecksum = Utils.AdlerHash(oldData);
if (existingChecksum.SequenceEqual(chunk.Checksum))
{
fs.Seek((long)chunk.Offset, SeekOrigin.Begin);
fs.Write(oldData, 0, oldData.Length);
#if DEBUG
Log.WriteDebug("FileDownloader", "{0} Found chunk ({1}), not downloading", file.FileName, chunk.Offset);
#endif
}
else
{
neededChunks.Add(chunk);
#if DEBUG
Log.WriteDebug("FileDownloader", "{0} Found chunk ({1}), but checksum differs", file.FileName, chunk.Offset);
#endif
}
}
else
{
neededChunks.Add(chunk);
}
}
}
}
else
{
neededChunks = chunks;
}
}
var downloadedSize = file.TotalSize - (ulong)neededChunks.Sum(x => x.UncompressedLength);
var chunkCancellation = new CancellationTokenSource();
var chunkTasks = new Task[neededChunks.Count];
Log.WriteInfo("FileDownloader", "Downloading {0} ({1} bytes, {2} out of {3} chunks)", file.FileName, downloadedSize, neededChunks.Count, chunks.Count);
for (var i = 0; i < chunkTasks.Length; i++)
{
var chunk = neededChunks[i];
chunkTasks[i] = TaskManager.Run(async() =>
{
try
{
chunkCancellation.Token.ThrowIfCancellationRequested();
await ChunkDownloadingSemaphore.WaitAsync(chunkCancellation.Token).ConfigureAwait(false);
var result = await DownloadChunk(job, chunk, downloadPath);
if (!result)
{
Log.WriteWarn("FileDownloader", "Failed to download chunk for {0}", file.FileName);
chunkCancellation.Cancel();
}
else
{
downloadedSize += chunk.UncompressedLength;
// Do not write progress info to log file
Console.WriteLine("{2} [{0,6:#00.00}%] {1}", downloadedSize / (float)file.TotalSize * 100.0f, file.FileName, job.DepotName);
}
}
finally
{
ChunkDownloadingSemaphore.Release();
}
}).Unwrap();
// Register error handler on inner task
TaskManager.RegisterErrorHandler(chunkTasks[i]);
}
await Task.WhenAll(chunkTasks).ConfigureAwait(false);
using (var fs = downloadPath.Open(FileMode.Open, FileAccess.ReadWrite))
{
fs.Seek(0, SeekOrigin.Begin);
using (var sha = SHA1.Create())
{
checksum = sha.ComputeHash(fs);
}
}
if (!file.FileHash.SequenceEqual(checksum))
{
IRC.Instance.SendOps("{0}[{1}]{2} Failed to correctly download {3}{4}",
Colors.OLIVE, job.DepotName, Colors.NORMAL, Colors.BLUE, file.FileName);
Log.WriteWarn("FileDownloader", "Failed to download file {0} ({1})", file.FileName, job.Server);
downloadPath.Delete();
return(EResult.DataCorruption);
}
Log.WriteInfo("FileDownloader", "Downloaded {0} from {1}", file.FileName, job.DepotName);
finalPath.Delete();
downloadPath.MoveTo(finalPath.FullName);
if (chunks.Count > 1)
{
File.WriteAllText(oldChunksFile, JsonConvert.SerializeObject(chunks, Formatting.None, JsonHandleAllReferences));
}
else if (File.Exists(oldChunksFile))
{
File.Delete(oldChunksFile);
}
return(EResult.OK);
}
public static void DownloadFilesFromDepot(DepotProcessor.ManifestJob job, DepotManifest depotManifest)
{
var files = depotManifest.Files.Where(x => IsFileNameMatching(job.DepotID, x.FileName)).ToList();
var filesUpdated = false;
Log.WriteDebug("FileDownloader", "Will download {0} files from depot {1}", files.Count(), job.DepotID);
foreach (var file in files)
{
string directory = Path.Combine(Application.Path, FILES_DIRECTORY, job.DepotID.ToString(), Path.GetDirectoryName(file.FileName));
string finalPath = Path.Combine(directory, Path.GetFileName(file.FileName));
string downloadPath = Path.GetTempFileName();
if (!Directory.Exists(directory))
{
Directory.CreateDirectory(directory);
}
else if (File.Exists(finalPath))
{
using (var fs = File.Open(finalPath, FileMode.Open))
{
using (var sha = new SHA1Managed())
{
if (file.FileHash.SequenceEqual(sha.ComputeHash(fs)))
{
Log.WriteDebug("FileDownloader", "{0} already matches the file we have", file.FileName);
continue;
}
}
}
}
Log.WriteInfo("FileDownloader", "Downloading {0} ({1} bytes, {2} chunks)", file.FileName, file.TotalSize, file.Chunks.Count);
uint count = 0;
byte[] checksum;
string lastError = "or checksum failed";
using (var fs = File.Open(downloadPath, FileMode.OpenOrCreate))
{
fs.SetLength((long)file.TotalSize);
var lockObject = new object();
// TODO: We *could* verify each chunk and only download needed ones
Parallel.ForEach(file.Chunks, (chunk, state) =>
{
var downloaded = false;
for (var i = 0; i <= 5; i++)
{
try
{
var chunkData = CDNClient.DownloadDepotChunk(job.DepotID, chunk, job.Server, job.CDNToken, job.DepotKey);
lock (lockObject)
{
fs.Seek((long)chunk.Offset, SeekOrigin.Begin);
fs.Write(chunkData.Data, 0, chunkData.Data.Length);
Log.WriteDebug("FileDownloader", "Downloaded {0} ({1}/{2})", file.FileName, ++count, file.Chunks.Count);
}
downloaded = true;
break;
}
catch (Exception e)
{
lastError = e.Message;
}
}
if (!downloaded)
{
state.Stop();
}
});
fs.Seek(0, SeekOrigin.Begin);
using (var sha = new SHA1Managed())
{
checksum = sha.ComputeHash(fs);
}
}
if (file.Chunks.Count == 0 || file.FileHash.SequenceEqual(checksum))
{
Log.WriteInfo("FileDownloader", "Downloaded {0} from {1}", file.FileName, Steam.GetAppName(job.ParentAppID));
if (File.Exists(finalPath))
{
File.Delete(finalPath);
}
File.Move(downloadPath, finalPath);
filesUpdated = true;
}
else
{
IRC.Instance.SendOps("{0}[{1}]{2} Failed to download {3}: Only {4} out of {5} chunks downloaded ({6})",
Colors.OLIVE, Steam.GetAppName(job.ParentAppID), Colors.NORMAL, file.FileName, count, file.Chunks.Count, lastError);
Log.WriteError("FileDownloader", "Failed to download {0}: Only {1} out of {2} chunks downloaded from {3} ({4})",
file.FileName, count, file.Chunks.Count, job.Server, lastError);
File.Delete(downloadPath);
}
}
if (filesUpdated)
{
var updateScript = Path.Combine(Application.Path, "files", "update.sh");
if (File.Exists(updateScript))
{
// YOLO
Process.Start(updateScript, job.DepotID.ToString());
}
}
}
/*
* Here be dragons.
*/
public static async Task <EResult> DownloadFilesFromDepot(DepotProcessor.ManifestJob job, DepotManifest depotManifest)
{
var filesRegex = Files[job.DepotID];
var files = depotManifest.Files.Where(x => filesRegex.IsMatch(x.FileName.Replace('\\', '/'))).ToList();
var downloadState = EResult.Fail;
ConcurrentDictionary <string, ExistingFileData> existingFileData;
await using (var db = await Database.GetConnectionAsync())
{
var data = db.ExecuteScalar <string>("SELECT `Value` FROM `LocalConfig` WHERE `ConfigKey` = @Key", new { Key = $"depot.{job.DepotID}" });
if (data != null)
{
existingFileData = JsonConvert.DeserializeObject <ConcurrentDictionary <string, ExistingFileData> >(data);
}
else
{
existingFileData = new ConcurrentDictionary <string, ExistingFileData>();
}
}
foreach (var file in existingFileData.Keys.Except(files.Select(x => x.FileName)))
{
Log.WriteWarn(nameof(FileDownloader), $"\"{file}\" no longer exists in manifest");
}
Log.WriteInfo($"FileDownloader {job.DepotID}", $"Will download {files.Count} files");
var downloadedFiles = 0;
var fileTasks = new Task[files.Count];
for (var i = 0; i < fileTasks.Length; i++)
{
var file = files[i];
fileTasks[i] = TaskManager.Run(async() =>
{
var existingFile = existingFileData.GetOrAdd(file.FileName, _ => new ExistingFileData());
EResult fileState;
try
{
await ChunkDownloadingSemaphore.WaitAsync().ConfigureAwait(false);
fileState = await DownloadFile(job, file, existingFile);
}
finally
{
ChunkDownloadingSemaphore.Release();
}
if (fileState == EResult.OK || fileState == EResult.SameAsPreviousValue)
{
existingFile.FileHash = file.FileHash;
downloadedFiles++;
}
if (fileState != EResult.SameAsPreviousValue)
{
// Do not write progress info to log file
Console.WriteLine($"{job.DepotName} [{downloadedFiles / (float) files.Count * 100.0f,6:#00.00}%] {files.Count - downloadedFiles} files left to download");
}
if (downloadState == EResult.DataCorruption)
{
return;
}
if (fileState == EResult.OK || fileState == EResult.DataCorruption)
{
downloadState = fileState;
}
});
}
await Task.WhenAll(fileTasks).ConfigureAwait(false);
await LocalConfig.Update($"depot.{job.DepotID}", JsonConvert.SerializeObject(existingFileData));
job.Result = downloadState switch
{
EResult.OK => EResult.OK,
EResult.DataCorruption => EResult.DataCorruption,
_ => EResult.Ignored
};
return(job.Result);
}
private static async Task <EResult> DownloadFile(DepotProcessor.ManifestJob job, DepotManifest.FileData file, ExistingFileData existingFile)
{
var directory = Path.Combine(Application.Path, "files", DownloadFolders[job.DepotID], Path.GetDirectoryName(file.FileName));
var finalPath = new FileInfo(Path.Combine(directory, Path.GetFileName(file.FileName)));
var downloadPath = new FileInfo(Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Path.GetRandomFileName(), ".steamdb_tmp")));
if (!Directory.Exists(directory))
{
Directory.CreateDirectory(directory);
}
else if (file.TotalSize == 0)
{
if (!finalPath.Exists)
{
await using (var _ = finalPath.Create())
{
// FileInfo.Create returns a stream but we don't need it
}
Log.WriteInfo($"FileDownloader {job.DepotID}", $"{file.FileName} created an empty file");
return(EResult.SameAsPreviousValue);
}
else if (finalPath.Length == 0)
{
#if DEBUG
Log.WriteDebug($"FileDownloader {job.DepotID}", $"{file.FileName} is already empty");
#endif
return(EResult.SameAsPreviousValue);
}
}
else if (existingFile.FileHash != null && file.FileHash.SequenceEqual(existingFile.FileHash))
{
#if DEBUG
Log.WriteDebug($"FileDownloader {job.DepotID}", $"{file.FileName} already matches the file we have");
#endif
return(EResult.SameAsPreviousValue);
}
using var sha = SHA1.Create();
var neededChunks = new List <DepotManifest.ChunkData>();
var chunks = file.Chunks.OrderBy(x => x.Offset).ToList();
await using (var fs = downloadPath.Open(FileMode.OpenOrCreate, FileAccess.ReadWrite))
{
fs.SetLength((long)file.TotalSize);
if (finalPath.Exists)
{
await using var fsOld = finalPath.Open(FileMode.Open, FileAccess.Read);
foreach (var chunk in chunks)
{
var oldChunk = existingFile.Chunks.FirstOrDefault(c => c.Value.SequenceEqual(chunk.ChunkID));
if (oldChunk.Value != null)
{
var oldData = new byte[chunk.UncompressedLength];
fsOld.Seek((long)oldChunk.Key, SeekOrigin.Begin);
fsOld.Read(oldData, 0, oldData.Length);
var existingChecksum = sha.ComputeHash(oldData);
if (existingChecksum.SequenceEqual(chunk.ChunkID))
{
fs.Seek((long)chunk.Offset, SeekOrigin.Begin);
fs.Write(oldData, 0, oldData.Length);
#if DEBUG
Log.WriteDebug($"FileDownloader {job.DepotID}", $"{file.FileName} Found chunk ({chunk.Offset}), not downloading");
#endif
}
else
{
neededChunks.Add(chunk);
#if DEBUG
Log.WriteDebug($"FileDownloader {job.DepotID}", $"{file.FileName} Found chunk ({chunk.Offset}), but checksum differs");
#endif
}
}
else
{
neededChunks.Add(chunk);
}
}
}
else
{
neededChunks = chunks;
}
}
using var chunkCancellation = new CancellationTokenSource();
var downloadedSize = file.TotalSize - (ulong)neededChunks.Sum(x => x.UncompressedLength);
var chunkTasks = new Task[neededChunks.Count];
Log.WriteInfo($"FileDownloader {job.DepotID}", $"Downloading {file.FileName} ({neededChunks.Count} out of {chunks.Count} chunks to download)");
for (var i = 0; i < chunkTasks.Length; i++)
{
var chunk = neededChunks[i];
chunkTasks[i] = TaskManager.Run(async() =>
{
var result = await DownloadChunk(job, chunk, downloadPath, chunkCancellation);
if (!result)
{
Log.WriteWarn($"FileDownloader {job.DepotID}", $"Failed to download chunk for {file.FileName} ({chunk.Offset})");
chunkCancellation.Cancel();
}
else
{
downloadedSize += chunk.UncompressedLength;
// Do not write progress info to log file
Console.WriteLine($"{job.DepotName} [{downloadedSize / (float) file.TotalSize * 100.0f,6:#00.00}%] {file.FileName}");
}
});
}
await Task.WhenAll(chunkTasks).ConfigureAwait(false);
byte[] checksum;
await using (var fs = downloadPath.Open(FileMode.Open, FileAccess.ReadWrite))
{
checksum = await sha.ComputeHashAsync(fs, chunkCancellation.Token);
}
if (!file.FileHash.SequenceEqual(checksum))
{
if (!job.DownloadCorrupted)
{
job.DownloadCorrupted = true;
IRC.Instance.SendOps($"{Colors.OLIVE}[{job.DepotName}]{Colors.NORMAL} Failed to correctly download {Colors.BLUE}{file.FileName}");
}
Log.WriteWarn($"FileDownloader {job.DepotID}", $"Hash check failed for {file.FileName} ({job.Server})");
downloadPath.Delete();
existingFile.FileHash = null;
existingFile.Chunks.Clear();
return(EResult.DataCorruption);
}
Log.WriteInfo($"FileDownloader {job.DepotID}", $"Downloaded {file.FileName}");
finalPath.Delete();
downloadPath.MoveTo(finalPath.FullName);
if (chunks.Count > 0)
{
existingFile.Chunks = chunks.ToDictionary(chunk => chunk.Offset, chunk => chunk.ChunkID);
}
else
{
existingFile.Chunks.Clear();
}
return(EResult.OK);
}
/*
* Here be dragons.
*/
public static EResult DownloadFilesFromDepot(uint appID, DepotProcessor.ManifestJob job, DepotManifest depotManifest)
{
var files = depotManifest.Files.Where(x => IsFileNameMatching(job.DepotID, x.FileName)).ToList();
var filesUpdated = false;
var filesAnyFailed = false;
var hashesFile = Path.Combine(Application.Path, "files", ".support", "hashes", string.Format("{0}.json", job.DepotID));
var hashes = new Dictionary <string, byte[]>();
if (File.Exists(hashesFile))
{
hashes = JsonConvert.DeserializeObject <Dictionary <string, byte[]> >(File.ReadAllText(hashesFile));
}
Log.WriteDebug("FileDownloader", "Will download {0} files from depot {1}", files.Count, job.DepotID);
Parallel.ForEach(files, new ParallelOptions {
MaxDegreeOfParallelism = 2
}, (file, state2) =>
{
string directory = Path.Combine(Application.Path, "files", DownloadFolders[job.DepotID], Path.GetDirectoryName(file.FileName));
string finalPath = Path.Combine(directory, Path.GetFileName(file.FileName));
string downloadPath = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Path.GetRandomFileName(), ".steamdb_tmp"));
if (!Directory.Exists(directory))
{
Directory.CreateDirectory(directory);
}
else if (file.TotalSize == 0)
{
if (File.Exists(finalPath))
{
var f = new FileInfo(finalPath);
if (f.Length == 0)
{
#if DEBUG
Log.WriteDebug("FileDownloader", "{0} is already empty", file.FileName);
#endif
return;
}
}
else
{
File.Create(finalPath);
Log.WriteInfo("FileDownloader", "{0} created an empty file", file.FileName);
return;
}
}
else if (hashes.ContainsKey(file.FileName) && file.FileHash.SequenceEqual(hashes[file.FileName]))
{
#if DEBUG
Log.WriteDebug("FileDownloader", "{0} already matches the file we have", file.FileName);
#endif
return;
}
var chunks = file.Chunks.OrderBy(x => x.Offset).ToList();
Log.WriteInfo("FileDownloader", "Downloading {0} ({1} bytes, {2} chunks)", file.FileName, file.TotalSize, chunks.Count);
uint count = 0;
byte[] checksum;
string lastError = "or checksum failed";
string oldChunksFile;
using (var sha = new SHA1Managed())
{
oldChunksFile = Path.Combine(Application.Path, "files", ".support", "chunks",
string.Format("{0}-{1}.json", job.DepotID, BitConverter.ToString(sha.ComputeHash(Encoding.UTF8.GetBytes(file.FileName))))
);
}
using (var fs = File.Open(downloadPath, FileMode.OpenOrCreate, FileAccess.ReadWrite))
{
fs.SetLength((long)file.TotalSize);
var lockObject = new object();
var neededChunks = new List <DepotManifest.ChunkData>();
if (File.Exists(oldChunksFile) && File.Exists(finalPath))
{
var oldChunks = JsonConvert.DeserializeObject <List <DepotManifest.ChunkData> >(
File.ReadAllText(oldChunksFile),
new JsonSerializerSettings {
PreserveReferencesHandling = PreserveReferencesHandling.All
}
);
using (var fsOld = File.Open(finalPath, FileMode.Open, FileAccess.Read))
{
foreach (var chunk in chunks)
{
var oldChunk = oldChunks.FirstOrDefault(c => c.ChunkID.SequenceEqual(chunk.ChunkID));
if (oldChunk != null)
{
var oldData = new byte[oldChunk.UncompressedLength];
fsOld.Seek((long)oldChunk.Offset, SeekOrigin.Begin);
fsOld.Read(oldData, 0, oldData.Length);
var existingChecksum = Utils.AdlerHash(oldData);
if (existingChecksum.SequenceEqual(chunk.Checksum))
{
fs.Seek((long)chunk.Offset, SeekOrigin.Begin);
fs.Write(oldData, 0, oldData.Length);
#if DEBUG
Log.WriteDebug("FileDownloader", "{0} Found chunk ({1}), not downloading ({2}/{3})", file.FileName, chunk.Offset, ++count, chunks.Count);
#endif
}
else
{
neededChunks.Add(chunk);
#if DEBUG
Log.WriteDebug("FileDownloader", "{0} Found chunk ({1}), but checksum differs", file.FileName, chunk.Offset);
#endif
}
}
else
{
neededChunks.Add(chunk);
}
}
}
}
else
{
neededChunks = chunks;
}
Parallel.ForEach(neededChunks, new ParallelOptions {
MaxDegreeOfParallelism = 3
}, (chunk, state) =>
{
var downloaded = false;
for (var i = 0; i <= 5; i++)
{
try
{
var chunkData = CDNClient.DownloadDepotChunk(job.DepotID, chunk, job.Server, job.CDNToken, job.DepotKey);
lock (lockObject)
{
fs.Seek((long)chunk.Offset, SeekOrigin.Begin);
fs.Write(chunkData.Data, 0, chunkData.Data.Length);
Log.WriteDebug("FileDownloader", "Downloaded {0} ({1}/{2})", file.FileName, ++count, chunks.Count);
}
downloaded = true;
break;
}
catch (Exception e)
{
lastError = e.Message;
}
Task.Delay(Utils.ExponentionalBackoff(i)).Wait();
}
if (!downloaded)
{
state.Stop();
}
});
fs.Seek(0, SeekOrigin.Begin);
using (var sha = new SHA1Managed())
{
checksum = sha.ComputeHash(fs);
}
}
if (file.FileHash.SequenceEqual(checksum))
{
Log.WriteInfo("FileDownloader", "Downloaded {0} from {1}", file.FileName, Steam.GetAppName(appID));
hashes[file.FileName] = checksum;
if (File.Exists(finalPath))
{
File.Delete(finalPath);
}
File.Move(downloadPath, finalPath);
if (chunks.Count > 1)
{
File.WriteAllText(oldChunksFile,
JsonConvert.SerializeObject(
chunks,
Formatting.None,
new JsonSerializerSettings {
PreserveReferencesHandling = PreserveReferencesHandling.All
}
)
);
}
else if (File.Exists(oldChunksFile))
{
File.Delete(oldChunksFile);
}
filesUpdated = true;
}
else
{
filesAnyFailed = true;
IRC.Instance.SendOps("{0}[{1}]{2} Failed to download {3}: Only {4} out of {5} chunks downloaded ({6})",
Colors.OLIVE, Steam.GetAppName(appID), Colors.NORMAL, file.FileName, count, chunks.Count, lastError);
Log.WriteError("FileDownloader", "Failed to download {0}: Only {1} out of {2} chunks downloaded from {3} ({4})",
file.FileName, count, chunks.Count, job.Server, lastError);
File.Delete(downloadPath);
}
});
if (filesAnyFailed)
{
using (var db = Database.GetConnection())
{
// Mark this depot for redownload
db.Execute("UPDATE `Depots` SET `LastManifestID` = 0 WHERE `DepotID` = @DepotID", new { job.DepotID });
}
IRC.Instance.SendOps("{0}[{1}]{2} Failed to download some files, not running update script to prevent broken diffs.",
Colors.OLIVE, Steam.GetAppName(appID), Colors.NORMAL);
}
else if (filesUpdated)
{
File.WriteAllText(hashesFile, JsonConvert.SerializeObject(hashes));
job.Result = EResult.OK;
}
else
{
job.Result = EResult.Ignored;
}
return(job.Result);
}