private static async Task <EResult> DownloadFile(DepotProcessor.ManifestJob job, DepotManifest.FileData file, byte[] hash)
{
var directory = Path.Combine(Application.Path, "files", DownloadFolders[job.DepotID], Path.GetDirectoryName(file.FileName));
var finalPath = new FileInfo(Path.Combine(directory, Path.GetFileName(file.FileName)));
var downloadPath = new FileInfo(Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Path.GetRandomFileName(), ".steamdb_tmp")));
if (!Directory.Exists(directory))
{
Directory.CreateDirectory(directory);
}
else if (file.TotalSize == 0)
{
if (!finalPath.Exists)
{
using (var _ = finalPath.Create())
{
// FileInfo.Create returns a stream but we don't need it
}
Log.WriteInfo("FileDownloader", "{0} created an empty file", file.FileName);
return(EResult.SameAsPreviousValue);
}
else if (finalPath.Length == 0)
{
#if DEBUG
Log.WriteDebug("FileDownloader", "{0} is already empty", file.FileName);
#endif
return(EResult.SameAsPreviousValue);
}
}
else if (hash != null && file.FileHash.SequenceEqual(hash))
{
#if DEBUG
Log.WriteDebug("FileDownloader", "{0} already matches the file we have", file.FileName);
#endif
return(EResult.SameAsPreviousValue);
}
byte[] checksum;
using (var sha = SHA1.Create())
{
checksum = sha.ComputeHash(Encoding.UTF8.GetBytes(file.FileName));
}
var neededChunks = new List <DepotManifest.ChunkData>();
var chunks = file.Chunks.OrderBy(x => x.Offset).ToList();
var oldChunksFile = Path.Combine(Application.Path, "files", ".support", "chunks", string.Format("{0}-{1}.json", job.DepotID, BitConverter.ToString(checksum)));
using (var fs = downloadPath.Open(FileMode.OpenOrCreate, FileAccess.ReadWrite))
{
fs.SetLength((long)file.TotalSize);
if (finalPath.Exists && File.Exists(oldChunksFile))
{
var oldChunks = JsonConvert.DeserializeObject <List <DepotManifest.ChunkData> >(File.ReadAllText(oldChunksFile), JsonHandleAllReferences);
using (var fsOld = finalPath.Open(FileMode.Open, FileAccess.Read))
{
foreach (var chunk in chunks)
{
var oldChunk = oldChunks.Find(c => c.ChunkID.SequenceEqual(chunk.ChunkID));
if (oldChunk != null)
{
var oldData = new byte[oldChunk.UncompressedLength];
fsOld.Seek((long)oldChunk.Offset, SeekOrigin.Begin);
fsOld.Read(oldData, 0, oldData.Length);
var existingChecksum = Utils.AdlerHash(oldData);
if (existingChecksum.SequenceEqual(chunk.Checksum))
{
fs.Seek((long)chunk.Offset, SeekOrigin.Begin);
fs.Write(oldData, 0, oldData.Length);
#if DEBUG
Log.WriteDebug("FileDownloader", "{0} Found chunk ({1}), not downloading", file.FileName, chunk.Offset);
#endif
}
else
{
neededChunks.Add(chunk);
#if DEBUG
Log.WriteDebug("FileDownloader", "{0} Found chunk ({1}), but checksum differs", file.FileName, chunk.Offset);
#endif
}
}
else
{
neededChunks.Add(chunk);
}
}
}
}
else
{
neededChunks = chunks;
}
}
var downloadedSize = file.TotalSize - (ulong)neededChunks.Sum(x => x.UncompressedLength);
var chunkCancellation = new CancellationTokenSource();
var chunkTasks = new Task[neededChunks.Count];
Log.WriteInfo("FileDownloader", "Downloading {0} ({1} bytes, {2} out of {3} chunks)", file.FileName, downloadedSize, neededChunks.Count, chunks.Count);
for (var i = 0; i < chunkTasks.Length; i++)
{
var chunk = neededChunks[i];
chunkTasks[i] = TaskManager.Run(async() =>
{
try
{
chunkCancellation.Token.ThrowIfCancellationRequested();
await ChunkDownloadingSemaphore.WaitAsync(chunkCancellation.Token).ConfigureAwait(false);
var result = await DownloadChunk(job, chunk, downloadPath);
if (!result)
{
Log.WriteWarn("FileDownloader", "Failed to download chunk for {0}", file.FileName);
chunkCancellation.Cancel();
}
else
{
downloadedSize += chunk.UncompressedLength;
// Do not write progress info to log file
Console.WriteLine("{2} [{0,6:#00.00}%] {1}", downloadedSize / (float)file.TotalSize * 100.0f, file.FileName, job.DepotName);
}
}
finally
{
ChunkDownloadingSemaphore.Release();
}
}).Unwrap();
// Register error handler on inner task
TaskManager.RegisterErrorHandler(chunkTasks[i]);
}
await Task.WhenAll(chunkTasks).ConfigureAwait(false);
using (var fs = downloadPath.Open(FileMode.Open, FileAccess.ReadWrite))
{
fs.Seek(0, SeekOrigin.Begin);
using (var sha = SHA1.Create())
{
checksum = sha.ComputeHash(fs);
}
}
if (!file.FileHash.SequenceEqual(checksum))
{
IRC.Instance.SendOps("{0}[{1}]{2} Failed to correctly download {3}{4}",
Colors.OLIVE, job.DepotName, Colors.NORMAL, Colors.BLUE, file.FileName);
Log.WriteWarn("FileDownloader", "Failed to download file {0} ({1})", file.FileName, job.Server);
downloadPath.Delete();
return(EResult.DataCorruption);
}
Log.WriteInfo("FileDownloader", "Downloaded {0} from {1}", file.FileName, job.DepotName);
finalPath.Delete();
downloadPath.MoveTo(finalPath.FullName);
if (chunks.Count > 1)
{
File.WriteAllText(oldChunksFile, JsonConvert.SerializeObject(chunks, Formatting.None, JsonHandleAllReferences));
}
else if (File.Exists(oldChunksFile))
{
File.Delete(oldChunksFile);
}
return(EResult.OK);
}
/*
* Here be dragons.
*/
public static EResult DownloadFilesFromDepot(uint appID, DepotProcessor.ManifestJob job, DepotManifest depotManifest)
{
var files = depotManifest.Files.Where(x => IsFileNameMatching(job.DepotID, x.FileName)).ToList();
var filesUpdated = false;
var filesAnyFailed = false;
var hashesFile = Path.Combine(Application.Path, "files", ".support", "hashes", string.Format("{0}.json", job.DepotID));
var hashes = new Dictionary <string, byte[]>();
if (File.Exists(hashesFile))
{
hashes = JsonConvert.DeserializeObject <Dictionary <string, byte[]> >(File.ReadAllText(hashesFile));
}
Log.WriteDebug("FileDownloader", "Will download {0} files from depot {1}", files.Count, job.DepotID);
Parallel.ForEach(files, new ParallelOptions {
MaxDegreeOfParallelism = 2
}, (file, state2) =>
{
string directory = Path.Combine(Application.Path, "files", DownloadFolders[job.DepotID], Path.GetDirectoryName(file.FileName));
string finalPath = Path.Combine(directory, Path.GetFileName(file.FileName));
string downloadPath = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Path.GetRandomFileName(), ".steamdb_tmp"));
if (!Directory.Exists(directory))
{
Directory.CreateDirectory(directory);
}
else if (file.TotalSize == 0)
{
if (File.Exists(finalPath))
{
var f = new FileInfo(finalPath);
if (f.Length == 0)
{
#if DEBUG
Log.WriteDebug("FileDownloader", "{0} is already empty", file.FileName);
#endif
return;
}
}
else
{
File.Create(finalPath);
Log.WriteInfo("FileDownloader", "{0} created an empty file", file.FileName);
return;
}
}
else if (hashes.ContainsKey(file.FileName) && file.FileHash.SequenceEqual(hashes[file.FileName]))
{
#if DEBUG
Log.WriteDebug("FileDownloader", "{0} already matches the file we have", file.FileName);
#endif
return;
}
var chunks = file.Chunks.OrderBy(x => x.Offset).ToList();
Log.WriteInfo("FileDownloader", "Downloading {0} ({1} bytes, {2} chunks)", file.FileName, file.TotalSize, chunks.Count);
uint count = 0;
byte[] checksum;
string lastError = "or checksum failed";
string oldChunksFile;
using (var sha = new SHA1Managed())
{
oldChunksFile = Path.Combine(Application.Path, "files", ".support", "chunks",
string.Format("{0}-{1}.json", job.DepotID, BitConverter.ToString(sha.ComputeHash(Encoding.UTF8.GetBytes(file.FileName))))
);
}
using (var fs = File.Open(downloadPath, FileMode.OpenOrCreate, FileAccess.ReadWrite))
{
fs.SetLength((long)file.TotalSize);
var lockObject = new object();
var neededChunks = new List <DepotManifest.ChunkData>();
if (File.Exists(oldChunksFile) && File.Exists(finalPath))
{
var oldChunks = JsonConvert.DeserializeObject <List <DepotManifest.ChunkData> >(
File.ReadAllText(oldChunksFile),
new JsonSerializerSettings {
PreserveReferencesHandling = PreserveReferencesHandling.All
}
);
using (var fsOld = File.Open(finalPath, FileMode.Open, FileAccess.Read))
{
foreach (var chunk in chunks)
{
var oldChunk = oldChunks.FirstOrDefault(c => c.ChunkID.SequenceEqual(chunk.ChunkID));
if (oldChunk != null)
{
var oldData = new byte[oldChunk.UncompressedLength];
fsOld.Seek((long)oldChunk.Offset, SeekOrigin.Begin);
fsOld.Read(oldData, 0, oldData.Length);
var existingChecksum = Utils.AdlerHash(oldData);
if (existingChecksum.SequenceEqual(chunk.Checksum))
{
fs.Seek((long)chunk.Offset, SeekOrigin.Begin);
fs.Write(oldData, 0, oldData.Length);
#if DEBUG
Log.WriteDebug("FileDownloader", "{0} Found chunk ({1}), not downloading ({2}/{3})", file.FileName, chunk.Offset, ++count, chunks.Count);
#endif
}
else
{
neededChunks.Add(chunk);
#if DEBUG
Log.WriteDebug("FileDownloader", "{0} Found chunk ({1}), but checksum differs", file.FileName, chunk.Offset);
#endif
}
}
else
{
neededChunks.Add(chunk);
}
}
}
}
else
{
neededChunks = chunks;
}
Parallel.ForEach(neededChunks, new ParallelOptions {
MaxDegreeOfParallelism = 3
}, (chunk, state) =>
{
var downloaded = false;
for (var i = 0; i <= 5; i++)
{
try
{
var chunkData = CDNClient.DownloadDepotChunk(job.DepotID, chunk, job.Server, job.CDNToken, job.DepotKey);
lock (lockObject)
{
fs.Seek((long)chunk.Offset, SeekOrigin.Begin);
fs.Write(chunkData.Data, 0, chunkData.Data.Length);
Log.WriteDebug("FileDownloader", "Downloaded {0} ({1}/{2})", file.FileName, ++count, chunks.Count);
}
downloaded = true;
break;
}
catch (Exception e)
{
lastError = e.Message;
}
Task.Delay(Utils.ExponentionalBackoff(i)).Wait();
}
if (!downloaded)
{
state.Stop();
}
});
fs.Seek(0, SeekOrigin.Begin);
using (var sha = new SHA1Managed())
{
checksum = sha.ComputeHash(fs);
}
}
if (file.FileHash.SequenceEqual(checksum))
{
Log.WriteInfo("FileDownloader", "Downloaded {0} from {1}", file.FileName, Steam.GetAppName(appID));
hashes[file.FileName] = checksum;
if (File.Exists(finalPath))
{
File.Delete(finalPath);
}
File.Move(downloadPath, finalPath);
if (chunks.Count > 1)
{
File.WriteAllText(oldChunksFile,
JsonConvert.SerializeObject(
chunks,
Formatting.None,
new JsonSerializerSettings {
PreserveReferencesHandling = PreserveReferencesHandling.All
}
)
);
}
else if (File.Exists(oldChunksFile))
{
File.Delete(oldChunksFile);
}
filesUpdated = true;
}
else
{
filesAnyFailed = true;
IRC.Instance.SendOps("{0}[{1}]{2} Failed to download {3}: Only {4} out of {5} chunks downloaded ({6})",
Colors.OLIVE, Steam.GetAppName(appID), Colors.NORMAL, file.FileName, count, chunks.Count, lastError);
Log.WriteError("FileDownloader", "Failed to download {0}: Only {1} out of {2} chunks downloaded from {3} ({4})",
file.FileName, count, chunks.Count, job.Server, lastError);
File.Delete(downloadPath);
}
});
if (filesAnyFailed)
{
using (var db = Database.GetConnection())
{
// Mark this depot for redownload
db.Execute("UPDATE `Depots` SET `LastManifestID` = 0 WHERE `DepotID` = @DepotID", new { job.DepotID });
}
IRC.Instance.SendOps("{0}[{1}]{2} Failed to download some files, not running update script to prevent broken diffs.",
Colors.OLIVE, Steam.GetAppName(appID), Colors.NORMAL);
}
else if (filesUpdated)
{
File.WriteAllText(hashesFile, JsonConvert.SerializeObject(hashes));
job.Result = EResult.OK;
}
else
{
job.Result = EResult.Ignored;
}
return(job.Result);
}
