Exemplo n.º 1
0
        private static void DownloadSteam3(uint appId, List <DepotDownloadInfo> depots)
        {
            ulong TotalBytesCompressed   = 0;
            ulong TotalBytesUncompressed = 0;

            foreach (var depot in depots)
            {
                ulong DepotBytesCompressed   = 0;
                ulong DepotBytesUncompressed = 0;

                Console.WriteLine("Downloading depot {0} - {1}", depot.id, depot.contentName);

                CancellationTokenSource cts = new CancellationTokenSource();

                ProtoManifest oldProtoManifest = null;
                ProtoManifest newProtoManifest = null;
                string        configDir        = Path.Combine(depot.installDir, CONFIG_DIR);

                ulong lastManifestId = INVALID_MANIFEST_ID;
                ConfigStore.TheConfig.LastManifests.TryGetValue(depot.id, out lastManifestId);

                // In case we have an early exit, this will force equiv of verifyall next run.
                ConfigStore.TheConfig.LastManifests[depot.id] = INVALID_MANIFEST_ID;
                ConfigStore.Save();

                if (lastManifestId != INVALID_MANIFEST_ID)
                {
                    var oldManifestFileName = Path.Combine(configDir, string.Format("{0}.bin", lastManifestId));
                    if (File.Exists(oldManifestFileName))
                    {
                        oldProtoManifest = ProtoManifest.LoadFromFile(oldManifestFileName);
                    }
                }

                if (lastManifestId == depot.manifestId && oldProtoManifest != null)
                {
                    newProtoManifest = oldProtoManifest;
                    Console.WriteLine("Already have manifest {0} for depot {1}.", depot.manifestId, depot.id);
                }
                else
                {
                    var newManifestFileName = Path.Combine(configDir, string.Format("{0}.bin", depot.manifestId));
                    if (newManifestFileName != null)
                    {
                        newProtoManifest = ProtoManifest.LoadFromFile(newManifestFileName);
                    }

                    if (newProtoManifest != null)
                    {
                        Console.WriteLine("Already have manifest {0} for depot {1}.", depot.manifestId, depot.id);
                    }
                    else
                    {
                        Console.Write("Downloading depot manifest...");

                        DepotManifest depotManifest = null;

                        while (depotManifest == null)
                        {
                            CDNClient client = null;
                            try
                            {
                                client = cdnPool.GetConnectionForDepot(appId, depot.id, depot.depotKey, CancellationToken.None);

                                depotManifest = client.DownloadManifest(depot.id, depot.manifestId);

                                cdnPool.ReturnConnection(client);
                            }
                            catch (WebException e)
                            {
                                cdnPool.ReturnBrokenConnection(client);

                                if (e.Status == WebExceptionStatus.ProtocolError)
                                {
                                    var response = e.Response as HttpWebResponse;
                                    if (response.StatusCode == HttpStatusCode.Unauthorized || response.StatusCode == HttpStatusCode.Forbidden)
                                    {
                                        Console.WriteLine("Encountered 401 for depot manifest {0} {1}. Aborting.", depot.id, depot.manifestId);
                                        break;
                                    }
                                    else
                                    {
                                        Console.WriteLine("Encountered error downloading depot manifest {0} {1}: {2}", depot.id, depot.manifestId, response.StatusCode);
                                    }
                                }
                                else
                                {
                                    Console.WriteLine("Encountered error downloading manifest for depot {0} {1}: {2}", depot.id, depot.manifestId, e.Status);
                                }
                            }
                            catch (Exception e)
                            {
                                cdnPool.ReturnBrokenConnection(client);
                                Console.WriteLine("Encountered error downloading manifest for depot {0} {1}: {2}", depot.id, depot.manifestId, e.Message);
                            }
                        }

                        if (depotManifest == null)
                        {
                            Console.WriteLine("\nUnable to download manifest {0} for depot {1}", depot.manifestId, depot.id);
                            return;
                        }

                        newProtoManifest = new ProtoManifest(depotManifest, depot.manifestId);
                        newProtoManifest.SaveToFile(newManifestFileName);

                        Console.WriteLine("Successfully downloaded manifest {0} for depot {1}", depot.manifestId, depot.id);

                        Console.WriteLine(" Done!");
                    }
                }

                newProtoManifest.Files.Sort((x, y) => { return(x.FileName.CompareTo(y.FileName)); });

                if (Config.DownloadManifestOnly)
                {
                    StringBuilder manifestBuilder = new StringBuilder();
                    string        txtManifest     = Path.Combine(depot.installDir, string.Format("manifest_{0}.txt", depot.id));

                    foreach (var file in newProtoManifest.Files)
                    {
                        if (file.Flags.HasFlag(EDepotFileFlag.Directory))
                        {
                            continue;
                        }

                        manifestBuilder.Append(string.Format("{0}\n", file.FileName));
                    }

                    File.WriteAllText(txtManifest, manifestBuilder.ToString());
                    continue;
                }

                ulong  complete_download_size = 0;
                ulong  size_downloaded        = 0;
                string stagingDir             = Path.Combine(depot.installDir, STAGING_DIR);

                var filesAfterExclusions = newProtoManifest.Files.AsParallel().Where(f => TestIsFileIncluded(f.FileName)).ToList();

                // Pre-process
                filesAfterExclusions.ForEach(file =>
                {
                    var fileFinalPath   = Path.Combine(depot.installDir, file.FileName);
                    var fileStagingPath = Path.Combine(stagingDir, file.FileName);

                    if (file.Flags.HasFlag(EDepotFileFlag.Directory))
                    {
                        Directory.CreateDirectory(fileFinalPath);
                        Directory.CreateDirectory(fileStagingPath);
                    }
                    else
                    {
                        // Some manifests don't explicitly include all necessary directories
                        Directory.CreateDirectory(Path.GetDirectoryName(fileFinalPath));
                        Directory.CreateDirectory(Path.GetDirectoryName(fileStagingPath));

                        complete_download_size += file.TotalSize;
                    }
                });

                filesAfterExclusions.Where(f => !f.Flags.HasFlag(EDepotFileFlag.Directory))
                .AsParallel().WithCancellation(cts.Token).WithDegreeOfParallelism(Config.MaxDownloads)
                .ForAll(file =>
                {
                    string fileFinalPath   = Path.Combine(depot.installDir, file.FileName);
                    string fileStagingPath = Path.Combine(stagingDir, file.FileName);

                    // This may still exist if the previous run exited before cleanup
                    if (File.Exists(fileStagingPath))
                    {
                        File.Delete(fileStagingPath);
                    }

                    FileStream fs = null;
                    List <ProtoManifest.ChunkData> neededChunks;
                    FileInfo fi = new FileInfo(fileFinalPath);
                    if (!fi.Exists)
                    {
                        // create new file. need all chunks
                        fs = File.Create(fileFinalPath);
                        fs.SetLength(( long )file.TotalSize);
                        neededChunks = new List <ProtoManifest.ChunkData>(file.Chunks);
                    }
                    else
                    {
                        // open existing
                        ProtoManifest.FileData oldManifestFile = null;
                        if (oldProtoManifest != null)
                        {
                            oldManifestFile = oldProtoManifest.Files.SingleOrDefault(f => f.FileName == file.FileName);
                        }

                        if (oldManifestFile != null)
                        {
                            neededChunks = new List <ProtoManifest.ChunkData>();

                            if (Config.VerifyAll || !oldManifestFile.FileHash.SequenceEqual(file.FileHash))
                            {
                                // we have a version of this file, but it doesn't fully match what we want

                                var matchingChunks = new List <ChunkMatch>();

                                foreach (var chunk in file.Chunks)
                                {
                                    var oldChunk = oldManifestFile.Chunks.FirstOrDefault(c => c.ChunkID.SequenceEqual(chunk.ChunkID));
                                    if (oldChunk != null)
                                    {
                                        matchingChunks.Add(new ChunkMatch(oldChunk, chunk));
                                    }
                                    else
                                    {
                                        neededChunks.Add(chunk);
                                    }
                                }

                                File.Move(fileFinalPath, fileStagingPath);

                                fs = File.Open(fileFinalPath, FileMode.Create);
                                fs.SetLength(( long )file.TotalSize);

                                using (var fsOld = File.Open(fileStagingPath, FileMode.Open))
                                {
                                    foreach (var match in matchingChunks)
                                    {
                                        fsOld.Seek(( long )match.OldChunk.Offset, SeekOrigin.Begin);

                                        byte[] tmp = new byte[match.OldChunk.UncompressedLength];
                                        fsOld.Read(tmp, 0, tmp.Length);

                                        byte[] adler = Util.AdlerHash(tmp);
                                        if (!adler.SequenceEqual(match.OldChunk.Checksum))
                                        {
                                            neededChunks.Add(match.NewChunk);
                                        }
                                        else
                                        {
                                            fs.Seek(( long )match.NewChunk.Offset, SeekOrigin.Begin);
                                            fs.Write(tmp, 0, tmp.Length);
                                        }
                                    }
                                }

                                File.Delete(fileStagingPath);
                            }
                        }
                        else
                        {
                            // No old manifest or file not in old manifest. We must validate.

                            fs = File.Open(fileFinalPath, FileMode.Open);
                            if (( ulong )fi.Length != file.TotalSize)
                            {
                                fs.SetLength(( long )file.TotalSize);
                            }

                            neededChunks = Util.ValidateSteam3FileChecksums(fs, file.Chunks.OrderBy(x => x.Offset).ToArray());
                        }

                        if (neededChunks.Count() == 0)
                        {
                            size_downloaded += file.TotalSize;
                            Console.WriteLine("{0,6:#00.00}% {1}", (( float )size_downloaded / ( float )complete_download_size) * 100.0f, fileFinalPath);
                            if (fs != null)
                            {
                                fs.Close();
                            }
                            return;
                        }
                        else
                        {
                            size_downloaded += (file.TotalSize - ( ulong )neededChunks.Select(x => ( long )x.UncompressedLength).Sum());
                        }
                    }

                    foreach (var chunk in neededChunks)
                    {
                        if (cts.IsCancellationRequested)
                        {
                            break;
                        }

                        string chunkID = Util.EncodeHexString(chunk.ChunkID);
                        CDNClient.DepotChunk chunkData = null;

                        while (!cts.IsCancellationRequested)
                        {
                            CDNClient client;
                            try
                            {
                                client = cdnPool.GetConnectionForDepot(appId, depot.id, depot.depotKey, cts.Token);
                            }
                            catch (OperationCanceledException)
                            {
                                break;
                            }

                            DepotManifest.ChunkData data = new DepotManifest.ChunkData();
                            data.ChunkID            = chunk.ChunkID;
                            data.Checksum           = chunk.Checksum;
                            data.Offset             = chunk.Offset;
                            data.CompressedLength   = chunk.CompressedLength;
                            data.UncompressedLength = chunk.UncompressedLength;

                            try
                            {
                                chunkData = client.DownloadDepotChunk(depot.id, data);
                                cdnPool.ReturnConnection(client);
                                break;
                            }
                            catch (WebException e)
                            {
                                cdnPool.ReturnBrokenConnection(client);

                                if (e.Status == WebExceptionStatus.ProtocolError)
                                {
                                    var response = e.Response as HttpWebResponse;
                                    if (response.StatusCode == HttpStatusCode.Unauthorized || response.StatusCode == HttpStatusCode.Forbidden)
                                    {
                                        Console.WriteLine("Encountered 401 for chunk {0}. Aborting.", chunkID);
                                        cts.Cancel();
                                        break;
                                    }
                                    else
                                    {
                                        Console.WriteLine("Encountered error downloading chunk {0}: {1}", chunkID, response.StatusCode);
                                    }
                                }
                                else
                                {
                                    Console.WriteLine("Encountered error downloading chunk {0}: {1}", chunkID, e.Status);
                                }
                            }
                            catch (Exception e)
                            {
                                cdnPool.ReturnBrokenConnection(client);
                                Console.WriteLine("Encountered unexpected error downloading chunk {0}: {1}", chunkID, e.Message);
                            }
                        }

                        if (chunkData == null)
                        {
                            Console.WriteLine("Failed to find any server with chunk {0} for depot {1}. Aborting.", chunkID, depot.id);
                            return;
                        }

                        TotalBytesCompressed   += chunk.CompressedLength;
                        DepotBytesCompressed   += chunk.CompressedLength;
                        TotalBytesUncompressed += chunk.UncompressedLength;
                        DepotBytesUncompressed += chunk.UncompressedLength;

                        fs.Seek(( long )chunk.Offset, SeekOrigin.Begin);
                        fs.Write(chunkData.Data, 0, chunkData.Data.Length);

                        size_downloaded += chunk.UncompressedLength;
                    }

                    fs.Close();

                    Console.WriteLine("{0,6:#00.00}% {1}", (( float )size_downloaded / ( float )complete_download_size) * 100.0f, fileFinalPath);
                });

                ConfigStore.TheConfig.LastManifests[depot.id] = depot.manifestId;
                ConfigStore.Save();

                Console.WriteLine("Depot {0} - Downloaded {1} bytes ({2} bytes uncompressed)", depot.id, DepotBytesCompressed, DepotBytesUncompressed);
            }

            Console.WriteLine("Total downloaded: {0} bytes ({1} bytes uncompressed) from {2} depots", TotalBytesCompressed, TotalBytesUncompressed, depots.Count);
        }
        private static async Task DownloadSteam3Async(uint appId, List <DepotDownloadInfo> depots, Action downloadCompleteAction = null)
        {
            ulong TotalBytesCompressed   = 0;
            ulong TotalBytesUncompressed = 0;

            foreach (var depot in depots)
            {
                ulong DepotBytesCompressed   = 0;
                ulong DepotBytesUncompressed = 0;

                DebugLog.WriteLine("ContentDownloader", "Downloading depot " + depot.id + " - " + depot.contentName);

                CancellationTokenSource cts = new CancellationTokenSource();
                cdnPool.ExhaustedToken = cts;

                ProtoManifest oldProtoManifest = null;
                ProtoManifest downloadManifest = null;
                string        configDir        = Path.Combine(depot.installDir, CONFIG_DIR);

                ulong lastManifestId = INVALID_MANIFEST_ID;
                ConfigStore.TheConfig.LastManifests.TryGetValue(depot.id, out lastManifestId);

                // In case we have an early exit, this will force equiv of verifyall next run.
                ConfigStore.TheConfig.LastManifests[depot.id] = INVALID_MANIFEST_ID;
                ConfigStore.Save();

                if (lastManifestId != INVALID_MANIFEST_ID)
                {
                    var oldManifestFileName = Path.Combine(configDir, string.Format("{0}.bin", lastManifestId));
                    DebugLog.WriteLine(DEBUG_NAME_FILES, "Checking if " + oldManifestFileName + " exists");
                    if (File.Exists(oldManifestFileName))
                    {
                        DebugLog.WriteLine(DEBUG_NAME_FILES, oldManifestFileName + " exists, reading!");
                        oldProtoManifest = ProtoManifest.LoadFromFile(oldManifestFileName);
                    }
                }

                if (lastManifestId == depot.manifestId && oldProtoManifest != null)
                {
                    downloadManifest = oldProtoManifest;
                    DebugLog.WriteLine("ContentDownloader", "Already have manifest " + depot.manifestId + " for depot " + depot.id + ".");
                }
                else
                {
                    var newManifestFileName = Path.Combine(configDir, string.Format("{0}.bin", depot.manifestId));
                    if (newManifestFileName != null)
                    {
                        downloadManifest = ProtoManifest.LoadFromFile(newManifestFileName);
                    }

                    if (downloadManifest != null)
                    {
                        DebugLog.WriteLine("ContentDownloader", "Already have manifest " + depot.manifestId + " for depot " + depot.id + ".");
                    }
                    else
                    {
                        DebugLog.WriteLine("ContentDownloader", "Downloading depot manifest...");

                        DepotManifest depotManifest = null;

                        while (depotManifest == null)
                        {
                            CDNClient client = null;
                            try
                            {
                                client = await cdnPool.GetConnectionForDepotAsync(appId, depot.id, depot.depotKey, CancellationToken.None).ConfigureAwait(false);

                                //client = await cdnPool.GetConnectionForDepotAsync(appId, depot.id, depot.depotKey, CancellationToken.None);

                                depotManifest = await client.DownloadManifestAsync(depot.id, depot.manifestId).ConfigureAwait(false);

                                //depotManifest = await client.DownloadManifestAsync(depot.id, depot.manifestId);

                                cdnPool.ReturnConnection(client);
                            }
                            catch (WebException e)
                            {
                                cdnPool.ReturnBrokenConnection(client);

                                if (e.Status == WebExceptionStatus.ProtocolError)
                                {
                                    var response = e.Response as HttpWebResponse;
                                    if (response.StatusCode == HttpStatusCode.Unauthorized || response.StatusCode == HttpStatusCode.Forbidden)
                                    {
                                        DebugLog.WriteLine("ContentDownloader", "Encountered 401 for depot manifest " + depot.id + " " + depot.manifestId + ". Aborting.");
                                        break;
                                    }
                                    else
                                    {
                                        DebugLog.WriteLine("ContentDownloader", "Encountered error downloading depot manifest " + depot.id + " " + depot.manifestId + ": " + response.StatusCode);
                                    }
                                }
                                else
                                {
                                    DebugLog.WriteLine("ContentDownloader", "Encountered error downloading manifest for depot " + depot.id + " " + depot.manifestId + ": " + e.Status);
                                }
                            }
                            catch (Exception e)
                            {
                                cdnPool.ReturnBrokenConnection(client);
                                DebugLog.WriteLine("ContentDownloader", "Encountered error downloading manifest for depot " + depot.id + " " + depot.manifestId + ": " + e.Message);
                            }
                        }

                        if (depotManifest == null)
                        {
                            DebugLog.WriteLine("ContentDownloader", "\nUnable to download manifest " + depot.manifestId + " for depot " + depot.id);
                            return;
                        }

                        downloadManifest = new ProtoManifest(depotManifest, depot.manifestId);
                        downloadManifest.SaveToFile(newManifestFileName);

                        DebugLog.WriteLine("ContentDownloader", "Done!");
                    }
                }

                downloadManifest.Files.Sort((x, y) => { return(x.FileName.CompareTo(y.FileName)); });
                if (downloadManifest != null)
                {
                    onManifestReceived?.Invoke(appId, depot.id, depot.contentName, downloadManifest);
                }

                if (Config.DownloadManifestOnly)
                {
                    continue;
                }

                complete_download_size = 0;
                size_downloaded        = 0;
                string stagingDir = Path.Combine(depot.installDir, STAGING_DIR);

                var filesAfterExclusions = downloadManifest.Files.Where(f => TestIsFileIncluded(f.FileName)).ToList();

                // Pre-process
                filesAfterExclusions.ForEach(file =>
                {
                    var fileFinalPath   = Path.Combine(depot.installDir, file.FileName);
                    var fileStagingPath = Path.Combine(stagingDir, file.FileName);

                    if (file.Flags.HasFlag(EDepotFileFlag.Directory))
                    {
                        Directory.CreateDirectory(fileFinalPath);
                        Directory.CreateDirectory(fileStagingPath);
                    }
                    else
                    {
                        // Some manifests don't explicitly include all necessary directories
                        Directory.CreateDirectory(Path.GetDirectoryName(fileFinalPath));
                        Directory.CreateDirectory(Path.GetDirectoryName(fileStagingPath));

                        complete_download_size += file.TotalSize;
                    }
                });

                var semaphore = new SemaphoreSlim(Config.MaxDownloads);
                var files     = filesAfterExclusions.Where(f => !f.Flags.HasFlag(EDepotFileFlag.Directory)).ToArray();
                var tasks     = new Task[files.Length];
                for (var i = 0; i < files.Length; i++)
                {
                    var file = files[i];
                    var task = Task.Run(async() =>
                    {
                        cts.Token.ThrowIfCancellationRequested();

                        try
                        {
                            await semaphore.WaitAsync().ConfigureAwait(false);
                            //await semaphore.WaitAsync();
                            cts.Token.ThrowIfCancellationRequested();

                            string fileFinalPath   = Path.Combine(depot.installDir, file.FileName);
                            string fileStagingPath = Path.Combine(stagingDir, file.FileName);

                            // This may still exist if the previous run exited before cleanup
                            DebugLog.WriteLine(DEBUG_NAME_FILES, "Checking if " + fileStagingPath + " exists");
                            if (File.Exists(fileStagingPath))
                            {
                                DebugLog.WriteLine(DEBUG_NAME_FILES, fileStagingPath + " exists, deleting!");
                                File.Delete(fileStagingPath);
                            }

                            List <ProtoManifest.ChunkData> neededChunks;
                            FileInfo fi = new FileInfo(fileFinalPath);
                            DebugLog.WriteLine(DEBUG_NAME_FILES, "Checking if " + fileFinalPath + " exists");
                            if (!fi.Exists)
                            {
                                // create new file. need all chunks
                                DebugLog.WriteLine(DEBUG_NAME_FILES, fileFinalPath + " does not exist, creating!");
                                using (FileStream fs = File.Create(fileFinalPath))
                                {
                                    fs.SetLength((long)file.TotalSize);
                                    neededChunks = new List <ProtoManifest.ChunkData>(file.Chunks);
                                }
                            }
                            else
                            {
                                // open existing
                                ProtoManifest.FileData oldManifestFile = null;
                                if (oldProtoManifest != null)
                                {
                                    oldManifestFile = oldProtoManifest.Files.SingleOrDefault(f => f.FileName == file.FileName);
                                }

                                if (oldManifestFile != null)
                                {
                                    neededChunks = new List <ProtoManifest.ChunkData>();

                                    if (Config.VerifyAll || !oldManifestFile.FileHash.SequenceEqual(file.FileHash))
                                    {
                                        // we have a version of this file, but it doesn't fully match what we want

                                        var matchingChunks = new List <ChunkMatch>();

                                        foreach (var chunk in file.Chunks)
                                        {
                                            var oldChunk = oldManifestFile.Chunks.FirstOrDefault(c => c.ChunkID.SequenceEqual(chunk.ChunkID));
                                            if (oldChunk != null)
                                            {
                                                matchingChunks.Add(new ChunkMatch(oldChunk, chunk));
                                            }
                                            else
                                            {
                                                neededChunks.Add(chunk);
                                            }
                                        }

                                        DebugLog.WriteLine(DEBUG_NAME_FILES, "Moving file " + fileFinalPath + " to " + fileStagingPath);
                                        File.Move(fileFinalPath, fileStagingPath);

                                        DebugLog.WriteLine(DEBUG_NAME_FILES, "Creating file " + fileFinalPath);
                                        using (FileStream fs = File.Open(fileFinalPath, FileMode.Create))
                                        {
                                            fs.SetLength((long)file.TotalSize);

                                            DebugLog.WriteLine(DEBUG_NAME_FILES, "Opening file " + fileStagingPath);
                                            using (var fsOld = File.Open(fileStagingPath, FileMode.Open))
                                            {
                                                foreach (var match in matchingChunks)
                                                {
                                                    fsOld.Seek((long)match.OldChunk.Offset, SeekOrigin.Begin);

                                                    byte[] tmp = new byte[match.OldChunk.UncompressedLength];
                                                    fsOld.Read(tmp, 0, tmp.Length);

                                                    byte[] adler = Util.AdlerHash(tmp);
                                                    if (!adler.SequenceEqual(match.OldChunk.Checksum))
                                                    {
                                                        neededChunks.Add(match.NewChunk);
                                                    }
                                                    else
                                                    {
                                                        fs.Seek((long)match.NewChunk.Offset, SeekOrigin.Begin);
                                                        fs.Write(tmp, 0, tmp.Length);
                                                    }
                                                }
                                            }
                                        }
                                        DebugLog.WriteLine(DEBUG_NAME_FILES, "Deleting file " + fileStagingPath);
                                        File.Delete(fileStagingPath);
                                    }
                                }
                                else
                                {
                                    // No old manifest or file not in old manifest. We must validate.

                                    DebugLog.WriteLine(DEBUG_NAME_FILES, "Opening file " + fileFinalPath);
                                    using (FileStream fs = File.Open(fileFinalPath, FileMode.Open))
                                    {
                                        if ((ulong)fi.Length != file.TotalSize)
                                        {
                                            fs.SetLength((long)file.TotalSize);
                                        }

                                        neededChunks = Util.ValidateSteam3FileChecksums(fs, file.Chunks.OrderBy(x => x.Offset).ToArray());
                                    }
                                }

                                if (neededChunks.Count() == 0)
                                {
                                    size_downloaded += file.TotalSize;
                                    DebugLog.WriteLine("ContentDownloader", DownloadPercent * 100.0f + "% " + fileFinalPath);
                                    return;
                                }
                                else
                                {
                                    size_downloaded += (file.TotalSize - ( ulong )neededChunks.Select(x => ( long )x.UncompressedLength).Sum());
                                }
                            }

                            DebugLog.WriteLine(DEBUG_NAME_FILES, "Opening file " + fileFinalPath);
                            using (FileStream fs = File.Open(fileFinalPath, FileMode.Open))
                            {
                                foreach (var chunk in neededChunks)
                                {
                                    if (cts.IsCancellationRequested)
                                    {
                                        break;
                                    }

                                    string chunkID = Util.EncodeHexString(chunk.ChunkID);
                                    CDNClient.DepotChunk chunkData = null;

                                    while (!cts.IsCancellationRequested)
                                    {
                                        CDNClient client;
                                        try
                                        {
                                            client = await cdnPool.GetConnectionForDepotAsync(appId, depot.id, depot.depotKey, cts.Token).ConfigureAwait(false);
                                            //client = await cdnPool.GetConnectionForDepotAsync(appId, depot.id, depot.depotKey, cts.Token);
                                        }
                                        catch (OperationCanceledException)
                                        {
                                            break;
                                        }

                                        DepotManifest.ChunkData data = new DepotManifest.ChunkData();
                                        data.ChunkID            = chunk.ChunkID;
                                        data.Checksum           = chunk.Checksum;
                                        data.Offset             = chunk.Offset;
                                        data.CompressedLength   = chunk.CompressedLength;
                                        data.UncompressedLength = chunk.UncompressedLength;

                                        try
                                        {
                                            //chunkData = await client.DownloadDepotChunkAsStreamAsync(depot.id, data).ConfigureAwait(false);
                                            chunkData = client.DownloadDepotChunkAsStreamAsync(depot.id, data).Result;
                                            //chunkData = await client.DownloadDepotChunkAsync(depot.id, data);
                                            cdnPool.ReturnConnection(client);
                                            break;
                                        }
                                        catch (WebException e)
                                        {
                                            cdnPool.ReturnBrokenConnection(client);

                                            if (e.Status == WebExceptionStatus.ProtocolError)
                                            {
                                                var response = e.Response as HttpWebResponse;
                                                if (response.StatusCode == HttpStatusCode.Unauthorized || response.StatusCode == HttpStatusCode.Forbidden)
                                                {
                                                    DebugLog.WriteLine("ContentDownloader", "Encountered 401 for chunk " + chunkID + ". Aborting.");
                                                    cts.Cancel();
                                                    break;
                                                }
                                                else
                                                {
                                                    DebugLog.WriteLine("ContentDownloader", "Encountered error downloading chunk " + chunkID + ": " + response.StatusCode);
                                                }
                                            }
                                            else
                                            {
                                                DebugLog.WriteLine("ContentDownloader", "Encountered error downloading chunk " + chunkID + ": " + e.Status);
                                            }
                                        }
                                        catch (Exception e)
                                        {
                                            cdnPool.ReturnBrokenConnection(client);
                                            DebugLog.WriteLine("ContentDownloader", "Encountered unexpected error downloading chunk " + chunkID + ": " + e.Message);
                                        }
                                    }

                                    if (chunkData == null)
                                    {
                                        DebugLog.WriteLine("ContentDownloader", "Failed to find any server with chunk " + chunkID + " for depot " + depot.id + ". Aborting.");
                                        cts.Cancel();
                                        return;
                                    }

                                    TotalBytesCompressed   += chunk.CompressedLength;
                                    DepotBytesCompressed   += chunk.CompressedLength;
                                    TotalBytesUncompressed += chunk.UncompressedLength;
                                    DepotBytesUncompressed += chunk.UncompressedLength;

                                    using (chunkData.DataStream)
                                    {
                                        fs.Seek((long)chunk.Offset, SeekOrigin.Begin);
                                        chunkData.DataStream.CopyTo(fs);
                                        //fs.Write(chunkData.Data, 0, chunkData.Data.Length);
                                    }

                                    size_downloaded += chunk.UncompressedLength;
                                }
                            }
                            DebugLog.WriteLine("ContentDownloader", DownloadPercent * 100.0f + "% " + fileFinalPath);
                        }
                        finally
                        {
                            semaphore.Release();
                        }
                    });

                    tasks[i] = task;
                }

                Task.WaitAll(tasks);

                ConfigStore.TheConfig.LastManifests[depot.id] = depot.manifestId;
                ConfigStore.Save();

                DebugLog.WriteLine("ContentDownloader", "Depot " + depot.id + " - Downloaded " + DepotBytesCompressed + " bytes (" + DepotBytesUncompressed + " bytes uncompressed)");
            }

            IsDownloading = false;
            downloadCompleteAction?.Invoke();
            onDownloadCompleted?.Invoke();
            DebugLog.WriteLine("ContentDownloader", "Total downloaded: " + TotalBytesCompressed + " bytes (" + TotalBytesUncompressed + " bytes uncompressed) from " + depots.Count + " depots");
        }
        private static async Task DownloadSteam3Async(uint appId, List <DepotDownloadInfo> depots)
        {
            ulong TotalBytesCompressed   = 0;
            ulong TotalBytesUncompressed = 0;

            foreach (var depot in depots)
            {
                ulong DepotBytesCompressed   = 0;
                ulong DepotBytesUncompressed = 0;

                Console.WriteLine("Downloading depot {0} - {1}", depot.id, depot.contentName);

                CancellationTokenSource cts = new CancellationTokenSource();
                cdnPool.ExhaustedToken = cts;

                ProtoManifest oldProtoManifest = null;
                ProtoManifest newProtoManifest = null;
                string        configDir        = Path.Combine(depot.installDir, CONFIG_DIR);

                ulong lastManifestId = INVALID_MANIFEST_ID;
                DepotConfigStore.Instance.InstalledManifestIDs.TryGetValue(depot.id, out lastManifestId);

                // In case we have an early exit, this will force equiv of verifyall next run.
                DepotConfigStore.Instance.InstalledManifestIDs[depot.id] = INVALID_MANIFEST_ID;
                DepotConfigStore.Save();

                if (lastManifestId != INVALID_MANIFEST_ID)
                {
                    var oldManifestFileName = Path.Combine(configDir, string.Format("{0}.bin", lastManifestId));

                    if (File.Exists(oldManifestFileName))
                    {
                        byte[] expectedChecksum, currentChecksum;

                        try
                        {
                            expectedChecksum = File.ReadAllBytes(oldManifestFileName + ".sha");
                        }
                        catch (IOException)
                        {
                            expectedChecksum = null;
                        }

                        oldProtoManifest = ProtoManifest.LoadFromFile(oldManifestFileName, out currentChecksum);

                        if (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum))
                        {
                            // We only have to show this warning if the old manifest ID was different
                            if (lastManifestId != depot.manifestId)
                            {
                                Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", lastManifestId);
                            }
                            oldProtoManifest = null;
                        }
                    }
                }

                if (lastManifestId == depot.manifestId && oldProtoManifest != null)
                {
                    newProtoManifest = oldProtoManifest;
                    Console.WriteLine("Already have manifest {0} for depot {1}.", depot.manifestId, depot.id);
                }
                else
                {
                    var newManifestFileName = Path.Combine(configDir, string.Format("{0}.bin", depot.manifestId));
                    if (newManifestFileName != null)
                    {
                        byte[] expectedChecksum, currentChecksum;

                        try
                        {
                            expectedChecksum = File.ReadAllBytes(newManifestFileName + ".sha");
                        }
                        catch (IOException)
                        {
                            expectedChecksum = null;
                        }

                        newProtoManifest = ProtoManifest.LoadFromFile(newManifestFileName, out currentChecksum);

                        if (newProtoManifest != null && (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum)))
                        {
                            Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", depot.manifestId);
                            newProtoManifest = null;
                        }
                    }

                    if (newProtoManifest != null)
                    {
                        Console.WriteLine("Already have manifest {0} for depot {1}.", depot.manifestId, depot.id);
                    }
                    else
                    {
                        Console.Write("Downloading depot manifest...");

                        DepotManifest depotManifest = null;

                        while (depotManifest == null)
                        {
                            Tuple <CDNClient.Server, string> connection = null;
                            try
                            {
                                connection = await cdnPool.GetConnectionForDepot(appId, depot.id, CancellationToken.None);

                                depotManifest = await cdnPool.CDNClient.DownloadManifestAsync(depot.id, depot.manifestId,
                                                                                              connection.Item1, connection.Item2, depot.depotKey).ConfigureAwait(false);

                                cdnPool.ReturnConnection(connection);
                            }
                            catch (SteamKitWebRequestException e)
                            {
                                cdnPool.ReturnBrokenConnection(connection);

                                if (e.StatusCode == HttpStatusCode.Unauthorized || e.StatusCode == HttpStatusCode.Forbidden)
                                {
                                    Console.WriteLine("Encountered 401 for depot manifest {0} {1}. Aborting.", depot.id, depot.manifestId);
                                    break;
                                }
                                else
                                {
                                    Console.WriteLine("Encountered error downloading depot manifest {0} {1}: {2}", depot.id, depot.manifestId, e.StatusCode);
                                }
                            }
                            catch (Exception e)
                            {
                                cdnPool.ReturnBrokenConnection(connection);
                                Console.WriteLine("Encountered error downloading manifest for depot {0} {1}: {2}", depot.id, depot.manifestId, e.Message);
                            }
                        }

                        if (depotManifest == null)
                        {
                            Console.WriteLine("\nUnable to download manifest {0} for depot {1}", depot.manifestId, depot.id);
                            return;
                        }

                        byte[] checksum;

                        newProtoManifest = new ProtoManifest(depotManifest, depot.manifestId);
                        newProtoManifest.SaveToFile(newManifestFileName, out checksum);
                        File.WriteAllBytes(newManifestFileName + ".sha", checksum);

                        Console.WriteLine(" Done!");
                    }
                }

                newProtoManifest.Files.Sort((x, y) => string.Compare(x.FileName, y.FileName, StringComparison.Ordinal));

                if (Config.DownloadManifestOnly)
                {
                    StringBuilder manifestBuilder = new StringBuilder();
                    string        txtManifest     = Path.Combine(depot.installDir, string.Format("manifest_{0}.txt", depot.id));

                    foreach (var file in newProtoManifest.Files)
                    {
                        if (file.Flags.HasFlag(EDepotFileFlag.Directory))
                        {
                            continue;
                        }

                        manifestBuilder.Append(string.Format("{0}\n", file.FileName));
                    }

                    File.WriteAllText(txtManifest, manifestBuilder.ToString());
                    continue;
                }

                ulong  complete_download_size = 0;
                ulong  size_downloaded        = 0;
                string stagingDir             = Path.Combine(depot.installDir, STAGING_DIR);

                var filesAfterExclusions = newProtoManifest.Files.AsParallel().Where(f => TestIsFileIncluded(f.FileName)).ToList();

                // Pre-process
                filesAfterExclusions.ForEach(file =>
                {
                    var fileFinalPath   = Path.Combine(depot.installDir, file.FileName);
                    var fileStagingPath = Path.Combine(stagingDir, file.FileName);

                    if (file.Flags.HasFlag(EDepotFileFlag.Directory))
                    {
                        Directory.CreateDirectory(fileFinalPath);
                        Directory.CreateDirectory(fileStagingPath);
                    }
                    else
                    {
                        // Some manifests don't explicitly include all necessary directories
                        Directory.CreateDirectory(Path.GetDirectoryName(fileFinalPath));
                        Directory.CreateDirectory(Path.GetDirectoryName(fileStagingPath));

                        complete_download_size += file.TotalSize;
                    }
                });

                var semaphore = new SemaphoreSlim(Config.MaxDownloads);
                var files     = filesAfterExclusions.Where(f => !f.Flags.HasFlag(EDepotFileFlag.Directory)).ToArray();
                var tasks     = new Task[files.Length];
                for (var i = 0; i < files.Length; i++)
                {
                    var file = files[i];
                    var task = Task.Run(async() =>
                    {
                        cts.Token.ThrowIfCancellationRequested();

                        try
                        {
                            await semaphore.WaitAsync().ConfigureAwait(false);
                            cts.Token.ThrowIfCancellationRequested();

                            string fileFinalPath   = Path.Combine(depot.installDir, file.FileName);
                            string fileStagingPath = Path.Combine(stagingDir, file.FileName);

                            // This may still exist if the previous run exited before cleanup
                            if (File.Exists(fileStagingPath))
                            {
                                File.Delete(fileStagingPath);
                            }

                            FileStream fs = null;
                            List <ProtoManifest.ChunkData> neededChunks;
                            FileInfo fi = new FileInfo(fileFinalPath);
                            if (!fi.Exists)
                            {
                                // create new file. need all chunks
                                fs = File.Create(fileFinalPath);
                                fs.SetLength(( long )file.TotalSize);
                                neededChunks = new List <ProtoManifest.ChunkData>(file.Chunks);
                            }
                            else
                            {
                                // open existing
                                ProtoManifest.FileData oldManifestFile = null;
                                if (oldProtoManifest != null)
                                {
                                    oldManifestFile = oldProtoManifest.Files.SingleOrDefault(f => f.FileName == file.FileName);
                                }

                                if (oldManifestFile != null)
                                {
                                    neededChunks = new List <ProtoManifest.ChunkData>();

                                    if (Config.VerifyAll || !oldManifestFile.FileHash.SequenceEqual(file.FileHash))
                                    {
                                        // we have a version of this file, but it doesn't fully match what we want

                                        var matchingChunks = new List <ChunkMatch>();

                                        foreach (var chunk in file.Chunks)
                                        {
                                            var oldChunk = oldManifestFile.Chunks.FirstOrDefault(c => c.ChunkID.SequenceEqual(chunk.ChunkID));
                                            if (oldChunk != null)
                                            {
                                                matchingChunks.Add(new ChunkMatch(oldChunk, chunk));
                                            }
                                            else
                                            {
                                                neededChunks.Add(chunk);
                                            }
                                        }

                                        File.Move(fileFinalPath, fileStagingPath);

                                        fs = File.Open(fileFinalPath, FileMode.Create);
                                        fs.SetLength(( long )file.TotalSize);

                                        using (var fsOld = File.Open(fileStagingPath, FileMode.Open))
                                        {
                                            foreach (var match in matchingChunks)
                                            {
                                                fsOld.Seek(( long )match.OldChunk.Offset, SeekOrigin.Begin);

                                                byte[] tmp = new byte[match.OldChunk.UncompressedLength];
                                                fsOld.Read(tmp, 0, tmp.Length);

                                                byte[] adler = Util.AdlerHash(tmp);
                                                if (!adler.SequenceEqual(match.OldChunk.Checksum))
                                                {
                                                    neededChunks.Add(match.NewChunk);
                                                }
                                                else
                                                {
                                                    fs.Seek(( long )match.NewChunk.Offset, SeekOrigin.Begin);
                                                    fs.Write(tmp, 0, tmp.Length);
                                                }
                                            }
                                        }

                                        File.Delete(fileStagingPath);
                                    }
                                }
                                else
                                {
                                    // No old manifest or file not in old manifest. We must validate.

                                    fs = File.Open(fileFinalPath, FileMode.Open);
                                    if (( ulong )fi.Length != file.TotalSize)
                                    {
                                        fs.SetLength(( long )file.TotalSize);
                                    }

                                    neededChunks = Util.ValidateSteam3FileChecksums(fs, file.Chunks.OrderBy(x => x.Offset).ToArray());
                                }

                                if (neededChunks.Count() == 0)
                                {
                                    size_downloaded += file.TotalSize;
                                    Console.WriteLine("{0,6:#00.00}% {1}", (( float )size_downloaded / ( float )complete_download_size) * 100.0f, fileFinalPath);
                                    if (fs != null)
                                    {
                                        fs.Dispose();
                                    }
                                    return;
                                }
                                else
                                {
                                    size_downloaded += (file.TotalSize - ( ulong )neededChunks.Select(x => ( long )x.UncompressedLength).Sum());
                                }
                            }

                            foreach (var chunk in neededChunks)
                            {
                                if (cts.IsCancellationRequested)
                                {
                                    break;
                                }

                                string chunkID = Util.EncodeHexString(chunk.ChunkID);
                                CDNClient.DepotChunk chunkData = null;

                                while (!cts.IsCancellationRequested)
                                {
                                    Tuple <CDNClient.Server, string> connection;
                                    try
                                    {
                                        connection = await cdnPool.GetConnectionForDepot(appId, depot.id, cts.Token);
                                    }
                                    catch (OperationCanceledException)
                                    {
                                        break;
                                    }

                                    DepotManifest.ChunkData data = new DepotManifest.ChunkData();
                                    data.ChunkID            = chunk.ChunkID;
                                    data.Checksum           = chunk.Checksum;
                                    data.Offset             = chunk.Offset;
                                    data.CompressedLength   = chunk.CompressedLength;
                                    data.UncompressedLength = chunk.UncompressedLength;

                                    try
                                    {
                                        chunkData = await cdnPool.CDNClient.DownloadDepotChunkAsync(depot.id, data,
                                                                                                    connection.Item1, connection.Item2, depot.depotKey).ConfigureAwait(false);
                                        cdnPool.ReturnConnection(connection);
                                        break;
                                    }
                                    catch (SteamKitWebRequestException e)
                                    {
                                        cdnPool.ReturnBrokenConnection(connection);

                                        if (e.StatusCode == HttpStatusCode.Unauthorized || e.StatusCode == HttpStatusCode.Forbidden)
                                        {
                                            Console.WriteLine("Encountered 401 for chunk {0}. Aborting.", chunkID);
                                            cts.Cancel();
                                            break;
                                        }
                                        else
                                        {
                                            Console.WriteLine("Encountered error downloading chunk {0}: {1}", chunkID, e.StatusCode);
                                        }
                                    }
                                    catch (Exception e)
                                    {
                                        cdnPool.ReturnBrokenConnection(connection);
                                        Console.WriteLine("Encountered unexpected error downloading chunk {0}: {1}", chunkID, e.Message);
                                    }
                                }

                                if (chunkData == null)
                                {
                                    Console.WriteLine("Failed to find any server with chunk {0} for depot {1}. Aborting.", chunkID, depot.id);
                                    cts.Cancel();
                                }

                                // Throw the cancellation exception if requested so that this task is marked failed
                                cts.Token.ThrowIfCancellationRequested();

                                TotalBytesCompressed   += chunk.CompressedLength;
                                DepotBytesCompressed   += chunk.CompressedLength;
                                TotalBytesUncompressed += chunk.UncompressedLength;
                                DepotBytesUncompressed += chunk.UncompressedLength;

                                fs.Seek(( long )chunk.Offset, SeekOrigin.Begin);
                                fs.Write(chunkData.Data, 0, chunkData.Data.Length);

                                size_downloaded += chunk.UncompressedLength;
                            }

                            fs.Dispose();

                            Console.WriteLine("{0,6:#00.00}% {1}", (( float )size_downloaded / ( float )complete_download_size) * 100.0f, fileFinalPath);
                        }
                        finally
                        {
                            semaphore.Release();
                        }
                    });

                    tasks[i] = task;
                }

                await Task.WhenAll(tasks).ConfigureAwait(false);

                DepotConfigStore.Instance.InstalledManifestIDs[depot.id] = depot.manifestId;
                DepotConfigStore.Save();

                Console.WriteLine("Depot {0} - Downloaded {1} bytes ({2} bytes uncompressed)", depot.id, DepotBytesCompressed, DepotBytesUncompressed);
            }

            Console.WriteLine("Total downloaded: {0} bytes ({1} bytes uncompressed) from {2} depots", TotalBytesCompressed, TotalBytesUncompressed, depots.Count);
        }
        private static ulong DownloadSteam3(List<DepotDownloadInfo> depots)
        {
            ulong TotalBytesCompressed = 0;
            ulong TotalBytesUncompressed = 0;
            ulong result = 0;

            foreach (var depot in depots)
            {
                result ^= depot.manifestId;
                ulong DepotBytesCompressed = 0;
                ulong DepotBytesUncompressed = 0;

                Console.WriteLine("Downloading depot {0} - {1}", depot.id, depot.contentName);
                Console.Write("Finding content servers...");

                List<CDNClient> cdnClients = null;

                Console.WriteLine(" Done!");

                ProtoManifest oldProtoManifest = null;
                ProtoManifest newProtoManifest = null;
                string configDir = Path.Combine(depot.installDir, CONFIG_DIR);

                ulong lastManifestId = INVALID_MANIFEST_ID;
                ConfigStore.TheConfig.LastManifests.TryGetValue(depot.id, out lastManifestId);

                // In case we have an early exit, this will force equiv of verifyall next run.
                ConfigStore.TheConfig.LastManifests[depot.id] = INVALID_MANIFEST_ID;
                ConfigStore.Save();

                if (lastManifestId != INVALID_MANIFEST_ID)
                {
                    var oldManifestFileName = Path.Combine(configDir, string.Format("{0}.bin", lastManifestId));
                    if (File.Exists(oldManifestFileName))
                        oldProtoManifest = ProtoManifest.LoadFromFile(oldManifestFileName);
                }

                if (lastManifestId == depot.manifestId && oldProtoManifest != null)
                {
                    newProtoManifest = oldProtoManifest;
                    Console.WriteLine("Already have manifest {0} for depot {1}.", depot.manifestId, depot.id);
                }
                else
                {
                    var newManifestFileName = Path.Combine(configDir, string.Format("{0}.bin", depot.manifestId));
                    if (newManifestFileName != null)
                    {
                        newProtoManifest = ProtoManifest.LoadFromFile(newManifestFileName);
                    }

                    if (newProtoManifest != null)
                    {
                        Console.WriteLine("Already have manifest {0} for depot {1}.", depot.manifestId, depot.id);
                    }
                    else
                    {
                        Console.Write("Downloading depot manifest...");

                        DepotManifest depotManifest = null;

                        cdnClients = CollectCDNClientsForDepot(depot);

                        foreach (var c in cdnClients)
                        {
                            try
                            {
                                depotManifest = c.DownloadManifest(depot.id, depot.manifestId);
                                break;
                            }
                            catch (WebException) { }
                            catch (SocketException) { }
                        }

                        if (depotManifest == null)
                        {
                            Console.WriteLine("\nUnable to download manifest {0} for depot {1}", depot.manifestId, depot.id);
                            return 0;
                        }

                        newProtoManifest = new ProtoManifest(depotManifest, depot.manifestId);
                        newProtoManifest.SaveToFile(newManifestFileName);

                        Console.WriteLine(" Done!");
                    }
                }

                newProtoManifest.Files.Sort((x, y) => { return x.FileName.CompareTo(y.FileName); });

                if (Config.DownloadManifestOnly)
                {
                    StringBuilder manifestBuilder = new StringBuilder();
                    string txtManifest = Path.Combine(depot.installDir, string.Format("manifest_{0}.txt", depot.id));

                    foreach (var file in newProtoManifest.Files)
                    {
                        if (file.Flags.HasFlag(EDepotFileFlag.Directory))
                            continue;

                        manifestBuilder.Append(string.Format("{0}\n", file.FileName));
                    }

                    File.WriteAllText(txtManifest, manifestBuilder.ToString());
                    continue;
                }
                #region post-manifest
                ulong complete_download_size = 0;
                ulong size_downloaded = 0;
                string stagingDir = Path.Combine(depot.installDir, STAGING_DIR);

                var filesAfterExclusions = newProtoManifest.Files.AsParallel().Where(f => TestIsFileIncluded(f.FileName)).ToList();

                // Pre-process
                filesAfterExclusions.ForEach(file =>
                {
                    var fileFinalPath = Path.Combine(depot.installDir, file.FileName);
                    var fileStagingPath = Path.Combine(stagingDir, file.FileName);

                    if (file.Flags.HasFlag(EDepotFileFlag.Directory))
                    {
                        Directory.CreateDirectory(fileFinalPath);
                        Directory.CreateDirectory(fileStagingPath);
                    }
                    else
                    {
                        // Some manifests don't explicitly include all necessary directories
                        Directory.CreateDirectory(Path.GetDirectoryName(fileFinalPath));
                        Directory.CreateDirectory(Path.GetDirectoryName(fileStagingPath));

                        complete_download_size += file.TotalSize;
                    }
                });

                var rand = new Random();

                filesAfterExclusions.Where(f => !f.Flags.HasFlag(EDepotFileFlag.Directory))
                    .AsParallel().WithDegreeOfParallelism(Config.MaxDownloads)
                    .ForAll(file =>
                {
                    string fileFinalPath = Path.Combine(depot.installDir, file.FileName);
                    string fileStagingPath = Path.Combine(stagingDir, file.FileName);

                    // This may still exist if the previous run exited before cleanup
                    if (File.Exists(fileStagingPath))
                    {
                        File.Delete(fileStagingPath);
                    }

                    FileStream fs = null;
                    List<ProtoManifest.ChunkData> neededChunks;
                    FileInfo fi = new FileInfo(fileFinalPath);
                    if (!fi.Exists)
                    {
                        // create new file. need all chunks
                        fs = File.Create(fileFinalPath);
                        fs.SetLength((long)file.TotalSize);
                        neededChunks = new List<ProtoManifest.ChunkData>(file.Chunks);
                    }
                    else
                    {
                        // open existing
                        ProtoManifest.FileData oldManifestFile = null;
                        if (oldProtoManifest != null)
                        {
                            oldManifestFile = oldProtoManifest.Files.SingleOrDefault(f => f.FileName == file.FileName);
                        }

                        if (oldManifestFile != null)
                        {
                            neededChunks = new List<ProtoManifest.ChunkData>();

                            if (Config.VerifyAll || !oldManifestFile.FileHash.SequenceEqual(file.FileHash))
                            {
                                // we have a version of this file, but it doesn't fully match what we want

                                var matchingChunks = new List<ChunkMatch>();

                                foreach (var chunk in file.Chunks)
                                {
                                    var oldChunk = oldManifestFile.Chunks.FirstOrDefault(c => c.ChunkID.SequenceEqual(chunk.ChunkID));
                                    if (oldChunk != null)
                                    {
                                        matchingChunks.Add(new ChunkMatch(oldChunk, chunk));
                                    }
                                    else
                                    {
                                        neededChunks.Add(chunk);
                                    }
                                }

                                File.Move(fileFinalPath, fileStagingPath);

                                fs = File.Open(fileFinalPath, FileMode.Create);
                                fs.SetLength((long)file.TotalSize);

                                using (var fsOld = File.Open(fileStagingPath, FileMode.Open))
                                {
                                    foreach (var match in matchingChunks)
                                    {
                                        fs.Seek((long)match.NewChunk.Offset, SeekOrigin.Begin);
                                        fsOld.Seek((long)match.OldChunk.Offset, SeekOrigin.Begin);

                                        byte[] tmp = new byte[match.OldChunk.UncompressedLength];
                                        fsOld.Read(tmp, 0, tmp.Length);
                                        fs.Write(tmp, 0, tmp.Length);
                                    }
                                }

                                File.Delete(fileStagingPath);
                            }
                        }
                        else
                        {
                            // No old manifest or file not in old manifest. We must validate.

                            fs = File.Open(fileFinalPath, FileMode.Open);
                            if ((ulong)fi.Length != file.TotalSize)
                            {
                                fs.SetLength((long)file.TotalSize);
                            }

                            neededChunks = Util.ValidateSteam3FileChecksums(fs, file.Chunks.OrderBy(x => x.Offset).ToArray());
                        }

                        if (neededChunks.Count() == 0)
                        {
                            size_downloaded += file.TotalSize;
                            Console.WriteLine("{0,6:#00.00}% {1}", ((float)size_downloaded / (float)complete_download_size) * 100.0f, fileFinalPath);
                            if (fs != null)
                                fs.Close();
                            return;
                        }
                        else
                        {
                            size_downloaded += (file.TotalSize - (ulong)neededChunks.Select(x => (int)x.UncompressedLength).Sum());
                        }
                    }

                    int cdnClientIndex = 0;
                    if (neededChunks.Count > 0 && cdnClients == null)
                    {
                        // If we didn't need to connect to get manifests, connect now.
                        cdnClients = CollectCDNClientsForDepot(depot);
                        cdnClientIndex = rand.Next(0, cdnClients.Count);
                    }

                    foreach (var chunk in neededChunks)
                    {
                        string chunkID = Util.EncodeHexString(chunk.ChunkID);

                        CDNClient.DepotChunk chunkData = null;
                        int idx = cdnClientIndex;
                        while (true)
                        {
                            DepotManifest.ChunkData data = new DepotManifest.ChunkData();
                            data.ChunkID = chunk.ChunkID;
                            data.Checksum = chunk.Checksum;
                            data.Offset = chunk.Offset;
                            data.CompressedLength = chunk.CompressedLength;
                            data.UncompressedLength = chunk.UncompressedLength;

                            try
                            {
                                chunkData = cdnClients[idx].DownloadDepotChunk(depot.id, data);
                                break;
                            }
                            catch
                            {
                                if (++idx >= cdnClients.Count)
                                    idx = 0;

                                if (idx == cdnClientIndex)
                                    break;
                            }
                        }

                        if (chunkData == null)
                        {
                            Console.WriteLine("Failed to find any server with chunk {0} for depot {1}. Aborting.", chunkID, depot);
                            return;
                        }

                        TotalBytesCompressed += chunk.CompressedLength;
                        DepotBytesCompressed += chunk.CompressedLength;
                        TotalBytesUncompressed += chunk.UncompressedLength;
                        DepotBytesUncompressed += chunk.UncompressedLength;

                        fs.Seek((long)chunk.Offset, SeekOrigin.Begin);
                        fs.Write(chunkData.Data, 0, chunkData.Data.Length);

                        size_downloaded += chunk.UncompressedLength;
                    }

                    fs.Close();

                    Console.WriteLine("{0,6:#00.00}% {1}", ((float)size_downloaded / (float)complete_download_size) * 100.0f, fileFinalPath);
                });

                ConfigStore.TheConfig.LastManifests[depot.id] = depot.manifestId;
                ConfigStore.Save();

                Console.WriteLine("Depot {0} - Downloaded {1} bytes ({2} bytes uncompressed)", depot.id, DepotBytesCompressed, DepotBytesUncompressed);
            }
                #endregion

            Console.WriteLine("Total downloaded: {0} bytes ({1} bytes uncompressed) from {2} depots", TotalBytesCompressed, TotalBytesUncompressed, depots.Count);
            return result;
        }
 public ChunkMatch(ProtoManifest.ChunkData oldChunk, ProtoManifest.ChunkData newChunk)
 {
     OldChunk = oldChunk;
     NewChunk = newChunk;
 }
Exemplo n.º 6
0
        // Validate a file against Steam3 Chunk data
        public static List<ProtoManifest.ChunkData> ValidateSteam3FileChecksums(FileStream fs, ProtoManifest.ChunkData[] chunkdata)
        {
            var neededChunks = new List<ProtoManifest.ChunkData>();
            int read;

            foreach (var data in chunkdata)
            {
                byte[] chunk = new byte[data.UncompressedLength];
                fs.Seek((long)data.Offset, SeekOrigin.Begin);
                read = fs.Read(chunk, 0, (int)data.UncompressedLength);

                byte[] tempchunk;
                if (read < data.UncompressedLength)
                {
                    tempchunk = new byte[read];
                    Array.Copy(chunk, 0, tempchunk, 0, read);
                }
                else
                {
                    tempchunk = chunk;
                }

                byte[] adler = AdlerHash(tempchunk);
                if (!adler.SequenceEqual(data.Checksum))
                {
                    neededChunks.Add(data);
                }
            }

            return neededChunks;
        }
Exemplo n.º 7
0
        private static async Task <DepotFilesData> ProcessDepotManifestAndFiles(CancellationTokenSource cts,
                                                                                uint appId, DepotDownloadInfo depot)
        {
            DepotDownloadCounter depotCounter = new DepotDownloadCounter();

            Console.WriteLine("Processing depot {0} - {1}", depot.id, depot.contentName);

            ProtoManifest oldProtoManifest = null;
            ProtoManifest newProtoManifest = null;
            string        configDir        = Path.Combine(depot.installDir, CONFIG_DIR);

            ulong lastManifestId = INVALID_MANIFEST_ID;

            DepotConfigStore.Instance.InstalledManifestIDs.TryGetValue(depot.id, out lastManifestId);

            // In case we have an early exit, this will force equiv of verifyall next run.
            DepotConfigStore.Instance.InstalledManifestIDs[depot.id] = INVALID_MANIFEST_ID;
            DepotConfigStore.Save();

            if (lastManifestId != INVALID_MANIFEST_ID)
            {
                var oldManifestFileName = Path.Combine(configDir, string.Format("{0}_{1}.bin", depot.id, lastManifestId));

                if (File.Exists(oldManifestFileName))
                {
                    byte[] expectedChecksum, currentChecksum;

                    try
                    {
                        expectedChecksum = File.ReadAllBytes(oldManifestFileName + ".sha");
                    }
                    catch (IOException)
                    {
                        expectedChecksum = null;
                    }

                    oldProtoManifest = ProtoManifest.LoadFromFile(oldManifestFileName, out currentChecksum);

                    if (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum))
                    {
                        // We only have to show this warning if the old manifest ID was different
                        if (lastManifestId != depot.manifestId)
                        {
                            Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", lastManifestId);
                        }
                        oldProtoManifest = null;
                    }
                }
            }

            if (lastManifestId == depot.manifestId && oldProtoManifest != null)
            {
                newProtoManifest = oldProtoManifest;
                Console.WriteLine("Already have manifest {0} for depot {1}.", depot.manifestId, depot.id);
            }
            else
            {
                var newManifestFileName = Path.Combine(configDir, string.Format("{0}_{1}.bin", depot.id, depot.manifestId));
                if (newManifestFileName != null)
                {
                    byte[] expectedChecksum, currentChecksum;

                    try
                    {
                        expectedChecksum = File.ReadAllBytes(newManifestFileName + ".sha");
                    }
                    catch (IOException)
                    {
                        expectedChecksum = null;
                    }

                    newProtoManifest = ProtoManifest.LoadFromFile(newManifestFileName, out currentChecksum);

                    if (newProtoManifest != null && (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum)))
                    {
                        Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", depot.manifestId);
                        newProtoManifest = null;
                    }
                }

                if (newProtoManifest != null)
                {
                    Console.WriteLine("Already have manifest {0} for depot {1}.", depot.manifestId, depot.id);
                }
                else
                {
                    Console.Write("Downloading depot manifest...");

                    DepotManifest depotManifest = null;

                    do
                    {
                        cts.Token.ThrowIfCancellationRequested();

                        CDNClient.Server connection = null;

                        try
                        {
                            connection = cdnPool.GetConnection(cts.Token);
                            var cdnToken = await cdnPool.AuthenticateConnection(appId, depot.id, connection);

                            depotManifest = await cdnPool.CDNClient.DownloadManifestAsync(depot.id, depot.manifestId,
                                                                                          connection, cdnToken, depot.depotKey).ConfigureAwait(false);

                            cdnPool.ReturnConnection(connection);
                        }
                        catch (TaskCanceledException)
                        {
                            Console.WriteLine("Connection timeout downloading depot manifest {0} {1}", depot.id, depot.manifestId);
                        }
                        catch (SteamKitWebRequestException e)
                        {
                            cdnPool.ReturnBrokenConnection(connection);

                            if (e.StatusCode == HttpStatusCode.Unauthorized || e.StatusCode == HttpStatusCode.Forbidden)
                            {
                                Console.WriteLine("Encountered 401 for depot manifest {0} {1}. Aborting.", depot.id, depot.manifestId);
                                break;
                            }
                            else
                            {
                                Console.WriteLine("Encountered error downloading depot manifest {0} {1}: {2}", depot.id, depot.manifestId, e.StatusCode);
                            }
                        }
                        catch (OperationCanceledException)
                        {
                            break;
                        }
                        catch (Exception e)
                        {
                            cdnPool.ReturnBrokenConnection(connection);
                            Console.WriteLine("Encountered error downloading manifest for depot {0} {1}: {2}", depot.id, depot.manifestId, e.Message);
                        }
                    }while (depotManifest == null);

                    if (depotManifest == null)
                    {
                        Console.WriteLine("\nUnable to download manifest {0} for depot {1}", depot.manifestId, depot.id);
                        cts.Cancel();
                    }

                    // Throw the cancellation exception if requested so that this task is marked failed
                    cts.Token.ThrowIfCancellationRequested();

                    byte[] checksum;

                    newProtoManifest = new ProtoManifest(depotManifest, depot.manifestId);
                    newProtoManifest.SaveToFile(newManifestFileName, out checksum);
                    File.WriteAllBytes(newManifestFileName + ".sha", checksum);

                    Console.WriteLine(" Done!");
                }
            }

            newProtoManifest.Files.Sort((x, y) => string.Compare(x.FileName, y.FileName, StringComparison.Ordinal));

            Console.WriteLine("Manifest {0} ({1})", depot.manifestId, newProtoManifest.CreationTime);

            if (Config.DownloadManifestOnly)
            {
                StringBuilder manifestBuilder = new StringBuilder();
                string        txtManifest     = Path.Combine(depot.installDir, string.Format("manifest_{0}_{1}.txt", depot.id, depot.manifestId));
                manifestBuilder.Append(string.Format("{0}\n\n", newProtoManifest.CreationTime));

                foreach (var file in newProtoManifest.Files)
                {
                    if (file.Flags.HasFlag(EDepotFileFlag.Directory))
                    {
                        continue;
                    }

                    manifestBuilder.Append(string.Format("{0}\n", file.FileName));
                    manifestBuilder.Append(string.Format("\t{0}\n", file.TotalSize));
                    manifestBuilder.Append(string.Format("\t{0}\n", BitConverter.ToString(file.FileHash).Replace("-", "")));
                }

                File.WriteAllText(txtManifest, manifestBuilder.ToString());
                return(null);
            }

            string stagingDir = Path.Combine(depot.installDir, STAGING_DIR);

            var filesAfterExclusions = newProtoManifest.Files.AsParallel().Where(f => TestIsFileIncluded(f.FileName)).ToList();
            var allFileNames         = new HashSet <string>(filesAfterExclusions.Count);

            // Pre-process
            filesAfterExclusions.ForEach(file =>
            {
                allFileNames.Add(file.FileName);

                var fileFinalPath   = Path.Combine(depot.installDir, file.FileName);
                var fileStagingPath = Path.Combine(stagingDir, file.FileName);

                if (file.Flags.HasFlag(EDepotFileFlag.Directory))
                {
                    Directory.CreateDirectory(fileFinalPath);
                    Directory.CreateDirectory(fileStagingPath);
                }
                else
                {
                    // Some manifests don't explicitly include all necessary directories
                    Directory.CreateDirectory(Path.GetDirectoryName(fileFinalPath));
                    Directory.CreateDirectory(Path.GetDirectoryName(fileStagingPath));

                    depotCounter.CompleteDownloadSize += file.TotalSize;
                }
            });

            return(new DepotFilesData
            {
                depotDownloadInfo = depot,
                depotCounter = depotCounter,
                stagingDir = stagingDir,
                manifest = newProtoManifest,
                previousManifest = oldProtoManifest,
                filteredFiles = filesAfterExclusions,
                allFileNames = allFileNames
            });
        }
 private async void FileSelector_LoadAsync(object sender, EventArgs e)
 {
     string Title = this.Text;
     string Password = "";
     if(ManifestID==ContentDownloader.INVALID_MANIFEST_ID)
         ManifestID = ContentDownloader.GetSteam3DepotManifestStatic(DepotID, AppID, Branch,ref Password);
     if (ManifestID == ContentDownloader.INVALID_MANIFEST_ID)
     {
         MessageBox.Show(Properties.Resources.NoManifestID, "FileSelector", MessageBoxButtons.OK, MessageBoxIcon.Exclamation);
         return;
     }
     this.labelManifestID.Text = "ManifestID:" + ManifestID.ToString();
     System.IO.Directory.CreateDirectory(Program.CacheDir);
     var localProtoManifest = DepotDownloader.ProtoManifest.LoadFromFile(string.Format("{0}/{1}.bin",Program.CacheDir, ManifestID));
     if (localProtoManifest!=null)
     {
         depotManifest = localProtoManifest;
     }
     else
     {
         this.Text = "Downloading File List...";
         this.Refresh();
         ContentDownloader.Steam3.RequestDepotKey(DepotID,AppID);
         if(!ContentDownloader.Steam3.DepotKeys.ContainsKey(DepotID))
         {
             MessageBox.Show(Properties.Resources.NoDepotKey, "FileSelector",MessageBoxButtons.OK,MessageBoxIcon.Exclamation);
             //return;
             //User may still need to use FileRegex
             Close();
             return;
         }
         byte[] depotKey = ContentDownloader.Steam3.DepotKeys[DepotID];
         //ContentDownloader.Steam3.RequestAppTicket(AppID);
         ContentDownloader.Steam3.RequestAppTicket(DepotID);
         var client = await DepotDownloader.ContentDownloader.GlobalCDNPool.GetConnectionForDepotAsync(AppID, DepotID, depotKey, System.Threading.CancellationToken.None).ConfigureAwait(true);
         var SteamKitdepotManifest = await client.DownloadManifestAsync(DepotID, ManifestID).ConfigureAwait(true);
         if (SteamKitdepotManifest != null)
         {
             localProtoManifest = new ProtoManifest(SteamKitdepotManifest, ManifestID);
             localProtoManifest.SaveToFile(string.Format("{0}/{1}.bin", Program.CacheDir, ManifestID));
             depotManifest = localProtoManifest;
         }
     }
     if(depotManifest!=null)
     {
         foreach (var file in localProtoManifest.Files)
         {
             //Process Dir
             var FilePathSplited = file.FileName.Split('\\');
             List<string> FilePathList = new List<string>(FilePathSplited);
             TreeNode LastNode=null;
             TreeNodeCollection FileNodes = this.treeViewFileList.Nodes;
             while (FilePathList.Count != 0)
             {
                 TreeNode TargetNode = null;
                 foreach (TreeNode Tn in FileNodes)
                 {
                     if (Tn.Text == FilePathList[0])
                     {
                         TargetNode = Tn;
                         break;
                     }
                 }
                 if (TargetNode == null)
                 {
                     LastNode = FileNodes.Add(FilePathList[0]);
                     FileNodes = LastNode.Nodes;
                 }
                 else
                 {
                     FileNodes = TargetNode.Nodes;
                 }
                 FilePathList.RemoveAt(0);
             }
             if(file.Flags!=SteamKit2.EDepotFileFlag.Directory)
             {
                 DepotMaxSize += file.TotalSize;
                 //if(LastNode!=null)
                 LastNode.Tag = file.TotalSize;
             }
         }
         float TargetSize = DepotMaxSize / 1024f;
         if(TargetSize<1024)
         {
             SizeDivisor = 1024f;
             UnitStr = "KB";
         }
         else if(TargetSize/1024f<1024)
         {
             SizeDivisor = 1024 * 1024f;
             UnitStr = "MB";
         }
         else
         {
             SizeDivisor = 1024 * 1024 * 1024f;
             UnitStr = "GB";
         }
         this.labelSize.Text = string.Format("{0}{1}/{2}{1}", 0.ToString("#0.00"), UnitStr, (DepotMaxSize / SizeDivisor).ToString("#0.00"));
     }
     this.Text = Title;
 }