private List <CharacterModel> LoadCharacterModels(bool unique) { if (_cache.TryLoadCache(out var cached)) { var validCached = cached.Where(x => x.UniqueCharacter == unique).ToList(); if (validCached.Any()) { return(validCached); } } else { cached = new List <CharacterModel>(); } var files = Prefetch.Load().Files.Keys; var modelBag = new ConcurrentBag <CharacterModel>(); var tasks = new ParallelTasks <string>( Environment.ProcessorCount, file => { if (IsValid(file, unique)) { foreach (var model in GetCharacterModels(file)) { modelBag.Add(model); } } }); tasks.Start(); tasks.AddItems(files); tasks.WaitForComplete(); GC.Collect(); var modelList = modelBag.ToList(); cached.AddRange(modelList); _cache.Save(cached); return(modelList); }
private List <T> LoadInternal() { if (_cache.TryLoadCache(out var cached)) { if (cached.Any()) { return(cached); } } else { cached = new List <T>(); } var files = Prefetch.Load().Files.Keys; var bag = new ConcurrentBag <T>(); var parallels = IoC.Debug.SingleThread ? 1 : Environment.ProcessorCount; var tasks = new ParallelTasks <string>( parallels, file => { if (_fileNameValidator(file) && !Ignored.Any(x => x.IsMatch(file))) { foreach (var item in _itemGetter(file)) { bag.Add(item); } } }); tasks.Start(); tasks.AddItems(files); tasks.WaitForComplete(); GC.Collect(); var itemList = bag.ToList(); cached.AddRange(itemList); _cache.Save(cached); return(itemList); }
public void BuildFromFileList(string physicalPathRoot, string[] sourceFiles) { var compressedBlocks = new ConcurrentBag <CompressBlock>(); var tasks = new ParallelTasks <CompressBlock>( Environment.ProcessorCount, block => { block.CompressBuffer = new byte[BlockSize * 2]; // Compress long compressedSize = Utility.OodleLZ.Compress( block.DataBuffer.AsSpan(), (int)block.DecompressedSize, block.CompressBuffer.AsSpan()); if (compressedSize == -1) { throw new Exception("Buffer compression failed"); } block.Size = (uint)compressedSize; block.DataBuffer = null; compressedBlocks.Add(block); }); tasks.Start(); AddFiles(physicalPathRoot, sourceFiles, tasks); tasks.WaitForComplete(); using var fs = File.Open(_archivePath, _allowOverwrite ? FileMode.Create : FileMode.CreateNew, FileAccess.ReadWrite, FileShare.None); using var archiveWriter = new BinaryWriter(fs, Encoding.UTF8, true); archiveWriter.BaseStream.Position = CalculateArchiveHeaderLength(sourceFiles.Length, compressedBlocks.Count); WriteBlockEntries(archiveWriter, compressedBlocks); archiveWriter.BaseStream.Position = 0; WriteArchiveHeaders(archiveWriter); }
/// <summary> /// Generate an archive from a set of file paths on disk. /// </summary> /// <param name="baseDirectoryRoot">Base directory to look for files</param> /// <param name="sourceCorePaths">List of files in Decima core path format. <see cref="baseDirectoryRoot"/> /// is prepended to each element.</param> public void BuildFromFileList(string baseDirectoryRoot, IEnumerable <string> sourceCorePaths) { var compressedBlocks = new ConcurrentBag <CompressBlock>(); var tasks = new ParallelTasks <CompressBlock>( Environment.ProcessorCount, block => { block.CompressBuffer = new byte[BlockSize * 2]; // Compress long compressedSize = HZDCoreEditor.Util.OodleLZ.Compress( block.DataBuffer.AsSpan(), (int)block.DecompressedSize, block.CompressBuffer.AsSpan()); if (compressedSize == -1) { throw new Exception("Buffer compression failed"); } block.Size = (uint)compressedSize; block.DataBuffer = null; compressedBlocks.Add(block); }); tasks.Start(); AddFiles(baseDirectoryRoot, sourceCorePaths, tasks); tasks.WaitForComplete(); using var fs = File.Open(_archivePath, _fileMode, FileAccess.ReadWrite, FileShare.None); using var archiveWriter = new BinaryWriter(fs, Encoding.UTF8, true); archiveWriter.BaseStream.Position = CalculateArchiveHeaderLength(sourceCorePaths.Count(), compressedBlocks.Count); WriteBlockEntries(archiveWriter, compressedBlocks); archiveWriter.BaseStream.Position = 0; WriteArchiveHeaders(archiveWriter); }