public UnpackedRenamedFileMapping LoadMappingData()
{
if (!fileInfo.Exists)
{
throw new Exception("Can't read from null file!");
}
log.Info("Loading renamed file mapping from file: " + fileInfo.FullName);
UnpackedRenamedFileMapping mapping = new UnpackedRenamedFileMapping();
using (FileStream fs = File.OpenRead(fileInfo.FullName))
{
int sig = fs.ReadByte();
while (sig != -1)
{
UnpackedRenamedFileMapping.RenamedFileMappingData data = new UnpackedRenamedFileMapping.RenamedFileMappingData();
data.ReadFromStream(fs, tempBuffer);
mapping[data.Key] = data;
sig = fs.ReadByte();
}
}
return(mapping);
}
public override void LoadFromDisk()
{
status.stopwatch.Reset();
status.stopwatch.Start();
status.UpdateProgress(0f);
log.Info("Loading unpacked bigfile from disk...");
UnpackedFileKeyMappingFile mappingFile = new UnpackedFileKeyMappingFile(new DirectoryInfo(fileOrDirectory));
renamedMapping = mappingFile.LoadMappingData();
status.UpdateProgress(0.2f);
SegmentHeader = segment.ReadSegmentHeader();
FileHeader = header.ReadHeader(ref SegmentHeader);
log.Info(string.Format("Count info offset: {0:X8}", SegmentHeader.InfoOffset));
version = BigFileVersions.GetVersion(FileHeader.BigFileVersion);
fileUtil.BigFileVersion = version;
filesAndFolders.Version = version;
log.Info(string.Format("Version: {0:X4}", FileHeader.BigFileVersion));
rawFolderInfos = filesAndFolders.ReadFolderInfos(ref SegmentHeader, ref FileHeader);
rawFileInfos = filesAndFolders.ReadFileInfos(ref SegmentHeader, ref FileHeader);
fileUtil.BigFileVersion = version;
filesAndFolders.Version = version;
status.UpdateProgress(0.4f);
rootFolder = fileUtil.CreateRootFolderTree(rawFolderInfos);
fileMap = fileUtil.CreateFileMappingData(rootFolder, rawFileInfos);
fileUtil.MapFilesToFolders(rootFolder, fileMap);
UnpackedFolderMapAndFilesList folderAndFiles = fileUtil.CreateFolderTreeAndFilesListFromDirectory(new DirectoryInfo(directory.FullName + "\\" + BigFileConst.UNPACK_DIR), renamedMapping, fileMap);
rootFolder = folderAndFiles.folderMap[0];
rawFileInfos = folderAndFiles.filesList;
rawFolderInfos = folderAndFiles.foldersList;
status.UpdateProgress(0.6f);
fileMap = fileUtil.CreateFileMappingData(folderAndFiles.folderMap[0], folderAndFiles.filesList);
status.UpdateProgress(0.8f);
fileUtil.MapFilesToFolders(rootFolder, fileMap);
status.UpdateProgress(1.0f);
status.stopwatch.Stop();
log.Info("Unpacked bigfile loaded!");
log.Info(" Time taken: " + status.TimeTaken + "ms");
}
public void SaveMappingData(UnpackedRenamedFileMapping mapping)
{
log.Info("Saving renamed mapping to file: " + fileInfo.FullName);
using (FileStream fs = File.Create(fileInfo.FullName))
{
foreach (KeyValuePair <int, UnpackedRenamedFileMapping.RenamedFileMappingData> kvp in mapping.KeyMap)
{
fs.WriteByte(0x1C);
kvp.Value.WriteToStream(fs, tempBuffer);
}
}
}
public UnpackedRenamedFileMapping CreateRenamedFileMapping(BigFileFolder folder, UnpackedRenamedFileMapping mapping = null, Dictionary <string, int> fileRenameCounts = null)
{
bool isFirst = mapping == null;
if (isFirst)
{
mapping = new UnpackedRenamedFileMapping();
fileRenameCounts = new Dictionary <string, int>();
stopwatch.Reset();
stopwatch.Start();
}
foreach (BigFileFile file in folder.Files)
{
string fullName = file.FullFolderPath + file.Name;
//if (extensionsList != null && extensionsList.ContainsKey(file.FileInfo.FileType))
//{
// fullName += "." + extensionsList[file.FileInfo.FileType];
//}
//else
//{
fullName += string.Format(".{0}", file.FileInfo.FileType);
//}
string fullNameLower = fullName.ToLowerInvariant();
if (fileRenameCounts.ContainsKey(fullNameLower))
{
fileRenameCounts[fullNameLower]++;
fullName += "_" + fileRenameCounts[fullNameLower];
}
else
{
fileRenameCounts.Add(fullNameLower, 1);
}
log.Debug(file.Name + " remapped to " + fullName);
UnpackedRenamedFileMapping.RenamedFileMappingData data = new UnpackedRenamedFileMapping.RenamedFileMappingData()
{
Key = file.FileInfo.Key,
OriginalName = file.Name,
FileName = fullName
};
mapping[file.FileInfo.Key] = data;
}
foreach (BigFileFolder childFolder in folder.SubFolders)
{
CreateRenamedFileMapping(childFolder, mapping, fileRenameCounts);
}
if (isFirst)
{
stopwatch.Stop();
diagData.CreateRenamedFileMapping = stopwatch.ElapsedMilliseconds;
}
return(mapping);
}
public BigFileUnpackOperationStatus UnpackBigfile(BigFileUnpackOptions options)
{
if (options.Threads > MAX_UNPACK_THREADS)
{
log.Error(string.Format("Can't have more threads than the max! ({0} > {1})", options.Threads, MAX_UNPACK_THREADS));
log.Error(" Threads will be clamped to the max!");
options.Threads = MAX_UNPACK_THREADS;
}
log.Info("Unpacking a bigfile to directory: \"" + options.Directory.FullName + "\"");
options.Log(log);
if (!options.Directory.Exists)
{
log.Info("Directory does not exist, creating it...");
Directory.CreateDirectory(options.Directory.FullName);
}
GenerateYetiMetadataFile(options.Directory, bigFile);
DirectoryInfo unpackDir = new DirectoryInfo(options.Directory.FullName + "\\" + BigFileConst.UNPACK_DIR);
log.Info("Creating unpack dir: " + unpackDir.FullName);
Directory.CreateDirectory(unpackDir.FullName);
log.Info("Creating unpacked directories...");
CreateDirectoriesFromTree(unpackDir, bigFile.RootFolder);
log.Info("Creating renamed mapping file...");
UnpackedRenamedFileMapping renamedMapping = CreateRenamedFileMapping(bigFile.RootFolder);
UnpackedFileKeyMappingFile mappingFile = new UnpackedFileKeyMappingFile(options.Directory);
mappingFile.SaveMappingData(renamedMapping);
log.Info("Mapping file saved!");
stopwatch.Reset();
stopwatch.Start();
log.Info("Beginning extract...");
verifyAndResetThreads(options.Threads);
if ((options.Flags & BigFileFlags.UseThreading) != 0)
{
int dividedCount = bigFile.FileMap.FilesList.Length / options.Threads;
int dividedRemainder = bigFile.FileMap.FilesList.Length % options.Threads;
log.Info("Divided files into " + options.Threads + " pools of " + dividedCount + " with " + dividedRemainder + " left over (to be tacked onto the last!)");
List <UnpackThreadInfo> usingThreads = new List <UnpackThreadInfo>();
for (int i = 0; i < options.Threads; i++)
{
unpackThreads[i].options = options;
unpackThreads[i].bigFile = bigFile;
unpackThreads[i].fileMapping = renamedMapping;
unpackThreads[i].startIndex = i * dividedCount;
unpackThreads[i].count = dividedCount;
unpackThreads[i].threadID = i;
unpackThreads[i].OnWorkDoneCallback = internal_OnThreadFinished;
}
unpackThreads[options.Threads - 1].count += dividedRemainder; //add the remainder onto the last info
for (int i = 0; i < options.Threads; i++)
{
ThreadPool.QueueUserWorkItem(internal_UnpackFiles, unpackThreads[i]);
usingThreads.Add(unpackThreads[i]);
}
return(new BigFileUnpackOperationStatus(usingThreads));
}
else //use the function for threads even without one
{
unpackThreads[0].options = options;
unpackThreads[0].bigFile = bigFile;
unpackThreads[0].fileMapping = renamedMapping;
unpackThreads[0].startIndex = 0;
unpackThreads[0].count = bigFile.FileMap.FilesList.Length;
unpackThreads[0].threadID = 0;
unpackThreads[0].OnWorkDoneCallback = internal_OnThreadFinished;
internal_UnpackFiles(unpackThreads[0]); //teehee
diagData.WriteUnpackedFiles = stopwatch.ElapsedMilliseconds;
stopwatch.Stop();
log.Info("Extract complete!");
diagData.DebugLog(log);
log.Info("Unpack complete!");
return(null);
}
}
public UnpackedFolderMapAndFilesList CreateFolderTreeAndFilesListFromDirectory(DirectoryInfo dir, UnpackedRenamedFileMapping mapping, FileMappingData defaultMappingData)
{
log.Info("Creating folder tree and files list from directory " + dir.FullName);
IBigFileFileInfo[] fileInfos = new IBigFileFileInfo[mapping.KeyMap.Count];
List <IBigFileFolderInfo> folderInfos = new List <IBigFileFolderInfo>();
Dictionary <short, BigFileFolder> folderMap = new Dictionary <short, BigFileFolder>();
short folderCount = 0;
int fileCount = 0;
Dictionary <string, UnpackedRenamedFileMapping.RenamedFileMappingData> temp = new Dictionary <string, UnpackedRenamedFileMapping.RenamedFileMappingData>(mapping.RenamedMap);
BigFileFolder recursion(DirectoryInfo directory, string dirName, BigFileFolder parentFolder)
{
IBigFileFolderInfo folderInfo = version.CreateFolderInfo();
folderInfo.Unknown_01 = 0;
folderInfo.PreviousFolder = parentFolder != null ? parentFolder.FolderIndex : (short)-1;
folderInfo.NextFolder = -1;
folderInfo.Unknown_02 = 0;
folderInfo.Name = parentFolder == null ? //oh my lawdy what is this EDIT 4/5/2018: what the f**k
"/".EncodeToBadString(length: 54) :
directory.Name.EncodeToBadString(length: 54);
folderInfos.Add(folderInfo);
BigFileFolder thisFolder = new BigFileFolder(folderCount, folderInfo, folderMap);
folderMap.Add(folderCount, thisFolder);
foreach (FileInfo file in directory.GetFiles())
{
if (file.Name.EndsWith(".header"))
{
continue;
}
string fileName = dirName + "//" + file.Name;
UnpackedRenamedFileMapping.RenamedFileMappingData mappingData = mapping[fileName];
temp.Remove(fileName);
IBigFileFileInfo fileInfo = version.CreateFileInfo();
defaultMappingData[mappingData.Key].FileInfo.Copy(fileInfo);
fileInfo.Key = mappingData.Key;
//fileInfo.FileNumber = fileCount;
fileInfo.Name = mappingData.OriginalName.EncodeToBadString(length: 60);
fileInfo.Folder = folderCount;
log.Debug("Add file " + file.FullName);
fileInfos[fileCount] = fileInfo;
fileCount++;
}
folderCount++;
foreach (DirectoryInfo dirInfo in directory.GetDirectories())
{
if (parentFolder == null) //ONLY THE FIRST RECURSION, PREVENTS ADDING WRONG FOLDERS WHEN PACKING
{
thisFolder.SubFolders.Add(recursion(dirInfo, dirInfo.Name, thisFolder));
}
else
{
thisFolder.SubFolders.Add(recursion(dirInfo, dirName + "/" + dirInfo.Name, thisFolder));
}
}
return(thisFolder);
}
recursion(dir, "", null);
if (fileCount != mapping.KeyMap.Count)
{
log.Error(string.Format("Missing {0} files!", temp.Count));
foreach (KeyValuePair <string, UnpackedRenamedFileMapping.RenamedFileMappingData> kvp in temp)
{
log.Error(string.Format(" >{0}", kvp.Value.FileName));
}
}
return(new UnpackedFolderMapAndFilesList()
{
folderMap = folderMap,
filesList = fileInfos,
foldersList = folderInfos.ToArray(),
});
}
