public void AddFileReferencesToFile(BigFileFile file, int[] header)
{
log.Debug("Loading file references for file: " + file.Name);
log.Debug(" Reference count: " + header.Length.ToString());
if (file.FileReferences != null)
{
log.Debug("File {0} (key{1:X8}) already has references loaded!", file.Name, file.FileInfo.Key);
return;
}
BigFileFile[] references = new BigFileFile[header.Length];
for (int i = 0; i < header.Length; i++)
{
references[i] = file.MappingData[header[i]];
}
file.FileReferences = references;
}
private BigFilePackOperationStatus internal_ThreadedPack(BigFilePackOptions options)
{
//set up threads
int dividedCount = bigFile.FileMap.FilesList.Length / options.Threads;
int dividedRemainder = bigFile.FileMap.FilesList.Length % options.Threads;
log.Info("Divided files into " + options.Threads + " pools of " + dividedCount + " with " + dividedRemainder + " left over (to be tacked onto the last!)");
BigFileFile[] files = new BigFileFile[bigFile.FileMap.FilesList.Length];
Array.Copy(bigFile.FileMap.FilesList, files, files.Length);
Array.Sort(files,
(fileA, fileB) =>
{
return(fileA.FileInfo.Offset.CompareTo(fileB.FileInfo.Offset));
});
List <BigFilePackInfo> infos = new List <BigFilePackInfo>();
for (int i = 0; i < options.Threads; i++)
{
packInfos[i].Options = options;
packInfos[i].startIndex = i * dividedCount;
packInfos[i].count = dividedCount;
packInfos[i].bigFile = bigFile;
packInfos[i].filesList = files;
}
packInfos[options.Threads - 1].count += dividedRemainder;
packInfos[options.Threads - 1].OnCompleted = internal_OnPackFinished; //the last thread gets the job of stitching together the chunks
for (int i = 0; i < options.Threads; i++)
{
ThreadPool.QueueUserWorkItem(internal_GenBigFileChunk, packInfos[i]);
infos.Add(packInfos[i]);
}
return(new BigFilePackOperationStatus(infos));
}
public void MapFilesToFolders(BigFileFolder rootFolder, FileMappingData mapping)
{
log.Info("Mapping files to folders...");
stopwatch.Reset();
stopwatch.Start();
for (int i = 0; i < mapping.FilesList.Length; i++)
{
BigFileFile file = mapping.FilesList[i];
if (file != null)
{
BigFileFolder folder = rootFolder.FolderMap[mapping.FilesList[i].FileInfo.Folder];
if (folder != null)
{
folder.Files.Add(mapping.FilesList[i]);
}
}
}
diagData.MapFilesToFolders = stopwatch.ElapsedMilliseconds;
stopwatch.Reset();
}
private void internal_UnpackFiles(object state)
{
UnpackThreadInfo info = state as UnpackThreadInfo;
info.isUnpacking = true;
info.stopwatch.Reset();
info.stopwatch.Start();
//int segmentDataOffset = info.bigFile.FileUtil.CalculateDataOffset(ref info.bigFile.SegmentHeader, ref info.bigFile.FileHeader);
//byte[] buffer = info.buffers[4];
BigFileFile[] files = new BigFileFile[info.count];
Array.Copy(info.bigFile.FileMap.FilesList, info.startIndex, files, 0, info.count);
IEnumerator <int[]> headers = info.bigFile.FileReader.ReadAllHeaders(files, info.buffers, info.options.Flags).GetEnumerator();
IEnumerator <int> data = info.bigFile.FileReader.ReadAllData(files, info.buffers, info.options.Flags).GetEnumerator();
for (int i = 0; i < files.Length; i++)
{
info.progress = i;
headers.MoveNext();
data.MoveNext();
log.Info("Unpacking file {0}", files[i].Name);
//********************************************//
//DON'T FORGET THE ******* UNPACK SUBDIRECTORY//
//********************************************//
string dataFileName = info.options.Directory.FullName + "\\"
+ BigFileConst.UNPACK_DIR + "\\"
+ info.fileMapping[files[i].FileInfo.Key].FileName;
string headerFileName = dataFileName + BigFileConst.UNPACKED_HEADER_FILE_EXTENSION;
using (FileStream dataFS = File.Create(dataFileName))
using (FileStream headerFS = File.Create(headerFileName))
{
int size = data.Current;
int[] header = headers.Current;
if (size != -1)
{
int headerCount = header.Length;
dataFS.Write(info.buffers[size], 0, size);
headerFS.Write(headerCount.ToByteArray(info.buffers[4]), 0, 4);
for (int j = 0; j < headerCount; j++)
{
headerFS.Write(header[j].ToByteArray(info.buffers[4]), 0, 4);
if (header[j] == 0)
{
log.Error("WTF");
}
}
}
else
{
log.Error("Can't unpack file {0} because size is -1", files[i].Name);
}
}
}
//int index = -1;
//foreach (int size in info.bigFile.FileReader.ReadAllRaw(files, info.buffers, info.options.Flags))
//{
// index++;
// info.progress = index;
// log.Info("Unpacking file {0}", files[index].Name);
// //********************************************//
// //DON'T FORGET THE ******* UNPACK SUBDIRECTORY//
// //********************************************//
// string dataFileName = info.options.Directory.FullName + "\\"
// + BigFileConst.UNPACK_DIR + "\\"
// + info.fileMapping[files[index].FileInfo.Key].FileName;
// string headerFileName = dataFileName + BigFileConst.UNPACKED_HEADER_FILE_EXTENSION;
// IEnumerable<int> p = info.bigFile.FileReader.ReadAllData(files, info.buffers, info.options.Flags);
// using (FileStream dataFS = File.Create(dataFileName))
// using (FileStream headerFS = File.Create(headerFileName))
// {
// if (size != -1)
// {
// int headerCount = BitConverter.ToInt32(info.buffers[size], 0);
// int dataOffset = headerCount * 4 + 4;
// if (dataOffset > size)
// {
// log.Error("Hmmm... something's wrong here");
// headerFS.Write(info.buffers[size], 0, size);
// }
// else
// {
// dataFS.Write(info.buffers[size], dataOffset, size - dataOffset);
// headerFS.Write(info.buffers[size], 0, dataOffset);
// }
// }
// else
// {
// log.Error("Can't unpack file {0} because size is -1", files[index].Name);
// }
// }
//}
log.Info("Unpack thread (ID:" + info.threadID + ") finished work!");
info.isUnpacking = false;
info.stopwatch.Stop();
info.OnWorkDoneCallback.Invoke(info);
}
public FileMappingData CreateFileMappingData(BigFileFolder rootFolder, IBigFileFileInfo[] fileInfos)
{
log.Info("Creating mapping data...");
log.Info("Creating files list... Count: " + fileInfos.Length);
stopwatch.Reset();
stopwatch.Start();
FileMappingData mappingData = new FileMappingData();
BigFileFile[] filesList = new BigFileFile[fileInfos.Length];
for (int i = 0; i < fileInfos.Length; i++)
{
BigFileFile newFile = null;
if (fileInfos[i] != null)
{
newFile = new BigFileFile(fileInfos[i], rootFolder.FolderMap[fileInfos[i].Folder]);
newFile.MappingData = mappingData;
filesList[i] = newFile;
}
else
{
log.Error(string.Format("File info at index {0} is null!", i));
}
}
diagData.CreateFilesList = stopwatch.ElapsedMilliseconds;
log.Info("List created!");
log.Info("Creating file mappings...");
stopwatch.Reset();
stopwatch.Start();
Dictionary <int, BigFileFile> fileKeyMapping = new Dictionary <int, BigFileFile>();
for (int i = 0; i < filesList.Length; i++)
{
if (fileInfos[i]?.Name == null)
{
continue;
}
if (!fileKeyMapping.ContainsKey(fileInfos[i].Key))
{
fileKeyMapping.Add(fileInfos[i].Key, filesList[i]);
}
else
{
log.Error("File key mapping already contains key " + fileInfos[i].Key + " (File: " + filesList[i].Name + ")");
}
if (filesList[i].FileInfo.FileNumber == -1)
{
log.Debug(string.Format("File number is -1! (key:{0:X8}) (offset:{1:X8})", fileInfos[i].Key, fileInfos[i].Offset));
}
}
log.Info("Mappings created!");
foreach (KeyValuePair <short, BigFileFolder> kvp in rootFolder.FolderMap)
{
kvp.Value.FileMap = mappingData;
}
diagData.CreateKeyAndNumMappings = stopwatch.ElapsedMilliseconds;
stopwatch.Reset();
mappingData.FilesList = filesList;
mappingData.KeyMapping = fileKeyMapping;
log.Info("mappingData count: {0}", mappingData.FilesList.Length);
return(mappingData);
}
private void internal_GenBigFileChunk(object state)
{
BigFilePackInfo info = state as BigFilePackInfo;
info.isPacking = true;
info.diag.StartStopwatch();
string tempDir = Environment.CurrentDirectory + BigFileConst.PACK_STAGING_DIR;
Directory.CreateDirectory(tempDir);
string chunkFileName = tempDir + info.Options.BigFileName + ".chunk" + info.ThreadID.ToString();
string metadataFilename = tempDir + info.Options.BigFileName + ".meta" + info.ThreadID.ToString();
log.Info("Generating bigfile chunk: " + chunkFileName);
using (FileStream chunkFS = new FileStream(chunkFileName, FileMode.Create, FileAccess.Write))
using (FileStream metaFS = new FileStream(metadataFilename, FileMode.Create, FileAccess.Write))
{
//fill 8 bytes to be filled with number of files chunked and final size later
metaFS.Write(BitConverter.GetBytes((long)0), 0, 8);
BigFileFile[] filesToWrite = new BigFileFile[info.count];
Array.Copy(info.filesList, info.startIndex, filesToWrite, 0, info.count);
log.Error("Thread ID {0} - First file is {1}", info.ThreadID, filesToWrite[0].Name);
log.Error("Thread ID {0} - Last file is {1}", info.ThreadID, filesToWrite[filesToWrite.Length - 1].Name);
BigFileFile currFile = null;
int index = -1;
foreach (int size in bigFile.FileReader.ReadAllRaw(filesToWrite, info.IOBuffers, info.Options.Flags))
{
index++;
currFile = filesToWrite[index];
int size2 = size;
//if (currFile.FileInfo.FileType == 0x3c)
//{
// log.Error("Skipped file {0}", currFile.Name);
// index++;
// continue;
//}
log.Debug("Packing file {0}, size: {1}, ZIP: {2}", currFile.Name, size, currFile.FileInfo.ZIP);
if (size2 < 0)
{
metaFS.Write(BitConverter.GetBytes(currFile.FileInfo.FileNumber), 0, 4);
metaFS.Write(BitConverter.GetBytes(currFile.FileInfo.Key), 0, 4);
metaFS.Write(BitConverter.GetBytes(-1), 0, 4);
log.Error("WAIT WHAT");
}
else
{
//write the file number, key, and offset to metadata file
metaFS.Write(BitConverter.GetBytes(currFile.FileInfo.FileNumber), 0, 4);
metaFS.Write(BitConverter.GetBytes(currFile.FileInfo.Key), 0, 4);
metaFS.Write(BitConverter.GetBytes((int)chunkFS.Position), 0, 4);
if (currFile.FileInfo.ZIP == 1 && (info.Options.Flags & BigFileFlags.Compress) != 0)
{
int sizePos = (int)chunkFS.Position;
chunkFS.Write(info.IOBuffers[8], 0, 8); //write 8 bytes of garbage to fill the space for decompressed and compressed size
using (ZlibStream zs = new ZlibStream(chunkFS, Ionic.Zlib.CompressionMode.Compress, true))
{
zs.Write(info.IOBuffers[size], 0, size);
}
int newPos = (int)chunkFS.Position;
int remainder = ((((newPos - sizePos) - 1) / 8 + 1) * 8) - (newPos - sizePos);
for (int i = 0; i < remainder; i++)
{
chunkFS.WriteByte(0x00);
}
newPos = (int)chunkFS.Position;
int compressedSize = newPos - sizePos - 4;
//go back to the file offset and write the compressed and decompressed sizes
chunkFS.Seek(sizePos, SeekOrigin.Begin);
chunkFS.Write(BitConverter.GetBytes(compressedSize), 0, 4);
chunkFS.Write(BitConverter.GetBytes(size), 0, 4);
chunkFS.Seek(newPos, SeekOrigin.Begin);
}
else
{
int sizePos = (int)chunkFS.Position;
chunkFS.Write(BitConverter.GetBytes(size), 0, 4);
chunkFS.Write(info.IOBuffers[size], 0, size);
int remainder = (((((int)chunkFS.Position - sizePos) - 1) / 8 + 1) * 8) - ((int)chunkFS.Position - sizePos);
for (int i = 0; i < remainder; i++)
{
chunkFS.WriteByte(0x00);
}
}
}
info.filesChunked++;
}
//write number of files chunked and final file size
metaFS.Seek(0, SeekOrigin.Begin);
metaFS.Write(BitConverter.GetBytes(info.filesChunked), 0, 4);
metaFS.Write(BitConverter.GetBytes(chunkFS.Length), 0, 4);
WinMessageBox.Show(chunkFS.Length.ToString(), "HA", WinMessageBoxFlags.btnOkay);
}
info.isPacking = false;
info.diag.StopStopwatch();
log.Info("Thread (ID: {0}) finished chunking work, time: {1,5}s", info.ThreadID, info.diag.StopwatchTime / 1000);
if (info.OnCompleted != null)
{
info.OnCompleted.Invoke(info);
}
}