public async Task <int> Open(Action <string> info)
{
using (MyDbContext _context = new MyDbContext(_settings.ConnectionString))
{
_context.Database.EnsureDeleted();
_context.Database.EnsureCreated();
}
IExtract extract = null;
switch (_settings.ExtractMethod)
{
case DataMethods.File:
{
var extract_C = new FileExtract(_dataBacklinkRowService, _dataListedInfoService, _settings)
{
ContainsExtraInfo = false
};
break;
}
case DataMethods.GoogleDocs:
{
extract = new SpreadSheetExtract(_dataBacklinkRowService, _dataListedInfoService, _settings);
break;
}
}
int i = await extract.Extract();
_settings.Positions = i;
_dbQueue = new DbQueue(_settings.ConnectionString, _settings.PlainHttpThreads, i);
DbQueue.Info = info;
return(i);
}
private async Task ExtractFilesAsync()
{
while (filesToExtract.Count > 0)
{
cancellationTokenSource.Token.ThrowIfCancellationRequested();
var packageFile = filesToExtract.Dequeue();
Invoke((Action)(() =>
{
extractProgressBar.Value = 100 - (int)((filesToExtract.Count / (float)initialFileCount) * 100.0f);
extractStatusLabel.Text = $"Extracting {packageFile.GetFullPath()}";
}));
var filePath = Path.Combine(path, packageFile.GetFullPath());
package.ReadEntry(packageFile, out var output, false);
Directory.CreateDirectory(Path.GetDirectoryName(filePath));
if (decompile && filePath.EndsWith("_c", StringComparison.Ordinal))
{
using (var resource = new Resource
{
FileName = filePath,
})
using (var memory = new MemoryStream(output))
{
try
{
resource.Read(memory);
var extension = FileExtract.GetExtension(resource);
if (extension == null)
{
filePath = filePath.Substring(0, filePath.Length - 2);
}
else
{
filePath = Path.ChangeExtension(filePath, extension);
}
output = FileExtract.Extract(resource).ToArray();
}
catch (Exception e)
{
Console.WriteLine($"Failed to extract '{packageFile.GetFullPath()}' - {e.Message}");
continue;
}
}
}
var stream = new FileStream(filePath, FileMode.Create);
await using (stream.ConfigureAwait(false))
{
await stream.WriteAsync(output, cancellationTokenSource.Token).ConfigureAwait(false);
}
}
}
static void InsertData(FileExtract fE, Source src)
{
string dataExtracted =
$"INSERT INTO t_file " +
$"( " +
$"{fE.Path}, " +
$"{fE.Name}, " +
$"{fE.Author}, " +
$"{fE.Date}, " +
$"{fE.LastModifiedDate}, " +
$"{researchNumberReference}, " +
$"{src.Name}, " +
$"(SELECT idSource FROM t_source WHERE {src.Name} = t_source.sourceName), " +
$"(SELECT idFile FROM t_file WHERE {fE.Path} = t_source.filePath) " +
$")" +
$"SELECT " +
$"filePath, " +
$"fileName, " +
$"fileAuthor, " +
$"fileDate, " +
$"fileModificationDate, " +
$"referenceNumber," +
$"t_source.sourceName, " +
$"t_fileSourceBind.idSource, " +
$"t_fileSourceBind.idFile" +
$"FROM t_file" +
$"INNER JOIN t_fileSourceBind ON t_file.idFile = t_fileSourceBind.idFile" +
$"INNER JOIN t_source ON t_fileSourceBind.idSource = t_source.idSource)";
}
public static void InsertDFile(FileExtract fE)
{
ConsoleWriter.WriteNotification($"INSERTING FILE {fE.Name} DATA-----------------------------------------------------------");
string dataExtracted =
$"INSERT INTO t_file " +
$"(" +
$"filePath, " +
$"fileName, " +
$"fileAuthor, " +
$"fileDate, " +
$"fileModificationDate, " +
$"referenceNumber " +
$")" +
$"VALUES" +
$"( " +
$"\"{fE.Path}\", " +
$"\"{fE.Name}\", " +
$"\"{fE.Author}\", " +
$"\"{Convert.ToString(fE.Date)}\", " +
$"\"{Convert.ToString(fE.LastModifiedDate)}\", " +
$"\"{researchNumberReference}\" " +
$")";
sqlite_cmd = sqlite_conn.CreateCommand();
sqlite_cmd.CommandText = dataExtracted;
sqlite_cmd.ExecuteNonQuery();
Sanitize(researchNumberReference);
}
public void SetTextExtract(FileExtract value)
{
_extractPosition = value;
OnPropertyChanged(ReflectionUtils.GetPropertyName(this, x => x.DisplayText));
OnPropertyChanged(ReflectionUtils.GetPropertyName(this, x => x.TextBeforeMatch));
OnPropertyChanged(ReflectionUtils.GetPropertyName(this, x => x.MatchText));
OnPropertyChanged(ReflectionUtils.GetPropertyName(this, x => x.TextAfterMatch));
}
private void DumpVPK(Package package, string type)
{
if (ExtFilterList != null && !ExtFilterList.Contains(type))
{
return;
}
if (!package.Entries.ContainsKey(type))
{
Console.WriteLine("There are no files of type \"{0}\".", type);
return;
}
var fileLoader = new BasicVpkFileLoader(package);
var entries = package.Entries[type];
foreach (var file in entries)
{
var extension = type;
var filePath = FixPathSlashes(file.GetFullPath());
if (FileFilter != null && !filePath.StartsWith(FileFilter, StringComparison.Ordinal))
{
continue;
}
if (OutputFile != null)
{
if (CachedManifest && OldPakManifest.TryGetValue(filePath, out var oldCrc32) && oldCrc32 == file.CRC32)
{
continue;
}
OldPakManifest[filePath] = file.CRC32;
}
Console.WriteLine("\t[archive index: {0:D3}] {1}", file.ArchiveIndex, filePath);
package.ReadEntry(file, out var output);
if (type.EndsWith("_c", StringComparison.Ordinal) && Decompile)
{
using var resource = new Resource();
using var memory = new MemoryStream(output);
try
{
resource.Read(memory);
extension = FileExtract.GetExtension(resource);
if (extension == null)
{
extension = type.Substring(0, type.Length - 2);
}
// TODO: Hook this up in FileExtract
if (resource.ResourceType == ResourceType.Mesh || resource.ResourceType == ResourceType.Model)
{
var outputExtension = GltfExportFormat;
var outputFile = Path.Combine(OutputFile, Path.ChangeExtension(filePath, outputExtension));
Directory.CreateDirectory(Path.GetDirectoryName(outputFile));
var exporter = new GltfModelExporter
{
ExportMaterials = GltfExportMaterials,
ProgressReporter = new ConsoleProgressReporter(),
FileLoader = fileLoader
};
if (resource.ResourceType == ResourceType.Mesh)
{
exporter.ExportToFile(file.GetFileName(), outputFile, new Mesh(resource));
}
else if (resource.ResourceType == ResourceType.Model)
{
exporter.ExportToFile(file.GetFileName(), outputFile, (Model)resource.DataBlock);
}
continue;
}
output = FileExtract.Extract(resource).ToArray();
}
catch (Exception e)
{
File.AppendAllText("exceptions.txt", $"---------------\nFile: {filePath}\nException: {e}\n\n");
lock (ConsoleWriterLock)
{
Console.ForegroundColor = ConsoleColor.DarkRed;
Console.WriteLine("\t" + e.Message + " on resource type " + type + ", extracting as-is");
Console.ResetColor();
}
}
}
if (OutputFile != null)
{
if (type != extension)
{
filePath = Path.ChangeExtension(filePath, extension);
}
DumpFile(filePath, output);
}
}
}
private void ProcessFile(string path, Stream stream)
{
var resource = new Resource();
try
{
resource.Read(stream);
var extension = FileExtract.GetExtension(resource);
if (extension == null)
{
extension = Path.GetExtension(path);
if (extension.EndsWith("_c", StringComparison.Ordinal))
{
extension = extension.Substring(0, extension.Length - 2);
}
}
if (CollectStats)
{
string id = string.Format("{0}_{1}", resource.ResourceType, resource.Version);
string info = string.Empty;
switch (resource.ResourceType)
{
case ResourceType.Texture:
var texture = (Texture)resource.DataBlock;
info = texture.Format.ToString();
texture.GenerateBitmap();
break;
case ResourceType.Sound:
info = ((Sound)resource.DataBlock).SoundType.ToString();
break;
}
if (!string.IsNullOrEmpty(info))
{
id = string.Concat(id, "_", info);
}
lock (stats)
{
if (stats.ContainsKey(id))
{
if (stats[id].Count++ < 10)
{
stats[id].FilePaths.Add(path);
}
}
else
{
stats.Add(id, new ResourceStat(resource, info, path));
}
}
if (resource.EditInfo != null && resource.EditInfo.Structs.ContainsKey(ResourceEditInfo.REDIStruct.SpecialDependencies))
{
lock (uniqueSpecialDependancies)
{
foreach (var dep in ((ValveResourceFormat.Blocks.ResourceEditInfoStructs.SpecialDependencies)resource.EditInfo.Structs[ResourceEditInfo.REDIStruct.SpecialDependencies]).List)
{
uniqueSpecialDependancies[string.Format("{0} \"{1}\"", dep.CompilerIdentifier, dep.String)] = path;
}
}
}
foreach (var block in resource.Blocks)
{
block.ToString();
}
}
if (OutputFile != null)
{
var data = FileExtract.Extract(resource);
var filePath = Path.ChangeExtension(path, extension);
if (IsInputFolder)
{
// I bet this is prone to breaking, is there a better way?
filePath = filePath.Remove(0, InputFile.TrimEnd(Path.DirectorySeparatorChar).Length + 1);
}
else
{
filePath = Path.GetFileName(filePath);
}
DumpFile(filePath, data, !IsInputFolder);
}
}
catch (Exception e)
{
File.AppendAllText("exceptions.txt", string.Format("---------------\nFile: {0}\nException: {1}\n\n", path, e));
lock (ConsoleWriterLock)
{
Console.ForegroundColor = ConsoleColor.Cyan;
Console.WriteLine(e);
Console.ResetColor();
}
}
if (CollectStats)
{
return;
}
//Console.WriteLine("\tInput Path: \"{0}\"", args[fi]);
//Console.WriteLine("\tResource Name: \"{0}\"", "???");
//Console.WriteLine("\tID: {0:x16}", 0);
lock (ConsoleWriterLock)
{
// Highlight resource type line if undetermined
if (resource.ResourceType == ResourceType.Unknown)
{
Console.ForegroundColor = ConsoleColor.Cyan;
}
Console.WriteLine("\tResource Type: {0} [Version {1}] [Header Version: {2}]", resource.ResourceType, resource.Version, resource.HeaderVersion);
Console.ResetColor();
}
Console.WriteLine("\tFile Size: {0} bytes", resource.FileSize);
Console.WriteLine(Environment.NewLine);
if (resource.ContainsBlockType(BlockType.RERL))
{
Console.WriteLine("--- Resource External Refs: ---");
Console.WriteLine("\t{0,-16} {1,-48}", "Id:", "Resource Name:");
foreach (var res in resource.ExternalReferences.ResourceRefInfoList)
{
Console.WriteLine("\t{0:X16} {1,-48}", res.Id, res.Name);
}
}
else
{
Console.WriteLine("--- (No External Resource References Found)");
}
Console.WriteLine(Environment.NewLine);
// TODO: Resource Deferred Refs:
Console.WriteLine("--- (No Deferred Resource References Found)");
Console.WriteLine(Environment.NewLine);
Console.WriteLine("--- Resource Blocks: Count {0} ---", resource.Blocks.Count);
foreach (var block in resource.Blocks)
{
Console.WriteLine("\t-- Block: {0,-4} Size: {1,-6} bytes [Offset: {2,6}]", block.Type, block.Size, block.Offset);
}
if (PrintAllBlocks || !string.IsNullOrEmpty(BlockToPrint))
{
Console.WriteLine(Environment.NewLine);
foreach (var block in resource.Blocks)
{
if (!PrintAllBlocks && BlockToPrint != block.Type.ToString())
{
continue;
}
Console.WriteLine("--- Data for block \"{0}\" ---", block.Type);
Console.WriteLine(block.ToString());
}
}
}
private void ExportToolStripMenuItem_Click(object sender, EventArgs e)
{
//ToolTipText is the full filename
var fileName = ((ToolStripMenuItem)sender).ToolTipText;
var tag = ((ToolStripMenuItem)sender).Tag as ExportData;
var resource = tag.Resource;
Console.WriteLine($"Export requested for {fileName}");
string[] extensions = null;
switch (resource.ResourceType)
{
case ResourceType.Sound:
//WAV or MP3
extensions = new[] { ((Sound)resource.DataBlock).SoundType.ToString().ToLower() };
break;
case ResourceType.Texture:
extensions = new[] { "png" };
break;
case ResourceType.PanoramaLayout:
extensions = new[] { "xml", "vxml" };
break;
case ResourceType.PanoramaScript:
extensions = new[] { "js", "vjs" };
break;
case ResourceType.PanoramaStyle:
extensions = new[] { "css", "vcss" };
break;
case ResourceType.Mesh:
extensions = new[] { "obj" };
break;
}
//Did we find a format we like?
if (extensions != null)
{
var dialog = new SaveFileDialog
{
FileName = Path.GetFileName(Path.ChangeExtension(fileName, extensions[0])),
InitialDirectory = Settings.Config.SaveDirectory,
DefaultExt = extensions[0],
};
var filter = string.Empty;
foreach (var extension in extensions)
{
filter += $"{extension} files (*.{extension})|*.{extension}|";
}
//Remove the last |
dialog.Filter = filter.Substring(0, filter.Length - 1);
var result = dialog.ShowDialog();
if (result == DialogResult.OK)
{
Settings.Config.SaveDirectory = Path.GetDirectoryName(dialog.FileName);
Settings.Save();
using (var stream = dialog.OpenFile())
{
// TODO: move this to FileExtract/VRF too
if (resource.ResourceType == ResourceType.Mesh)
{
using (var objStream = new StreamWriter(stream))
using (var mtlStream = new StreamWriter(Path.ChangeExtension(dialog.FileName, "mtl")))
{
MeshWriter.WriteObject(objStream, mtlStream, Path.GetFileNameWithoutExtension(dialog.FileName), resource);
}
foreach (var texture in tag.VrfGuiContext.MaterialLoader.LoadedTextures)
{
Console.WriteLine($"Exporting texture for mesh: {texture}");
var textureResource = tag.VrfGuiContext.LoadFileByAnyMeansNecessary(texture + "_c");
var textureImage = SKImage.FromBitmap(((Texture)textureResource.DataBlock).GenerateBitmap());
using (var texStream = new FileStream(Path.Combine(Path.GetDirectoryName(dialog.FileName), Path.GetFileNameWithoutExtension(texture) + ".png"), FileMode.Create, FileAccess.Write))
using (var data = textureImage.Encode(SKEncodedImageFormat.Png, 100))
{
data.SaveTo(texStream);
}
}
}
else
{
var data = FileExtract.Extract(resource).ToArray();
stream.Write(data, 0, data.Length);
}
}
}
}
Console.WriteLine($"Export requested for {fileName} Complete");
}
private void ExtractFiles(object sender, bool decompile)
{
TreeViewWithSearchResults.TreeViewPackageTag package = null;
TreeNode selectedNode = null;
// the context menu can come from a TreeView or a ListView depending on where the user clicked to extract
// each option has a difference in where we can get the values to extract
if (((ContextMenuStrip)((ToolStripMenuItem)sender).Owner).SourceControl is TreeView)
{
var tree = ((ContextMenuStrip)((ToolStripMenuItem)sender).Owner).SourceControl as TreeView;
selectedNode = tree.SelectedNode;
package = tree.Tag as TreeViewWithSearchResults.TreeViewPackageTag;
}
else if (((ContextMenuStrip)((ToolStripMenuItem)sender).Owner).SourceControl is ListView)
{
var listView = ((ContextMenuStrip)((ToolStripMenuItem)sender).Owner).SourceControl as ListView;
selectedNode = listView.SelectedItems[0].Tag as TreeNode;
package = listView.Tag as TreeViewWithSearchResults.TreeViewPackageTag;
}
if (selectedNode.Tag.GetType() == typeof(PackageEntry))
{
// We are a file
var file = selectedNode.Tag as PackageEntry;
var fileName = file.GetFileName();
package.Package.ReadEntry(file, out var output);
if (decompile && fileName.EndsWith("_c", StringComparison.Ordinal))
{
using (var resource = new Resource())
using (var memory = new MemoryStream(output))
{
resource.Read(memory);
var extension = FileExtract.GetExtension(resource);
if (extension == null)
{
fileName = fileName.Substring(0, fileName.Length - 2);
}
else
{
fileName = Path.ChangeExtension(fileName, extension);
}
output = FileExtract.Extract(resource).ToArray();
}
}
var dialog = new SaveFileDialog
{
InitialDirectory = Settings.Config.SaveDirectory,
Filter = "All files (*.*)|*.*",
FileName = fileName,
};
var userOK = dialog.ShowDialog();
if (userOK == DialogResult.OK)
{
Settings.Config.SaveDirectory = Path.GetDirectoryName(dialog.FileName);
Settings.Save();
using (var stream = dialog.OpenFile())
{
stream.Write(output, 0, output.Length);
}
}
}
else
{
//We are a folder
var dialog = new FolderBrowserDialog();
if (dialog.ShowDialog() == DialogResult.OK)
{
var extractDialog = new ExtractProgressForm(package.Package, selectedNode, dialog.SelectedPath, decompile);
extractDialog.ShowDialog();
}
}
}
private void DumpVPK(Package package, string type)
{
if (ExtFilterList != null && !ExtFilterList.Contains(type))
{
return;
}
if (!package.Entries.ContainsKey(type))
{
Console.WriteLine("There are no files of type \"{0}\".", type);
return;
}
var entries = package.Entries[type];
foreach (var file in entries)
{
var extension = type;
var filePath = FixPathSlashes(file.GetFullPath());
if (FileFilter != null && !filePath.StartsWith(FileFilter, StringComparison.Ordinal))
{
continue;
}
if (OutputFile != null)
{
if (CachedManifest && OldPakManifest.TryGetValue(filePath, out var oldCrc32) && oldCrc32 == file.CRC32)
{
continue;
}
OldPakManifest[filePath] = file.CRC32;
}
Console.WriteLine("\t[archive index: {0:D3}] {1}", file.ArchiveIndex, filePath);
package.ReadEntry(file, out var output);
if (type.EndsWith("_c", StringComparison.Ordinal) && Decompile)
{
using (var resource = new Resource())
{
using (var memory = new MemoryStream(output))
{
try
{
resource.Read(memory);
extension = FileExtract.GetExtension(resource);
if (extension == null)
{
extension = type.Substring(0, type.Length - 2);
}
output = FileExtract.Extract(resource).ToArray();
}
catch (Exception e)
{
lock (ConsoleWriterLock)
{
Console.ForegroundColor = ConsoleColor.DarkRed;
Console.WriteLine("\t" + e.Message + " on resource type " + type + ", extracting as-is");
Console.ResetColor();
}
}
}
}
}
if (OutputFile != null)
{
if (type != extension)
{
filePath = Path.ChangeExtension(filePath, extension);
}
DumpFile(filePath, output);
}
}
}
public void SetTextExtract(FileExtract value)
{
_fileExtract = value;
OnPropertyChanged("DisplayText");
}
public static void Export(string fileName, ExportData exportData)
{
var resource = exportData.Resource;
var extension = FileExtract.GetExtension(resource);
if (extension == null)
{
Console.WriteLine($"Export for \"{fileName}\" has no suitable extension");
return;
}
var filter = $"{extension} file|*.{extension}";
if (resource.ResourceType == ResourceType.Mesh || resource.ResourceType == ResourceType.Model)
{
if (exportData.FileType == ExportFileType.GLB)
{
extension = "glb";
filter = $"GLB file|*.glb|{filter}";
}
else
{
extension = "gltf";
filter = $"glTF file|*.gltf|{filter}";
}
}
var dialog = new SaveFileDialog
{
FileName = Path.GetFileName(Path.ChangeExtension(fileName, extension)),
InitialDirectory = Settings.Config.SaveDirectory,
DefaultExt = extension,
Filter = filter,
};
var result = dialog.ShowDialog();
if (result != DialogResult.OK)
{
Console.WriteLine($"Export for \"{fileName}\" cancelled");
return;
}
Console.WriteLine($"Export for \"{fileName}\" started to \"{extension}\"");
Settings.Config.SaveDirectory = Path.GetDirectoryName(dialog.FileName);
Settings.Save();
var extractDialog = new GenericProgressForm();
extractDialog.OnProcess += (_, __) =>
{
if (resource.ResourceType == ResourceType.Mesh && dialog.FilterIndex == 1)
{
var exporter = new GltfModelExporter
{
ProgressReporter = new Progress <string>(extractDialog.SetProgress),
FileLoader = exportData.VrfGuiContext.FileLoader,
};
exporter.ExportToFile(fileName, dialog.FileName, new Mesh(resource));
}
else if (resource.ResourceType == ResourceType.Model && dialog.FilterIndex == 1)
{
var exporter = new GltfModelExporter
{
ProgressReporter = new Progress <string>(extractDialog.SetProgress),
FileLoader = exportData.VrfGuiContext.FileLoader,
};
exporter.ExportToFile(fileName, dialog.FileName, (Model)resource.DataBlock);
}
else
{
var data = FileExtract.Extract(resource).ToArray();
using var stream = dialog.OpenFile();
stream.Write(data, 0, data.Length);
}
Console.WriteLine($"Export for \"{fileName}\" completed");
};
extractDialog.ShowDialog();
}
