static void Main(string[] args)
{
//Read config
config = JsonConvert.DeserializeObject <ExtractorConfig>(File.ReadAllText("config.json"));
//Generate a revision tag and make it's folder structure
log = new FileStream(config.GetProfile().log_path, FileMode.Create);
//Do some logging
Log.WriteHeader("Data created " + DateTime.Now.ToShortDateString() + " " + DateTime.Now.ToLongTimeString() + " LOCAL");
Log.WriteHeader("Deploying to " + config.enviornment);
Log.WriteHeader("(C) RomanPort 2019");
//Init the install
ArkInstall install = ArkInstall.GetInstall(config.install_path);
//Create a patch and run
DeltaExportPatch patch = new DeltaExportPatch(install);
patch.Go();
//Finish
log.Flush();
log.Close();
Console.ReadLine();
}
public static void ProcessImages(List <string> readErrors, DeltaExportPatch patch)
{
//Clean up any old and bad paths
Console.WriteLine("Cleaning up old image conversions...");
if (Directory.Exists("./Lib/UModel/in_temp/"))
{
Directory.Delete("./Lib/UModel/in_temp/", true);
}
if (Directory.Exists("./Lib/UModel/out_temp/"))
{
Directory.Delete("./Lib/UModel/out_temp/", true);
}
//Make structre
Directory.CreateDirectory("./Lib/UModel/in_temp/");
Directory.CreateDirectory("./Lib/UModel/out_temp/");
//Get the queue
var queue = patch.queued_images;
//First, we copy all packages to a temporary path with their index
Console.WriteLine($"Now copying {queue.Count} images...");
for (int i = 0; i < queue.Count; i++)
{
string source = queue[i].pathname;
File.Copy(source, $"./Lib/UModel/in_temp/{i}.uasset");
}
//Now, run the conversion
Console.WriteLine("Now converting images using UModel...");
Process p = Process.Start(new ProcessStartInfo
{
Arguments = "",
FileName = "go.bat",
WorkingDirectory = "Lib\\UModel\\",
UseShellExecute = true
});
p.WaitForExit();
//Now, load and process these images
int ok = 0;
Console.WriteLine($"Now processing {queue.Count} images...");
for (int i = 0; i < queue.Count; i += 1)
{
QueuedImage q = queue[i];
bool status = false;
try
{
//Get the directory. It's a little janky, as files are stored in subdirs
string[] results = Directory.GetFiles($"./Lib/UModel/out_temp/{i}/");
if (results.Length != 1)
{
throw new Exception("None or too many results found for image.");
}
//Open FileStream on this
using (FileStream imgStream = new FileStream(results[0], FileMode.Open, FileAccess.Read))
{
//Now, begin reading the TGA data https://en.wikipedia.org/wiki/Truevision_TGA
IOMemoryStream imgReader = new IOMemoryStream(imgStream, true);
imgReader.position += 3 + 5; //Skip intro, it will always be known
imgReader.ReadShort(); //Will always be 0
imgReader.ReadShort(); //Will aways be 0
short width = imgReader.ReadShort();
short height = imgReader.ReadShort();
byte colorDepth = imgReader.ReadByte();
imgReader.ReadByte();
//Now, we can begin reading image data
//This appears to be bugged for non-square images right now.
using (Image <Rgba32> img = new Image <Rgba32>(width, height))
{
//Read file
byte[] channels;
for (int y = 0; y < height; y++)
{
for (int x = 0; x < width; x++)
{
if (colorDepth == 32)
{
//Read four channels
channels = imgReader.ReadBytes(4);
//Set pixel
img[x, width - y - 1] = new Rgba32(channels[2], channels[1], channels[0], channels[3]);
}
else if (colorDepth == 24)
{
//Read three channels
channels = imgReader.ReadBytes(3);
//Set pixel
img[x, width - y - 1] = new Rgba32(channels[2], channels[1], channels[0]);
}
}
}
//Apply mods
if (q.mods == ImageModifications.White)
{
ApplyWhiteMod(img);
}
//Save original image
using (MemoryStream ms = new MemoryStream())
{
img.SaveAsPng(ms);
ms.Position = 0;
patch.asset_manager.Upload(Program.config.GetProfile().upload_images + q.hiId + FORMAT_TYPE, ms);
}
//Now, downscale
img.Mutate(x => x.Resize(64, 64));
//Save thumbnail
using (MemoryStream ms = new MemoryStream())
{
img.SaveAsPng(ms, new PngEncoder
{
CompressionLevel = 9
});
ms.Position = 0;
patch.asset_manager.Upload(Program.config.GetProfile().upload_images + q.loId + FORMAT_TYPE, ms);
}
status = true;
ok++;
}
}
} catch (Exception ex)
{
Console.WriteLine($"Failed to process image {q.classname} with error {ex.Message}");
readErrors.Add($"Failed to process image {q.classname} with error {ex.Message} {ex.StackTrace}");
}
//Now, add to persistent storage
patch.persist.external_assets.Add(new DeltaExportBranchExternalAsset
{
name = q.name,
patch = patch.tag,
sha1 = q.sha1,
time = DateTime.UtcNow,
id_hires = q.hiId,
id_lores = q.loId,
ok = status
});
}
Log.WriteSuccess("ImageTool", $"Processed and uploading {ok}/{queue.Count} images.");
queue.Clear();
//Clean up any old and bad paths
Console.WriteLine("Cleaning up...");
if (Directory.Exists("./Lib/UModel/in_temp/"))
{
Directory.Delete("./Lib/UModel/in_temp/", true);
}
if (Directory.Exists("./Lib/UModel/out_temp/"))
{
Directory.Delete("./Lib/UModel/out_temp/", true);
}
}
public static DeltaAsset QueueImage(string classname, string pathname, ImageModifications mods, DeltaExportPatch patch)
{
//SHA1 this file for version control
byte[] hashBytes = new SHA1Managed().ComputeHash(File.ReadAllBytes(pathname));
string hash = string.Concat(hashBytes.Select(b => b.ToString("x2")));
//Get the game pathname
ArkAsset asset = ArkAsset.GetAssetFromFolderPath(pathname, patch.installation);
string namespacedName = asset.fullName;
//Check for matching file pathnames and versions
var matches = patch.persist.external_assets.Where(x => x.sha1 == hash && x.name == namespacedName).ToArray();
var queuedMatches = patch.queued_images.Where(x => x.name == namespacedName).ToArray();
//Create or get
string hiId;
string loId;
if (matches.Length >= 1)
{
//We'll use this existing image
hiId = matches[0].id_hires;
loId = matches[0].id_lores;
}
else if (queuedMatches.Length >= 1)
{
//We'll use the existing queued image
hiId = queuedMatches[0].hiId;
loId = queuedMatches[0].loId;
}
else
{
//We'll generate this
//Generate IDs for the high res image and thumbnail
hiId = GenerateUniqueImageID(patch);
loId = GenerateUniqueImageID(patch);
//Now, create an object and add it to a queue
patch.queued_images.Add(new QueuedImage
{
classname = classname,
pathname = pathname,
hiId = hiId,
loId = loId,
mods = mods,
sha1 = hash,
name = namespacedName
});
}
//Now, create an ArkImage
DeltaAsset r = new DeltaAsset
{
image_thumb_url = GetAssetsUrl() + loId + FORMAT_TYPE,
image_url = GetAssetsUrl() + hiId + FORMAT_TYPE
};
return(r);
}
private static string GenerateUniqueImageID(DeltaExportPatch patch)
{
string id = GenerateUnsafeImageID();
return(id);
}
public static DeltaAsset QueueImage(ClassnamePathnamePair name, ImageModifications mods, DeltaExportPatch patch)
{
return(QueueImage(name.classname, name.pathname, mods, patch));
}
public static ItemEntry ConvertEntry(UAssetFileBlueprint bp, UAssetCacheBlock cache, DeltaExportPatch patch)
{
//Open reader
PropertyReader reader = new PropertyReader(bp.GetFullProperties(cache));
//Get primary icon
DeltaAsset icon;
if (reader.GetProperty <ObjectProperty>("ItemIcon") != null)
{
icon = ImageTool.QueueImage(reader.GetProperty <ObjectProperty>("ItemIcon").GetReferencedFile(), ImageTool.ImageModifications.None, patch);
}
else
{
icon = DeltaAsset.MISSING_ICON;
}
//Get broken icon
DeltaAsset brokenIcon;
if (reader.GetProperty <ObjectProperty>("BrokenImage") != null)
{
brokenIcon = ImageTool.QueueImage(reader.GetProperty <ObjectProperty>("BrokenImage").GetReferencedFile(), ImageTool.ImageModifications.None, patch);
}
else
{
brokenIcon = DeltaAsset.MISSING_ICON;
}
//Get the array of UseItemAddCharacterStatusValues
ArrayProperty statusValuesArray = reader.GetProperty <ArrayProperty>("UseItemAddCharacterStatusValues");
Dictionary <string, ItemEntry_ConsumableAddStatusValue> statusValues = new Dictionary <string, ItemEntry_ConsumableAddStatusValue>();
if (statusValuesArray != null)
{
foreach (var i in statusValuesArray.props)
{
StructProperty sv = (StructProperty)i;
var svp = ((PropListStruct)sv.data).propsList;
var svpr = new PropertyReader(svp);
string type = svpr.GetProperty <ByteProperty>("StatusValueType").enumValue;
ItemEntry_ConsumableAddStatusValue sve = ArkItemEntry_ConsumableAddStatusValueReader.Convert(svpr, type);
statusValues.Add(type, sve);
}
}
//Create
ItemEntry e = new ItemEntry
{
hideFromInventoryDisplay = reader.GetPropertyBool("bHideFromInventoryDisplay", false),
useItemDurability = reader.GetPropertyBool("bUseItemDurability", false),
isTekItem = reader.GetPropertyBool("bTekItem", false),
allowUseWhileRiding = reader.GetPropertyBool("bAllowUseWhileRiding", false),
name = reader.GetPropertyString("DescriptiveNameBase", null),
description = reader.GetPropertyString("ItemDescription", null),
spoilingTime = reader.GetPropertyFloat("SpolingTime", 0),
baseItemWeight = reader.GetPropertyFloat("BaseItemWeight", 0),
useCooldownTime = reader.GetPropertyFloat("MinimumUseInterval", 0),
baseCraftingXP = reader.GetPropertyFloat("BaseCraftingXP", 0),
baseRepairingXP = reader.GetPropertyFloat("BaseRepairingXP", 0),
maxItemQuantity = reader.GetPropertyInt("MaxItemQuantity", 0),
classname = DeltaDataExtractor.Program.TrimArkClassname(bp.classname),
icon = icon,
broken_icon = brokenIcon,
addStatusValues = statusValues
};
return(e);
}
public static List <DinosaurEntry> ExtractDinos(UAssetCacheBlock cache, List <ArkAsset> assets, DeltaExportPatch patch, PropertyReader primalDataReader, Dictionary <string, PropertyReader> dinoEntries)
{
//Loop through assets and import them
List <DinosaurEntry> dinos = new List <DinosaurEntry>();
foreach (var a in assets)
{
//Open file
UAssetFileBlueprint f;
try
{
f = UAssetFileBlueprint.OpenFile(a.filename, false, a.name, a.installation.contentFolder);
} catch (Exception ex)
{
continue;
}
//Convert file
try
{
//Create a dino entry
DinosaurEntry entry = ArkDinoEntryConverter.Convert(f, cache, patch, primalDataReader, dinoEntries);
dinos.Add(entry);
} catch (Exception ex)
{
continue;
}
}
return(dinos);
}
public static List <ItemEntry> ExtractItems(UAssetCacheBlock cache, List <ArkAsset> assets, DeltaExportPatch patch)
{
//Loop through all assets
List <ItemEntry> items = new List <ItemEntry>();
foreach (var a in assets)
{
//Open
DateTime start = DateTime.UtcNow;
UAssetFileBlueprint bp;
try
{
bp = UAssetFileBlueprint.OpenFile(a.filename, false, a.name, a.installation.contentFolder);
}
catch (Exception ex)
{
Log.WriteError("Item Extractor", "Failed to open file " + a.fullName + ": " + ex.Message + ex.StackTrace);
continue;
}
//Decode
try
{
ItemEntry entry = ArkItemEntryReader.ConvertEntry(bp, cache, patch);
items.Add(entry);
TimeSpan time = DateTime.UtcNow - start;
Log.WriteInfo("Item Extractor", "Successfully opened and converted " + a.fullName + " in " + Math.Round(time.TotalMilliseconds) + "ms");
}
catch (Exception ex)
{
Log.WriteError("Item Extractor", "Failed to import " + a.fullName + ": " + ex.Message + ex.StackTrace);
continue;
}
}
Log.WriteSuccess("Item Extractor", "Extracted " + items.Count + "/" + assets.Count + " items.");
return(items);
}
public static DinosaurEntry Convert(UAssetFileBlueprint f, UAssetCacheBlock cache, DeltaExportPatch patch, PropertyReader primalDataReader, Dictionary <string, PropertyReader> dinoEntries)
{
//Open reader
PropertyReader reader = new PropertyReader(f.GetFullProperties(cache));
//Get the dino settings
UAssetFileBlueprint settingsFileAdult = ArkDinoFoodConverter.GetAdultFile(f, cache);
UAssetFileBlueprint settingsFileBaby = ArkDinoFoodConverter.GetBabyFile(f, cache);
//Get status component
UAssetFileBlueprint statusComponent = ArkDinoEntryStatusConverter.GetFile(f, cache);
PropertyReader statusReader = new PropertyReader(statusComponent.GetFullProperties(cache));
//Use name tag to find entry
string tag = reader.GetPropertyStringOrName("DinoNameTag");
PropertyReader entry = dinoEntries[tag];
//Now, load the material used for the dino image
UAssetFileMaterial entryMaterial = entry.GetProperty <ObjectProperty>("DinoMaterial").GetReferencedFileMaterial();
UAssetFileMaterial.TextureParameterValue entryMaterialTexture = entryMaterial.textureParameters[0];
ClassnamePathnamePair entryTexture = entryMaterialTexture.prop.GetReferencedFile();
//Read
DinosaurEntry e = new DinosaurEntry
{
screen_name = reader.GetPropertyString("DescriptiveName", null),
colorizationIntensity = reader.GetPropertyFloat("ColorizationIntensity", 1),
babyGestationSpeed = reader.GetPropertyFloat("BabyGestationSpeed", -1),
extraBabyGestationSpeedMultiplier = reader.GetPropertyFloat("ExtraBabyGestationSpeedMultiplier", -1),
babyAgeSpeed = reader.GetPropertyFloat("BabyAgeSpeed", null),
extraBabyAgeMultiplier = reader.GetPropertyFloat("ExtraBabyAgeSpeedMultiplier", -1),
useBabyGestation = reader.GetPropertyBool("bUseBabyGestation", false),
statusComponent = ArkDinoEntryStatusConverter.Convert(statusComponent, statusReader),
adultFoods = ArkDinoFoodConverter.Convert(settingsFileAdult, cache),
childFoods = ArkDinoFoodConverter.Convert(settingsFileBaby, cache),
classname = DeltaDataExtractor.Program.TrimArkClassname(f.classname),
icon = ImageTool.QueueImage(entryTexture, ImageTool.ImageModifications.None, patch),
};
//Finally, read stats
ArkStatsRipper.DoRipStats(statusReader, e);
return(e);
}
