public static void LoadExternalDataForTensor(TensorProto tensor, string base_dir)
{
if (tensor.RawData != null)
{
// already loaded
return;
}
var info = new ExternalDataInfo(tensor);
var file_location = _sanitize_path(info.location);
var external_data_file_path = Path.Combine(base_dir, file_location);
using (var data_file = File.Open(external_data_file_path, FileMode.Open))
{
if (info.offset != null)
{
data_file.Seek(info.offset.Value, SeekOrigin.Begin);
}
byte[] bytes = null;
if (info.length != null)
{
bytes = new byte[info.length.Value];
data_file.Read(bytes, 0, info.length.Value);
}
else
{
bytes = new byte[data_file.Length];
data_file.Read(bytes, 0, (int)data_file.Length);
}
tensor.RawData = ByteString.CopyFrom(bytes);
}
}
public static void SaveModel(ModelProto proto, string f, object format = null)
{
var model_filepath = _get_file_path(f);
if (!string.IsNullOrWhiteSpace(model_filepath))
{
var basepath = Path.GetDirectoryName(model_filepath);
proto = ExternalDataInfo.WriteExternalDataTensors(proto, basepath);
}
var s = _serialize(proto);
_save_bytes(s, f);
}
public static void SaveExternalData(TensorProto tensor, string base_path)
{
var info = new ExternalDataInfo(tensor);
var external_data_file_path = Path.Combine(base_path, info.location);
// Retrieve the tensor's data from raw_data or load external file
if (tensor.RawData == null)
{
throw new Exception("raw_data field doesn't exist.");
}
// Create file if it doesn't exist
if (!File.Exists(external_data_file_path))
{
using (var f = File.CreateText(external_data_file_path))
{
f.Write("ab");
}
}
// Open file for reading and writing at random locations ('r+b')
using (var data_file = File.OpenRead(external_data_file_path))
{
data_file.Seek(0, SeekOrigin.End);
if (info.offset != null)
{
// Pad file to required offset if needed
var file_size = data_file.Length;
if (info.offset > file_size)
{
List <byte> writeBytes = new List <byte>();
for (int i = 0; i < (info.offset - file_size); i++)
{
writeBytes.AddRange(Encoding.UTF8.GetBytes("\0"));
}
data_file.Write(writeBytes.ToArray());
}
data_file.Seek(info.offset.Value, SeekOrigin.Begin);
}
var offset = (int)data_file.Length;
data_file.Write(tensor.RawData.ToByteArray());
SetExternalData(tensor, info.location, offset, (int)data_file.Length - offset);
}
}
public static ModelProto LoadModel(string f, object format = null, bool load_external_data = true)
{
var s = _load_bytes(f);
var model = LoadModelFromString(s, format: format);
if (load_external_data)
{
var model_filepath = _get_file_path(f);
if (!string.IsNullOrWhiteSpace(model_filepath))
{
var base_dir = Path.GetDirectoryName(model_filepath);
ExternalDataInfo.LoadExternalDataForModel(model, base_dir);
}
}
return(model);
}
