public static void LoadExternalDataForTensor(TensorProto tensor, string base_dir) { if (tensor.RawData != null) { // already loaded return; } var info = new ExternalDataInfo(tensor); var file_location = _sanitize_path(info.location); var external_data_file_path = Path.Combine(base_dir, file_location); using (var data_file = File.Open(external_data_file_path, FileMode.Open)) { if (info.offset != null) { data_file.Seek(info.offset.Value, SeekOrigin.Begin); } byte[] bytes = null; if (info.length != null) { bytes = new byte[info.length.Value]; data_file.Read(bytes, 0, info.length.Value); } else { bytes = new byte[data_file.Length]; data_file.Read(bytes, 0, (int)data_file.Length); } tensor.RawData = ByteString.CopyFrom(bytes); } }
public static void SaveModel(ModelProto proto, string f, object format = null) { var model_filepath = _get_file_path(f); if (!string.IsNullOrWhiteSpace(model_filepath)) { var basepath = Path.GetDirectoryName(model_filepath); proto = ExternalDataInfo.WriteExternalDataTensors(proto, basepath); } var s = _serialize(proto); _save_bytes(s, f); }
public static void SaveExternalData(TensorProto tensor, string base_path) { var info = new ExternalDataInfo(tensor); var external_data_file_path = Path.Combine(base_path, info.location); // Retrieve the tensor's data from raw_data or load external file if (tensor.RawData == null) { throw new Exception("raw_data field doesn't exist."); } // Create file if it doesn't exist if (!File.Exists(external_data_file_path)) { using (var f = File.CreateText(external_data_file_path)) { f.Write("ab"); } } // Open file for reading and writing at random locations ('r+b') using (var data_file = File.OpenRead(external_data_file_path)) { data_file.Seek(0, SeekOrigin.End); if (info.offset != null) { // Pad file to required offset if needed var file_size = data_file.Length; if (info.offset > file_size) { List <byte> writeBytes = new List <byte>(); for (int i = 0; i < (info.offset - file_size); i++) { writeBytes.AddRange(Encoding.UTF8.GetBytes("\0")); } data_file.Write(writeBytes.ToArray()); } data_file.Seek(info.offset.Value, SeekOrigin.Begin); } var offset = (int)data_file.Length; data_file.Write(tensor.RawData.ToByteArray()); SetExternalData(tensor, info.location, offset, (int)data_file.Length - offset); } }
public static ModelProto LoadModel(string f, object format = null, bool load_external_data = true) { var s = _load_bytes(f); var model = LoadModelFromString(s, format: format); if (load_external_data) { var model_filepath = _get_file_path(f); if (!string.IsNullOrWhiteSpace(model_filepath)) { var base_dir = Path.GetDirectoryName(model_filepath); ExternalDataInfo.LoadExternalDataForModel(model, base_dir); } } return(model); }