public DataHandler(DataHandlerArgs args)
{
this.args = args;
if (args.StepsPerExecution == null)
{
_steps_per_execution = tf.Variable(1);
_steps_per_execution_value = 1;
}
else
{
_steps_per_execution = args.StepsPerExecution;
_steps_per_execution_value = args.StepsPerExecution.numpy();
}
_adapter = new TensorLikeDataAdapter(new TensorLikeDataAdapterArgs
{
X = args.X,
Y = args.Y,
BatchSize = args.BatchSize,
Steps = args.StepsPerEpoch,
Epochs = args.Epochs - args.InitialEpoch,
Shuffle = args.Shuffle,
MaxQueueSize = args.MaxQueueSize,
Worker = args.Workers,
UseMultiprocessing = args.UseMultiprocessing,
Model = args.Model
});
_dataset = _adapter.GetDataset();
_inferred_steps = _infer_steps(args.StepsPerEpoch, _dataset);
_current_step = 0;
_step_increment = args.StepsPerExecution.numpy() - 1;
_insufficient_data = false;
}
public FilterDataset(IDatasetV2 input_dataset,
Func <Tensor, bool> predicate_func) : base(input_dataset)
{
Func <Tensors, Tensors> predicate_func_update = x =>
{
var result = predicate_func(x);
return(constant_op.constant(result));
};
var func = new ConcreteFunction($"{predicate_func.Method.Name}_{Tensorflow.ops.uid_function()}");
func.Enter();
var inputs = new Tensors();
foreach (var input in input_dataset.element_spec)
{
inputs.Add(tf.placeholder(input.dtype, shape: input.shape, name: "arg"));
}
var outputs = predicate_func_update(inputs);
func.ToGraph(inputs, outputs);
func.Exit();
structure = func.OutputStructure;
variant_tensor = ops.filter_dataset(input_dataset.variant_tensor,
func,
output_types,
output_shapes);
}
public MapDataset(IDatasetV2 input_dataset,
Func <Tensors, Tensors> map_func,
bool use_inter_op_parallelism = true,
bool preserve_cardinality = false,
bool use_legacy_function = false) : base(input_dataset)
{
var func = new ConcreteFunction($"{map_func.Method.Name}_{Tensorflow.ops.uid_function()}");
func.Enter();
var inputs = new Tensors();
foreach (var input in input_dataset.element_spec)
{
inputs.Add(tf.placeholder(input.dtype, shape: input.shape, name: "arg"));
}
var outputs = map_func(inputs);
func.ToGraph(inputs, outputs);
func.Exit();
structure = func.OutputStructure;
variant_tensor = ops.map_dataset(input_dataset.variant_tensor,
func,
output_types,
output_shapes,
use_inter_op_parallelism: use_inter_op_parallelism,
preserve_cardinality: preserve_cardinality);
}
public KeyValuePair <string, float>[] evaluate(IDatasetV2 x)
{
data_handler = new DataHandler(new DataHandlerArgs
{
Dataset = x,
Model = this,
StepsPerExecution = _steps_per_execution
});
Binding.tf_output_redirect.WriteLine($"Testing...");
IEnumerable <(string, Tensor)> logs = null;
foreach (var(epoch, iterator) in data_handler.enumerate_epochs())
{
reset_metrics();
// callbacks.on_epoch_begin(epoch)
// data_handler.catch_stop_iteration();
foreach (var step in data_handler.steps())
{
// callbacks.on_train_batch_begin(step)
logs = test_function(iterator);
}
Binding.tf_output_redirect.WriteLine($"iterator: {epoch + 1}, " + string.Join(", ", logs.Select(x => $"{x.Item1}: {(float)x.Item2}")));
}
return(logs.Select(x => new KeyValuePair <string, float>(x.Item1, (float)x.Item2)).ToArray());
}
public TakeDataset(IDatasetV2 input_dataset, int count) :
base(input_dataset)
{
_count = tf.convert_to_tensor(count, dtype: dtypes.int64, name: "count");
variant_tensor = ops.take_dataset(input_dataset.variant_tensor, _count,
output_types, output_shapes);
}
void _create_iterator(IDatasetV2 dataset)
{
dataset = dataset.apply_options();
_dataset = dataset;
_element_spec = dataset.element_spec;
(_iterator_resource, _deleter) = ops.anonymous_iterator_v2(_dataset.output_types, _dataset.output_shapes);
ops.make_iterator(dataset.variant_tensor, _iterator_resource);
}
public override void adapt(IDatasetV2 data, bool reset_state = true)
{
if (!reset_state)
{
throw new ValueError("IndexLookup does not support streaming adapts.");
}
base.adapt(data, reset_state);
}
public RepeatDataset(IDatasetV2 input_dataset, int count = -1) :
base(input_dataset)
{
_count = constant_op.constant(count, dtype: TF_DataType.TF_INT64, name: "count");
variant_tensor = ops.repeat_dataset(input_dataset.variant_tensor,
_count,
input_dataset.output_types,
input_dataset.output_shapes);
}
public FlatMapDataset(IDatasetV2 input_dataset,
Func <Tensor, IDatasetV2> map_func) : base(input_dataset)
{
var func = new ConcreteFunction(map_func, input_dataset.element_spec[0].dtype);
variant_tensor = ops.flat_map_dataset(input_dataset.variant_tensor,
func,
output_types,
output_shapes);
}
public ModelDataset(IDatasetV2 input_dataset,
AutotuneAlgorithm algorithm,
long cpu_budget) :
base(input_dataset)
{
variant_tensor = ops.model_dataset(input_dataset.variant_tensor,
output_types,
output_shapes,
algorithm,
cpu_budget);
}
/// <summary>
/// Fits the state of the preprocessing layer to the dataset.
/// </summary>
/// <param name="data"></param>
/// <param name="reset_state"></param>
public void adapt(IDatasetV2 data, bool reset_state = true)
{
var shape = data.output_shapes[0];
if (shape.rank == 1)
{
data = data.map(tensor => array_ops.expand_dims(tensor, -1));
}
build(data.variant_tensor);
var preprocessed_inputs = data.map(_preprocess);
}
public CacheDataset(IDatasetV2 input_dataset,
string filename = "") :
base(input_dataset)
{
_filename = tf.convert_to_tensor(filename, dtype: TF_DataType.TF_STRING, name: "filename");
variant_tensor = ops.cache_dataset_v2(input_dataset.variant_tensor,
_filename,
ops.dummy_memory_cache(),
output_types,
output_shapes);
}
IDatasetV2 sequences_from_indices(Tensor array, IDatasetV2 indices_ds, int start_index, int?end_index)
{
var dataset = tf.data.Dataset.from_tensors(array[new Slice(start: start_index, stop: end_index)]);
dataset = tf.data.Dataset.zip(dataset.repeat(), indices_ds)
.map(x =>
{
var(steps, indx) = x;
return(array_ops.gather(steps, indx));
}, num_parallel_calls: -1);
return(dataset);
}
public virtual void adapt(IDatasetV2 data, bool reset_state = true)
{
IAccumulator accumulator;
if (!reset_state)
{
accumulator = combiner.Restore();
}
var next_data = data.make_one_shot_iterator();
var data_element = next_data.next();
}
/// <summary>
/// Fits the state of the preprocessing layer to the dataset.
/// </summary>
/// <param name="data"></param>
/// <param name="reset_state"></param>
public override void adapt(IDatasetV2 data, bool reset_state = true)
{
var shape = data.output_shapes[0];
if (shape.ndim == 1)
{
data = data.map(tensor => array_ops.expand_dims(tensor, -1));
}
build(data.variant_tensor);
var preprocessed_inputs = data.map(_preprocess);
_index_lookup_layer.adapt(preprocessed_inputs);
}
public MapDataset(IDatasetV2 input_dataset,
Func <Tensor, Tensor> map_func,
bool use_inter_op_parallelism = true,
bool preserve_cardinality = false,
bool use_legacy_function = false) : base(input_dataset)
{
var func = new ConcreteFunction(map_func, input_dataset.element_spec[0].dtype);
variant_tensor = ops.map_dataset(input_dataset.variant_tensor,
func,
output_types,
output_shapes);
}
public PrefetchDataset(IDatasetV2 input_dataset,
long buffer_size = -1,
int?slack_period = null) :
base(input_dataset)
{
_buffer_size = tf.convert_to_tensor(buffer_size, dtype: TF_DataType.TF_INT64, name: "buffer_size");
variant_tensor = ops.prefetch_dataset(input_dataset.variant_tensor,
_buffer_size,
input_dataset.output_types,
input_dataset.output_shapes,
slack_period: slack_period);
}
public ShardDataset(IDatasetV2 input_dataset,
int num_shards,
int index) : base(input_dataset)
{
_num_shards = tf.convert_to_tensor(num_shards, dtype: TF_DataType.TF_INT64, name: "num_shards");
_index = tf.convert_to_tensor(index, dtype: TF_DataType.TF_INT64, name: "index");
variant_tensor = ops.shard_dataset
(input_dataset.variant_tensor,
num_shards: _num_shards,
index: _index,
input_dataset.output_types,
input_dataset.output_shapes);
}
float accuracy_test = 0f; // 测试集准确率
public void Main()
{
((x_train, y_train), (x_test, y_test)) = tf.keras.datasets.mnist.load_data(); // 下载 或 加载本地 MNIST
(x_train, x_test) = (x_train.reshape((-1, num_features)), x_test.reshape((-1, num_features))); // 输入数据展平
(x_train, x_test) = (x_train / 255f, x_test / 255f); // 归一化
train_data = tf.data.Dataset.from_tensor_slices(x_train, y_train); //转换为 Dataset 格式
train_data = train_data.repeat()
.shuffle(5000)
.batch(batch_size)
.prefetch(1)
.take(training_steps);// 数据预处理
// 随机初始化网络权重变量，并打包成数组方便后续梯度求导作为参数。
var random_normal = tf.initializers.random_normal_initializer();
h1 = tf.Variable(random_normal.Apply(new InitializerArgs((num_features, n_hidden_1))));
h2 = tf.Variable(random_normal.Apply(new InitializerArgs((n_hidden_1, n_hidden_2))));
wout = tf.Variable(random_normal.Apply(new InitializerArgs((n_hidden_2, num_classes))));
b1 = tf.Variable(tf.zeros(n_hidden_1));
b2 = tf.Variable(tf.zeros(n_hidden_2));
bout = tf.Variable(tf.zeros(num_classes));
var trainable_variables = new IVariableV1[] { h1, h2, wout, b1, b2, bout };
// 采用随机梯度下降优化器
var optimizer = tf.optimizers.SGD(learning_rate);
// 训练模型
foreach (var(step, (batch_x, batch_y)) in enumerate(train_data, 1))
{
// 运行优化器 进行模型权重 w 和 b 的更新
run_optimization(optimizer, batch_x, batch_y, trainable_variables);
if (step % display_step == 0)
{
var pred = neural_net(batch_x);
var loss = cross_entropy(pred, batch_y);
var acc = accuracy(pred, batch_y);
print($"step: {step}, loss: {(float)loss}, accuracy: {(float)acc}");
}
}
// 在测试集上对训练后的模型进行预测准确率性能评估
{
var pred = neural_net(x_test);
accuracy_test = (float)accuracy(pred, y_test);
print($"Test Accuracy: {accuracy_test}");
}
}
public MapDataset(IDatasetV2 input_dataset,
Func <Tensor, Tensor> map_func,
bool use_inter_op_parallelism = true,
bool preserve_cardinality = false,
bool use_legacy_function = false) : base(input_dataset)
{
foreach (var input in input_dataset)
{
var data = map_func(input.Item1);
}
variant_tensor = ops.map_dataset(input_dataset.variant_tensor,
output_types,
output_shapes);
}
int _infer_steps(int steps_per_epoch, IDatasetV2 dataset)
{
if (steps_per_epoch > -1)
{
return(steps_per_epoch);
}
var adapter_steps = _adapter.GetSize();
if (adapter_steps > -1)
{
return(adapter_steps);
}
throw new NotImplementedException("");
}
public ConcatenateDataset(IDatasetV2 input_dataset, IDatasetV2 dataset_to_concatenate)
{
_input_dataset = input_dataset;
_dataset_to_concatenate = dataset_to_concatenate;
var _structure = new List <TensorSpec>();
foreach (var(i, spec) in enumerate(dataset_to_concatenate.element_spec))
{
var shape = _input_dataset.output_shapes[i].most_specific_compatible_shape(spec.shape);
_structure.Add(new TensorSpec(shape, dtype: spec.dtype));
}
structure = _structure.ToArray();
variant_tensor = ops.concatenate_dataset(input_dataset.variant_tensor,
dataset_to_concatenate.variant_tensor,
output_types, output_shapes);
}
public override void PrepareData()
{
// Prepare MNIST data.
((x_train, y_train), (x_test, y_test)) = tf.keras.datasets.mnist.load_data();
// Flatten images to 1-D vector of 784 features (28*28).
(x_train, x_test) = (x_train.reshape((-1, num_features)), x_test.reshape((-1, num_features)));
// Normalize images value from [0, 255] to [0, 1].
(x_train, x_test) = (x_train / 255f, x_test / 255f);
// Use tf.data API to shuffle and batch data.
train_data = tf.data.Dataset.from_tensor_slices(x_train, y_train);
train_data = train_data.repeat()
.shuffle(5000)
.batch(batch_size)
.prefetch(1)
.take(training_steps);
}
IDatasetV2 slice_inputs(IDatasetV2 indices_dataset, Tensors elements)
{
var dataset = tf.data.Dataset.from_tensors(elements).repeat();
dataset = tf.data.Dataset.zip(indices_dataset, dataset);
dataset = dataset.map(inputs =>
{
var indices = inputs[0];
var results = inputs.Skip(1)
.Select(x => gen_array_ops.gather_v2(x, indices, 0))
.ToArray();
return(new Tensors(results));
}, -1);
return(dataset.with_options(new DatasetOptions {
}));
}
int _infer_steps(int steps_per_epoch, IDatasetV2 dataset)
{
if (steps_per_epoch > -1)
{
return(steps_per_epoch);
}
var adapter_steps = _adapter.GetSize();
if (adapter_steps > -1)
{
return(adapter_steps);
}
var size = dataset.dataset_cardinality();
return(size.numpy());
}
public TensorLikeDataAdapter(TensorLikeDataAdapterArgs args)
{
this.args = args;
_process_tensorlike();
num_samples = args.X.shape[0];
var batch_size = args.BatchSize == -1 ? 32 : args.BatchSize;
_batch_size = batch_size;
_size = Convert.ToInt32(Math.Ceiling(num_samples / (batch_size + 0f)));
num_full_batches = num_samples / batch_size;
var _partial_batch_size = num_samples % batch_size;
var indices_dataset = tf.data.Dataset.range(1);
indices_dataset = indices_dataset.repeat();
indices_dataset = indices_dataset.map(permutation).prefetch(1);
indices_dataset = indices_dataset.flat_map(slice_batch_indices);
_dataset = slice_inputs(indices_dataset, args.X, args.Y);
}
