public static void SerializeNet(string path, Network net)
{
var doc = new XDocument();
var rootElement = new XElement("Network");
var layersElement = new XElement("Layers");
rootElement.Add(layersElement);
foreach (var layer in net.Layers)
{
SerializeLayer(layersElement, layer);
}
doc.Add(rootElement);
doc.Save(path);
}
/// <summary>
/// Load data into existing network
/// </summary>
public static void DeserializeNet(string path, Network net, params SerializationOptions[] options)
{
var doc = XDocument.Load(path);
var rootElement = doc.Root;
var layerElements = rootElement.Element("Layers").Elements("Layer");
foreach (var layerElement in layerElements)
{
var id = layerElement.GetElementValue("Id");
var layer = net.GetLayerById(id);
if (layer != null)
{
DeserializeLayer(layerElement, layer, options);
}
}
}
public static Network CreateNetworkRelu(GPUModule module, int minibatchSize)
{
var net = new Network(module, minibatchSize: minibatchSize);
net.AddInputLayer(Constants.TOTAL_VALUE_COUNT, sparseDataSize: minibatchSize * RawRecord.FEATURE_COUNT * 2);
net.AddLabelLayer(1);
var fc1 = net.AddFullyConnectedLayer(128, "FC1");
fc1.Weights.InitValuesUniformCPU(0.1f);
fc1.L2Regularization = 0.00001f;
fc1.RegularizationRatio = 10;
net.AddReluLayer("REL1");
var fc2 = net.AddFullyConnectedLayer(256, "FC2");
fc2.Weights.InitValuesUniformCPU(0.1f);
net.AddReluLayer("REL2");
net.AddDropoutLayer();
var sm = net.AddSoftmaxLayer(2, "SMAX");
sm.Weights.InitValuesUniformCPU(0.1f);
return net;
}
public AsciiDumper(Network net)
{
_network = net;
}
public SGDTrainer(Network net,DataProvider trainDataProvider, DataProvider testDataProvider)
{
_network = net;
_testDataProvider = testDataProvider;
_trainDataProvider = trainDataProvider;
}
public CriteoTrainer(Network net, OneHotRecordProvider trainDataProvider, OneHotRecordProvider testDataProvider)
: base(net, trainDataProvider, testDataProvider)
{
}
public static void MakeSubmission(Network network, string testsetPath, string targetPath)
{
Console.WriteLine("Making submission");
var batchSize = Constants.MINIBATCH_SIZE;
var recs = OneHotRecordReadOnly.LoadBinary(testsetPath);
var sparseIndices = new int[batchSize][];
var sparseValues = new float[batchSize][];
var labels = new float[batchSize];
var ids = new int[batchSize];
var submissionLines = new List<SubmissionLine>();
var firstBatch = true;
for (var recNo = 0; recNo < recs.Count; recNo++)
{
var record = recs[recNo];
var label = record.Label;
var id = record.Id;
labels[recNo % batchSize] = label;
ids[recNo % batchSize] = id;
record.CopyDataToSparseArray(sparseIndices, sparseValues, recNo % batchSize);
if ((((recNo + 1) % batchSize) == 0) || (recNo == (recs.Count-1)))
{
network.InputLayer.SetSparseData(sparseValues.ToList(), sparseIndices.ToList());
network.LabelLayer.SetData(labels);
network.Calculate(train: false);
network.CostLayer.CopyToHost();
for (var i = 0; i < batchSize; i++)
{
if (ids[i] == 0) continue;
var chance = network.CostLayer.Outputs[i, 1];
var line = new SubmissionLine();
line.Id = ids[i];
line.Chance = chance;
submissionLines.Add(line);
}
Array.Clear(ids, 0, ids.Length);
}
if (recNo % 100000 == 0)
{
Console.WriteLine("line : " + recNo);
}
}
SubmissionLine.SaveSubmission(targetPath, submissionLines);
}
public static void Train(GPUModule module, List<OneHotRecordReadOnly> allTrainRecords, Network net, float learnRate = 0.02f, float momentum = 0.5f, int epochsBeforeMergeHoldout = 30, int totalEpochs = 50, string tmpDir = null)
{
// use roughly last day for validation
var trainCount = allTrainRecords.Count;
var holdoutCount = trainCount / 7;
trainCount = trainCount - holdoutCount;
var holdoutRecords = allTrainRecords.Skip(trainCount).ToList();
holdoutRecords.Shuffle();
var trainRecords = allTrainRecords.Take(trainCount).ToList();
var trainProvider = new OneHotRecordProvider(module, trainRecords, "train", shuffleEveryEpoch: true);
//var trainProvider = new ClicksProvider(module, TRAINSET_BIN_PATH, "train");
trainProvider._currentEpoch = 0;
var holdoutProvider = new OneHotRecordProvider(module, holdoutRecords, "test");// new ClicksProvider(module, TESTSET_BIN_PATH, "test");
//var testProvider = new ClicksProvider(module, TESTSET_BIN_PATH, "test");
holdoutProvider._currentEpoch = 0;
var trainer = new CriteoTrainer(net, trainProvider, holdoutProvider);
trainer.Train(learnRate, momentum, epocsBeforeReport: 40, epocsBeforeMergeHoldout: epochsBeforeMergeHoldout, totalEpochs: totalEpochs, workDir: tmpDir);
}