public static void MakeSubmission(Network network, string testsetPath, string targetPath)
{
Console.WriteLine("Making submission");
var batchSize = Constants.MINIBATCH_SIZE;
var recs = OneHotRecordReadOnly.LoadBinary(testsetPath);
var sparseIndices = new int[batchSize][];
var sparseValues = new float[batchSize][];
var labels = new float[batchSize];
var ids = new int[batchSize];
var submissionLines = new List <SubmissionLine>();
var firstBatch = true;
for (var recNo = 0; recNo < recs.Count; recNo++)
{
var record = recs[recNo];
var label = record.Label;
var id = record.Id;
labels[recNo % batchSize] = label;
ids[recNo % batchSize] = id;
record.CopyDataToSparseArray(sparseIndices, sparseValues, recNo % batchSize);
if ((((recNo + 1) % batchSize) == 0) || (recNo == (recs.Count - 1)))
{
network.InputLayer.SetSparseData(sparseValues.ToList(), sparseIndices.ToList());
network.LabelLayer.SetData(labels);
network.Calculate(train: false);
network.CostLayer.CopyToHost();
for (var i = 0; i < batchSize; i++)
{
if (ids[i] == 0)
{
continue;
}
var chance = network.CostLayer.Outputs[i, 1];
var line = new SubmissionLine();
line.Id = ids[i];
line.Chance = chance;
submissionLines.Add(line);
}
Array.Clear(ids, 0, ids.Length);
}
if (recNo % 100000 == 0)
{
Console.WriteLine("line : " + recNo);
}
}
SubmissionLine.SaveSubmission(targetPath, submissionLines);
}
public static void CombineSubmission(string dstPath, params string[] srcFiles)
{
var submissionIndex = new Dictionary <int, SubmissionLine>();
var submissionList = new List <SubmissionLine>();
for (var i = 0; i < srcFiles.Length; i++)
{
var path = srcFiles[i];
Console.WriteLine("Processing submission : " + Path.GetFileName(path));
var lineNo = 0;
foreach (var line in File.ReadLines(path))
{
lineNo++;
if (lineNo == 1)
{
continue;
}
var values = line.Split(',');
var id = Int32.Parse(values[0]);
var chance = float.Parse(values[1], CultureInfo.InvariantCulture);
if (!submissionIndex.ContainsKey(id))
{
var newLine = new SubmissionLine();
newLine.Id = id;
newLine.Chance = 0f;
submissionIndex[id] = newLine;
submissionList.Add(newLine);
}
var subLine = submissionIndex[id];
subLine.Chance += chance;
if (lineNo % 1000000 == 0)
{
Console.WriteLine("line : " + lineNo);
}
}
}
foreach (var submission in submissionList)
{
submission.Chance = submission.Chance / srcFiles.Length;
}
SubmissionLine.SaveSubmission(dstPath, submissionList);
}
public static void CombineSubmission(string dstPath, params string[] srcFiles)
{
var submissionIndex = new Dictionary<int, SubmissionLine>();
var submissionList = new List<SubmissionLine>();
for (var i = 0; i < srcFiles.Length; i++)
{
var path = srcFiles[i];
Console.WriteLine("Processing submission : " + Path.GetFileName(path));
var lineNo = 0;
foreach (var line in File.ReadLines(path))
{
lineNo++;
if (lineNo == 1) continue;
var values = line.Split(',');
var id = Int32.Parse(values[0]);
var chance = float.Parse(values[1], CultureInfo.InvariantCulture);
if (!submissionIndex.ContainsKey(id))
{
var newLine = new SubmissionLine();
newLine.Id = id;
newLine.Chance = 0f;
submissionIndex[id] = newLine;
submissionList.Add(newLine);
}
var subLine = submissionIndex[id];
subLine.Chance += chance;
if (lineNo % 1000000 == 0)
{
Console.WriteLine("line : " + lineNo);
}
}
}
foreach (var submission in submissionList)
{
submission.Chance = submission.Chance / srcFiles.Length;
}
SubmissionLine.SaveSubmission(dstPath, submissionList);
}
public static void MakeSubmission(Network network, string testsetPath, string targetPath)
{
Console.WriteLine("Making submission");
var batchSize = Constants.MINIBATCH_SIZE;
var recs = OneHotRecordReadOnly.LoadBinary(testsetPath);
var sparseIndices = new int[batchSize][];
var sparseValues = new float[batchSize][];
var labels = new float[batchSize];
var ids = new int[batchSize];
var submissionLines = new List<SubmissionLine>();
var firstBatch = true;
for (var recNo = 0; recNo < recs.Count; recNo++)
{
var record = recs[recNo];
var label = record.Label;
var id = record.Id;
labels[recNo % batchSize] = label;
ids[recNo % batchSize] = id;
record.CopyDataToSparseArray(sparseIndices, sparseValues, recNo % batchSize);
if ((((recNo + 1) % batchSize) == 0) || (recNo == (recs.Count-1)))
{
network.InputLayer.SetSparseData(sparseValues.ToList(), sparseIndices.ToList());
network.LabelLayer.SetData(labels);
network.Calculate(train: false);
network.CostLayer.CopyToHost();
for (var i = 0; i < batchSize; i++)
{
if (ids[i] == 0) continue;
var chance = network.CostLayer.Outputs[i, 1];
var line = new SubmissionLine();
line.Id = ids[i];
line.Chance = chance;
submissionLines.Add(line);
}
Array.Clear(ids, 0, ids.Length);
}
if (recNo % 100000 == 0)
{
Console.WriteLine("line : " + recNo);
}
}
SubmissionLine.SaveSubmission(targetPath, submissionLines);
}
