public static void MakeSubmission(Network network, string testsetPath, string targetPath) { Console.WriteLine("Making submission"); var batchSize = Constants.MINIBATCH_SIZE; var recs = OneHotRecordReadOnly.LoadBinary(testsetPath); var sparseIndices = new int[batchSize][]; var sparseValues = new float[batchSize][]; var labels = new float[batchSize]; var ids = new int[batchSize]; var submissionLines = new List <SubmissionLine>(); var firstBatch = true; for (var recNo = 0; recNo < recs.Count; recNo++) { var record = recs[recNo]; var label = record.Label; var id = record.Id; labels[recNo % batchSize] = label; ids[recNo % batchSize] = id; record.CopyDataToSparseArray(sparseIndices, sparseValues, recNo % batchSize); if ((((recNo + 1) % batchSize) == 0) || (recNo == (recs.Count - 1))) { network.InputLayer.SetSparseData(sparseValues.ToList(), sparseIndices.ToList()); network.LabelLayer.SetData(labels); network.Calculate(train: false); network.CostLayer.CopyToHost(); for (var i = 0; i < batchSize; i++) { if (ids[i] == 0) { continue; } var chance = network.CostLayer.Outputs[i, 1]; var line = new SubmissionLine(); line.Id = ids[i]; line.Chance = chance; submissionLines.Add(line); } Array.Clear(ids, 0, ids.Length); } if (recNo % 100000 == 0) { Console.WriteLine("line : " + recNo); } } SubmissionLine.SaveSubmission(targetPath, submissionLines); }
public static void CombineSubmission(string dstPath, params string[] srcFiles) { var submissionIndex = new Dictionary <int, SubmissionLine>(); var submissionList = new List <SubmissionLine>(); for (var i = 0; i < srcFiles.Length; i++) { var path = srcFiles[i]; Console.WriteLine("Processing submission : " + Path.GetFileName(path)); var lineNo = 0; foreach (var line in File.ReadLines(path)) { lineNo++; if (lineNo == 1) { continue; } var values = line.Split(','); var id = Int32.Parse(values[0]); var chance = float.Parse(values[1], CultureInfo.InvariantCulture); if (!submissionIndex.ContainsKey(id)) { var newLine = new SubmissionLine(); newLine.Id = id; newLine.Chance = 0f; submissionIndex[id] = newLine; submissionList.Add(newLine); } var subLine = submissionIndex[id]; subLine.Chance += chance; if (lineNo % 1000000 == 0) { Console.WriteLine("line : " + lineNo); } } } foreach (var submission in submissionList) { submission.Chance = submission.Chance / srcFiles.Length; } SubmissionLine.SaveSubmission(dstPath, submissionList); }
public static void CombineSubmission(string dstPath, params string[] srcFiles) { var submissionIndex = new Dictionary<int, SubmissionLine>(); var submissionList = new List<SubmissionLine>(); for (var i = 0; i < srcFiles.Length; i++) { var path = srcFiles[i]; Console.WriteLine("Processing submission : " + Path.GetFileName(path)); var lineNo = 0; foreach (var line in File.ReadLines(path)) { lineNo++; if (lineNo == 1) continue; var values = line.Split(','); var id = Int32.Parse(values[0]); var chance = float.Parse(values[1], CultureInfo.InvariantCulture); if (!submissionIndex.ContainsKey(id)) { var newLine = new SubmissionLine(); newLine.Id = id; newLine.Chance = 0f; submissionIndex[id] = newLine; submissionList.Add(newLine); } var subLine = submissionIndex[id]; subLine.Chance += chance; if (lineNo % 1000000 == 0) { Console.WriteLine("line : " + lineNo); } } } foreach (var submission in submissionList) { submission.Chance = submission.Chance / srcFiles.Length; } SubmissionLine.SaveSubmission(dstPath, submissionList); }
public static void MakeSubmission(Network network, string testsetPath, string targetPath) { Console.WriteLine("Making submission"); var batchSize = Constants.MINIBATCH_SIZE; var recs = OneHotRecordReadOnly.LoadBinary(testsetPath); var sparseIndices = new int[batchSize][]; var sparseValues = new float[batchSize][]; var labels = new float[batchSize]; var ids = new int[batchSize]; var submissionLines = new List<SubmissionLine>(); var firstBatch = true; for (var recNo = 0; recNo < recs.Count; recNo++) { var record = recs[recNo]; var label = record.Label; var id = record.Id; labels[recNo % batchSize] = label; ids[recNo % batchSize] = id; record.CopyDataToSparseArray(sparseIndices, sparseValues, recNo % batchSize); if ((((recNo + 1) % batchSize) == 0) || (recNo == (recs.Count-1))) { network.InputLayer.SetSparseData(sparseValues.ToList(), sparseIndices.ToList()); network.LabelLayer.SetData(labels); network.Calculate(train: false); network.CostLayer.CopyToHost(); for (var i = 0; i < batchSize; i++) { if (ids[i] == 0) continue; var chance = network.CostLayer.Outputs[i, 1]; var line = new SubmissionLine(); line.Id = ids[i]; line.Chance = chance; submissionLines.Add(line); } Array.Clear(ids, 0, ids.Length); } if (recNo % 100000 == 0) { Console.WriteLine("line : " + recNo); } } SubmissionLine.SaveSubmission(targetPath, submissionLines); }