| public FillColumn ( int InputKey, float InputVal, int IdxCol ) : void | ||
| InputKey | int | |
| InputVal | float | |
| IdxCol | int | |
| 리턴 | void | |
/*
* Load the label data (single column) and return it in a sparse matrix
*/
public static SparseMatrix LabelDataLoader(string LabelDataFile, int nOutput, string OutputType)
{
Console.WriteLine("==================================================");
Console.WriteLine("Scanning the file: {0}", LabelDataFile);
StreamReader LabelDataStream = new StreamReader(LabelDataFile);
string StrLine;
int nLine = 0;
int nSamples = 0;
List<string> AllRawLabel = new List<string>();
// Load all the data
nLine = 0;
while ((StrLine = LabelDataStream.ReadLine())!=null)
{
AllRawLabel.Add(StrLine);
nLine++;
if (nLine % 10000 == 0)
{
Console.Write("Number of lines (samples): {0}\r", nLine);
}
}
Console.WriteLine("Number of lines (samples): {0}", nLine);
Console.WriteLine("Finished scanning the input data file");
LabelDataStream.Close();
// Parse the raw text into actual labels to be used in the learning algorithm
Console.WriteLine("Loading input data...");
nSamples = nLine;
SparseMatrix LabelData = new SparseMatrix(nOutput, nSamples);
nLine = 0;
switch (OutputType)
{
case "linearCE":
Parallel.For(0, LabelData.nCols, new ParallelOptions { MaxDegreeOfParallelism = MatrixOperation.MaxMultiThreadDegree }, IdxCol =>
{
int[] Key = new int[1];
float[] Val = new float[1];
if (!int.TryParse(AllRawLabel[IdxCol], out Key[0]))
{
Key[0] = (int)float.Parse(AllRawLabel[IdxCol]);
}
Val[0] = 1.0f;
LabelData.FillColumn(Key, Val, IdxCol);
Interlocked.Increment(ref nLine);
if (nLine % 10000 == 0)
{
Console.Write("Number of lines (samples): {0}\r", nLine);
}
});
break;
case "softmaxCE":
Parallel.For(0, LabelData.nCols, new ParallelOptions { MaxDegreeOfParallelism = MatrixOperation.MaxMultiThreadDegree }, IdxCol =>
{
int[] Key = new int[1];
float[] Val = new float[1];
if (!int.TryParse(AllRawLabel[IdxCol], out Key[0]))
{
Key[0] = (int)float.Parse(AllRawLabel[IdxCol]);
}
Val[0] = 1.0f;
LabelData.FillColumn(Key, Val, IdxCol);
Interlocked.Increment(ref nLine);
if (nLine % 10000 == 0)
{
Console.Write("Number of lines (samples): {0}\r", nLine);
}
});
break;
case "linearQuad":
Parallel.For(0, LabelData.nCols, new ParallelOptions { MaxDegreeOfParallelism = MatrixOperation.MaxMultiThreadDegree }, IdxCol =>
{
string[] StrLineSplit = AllRawLabel[IdxCol].Split('\t');
int[] Key = new int[StrLineSplit.Length];
float[] Val = new float[StrLineSplit.Length];
for (int IdxOutput = 0; IdxOutput < StrLineSplit.Length; IdxOutput++)
{
Key[IdxOutput] = IdxOutput;
Val[IdxOutput] = float.Parse(StrLineSplit[IdxOutput]);
}
LabelData.FillColumn(Key, Val, IdxCol);
Interlocked.Increment(ref nLine);
if (nLine % 10000 == 0)
{
Console.Write("Number of lines (samples): {0}\r", nLine);
}
});
break;
default:
throw new Exception("Unknown OutputType are supported.");
}
Console.Write("Number of lines (samples): {0}\n", nLine);
Console.WriteLine("==================================================");
return LabelData;
}
/*
* Load input data (tsv format) and return it in a sparse matrix
*/
public static SparseMatrix InputDataLoader(string InputDataFile, int nInput)
{
Console.WriteLine("==================================================");
// Scan through the entire file to get the number of lines (number of samples)
Console.WriteLine("Scanning the file: {0}", InputDataFile);
StreamReader InputDataStream = new StreamReader(InputDataFile);
string StrLine;
int nLine = 0;
int nSamples = 0;
List<string> AllRawInput = new List<string>();
while ((StrLine = InputDataStream.ReadLine()) != null)
{
AllRawInput.Add(StrLine);
nLine++;
if (nLine % 10000 == 0)
{
Console.Write("Number of lines (samples): {0}\r", nLine);
}
}
nSamples = nLine;
Console.Write("Number of lines (samples): {0}\n", nLine);
Console.WriteLine("Finished scanning the input data file");
InputDataStream.Close();
// Parse each line and store it into each column of the sparse matrix
Console.WriteLine("Loading input data...");
SparseMatrix InputDataMatrix = new SparseMatrix(nInput, nLine);
nLine = 0;
int nTotNonzero = 0;
int nEmptyLine = 0;
Parallel.For(0, InputDataMatrix.nCols, new ParallelOptions { MaxDegreeOfParallelism = MatrixOperation.MaxMultiThreadDegree }, IdxCol =>
{
string[] StrLineSplit = AllRawInput[IdxCol].Split('\t');
int nNonzero = StrLineSplit.Length;
Interlocked.Add(ref nTotNonzero, nNonzero);
int[] Key = null;
float[] Val = null;
Key = new int[nNonzero];
Val = new float[nNonzero];
string[] StrKeyValPair = null;
if (StrLineSplit.Length == 1 && StrLineSplit[0] == "")
{
Key[0] = 0;
Val[0] = 0.0f;
Interlocked.Increment(ref nEmptyLine);
}
else
{
for (int Idx = 0; Idx < nNonzero; Idx++)
{
StrKeyValPair = StrLineSplit[Idx].Split(':');
Key[Idx] = int.Parse(StrKeyValPair[0]);
Val[Idx] = float.Parse(StrKeyValPair[1]);
}
}
InputDataMatrix.FillColumn(Key, Val, IdxCol);
Interlocked.Increment(ref nLine);
if (nLine % 10000 == 0)
{
Console.Write("Number of lines (samples): {0}, with {1} empty lines.\r", nLine, nEmptyLine);
}
});
Console.Write("Number of lines (samples): {0}, with {1} empty lines.\n", nLine, nEmptyLine);
Console.WriteLine("Finished loading the input data file");
Console.WriteLine("# Samples = {0}, # Inputs = {1}", InputDataMatrix.nCols, InputDataMatrix.nRows);
Console.WriteLine("# Nonzeros = {0}/{1} ({2}%)", nTotNonzero, (long)InputDataMatrix.nRows * (long)InputDataMatrix.nCols, (((float)nTotNonzero) / ((float)InputDataMatrix.nRows * InputDataMatrix.nCols)) * 100);
Console.WriteLine("==================================================");
return InputDataMatrix;
}