private unsafe static COLUMNS.MGSpan[] ParseLine(COLUMNS.MGSpan line, char delimiter, COLUMNS.MGSpan[] columns)
{
//return line.Split(delimiter);
int col = 0;
int linelen = line.Count + line.Start;
int index = line.Start;
fixed(char *l = line.buf)
{
while (index < linelen)
{
if (*(l + index) != '\"')
{
// non quoted
var next = -1;
for (int i = index; i < linelen; i++)
{
if (*(l + i) == delimiter)
{
next = i;
break;
}
}
if (next < 0)
{
columns[col++] = new COLUMNS.MGSpan(line.buf, index, linelen - index);
break;
}
columns[col++] = new COLUMNS.MGSpan(line.buf, index, next - index);
index = next + 1;
}
else
{
// quoted string change "" -> "
int qc = 1;
int start = index;
char c = *(l + ++index);
// find matching quote until delim or EOL
while (index++ < linelen)
{
if (c == '\"')
{
qc++;
}
if (c == delimiter && qc % 2 == 0)
{
break;
}
c = *(l + index);
}
var s = new string(line.buf, start + 1, index - start - 3).Replace("\"\"", "\""); // ugly
columns[col++] = new COLUMNS.MGSpan(s.ToCharArray(), 0, s.Length);
}
}
}
return(columns);
}
public static List <T> ReadFile <T>(string filename, bool hasheader, char delimiter, ToOBJ <T> mapper) //where T : new()
{
COLUMNS.MGSpan[] cols;
List <T> list = new List <T>(10000);
int linenum = 0;
CreateObject co = FastCreateInstance <T>();
var br = new BufReader(File.OpenText(filename), 64 * 1024);
var line = br.ReadLine();
if (line.Count == 0)
{
return(list);
}
if (hasheader)
{
// actual col count
int cc = CountOccurence(line, delimiter);
if (cc == 0)
{
throw new Exception("File does not have '" + delimiter + "' as a delimiter");
}
cols = new COLUMNS.MGSpan[cc + 1];
}
else
{
cols = new COLUMNS.MGSpan[_COLCOUNT];
}
while (true)
{
try
{
line = br.ReadLine();
linenum++;
if (line.Count == 0)
{
break;
}
var c = ParseLine(line, delimiter, cols);
T o = (T)co();
//new T();
var b = mapper(o, new COLUMNS(c));
if (b)
{
list.Add(o);
}
}
catch (Exception ex)
{
throw new Exception("error on line " + linenum + "\r\n" + line, ex);
}
}
return(list);
}
private unsafe static int CountOccurence(COLUMNS.MGSpan text, char c)
{
int count = 0;
int len = text.Count + text.Start;
int index = text.Start;
fixed(char *s = text.buf)
{
while (index++ < len)
{
char ch = *(s + index);
if (ch == c)
{
count++;
}
}
}
return(count);
}
private static List <T> ReadData <T>(TextReader sr, bool hasheader, int colcount, char delimiter, ToOBJ <T> mapper)
{
COLUMNS.MGSpan[] cols;
List <T> list = new List <T>(10000);
int linenum = 0;
CreateObject co = FastCreateInstance <T>();
var br = new BufReader(sr, 64 * 1024);
COLUMNS.MGSpan line = new COLUMNS.MGSpan();
if (hasheader)
{
line = br.ReadLine();
if (line.Count == 0)
{
return(list);
}
ColName = line.ToString().Split(delimiter);
// actual col count
int cc = CountOccurence(line, delimiter);
if (cc == 0)
{
throw new Exception("File does not have '" + delimiter + "' as a delimiter");
}
cols = new COLUMNS.MGSpan[cc + 1];
}
else
{
cols = new COLUMNS.MGSpan[colcount];
ColName = new string[colcount];
}
while (true)
{
try
{
line = br.ReadLine();
linenum++;
if (line.Count == 0)
{
break;
}
var c = ParseLine(line, delimiter, cols);
T o = (T)co();
//new T();
var b = mapper(o, new COLUMNS(c));
if (b)
{
list.Add(o);
}
}
catch (Exception ex)
{
sr.Close();
throw new Exception("error on line " + linenum + "\r\n" + line, ex);
}
}
sr.Close();
return(list);
}
