/// <summary>
/// Get a path relative to the base path.
/// </summary>
/// <param name="basepath">A base path.</param>
/// <param name="files">A list of files under the base path.</param>
/// <returns>A realtive file path.</returns>
private string GetRelativePath(string basepath, IMultiStreamSource files)
{
Contracts.CheckNonEmpty(basepath, nameof(basepath));
string path = files.GetPathOrNull(0);
_host.CheckNonEmpty(path, nameof(path));
var relativePath = PartitionedPathUtils.MakePathRelative(basepath, path);
return relativePath;
}
private void ThreadProc()
{
Contracts.Assert(_batchSize >= 2);
try
{
if (_limit <= 0)
{
return;
}
long total = 0;
long batch = -1;
for (int ifile = 0; ifile < _files.Count; ifile++)
{
string path = _files.GetPathOrNull(ifile);
using (var rdr = _files.OpenTextReader(ifile))
{
string text;
long line = 0;
for (; ;)
{
// REVIEW: Avoid allocating a string for every line. This would probably require
// introducing a CharSpan type (similar to DvText but based on char[] or StringBuilder)
// and implementing all the necessary conversion functionality on it. See task 3871.
text = rdr.ReadLine();
if (text == null)
{
goto LNext;
}
line++;
if (text.Length > 0 && text[0] != '#' && !text.StartsWith("//"))
{
break;
}
}
// REVIEW: Use a pool of batches?
int index = 0;
var infos = new LineInfo[_batchSize];
if (!_hasHeader)
{
// Not a header or comment, so first line is a real line.
infos[index++] = new LineInfo(line, text);
if (++total >= _limit)
{
PostPartial(path, total - index, ref batch, index, infos);
return;
}
}
for (; ;)
{
if (_abort)
{
return;
}
text = rdr.ReadLine();
if (text == null)
{
// We're done with this file. Queue the last partial batch.
PostPartial(path, total - index, ref batch, index, infos);
goto LNext;
}
line++;
// Filter out comments and empty strings.
if (text.Length >= 2)
{
// Don't use string.StartsWith("//") - it is too slow.
if (text[0] == '/' && text[1] == '/')
{
continue;
}
}
else if (text.Length == 0)
{
continue;
}
infos[index] = new LineInfo(line, text);
if (++index >= infos.Length)
{
batch++;
var lines = new LineBatch(path, total - index + 1, batch, infos);
while (!_queue.TryAdd(lines, TimeOut))
{
if (_abort)
{
return;
}
}
infos = new LineInfo[_batchSize];
index = 0;
}
if (++total >= _limit)
{
PostPartial(path, total - index, ref batch, index, infos);
return;
}
}
LNext:
;
}
}
}
catch (Exception ex)
{
while (!_queue.TryAdd(new LineBatch(ex), TimeOut))
{
if (_abort)
{
return;
}
}
}
finally
{
_queue.CompleteAdding();
}
}