/// <summary> /// Get a path relative to the base path. /// </summary> /// <param name="basepath">A base path.</param> /// <param name="files">A list of files under the base path.</param> /// <returns>A realtive file path.</returns> private string GetRelativePath(string basepath, IMultiStreamSource files) { Contracts.CheckNonEmpty(basepath, nameof(basepath)); string path = files.GetPathOrNull(0); _host.CheckNonEmpty(path, nameof(path)); var relativePath = PartitionedPathUtils.MakePathRelative(basepath, path); return relativePath; }
private void ThreadProc() { Contracts.Assert(_batchSize >= 2); try { if (_limit <= 0) { return; } long total = 0; long batch = -1; for (int ifile = 0; ifile < _files.Count; ifile++) { string path = _files.GetPathOrNull(ifile); using (var rdr = _files.OpenTextReader(ifile)) { string text; long line = 0; for (; ;) { // REVIEW: Avoid allocating a string for every line. This would probably require // introducing a CharSpan type (similar to DvText but based on char[] or StringBuilder) // and implementing all the necessary conversion functionality on it. See task 3871. text = rdr.ReadLine(); if (text == null) { goto LNext; } line++; if (text.Length > 0 && text[0] != '#' && !text.StartsWith("//")) { break; } } // REVIEW: Use a pool of batches? int index = 0; var infos = new LineInfo[_batchSize]; if (!_hasHeader) { // Not a header or comment, so first line is a real line. infos[index++] = new LineInfo(line, text); if (++total >= _limit) { PostPartial(path, total - index, ref batch, index, infos); return; } } for (; ;) { if (_abort) { return; } text = rdr.ReadLine(); if (text == null) { // We're done with this file. Queue the last partial batch. PostPartial(path, total - index, ref batch, index, infos); goto LNext; } line++; // Filter out comments and empty strings. if (text.Length >= 2) { // Don't use string.StartsWith("//") - it is too slow. if (text[0] == '/' && text[1] == '/') { continue; } } else if (text.Length == 0) { continue; } infos[index] = new LineInfo(line, text); if (++index >= infos.Length) { batch++; var lines = new LineBatch(path, total - index + 1, batch, infos); while (!_queue.TryAdd(lines, TimeOut)) { if (_abort) { return; } } infos = new LineInfo[_batchSize]; index = 0; } if (++total >= _limit) { PostPartial(path, total - index, ref batch, index, infos); return; } } LNext: ; } } } catch (Exception ex) { while (!_queue.TryAdd(new LineBatch(ex), TimeOut)) { if (_abort) { return; } } } finally { _queue.CompleteAdding(); } }