/// <summary>Scan over the given range for lines</summary> private static List <RangeI> ScanForLines(Stream src, State d, bool bwd_first, RangeI scan_range, int bwd_lines, int fwd_lines, byte[] buf, ProgressFunc progress) { var line_index = new List <RangeI>(); if (bwd_lines == 0 && fwd_lines == 0) { return(line_index); } // Line index buffers for collecting the results var fwd_line_buf = new List <RangeI>(fwd_lines); var bwd_line_buf = new List <RangeI>(bwd_lines); // Data used in the 'add_line' callback. Updated for forward and backward passes var lbd = new LineBufferData { line_buf = null, // pointer to either 'fwd_line_buf' or 'bwd_line_buf' line_limit = 0, // Caps the number of lines read for each of the forward and backward searches }; // Callback for adding line byte ranges to a line buffer bool AddLine(RangeI line, long baddr, long fend, byte[] bf, Encoding enc) { if (line.Empty && d.m_ignore_blanks) { return(true); } // Test 'text' against each filter to see if it's included // Note: not caching this string because we want to read immediate data // from the file to pick up file changes. string text = d.m_encoding.GetString(buf, (int)line.Beg, (int)line.Size); if (!PassesFilters(text, d.m_filters)) { return(true); } // Convert the byte range to a file range line = line.Shift(baddr); Debug.Assert(new RangeI(0, d.m_fileend).Contains(line)); lbd.line_buf.Add(line); Debug.Assert(lbd.line_buf.Count <= lbd.line_limit); return((fwd_line_buf.Count + bwd_line_buf.Count) < lbd.line_limit); } // Callback for updating progress bool Progress(long _, long __) { var scanned = fwd_line_buf.Count + bwd_line_buf.Count; return(progress(scanned, lbd.line_limit)); } // Scan twice, starting in the direction of the smallest range so that any // unused cache space is used by the search in the other direction. var scan_from = Math_.Clamp(d.m_filepos, scan_range.Beg, scan_range.End); for (int a = 0; a != 2; ++a, bwd_first = !bwd_first) { lbd.line_buf = bwd_first ? bwd_line_buf : fwd_line_buf; lbd.line_limit += bwd_first ? bwd_lines : fwd_lines; if ((bwd_line_buf.Count + fwd_line_buf.Count) < lbd.line_limit) { var length = bwd_first ? scan_from - scan_range.Beg : scan_range.End - scan_from; if (!FindLines(src, scan_from, d.m_fileend, bwd_first, length, AddLine, d.m_encoding, d.m_line_end, buf, Progress)) { break; } } } // Scanning backward adds lines to the line index in reverse order. bwd_line_buf.Reverse(); // 'line_index' should be a contiguous block of byte offset ranges for // the lines around 'd.m_filepos'. If 'reload' is false, then the line // index will only contain byte offset ranges that are not currently cached. line_index.Capacity = bwd_line_buf.Count + fwd_line_buf.Count; line_index.AddRange(bwd_line_buf); line_index.AddRange(fwd_line_buf); // Return the found line ranges return(line_index); }
/// <summary>The grunt work of building the new line index.</summary> private static void BuildLineIndexAsync(BLIData d, Action <BLIData, RangeI, List <RangeI>, Exception> on_complete) { // This method runs in a background thread // All we're doing here is loading data around 'd.filepos' so that there are an equal number // of lines on either side. This can be optimised however because the existing range of // cached data probably overlaps the range we want loaded. try { Log.Write(ELogLevel.Info, "BLIAsync", $"build started. (id {d.build_issue}, reload {d.reload})"); if (BuildCancelled(d.build_issue)) { return; } using (d.file) { // A temporary buffer for reading sections of the file var buf = new byte[d.max_line_length]; // Seek to the first line that starts immediately before 'filepos' d.filepos = FindLineStart(d.file, d.filepos, d.fileend, d.row_delim, d.encoding, buf); if (BuildCancelled(d.build_issue)) { return; } // Determine the range to scan and the number of lines in each direction var scan_backward = (d.fileend - d.filepos) > (d.filepos - 0); // scan in the most bound direction first var scan_range = CalcBufferRange(d.filepos, d.fileend, d.file_buffer_size); var line_range = CalcLineRange(d.line_cache_count); var bwd_lines = line_range.Begi; var fwd_lines = line_range.Endi; // Incremental loading - only load what isn't already cached. // If the 'filepos' is left of the cache centre, try to extent in left direction first. // If the scan range in that direction is empty, try extending at the other end. The // aim is to try to get d.line_index_count as close to d.line_cache_count as possible // without loading data that is already cached. #region Incremental loading if (!d.reload && !d.cached_whole_line_range.Empty) { // Determine the direction the cached range is moving based on where 'filepos' is relative // to the current cache centre and which range contains an valid area to be scanned. // With incremental scans we can only update one side of the cache because the returned line index has to // be a contiguous block of lines. This means one of 'bwd_lines' or 'fwd_lines' must be zero. var Lrange = new RangeI(scan_range.Beg, d.cached_whole_line_range.Beg); var Rrange = new RangeI(d.cached_whole_line_range.End, scan_range.End); var dir = (!Lrange.Empty && !Rrange.Empty) ? Math.Sign(2 * d.filepos_line_index - d.line_cache_count) : (!Lrange.Empty) ? -1 : (!Rrange.Empty) ? +1 : 0; // Determine the number of lines to scan, based on direction if (dir < 0) { scan_backward = true; scan_range = Lrange; bwd_lines -= Math_.Clamp(d.filepos_line_index - 0, 0, bwd_lines); fwd_lines = 0; } else if (dir > 0) { scan_backward = false; scan_range = Rrange; bwd_lines = 0; fwd_lines -= Math_.Clamp(d.line_index_count - d.filepos_line_index - 1, 0, fwd_lines); } else if (dir == 0) { bwd_lines = 0; fwd_lines = 0; scan_range = RangeI.Zero; } } #endregion Debug.Assert(bwd_lines + fwd_lines <= d.line_cache_count); // Build the collection of line byte ranges to add to the cache var line_index = new List <RangeI>(); if (bwd_lines != 0 || fwd_lines != 0) { // Line index buffers for collecting the results var fwd_line_buf = new List <RangeI>(); var bwd_line_buf = new List <RangeI>(); // Data used in the 'add_line' callback. Updated for forward and backward passes var lbd = new LineBufferData { line_buf = null, // pointer to either 'fwd_line_buf' or 'bwd_line_buf' line_limit = 0, // Caps the number of lines read for each of the forward and backward searches }; // Callback for adding line byte ranges to a line buffer AddLineFunc add_line = (line, baddr, fend, bf, enc) => { if (line.Empty && d.ignore_blanks) { return(true); } // Test 'text' against each filter to see if it's included // Note: not caching this string because we want to read immediate data // from the file to pick up file changes. string text = d.encoding.GetString(buf, (int)line.Beg, (int)line.Size); if (!PassesFilters(text, d.filters)) { return(true); } // Convert the byte range to a file range line = line.Shift(baddr); Debug.Assert(new RangeI(0, d.fileend).Contains(line)); lbd.line_buf.Add(line); Debug.Assert(lbd.line_buf.Count <= lbd.line_limit); return((fwd_line_buf.Count + bwd_line_buf.Count) < lbd.line_limit); }; // Callback for updating progress ProgressFunc progress = (scanned, length) => { int numer = fwd_line_buf.Count + bwd_line_buf.Count, denom = lbd.line_limit; return(d.progress(numer, denom) && !BuildCancelled(d.build_issue)); }; // Scan twice, starting in the direction of the smallest range so that any // unused cache space is used by the search in the other direction var scan_from = Math_.Clamp(d.filepos, scan_range.Beg, scan_range.End); for (int a = 0; a != 2; ++a, scan_backward = !scan_backward) { if (BuildCancelled(d.build_issue)) { return; } lbd.line_buf = scan_backward ? bwd_line_buf : fwd_line_buf; lbd.line_limit += scan_backward ? bwd_lines : fwd_lines; if ((bwd_line_buf.Count + fwd_line_buf.Count) < lbd.line_limit) { var length = scan_backward ? scan_from - scan_range.Beg : scan_range.End - scan_from; FindLines(d.file, scan_from, d.fileend, scan_backward, length, add_line, d.encoding, d.row_delim, buf, progress); } } // Scanning backward adds lines to the line index in reverse order. bwd_line_buf.Reverse(); // 'line_index' should be a contiguous block of byte offset ranges for // the lines around 'd.filepos'. If 'd.reload' is false, then the line // index will only contain byte offset ranges that are not currently cached. line_index.Capacity = bwd_line_buf.Count + fwd_line_buf.Count; line_index.AddRange(bwd_line_buf); line_index.AddRange(fwd_line_buf); } // Job done on_complete(d, scan_range, line_index, null); } } catch (Exception ex) { on_complete(d, RangeI.Zero, null, ex); } }