|
public ReadBlockAsync ( char buffer, int index, int count ) : Task |
||
| buffer | char | |
| index | int | |
| count | int | |
| return | Task |
|
protected static async Task <object> ReadStreamInChuncksAsync(Stream stream, int?maxLength = null)
{
if (stream == null || stream?.Length == 0)
{
return("");
}
string hasMore = maxLength.HasValue ? (stream.Length > maxLength.Value ? "..." : "") : "";
int readChunkBufferength = 4096;
if (maxLength.HasValue && maxLength < readChunkBufferength)
{
readChunkBufferength = maxLength.Value;
}
stream.Seek(0, SeekOrigin.Begin);
using (var textwriter = new StringWriter())
{
using (var reader = new System.IO.StreamReader(stream, System.Text.Encoding.Default, true, 1024, true))
{
var readChunk = new char[readChunkBufferength];
int readChunkLength = 0;
int alreadyRead = 0;
do
{
readChunkLength = await reader.ReadBlockAsync(readChunk, 0, readChunkBufferength);
alreadyRead += readChunkLength;
await textwriter.WriteAsync(readChunk, 0, readChunkLength);
if (maxLength.HasValue && alreadyRead + readChunkBufferength > maxLength)
{
readChunkBufferength = maxLength.Value - alreadyRead;
}
}while (readChunkLength > 0 && readChunkBufferength > 0);
if (hasMore.Length > 0)
{
return(textwriter.ToString() + hasMore);
}
else
{
try
{
return(System.Text.Json.JsonSerializer.Deserialize <System.Text.Json.JsonElement>(textwriter.ToString()));
}
catch
{
return(textwriter.ToString());
}
}
}
}
stream.Seek(0, SeekOrigin.Begin);
}
public override async Task ReadAsync(StreamReader reader)
{
await base.ReadAsync(reader);
// Line ending
var buffer = new char[2];
var read = await reader.ReadBlockAsync(buffer, 0, buffer.Length);
if (read != buffer.Length || buffer != "\r\n".ToArray())
throw new LeituraException("Final de linha não encontrado.");
}
public async Task ProcessArgs(string[] args)
{
if (args.Length == 0)
{
ShowHelp();
}
else
{
var errors = _options.Parse(args);
if (errors.Count == 0)
{
if (ValidateOptions())
{
var watch = Stopwatch.StartNew();
Language language = LanguageFactory.CreateLanguage(_language);
const int bufferSize = 1024 * 8;
bool done = false;
const int parserThreads = 8;
ConcurrentQueue<string>[] buckets = InitializeConcurrentQueueTiles(parserThreads);
var producer = Task.Factory.StartNew(() =>
{
int index = 0;
foreach (var file in Directory.EnumerateFiles(_directory, language.Pattern, SearchOption.AllDirectories))
{
if (_ignorePattern == null || _ignorePattern.IsMatch(file) == false)
{
var tile = buckets[index++ % parserThreads];
tile.Enqueue(file);
}
}
lock (_out)
{
_out.WriteLine("Added {0} files to be processed.", index);
}
done = true;
});
Task[] consumers = new Task[parserThreads];
Parser parser = language.Parser;
for (int i = 0; i < parserThreads; i++)
{
var tile = buckets[i];
consumers[i] = Task.Factory.StartNew(async () =>
{
while (!done)
{
string file;
if (!tile.TryDequeue(out file))
continue;
Document document = new Document(file);
using (var sr = new StreamReader(file, Encoding.UTF8, true, bufferSize))
{
char[] buffer = new char[bufferSize];
while (true)
{
int bytesRead = await sr.ReadBlockAsync(buffer, 0, bufferSize).ConfigureAwait(false);
if (bytesRead == 0)
break;
// todo: pass bytesRead into the parser?
// maybe the bounds check would get eliminated?
parser.ParseChunk(document, buffer, (comment) =>
{
var result = TextAnalyzer.Analyze(comment.Text);
double score = Score(comment, result);
if (score > 0)
{
lock (_out)
{
_out.WriteLine(string.Format("{0} [{1}-{2}]: {3}", comment.File, comment.Start, comment.End, comment.Text));
}
}
});
}
}
}
}, TaskCreationOptions.PreferFairness);
}
await producer;
await Task.WhenAll(consumers);
watch.Stop();
_out.WriteLine("elapsed: " + watch.ElapsedMilliseconds + "ms");
}
}
else
{
foreach (var error in errors)
{
_error.WriteLine("Invalid Argument: {0}", error);
}
}
}
_out.WriteLine();
_error.Flush();
_out.Flush();
}
public AsyncReader(StreamReader reader, int length)
{
_buffer = new char[length];
_readerTask = reader.ReadBlockAsync(_buffer, 0, length);
}
private async Task ReadTextAsync(string path)
{
text.Text = "";
using (StreamReader file = new StreamReader(path))
{
status.Text = "";
status.Foreground = Brushes.Red;
currentCount = 0;
switch (loadMethod)
{
case LoadMethod.LineByLine:
// ==== Loading the file one line at a time ====
// Too much updating of the form, so it is essentially frozen.
// Takes the longest to load, by far.
string line;
while ((line = await file.ReadLineAsync()) != null)
{
text.AppendText(line + Environment.NewLine);
currentCount++;
status.Text = currentCount.ToString() + " lines";
}
break;
case LoadMethod.Block:
// ==== Loading the file in blocks ====
// Note: The main freeze in load time is in the final refreshing of the WPF form.
// The size of the buffer, set by 'count', is critical in how noticeable this is.
// At 2,000 (3:00 min load!, 270 MB memory), the forms starts out smooth but it is harder to interact with the frequency of loading.
// Changing the form causes a long freeze before it can resume upating the layout.
// At 20,000 (1:00 min load, 325 MB memory), the form is reasonably responsive and does not freeze up bad when text is typed.
// It seems this is the buffer sweet spot for size.
// Second loading causes big jump in memory, which is dropped near the end to be back near the original state.
// At 200,000 (1:00 min load, 450 MB memory), the blocks appended are large enough to freeze the form.
int index = 0;
int count = 2000;
int bufferCount = 0;
int bufferInterval = 10;
char[] buffer = new char[count];
StringBuilder blockTextComplete = new StringBuilder();
while ( await file.ReadBlockAsync(buffer, index, count) >= count)
{
blockTextComplete.Append(new string(buffer));
// Load first block to let user work with it while the rest loads
// After, load at buffer intervals
if (currentCount == 0 || bufferCount == bufferInterval * count)
{
text.AppendText(await AppendBlock(blockTextComplete));
blockTextComplete.Clear();
bufferCount = 0;
}
bufferCount += count;
currentCount += count;
status.Text = currentCount.ToString() + " characters";
}
text.AppendText(await AppendBlock(blockTextComplete));
blockTextComplete.Clear();
break;
case LoadMethod.File:
// ==== Loading the entire file at once ====
// 5-10 sec, 350 MB memory. Second load is much longer and takes double the memory.
// Loading in WPF is very fast. It seems that it is much faster to load the data all at once than in many increments, including total frozen time on the form.
stopwatch.Tick += stopwatch_Tick;
stopwatch.Start();
text.Text = await file.ReadToEndAsync();
if (text.Text != null && stopwatch.IsEnabled)
{
stopwatch.Stop();
}
break;
default:
break;
}
status.Foreground = Brushes.Green;
status.Text = "Loaded!";
}
textChanged = false;
}
/// <summary>
/// Formats the contents of an HTTP request or response body into a string.
/// </summary>
/// <param name="bodyStream"></param>
/// <returns>
/// The request or response body stream to use (must replace the current stream).
/// </returns>
private async Task<string> FormatBodyStreamAsync(Stream bodyStream)
{
if ((bodyStream == null) || !bodyStream.CanRead || !bodyStream.CanSeek)
{
return string.Empty;
}
// Creo un vettore di caratteri usato come buffer temporaneo.
var buffer = new char[512];
// E' fondamentale che lo stream del body rimanga aperto.
using (var bodyReader = new StreamReader(bodyStream, PortableEncoding.UTF8WithoutBOM, false, buffer.Length, true))
using (var psb = _stringBuilderPool.GetObject())
using (var stringWriter = new StringWriter(psb.StringBuilder, CultureInfo.InvariantCulture))
{
// Riporto all'inizio la posizione dello stream per poterlo copiare in una stringa.
bodyStream.Position = 0;
// Avvio il ciclo di lettura a blocchi.
var readIndex = 0;
var readChars = 0;
do
{
// Leggo un blocco di caratteri dallo stream.
readChars = await bodyReader.ReadBlockAsync(buffer, 0, buffer.Length);
readIndex += readChars;
// Scrivo i caratteri letti.
await stringWriter.WriteAsync(buffer, 0, readChars);
} while (readChars == buffer.Length && readIndex < _settings.MaxLoggedBodyLength);
// Riporto all'inizio la posizione dello stream per consentire agli altri di leggerlo.
bodyStream.Position = 0;
return stringWriter.ToString();
}
}
