static bool HeuristicallyDetectWhetherMultithreadingMakesSense(CreateParserParams parserParams, TextStreamPositioningParams textStreamPositioningParams) { #if SILVERLIGHT return(false); #else if (System.Environment.ProcessorCount == 1) { return(false); } long approxBytesToRead; if (parserParams.Direction == MessagesParserDirection.Forward) { approxBytesToRead = new TextStreamPosition(parserParams.Range.Value.End, textStreamPositioningParams).StreamPositionAlignedToBlockSize - new TextStreamPosition(parserParams.StartPosition, textStreamPositioningParams).StreamPositionAlignedToBlockSize; } else { approxBytesToRead = new TextStreamPosition(parserParams.StartPosition, textStreamPositioningParams).StreamPositionAlignedToBlockSize - new TextStreamPosition(parserParams.Range.Value.Begin, textStreamPositioningParams).StreamPositionAlignedToBlockSize; } if (approxBytesToRead < MultiThreadedStrategy <int> .GetBytesToParsePerThread(textStreamPositioningParams) * 2) { return(false); } return(true); #endif }
public IPositionedMessagesParser CreateParser(CreateParserParams parserParams) { parserParams.EnsureRangeIsSet(this); var strategiesCache = new StreamParser.StrategiesCache() { MultiThreadedStrategy = multiThreadedStrategy, SingleThreadedStrategy = singleThreadedStrategy }; DejitteringParams?dejitteringParams = GetDejitteringParams(); if (dejitteringParams != null && (parserParams.Flags & MessagesParserFlag.DisableDejitter) == 0) { return(new DejitteringMessagesParser( underlyingParserParams => new StreamParser( this, EnsureParserRangeDoesNotExceedReadersBoundaries(underlyingParserParams), textStreamPositioningParams, settingsAccessor, strategiesCache ), parserParams, dejitteringParams.Value )); } return(new StreamParser( this, parserParams, textStreamPositioningParams, settingsAccessor, strategiesCache )); }
private CreateParserParams EnsureParserRangeDoesNotExceedReadersBoundaries(CreateParserParams p) { if (p.Range != null) { p.Range = FileRange.Range.Intersect(p.Range.Value, new FileRange.Range(BeginPosition, EndPosition)).Common; } return(p); }
public static async Task <StreamParser> Create( IPositionedMessagesReader owner, CreateParserParams p, TextStreamPositioningParams textStreamPositioningParams, IGlobalSettingsAccessor globalSettings, StrategiesCache strategiesCache) { var parser = new StreamParser(owner, p, textStreamPositioningParams, globalSettings, strategiesCache); await parser.Strategy.ParserCreated(parser.InitialParams); return(parser); }
async Task CreateUnderlyingParserAndInitJitterBuffer(Func <CreateParserParams, Task <IPositionedMessagesParser> > underlyingParserFactory) { CreateParserParams reversedParserParams = originalParams; reversedParserParams.Range = null; reversedParserParams.Direction = GetOppositeDirection(originalParams.Direction); reversedParserParams.Flags |= MessagesParserFlag.DisableMultithreading; int reversedMessagesQueued = 0; await DisposableAsync.Using(await underlyingParserFactory(reversedParserParams), async reversedParser => { var tmp = new List <PostprocessedMessage>(); for (int i = 0; i < jitterBufferSize; ++i) { var tmpMsg = await reversedParser.ReadNextAndPostprocess(); if (tmpMsg.Message == null) { break; } tmp.Add(tmpMsg); } tmp.Reverse(); foreach (var tmpMsg in tmp) { jitterBuffer.Enqueue(new Entry() { data = tmpMsg, index = currentIndex++ }); positionsBuffer.Push(new MessagesPositions(tmpMsg.Message)); ++reversedMessagesQueued; } }); enumerator = await ReadAddMessagesFromRangeCompleteJitterBuffer(underlyingParserFactory).GetEnumerator(); for (int i = 0; i < jitterBufferSize; ++i) { var tmp = await LoadNextMessage(); reversedMessagesQueued -= tmp.DequeuedMessages; if (tmp.LoadedMessage == null) { break; } } for (int i = 0; i < reversedMessagesQueued && jitterBuffer.Count > 0; ++i) { jitterBuffer.Dequeue(); positionsBuffer.Pop(); } }
private StreamParser( IPositionedMessagesReader owner, CreateParserParams p, TextStreamPositioningParams textStreamPositioningParams, IGlobalSettingsAccessor globalSettings, StrategiesCache strategiesCache ) { p.EnsureRangeIsSet(owner); this.InitialParams = p; this.isSequentialReadingParser = (p.Flags & MessagesParserFlag.HintParserWillBeUsedForMassiveSequentialReading) != 0; this.multithreadingDisabled = (p.Flags & MessagesParserFlag.DisableMultithreading) != 0 || globalSettings.MultithreadedParsingDisabled; CreateParsingStrategy(p, textStreamPositioningParams, strategiesCache, out this.Strategy); }
private DejitteringMessagesParser(CreateParserParams originalParams, int jitterBufferSize) { if (jitterBufferSize < 1) { throw new ArgumentException("jitterBufferSize must be equal to or geater than 1"); } if (originalParams.Range == null) { throw new ArgumentNullException("DejitteringMessagesParser does not support unspecified positions range", "originalParams.Range"); } this.originalParams = originalParams; this.originalParams.EnsureStartPositionIsInRange(); this.jitterBufferSize = jitterBufferSize; this.jitterBuffer = new VCSKicksCollection.PriorityQueue <Entry>(new Comparer(originalParams.Direction, jitterBufferSize)); this.positionsBuffer = new Generic.CircularBuffer <MessagesPositions>(jitterBufferSize + 1); }
IEnumerableAsync <PostprocessedMessage> ReadAddMessagesFromRangeCompleteJitterBuffer( Func <CreateParserParams, Task <IPositionedMessagesParser> > underlyingParserFactory) { return(EnumerableAsync.Produce <PostprocessedMessage>(async yieldAsync => { CreateParserParams mainParserParams = originalParams; //mainParserParams.Range = null; await DisposableAsync.Using(await underlyingParserFactory(mainParserParams), async mainParser => { for (; ;) { var msg = await mainParser.ReadNextAndPostprocess(); if (msg.Message == null) { break; } if (!await yieldAsync.YieldAsync(msg)) { break; } } }); CreateParserParams jitterBufferCompletionParams = originalParams; jitterBufferCompletionParams.Flags |= MessagesParserFlag.DisableMultithreading; jitterBufferCompletionParams.Range = null; jitterBufferCompletionParams.StartPosition = originalParams.Direction == MessagesParserDirection.Forward ? originalParams.Range.Value.End : originalParams.Range.Value.Begin; await DisposableAsync.Using(await underlyingParserFactory(jitterBufferCompletionParams), async completionParser => { for (int i = 0; i < jitterBufferSize; ++i) { var msg = await completionParser.ReadNextAndPostprocess(); if (msg.Message == null) { break; } if (!await yieldAsync.YieldAsync(msg)) { break; } } }); })); }
void CreateParsingStrategy( CreateParserParams parserParams, TextStreamPositioningParams textStreamPositioningParams, StrategiesCache strategiesCache, out BaseStrategy strategy) { bool useMultithreadedStrategy; if (multithreadingDisabled) { useMultithreadedStrategy = false; } else if (!isSequentialReadingParser) { useMultithreadedStrategy = false; } else { useMultithreadedStrategy = HeuristicallyDetectWhetherMultithreadingMakesSense(parserParams, textStreamPositioningParams); } //useMultithreadedStrategy = false; Lazy <BaseStrategy> strategyToTryFirst; Lazy <BaseStrategy> strategyToTrySecond; if (useMultithreadedStrategy) { strategyToTryFirst = strategiesCache.MultiThreadedStrategy; strategyToTrySecond = strategiesCache.SingleThreadedStrategy; } else { strategyToTryFirst = strategiesCache.SingleThreadedStrategy; strategyToTrySecond = strategiesCache.MultiThreadedStrategy; } strategy = strategyToTryFirst.Value; if (strategy == null) { strategy = strategyToTrySecond.Value; } }
public static async Task <DejitteringMessagesParser> Create(Func <CreateParserParams, Task <IPositionedMessagesParser> > underlyingParserFactory, CreateParserParams originalParams, int jitterBufferSize) { if (underlyingParserFactory == null) { throw new ArgumentNullException("underlyingParserFactory"); } var parser = new DejitteringMessagesParser(originalParams, jitterBufferSize); try { await parser.CreateUnderlyingParserAndInitJitterBuffer(underlyingParserFactory); } catch { await parser.Dispose(); throw; } return(parser); }
IEnumerable <PostprocessedMessage> ReadAddMessagesFromRangeCompleteJitterBuffer(Func <CreateParserParams, IPositionedMessagesParser> underlyingParserFactory) { CreateParserParams mainParserParams = originalParams; //mainParserParams.Range = null; using (var mainParser = underlyingParserFactory(mainParserParams)) { for (; ;) { var msg = mainParser.ReadNextAndPostprocess(); if (msg.Message == null) { break; } yield return(msg); } } CreateParserParams jitterBufferCompletionParams = originalParams; jitterBufferCompletionParams.Flags |= MessagesParserFlag.DisableMultithreading; jitterBufferCompletionParams.Range = null; jitterBufferCompletionParams.StartPosition = originalParams.Direction == MessagesParserDirection.Forward ? originalParams.Range.Value.End : originalParams.Range.Value.Begin; using (var completionParser = underlyingParserFactory(jitterBufferCompletionParams)) { for (int i = 0; i < jitterBufferSize; ++i) { var msg = completionParser.ReadNextAndPostprocess(); if (msg.Message == null) { break; } yield return(msg); } } }
public IPositionedMessagesParser CreateParser(CreateParserParams p) { return(underliyingReader.CreateParser(p)); }
public DejitteringMessagesParser(Func <CreateParserParams, IPositionedMessagesParser> underlyingParserFactory, CreateParserParams originalParams, DejitteringParams config) : this(underlyingParserFactory, originalParams, config.JitterBufferSize) { }
public static Task <DejitteringMessagesParser> Create(Func <CreateParserParams, Task <IPositionedMessagesParser> > underlyingParserFactory, CreateParserParams originalParams, DejitteringParams config) { return(Create(underlyingParserFactory, originalParams, config.JitterBufferSize)); }