internal static IEnumerable<Utoken> Cook(this IEnumerable<UtokenBase> utokens, IPostProcessHelper postProcessHelper) { return FormatYielder.PostProcess(utokens, postProcessHelper); }
internal static IEnumerable<Utoken> PostProcess(IEnumerable<UtokenBase> utokens, IPostProcessHelper postProcessHelper) { return utokens .DebugWriteLines(tsRaw) .CalculateDeferredUtokens(postProcessHelper) .DebugWriteLines(tsCalculatedDeferred) .FilterInsertedUtokens(postProcessHelper) .DebugWriteLines(tsFiltered) .Flatten() .DebugWriteLines(tsFlattened) .ProcessControls(postProcessHelper) .DebugWriteLines(tsProcessedControls) .Decorate(postProcessHelper) ; }
internal static IEnumerable <Utoken> PostProcess(IEnumerable <UtokenBase> utokens, IPostProcessHelper postProcessHelper) { return(utokens.DebugWriteLines(tsRaw) .CalculateDeferredUtokens(postProcessHelper).DebugWriteLines(tsCalculatedDeferred) .FilterInsertedUtokens(postProcessHelper).DebugWriteLines(tsFiltered) .Flatten().DebugWriteLines(tsFlattened) .ProcessControls(postProcessHelper).DebugWriteLines(tsProcessedControls) .Decorate(postProcessHelper) ); }
public static IEnumerable<Utoken> Decorate(this IEnumerable<Utoken> utokens, IPostProcessHelper postProcessHelper) { var formatter = postProcessHelper.Formatter; foreach (Utoken utoken in utokens) { ((UtokenBase)utoken).Decoration = formatter.GetDecoration(utoken); yield return utoken; } }
public static IEnumerable<UtokenBase> CalculateDeferredUtokens(this IEnumerable<UtokenBase> utokens, IPostProcessHelper postProcessHelper) { #if DEBUG int maxBufferSizeForDebug = 0; #endif var utokensBuffer = new Queue<UtokenBase>(); /* * When dealing with right-to-left unparsing, in case of left-context-specific indentation (which is deferred in this case) * we have to consume more than one utokens in order to be able to calculate the indentation. Thus, we prefetch some more utokens * instead of consuming them one-by-one. This prefetch behavior causes a significant speed-up. * */ const int initialPrefetchCountForDeferred = 100; int prefetchCount = 0; int prefetchedCount = 0; /* * NOTE: During unparsing from right-to-left the leftmost child's left sibling will be set _after_ the last child was consumed, * therefore we might have unprocessed elements (deferred and normal utokens) in the buffer which could not be calculated * when we were consuming the last utoken, but became calculable after the last utoken has been consumed * (despite the fact that we did not consume another utoken in this step). * * Basically, we should execute the body of the loop one more time after the loop. This can be done the most easily by consuming * an extra null element after the real utokens. * */ foreach (UtokenBase _utoken in utokens.Concat((UtokenBase)null)) { utokensBuffer.Enqueue(_utoken); #region Prefetch when right-to-left if (postProcessHelper.Direction == Unparser.Direction.RightToLeft && prefetchedCount < prefetchCount && _utoken != null) { prefetchedCount++; continue; } else prefetchedCount = 0; #endregion #if DEBUG maxBufferSizeForDebug = Math.Max(maxBufferSizeForDebug, utokensBuffer.Count); #endif FormatYielder.tsCalculatedDeferredDetailed.Debug("Consumed and enqueued utoken: {0} (queue size is now {1})", _utoken != null ? _utoken.ToString() : "extra <<NULL>> utoken after last real utoken", utokensBuffer.Count); LProcessBufferWithoutConsumingNewUtoken: if (utokensBuffer.Peek() is DeferredUtokens) { DeferredUtokens deferredUtokens = (DeferredUtokens)utokensBuffer.Peek(); IEnumerable<UtokenBase> calculatedUtokens; try { calculatedUtokens = deferredUtokens.GetUtokens(); utokensBuffer.Dequeue(); // postProcessHelper.UnlinkChildFromChildPrevSiblingIfNotFullUnparse(deferredUtokens.Self); prefetchCount = 0; FormatYielder.tsCalculatedDeferredDetailed.Debug("Calculated: {0}", deferredUtokens); } catch (NonCalculatedException) { FormatYielder.tsCalculatedDeferredDetailed.Debug("Tried to calculate but failed: {0}", deferredUtokens); if (_utoken == null) { // after the last real utoken every deferred utokens should be calculable FormatYielder.tsCalculatedDeferredDetailed.Debug("ERROR: Calculate should not fail after last real token"); #if DEBUG DebugUnprocessedBuffer(utokensBuffer, maxBufferSizeForDebug); #endif throw; } if (prefetchCount == 0) prefetchCount = initialPrefetchCountForDeferred; else prefetchCount *= 2; continue; } FormatYielder.tsCalculatedDeferredDetailed.Debug("Successfully calculated: {0}", deferredUtokens); FormatYielder.tsCalculatedDeferredDetailed.Indent(); foreach (UtokenBase calculatedUtoken in calculatedUtokens) { FormatYielder.tsCalculatedDeferredDetailed.Debug("Yielded calculated: {0}", calculatedUtoken); yield return calculatedUtoken; } FormatYielder.tsCalculatedDeferredDetailed.Unindent(); } else if (utokensBuffer.Peek() == null) { // the extra null element after the real elements -> just remove it from the queue utokensBuffer.Dequeue(); } else { FormatYielder.tsCalculatedDeferredDetailed.Debug("Yielded normal: {0}", utokensBuffer.Peek()); yield return utokensBuffer.Dequeue(); } if (utokensBuffer.Count > 0) { FormatYielder.tsCalculatedDeferredDetailed.Debug("Queue has elements -> process without consuming new utoken (queue size is {0})", utokensBuffer.Count); goto LProcessBufferWithoutConsumingNewUtoken; } } #if DEBUG DebugUnprocessedBuffer(utokensBuffer, maxBufferSizeForDebug); #endif Debug.Assert(utokensBuffer.Count == 0, "unprocessed items in buffer (non-calculated unparsable objects remained)"); }
/* * utokens contains several InsertedUtokens "sessions" which consist of "Right", "Left" and "Between" InsertedUtokens. * A session looks like this: (Right)*((Between)?(Left)?)* * * Note that "Between" and "Left" InsertedUtokens are mixed with each other. * * "Right" InsertedUtokens are handled so that in case of equal priorities we will choose the right one. * "Between" InsertedUtokens are handled so that in case of equal priorities we will choose the left one. * "Left" InsertedUtokens are handled so that in case of equal priorities we will choose the left one. * * Since "Between" and "Left" InsertedUtokens are handled in the same way, we can handle them mixed. * * We handle the InsertedUtokens like this in order to ensure that the InsertedUtokens belonging to the * outer (in sense of structure) bnfterms always being preferred against inner bnfterms in case of equal priorities. * * InsertedUtokens belonging to the same bnfterm with equal priorities are handled so that the several kind * of InsertedUtokens have strength in descending order: Between, Left, Right * */ public static IEnumerable<UtokenBase> FilterInsertedUtokens(this IEnumerable<UtokenBase> utokens, IPostProcessHelper postProcessHelper) { InsertedUtokens prevInsertedUtokensToBeYield = null; var nonOverridableSkipThroughBuffer = new Queue<UtokenBase>(); foreach (UtokenBase utoken in utokens.Concat((UtokenBase)null)) { if (utoken is InsertedUtokens) { InsertedUtokens nextInsertedUtokens = (InsertedUtokens)utoken; var switchToNextInsertedUtokens = new Lazy<bool>(() => IsNextStronger(prevInsertedUtokensToBeYield, nextInsertedUtokens, postProcessHelper.Direction)); if (nextInsertedUtokens.Behavior == Behavior.Overridable) { prevInsertedUtokensToBeYield = switchToNextInsertedUtokens.Value ? nextInsertedUtokens : prevInsertedUtokensToBeYield; } else if (nextInsertedUtokens.Behavior == Behavior.NonOverridableSkipThrough) { if (postProcessHelper.Direction == Unparser.Direction.LeftToRight) yield return nextInsertedUtokens; else nonOverridableSkipThroughBuffer.Enqueue(nextInsertedUtokens); } else if (nextInsertedUtokens.Behavior == Behavior.NonOverridableSeparator) { if (!switchToNextInsertedUtokens.Value) yield return prevInsertedUtokensToBeYield; while (nonOverridableSkipThroughBuffer.Count > 0) yield return nonOverridableSkipThroughBuffer.Dequeue(); yield return nextInsertedUtokens; prevInsertedUtokensToBeYield = null; } } else if (utoken is UtokenControl && ((UtokenControl)utoken).IsIndent()) { // handle it as it were a NonOverridableSkipThrough if (postProcessHelper.Direction == Unparser.Direction.LeftToRight) yield return utoken; else nonOverridableSkipThroughBuffer.Enqueue(utoken); } else { if (prevInsertedUtokensToBeYield != null) { yield return prevInsertedUtokensToBeYield; prevInsertedUtokensToBeYield = null; } while (nonOverridableSkipThroughBuffer.Count > 0) yield return nonOverridableSkipThroughBuffer.Dequeue(); if (utoken != null) yield return utoken; } } }
public static IEnumerable<Utoken> ProcessControls(this IEnumerable<UtokenBase> utokens, IPostProcessHelper postProcessHelper) { var formatter = postProcessHelper.Formatter; int indentLevel = 0; int indentLevelForCurrentLine = 0; Stack<int> storedIndentLevel = new Stack<int>(); bool allowWhitespaceBetweenUtokens = true; UtokenBase prevNotControlUtoken = null; /* * We have to yield the indentation after consuming the last utoken (regardless of left-to-right or right-to-left unparse), * so we use an extra null utoken. * * e.g.: In case of right-to-left unparse if utokens enumerable is not empty and the last processed utoken (which is the leftmost utoken) * is not a line separator (which is the common case) then we didn't yielded the utokenindent for the last processed line * (which is the topmost line), so we yield it now when processing the extra null utoken. * */ foreach (UtokenBase utoken in utokens.Concat((UtokenBase)null)) { if (IsControl(utoken)) { UtokenControl utokenControl = (UtokenControl)utoken; switch (utokenControl.kind) { case UtokenControl.Kind.IncreaseIndentLevel: indentLevel++; break; case UtokenControl.Kind.DecreaseIndentLevel: indentLevel--; break; case UtokenControl.Kind.SetIndentLevelToNone: storedIndentLevel.Push(indentLevel); indentLevel = 0; break; case UtokenControl.Kind.RestoreIndentLevel: indentLevel = storedIndentLevel.Pop(); break; case UtokenControl.Kind.NoWhitespace: allowWhitespaceBetweenUtokens = false; break; default: throw new InvalidOperationException(string.Format("Unknown UtokenControl '{0}'", utokenControl.kind)); } } else if (utoken == UtokenValueControl.NoWhitespace) { allowWhitespaceBetweenUtokens = false; } else { if (postProcessHelper.Direction == Unparser.Direction.RightToLeft && IsLineSeparator(utoken) && (formatter.IndentEmptyLines || !IsLineSeparator(prevNotControlUtoken))) yield return new UtokenIndent(indentLevelForCurrentLine); indentLevelForCurrentLine = indentLevel; if (postProcessHelper.Direction == Unparser.Direction.LeftToRight && IsLineSeparator(prevNotControlUtoken) && (formatter.IndentEmptyLines || !IsLineSeparator(utoken))) yield return new UtokenIndent(indentLevelForCurrentLine); if (allowWhitespaceBetweenUtokens && prevNotControlUtoken != null && utoken != null && !IsWhitespace(prevNotControlUtoken) && !IsWhitespace(utoken)) yield return UtokenWhitespace.WhiteSpaceBetweenUtokens(); if (utoken != null) yield return (Utoken)utoken; allowWhitespaceBetweenUtokens = true; prevNotControlUtoken = utoken; } } }
public static IEnumerable <UtokenBase> CalculateDeferredUtokens(this IEnumerable <UtokenBase> utokens, IPostProcessHelper postProcessHelper) { #if DEBUG int maxBufferSizeForDebug = 0; #endif var utokensBuffer = new Queue <UtokenBase>(); /* * When dealing with right-to-left unparsing, in case of left-context-specific indentation (which is deferred in this case) * we have to consume more than one utokens in order to be able to calculate the indentation. Thus, we prefetch some more utokens * instead of consuming them one-by-one. This prefetch behavior causes a significant speed-up. * */ const int initialPrefetchCountForDeferred = 100; int prefetchCount = 0; int prefetchedCount = 0; /* * NOTE: During unparsing from right-to-left the leftmost child's left sibling will be set _after_ the last child was consumed, * therefore we might have unprocessed elements (deferred and normal utokens) in the buffer which could not be calculated * when we were consuming the last utoken, but became calculable after the last utoken has been consumed * (despite the fact that we did not consume another utoken in this step). * * Basically, we should execute the body of the loop one more time after the loop. This can be done the most easily by consuming * an extra null element after the real utokens. * */ foreach (UtokenBase _utoken in utokens.Concat((UtokenBase)null)) { utokensBuffer.Enqueue(_utoken); #region Prefetch when right-to-left if (postProcessHelper.Direction == Unparser.Direction.RightToLeft && prefetchedCount < prefetchCount && _utoken != null) { prefetchedCount++; continue; } else { prefetchedCount = 0; } #endregion #if DEBUG maxBufferSizeForDebug = Math.Max(maxBufferSizeForDebug, utokensBuffer.Count); #endif FormatYielder.tsCalculatedDeferredDetailed.Debug("Consumed and enqueued utoken: {0} (queue size is now {1})", _utoken != null ? _utoken.ToString() : "extra <<NULL>> utoken after last real utoken", utokensBuffer.Count); LProcessBufferWithoutConsumingNewUtoken: if (utokensBuffer.Peek() is DeferredUtokens) { DeferredUtokens deferredUtokens = (DeferredUtokens)utokensBuffer.Peek(); IEnumerable <UtokenBase> calculatedUtokens; try { calculatedUtokens = deferredUtokens.GetUtokens(); utokensBuffer.Dequeue(); // postProcessHelper.UnlinkChildFromChildPrevSiblingIfNotFullUnparse(deferredUtokens.Self); prefetchCount = 0; FormatYielder.tsCalculatedDeferredDetailed.Debug("Calculated: {0}", deferredUtokens); } catch (NonCalculatedException) { FormatYielder.tsCalculatedDeferredDetailed.Debug("Tried to calculate but failed: {0}", deferredUtokens); if (_utoken == null) { // after the last real utoken every deferred utokens should be calculable FormatYielder.tsCalculatedDeferredDetailed.Debug("ERROR: Calculate should not fail after last real token"); #if DEBUG DebugUnprocessedBuffer(utokensBuffer, maxBufferSizeForDebug); #endif throw; } if (prefetchCount == 0) { prefetchCount = initialPrefetchCountForDeferred; } else { prefetchCount *= 2; } continue; } FormatYielder.tsCalculatedDeferredDetailed.Debug("Successfully calculated: {0}", deferredUtokens); FormatYielder.tsCalculatedDeferredDetailed.Indent(); foreach (UtokenBase calculatedUtoken in calculatedUtokens) { FormatYielder.tsCalculatedDeferredDetailed.Debug("Yielded calculated: {0}", calculatedUtoken); yield return(calculatedUtoken); } FormatYielder.tsCalculatedDeferredDetailed.Unindent(); } else if (utokensBuffer.Peek() == null) { // the extra null element after the real elements -> just remove it from the queue utokensBuffer.Dequeue(); } else { FormatYielder.tsCalculatedDeferredDetailed.Debug("Yielded normal: {0}", utokensBuffer.Peek()); yield return(utokensBuffer.Dequeue()); } if (utokensBuffer.Count > 0) { FormatYielder.tsCalculatedDeferredDetailed.Debug("Queue has elements -> process without consuming new utoken (queue size is {0})", utokensBuffer.Count); goto LProcessBufferWithoutConsumingNewUtoken; } } #if DEBUG DebugUnprocessedBuffer(utokensBuffer, maxBufferSizeForDebug); #endif Debug.Assert(utokensBuffer.Count == 0, "unprocessed items in buffer (non-calculated unparsable objects remained)"); }
public static IEnumerable <Utoken> Decorate(this IEnumerable <Utoken> utokens, IPostProcessHelper postProcessHelper) { var formatter = postProcessHelper.Formatter; foreach (Utoken utoken in utokens) { ((UtokenBase)utoken).Decoration = formatter.GetDecoration(utoken); yield return(utoken); } }
public static IEnumerable <Utoken> ProcessControls(this IEnumerable <UtokenBase> utokens, IPostProcessHelper postProcessHelper) { var formatter = postProcessHelper.Formatter; int indentLevel = 0; int indentLevelForCurrentLine = 0; Stack <int> storedIndentLevel = new Stack <int>(); bool allowWhitespaceBetweenUtokens = true; UtokenBase prevNotControlUtoken = null; /* * We have to yield the indentation after consuming the last utoken (regardless of left-to-right or right-to-left unparse), * so we use an extra null utoken. * * e.g.: In case of right-to-left unparse if utokens enumerable is not empty and the last processed utoken (which is the leftmost utoken) * is not a line separator (which is the common case) then we didn't yielded the utokenindent for the last processed line * (which is the topmost line), so we yield it now when processing the extra null utoken. * */ foreach (UtokenBase utoken in utokens.Concat((UtokenBase)null)) { if (IsControl(utoken)) { UtokenControl utokenControl = (UtokenControl)utoken; switch (utokenControl.kind) { case UtokenControl.Kind.IncreaseIndentLevel: indentLevel++; break; case UtokenControl.Kind.DecreaseIndentLevel: indentLevel--; break; case UtokenControl.Kind.SetIndentLevelToNone: storedIndentLevel.Push(indentLevel); indentLevel = 0; break; case UtokenControl.Kind.RestoreIndentLevel: indentLevel = storedIndentLevel.Pop(); break; case UtokenControl.Kind.NoWhitespace: allowWhitespaceBetweenUtokens = false; break; default: throw new InvalidOperationException(string.Format("Unknown UtokenControl '{0}'", utokenControl.kind)); } } else if (utoken == UtokenValueControl.NoWhitespace) { allowWhitespaceBetweenUtokens = false; } else { if (postProcessHelper.Direction == Unparser.Direction.RightToLeft && IsLineSeparator(utoken) && (formatter.IndentEmptyLines || !IsLineSeparator(prevNotControlUtoken))) { yield return(new UtokenIndent(indentLevelForCurrentLine)); } indentLevelForCurrentLine = indentLevel; if (postProcessHelper.Direction == Unparser.Direction.LeftToRight && IsLineSeparator(prevNotControlUtoken) && (formatter.IndentEmptyLines || !IsLineSeparator(utoken))) { yield return(new UtokenIndent(indentLevelForCurrentLine)); } if (allowWhitespaceBetweenUtokens && prevNotControlUtoken != null && utoken != null && !IsWhitespace(prevNotControlUtoken) && !IsWhitespace(utoken)) { yield return(UtokenWhitespace.WhiteSpaceBetweenUtokens()); } if (utoken != null) { yield return((Utoken)utoken); } allowWhitespaceBetweenUtokens = true; prevNotControlUtoken = utoken; } } }
/* * utokens contains several InsertedUtokens "sessions" which consist of "Right", "Left" and "Between" InsertedUtokens. * A session looks like this: (Right)*((Between)?(Left)?)* * * Note that "Between" and "Left" InsertedUtokens are mixed with each other. * * "Right" InsertedUtokens are handled so that in case of equal priorities we will choose the right one. * "Between" InsertedUtokens are handled so that in case of equal priorities we will choose the left one. * "Left" InsertedUtokens are handled so that in case of equal priorities we will choose the left one. * * Since "Between" and "Left" InsertedUtokens are handled in the same way, we can handle them mixed. * * We handle the InsertedUtokens like this in order to ensure that the InsertedUtokens belonging to the * outer (in sense of structure) bnfterms always being preferred against inner bnfterms in case of equal priorities. * * InsertedUtokens belonging to the same bnfterm with equal priorities are handled so that the several kind * of InsertedUtokens have strength in descending order: Between, Left, Right * */ public static IEnumerable <UtokenBase> FilterInsertedUtokens(this IEnumerable <UtokenBase> utokens, IPostProcessHelper postProcessHelper) { InsertedUtokens prevInsertedUtokensToBeYield = null; var nonOverridableSkipThroughBuffer = new Queue <UtokenBase>(); foreach (UtokenBase utoken in utokens.Concat((UtokenBase)null)) { if (utoken is InsertedUtokens) { InsertedUtokens nextInsertedUtokens = (InsertedUtokens)utoken; var switchToNextInsertedUtokens = new Lazy <bool>(() => IsNextStronger(prevInsertedUtokensToBeYield, nextInsertedUtokens, postProcessHelper.Direction)); if (nextInsertedUtokens.Behavior == Behavior.Overridable) { prevInsertedUtokensToBeYield = switchToNextInsertedUtokens.Value ? nextInsertedUtokens : prevInsertedUtokensToBeYield; } else if (nextInsertedUtokens.Behavior == Behavior.NonOverridableSkipThrough) { if (postProcessHelper.Direction == Unparser.Direction.LeftToRight) { yield return(nextInsertedUtokens); } else { nonOverridableSkipThroughBuffer.Enqueue(nextInsertedUtokens); } } else if (nextInsertedUtokens.Behavior == Behavior.NonOverridableSeparator) { if (!switchToNextInsertedUtokens.Value) { yield return(prevInsertedUtokensToBeYield); } while (nonOverridableSkipThroughBuffer.Count > 0) { yield return(nonOverridableSkipThroughBuffer.Dequeue()); } yield return(nextInsertedUtokens); prevInsertedUtokensToBeYield = null; } } else if (utoken is UtokenControl && ((UtokenControl)utoken).IsIndent()) { // handle it as it were a NonOverridableSkipThrough if (postProcessHelper.Direction == Unparser.Direction.LeftToRight) { yield return(utoken); } else { nonOverridableSkipThroughBuffer.Enqueue(utoken); } } else { if (prevInsertedUtokensToBeYield != null) { yield return(prevInsertedUtokensToBeYield); prevInsertedUtokensToBeYield = null; } while (nonOverridableSkipThroughBuffer.Count > 0) { yield return(nonOverridableSkipThroughBuffer.Dequeue()); } if (utoken != null) { yield return(utoken); } } } }
internal static IEnumerable <Utoken> Cook(this IEnumerable <UtokenBase> utokens, IPostProcessHelper postProcessHelper) { return(FormatYielder.PostProcess(utokens, postProcessHelper)); }