static void Main(string[] args)
{
var before = "a or (a or b)";
var after = "a or b";
var examples = new List <Tuple <string, string> >()
{
Tuple.Create(before, after)
};
var refazer = new Refazer4Python();
var transformation = refazer.LearnTransformations(examples.ToList()).First();
foreach (var mistake in examples)
{
var output = refazer.Apply(transformation, mistake.Item1);
Console.Out.WriteLine("");
var isFixed = false;
foreach (var newCode in output)
{
var unparser = new Unparser();
isFixed = mistake.Item2.Equals(newCode);
if (isFixed)
{
break;
}
}
Console.WriteLine("Before: " + mistake.Item1);
Console.WriteLine("Expected After: " + mistake.Item2);
Console.WriteLine("Actual After: " + output.FirstOrDefault());
Console.WriteLine(isFixed);
}
}
public IEnumerable <string> Apply(Transformation transformation, string program)
{
var unparser = new Unparser();
var result = transformation.GetSynthesizedProgram().Invoke(CreateInputState(program)) as IEnumerable <PythonNode>;
return(result == null ? new List <string>() : result.Select(x => unparser.Unparse(x)));
}
public GrammarToken(Unparser grammar, int start, GrammarToken parent)
{
Raw = "";
Grammar = grammar;
Start = start;
Parent = parent;
}
protected override int? GetChildrenPriority(IUnparser unparser, object astValue, Unparser.Children children, Unparser.Direction direction)
{
UnparsableAst mainChild = children.Single(childValue => IsMainChild(childValue.BnfTerm));
if (astValue.GetType() == this.domainType)
return unparser.GetPriority(mainChild);
else
{
IBnfiTerm mainChildWithDomainType = mainChild.BnfTerm as IBnfiTerm;
if (mainChildWithDomainType == null || mainChildWithDomainType.DomainType == null)
{
throw new UnparseException(string.Format("Cannot unparse '{0}' (type: '{1}'). BnfTerm '{2}' is not an IBnfiTerm or it has no domain type.",
astValue, astValue.GetType().Name, mainChild.BnfTerm));
}
int? priority = mainChildWithDomainType.DomainType == typeof(object)
? int.MinValue
: 0 - mainChildWithDomainType.DomainType.GetInheritanceDistance(astValue);
Unparser.tsPriorities.Indent();
priority.DebugWriteLinePriority(Unparser.tsPriorities, mainChild);
Unparser.tsPriorities.Unindent();
return priority;
}
}
public IEnumerable <string> Generate(string before, string after)
{
var result = new List <string>();
//parse code
var beforeAst = Parse(before);
var afterAast = Parse(after);
//run tree edit distance
var zss = new PythonZss(beforeAst, afterAast);
var editDistance = zss.Compute();
//get primary edits
var rootAndNonRootEdits = WitnessFunctions.SplitEditsByRootsAndNonRoots(editDistance);
//replace insert and delete by update
var unparser = new Unparser();
foreach (var edit in rootAndNonRootEdits.Item1)
{
if (edit is Update)
{
result.Add("Update " + unparser.Unparse(edit.TargetNode) + " to " + unparser.Unparse(edit.ModifiedNode));
}
else if (edit is Insert)
{
result.Add("Insert " + unparser.Unparse(edit.ModifiedNode));
}
}
//for each edit, create a hint
return(result);
}
public static Task <string> AsTextAsync(this IEnumerable <Utoken> utokens, Unparser unparser, CancellationToken cancellationToken)
{
return(unparser.Lock.LockAsync()
.ContinueWith(
task =>
{
using (task.Result)
return TaskEx.Run(
() =>
string.Concat(
utokens.Select(
utoken =>
{
cancellationToken.ThrowIfCancellationRequested();
return utoken.ToText(unparser.Formatter);
}
)
)
,
cancellationToken
);
},
cancellationToken
)
.Unwrap());
}
public static Task<string> AsTextAsync(this IEnumerable<Utoken> utokens, Unparser unparser, CancellationToken cancellationToken)
{
return unparser.Lock.LockAsync()
.ContinueWith(
task =>
{
using (task.Result)
return TaskEx.Run(
() =>
string.Concat(
utokens.Select(
utoken =>
{
cancellationToken.ThrowIfCancellationRequested();
return utoken.ToText(unparser.Formatter);
}
)
)
,
cancellationToken
);
},
cancellationToken
)
.Unwrap();
}
private ExpressionUnparser(ExpressionUnparser that, Unparser newUnparser)
{
this.unparser = newUnparser;
this.expressionsThatCanCauseOthersBeingParenthesized = that.expressionsThatCanCauseOthersBeingParenthesized;
this.expressionThatMayNeedParenthesesToParentheses = that.expressionThatMayNeedParenthesesToParentheses;
this.bnfTermToBnfTermKind = that.bnfTermToBnfTermKind;
this.direction = that.direction;
}
public ExpressionUnparser Spawn(Unparser newUnparser)
{
if (this.OngoingExpressionUnparse)
{
throw new InvalidOperationException("Cannot spawn unparser during an ongoing expression unparse");
}
return(new ExpressionUnparser(this, newUnparser));
}
public void HeroGrammarWorks()
{
var grammar = Grammar.FromFile("Grammars/hero-grammar.json");
var unparser = new Unparser(grammar);
var output = unparser.Generate(0);
var test = "Once upon a time, Chiaki the time captain left his home. Chiaki went home.";
Assert.Equal(test, output);
}
private void RegisteringExpressionsThatNeedParentheses(NonTerminal nonTerminal)
{
foreach (BnfTermList childBnfTerms in Unparser.GetChildBnfTermListsLeftToRight(nonTerminal))
{
Unparse(
new UnparsableAst(nonTerminal, astValue: null),
childBnfTerms.Select(childBnfTerm => new UnparsableAst(childBnfTerm, astValue: null)),
initialization: true
);
}
}
private static void Main(string[] args)
{
var grammar = Grammar.FromFile("Tracery.Testing/Grammars/readme-grammar.json");
var unparser = new Unparser(grammar);
foreach (var i in new[] { 0, 1, 2, 3, 4 })
{
var output = unparser.Generate(i);
Console.WriteLine($"#{i}: {output}\n");
}
}
public TargetSeed(ApplicationDbContext context, IHostingEnvironment env,
IVersionNumberFactory versionNumberFactory,
ISlugifier slugifier,
Func <string, Unparser> unparserFactory,
IconRandomizer iconRandomizer)
{
_context = context;
_env = env;
_versionNumberFactory = versionNumberFactory;
_slugifier = slugifier;
_iconRandomizer = iconRandomizer;
_unparser = unparserFactory("Entities/target.json");
}
public UserIdentitySeed(ApplicationDbContext context,
ILogger <UserIdentitySeed> logger,
IHostingEnvironment env,
ISlugifier slugifier,
Func <string, Unparser> unparserFactory,
IconRandomizer iconRandomizer,
UserManager <UserIdentity> userManager)
{
_context = context;
_logger = logger;
_env = env;
_slugifier = slugifier;
_iconRandomizer = iconRandomizer;
_userManager = userManager;
_unparser = unparserFactory("all.json");
}
public static void InitializeUnparser(TestContext testContext)
{
CommonTest.InitializeParser();
Directory.CreateDirectory(actualUnparsedFilesDir);
unparser = new Unparser(grammar);
formatter2 = new SpecialFormatter(grammar)
{
IndentEmptyLines = false
};
formatter3 = new SpecialFormatter(grammar)
{
IndentEmptyLines = true
};
grammar.UnparseControl.ClearPrecedenceBasedParenthesesForExpressions(); // we want to test the automatic parentheses stuff
}
public static Func <string, Unparser> UnparserFactory(IServiceProvider di)
{
var logger = di.GetRequiredService <ILogger <TraceryUnparser> >();
var config = di.GetRequiredService <IConfiguration>();
var slugifier = di.GetRequiredService <ISlugifier>();
var grammarPath = config.GetValue <string>("Database:TraceryGrammarPath");
string Capitalize(string s)
{
if (string.IsNullOrEmpty(s))
{
return(s);
}
var result = char.ToUpper(s[0]).ToString();
if (s.Length > 1)
{
result += s.Substring(1);
}
return(result);
}
string TitleCase(string s)
{
var parts = s.Split(null)
.Where(p => p.Any())
.Select(p => char.ToUpper(p[0]) + p.Substring(1));
return(String.Join(" ", parts));
}
return(filename => {
var fullPath = Path.Combine(grammarPath, filename);
var grammar = Grammar.FromFile(fullPath);
var unparser = new Unparser(grammar);
unparser.Modifiers["capitalize"] = Capitalize;
unparser.Modifiers["title"] = TitleCase;
unparser.Modifiers["slug"] = slugifier.Slugify;
return unparser;
});
}
private static void FixSubmission(Tuple <ProgramNode, Models.Transformation> transformationTuple, int experiementId, int questionId,
Tuple <Submission, State> submission)
{
try
{
if (!submission.Item1.IsFixed)
{
var manager = new TestManager();
var mistake = new Mistake();
mistake.before = submission.Item1.Code;
var fixer = new SubmissionFixer();
var unparser = new Unparser();
var fixedCode = fixer.TryFix(manager.GetTests(questionId), transformationTuple.Item1, submission.Item2, unparser);
if (fixedCode != null)
{
var refazerDb2 = new RefazerDbContext();
submission.Item1.IsFixed = true;
var updatedSub = refazerDb2.Submissions.SingleOrDefault(e => e.ID == submission.Item1.ID);
if (updatedSub != null)
{
updatedSub.IsFixed = true;
}
var trans = refazerDb2.Transformations.First(x => x.ID == transformationTuple.Item2.ID);
var fix = new Fix()
{
FixedCode = fixedCode,
SessionId = experiementId,
SubmissionId = submission.Item1.SubmissionId,
QuestionId = questionId,
Transformation = trans
};
refazerDb2.Fixes.Add(fix);
refazerDb2.SaveChanges();
Trace.TraceWarning(string.Format("Submission fixed: {0}, Session: {1}, Transformation: {2}, Time: {3}",
submission.Item1.SubmissionId, transformationTuple.Item2.SessionId, transformationTuple.Item2.ID, DateTime.Now));
}
}
}
catch (Exception e)
{
Trace.TraceError("Exception was thrown when applying fixes.");
Trace.TraceError(e.Message);
}
}
static void Main(string[] args)
{
var grammar = new UDLGrammar();
//Console.WriteLine(grammar.GetNonTerminalsAsText());
//Console.WriteLine();
//Console.WriteLine(grammar.GetNonTerminalsAsText(omitBoundMembers: true));
var parser = new Parser(grammar);
ParseTree parseTree = parser.Parse(File.ReadAllText(path), path);
Unparser unparser = new Unparser(grammar);
Directory.CreateDirectory("unparse_logs");
var stream = File.Create(@"unparse_logs\unparsed_text");
unparser.Unparse(parseTree.Root.AstNode).WriteToStream(stream, unparser);
//string str = unparser.Unparse(parseTree.Root.AstNode).AsString(unparser);
//Console.WriteLine(str);
}
public static void AssertCorrectTransformation(IEnumerable <Tuple <string, string> > examples)
{
var refazer = new Refazer4Python();
var transformation = refazer.LearnTransformations(examples.ToList()).First();
foreach (var mistake in examples)
{
var output = refazer.Apply(transformation, mistake.Item1);
Console.Out.WriteLine("");
var isFixed = false;
foreach (var newCode in output)
{
var unparser = new Unparser();
isFixed = mistake.Item2.Equals(newCode);
if (isFixed)
{
break;
}
}
Assert.IsTrue(isFixed);
}
}
public ExpressionUnparser(Unparser unparser, UnparseControl unparseControl)
{
this.unparser = unparser;
InitParentheses(unparseControl);
}
public static async Task <string> AsTextAsync(this IEnumerable <Utoken> utokens, Unparser unparser, CancellationToken cancellationToken)
{
using (await unparser.Lock.LockAsync())
{
return(await Task.Run(
() =>
{
return string.Concat(
utokens.Select(
utoken =>
{
cancellationToken.ThrowIfCancellationRequested();
return utoken.ToText(unparser.Formatter);
}
)
);
},
cancellationToken
));
}
}
// static string path2 = @"..\..\..\Sarcasm.UnitTest\Test files\Binary1.expr";
static void Main(string[] args)
{
var stopwatch = Stopwatch.StartNew();
stopwatch.Start();
var grammar = new MiniPLG.GrammarP();
ShowTimeAndRestart(stopwatch, "Creation of grammar");
var parser = MultiParser.Create(grammar);
ShowTimeAndRestart(stopwatch, "Creation of parser");
var parseTree = parser.Parse(File.ReadAllText(path), path);
ShowTimeAndRestart(stopwatch, "Parsing");
var astRootValue = parseTree.RootAstValue;
// string generatedCode = new MiniPLC.CSharpGenerator().Generate(astRootValue);
// string generatedCode = new MiniPLC.CppGenerator().Generate(astRootValue);
#if false
var jsonGrammar = new JsonGrammar();
Unparser universalUnparser = new Unparser(jsonGrammar);
var universalParser = MultiParser.Create(jsonGrammar);
string text = universalUnparser.Unparse(astRootValue).AsText(universalUnparser);
var foo = universalUnparser.Unparse(astRootValue).Select(utoken => utoken.GetDecoration()).ToList();
var parseTree2 = universalParser.Parse(text);
var astRootValue2 = parseTree2.RootAstValue;
string text2 = universalUnparser.Unparse(astRootValue2).AsText(universalUnparser);
bool eq = text == text2;
#endif
Unparser unparser = new Unparser(grammar);
ShowTimeAndRestart(stopwatch, "Creation of unparser");
//var utokens = unparser.Unparse(astRootValue).ToList();
//ShowTimeAndRestart(stopwatch, "Unparsing to utokens");
//unparser.EnableParallelProcessing = false;
//string unparsedText = unparser.Unparse(astRootValue).AsText(unparser);
//var document = parseTree.GetDocument();
//string unparsedText = unparser.Unparse(document).AsText(unparser);
//ShowTimeAndRestart(stopwatch, "Converting utokens to string");
unparser.EnableParallelProcessing = false;
//foreach (Utoken utoken in unparser.Unparse(astRootValue))
// Console.WriteLine(utoken.ToString());
unparser.Unparse(astRootValue).ConsumeAll();
ShowTimeAndRestart(stopwatch, "Sequential unparsing to string");
unparser.EnableParallelProcessing = true;
//foreach (Utoken utoken in unparser.Unparse(astRootValue))
// Console.WriteLine(utoken.ToString());
unparser.Unparse(astRootValue).ConsumeAll();
ShowTimeAndRestart(stopwatch, "Parallel unparsing to string");
//var utokensReverse = unparser.Unparse(astRootValue, Unparser.Direction.RightToLeft).ToList();
//ShowTimeAndRestart(stopwatch, "Reverse unparsing to utokens");
unparser.EnableParallelProcessing = false;
unparser.Unparse(astRootValue, Unparser.Direction.RightToLeft).ConsumeAll();
ShowTimeAndRestart(stopwatch, "Reverse sequential unparsing to string");
unparser.EnableParallelProcessing = true;
unparser.Unparse(astRootValue, Unparser.Direction.RightToLeft).ConsumeAll();
ShowTimeAndRestart(stopwatch, "Reverse parallel unparsing to string");
//stopwatch.Stop();
//Console.WriteLine(stopwatch.ElapsedMilliseconds);
//stopwatch.Start();
//var parser2 = MultiParser.Create(grammar);
//stopwatch.Stop();
//Console.WriteLine(stopwatch.ElapsedMilliseconds);
//var parseTree2 = parser.Parse(File.ReadAllText(path2), path2, grammar.B.Expression);
//var parseTree3 = parser.Parse(File.ReadAllText(path2), path2, (NonTerminal)grammar.B.Expression);
//MiniPL.DomainDefinitions.Expression astValue2 = parseTree2.RootAstValue;
//Unparser unparser = new Unparser(grammar);
//Directory.CreateDirectory("unparse_logs");
//var stream = File.Create(@"unparse_logs\unparsed_text");
//unparser.Unparse(parseTree.Root.AstNode).WriteToStream(stream, unparser);
////string str = unparser.Unparse(parseTree.Root.AstNode).AsString(unparser);
////Console.WriteLine(str);
}
/// <exception cref="UnparsableAst.NonCalculatedException">
/// If topLeft is non-calculated or thrown out.
/// </exception>
private static IEnumerable<UtokenBase> _YieldIndentation(UnparsableAst self, ref BlockIndentation blockIndentationParameter, Unparser.Direction direction,
FormatYielder formatYielder, bool left)
{
if (!left && direction == Unparser.Direction.RightToLeft && blockIndentationParameter.IsDeferred() && blockIndentationParameter.LeftDeferred != null)
{
/*
* We are in a right-to-left unparse and this deferred right indentation depends on a deferred left indentation,
* so let's try to calculate the deferred left indentation, which - if succeed - will change the shared blockIndentation
* object state from deferred to a valid indentation.
*
* (If the left indentation wasn't deferred then it would be calculated which means that the shared blockIndentation
* object won't be deferred.)
* */
blockIndentationParameter.LeftDeferred.CalculateUtokens();
// either CalculateUtokens succeeded, and blockIndentation is not deferred, or CalculateUtokens threw a NonCalculatedException exception
Debug.Assert(!blockIndentationParameter.IsDeferred(), "CalculateUtokens succeeded, but blockIndentation remained deferred");
}
if (blockIndentationParameter == BlockIndentation.ToBeSet || blockIndentationParameter.IsDeferred())
{
UnparsableAst leftObject = GetLeftTerminalLeave(self);
BlockIndentation blockIndentation = formatYielder._GetBlockIndentation(leftObject, self);
// NOTE: topAncestorCacheForLeft gets updated by GetUsedLeftsFromTopToBottomB
if (blockIndentation != BlockIndentation.IndentNotNeeded)
Unparser.tsUnparse.Debug("blockindentation {0} for leftTarget '{1}' and for target '{2}'", blockIndentation, leftObject, self);
Debug.Assert(!blockIndentation.IsDeferred());
if (blockIndentationParameter == BlockIndentation.ToBeSet)
blockIndentationParameter = blockIndentation;
else
blockIndentationParameter.CopyKindFrom(blockIndentation);
}
if (direction == Unparser.Direction.LeftToRight)
{
return left
? BlockIndentationToUtokenControlBefore(blockIndentationParameter)
: BlockIndentationToUtokenControlAfter(blockIndentationParameter);
}
else
{
return left
? BlockIndentationToUtokenControlAfter(blockIndentationParameter)
: BlockIndentationToUtokenControlBefore(blockIndentationParameter);
}
}
public static async Task WriteToStreamAsync(this IEnumerable <Utoken> utokens, Stream stream, Unparser unparser)
{
await utokens.WriteToStreamAsync(stream, unparser, CancellationToken.None);
}
public static Task<string> AsTextAsync(this IEnumerable<Utoken> utokens, Unparser unparser)
{
return unparser.Lock.LockAsync()
.ContinueWith(task => { using (task.Result) return TaskEx.Run(() => utokens.AsText(unparser)); })
.Unwrap();
}
public static Task <string> AsTextAsync(this IEnumerable <Utoken> utokens, Unparser unparser)
{
return(unparser.Lock.LockAsync()
.ContinueWith(task => { using (task.Result) return TaskEx.Run(() => utokens.AsText(unparser)); })
.Unwrap());
}
public static async Task<string> AsTextAsync(this IEnumerable<Utoken> utokens, Unparser unparser)
{
using (await unparser.Lock.LockAsync())
return await Task.Run(() => utokens.AsText(unparser));
}
private BnfTermKind CalculateBnfTermKindForNonTerminal(NonTerminal current)
{
BnfTermKind?currentKind = null;
var childOperators = new List <BnfTerm>();
foreach (BnfTermList children in Unparser.GetChildBnfTermListsLeftToRight(current))
{
int operatorCount = 0;
int leftParenthesisCount = 0;
int rightParenthesisCount = 0;
int otherCount = 0;
/*
* NOTE: we should not read bnfTermToBnfTermKind in this method (because of recursive definitions in grammar),
* that's why we store BnfTermKind as well
* */
BnfTerm prevChild = null;
BnfTermKind? prevChildKind = null;
ParenthesizedExpression parentheses = null;
BnfTerm childExpression = null;
if (children.Count == 0)
{
continue;
}
BnfTerm childOperator = null;
foreach (BnfTerm child in children)
{
BnfTermKind childKind = CalculateBnfTermKind(child);
Debug.Assert(childKind != BnfTermKind.Undetermined);
#region Handle childKind
switch (childKind)
{
case BnfTermKind.Operator:
operatorCount++;
childOperator = child;
childExpression = null;
parentheses = null;
break;
case BnfTermKind.LeftParenthesis:
leftParenthesisCount++;
parentheses = new ParenthesizedExpression()
{
LeftParenthesis = child
};
childExpression = null;
break;
case BnfTermKind.RightParenthesis:
rightParenthesisCount++;
if (prevChildKind == BnfTermKind.Other && childExpression == prevChild)
{
Debug.Assert(parentheses != null && parentheses.LeftParenthesis != null && parentheses.Expression != null);
parentheses.RightParenthesis = child;
ParenthesizedExpression registeredParentheses;
if (_expressionToParentheses.TryGetValue(childExpression, out registeredParentheses))
{
if (parentheses != registeredParentheses)
{
_expressionToParentheses[childExpression] = null; // ambiguous parentheses -> set to null, and check later (we might not need the parentheses at all)
}
}
else
{
_expressionToParentheses.Add(childExpression, parentheses);
}
}
childExpression = null;
parentheses = null;
break;
case BnfTermKind.Other:
otherCount++;
if (prevChildKind == BnfTermKind.LeftParenthesis)
{
Debug.Assert(parentheses != null && parentheses.LeftParenthesis != null);
childExpression = child;
parentheses.Expression = childExpression;
}
else
{
childExpression = null;
parentheses = null;
}
break;
}
#endregion
if (childKind != BnfTermKind.GrammarHint)
{
prevChild = child;
prevChildKind = childKind;
}
}
#region Determine childListLooksLike
BnfTermKind?childListLooksLike;
if (operatorCount == 1 && leftParenthesisCount == 0 && rightParenthesisCount == 0 && otherCount == 0)
{
childListLooksLike = BnfTermKind.Operator;
}
else if (leftParenthesisCount == 1 && operatorCount == 0 && rightParenthesisCount == 0 && otherCount == 0)
{
childListLooksLike = BnfTermKind.LeftParenthesis;
}
else if (rightParenthesisCount == 1 && operatorCount == 0 && leftParenthesisCount == 0 && otherCount == 0)
{
childListLooksLike = BnfTermKind.RightParenthesis;
}
else
{
childListLooksLike = BnfTermKind.Other;
}
#endregion
if (childListLooksLike == BnfTermKind.Operator)
{
childOperators.Add(childOperator);
}
#region Determine currentKind
if ((currentKind == null || currentKind == BnfTermKind.Operator) && childListLooksLike == BnfTermKind.Operator)
{
currentKind = BnfTermKind.Operator;
}
else if ((currentKind == null || currentKind == BnfTermKind.LeftParenthesis) && childListLooksLike == BnfTermKind.LeftParenthesis)
{
currentKind = BnfTermKind.LeftParenthesis;
}
else if ((currentKind == null || currentKind == BnfTermKind.RightParenthesis) && childListLooksLike == BnfTermKind.RightParenthesis)
{
currentKind = BnfTermKind.RightParenthesis;
}
else
{
currentKind = BnfTermKind.Other;
}
#endregion
}
if (currentKind == BnfTermKind.Operator)
{
HandleOperators(current, childOperators);
}
return(currentKind ?? BnfTermKind.Other);
}
public static async Task <string> AsTextAsync(this IEnumerable <Utoken> utokens, Unparser unparser)
{
using (await unparser.Lock.LockAsync())
return(await Task.Run(() => utokens.AsText(unparser)));
}
public TagToken(Unparser grammar, int start, GrammarToken parent) : base(grammar, start, parent)
{
Type = TagType.Tag;
}
// static string path2 = @"..\..\..\Sarcasm.UnitTest\Test files\Binary1.expr";
static void Main(string[] args)
{
var stopwatch = Stopwatch.StartNew();
stopwatch.Start();
var grammar = new MiniPLG.GrammarP();
ShowTimeAndRestart(stopwatch, "Creation of grammar");
var parser = MultiParser.Create(grammar);
ShowTimeAndRestart(stopwatch, "Creation of parser");
var parseTree = parser.Parse(File.ReadAllText(path), path);
ShowTimeAndRestart(stopwatch, "Parsing");
var astRootValue = parseTree.RootAstValue;
// string generatedCode = new MiniPLC.CSharpGenerator().Generate(astRootValue);
// string generatedCode = new MiniPLC.CppGenerator().Generate(astRootValue);
#if false
var jsonGrammar = new JsonGrammar();
Unparser universalUnparser = new Unparser(jsonGrammar);
var universalParser = MultiParser.Create(jsonGrammar);
string text = universalUnparser.Unparse(astRootValue).AsText(universalUnparser);
var foo = universalUnparser.Unparse(astRootValue).Select(utoken => utoken.GetDecoration()).ToList();
var parseTree2 = universalParser.Parse(text);
var astRootValue2 = parseTree2.RootAstValue;
string text2 = universalUnparser.Unparse(astRootValue2).AsText(universalUnparser);
bool eq = text == text2;
#endif
Unparser unparser = new Unparser(grammar);
ShowTimeAndRestart(stopwatch, "Creation of unparser");
//var utokens = unparser.Unparse(astRootValue).ToList();
//ShowTimeAndRestart(stopwatch, "Unparsing to utokens");
//unparser.EnableParallelProcessing = false;
//string unparsedText = unparser.Unparse(astRootValue).AsText(unparser);
//var document = parseTree.GetDocument();
//string unparsedText = unparser.Unparse(document).AsText(unparser);
//ShowTimeAndRestart(stopwatch, "Converting utokens to string");
unparser.EnableParallelProcessing = false;
//foreach (Utoken utoken in unparser.Unparse(astRootValue))
// Console.WriteLine(utoken.ToString());
unparser.Unparse(astRootValue).ConsumeAll();
ShowTimeAndRestart(stopwatch, "Sequential unparsing to string");
unparser.EnableParallelProcessing = true;
//foreach (Utoken utoken in unparser.Unparse(astRootValue))
// Console.WriteLine(utoken.ToString());
unparser.Unparse(astRootValue).ConsumeAll();
ShowTimeAndRestart(stopwatch, "Parallel unparsing to string");
//var utokensReverse = unparser.Unparse(astRootValue, Unparser.Direction.RightToLeft).ToList();
//ShowTimeAndRestart(stopwatch, "Reverse unparsing to utokens");
unparser.EnableParallelProcessing = false;
unparser.Unparse(astRootValue, Unparser.Direction.RightToLeft).ConsumeAll();
ShowTimeAndRestart(stopwatch, "Reverse sequential unparsing to string");
unparser.EnableParallelProcessing = true;
unparser.Unparse(astRootValue, Unparser.Direction.RightToLeft).ConsumeAll();
ShowTimeAndRestart(stopwatch, "Reverse parallel unparsing to string");
//stopwatch.Stop();
//Console.WriteLine(stopwatch.ElapsedMilliseconds);
//stopwatch.Start();
//var parser2 = MultiParser.Create(grammar);
//stopwatch.Stop();
//Console.WriteLine(stopwatch.ElapsedMilliseconds);
//var parseTree2 = parser.Parse(File.ReadAllText(path2), path2, grammar.B.Expression);
//var parseTree3 = parser.Parse(File.ReadAllText(path2), path2, (NonTerminal)grammar.B.Expression);
//MiniPL.DomainDefinitions.Expression astValue2 = parseTree2.RootAstValue;
//Unparser unparser = new Unparser(grammar);
//Directory.CreateDirectory("unparse_logs");
//var stream = File.Create(@"unparse_logs\unparsed_text");
//unparser.Unparse(parseTree.Root.AstNode).WriteToStream(stream, unparser);
////string str = unparser.Unparse(parseTree.Root.AstNode).AsString(unparser);
////Console.WriteLine(str);
}
private static bool IsNextStronger(InsertedUtokens prevInsertedUtokens, InsertedUtokens nextInsertedUtokens, Unparser.Direction direction)
{
int compareResult = InsertedUtokens.Compare(prevInsertedUtokens, nextInsertedUtokens);
if (compareResult == 0)
{
if (direction == Unparser.Direction.LeftToRight)
return prevInsertedUtokens.kind == InsertedUtokens.Kind.Right;
else
return nextInsertedUtokens.kind != InsertedUtokens.Kind.Right;
}
else
return compareResult < 0;
}
public static async Task WriteToStreamAsync(this IEnumerable <Utoken> utokens, Stream stream, Unparser unparser, CancellationToken cancellationToken)
{
using (await unparser.Lock.LockAsync())
{
using (StreamWriter sw = new StreamWriter(stream))
{
foreach (Utoken utoken in utokens)
{
cancellationToken.ThrowIfCancellationRequested();
await sw.WriteAsync(utoken.ToText(unparser.Formatter));
}
}
}
}
public static void WriteToStream(this IEnumerable<Utoken> utokens, Stream stream, Unparser unparser)
{
utokens.WriteToStream(stream, unparser.Formatter);
}
public ActionToken(Unparser grammar, int start, GrammarToken parent) : base(grammar, start, parent)
{
Type = TagType.Action;
}
public static async Task WriteToStreamAsync(this IEnumerable<Utoken> utokens, Stream stream, Unparser unparser, CancellationToken cancellationToken)
{
using (await unparser.Lock.LockAsync())
{
using (StreamWriter sw = new StreamWriter(stream))
{
foreach (Utoken utoken in utokens)
{
cancellationToken.ThrowIfCancellationRequested();
await sw.WriteAsync(utoken.ToText(unparser.Formatter));
}
}
}
}
public static async Task<string> AsTextAsync(this IEnumerable<Utoken> utokens, Unparser unparser, CancellationToken cancellationToken)
{
using (await unparser.Lock.LockAsync())
{
return await Task.Run(
() =>
{
return string.Concat(
utokens.Select(
utoken =>
{
cancellationToken.ThrowIfCancellationRequested();
return utoken.ToText(unparser.Formatter);
}
)
);
},
cancellationToken
);
}
}
public static string AsText(this IEnumerable <Utoken> utokens, Unparser unparser)
{
return(utokens.AsText(unparser.Formatter));
}
public static async Task WriteToStreamAsync(this IEnumerable<Utoken> utokens, Stream stream, Unparser unparser)
{
await utokens.WriteToStreamAsync(stream, unparser, CancellationToken.None);
}
protected override IEnumerable<UnparsableAst> GetChildren(Unparser.ChildBnfTerms childBnfTerms, object astValue, Unparser.Direction direction)
{
return childBnfTerms.Select(childBnfTerm => new UnparsableAst(childBnfTerm, astValue));
}
public static string AsText(this IEnumerable<Utoken> utokens, Unparser unparser)
{
return utokens.AsText(unparser.Formatter);
}
/// <exception cref="UnparsableAst.NonCalculatedException">
/// If topLeft is non-calculated or thrown out.
/// </exception>
private static IEnumerable<UtokenBase> _YieldIndentation(UnparsableAst self, BlockIndentation blockIndentationParameter, Unparser.Direction direction,
FormatYielder formatYielder, bool left)
{
#if DEBUG
BlockIndentation originalBlockIndentationParameter = blockIndentationParameter;
#endif
var utokens = _YieldIndentation(self, ref blockIndentationParameter, direction, formatYielder, left);
#if DEBUG
Debug.Assert(object.ReferenceEquals(blockIndentationParameter, originalBlockIndentationParameter),
string.Format("unwanted change of blockIndentationParameter reference in _YieldIndentation ({0})", left ? "left" : "right"));
#endif
return utokens;
}
public static JObject Unparse(ISirenEntity entity)
{
return(Unparser.UnparseDocument(entity));
}
public static void WriteToStream(this IEnumerable <Utoken> utokens, Stream stream, Unparser unparser)
{
utokens.WriteToStream(stream, unparser.Formatter);
}
