public static void Main(string[] args)
        {
            var code1 = @"using System;

                        class Hello
                                {
                                    public static readonly void Main(){
                                        foreach(var h in asdf){
                                             h.Write(test);
                                        }
    
                                        var foo = new Bar(1,2,3,4,5,SomeProperty,""MyString"");
                                        var foo1 = sdf(x => x.sdf);
                                        var foo1 = sdf((x,y,z) => x.sdf);
                                        var foo2 = sdf();
                                        Assert.Equal(SomeInt, new int[] {1,2,3,4});

                                        Main();
                                    }

                                    public static int SomeInt = 2;
                                    public static string SomeProperty {get; set;};

                                    
                                }

                     class Hello2 {
                        public static int sdf = 3;
                     }
";

            var syntaxTree = CSharpSyntaxTree.ParseText(code1);

            var nodes = syntaxTree.GetRoot().DescendantNodes().ToList();

            Console.WriteLine(Canonicalization.CanonicalizeSyntaxNode(syntaxTree.GetRoot(), extractAllVariablesFirst: true).CanonicalSyntaxNode.GetText());

            Console.ReadLine();
        }
        public static IEnumerable <object> ProcessSingleRevision(string jsonLine, JsonSyntaxTreeHelper jsonSyntaxTreeHelper)
        {
            var entry = JObject.Parse(jsonLine);

            var previousFile = entry["prev_file"].ToString();
            var updatedFile  = entry["updated_file"].ToString();

            // Console.WriteLine($"Processing {entry["id"]}");

            // File.WriteAllText("a.original.cs", previousFile);
            // File.WriteAllText("b.original.cs", updatedFile);

            var previousFileAst = CSharpSyntaxTree.ParseText(previousFile);
            var updatedFileAst  = CSharpSyntaxTree.ParseText(updatedFile);

            (SyntaxNode canonicalPrevFileAst, Dictionary <string, string> prevFileVariableNameMap)       = Canonicalization.CanonicalizeSyntaxNode(previousFileAst.GetRoot(), extractAllVariablesFirst: true);
            (SyntaxNode canonicalUpdatedFileAst, Dictionary <string, string> updatedFileVariableNameMap) = Canonicalization.CanonicalizeSyntaxNode(updatedFileAst.GetRoot(), prevFileVariableNameMap);

            var prevCodeFile    = canonicalPrevFileAst.GetText();
            var updatedCodeFile = canonicalUpdatedFileAst.GetText();

            var prevFileTokens    = canonicalPrevFileAst.DescendantTokens().ToList();
            var updatedFileTokens = canonicalUpdatedFileAst.DescendantTokens().ToList();

            var changesInRevision = GetChangesBetweenAsts(canonicalPrevFileAst.SyntaxTree, canonicalUpdatedFileAst.SyntaxTree);

            // File.WriteAllText("a.canonical.cs", canonicalPrevFileAst.GetText().ToString());
            // File.WriteAllText("b.canonical.cs", canonicalUpdatedFileAst.GetText().ToString());

            var prevTokenIndex    = new TokenIndex(prevFileTokens);
            var updatedTokenIndex = new TokenIndex(updatedFileTokens);

            var changeId = 0;

            foreach (var change in changesInRevision)
            {
                var prevCodeChunkLineSpan    = canonicalPrevFileAst.SyntaxTree.GetLineSpan(change.BeforeSpan.ChangeSpan);
                var updatedCodeChunkLineSpan = canonicalUpdatedFileAst.SyntaxTree.GetLineSpan(change.AfterSpan.ChangeSpan);

                var prevCodeChunkLineSpanStart = prevCodeFile.Lines[prevCodeChunkLineSpan.StartLinePosition.Line].Span.Start;
                var prevCodeChunkSpanEnd       = prevCodeFile.Lines[prevCodeChunkLineSpan.EndLinePosition.Line].Span.End;

                var updatedCodeChunkLineSpanStart = updatedCodeFile.Lines[updatedCodeChunkLineSpan.StartLinePosition.Line].Span.Start;
                var updatedCodeChunkSpanEnd       = updatedCodeFile.Lines[updatedCodeChunkLineSpan.EndLinePosition.Line].Span.End;

                // only consider changes of equal number of lines
                if (prevCodeChunkLineSpan.EndLinePosition.Line - prevCodeChunkLineSpan.StartLinePosition.Line
                    != updatedCodeChunkLineSpan.EndLinePosition.Line - updatedCodeChunkLineSpan.StartLinePosition.Line)
                {
                    continue;
                }

                // TODO: remove trivial change

                // only consider SyntaxKind in allowedSytaxKinds
                var prevCodeChunkNodes = GetNodesByLineSpan(canonicalPrevFileAst, prevCodeFile,
                                                            prevCodeChunkLineSpan.StartLinePosition.Line, prevCodeChunkLineSpan.EndLinePosition.Line);
                if (prevCodeChunkNodes.Any(node => !allowedSytaxKinds.Contains(node.Kind())))
                {
                    continue;
                }

                var updatedCodeChunkNodes = GetNodesByLineSpan(canonicalUpdatedFileAst, updatedCodeFile,
                                                               updatedCodeChunkLineSpan.StartLinePosition.Line, updatedCodeChunkLineSpan.EndLinePosition.Line);
                if (updatedCodeChunkNodes.Any(node => !allowedSytaxKinds.Contains(node.Kind())))
                {
                    continue;
                }

                var previousCodeChunkTokens = prevTokenIndex
                                              .GetTokensInSpan(prevCodeChunkLineSpanStart, prevCodeChunkSpanEnd)
                                              .Select(token => token.ValueText)
                                              .Where(token => !string.IsNullOrWhiteSpace(token) && !string.IsNullOrEmpty(token))
                                              .ToArray();

                var updatedsCodeChunkTokens = updatedTokenIndex
                                              .GetTokensInSpan(updatedCodeChunkLineSpanStart, updatedCodeChunkSpanEnd)
                                              .Select(token => token.ValueText)
                                              .Where(token => !string.IsNullOrWhiteSpace(token) && !string.IsNullOrEmpty(token))
                                              .ToArray();

                if (previousCodeChunkTokens.Length > 0 && updatedsCodeChunkTokens.Length > 0 &&
                    IsValidCodeChunkTokens(previousCodeChunkTokens) && IsValidCodeChunkTokens(updatedsCodeChunkTokens) &&
                    !previousCodeChunkTokens.SequenceEqual(updatedsCodeChunkTokens))
                {
                    var changeSha = entry["id"] + "_" + changeId;

                    var prevCodeChunkBlockStmt    = SyntaxFactory.Block(prevCodeChunkNodes.Select(node => (StatementSyntax)node));
                    var updatedCodeChunkBlockStmt = SyntaxFactory.Block(updatedCodeChunkNodes.Select(node => (StatementSyntax)node));

                    IDictionary <string, string> zeroIndexedVariableNameMap;
                    (prevCodeChunkBlockStmt, updatedCodeChunkBlockStmt, zeroIndexedVariableNameMap) =
                        zeroIndexVariableNames(prevCodeChunkBlockStmt, updatedCodeChunkBlockStmt);

                    var prevCodeChunkBlockStmtTokens      = prevCodeChunkBlockStmt.DescendantTokens().Skip(1).SkipLast(1).ToArray();
                    var prevCodeChunkBlackStmtTokensIndex = new TokenIndex(prevCodeChunkBlockStmtTokens).InitInvertedIndex();

                    var updatedCodeChunkBlockStmtTokens      = updatedCodeChunkBlockStmt.DescendantTokens().Skip(1).SkipLast(1).ToArray();
                    var updatedCodeChunkBlockStmtTokensIndex = new TokenIndex(updatedCodeChunkBlockStmtTokens).InitInvertedIndex();

                    var prevCodeBlockJObject    = jsonSyntaxTreeHelper.GetJObjectForSyntaxNode(prevCodeChunkBlockStmt, prevCodeChunkBlackStmtTokensIndex);
                    var updatedCodeBlockJObject = jsonSyntaxTreeHelper.GetJObjectForSyntaxNode(updatedCodeChunkBlockStmt, updatedCodeChunkBlockStmtTokensIndex);

                    var precedingContextTokens  = prevTokenIndex.GetTokensInSpan(change.BeforeSpan.SpanOfPrecedingContext);
                    var succeedingContextTokens = updatedTokenIndex.GetTokensInSpan(change.BeforeSpan.SpanOfSucceedingContext);

                    precedingContextTokens  = zeroIndexVariableNames(precedingContextTokens, zeroIndexedVariableNameMap);
                    succeedingContextTokens = zeroIndexVariableNames(succeedingContextTokens, zeroIndexedVariableNameMap);

                    var prevCodeChunkBlockStmtTextTokens =
                        prevCodeChunkBlockStmtTokens.Select(token => token.ValueText).ToArray();
                    var updatedCodeChunkBlockStmtTextTokens =
                        updatedCodeChunkBlockStmtTokens.Select(token => token.ValueText).ToArray();

                    var prevCodeTextChunk = Utils.ExtractCodeTextFromBraces(prevCodeChunkBlockStmt.GetText().ToString());
                    prevCodeTextChunk = Utils.RemoveLeadingWhiteSpace(prevCodeTextChunk, naive: true);

                    var updatedCodeTextChunk = Utils.ExtractCodeTextFromBraces(updatedCodeChunkBlockStmt.GetText().ToString());
                    updatedCodeTextChunk = Utils.RemoveLeadingWhiteSpace(updatedCodeTextChunk, naive: true);

                    var precedingContextTextTokens  = precedingContextTokens.Select(token => token.ValueText).ToArray();
                    var succeedingContextTextTokens = succeedingContextTokens.Select(token => token.ValueText).ToArray();

                    var result = new
                    {
                        Id                     = changeSha,
                        PrevCodeChunk          = prevCodeTextChunk,
                        UpdatedCodeChunk       = updatedCodeTextChunk,
                        PrevCodeChunkTokens    = prevCodeChunkBlockStmtTextTokens,
                        UpdatedCodeChunkTokens = updatedCodeChunkBlockStmtTextTokens,
                        PrevCodeAST            = prevCodeBlockJObject,
                        UpdatedCodeAST         = updatedCodeBlockJObject,
                        PrecedingContext       = precedingContextTextTokens,
                        SucceedingContext      = succeedingContextTextTokens,
                        CommitMessage          = entry["message"]
                    };

                    changeId += 1;

                    yield return(result);
                }
            }
        }