public static LineageGraphSankey Transform(LineageGraph graph)
        {
            LineageGraphSankey result = new LineageGraphSankey(graph.Id, graph.Type);

            graph.Properties.ForEach(akv => result.AddProperty(akv.Key, akv.Value));

            if (graph.Vertices.Count > 0)
            {
                Dictionary <string, int> idIdx = new Dictionary <string, int>();

                for (int i = 0; i < graph.Vertices.Count; i++)
                {
                    LineageVertex       oldVertex = graph.Vertices[i];
                    LineageVertexSankey newVertex = new LineageVertexSankey(oldVertex.Id, i, oldVertex.Type);

                    newVertex.Properties = oldVertex.Properties;

                    result.AddVertex(newVertex);

                    idIdx.Add(oldVertex.Id, i);
                }

                if (graph.Edges.Count > 0)
                {
                    for (int i = 0; i < graph.Edges.Count; i++)
                    {
                        LineageEdge       oldEdge = graph.Edges[i];
                        LineageEdgeSankey newEdge = new LineageEdgeSankey(oldEdge.Id, oldEdge.Type, idIdx[oldEdge.VSourceId], idIdx[oldEdge.VDestinationId]);

                        newEdge.Properties = oldEdge.Properties;

                        result.AddEdge(newEdge);
                    }
                }
            }

            return(result);
        }
Exemple #2
0
        public static LineageGraph Transfrom(Lineage lineage)
        {
            var result = new LineageGraph(Guid.NewGuid().ToString(), "lineage");

            result.AddProperty("executedOnUTC", DateTimeOffset.Now.ToUnixTimeMilliseconds().ToString());

            #region cluster node

            var rootNodeId = lineage.Clustername + "-" + Guid.NewGuid().ToString();

            var rootVertex = new LineageVertex(rootNodeId, "cluster");
            rootVertex
            .AddProperty("name", lineage.Clustername);

            result.AddVertex(rootVertex);

            #endregion

            #region database nodes

            foreach (var aDatabase in lineage.Databases)
            {
                var databaseVertexId = aDatabase.Key;

                var databaseVertex = new LineageVertex(databaseVertexId, "database");
                databaseVertex
                .AddProperty("name", aDatabase.Value.Name);

                result.AddVertex(databaseVertex);

                var databaseEdge = new LineageEdge(Guid.NewGuid().ToString(), "hasDatabase", rootNodeId, databaseVertexId);

                result.AddEdge(databaseEdge);

                #region table nodes

                var tableIdx = new Dictionary <String, String>();

                foreach (var aTable in aDatabase.Value.Tables)
                {
                    var tableVertexId = databaseVertexId + aTable.Key;

                    tableIdx.Add(aTable.Key, tableVertexId);

                    var tableVertex = new LineageVertex(tableVertexId, "table");
                    tableVertex
                    .AddProperty("name", aTable.Key);

                    result.AddVertex(tableVertex);

                    #region update policies

                    if (aTable.Value.UpdatePolicies.Count > 0)
                    {
                        foreach (var aUpdatePolicy in aTable.Value.UpdatePolicies)
                        {
                            var upEdgeId            = Guid.NewGuid().ToString();
                            var sourceVertexId      = databaseVertexId + aUpdatePolicy.Source;
                            var destinationVertexId = tableVertexId;

                            var upEdge = new LineageEdge(upEdgeId, "propagatesViaUp", sourceVertexId, destinationVertexId);

                            upEdge.AddProperty("IsEnabled", aUpdatePolicy.IsEnabled.ToString());
                            upEdge.AddProperty("IsTransactional", aUpdatePolicy.IsTransactional.ToString());
                            upEdge.AddProperty("PropagateIngestionProperties", aUpdatePolicy.PropagateIngestionProperties.ToString());
                            upEdge.AddProperty("Query", aUpdatePolicy.Query.ToString());

                            result.AddEdge(upEdge);
                        }
                    }
                    else
                    {
                        var tableEdge = new LineageEdge(Guid.NewGuid().ToString(), "hasInternalTable", databaseVertexId, tableVertexId);

                        result.AddEdge(tableEdge);
                    }

                    #endregion
                }

                #endregion

                #region external tables

                var externalTableIdx = new Dictionary <String, String>();

                foreach (var anExternalTable in aDatabase.Value.ExternalTables)
                {
                    var externalTableVertexId = databaseVertexId + anExternalTable.Key;
                    externalTableIdx.Add(anExternalTable.Key, externalTableVertexId);
                }

                #endregion

                #region continous export

                var validExternalTableNames = new HashSet <String>();

                foreach (var aContinousExport in aDatabase.Value.ContinousExport)
                {
                    var ceEdgeId = databaseVertexId + aContinousExport.ExternalTableName;

                    var sourceTableName = ExtractTable(aContinousExport.CurserScopedQuery);

                    if (tableIdx.ContainsKey(sourceTableName) && externalTableIdx.ContainsKey(aContinousExport.ExternalTableName))
                    {
                        var sourceTable   = tableIdx[ExtractTable(aContinousExport.CurserScopedQuery)];
                        var externalTable = externalTableIdx[aContinousExport.ExternalTableName];

                        if (!validExternalTableNames.Contains(aContinousExport.ExternalTableName))
                        {
                            var externalTableVertex = new LineageVertex(externalTable, "externalTable");
                            externalTableVertex
                            .AddProperty("name", aContinousExport.ExternalTableName);
                            result.AddVertex(externalTableVertex);

                            validExternalTableNames.Add(aContinousExport.ExternalTableName);
                        }

                        var ceEdge = new LineageEdge(ceEdgeId, "exportsTo", sourceTable, externalTable);

                        ceEdge.AddProperty("Query", aContinousExport.Query);
                        ceEdge.AddProperty("Name", aContinousExport.ExternalTableName);

                        result.AddEdge(ceEdge);
                    }
                }

                #endregion
            }

            #endregion

            return(result);
        }