static int Export(AzureDataCatalog td, StreamWriter sw, string searchString) { const int countPerPage = 100; bool firstTime = true; int startPage = 1; int totalResultsCount = 0; int totalExportedSuccessfully = 0; sw.Write("{\"catalog\":["); do { string results = td.Search(searchString, startPage, countPerPage); if (results == null) { return(0); } if (firstTime) { totalResultsCount = (int)JObject.Parse(results)["totalResults"]; } var assetList = JObject.Parse(results)["results"].Children(); foreach (JObject asset in assetList["content"]) { if (firstTime) { firstTime = false; } else { sw.Write(","); } (asset["properties"] as JObject).Remove("containerId"); var annotationsNode = asset.SelectToken("annotations") as JObject; if (annotationsNode != null) { bool previewExists = annotationsNode.Remove("previews"); bool columnsDataProfilesExists = annotationsNode.Remove("columnsDataProfiles"); bool tableDataProfilesExist = annotationsNode.Remove("tableDataProfiles"); if (previewExists || columnsDataProfilesExists || tableDataProfilesExist) { var fullAsset = JObject.Parse(td.Get(asset["id"].ToString())); if (previewExists) { annotationsNode.Add("previews", fullAsset["annotations"]["previews"]); } if (columnsDataProfilesExists) { annotationsNode.Add("columnsDataProfiles", fullAsset["annotations"]["columnsDataProfiles"]); } if (tableDataProfilesExist) { annotationsNode.Add("tableDataProfiles", fullAsset["annotations"]["tableDataProfiles"]); } } } //Set contributor equal to "Everyone" on all nodes. This allows them to be updated by others later. Ideally we would preserve the contributor but that requires //a special platform to enable it. JToken contributor = JObject.Parse("{'role': 'Contributor','members': [{'objectId': '00000000-0000-0000-0000-000000000201'}]}"); var roles = new JArray(); roles.Add(contributor); foreach (var rolesNode in asset.SelectTokens("$..roles").ToList()) { rolesNode.Replace(roles); } RemoveSystemProperties(asset); sw.Write(JsonConvert.SerializeObject(asset)); totalExportedSuccessfully++; if (totalExportedSuccessfully % 10 == 0) { Console.Write("."); } } startPage++; } while ((startPage - 1) * countPerPage < totalResultsCount); sw.Write("]}"); Console.WriteLine(""); return(totalExportedSuccessfully); }
static int Export(AzureDataCatalog td, StreamWriter sw, string searchString) { const int countPerPage = 100; bool firstTime = true; int startPage = 1; int totalResultsCount = 0; int totalExportedSuccessfully = 0; sw.Write("{\"catalog\":["); do { string results = td.Search(searchString, startPage, countPerPage); if (results == null) { return(0); } if (firstTime) { totalResultsCount = (int)JObject.Parse(results)["totalResults"]; } var assetList = JObject.Parse(results)["results"].Children(); foreach (JObject asset in assetList["content"]) { if (firstTime) { firstTime = false; } else { sw.Write(","); } (asset["properties"] as JObject).Remove("containerId"); var annotationsNode = asset.SelectToken("annotations") as JObject; // pseudo code to write out new format // create loop and write table name to each row // loop again within loop write the column // within loop write the data types // query for the description if column has it then write it //testing github commit //another test github var table_desc = asset.SelectToken("descriptions"); JToken table_description = JObject.Parse("{'descriptions': ['properties': {'description':]}}"); //add custom code here to parse out differently. // ...from asset ...path to the list of cols and data types "schema": { // "id": "https://3740b0dc-672b-4bd1-adac-07e3e2a0c934-coapocdatacatalog.api.datacatalog.azure.com/catalogs/coapocdatacatalog/views/tables/cae113ca-a8c8-459a-9182-3685d76b4e9f/schema", //"type": "Microsoft.DataSource.Schema.1", //"timestamp": "2017-07-14T18:54:39.4329161Z", //"properties": { // "columns": [ // { // "name": "PROVID", // ...from data set...path to overal documentation for the table....this is the wiki page // "documentation": { // "id": "https://3740b0dc-672b-4bd1-adac-07e3e2a0c934-coapocdatacatalog.api.datacatalog.azure.com/catalogs/coapocdatacatalog/views/tables/cae113ca-a8c8-459a-9182-3685d76b4e9f/documentation", //"type": "Microsoft.Documentation.1", //"timestamp": "2018-07-30T16:23:13.4816962Z", //"properties": { // "mimeType": "text/html", // "content": "<p><strong>ODS - Provider Specialty Materialized View</strong></p><p>This view is used to show the primary specialty for a provider and associated provider information. This materialized view was created to simplify the query logic to ease the burden and simplify access to this information.</p><p><strong>Change Log:</strong></p><ul><li>20180213 - Updated documenation to reflect a code change</li><li>20150812 - Created documentation</li></ul><p><a href=\"http://example.coccess.com/docs\" target=\"_blank\">Provider Specialty MV Documentation</a></p><p> </p>", // "fromSourceS // ...from asset ...path to the table description // "descriptions": [ //{ // "id": "https://3740b0dc-672b-4bd1-adac-07e3e2a0c934-coapocdatacatalog.api.datacatalog.azure.com/catalogs/coapocdatacatalog/views/tables/cae113ca-a8c8-459a-9182-3685d76b4e9f/descriptions/9a3141958b374fc1bf8d4ea9f5696c93", // "type": "Microsoft.Description.1", // "timestamp": "2018-07-27T15:23:21.9458843Z", // "properties": { // "description": "The specialty information in this MV is specifically built and designed for communication with state based on their specialty roll-ups.", // "fromSourceSystem": false, // "key": "9a3141958b374fc1bf8d4ea9f5696c93" if (annotationsNode != null) { bool previewExists = annotationsNode.Remove("previews"); bool columnsDataProfilesExists = annotationsNode.Remove("columnsDataProfiles"); bool tableDataProfilesExist = annotationsNode.Remove("tableDataProfiles"); if (previewExists || columnsDataProfilesExists || tableDataProfilesExist) { var fullAsset = JObject.Parse(td.Get(asset["id"].ToString())); if (previewExists) { annotationsNode.Add("previews", fullAsset["annotations"]["previews"]); } if (columnsDataProfilesExists) { annotationsNode.Add("columnsDataProfiles", fullAsset["annotations"]["columnsDataProfiles"]); } if (tableDataProfilesExist) { annotationsNode.Add("tableDataProfiles", fullAsset["annotations"]["tableDataProfiles"]); } } } //Set contributor equal to "Everyone" on all nodes. This allows them to be updated by others later. Ideally we would preserve the contributor but that requires //a special platform to enable it. JToken contributor = JObject.Parse("{'role': 'Contributor','members': [{'objectId': '00000000-0000-0000-0000-000000000201'}]}"); var roles = new JArray(); roles.Add(contributor); foreach (var rolesNode in asset.SelectTokens("$..roles").ToList()) { rolesNode.Replace(roles); } RemoveSystemProperties(asset); sw.Write(JsonConvert.SerializeObject(asset)); totalExportedSuccessfully++; if (totalExportedSuccessfully % 10 == 0) { Console.Write("."); } } startPage++; } while ((startPage - 1) * countPerPage < totalResultsCount); sw.Write("]}"); Console.WriteLine(""); return(totalExportedSuccessfully); }