static int Export(AzureDataCatalog td, StreamWriter sw, string searchString)
{
const int countPerPage = 100;
bool firstTime = true;
int startPage = 1;
int totalResultsCount = 0;
int totalExportedSuccessfully = 0;
sw.Write("{\"catalog\":[");
do
{
string results = td.Search(searchString, startPage, countPerPage);
if (results == null)
{
return(0);
}
if (firstTime)
{
totalResultsCount = (int)JObject.Parse(results)["totalResults"];
}
var assetList = JObject.Parse(results)["results"].Children();
foreach (JObject asset in assetList["content"])
{
if (firstTime)
{
firstTime = false;
}
else
{
sw.Write(",");
}
(asset["properties"] as JObject).Remove("containerId");
var annotationsNode = asset.SelectToken("annotations") as JObject;
if (annotationsNode != null)
{
bool previewExists = annotationsNode.Remove("previews");
bool columnsDataProfilesExists = annotationsNode.Remove("columnsDataProfiles");
bool tableDataProfilesExist = annotationsNode.Remove("tableDataProfiles");
if (previewExists || columnsDataProfilesExists || tableDataProfilesExist)
{
var fullAsset = JObject.Parse(td.Get(asset["id"].ToString()));
if (previewExists)
{
annotationsNode.Add("previews", fullAsset["annotations"]["previews"]);
}
if (columnsDataProfilesExists)
{
annotationsNode.Add("columnsDataProfiles", fullAsset["annotations"]["columnsDataProfiles"]);
}
if (tableDataProfilesExist)
{
annotationsNode.Add("tableDataProfiles", fullAsset["annotations"]["tableDataProfiles"]);
}
}
}
//Set contributor equal to "Everyone" on all nodes. This allows them to be updated by others later. Ideally we would preserve the contributor but that requires
//a special platform to enable it.
JToken contributor = JObject.Parse("{'role': 'Contributor','members': [{'objectId': '00000000-0000-0000-0000-000000000201'}]}");
var roles = new JArray();
roles.Add(contributor);
foreach (var rolesNode in asset.SelectTokens("$..roles").ToList())
{
rolesNode.Replace(roles);
}
RemoveSystemProperties(asset);
sw.Write(JsonConvert.SerializeObject(asset));
totalExportedSuccessfully++;
if (totalExportedSuccessfully % 10 == 0)
{
Console.Write(".");
}
}
startPage++;
} while ((startPage - 1) * countPerPage < totalResultsCount);
sw.Write("]}");
Console.WriteLine("");
return(totalExportedSuccessfully);
}
static int Export(AzureDataCatalog td, StreamWriter sw, string searchString)
{
const int countPerPage = 100;
bool firstTime = true;
int startPage = 1;
int totalResultsCount = 0;
int totalExportedSuccessfully = 0;
sw.Write("{\"catalog\":[");
do
{
string results = td.Search(searchString, startPage, countPerPage);
if (results == null)
{
return(0);
}
if (firstTime)
{
totalResultsCount = (int)JObject.Parse(results)["totalResults"];
}
var assetList = JObject.Parse(results)["results"].Children();
foreach (JObject asset in assetList["content"])
{
if (firstTime)
{
firstTime = false;
}
else
{
sw.Write(",");
}
(asset["properties"] as JObject).Remove("containerId");
var annotationsNode = asset.SelectToken("annotations") as JObject;
// pseudo code to write out new format
// create loop and write table name to each row
// loop again within loop write the column
// within loop write the data types
// query for the description if column has it then write it
//testing github commit
//another test github
var table_desc = asset.SelectToken("descriptions");
JToken table_description = JObject.Parse("{'descriptions': ['properties': {'description':]}}");
//add custom code here to parse out differently.
// ...from asset ...path to the list of cols and data types "schema": {
// "id": "https://3740b0dc-672b-4bd1-adac-07e3e2a0c934-coapocdatacatalog.api.datacatalog.azure.com/catalogs/coapocdatacatalog/views/tables/cae113ca-a8c8-459a-9182-3685d76b4e9f/schema",
//"type": "Microsoft.DataSource.Schema.1",
//"timestamp": "2017-07-14T18:54:39.4329161Z",
//"properties": {
// "columns": [
// {
// "name": "PROVID",
// ...from data set...path to overal documentation for the table....this is the wiki page
// "documentation": {
// "id": "https://3740b0dc-672b-4bd1-adac-07e3e2a0c934-coapocdatacatalog.api.datacatalog.azure.com/catalogs/coapocdatacatalog/views/tables/cae113ca-a8c8-459a-9182-3685d76b4e9f/documentation",
//"type": "Microsoft.Documentation.1",
//"timestamp": "2018-07-30T16:23:13.4816962Z",
//"properties": {
// "mimeType": "text/html",
// "content": "<p><strong>ODS - Provider Specialty Materialized View</strong></p><p>This view is used to show the primary specialty for a provider and associated provider information. This materialized view was created to simplify the query logic to ease the burden and simplify access to this information.</p><p><strong>Change Log:</strong></p><ul><li>20180213 - Updated documenation to reflect a code change</li><li>20150812 - Created documentation</li></ul><p><a href=\"http://example.coccess.com/docs\" target=\"_blank\">Provider Specialty MV Documentation</a></p><p> </p>",
// "fromSourceS
// ...from asset ...path to the table description
// "descriptions": [
//{
// "id": "https://3740b0dc-672b-4bd1-adac-07e3e2a0c934-coapocdatacatalog.api.datacatalog.azure.com/catalogs/coapocdatacatalog/views/tables/cae113ca-a8c8-459a-9182-3685d76b4e9f/descriptions/9a3141958b374fc1bf8d4ea9f5696c93",
// "type": "Microsoft.Description.1",
// "timestamp": "2018-07-27T15:23:21.9458843Z",
// "properties": {
// "description": "The specialty information in this MV is specifically built and designed for communication with state based on their specialty roll-ups.",
// "fromSourceSystem": false,
// "key": "9a3141958b374fc1bf8d4ea9f5696c93"
if (annotationsNode != null)
{
bool previewExists = annotationsNode.Remove("previews");
bool columnsDataProfilesExists = annotationsNode.Remove("columnsDataProfiles");
bool tableDataProfilesExist = annotationsNode.Remove("tableDataProfiles");
if (previewExists || columnsDataProfilesExists || tableDataProfilesExist)
{
var fullAsset = JObject.Parse(td.Get(asset["id"].ToString()));
if (previewExists)
{
annotationsNode.Add("previews", fullAsset["annotations"]["previews"]);
}
if (columnsDataProfilesExists)
{
annotationsNode.Add("columnsDataProfiles", fullAsset["annotations"]["columnsDataProfiles"]);
}
if (tableDataProfilesExist)
{
annotationsNode.Add("tableDataProfiles", fullAsset["annotations"]["tableDataProfiles"]);
}
}
}
//Set contributor equal to "Everyone" on all nodes. This allows them to be updated by others later. Ideally we would preserve the contributor but that requires
//a special platform to enable it.
JToken contributor = JObject.Parse("{'role': 'Contributor','members': [{'objectId': '00000000-0000-0000-0000-000000000201'}]}");
var roles = new JArray();
roles.Add(contributor);
foreach (var rolesNode in asset.SelectTokens("$..roles").ToList())
{
rolesNode.Replace(roles);
}
RemoveSystemProperties(asset);
sw.Write(JsonConvert.SerializeObject(asset));
totalExportedSuccessfully++;
if (totalExportedSuccessfully % 10 == 0)
{
Console.Write(".");
}
}
startPage++;
} while ((startPage - 1) * countPerPage < totalResultsCount);
sw.Write("]}");
Console.WriteLine("");
return(totalExportedSuccessfully);
}
