static void Main(string[] args)
{
if (args.Length < 2)
{
System.Console.WriteLine("-import file_path or -export file_path [-search search_string]");
Console.ReadLine();
return;
}
string operation = args[0];
string filePath = args[1];
AzureDataCatalog td = new AzureDataCatalog();
if (operation.Equals("-export"))
{
string searchString = "";
if ((args.Length >= 4) && (args[2].ToLower().Equals("-search")))
{
//searchString = args[3];
searchString = "tags=VSA";
}
using (StreamWriter sw = new StreamWriter(filePath, false))
{
int count = Export(td, sw, searchString);
System.Console.WriteLine("Items Exported: " + count);
}
}
else if (operation.Equals("-import"))
{
Import(td, filePath);
}
else
{
System.Console.WriteLine("Must specify either import or export");
}
Console.ReadLine();
}
static int Export(AzureDataCatalog td, StreamWriter sw, string searchString)
{
const int countPerPage = 100;
bool firstTime = true;
int startPage = 1;
int totalResultsCount = 0;
int totalExportedSuccessfully = 0;
sw.Write("{\"catalog\":[");
do
{
string results = td.Search(searchString, startPage, countPerPage);
if (results == null)
{
return(0);
}
if (firstTime)
{
totalResultsCount = (int)JObject.Parse(results)["totalResults"];
}
var assetList = JObject.Parse(results)["results"].Children();
foreach (JObject asset in assetList["content"])
{
if (firstTime)
{
firstTime = false;
}
else
{
sw.Write(",");
}
(asset["properties"] as JObject).Remove("containerId");
var annotationsNode = asset.SelectToken("annotations") as JObject;
if (annotationsNode != null)
{
bool previewExists = annotationsNode.Remove("previews");
bool columnsDataProfilesExists = annotationsNode.Remove("columnsDataProfiles");
bool tableDataProfilesExist = annotationsNode.Remove("tableDataProfiles");
if (previewExists || columnsDataProfilesExists || tableDataProfilesExist)
{
var fullAsset = JObject.Parse(td.Get(asset["id"].ToString()));
if (previewExists)
{
annotationsNode.Add("previews", fullAsset["annotations"]["previews"]);
}
if (columnsDataProfilesExists)
{
annotationsNode.Add("columnsDataProfiles", fullAsset["annotations"]["columnsDataProfiles"]);
}
if (tableDataProfilesExist)
{
annotationsNode.Add("tableDataProfiles", fullAsset["annotations"]["tableDataProfiles"]);
}
}
}
//Set contributor equal to "Everyone" on all nodes. This allows them to be updated by others later. Ideally we would preserve the contributor but that requires
//a special platform to enable it.
JToken contributor = JObject.Parse("{'role': 'Contributor','members': [{'objectId': '00000000-0000-0000-0000-000000000201'}]}");
var roles = new JArray();
roles.Add(contributor);
foreach (var rolesNode in asset.SelectTokens("$..roles").ToList())
{
rolesNode.Replace(roles);
}
RemoveSystemProperties(asset);
sw.Write(JsonConvert.SerializeObject(asset));
totalExportedSuccessfully++;
if (totalExportedSuccessfully % 10 == 0)
{
Console.Write(".");
}
}
startPage++;
} while ((startPage - 1) * countPerPage < totalResultsCount);
sw.Write("]}");
Console.WriteLine("");
return(totalExportedSuccessfully);
}
static void Import(AzureDataCatalog td, string exportedCatalogFilePath)
{
int totalAssetsImportSucceeded = 0;
int totalAssetsImportFailed = 0;
System.IO.StreamReader sr = new StreamReader(exportedCatalogFilePath);
JsonTextReader reader = new JsonTextReader(sr);
StringWriter sw = new StringWriter(new StringBuilder());
JsonTextWriter jtw = new JsonTextWriter(sw);
reader.Read();
if (reader.TokenType != JsonToken.StartObject)
{
throw new Exception("Invalid Json. Expected StartObject");
}
reader.Read();
if ((reader.TokenType != JsonToken.PropertyName) || (!reader.Value.ToString().Equals("catalog")))
{
throw new Exception("Invalid Json. Expected catalog array");
}
reader.Read();
if (reader.TokenType != JsonToken.StartArray)
{
throw new Exception("Invalid Json. Expected StartArray");
}
while (reader.Read())
{
if (reader.TokenType == JsonToken.EndArray)
{
break;
}
jtw.WriteToken(reader);
JObject asset = JObject.Parse(sw.ToString());
string id = asset["id"].ToString();
asset.Remove("id");
string[] idInfo = id.Split(new char[] { '/' });
string newid;
string UpdateResponse = td.Update(asset.ToString(), idInfo[idInfo.Length - 2], out newid);
if ((UpdateResponse != null) && (!string.IsNullOrEmpty(newid)))
{
totalAssetsImportSucceeded++;
if (totalAssetsImportSucceeded % 50 == 0)
{
System.Console.WriteLine(totalAssetsImportSucceeded + "Assets Imported Succesfully");
}
}
else
{
totalAssetsImportFailed++;
}
//reset local variables for next iteration
sw = new StringWriter(new StringBuilder());
jtw = new JsonTextWriter(sw);
}
Console.WriteLine("Total Imported Success: " + totalAssetsImportSucceeded);
Console.WriteLine("Total Imported Failed: " + totalAssetsImportFailed);
}
static int Export(AzureDataCatalog td, StreamWriter sw, string searchString)
{
const int countPerPage = 100;
bool firstTime = true;
int startPage = 1;
int totalResultsCount = 0;
int totalExportedSuccessfully = 0;
sw.Write("{\"catalog\":[");
do
{
string results = td.Search(searchString, startPage, countPerPage);
if (results == null)
{
return(0);
}
if (firstTime)
{
totalResultsCount = (int)JObject.Parse(results)["totalResults"];
}
var assetList = JObject.Parse(results)["results"].Children();
foreach (JObject asset in assetList["content"])
{
if (firstTime)
{
firstTime = false;
}
else
{
sw.Write(",");
}
(asset["properties"] as JObject).Remove("containerId");
var annotationsNode = asset.SelectToken("annotations") as JObject;
// pseudo code to write out new format
// create loop and write table name to each row
// loop again within loop write the column
// within loop write the data types
// query for the description if column has it then write it
//testing github commit
//another test github
var table_desc = asset.SelectToken("descriptions");
JToken table_description = JObject.Parse("{'descriptions': ['properties': {'description':]}}");
//add custom code here to parse out differently.
// ...from asset ...path to the list of cols and data types "schema": {
// "id": "https://3740b0dc-672b-4bd1-adac-07e3e2a0c934-coapocdatacatalog.api.datacatalog.azure.com/catalogs/coapocdatacatalog/views/tables/cae113ca-a8c8-459a-9182-3685d76b4e9f/schema",
//"type": "Microsoft.DataSource.Schema.1",
//"timestamp": "2017-07-14T18:54:39.4329161Z",
//"properties": {
// "columns": [
// {
// "name": "PROVID",
// ...from data set...path to overal documentation for the table....this is the wiki page
// "documentation": {
// "id": "https://3740b0dc-672b-4bd1-adac-07e3e2a0c934-coapocdatacatalog.api.datacatalog.azure.com/catalogs/coapocdatacatalog/views/tables/cae113ca-a8c8-459a-9182-3685d76b4e9f/documentation",
//"type": "Microsoft.Documentation.1",
//"timestamp": "2018-07-30T16:23:13.4816962Z",
//"properties": {
// "mimeType": "text/html",
// "content": "<p><strong>ODS - Provider Specialty Materialized View</strong></p><p>This view is used to show the primary specialty for a provider and associated provider information. This materialized view was created to simplify the query logic to ease the burden and simplify access to this information.</p><p><strong>Change Log:</strong></p><ul><li>20180213 - Updated documenation to reflect a code change</li><li>20150812 - Created documentation</li></ul><p><a href=\"http://example.coccess.com/docs\" target=\"_blank\">Provider Specialty MV Documentation</a></p><p> </p>",
// "fromSourceS
// ...from asset ...path to the table description
// "descriptions": [
//{
// "id": "https://3740b0dc-672b-4bd1-adac-07e3e2a0c934-coapocdatacatalog.api.datacatalog.azure.com/catalogs/coapocdatacatalog/views/tables/cae113ca-a8c8-459a-9182-3685d76b4e9f/descriptions/9a3141958b374fc1bf8d4ea9f5696c93",
// "type": "Microsoft.Description.1",
// "timestamp": "2018-07-27T15:23:21.9458843Z",
// "properties": {
// "description": "The specialty information in this MV is specifically built and designed for communication with state based on their specialty roll-ups.",
// "fromSourceSystem": false,
// "key": "9a3141958b374fc1bf8d4ea9f5696c93"
if (annotationsNode != null)
{
bool previewExists = annotationsNode.Remove("previews");
bool columnsDataProfilesExists = annotationsNode.Remove("columnsDataProfiles");
bool tableDataProfilesExist = annotationsNode.Remove("tableDataProfiles");
if (previewExists || columnsDataProfilesExists || tableDataProfilesExist)
{
var fullAsset = JObject.Parse(td.Get(asset["id"].ToString()));
if (previewExists)
{
annotationsNode.Add("previews", fullAsset["annotations"]["previews"]);
}
if (columnsDataProfilesExists)
{
annotationsNode.Add("columnsDataProfiles", fullAsset["annotations"]["columnsDataProfiles"]);
}
if (tableDataProfilesExist)
{
annotationsNode.Add("tableDataProfiles", fullAsset["annotations"]["tableDataProfiles"]);
}
}
}
//Set contributor equal to "Everyone" on all nodes. This allows them to be updated by others later. Ideally we would preserve the contributor but that requires
//a special platform to enable it.
JToken contributor = JObject.Parse("{'role': 'Contributor','members': [{'objectId': '00000000-0000-0000-0000-000000000201'}]}");
var roles = new JArray();
roles.Add(contributor);
foreach (var rolesNode in asset.SelectTokens("$..roles").ToList())
{
rolesNode.Replace(roles);
}
RemoveSystemProperties(asset);
sw.Write(JsonConvert.SerializeObject(asset));
totalExportedSuccessfully++;
if (totalExportedSuccessfully % 10 == 0)
{
Console.Write(".");
}
}
startPage++;
} while ((startPage - 1) * countPerPage < totalResultsCount);
sw.Write("]}");
Console.WriteLine("");
return(totalExportedSuccessfully);
}
