static void Main(string[] args)
{
if (args.Length < 2)
{
System.Console.WriteLine("-import file_path or -export file_path [-search search_string]");
Console.ReadLine();
return;
}
string operation = args[0];
string filePath = args[1];
AzureDataCatalog td = new AzureDataCatalog();
if (operation.Equals("-export"))
{
string searchString = "";
if ((args.Length >= 4) && (args[2].ToLower().Equals("-search")))
{
searchString = args[3];
}
using (StreamWriter sw = new StreamWriter(filePath, false))
{
int count = Export(td, sw, searchString);
System.Console.WriteLine("Items Exported: " + count);
}
}
else if (operation.Equals("-import"))
{
Import(td, filePath);
}
else
{
System.Console.WriteLine("Must specify either import or export");
}
Console.ReadLine();
}
static int Export(AzureDataCatalog td, StreamWriter sw, string searchString)
{
const int countPerPage = 100;
bool firstTime = true;
int startPage = 1;
int totalResultsCount = 0;
int totalExportedSuccessfully = 0;
sw.Write("{\"catalog\":[");
do
{
string results = td.Search(searchString, startPage, countPerPage);
if (results == null)
{
return 0;
}
if (firstTime)
{
totalResultsCount = (int)JObject.Parse(results)["totalResults"];
}
var assetList = JObject.Parse(results)["results"].Children();
foreach (JObject asset in assetList["content"])
{
if (firstTime)
{
firstTime = false;
}
else
{
sw.Write(",");
}
(asset["properties"] as JObject).Remove("containerId");
var annotationsNode = asset.SelectToken("annotations") as JObject;
if (annotationsNode != null)
{
bool previewExists = annotationsNode.Remove("previews");
bool columnsDataProfilesExists = annotationsNode.Remove("columnsDataProfiles");
bool tableDataProfilesExist = annotationsNode.Remove("tableDataProfiles");
if (previewExists || columnsDataProfilesExists || tableDataProfilesExist)
{
var fullAsset = JObject.Parse(td.Get(asset["id"].ToString()));
if (previewExists)
{
annotationsNode.Add("previews", fullAsset["annotations"]["previews"]);
}
if (columnsDataProfilesExists)
{
annotationsNode.Add("columnsDataProfiles", fullAsset["annotations"]["columnsDataProfiles"]);
}
if (tableDataProfilesExist)
{
annotationsNode.Add("tableDataProfiles", fullAsset["annotations"]["tableDataProfiles"]);
}
}
}
//Set contributor equal to "Everyone" on all nodes. This allows them to be updated by others later. Ideally we would preserve the contributor but that requires
//a special platform to enable it.
JToken contributor = JObject.Parse("{'role': 'Contributor','members': [{'objectId': '00000000-0000-0000-0000-000000000201'}]}");
var roles = new JArray();
roles.Add(contributor);
foreach (var rolesNode in asset.SelectTokens("$..roles").ToList())
{
rolesNode.Replace(roles);
}
RemoveSystemProperties(asset);
sw.Write(JsonConvert.SerializeObject(asset));
totalExportedSuccessfully++;
if (totalExportedSuccessfully % 10 == 0)
{
Console.Write(".");
}
}
startPage++;
} while ((startPage - 1) * countPerPage < totalResultsCount);
sw.Write("]}");
Console.WriteLine("");
return totalExportedSuccessfully;
}
static void Import(AzureDataCatalog td, string exportedCatalogFilePath)
{
int totalAssetsImportSucceeded = 0;
int totalAssetsImportFailed = 0;
System.IO.StreamReader sr = new StreamReader(exportedCatalogFilePath);
JsonTextReader reader = new JsonTextReader(sr);
StringWriter sw = new StringWriter(new StringBuilder());
JsonTextWriter jtw = new JsonTextWriter(sw);
reader.Read();
if (reader.TokenType != JsonToken.StartObject)
{
throw new Exception("Invalid Json. Expected StartObject");
}
reader.Read();
if ((reader.TokenType != JsonToken.PropertyName) || (!reader.Value.ToString().Equals("catalog")))
{
throw new Exception("Invalid Json. Expected catalog array");
}
reader.Read();
if (reader.TokenType != JsonToken.StartArray)
{
throw new Exception("Invalid Json. Expected StartArray");
}
while (reader.Read())
{
if (reader.TokenType == JsonToken.EndArray)
break;
jtw.WriteToken(reader);
JObject asset = JObject.Parse(sw.ToString());
string id = asset["id"].ToString();
asset.Remove("id");
string[] idInfo = id.Split(new char[] { '/' });
string newid;
string UpdateResponse = td.Update(asset.ToString(), idInfo[idInfo.Length - 2], out newid);
if ((UpdateResponse != null) && (!string.IsNullOrEmpty(newid)))
{
totalAssetsImportSucceeded++;
if (totalAssetsImportSucceeded % 50 == 0)
{
System.Console.WriteLine(totalAssetsImportSucceeded + "Assets Imported Succesfully");
}
}
else
{
totalAssetsImportFailed++;
}
//reset local variables for next iteration
sw = new StringWriter(new StringBuilder());
jtw = new JsonTextWriter(sw);
}
Console.WriteLine("Total Imported Success: " + totalAssetsImportSucceeded);
Console.WriteLine("Total Imported Failed: " + totalAssetsImportFailed);
}
