private static void UpdateCosmosVCToBuild()
{
var kenshoDatasetIds = GetKenshoDatasetIds();
var jsonFilePaths = ReadFile.GetAllFilePath(FolderPath);
foreach (var jsonFilePath in jsonFilePaths)
{
if (jsonFilePath.ToLower().Contains(@"\dataset"))
{
string fileContent = ReadFile.ThirdMethod(jsonFilePath);
JsonSerializerSettings serializer = new JsonSerializerSettings
{
DateParseHandling = DateParseHandling.None
};
JObject gitDatasetJObject = JsonConvert.DeserializeObject <JObject>(fileContent, serializer);
if (gitDatasetJObject["dataFabric"]?.ToString().ToLower().Equals("cosmosstream") == true)
{
if (kenshoDatasetIds.Contains(gitDatasetJObject["id"].ToString()))
{
Console.WriteLine($"Not update kensho dataset: {gitDatasetJObject["id"]?.ToString()}");
continue;
}
bool update = false;
if (gitDatasetJObject["connectionInfo"]["cosmosVC"]?.ToString().ToLower().Trim('/').Equals("https://cosmos14.osdinfra.net/cosmos/ideas.prod.build") == true)
{
update = true;
}
else if (gitDatasetJObject["connectionInfo"]["cosmosVC"]?.ToString().ToLower().Trim('/').Equals("https://cosmos14.osdinfra.net/cosmos/ideas.prod") == true ||
gitDatasetJObject["connectionInfo"]["cosmosVC"]?.ToString().ToLower().Trim('/').Equals("https://cosmos14.osdinfra.net/cosmos/ideas.private.data") == true)
{
if (gitDatasetJObject["connectionInfo"]["streamPath"].ToString().ToLower().Trim('/').StartsWith("share"))
{
update = true;
}
else
{
var streamPath = gitDatasetJObject["connectionInfo"]["streamPath"].ToString();
Console.WriteLine($"Not update dataset: {gitDatasetJObject["id"]?.ToString()}, streamPath: {streamPath}");
}
}
else
{
var cosmosVC = gitDatasetJObject["connectionInfo"]["cosmosVC"].ToString();
Console.WriteLine($"Not update dataset: {gitDatasetJObject["id"]?.ToString()}, cosmosVC: {cosmosVC}");
}
if (update)
{
gitDatasetJObject["connectionInfo"]["cosmosVC"] = "https://cosmos14.osdinfra.net/cosmos/IDEAs.Prod.Build/";
if (gitDatasetJObject["connectionInfo"]["dataLakeStore"]?.ToString().Length > 0)
{
gitDatasetJObject["connectionInfo"]["dataLakeStore"] = "ideas-prod-build-c14.azuredatalakestore.net";
}
WriteFile.FirstMethod(jsonFilePath, gitDatasetJObject.ToString());
}
}
}
}
}
private static void UpsertDatasetsIntoDB()
{
Initialize("datacop-ppe");
Console.WriteLine("Upsert Datasets Into DB:");
AzureCosmosDBClient azureCosmosDBClient = new AzureCosmosDBClient("DataCop", "Dataset");
var datasetIds = new HashSet <string>()
{
@"618319c3-e263-4f02-bb24-041476954dc9",
};
var datasetJsonFilePaths = ReadFile.GetAllFilePath(FolderPath);
foreach (var datasetJsonFilePath in datasetJsonFilePaths)
{
if (datasetJsonFilePath.ToLower().Contains(@"\dataset"))
{
string fileContent = ReadFile.ThirdMethod(datasetJsonFilePath);
JsonSerializerSettings serializer = new JsonSerializerSettings
{
DateParseHandling = DateParseHandling.None
};
JObject gitDatasetJObject = JsonConvert.DeserializeObject <JObject>(fileContent, serializer);
string id = gitDatasetJObject["id"].ToString();
if (datasetIds.Contains(id))
{
Console.WriteLine(id);
azureCosmosDBClient.UpsertDocumentAsync(gitDatasetJObject).Wait();
}
}
}
}
private static void UpdateAllSqlKeyVaultName()
{
var datasetJsonFilePaths = ReadFile.GetAllFilePath(FolderPath);
foreach (var datasetJsonFilePath in datasetJsonFilePaths)
{
if (datasetJsonFilePath.ToLower().Contains("dataset"))
{
string fileContent = ReadFile.ThirdMethod(datasetJsonFilePath);
JsonSerializerSettings serializer = new JsonSerializerSettings
{
DateParseHandling = DateParseHandling.None
};
JObject gitDatasetJObject = JsonConvert.DeserializeObject <JObject>(fileContent, serializer);
if (gitDatasetJObject["dataFabric"]?.ToString().ToLower().Equals("sql") == true)
{
if (gitDatasetJObject["connectionInfo"]["auth"]["keyVaultName"]?.ToString().ToLower().Equals("datacopprod") == true)
{
gitDatasetJObject["connectionInfo"]["auth"]["keyVaultName"] = "datacop-prod";
WriteFile.FirstMethod(datasetJsonFilePath, gitDatasetJObject.ToString());
Console.WriteLine(datasetJsonFilePath);
}
else if (gitDatasetJObject["connectionInfo"]["auth"]["keyVaultName"]?.ToString().ToLower().Equals("datacop-prod") == true)
{
}
else
{
Console.WriteLine(gitDatasetJObject["id"]);
Console.WriteLine(gitDatasetJObject["connectionInfo"]["auth"]["keyVaultName"]);
}
}
}
}
}
private static void UpdateCosmosViewScripts()
{
var datasetTestJsonFilePaths = ReadFile.GetAllFilePath(FolderPath);
int count = 0;
foreach (var datasetTestJsonFilePath in datasetTestJsonFilePaths)
{
if (datasetTestJsonFilePath.ToLower().Contains(@"\monitor"))
{
string fileContent = ReadFile.ThirdMethod(datasetTestJsonFilePath);
JsonSerializerSettings serializer = new JsonSerializerSettings
{
DateParseHandling = DateParseHandling.None
};
JObject gitDatasetTestJObject = JsonConvert.DeserializeObject <JObject>(fileContent, serializer);
if (gitDatasetTestJObject["dataFabric"]?.ToString().ToLower().Equals("cosmosview") == true)
{
count++;
var cosmosScriptContent = gitDatasetTestJObject["testContent"]["cosmosScriptContent"].ToString();
var index = cosmosScriptContent.IndexOf("OUTPUT ViewSamples");
cosmosScriptContent = cosmosScriptContent.Substring(0, index);
cosmosScriptContent += " Count = SELECT COUNT() AS NumSessions FROM ViewSamples; OUTPUT Count TO SSTREAM \"/my/output.ss\"";
gitDatasetTestJObject["testContent"]["cosmosScriptContent"] = cosmosScriptContent;
WriteFile.FirstMethod(datasetTestJsonFilePath, gitDatasetTestJObject.ToString());
}
}
}
Console.WriteLine($"count: {count}");
}
private static void UpsertDatasetTestsIntoDB()
{
Console.WriteLine("Upsert DatasetTests Into DB:");
Initialize("datacop-ppe");
AzureCosmosDBClient azureCosmosDBClient = new AzureCosmosDBClient("DataCop", "DatasetTest");
var datasetTestIds = new HashSet <string>()
{
@"d2e33866-5d1c-4670-90c2-dd8d0c29fcdc",
};
var datasetTestJsonFilePaths = ReadFile.GetAllFilePath(FolderPath);
foreach (var datasetTestJsonFilePath in datasetTestJsonFilePaths)
{
if (datasetTestJsonFilePath.ToLower().Contains(@"\monitor"))
{
string fileContent = ReadFile.ThirdMethod(datasetTestJsonFilePath);
JsonSerializerSettings serializer = new JsonSerializerSettings
{
DateParseHandling = DateParseHandling.None
};
JObject gitDatasetTestJObject = JsonConvert.DeserializeObject <JObject>(fileContent, serializer);
string id = gitDatasetTestJObject["id"].ToString();
if (datasetTestIds.Contains(id))
{
Console.WriteLine(id);
azureCosmosDBClient.UpsertDocumentAsync(gitDatasetTestJObject).Wait();
}
}
}
}
private static void UpdateAllDatasetJsonForMergingADLSCosmos()
{
var datasetJsonFilePaths = ReadFile.GetAllFilePath(FolderPath);
foreach (var datasetJsonFilePath in datasetJsonFilePaths)
{
Console.WriteLine(datasetJsonFilePath);
UpdateDatasetConnectionInfoForMergingADLSCosmos(datasetJsonFilePath);
}
}
private static void ShowEnabledDatasets()
{
string rootFolder = Path.Combine(FolderPath, @"PROD\CFR");
var jsonFilePaths = ReadFile.GetAllFilePath(rootFolder);
HashSet <string> datasetIds = new HashSet <string>();
Dictionary <string, JObject> datasetTests = new Dictionary <string, JObject>();
foreach (var jsonFilePath in jsonFilePaths)
{
if (
//!datasetJsonFilePath.ToLower().Contains(@"\cfrinbuild2") && // They are all in the folder "CFRInBuild2"
(jsonFilePath.ToLower().Contains(@"\dataset") || jsonFilePath.ToLower().Contains(@"\monitor")))
{
string fileContent = ReadFile.ThirdMethod(jsonFilePath);
JsonSerializerSettings serializer = new JsonSerializerSettings
{
DateParseHandling = DateParseHandling.None
};
JObject jObject = JsonConvert.DeserializeObject <JObject>(fileContent, serializer);
if (jsonFilePath.ToLower().Contains(@"\dataset"))
{
if (bool.Parse(jObject["isEnabled"].ToString()))
{
datasetIds.Add(jObject["id"].ToString());
}
}
else
{
if (jObject["status"].ToString().Equals("Enabled"))
{
datasetTests.Add(jsonFilePath, jObject);
}
}
}
}
foreach (var datasetTest in datasetTests)
{
var datasetTestPath = datasetTest.Key;
var datasetTestJson = datasetTest.Value;
if (datasetIds.Contains(datasetTestJson["datasetId"].ToString()))
{
Console.WriteLine(datasetTestJson["id"]);
datasetTestJson["status"] = "Disabled";
WriteFile.FirstMethod(datasetTestPath, datasetTestJson.ToString());
}
}
}
private static HashSet <string> GetKenshoDatasetIds()
{
var kenshoDatasetIds = new HashSet <string>();
var jsonFilePaths = ReadFile.GetAllFilePath(FolderPath);
foreach (var jsonFilePath in jsonFilePaths)
{
if (jsonFilePath.ToLower().Contains(@"\kenshodata"))
{
string fileContent = ReadFile.ThirdMethod(jsonFilePath);
JsonSerializerSettings serializer = new JsonSerializerSettings
{
DateParseHandling = DateParseHandling.None
};
JObject gitKenshoJObject = JsonConvert.DeserializeObject <JObject>(fileContent, serializer);
kenshoDatasetIds.Add(gitKenshoJObject["datasetId"].ToString());
}
}
return(kenshoDatasetIds);
}
private static void UpdateStoreForSpark()
{
var datasetTestJsonFilePaths = ReadFile.GetAllFilePath(FolderPath);
HashSet <string> datasetIds = new HashSet <string>();
foreach (var datasetTestJsonFilePath in datasetTestJsonFilePaths)
{
if (datasetTestJsonFilePath.ToLower().Contains(@"\monitor"))
{
string fileContent = ReadFile.ThirdMethod(datasetTestJsonFilePath);
JsonSerializerSettings serializer = new JsonSerializerSettings
{
DateParseHandling = DateParseHandling.None
};
JObject gitDatasetTestJObject = JsonConvert.DeserializeObject <JObject>(fileContent, serializer);
if (gitDatasetTestJObject["testContentType"]?.ToString().ToLower().Equals("spark") == true)
{
var datasetId = gitDatasetTestJObject["datasetId"]?.ToString();
datasetIds.Add(datasetId);
//var cosmosScriptContent = gitDatasetTestJObject["testContent"]["cosmosScriptContent"].ToString();
//var index = cosmosScriptContent.IndexOf("OUTPUT ViewSamples");
//cosmosScriptContent = cosmosScriptContent.Substring(0, index);
//cosmosScriptContent += " Count = SELECT COUNT() AS NumSessions FROM ViewSamples; OUTPUT Count TO SSTREAM \"/my/output.ss\"";
//gitDatasetTestJObject["testContent"]["cosmosScriptContent"] = cosmosScriptContent;
//WriteFile.FirstMethod(datasetTestJsonFilePath, gitDatasetTestJObject.ToString());
}
}
}
foreach (var datasetId in datasetIds)
{
Console.WriteLine(datasetId);
}
Console.WriteLine($"count: {datasetIds.Count}");
}
private static void DisableDatasets(HashSet <string> datasetIds)
{
var datasetJsonFilePaths = ReadFile.GetAllFilePath(FolderPath);
foreach (var datasetJsonFilePath in datasetJsonFilePaths)
{
if (datasetJsonFilePath.ToLower().Contains("dataset"))
{
string fileContent = ReadFile.ThirdMethod(datasetJsonFilePath);
JsonSerializerSettings serializer = new JsonSerializerSettings
{
DateParseHandling = DateParseHandling.None
};
JObject gitDatasetJObject = JsonConvert.DeserializeObject <JObject>(fileContent, serializer);
string id = gitDatasetJObject["id"].ToString();
if (datasetIds.Contains(id))
{
Console.WriteLine(id);
gitDatasetJObject["isEnabled"] = false;
WriteFile.FirstMethod(datasetJsonFilePath, gitDatasetJObject.ToString());
}
}
}
}
private static void UpdateSomeCFRToAdls()
{
var datasetIds = new HashSet <string>()
{
"CFR_GAL_ReportUXWeekly",
"CFR_WWPPE_Cubes_ProPlusUsage",
"CFR_WW_ReportUXWeekly",
"CFR_WW_ReportUXAndAPIWeekly",
"CFR_GAL_ReportUXAndAPIWeekly",
"CFR_CookedUXLog",
"CFR_CookedKustoAPILog",
"CFR_WWPPE_Cubes_FormsActivity",
"CFR_WWPPE_Cubes_FormsProActivity",
"CFR_ReportGraphAPIAndExportWeekly",
"CFR_ReportGraphAPIAndExportMonthly",
"CFR_GAL_ReportUXMonthly",
"CFR_WW_ReportUXMonthly",
"CFR_WW_ReportUXAndAPIMonthly",
"CFR_GAL_ReportUXAndAPIMonthly",
"CFR_WWPROD_Cubes_ProPlusUsage",
"CFR_WWPROD_Cubes_FormsActivity",
"CFR_WWPROD_Cubes_FormsProActivity",
};
var jsonFilePaths = ReadFile.GetAllFilePath(FolderPath);
foreach (var jsonFilePath in jsonFilePaths)
{
if (jsonFilePath.ToLower().Contains(@"\datasets"))
{
string fileContent = ReadFile.ThirdMethod(jsonFilePath);
JsonSerializerSettings serializer = new JsonSerializerSettings
{
DateParseHandling = DateParseHandling.None
};
JObject gitDatasetJObject = JsonConvert.DeserializeObject <JObject>(fileContent, serializer);
string id = gitDatasetJObject["id"].ToString();
if (datasetIds.Contains(id))
{
var connectionInfo = JObject.Parse(gitDatasetJObject["connectionInfo"].ToString());
connectionInfo.Remove("cosmosVC");
string streamPath = connectionInfo["streamPath"].ToString().Trim(new char[] { '/' });
var newConnectionInfo = new JObject();
if (streamPath.ToLower().StartsWith("shares/cfr.ppe"))
{
newConnectionInfo["dataLakeStore"] = "cfr-ppe-c14.azuredatalakestore.net";
newConnectionInfo["streamPath"] = streamPath.Substring(14);
}
else if (streamPath.ToLower().StartsWith("shares/cfr.prod"))
{
newConnectionInfo["dataLakeStore"] = "cfr-prod-c14.azuredatalakestore.net";
newConnectionInfo["streamPath"] = streamPath.Substring(15);
}
else
{
continue;
}
gitDatasetJObject["connectionInfo"] = newConnectionInfo;
gitDatasetJObject["dataFabric"] = "ADLS";
Console.WriteLine($"Update datasets: {id}");
WriteFile.FirstMethod(jsonFilePath, gitDatasetJObject.ToString());
}
}
// Update datasetTests
if (jsonFilePath.ToLower().Contains(@"\monitors"))
{
string fileContent = ReadFile.ThirdMethod(jsonFilePath);
JsonSerializerSettings serializer = new JsonSerializerSettings
{
DateParseHandling = DateParseHandling.None
};
JObject gitDatasetTestJObject = JsonConvert.DeserializeObject <JObject>(fileContent, serializer);
string id = gitDatasetTestJObject["id"].ToString();
string datasetId = gitDatasetTestJObject["datasetId"].ToString();
if (datasetIds.Contains(datasetId))
{
gitDatasetTestJObject["testContentType"] = "AdlsAvailability";
gitDatasetTestJObject["dataFabric"] = "ADLS";
Console.WriteLine($"Update datasetTests: {id}");
WriteFile.FirstMethod(jsonFilePath, gitDatasetTestJObject.ToString());
}
}
}
}