private static void CheckAdlsFileExistsDemo()
{
var endDate = DateTime.UtcNow.Date.AddDays(-2);
var startDate = endDate.AddDays(-10);
string pathFormat = @"shares/IDEAs.Prod.Data/Publish/Usage/User/Commercial/Actions/EXO/Streams/v1/%Y/%m/WorldwidePIIDetectSDFV2Stat_%Y_%m_%d.ss";
var streamPaths = StreamSetUtils.GenerateStreamsetPaths(pathFormat, startDate, endDate, null, null, Grain.Daily);
Console.WriteLine("CheckAdlsFileExistsDemo: ");
Console.WriteLine(@"For datalake: 'skypedata-adhoc-c11.azuredatalakestore.net'...");
Console.WriteLine(@"For datalake: 'ideas-sensitive-build-c14.azuredatalakestore.net'...");
Console.WriteLine(dataLakeClient.CheckExists("ideas-sensitive-build-c14.azuredatalakestore.net",
@"/shares/IDEAs.Prod.Data/Private/Features/TPID/Commercial/FeatureStore/pbi/Views/v1/pbi.view"));
Console.WriteLine(@"For datalake: 'ideas-prod-build-c14.azuredatalakestore.net'...");
Console.WriteLine(dataLakeClient.CheckExists("ideas-prod-build-c14.azuredatalakestore.net",
@"/shares/IDEAs.Prod.Data/Publish/Usage/User/Commercial/CustomerFacing/Ppe/Cubes/TeamsActivityUserRange/Streams/v1/2021/01/TeamsActivityUser_180D_2021_01_11.ss"));
Console.WriteLine(dataLakeClient.CheckExists("ideas-prod-build-c14.azuredatalakestore.net",
@"/shares/CFR.ppe/Internal/Pfizer/MailboxState_2021_01_11.tsv"));
//foreach (var streamPath in streamPaths)
//{
// Console.WriteLine(dataLakeClient.CheckExists("ideas-prod-build-c14.azuredatalakestore.net", streamPath));
//}
//Console.WriteLine(@"For datalake: 'ideas-prod-c14.azuredatalakestore.net'...");
//foreach (var streamPath in streamPaths)
//{
// Console.WriteLine(dataLakeClient.CheckExists("ideas-prod-c14.azuredatalakestore.net", streamPath));
//}
// We don't have access to store "ideas-ppe-c14.azuredatalakestore.net". It will throw exception: forbidden
//Console.WriteLine(dataLakeClient.CheckExists("ideas-ppe-c14.azuredatalakestore.net", "/local/build/user/dekashet/TeamsMeetingProdAfterTimeZoneFixApril18th/directViewCodeWithAdjustEndDate.csv"));
}
private static void CheckPermission()
{
Console.WriteLine("CheckPermission:");
DateTime startDate = DateTime.Now.Date.AddDays(-10);
DateTime endDate = DateTime.Now.Date.AddDays(-3);
var dataLakeStores = new List <string>
{
//@"ideas-ppe-c14.azuredatalakestore.net",
@"ideas-prod-c14.azuredatalakestore.net",
@"ideas-prod-data-c14.azuredatalakestore.net",
@"ideas-prod-build-c14.azuredatalakestore.net",
};
var pathPrefixs = new List <string>()
{
@"Publish/Usage/Tenant/Commercial/ProductivityScore/Prod/ContentCollab/TenantScores_028D/Streams/v1/%Y/%m/TenantScores_028D_%Y_%m_%d.ss",
@"shares/IDEAs.Prod.Data/Publish/Usage/Tenant/Commercial/ProductivityScore/Prod/ContentCollab/TenantScores_028D/Streams/v1/%Y/%m/TenantScores_028D_%Y_%m_%d.ss"
};
var checkingPaths = new List <string>();
foreach (var pathPrefix in pathPrefixs)
{
string path;
if (pathPrefix.ToString().EndsWith(".ss") || pathPrefix.ToString().EndsWith(".view"))
{
path = pathPrefix;
}
else
{
path = pathPrefix + "Test.ss";
}
checkingPaths.AddRange(StreamSetUtils.GenerateStreamsetPaths(path, startDate, endDate, null, null, Grain.Daily));
}
foreach (var dataLakeStore in dataLakeStores)
{
Console.WriteLine($"DataLakeStore: {dataLakeStore}");
foreach (var checkingPath in checkingPaths)
{
if (!dataLakeClient.CheckPermission(dataLakeStore, checkingPath))
{
Console.WriteLine($"Have no permission for pathPrefix: '{checkingPath}'");
}
}
}
}
/// <summary>
/// Gets the previous stream path.
/// </summary>
/// <returns>System.String.</returns>
public string GetPreviousStreamPath()
{
var date = GetPreviousDate();
return(StreamSetUtils.GenerateStreamsetPaths(StreamPath, date, date, null, null, Frequency).First());
}
public string GetCurrentTargetStreamPath()
{
return(StreamSetUtils.GenerateStreamsetPaths(this.TargetStreamPath, Date, Date, null, null, Frequency).First());
}
