async T.Task IEtlJobs.ExecuteAsync(EtlPackages etlPackage, int dataSourceFetchId)
{
var fetch = await GlobalContext.DataSourceFetches.Include(z => z.DataSource).Include(z => z.DataSourceFetchDataSourceFetchItems).SingleAsync(z => z.DataSource.TenantId == this.JobInfo.TenantId && z.DataSourceFetchId == dataSourceFetchId);
Directory.CreateDirectory(FetchFolder);
var fetchItems = fetch.DataSourceFetchDataSourceFetchItems.Where(z => z.DataSourceFetchItemTypeStringValue == DataSourceFetchItem.DataSourceFetchItemTypes.Original.ToString());
Stuff.TaskWaitAllForEach(
fetchItems,
async fi => {
var fn = $"{FetchFolder}{fi.DataSourceFetchItemId}\\{fi.Name}";
Directory.CreateDirectory(Path.GetDirectoryName(fn));
var uri = new Uri(fi.Url);
if (uri.Scheme == BlobStorageServices.AzureBlobServiceProtocol)
{
uri = BlobStorageServices.GetReadonlySharedAccessSignatureUrl(BlobConfigOptions.Value, uri);
}
using (var client = HttpClientFactory.Create())
{
using (var st = await client.GetStreamAsync(uri))
{
using (var fst = File.Create(fn))
{
await st.CopyToAsync(fst);
}
}
if (MimeType.Application.Zip.DoesExtensionMatch(fn))
{
var unzipFolder = Path.Combine(Path.GetDirectoryName(fn), Path.GetFileNameWithoutExtension(fn));
ZipFile.ExtractToDirectory(fn, unzipFolder);
}
}
});
switch (etlPackage)
{
case EtlPackages.CmsGov:
await LoadCmsGovAsync(fetch);
return;
case EtlPackages.InternationalClassificationDiseases:
await LoadInternationalClassificationDiseasesAsync(fetch);
break;
case EtlPackages.NationalDrugCodes:
await LoadNationalDrugCodeAsync(fetch);
break;
case EtlPackages.ZipCodes:
await LoadZipCodesAsync(fetch);
break;
default:
throw new UnexpectedSwitchValueException(etlPackage);
}
}
private async Task GetEm(string folder)
{
var urls = new[] {
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_Hist14_20140401_20150331_20170512.ctl.pgp",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_20170401_20170430_20170512.ctl.pgp",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_Hist14_20140401_20150331_20170512.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_Hist15_20150401_20160331_20170512.ctl.pgp",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_20170501_20170531_20170607.ctl",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_20170501_20170531_20170608.ctl",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_20170501_20170531_20170607.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_Hist16_20160401_20170331_20170512.ctl",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_Hist14_20140401_20150331_20170512.ctl.pgp",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_20170401_20170430_20170512.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_Hist15_20150401_20160331_20170512.ctl",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_20170401_20170430_20170512.txt.pgp",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_Hist14_20140401_20150331_20170512.txt.pgp",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_Hist14_20140401_20150331_20170512.txt.pgp",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_20170401_20170430_20170512.ctl.pgp",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_Hist15_20150401_20160331_20170512.txt.pgp",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_20170601_20170630_20170710.ctl",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_Hist16_20160401_20170331_20170512.txt.pgp",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_Hist15_20150401_20160331_20170512.ctl.pgp",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_Hist14_20140401_20150331_20170512.ctl.pgp",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_20170501_20170531_20170607.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_20170401_20170430_20170512.ctl",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_20170401_20170430_20170512.ctl.pgp",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_Hist14_20140401_20150331_20170512.txt.pgp",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_20170601_20170630_20170709.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_Hist15_20150401_20160331_20170512.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_20170401_20170430_20170512.ctl",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/BPA/BPA_RX_20170601_20170630_.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/BPA/BPA_RX_20170701_20170731_.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/sfgtestfile.txt.pgp",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/BPA/BPA_ELIG_20170501_20170531_.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/BPA/BPA_ELIG_20170601_20170630_.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_Hist14_20140401_20150331_20170512.ctl",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/BPA/BPA_RX_20170501_20170531_.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_Hist16_20160401_20170331_20170512.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_20170601_20170630_20170709.ctl",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_Hist16_20160401_20170331_20170512.ctl.pgp",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/BPA/BPA_ELIG_20140101_20170430_.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_Hist14_20140401_20150331_20170512.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_Hist15_20150401_20160331_20170512.ctl",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_Hist15_20150401_20160331_20170512.ctl.pgp",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_Hist14_20140401_20150331_20170512.ctl",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_20170401_20170430_20170512.ctl",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_20170501_20170531_20170607.ctl",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_20170401_20170430_20170512.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_Hist16_20160401_20170331_20170512.ctl",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_Hist16_20160401_20170331_20170512.ctl.pgp",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_20170501_20170531_20170608.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_Hist14_20140401_20150331_20170512.ctl",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_Hist14_20140401_20150331_20170512.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_20170601_20170630_20170710.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_Hist16_20160401_20170331_20170512.ctl.pgp",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_Hist16_20160401_20170331_20170512.ctl",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_Hist15_20150401_20160331_20170512.ctl",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_20170401_20170430_20170512.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_Hist15_20150401_20160331_20170512.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/BPA/BPA_ELIG_20170701_20170731_.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/BPA/BPA_MED_20170701_20170731_.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_20170401_20170430_20170512.txt.pgp",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_Hist14_20140401_20150331_20170512.ctl",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_Hist15_20150401_20160331_20170512.txt.pgp",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_20170401_20170430_20170512.ctl",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_20170601_20170630_20170709.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_Hist14_20140401_20150331_20170512.ctl",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_20170401_20170430_20170512.txt.pgp",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_Hist15_20150401_20160331_20170512.ctl",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_20170601_20170630_20170709.ctl",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_20170401_20170430_20170512.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_Hist14_20140401_20150331_20170512.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_Hist14_20140401_20150331_20170512.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_Hist16_20160401_20170331_20170512.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_Hist16_20160401_20170331_20170512.txt.pgp",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/BPA/BPA_MED_20170501_20170531_.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/sfgtestfile.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_Hist14_20140401_20150331_20170512.ctl",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_Hist15_20150401_20160331_20170512.txt.pgp",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_Hist16_20160401_20170331_20170512.txt.pgp",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MBRSHP_Hist15_20150401_20160331_20170512.ctl",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/BPA/BPA_MED_20170601_20170630_.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_Hist15_20150401_20160331_20170512.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_MDCLM_Hist16_20160401_20170331_20170512.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/[email protected]/PrideIndust_TraffK_RXCLM_Hist16_20160401_20170331_20170512.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/BPA/BPA_RX_20140101_20170430_.txt",
"abs://traffkhipprod.blob.core.windows.net/secure/Pride/85/20170806/085545/BPA/BPA_MED_20140101_20170430_.txt",
};
var blobConfig = ServiceProvider.GetRequiredService <BSSC>().BlobConfigOptions.Value;
var httpClientFactory = ServiceProvider.GetRequiredService <Utility.IHttpClientFactory>();
int x = 0;
foreach (var u in urls)
{
var uri = new Uri(u);
if (uri.Scheme == BlobStorageServices.AzureBlobServiceProtocol)
{
uri = BlobStorageServices.GetReadonlySharedAccessSignatureUrl(blobConfig, uri);
}
using (var client = httpClientFactory.Create())
{
var parts = uri.LocalPath.Replace("/", "\\").Split(new[] { '\\' }, StringSplitOptions.RemoveEmptyEntries);
string fn = Path.Combine(folder, parts.Skip(5).Format("\\"));
Trace.WriteLine($"download {++x}/{urls.Length} from {u} to {fn}");
using (var st = await client.GetStreamAsync(uri))
{
Directory.CreateDirectory(Path.GetDirectoryName(fn));
using (var fst = File.Create(fn))
{
await st.CopyToAsync(fst);
}
}
}
}
}