public static string GetName(string fileName, string stateCode, CensusFileType censusFileType)
{
string fileTypeName = censusFileType.GetName();
string fileTypeShortName = censusFileType.GetShortName();
fileName = fileName.Replace($"20101.{censusFileType.GetShortName().ToLowerInvariant()}", "")
.Replace($"2010.{censusFileType.GetShortName().ToLowerInvariant()}", "")
.Replace($".{censusFileType.GetShortName().ToLowerInvariant()}", "");
switch (censusFileType)
{
case CensusFileType.DemographicProfileWithSF1Geos:
return("Part1");
case CensusFileType.Redistricting:
return("PL");
case CensusFileType.SF1CongressionalDistricts113:
case CensusFileType.SummaryOne:
return(fileName.Replace(stateCode, $"{fileTypeShortName}_"));
case CensusFileType.SummaryTwo:
return(fileName.Replace(fileName.Substring(2, 3), "").Replace(stateCode, $"{fileTypeShortName}_Segment_"));
case CensusFileType.UrbanAreaUpdate:
case CensusFileType.AdvanceGroupQuarters:
case CensusFileType.AIANSummaryFile:
case CensusFileType.IslandAreasDetailedCrossTabulations:
case CensusFileType.IslandAreas_DPSF:
case CensusFileType.IslandAreas_IASF:
case CensusFileType.IslandAreas_PUMS:
case CensusFileType.Stateside_PUMS:
return(null);
default:
throw new ArgumentOutOfRangeException(nameof(censusFileType), censusFileType, null);
}
}
public static void ProcessCensusData(CensusFileType censusFileType)
{
SSISApp = new SSISHelper();
string fileTypeName = censusFileType.GetName();
string fileTypeShortName = censusFileType.GetShortName();
var dataArchives = GetDataArchives(censusFileType);
foreach (KeyValuePair <string, IEnumerable <ZipArchiveEntry> > file in dataArchives)
{
Console.WriteLine($"{file.Key} | {file.Value.Count()} files");
foreach (ZipArchiveEntry entry in file.Value)
{
Console.WriteLine($"Processing {entry.FullName}");
Console.WriteLine($"\t{entry.Name} | {entry.CompressedLength} | {entry.Length} | {entry.LastWriteTime}");
bool isGeo = entry.Name.ToLowerInvariant()
.Contains("geo");
string name = isGeo
? GetGeoType(censusFileType)
: GetName(entry.Name, file.Key, censusFileType);
string outputFilePath = $"{Program.OutputPath}\\{entry.Name}";
string tableName = $"[{Program.DefaultDatabase}].[{fileTypeName}].[{name}]";
//SqlHelper.TruncateTable(tableName);
int rowsAffected = 0;
if (!File.Exists(outputFilePath))
{
using (StreamReader sr = new StreamReader(entry.Open()))
{
using (FileStream sw = File.Create(outputFilePath))
{
Console.Write($"Copying file to {outputFilePath}. . . Please wait!");
sr.BaseStream.CopyTo(sw);
}
}
}
if (isGeo)
{
Console.WriteLine($"\rFile copied successfully.\t\t\t\t\t");
Console.Write($"\rProcessing {entry.Name} as a Flat File into {tableName}\t\t\t\t\t");
string mapName = $"{Program.BaseNamespace}.{Program.Namespace}.Mapping.{fileTypeName}_{name}Map";
Type mapType = Type.GetType(mapName);
dynamic map = mapType == null
? null
: Activator.CreateInstance(mapType);
var result = SSISApp.ProcessFlatFile(outputFilePath, map);
Console.WriteLine($"SSIS Flat File Task Result: {result}");
}
else
{
rowsAffected += SqlHelper.BulkCSVInsert(tableName, outputFilePath);
}
Console.WriteLine($"\r{rowsAffected} records affected from processing {entry.Name} into {tableName}\t\t\t\t\t");
if (File.Exists(outputFilePath))
{
File.Delete(outputFilePath);
}
}
}
SSISApp.Save();
}
