public void TestFileDownload()
{
var oldConfig = Settings.ConfigFile;
try
{
// Wyoming has smallest file to download
var manager = new AcsDataManager(AcsState.Wyoming, BaseDir, "2012");
manager.SummaryLevel = "150";
//Settings.ConfigFile = new Config("configs\\AcsAlchemist.json.2012.config");
if (manager.CheckCensusAggregatedDataFile())
{
string filename = manager.GetLocalBlockGroupZipFileName();
Assert.IsTrue(File.Exists(filename), "File wasn't downloaded!");
}
else
{
Assert.Fail("Some error was thrown during the download");
}
}
finally
{
Settings.ConfigFile = oldConfig;
}
}
/// <summary>
/// Performs our main 'lifecycle'
/// </summary>
/// <returns></returns>
public bool ExecuteJob()
{
_manager = null; _cancelled = false;
this.UpdateProgress(0);
if (IsCancelled()) { _log.Debug("Job Cancelled..."); return false; }
DateTime startTime = DateTime.Now;
try
{
if (!string.IsNullOrEmpty(DisplaySummaryLevels))
{
Utilities.DisplayEnum<BoundaryLevels>("Summary Levels:", "{1} - {0}",
new HashSet<string>(new string[] { "None" })
);
return true;
}
if (!string.IsNullOrEmpty(DisplayStateCodes))
{
Utilities.DisplayEnumKeysOnly("State Codes:", typeof(AcsState),
//Do not display internal control value
new HashSet<string>(new string[] { "None" })
);
return true;
}
if (!string.IsNullOrEmpty(ListYears))
{
var years = Settings.LoadYearConfigs();
//_log.DebugFormat("I found {0} year config files ", years.Count);
_log.InfoFormat("I found {0} years available:", years.Count);
foreach (var key in years.Keys)
{
//_log.DebugFormat("{0} - {1}", key, years[key].GetFilename());
_log.InfoFormat(" * {0} ", key);
}
_log.InfoFormat(Environment.NewLine + "Done!");
return true;
}
if (this.State == AcsState.None)
{
_log.Error("Invalid State selected, please select a state from the list and try again.");
return false;
}
if ((string.IsNullOrEmpty(this.JobName)) || (this.JobName == true.ToString()))
{
this.JobName = string.Format("{0}_{1}_{2}", this.Year, this.State, DateTime.Now.ToShortDateString().Replace('/', '_'));
_log.DebugFormat("Jobname was empty, using {0}", this.JobName);
}
WorkingFolder = FileUtilities.CleanPath(WorkingFolder);
var manager = new AcsDataManager(this.State, WorkingFolder, this.Year);
this._manager = manager;
manager.WorkOffline = this.WorkOffline;
//TODO: check for bad combinations of inputs
manager.SummaryLevel = this.SummaryLevel;
manager.ExportFilterFilename = this.ExportFilterShapefile;
manager.DesiredVariablesFilename = IncludedVariableFile;
manager.ReusePreviousJobTable = (!string.IsNullOrEmpty(this.ReusePreviousJobTable));
manager.OutputProjectionFilename = this.OutputProjection;
manager.PreserveJam = (!string.IsNullOrEmpty(this.PreserveJam));
manager.AddStrippedGEOIDcolumn = (!string.IsNullOrEmpty(this.AddStrippedGEOIDcolumn));
manager.AddGeometryAttributesToOutput = (!string.IsNullOrEmpty(this.AddGeometryAttributesToOutput));
manager.OutputFolder = FileUtilities.CleanPath(OutputFolder);
manager.IncludeEmptyGridCells = (!string.IsNullOrEmpty(this.IncludeEmptyGridCells));
if (FileUtilities.SafePathEnsure(OutputFolder) != OutputFolder)
{
_log.ErrorFormat("Unable to set or create output folder, ( {0} ) exiting", OutputFolder);
_log.FatalFormat("Unable to set or create output folder, ( {0} ) exiting", OutputFolder);
return false;
}
this.UpdateProgress(25);
if (IsCancelled()) { _log.Debug("Job Cancelled..."); return false; }
if (string.IsNullOrEmpty(this.OutputProjection))
{
_log.Warn(Constants.Warning_MissingProjection);
}
_log.Debug("\r\n/*************************************/");
_log.Info(" Loading Prerequisites...");
_log.Debug("/*************************************/");
bool hasPrerequesites = true;
hasPrerequesites &= !IsCancelled() && manager.CheckColumnMappingsFile();
hasPrerequesites &= !IsCancelled() && manager.CheckCensusAggregatedDataFile();
hasPrerequesites &= !IsCancelled() && manager.CheckDatabase();
hasPrerequesites &= !IsCancelled() && manager.CheckShapefiles();
if (!hasPrerequesites)
{
_log.Info("Loading Prerequisites... Failed!");
_log.Error("Import cannot continue, one or more prerequisites failed!");
return false;
}
_log.Debug("Loading Prerequisites... Done!\r\n");
this.UpdateProgress(35);
if (IsCancelled()) { _log.Debug("Job Cancelled..."); return false; }
if (!string.IsNullOrEmpty(IncludedVariableFile) && !string.IsNullOrEmpty(this.JobName))
{
_log.Info("\r\n/*************************************/");
_log.Info(" Importing requested variables... ");
_log.Info("/*************************************/");
if (!manager.CheckBuildVariableTable(this.JobName))
{
_log.Error("Importing requested variables... Failed! A problem was detected, exiting.");
return false;
}
else
{
_log.Debug("Importing requested variables... Done!");
}
}
this.UpdateProgress(50);
if (IsCancelled()) { _log.Debug("Job Cancelled..."); return false; }
if (!string.IsNullOrEmpty(ExportToShapefile))
{
_log.Debug("\r\n/*************************************/");
_log.Info(" Exporting to shapefile... ");
_log.Debug("/*************************************/");
if (!manager.ExportShapefile(this.JobName))
{
_log.Error("There was an error while exporting the shapefile");
_log.Debug("\r\nExporting to shapefile... Failed!");
}
else
{
_log.Debug("Exporting to shapefile... Done!");
}
}
else
{
_log.Debug("\r\n/**** No shapefile export requested ****/\r\n");
}
this.UpdateProgress(75);
if (IsCancelled()) { _log.Debug("Job Cancelled..."); return false; }
if (!string.IsNullOrEmpty(ExportToGrid))
{
_log.Debug("\r\n/*************************************/");
_log.Info(" Exporting to gridded shapefile...");
_log.Debug("/*************************************/");
manager.GridEnvelopeFilename = GridEnvelope;
manager.SetGridParam(ExportToGrid);
_log.DebugFormat("Exporting all requested variables to fishnet shapefile with grid cell size {0} ", ExportToGrid);
manager.ExportGriddedShapefile(this.JobName);
_log.Debug("Exporting to gridded shapefile... Done!");
}
else
{
_log.Debug("\r\n/**** No gridded shapefile export requested ****/\r\n");
}
this.UpdateProgress(100);
_log.Info("Done!");
return true;
}
catch (Exception ex)
{
_log.Error("Error thrown during import job ", ex);
_log.Fatal("Please see the errors log file for details");
}
finally
{
TimeSpan elapsed = DateTime.Now - startTime;
if (IsCancelled())
{
_log.DebugFormat("Job was cancelled and stopped at {0} seconds", elapsed.TotalSeconds);
}
else
{
_log.DebugFormat("Job completed in {0} seconds", elapsed.TotalSeconds);
}
}
return false;
}
public void CheckTooManyColumnsFail()
{
string basePath = FileUtilities.PathEnsure(BaseDir, "TestData");
string TooManyVariablesFile = Path.Combine(basePath, "TooManyColumns.txt");
if (!File.Exists(TooManyVariablesFile))
{
int maxColumns = 255, max = (maxColumns / 2) + 1;
StringBuilder sb = new StringBuilder();
for (int i = 0; i < max; i++)
{
sb.Append("COLUMN").Append(i).Append(Environment.NewLine);
}
File.WriteAllText(TooManyVariablesFile, sb.ToString());
}
var manager = new AcsDataManager(AcsState.Wyoming);
manager.WorkingPath = basePath;
manager.DesiredVariablesFilename = TooManyVariablesFile;
Assert.IsFalse(manager.CheckBuildVariableTable("TestTooMany"));
}
private void AssertFailedImport(string filename, AcsDataManager man, DbConnection conn)
{
Assert.IsTrue(File.Exists(filename), "Could not find test file " + filename);
man.DesiredVariablesFilename = filename;
DataTable dt = man.GetRequestedVariables(conn);
Assert.IsTrue(dt == null, "Non-null DataTable returned for file " + filename);
}
protected string GetShapePath(AcsDataManager man, string filename)
{
return Path.Combine(man.ShapePath, Path.GetFileName(filename));
}
protected AcsDataManager GetManager(string year)
{
AcsDataManager m = new AcsDataManager(AcsState.Wyoming, Path.Combine(BaseDir, "Working"), year);
m.WorkingPath = FileUtilities.PathEnsure(BaseDir, "TestData");
string dbPath = FileUtilities.PathEnsure(m.WorkingPath, "database");
m.DBFilename = FileUtilities.PathCombine(dbPath, Settings.CurrentAcsDirectory + ".sqlite");
m.ShapePath = FileUtilities.PathEnsure(m.WorkingPath, "shapes");
m.CurrentDataPath = m.WorkingPath;
m.DbClient = DataClient.GetDefaultClient(m.DBFilename);
return m;
}