This class does most of the heavy lifting of the ACS Alchemist
Inheritance: IDisposable
        public void TestFileDownload()
        {
            var oldConfig = Settings.ConfigFile;
            try
            {
                // Wyoming has smallest file to download
                var manager = new AcsDataManager(AcsState.Wyoming, BaseDir, "2012");
                manager.SummaryLevel = "150";

                //Settings.ConfigFile = new Config("configs\\AcsAlchemist.json.2012.config");
                if (manager.CheckCensusAggregatedDataFile())
                {
                    string filename = manager.GetLocalBlockGroupZipFileName();

                    Assert.IsTrue(File.Exists(filename), "File wasn't downloaded!");
                }
                else
                {
                    Assert.Fail("Some error was thrown during the download");
                }
            }
            finally
            {
                Settings.ConfigFile = oldConfig;
            }
        }
Esempio n. 2
0
        /// <summary>
        /// Performs our main 'lifecycle'
        /// </summary>
        /// <returns></returns>
        public bool ExecuteJob()
        {
            _manager = null; _cancelled = false;

            this.UpdateProgress(0);
            if (IsCancelled()) { _log.Debug("Job Cancelled..."); return false; }

            DateTime startTime = DateTime.Now;
            try
            {
                if (!string.IsNullOrEmpty(DisplaySummaryLevels))
                {
                    Utilities.DisplayEnum<BoundaryLevels>("Summary Levels:", "{1} - {0}",
                        new HashSet<string>(new string[] { "None" })
                        );
                    return true;
                }
                if (!string.IsNullOrEmpty(DisplayStateCodes))
                {
                    Utilities.DisplayEnumKeysOnly("State Codes:", typeof(AcsState),
                        //Do not display internal control value
                        new HashSet<string>(new string[] { "None" })
                        );
                    return true;
                }
                if (!string.IsNullOrEmpty(ListYears))
                {
                    var years = Settings.LoadYearConfigs();
                    //_log.DebugFormat("I found {0} year config files ", years.Count);
                    _log.InfoFormat("I found {0} years available:", years.Count);
                    foreach (var key in years.Keys)
                    {
                        //_log.DebugFormat("{0} - {1}", key, years[key].GetFilename());

                        _log.InfoFormat(" * {0} ", key);
                    }
                    _log.InfoFormat(Environment.NewLine + "Done!");
                    return true;
                }

                if (this.State == AcsState.None)
                {
                    _log.Error("Invalid State selected, please select a state from the list and try again.");
                    return false;
                }

                if ((string.IsNullOrEmpty(this.JobName)) || (this.JobName == true.ToString()))
                {
                    this.JobName = string.Format("{0}_{1}_{2}", this.Year, this.State, DateTime.Now.ToShortDateString().Replace('/', '_'));
                    _log.DebugFormat("Jobname was empty, using {0}", this.JobName);
                }

                WorkingFolder = FileUtilities.CleanPath(WorkingFolder);
                var manager = new AcsDataManager(this.State, WorkingFolder, this.Year);
                this._manager = manager;
                manager.WorkOffline = this.WorkOffline;
                //TODO: check for bad combinations of inputs
                manager.SummaryLevel = this.SummaryLevel;
                manager.ExportFilterFilename = this.ExportFilterShapefile;
                manager.DesiredVariablesFilename = IncludedVariableFile;
                manager.ReusePreviousJobTable = (!string.IsNullOrEmpty(this.ReusePreviousJobTable));
                manager.OutputProjectionFilename = this.OutputProjection;
                manager.PreserveJam = (!string.IsNullOrEmpty(this.PreserveJam));
                manager.AddStrippedGEOIDcolumn = (!string.IsNullOrEmpty(this.AddStrippedGEOIDcolumn));
                manager.AddGeometryAttributesToOutput = (!string.IsNullOrEmpty(this.AddGeometryAttributesToOutput));
                manager.OutputFolder = FileUtilities.CleanPath(OutputFolder);
                manager.IncludeEmptyGridCells = (!string.IsNullOrEmpty(this.IncludeEmptyGridCells));

                if (FileUtilities.SafePathEnsure(OutputFolder) != OutputFolder)
                {
                    _log.ErrorFormat("Unable to set or create output folder, ( {0} ) exiting", OutputFolder);
                    _log.FatalFormat("Unable to set or create output folder, ( {0} ) exiting", OutputFolder);
                    return false;
                }

                this.UpdateProgress(25);
                if (IsCancelled()) { _log.Debug("Job Cancelled..."); return false; }

                if (string.IsNullOrEmpty(this.OutputProjection))
                {
                    _log.Warn(Constants.Warning_MissingProjection);
                }

                _log.Debug("\r\n/*************************************/");
                _log.Info("   Loading Prerequisites...");
                _log.Debug("/*************************************/");

                bool hasPrerequesites = true;
                hasPrerequesites &= !IsCancelled() && manager.CheckColumnMappingsFile();
                hasPrerequesites &= !IsCancelled() && manager.CheckCensusAggregatedDataFile();
                hasPrerequesites &= !IsCancelled() && manager.CheckDatabase();
                hasPrerequesites &= !IsCancelled() && manager.CheckShapefiles();

                if (!hasPrerequesites)
                {
                    _log.Info("Loading Prerequisites... Failed!");
                    _log.Error("Import cannot continue, one or more prerequisites failed!");
                    return false;
                }
                _log.Debug("Loading Prerequisites... Done!\r\n");
                this.UpdateProgress(35);
                if (IsCancelled()) { _log.Debug("Job Cancelled..."); return false; }

                if (!string.IsNullOrEmpty(IncludedVariableFile) && !string.IsNullOrEmpty(this.JobName))
                {
                    _log.Info("\r\n/*************************************/");
                    _log.Info("   Importing requested variables...    ");
                    _log.Info("/*************************************/");

                    if (!manager.CheckBuildVariableTable(this.JobName))
                    {
                        _log.Error("Importing requested variables... Failed! A problem was detected, exiting.");
                        return false;
                    }
                    else
                    {
                        _log.Debug("Importing requested variables... Done!");
                    }
                }
                this.UpdateProgress(50);
                if (IsCancelled()) { _log.Debug("Job Cancelled..."); return false; }

                if (!string.IsNullOrEmpty(ExportToShapefile))
                {
                    _log.Debug("\r\n/*************************************/");
                    _log.Info("   Exporting to shapefile... ");
                    _log.Debug("/*************************************/");

                    if (!manager.ExportShapefile(this.JobName))
                    {
                        _log.Error("There was an error while exporting the shapefile");

                        _log.Debug("\r\nExporting to shapefile... Failed!");
                    }
                    else
                    {
                        _log.Debug("Exporting to shapefile... Done!");
                    }
                }
                else
                {
                    _log.Debug("\r\n/****  No shapefile export requested ****/\r\n");
                }
                this.UpdateProgress(75);
                if (IsCancelled()) { _log.Debug("Job Cancelled..."); return false; }

                if (!string.IsNullOrEmpty(ExportToGrid))
                {
                    _log.Debug("\r\n/*************************************/");
                    _log.Info("   Exporting to gridded shapefile...");
                    _log.Debug("/*************************************/");

                    manager.GridEnvelopeFilename = GridEnvelope;
                    manager.SetGridParam(ExportToGrid);

                    _log.DebugFormat("Exporting all requested variables to fishnet shapefile with grid cell size {0} ", ExportToGrid);
                    manager.ExportGriddedShapefile(this.JobName);

                    _log.Debug("Exporting to gridded shapefile... Done!");
                }
                else
                {
                    _log.Debug("\r\n/****  No gridded shapefile export requested ****/\r\n");
                }
                this.UpdateProgress(100);

                _log.Info("Done!");

                return true;
            }
            catch (Exception ex)
            {
                _log.Error("Error thrown during import job ", ex);
                _log.Fatal("Please see the errors log file for details");
            }
            finally
            {
                TimeSpan elapsed = DateTime.Now - startTime;
                if (IsCancelled())
                {
                    _log.DebugFormat("Job was cancelled and stopped at {0} seconds", elapsed.TotalSeconds);
                }
                else
                {
                    _log.DebugFormat("Job completed in {0} seconds", elapsed.TotalSeconds);
                }
            }

            return false;
        }
        public void CheckTooManyColumnsFail()
        {
            string basePath = FileUtilities.PathEnsure(BaseDir, "TestData");
            string TooManyVariablesFile = Path.Combine(basePath, "TooManyColumns.txt");

            if (!File.Exists(TooManyVariablesFile))
            {
                int maxColumns = 255, max = (maxColumns / 2) + 1;
                StringBuilder sb = new StringBuilder();
                for (int i = 0; i < max; i++)
                {
                    sb.Append("COLUMN").Append(i).Append(Environment.NewLine);
                }
                File.WriteAllText(TooManyVariablesFile, sb.ToString());
            }

            var manager = new AcsDataManager(AcsState.Wyoming);
            manager.WorkingPath = basePath;
            manager.DesiredVariablesFilename = TooManyVariablesFile;
            Assert.IsFalse(manager.CheckBuildVariableTable("TestTooMany"));
        }
        private void AssertFailedImport(string filename, AcsDataManager man, DbConnection conn)
        {
            Assert.IsTrue(File.Exists(filename), "Could not find test file " + filename);
            man.DesiredVariablesFilename = filename;

            DataTable dt = man.GetRequestedVariables(conn);
            Assert.IsTrue(dt == null, "Non-null DataTable returned for file " + filename);
        }
 protected string GetShapePath(AcsDataManager man, string filename)
 {
     return Path.Combine(man.ShapePath, Path.GetFileName(filename));
 }
        protected AcsDataManager GetManager(string year)
        {
            AcsDataManager m = new AcsDataManager(AcsState.Wyoming, Path.Combine(BaseDir, "Working"), year);
            m.WorkingPath = FileUtilities.PathEnsure(BaseDir, "TestData");

            string dbPath = FileUtilities.PathEnsure(m.WorkingPath, "database");
            m.DBFilename = FileUtilities.PathCombine(dbPath, Settings.CurrentAcsDirectory + ".sqlite");
            m.ShapePath = FileUtilities.PathEnsure(m.WorkingPath, "shapes");
            m.CurrentDataPath = m.WorkingPath;
            m.DbClient = DataClient.GetDefaultClient(m.DBFilename);

            return m;
        }