public static List <Record> ToRecords(this Dataset dataset, string csvFile, string term)
        {
            using (StreamReader reader = new StreamReader(csvFile, Encoding.UTF8))
                using (CsvReader csv = new CsvReader(reader))
                {
                    Console.WriteLine($"Processing '{Path.GetFileName(csvFile)}'");

                    // Danger, Will Robinson!
                    //
                    // CsvHelper stores property names in couple of Dictionaries that use a string key.
                    // These dictionaries are case sensitive, because C# supports properties the same name as long as the cases are different (e.g. "discriminator" and "Discriminator" are different properties)
                    //
                    // But, we want a case _insensitive_ match for when a new csv file differs in column header case only (e.g. "Territorial Authority" and "Territorial authority") so we don't have to fix our Dataset configuration
                    // As per CsvHelper's documentation, we're expected to use the PrepareHeaderForMatch property to call ToLower() on all column headers:
                    //   - https://joshclose.github.io/CsvHelper/getting-started/
                    //   - https://github.com/JoshClose/CsvHelper/issues/1183
                    //
                    // That violates Microsoft's sting comparison guidelines, however: https://docs.microsoft.com/en-us/dotnet/csharp/how-to/compare-strings
                    //
                    // I'd prefer to replace the Dictionaries with ones that use a case insensitive match:
                    //
                    //  - csv.Context.NamedIndexes = new Dictionary<string, List<int>>(StringComparer.OrdinalIgnoreCase);
                    //  - csv.Context.NamedIndexCache = new Dictionary<string, (string, int)>(StringComparer.OrdinalIgnoreCase);
                    //
                    // ...which works right now, but reaching into CsvHelper's internals feels kind of fragile.
                    //
                    // So, we'll grumble and go ahead with ToLower()
                    csv.Configuration.PrepareHeaderForMatch = (header, index) => header.ToLowerInvariant();

                    // If we have more or fewer headers or properties than we expect, just keep going
                    csv.Configuration.HeaderValidated   = null;
                    csv.Configuration.MissingFieldFound = null;

                    csv.Configuration.ReadingExceptionOccurred = exception =>
                    {
                        if (exception is TypeConverterException)
                        {
                            // Some text in the Value column, instead of a number
                            // In most datasets, Value is null and we're supplied a null reason to indicate why we don't have a value
                            // However, some datasets include the null reason in the value column and cause type conversion errors
                            // We'll just skip these rows
                            return(false);
                        }

                        return(true);
                    };

                    var map = new RecordMap()
                              .Map(r => r.Selector, dataset.Selector ?? "Territorial Authority")
                              .Map(r => r.Date, dataset.Date);

                    map.Map(r => r.Measures)
                    .ConvertUsing(rr => dataset
                                  .Measure
                                  .Select((measure, index) => new { measure, index })
                                  .ToDictionary(
                                      mi => mi.measure.Name,
                                      mi => new ColumnValue
                    {
                        Index     = mi.index,
                        Column    = mi.measure.Name,
                        Separator = mi.measure.Separator ?? " — ",
                        Value     = rr.GetField <string>(mi.measure.Name)
                    })
                                  );

                    map.Map(r => r.Categories)
                    .ConvertUsing(rr => dataset
                                  .Category
                                  .Select((category, index) => new { category, index })
                                  .ToDictionary(ci =>
                                                ci.category.Name,
                                                ci => new ColumnValue
                    {
                        Index     = ci.index,
                        Column    = ci.category.Name,
                        Separator = ci.category.Separator ?? " — ",
                        Value     = rr.GetField <string>(ci.category.Name)
                    })
                                  );

                    map
                    .Map(r => r.Value, dataset.Value ?? "Value")
                    .Map(r => r.ValueUnit, dataset.ValueUnit ?? "Value Unit")
                    .Map(r => r.ValueLabel, dataset.ValueLabel ?? "Value Label")
                    .Map(r => r.NullReason, dataset.NullReason ?? "Null Reason");

                    csv.Configuration.RegisterClassMap(map);

                    return(dataset.ToRecords(csv, term));
                }
        }
        public static List <Record> ToRecords(this Dataset dataset, CsvReader csv, string term)
        {
            HashSet <string> terms = new HashSet <string>(StringComparer.OrdinalIgnoreCase)
            {
                term
            };

            if (!string.IsNullOrEmpty(dataset.TermMapping))
            {
                using (StreamReader file = File.OpenText(dataset.TermMapping))
                {
                    Dictionary <string, List <string> > mapping = new Dictionary <string, List <string> >(StringComparer.OrdinalIgnoreCase);
                    new JsonSerializer().Populate(file, mapping);

                    if (mapping.TryGetValue(term, out List <string> maps))
                    {
                        foreach (string map in maps)
                        {
                            terms.Add(map);
                        }
                    }
                }
            }

            bool hasMeasureExclusions = dataset.Measure.Any(c => c.Exclude != null && c.Exclude.Any());
            Dictionary <string, HashSet <string> > measureExclusions = dataset.Measure.Where(c => c.Exclude != null).ToDictionary(c => c.Name, c => new HashSet <string>(c.Exclude, StringComparer.OrdinalIgnoreCase), StringComparer.OrdinalIgnoreCase);

            bool hasMeasureInclusions = dataset.Measure.Any(c => c.Include != null && c.Include.Any());
            Dictionary <string, Dictionary <string, (int Index, Include Include)> > measureInclusions = dataset.Measure.Where(c => c.Include != null).ToDictionary(c => c.Name, c => c.Include.Select((include, index) => (Index: index, Include: include)).ToDictionary(ii => ii.Include.Value, i => i, StringComparer.OrdinalIgnoreCase), StringComparer.OrdinalIgnoreCase);

            bool hasCategoryExclusions = dataset.Category.Any(c => c.Exclude != null && c.Exclude.Any());
            Dictionary <string, HashSet <string> > categoryExclusions = dataset.Category.Where(c => c.Exclude != null).ToDictionary(c => c.Name, c => new HashSet <string>(c.Exclude, StringComparer.OrdinalIgnoreCase), StringComparer.OrdinalIgnoreCase);

            bool hasCategoryInclusions = dataset.Category.Any(c => c.Include != null && c.Include.Any());
            Dictionary <string, Dictionary <string, (int Index, Include Include)> > categoryInclusions = dataset.Category.Where(c => c.Include != null).ToDictionary(c => c.Name, c => c.Include.Select((include, index) => (Index: index, Include: include)).ToDictionary(ii => ii.Include.Value, i => i, StringComparer.OrdinalIgnoreCase), StringComparer.OrdinalIgnoreCase);

            List <Record> set = new List <Record>();

            int countRecords            = 0;
            int countMissingSelector    = 0;
            int countExcludedBySelector = 0;
            int countExcludedByMeasure  = 0;
            int countExcludedByGroup    = 0;
            int countExcludedByCategory = 0;
            int countExcludedByValue    = 0;

            foreach (Record r in csv.GetRecords <Record>())
            {
                countRecords++;

                if (r.Selector == null)
                {
                    countMissingSelector++;
                    continue;
                }

                if (!terms.Contains(r.Selector) && !term.Equals(dataset.AllSelectorsMatchTerm, StringComparison.OrdinalIgnoreCase))
                {
                    countExcludedBySelector++;
                    continue;
                }

                if (hasMeasureExclusions && r.Measures.Any(m => measureExclusions.TryGetValue(m.Key, out var exclusions) && exclusions.Contains(m.Value.Value)))
                {
                    countExcludedByMeasure++;
                    continue;
                }

                if (hasMeasureInclusions && r.Measures.Any(m => measureInclusions.TryGetValue(m.Key, out var inclusions) && inclusions.Any() && !inclusions.ContainsKey(m.Value.Value)))
                {
                    countExcludedByMeasure++;
                    continue;
                }

                if (hasCategoryExclusions && r.Categories.Any(m => categoryExclusions.TryGetValue(m.Key, out var exclusions) && exclusions.Contains(m.Value.Value)))
                {
                    countExcludedByCategory++;
                    continue;
                }

                if (hasCategoryInclusions && r.Categories.Any(c => categoryInclusions.TryGetValue(c.Key, out var inclusions) && inclusions.Any() && !inclusions.ContainsKey(c.Value.Value)))
                {
                    countExcludedByCategory++;
                    continue;
                }

                // If we've selected this row without a direct match to the supplied term, it's because we've got some mapping somewhere in the pipeline
                // Prepend the term to the row's selector, so we get something like "Auckland — Waitemata DHB"
                if (!term.Equals(r.Selector, StringComparison.OrdinalIgnoreCase))
                {
                    r.Selector = $"{term} — {r.Selector}";
                }

                if (hasMeasureInclusions)
                {
                    foreach (var m in r.Measures)
                    {
                        if (measureInclusions.TryGetValue(m.Key, out var mi) && mi.TryGetValue(m.Value.Value, out var ii))
                        {
                            m.Value.Label = ii.Include.Label;

                            // TODO: Move this out to Measure?
                            r.ConvertToPercentage = r.ConvertToPercentage || ii.Include.ConvertToPercentage;
                        }
                    }
                }

                if (hasCategoryInclusions)
                {
                    foreach (var c in r.Categories)
                    {
                        if (categoryInclusions.TryGetValue(c.Key, out var ci) && ci.TryGetValue(c.Value.Value, out var ii))
                        {
                            c.Value.Label = ii.Include.Label;

                            // TODO: Move this out to Category?
                            r.ConvertToPercentage = r.ConvertToPercentage || ii.Include.ConvertToPercentage;
                        }
                    }
                }

                if (r.Value != null && r.ValueLabel == "NZD thousands")
                {
                    r.ValueLabel = "NZD";
                    r.Value      = r.Value * 1000;
                }

                r.Parent    = dataset.Parent;
                r.Uri       = dataset.Source;
                r.DateLabel = dataset.Date;

                set.Add(r);
            }

            var ordered = set
                          .GroupBy(r => new { r.Selector, Measure = r.MeasureFormatted(), Category = r.CategoryFormatted() })
                          .Select(g => g
                                  .OrderByDescending(r => r.Date)
                                  .ThenByDescending(r => r.Value) // Case where Auckland appears twice for the same year because super city
                                  .First()
                                  )
                          .GroupBy(r => new { r.Selector, Measure = r.MeasureFormatted() })
                          .Select(g =>
            {
                decimal?total = g.Sum(r => r.Value);

                if (total == null || total == 0)
                {
                    return(g);
                }

                foreach (Record r in g.Where(r => r.ConvertToPercentage && r.Value != null))
                {
                    r.ValueUnit  = "percentage";
                    r.ValueLabel = r.ValueLabel.ReplaceCaseInsensitive("Number", "%");
                    r.Value      = (r.Value / total) * 100; // Multiple by 100 to stay consistent with other percentage values that are natively 100-based
                }

                return(g);
            })
                          .SelectMany(g => g)
                          .Where(r =>
            {
                // If we're excluding zero values, pull them out here
                if (dataset.ExcludeZeroValues && r.Value == 0)
                {
                    countExcludedByValue++;
                    return(false);
                }

                return(true);
            })
                          .OrderBy(r => r.Selector);

            foreach (KeyValuePair <string, Dictionary <string, (int Index, Include Include)> > measureInclusion in measureInclusions)
            {
                ordered = ordered
                          .ThenBy(r =>
                {
                    if (!r.Measures.TryGetValue(measureInclusion.Key, out var column))
                    {
                        return((int?)null);
                    }

                    if (!measureInclusion.Value.TryGetValue(column.Value, out var inclusion))
                    {
                        return((int?)null);
                    }

                    return(inclusion.Index);
                });
            }

            foreach (Column column in dataset.Measure)
            {
                ordered = ordered
                          .ThenBy(r => r.Measures[column.Name].Value, StringComparer.OrdinalIgnoreCase);
            }

            foreach (KeyValuePair <string, Dictionary <string, (int Index, Include Include)> > categoryInclusion in categoryInclusions)
            {
                ordered = ordered
                          .ThenBy(r =>
                {
                    if (!r.Categories.TryGetValue(categoryInclusion.Key, out var column))
                    {
                        return((int?)null);
                    }

                    if (!categoryInclusion.Value.TryGetValue(column.Value, out var inclusion))
                    {
                        return((int?)null);
                    }

                    return(inclusion.Index);
                });
            }

            set = ordered.ToList();

            Console.WriteLine($" - {countRecords} records read");

            if (countMissingSelector > 0)
            {
                Console.WriteLine($" - {countMissingSelector} records missing \"selector\"");
            }

            if (countExcludedBySelector > 0)
            {
                Console.WriteLine($" - {countExcludedBySelector} records excluded by \"selector\"");
            }

            if (countExcludedByMeasure > 0)
            {
                Console.WriteLine($" - {countExcludedByMeasure} records excluded by \"measure\"");
            }

            if (countExcludedByGroup > 0)
            {
                Console.WriteLine($" - {countExcludedByGroup} records excluded by \"group\"");
            }

            if (countExcludedByCategory > 0)
            {
                Console.WriteLine($" - {countExcludedByCategory} records excluded by \"category\"");
            }

            if (countExcludedByValue > 0)
            {
                Console.WriteLine($" - {countExcludedByValue} records excluded by \"value\"");
            }

            if (set.Count == 0)
            {
                Console.ForegroundColor = ConsoleColor.Red;
            }

            Console.WriteLine($" - {set.Count} records included");
            Console.ResetColor();
            Console.WriteLine();

            return(set);
        }