/// <summary>
        /// Streams all records of a CSV file using the specified delimiter.
        /// </summary>
        /// <param name="fileInfo">The CSV file to read.</param>
        /// <param name="readHeader"><c>true</c> if the first row contains the header.</param>
        /// <param name="delimiter">The delimiter for the values.</param>
        /// <returns>The records of this file.</returns>
        public IEnumerable <IReferenceRecord> StreamFile(FileInfo fileInfo, bool readHeader, char delimiter)
        {
            var header = new List <string>();

            using (var reader = new StreamReader(fileInfo.OpenRead()))
            {
                string    line;
                const int ChunkSize = 1000000;
                while (null != (line = reader.ReadLine()))
                {
                    var splittedLine = line.Split(delimiter);
                    if (!header.Any())
                    {
                        // Read the header
                        if (readHeader)
                        {
                            foreach (var headerValue in splittedLine)
                            {
                                header.Add(headerValue);
                            }
                            continue;
                        }
                        else
                        {
                            for (var index = 0; index < splittedLine.Length; index++)
                            {
                                header.Add(string.Format(@"FIELD{0}", index + 1));
                            }
                        }
                    }

                    if (header.Count == splittedLine.Length)
                    {
                        // Read the values
                        IReferenceRecord record = new ReferenceRecord();
                        for (var index = 0; index < header.Count; index++)
                        {
                            var key   = header[index];
                            var value = splittedLine[index];
                            if (null != ValueFactory)
                            {
                                value = ValueFactory.CreateValue(value);
                            }
                            if (null != RecordBuilder)
                            {
                                record = RecordBuilder.BuildRecord(record, key, value);
                            }
                            else if (!record.Properties.ContainsKey(key))
                            {
                                record.Properties.Add(key, value);
                            }
                        }

                        // Stream the record
                        yield return(record);
                    }
                }
            }
        }
        private static ReferenceDataset CreateRandomDataset(ulong recordCount)
        {
            var referenceDataset = new ReferenceDataset();
            for (ulong index = 0; index < recordCount; index++)
            {
                var record = new ReferenceRecord { Geometry = CreateRandomPoint() };
                record.Properties.Add(@"Name", @"Alexanderplatz");

                referenceDataset.Records.Add(record);
            }
            return referenceDataset;
        }
        private static ReferenceDataset CreateRandomDataset(ulong recordCount)
        {
            var referenceDataset = new ReferenceDataset();

            for (ulong index = 0; index < recordCount; index++)
            {
                var record = new ReferenceRecord {
                    Geometry = CreateRandomPoint()
                };
                record.Properties.Add(@"Name", @"Alexanderplatz");

                referenceDataset.Records.Add(record);
            }
            return(referenceDataset);
        }
        public void TestCreateSinglePropertyBuilder()
        {
            const string Key = @"Name";
            var recordBuilder = new SinglePropertyRecordBuilder(Key);

            IReferenceRecord record = new ReferenceRecord();
            record = recordBuilder.BuildRecord(record, Key, string.Empty);
            Assert.AreEqual(1, record.Properties.Count, @"The property count does not match!");
            Assert.AreEqual(string.Empty, record.Properties[Key], @"The property value does not match!");

            const string Value = @"Alexanderplatz";
            record = recordBuilder.BuildRecord(record, Key, Value);
            Assert.AreEqual(1, record.Properties.Count, @"The property count does not match!");
            Assert.AreEqual(Value, record.Properties[Key], @"The property value does not match!");
        }
Пример #5
0
        private string GetComparisonValue(ReferenceRecord reference)
        {
            const int pad = 7;

            var val = ComparedValue.Value / (UseInt ? (UseMultiThreaded ? reference.GInopsMT : reference.GInopsST)
                : (UseMultiThreaded ? reference.GFlopsMT : reference.GFlopsST));

            if (val > 0.01 && val < 10)
            {
                return((val.ToString("F2") + "x").PadRight(pad));
            }

            if (val > 10 && val < 1000)
            {
                return((val.ToString("F0") + "x").PadRight(pad));
            }

            var s = val.ToString("E0");

            try
            {
                var ss = s.Split('E');

                s = ss[1].TrimStart(new char[] { '+', '0' });

                if (s.StartsWith("-"))
                {
                    s = "-" + s.TrimStart(new char[] { '-', '0' });
                }

                s = s.Replace("-", "¯")
                    .Replace("0", "⁰")
                    .Replace("1", "¹")
                    .Replace("2", "²")
                    .Replace("3", "³")
                    .Replace("4", "⁴")
                    .Replace("5", "⁵")
                    .Replace("6", "⁶")
                    .Replace("7", "⁷")
                    .Replace("8", "⁸")
                    .Replace("9", "⁹");

                s = ss[0] + "·10" + s;
            }
            catch { }

            return(s.PadRight(pad));
        }
        public void TestCreateSinglePropertyBuilder()
        {
            const string Key           = @"Name";
            var          recordBuilder = new SinglePropertyRecordBuilder(Key);

            IReferenceRecord record = new ReferenceRecord();

            record = recordBuilder.BuildRecord(record, Key, string.Empty);
            Assert.AreEqual(1, record.Properties.Count, @"The property count does not match!");
            Assert.AreEqual(string.Empty, record.Properties[Key], @"The property value does not match!");

            const string Value = @"Alexanderplatz";

            record = recordBuilder.BuildRecord(record, Key, Value);
            Assert.AreEqual(1, record.Properties.Count, @"The property count does not match!");
            Assert.AreEqual(Value, record.Properties[Key], @"The property value does not match!");
        }
Пример #7
0
        public void TestWritingSimpleRecord()
        {
            var          records = new List <IReferenceRecord>();
            var          record  = new ReferenceRecord();
            const string Key     = @"Name";
            const string Value   = @"Alexanderplatz";

            record.Properties.Add(Key, Value);
            records.Add(record);

            var indexWriter = new IndexFileWriter();
            var indexFile   = new FileInfo(Path.GetTempFileName());

            indexWriter.WriteFile(indexFile, records);

            var indexReader    = new IndexFileReader();
            var writtenRecords = indexReader.ReadFile(indexFile);
            var writtenRecord  = Enumerable.First <IReferenceRecord>(writtenRecords);

            Assert.AreEqual(record.Properties.Count, writtenRecord.Properties.Count, @"The number of properties do not match!");
            Assert.IsTrue(writtenRecord.Properties.ContainsKey(Key), @"There is no property having a key equal to 'Name'!");
            Assert.AreEqual(record.Properties[Key], writtenRecord.Properties[Key], @"The property values do not match!");
        }
Пример #8
0
        public string RecordCompare()
        {
            Dictionary <string, Dictionary <string, dynamic> > description = new Dictionary <string, Dictionary <string, dynamic> >();

            foreach (PropertyInfo property in typeof(DeathRecord).GetProperties().OrderBy(p => ((Property)p.GetCustomAttributes().First()).Priority))
            {
                // Grab property annotation for this property
                Property info = (Property)property.GetCustomAttributes().First();

                // Skip properties that shouldn't be serialized.
                if (!info.Serialize)
                {
                    continue;
                }

                // Skip properties that are lost in the IJE format (if the test is a roundtrip).
                if (!info.CapturedInIJE && (Type != null && Type.Contains("Roundtrip")))
                {
                    continue;
                }

                // Add category if it doesn't yet exist
                if (!description.ContainsKey(info.Category))
                {
                    description.Add(info.Category, new Dictionary <string, dynamic>());
                }

                // Add the new property to the category
                Dictionary <string, dynamic> category = description[info.Category];
                category[property.Name] = new Dictionary <string, dynamic>();

                // Add the attributes of the property
                category[property.Name]["Name"]        = info.Name;
                category[property.Name]["Type"]        = info.Type.ToString();
                category[property.Name]["Description"] = info.Description;

                // Add snippets for reference
                FHIRPath path    = (FHIRPath)property.GetCustomAttributes().Last();
                var      matches = ReferenceRecord.GetRecord().GetITypedElement().Select(path.Path);
                if (matches.Count() > 0)
                {
                    if (info.Type == Property.Types.TupleCOD)
                    {
                        // Make sure to grab all of the Conditions for COD
                        string xml  = "";
                        string json = "";
                        foreach (var match in matches)
                        {
                            xml  += match.ToXml();
                            json += match.ToJson() + ",";
                        }
                        category[property.Name]["SnippetXML"]  = xml;
                        category[property.Name]["SnippetJSON"] = "[" + json + "]";
                    }
                    else if (!String.IsNullOrWhiteSpace(path.Element))
                    {
                        // Since there is an "Element" for this path, we need to be more
                        // specific about what is included in the snippets.
                        XElement root = XElement.Parse(matches.First().ToXml());
                        XElement node = root.DescendantsAndSelf("{http://hl7.org/fhir}" + path.Element).FirstOrDefault();
                        if (node != null)
                        {
                            node.Name = node.Name.LocalName;
                            category[property.Name]["SnippetXML"] = node.ToString();
                        }
                        else
                        {
                            category[property.Name]["SnippetXML"] = "";
                        }
                        Dictionary <string, dynamic> jsonRoot =
                            JsonConvert.DeserializeObject <Dictionary <string, dynamic> >(matches.First().ToJson(),
                                                                                          new JsonSerializerSettings()
                        {
                            DateParseHandling = DateParseHandling.None
                        });
                        if (jsonRoot != null && jsonRoot.Keys.Contains(path.Element))
                        {
                            category[property.Name]["SnippetJSON"] = "{" + $"\"{path.Element}\": \"{jsonRoot[path.Element]}\"" + "}";
                        }
                        else
                        {
                            category[property.Name]["SnippetJSON"] = "";
                        }
                    }
                    else
                    {
                        category[property.Name]["SnippetXML"]  = matches.First().ToXml();
                        category[property.Name]["SnippetJSON"] = matches.First().ToJson();
                    }
                }
                else
                {
                    category[property.Name]["SnippetXML"]  = "";
                    category[property.Name]["SnippetJSON"] = "";
                }

                // Add snippets for test
                FHIRPath pathTest    = (FHIRPath)property.GetCustomAttributes().Last();
                var      matchesTest = TestRecord.GetRecord().GetITypedElement().Select(pathTest.Path);
                if (matchesTest.Count() > 0)
                {
                    if (info.Type == Property.Types.TupleCOD)
                    {
                        // Make sure to grab all of the Conditions for COD
                        string xml  = "";
                        string json = "";
                        foreach (var match in matchesTest)
                        {
                            xml  += match.ToXml();
                            json += match.ToJson() + ",";
                        }
                        category[property.Name]["SnippetXMLTest"]  = xml;
                        category[property.Name]["SnippetJSONTest"] = "[" + json + "]";
                    }
                    else if (!String.IsNullOrWhiteSpace(pathTest.Element))
                    {
                        // Since there is an "Element" for this path, we need to be more
                        // specific about what is included in the snippets.
                        XElement root = XElement.Parse(matchesTest.First().ToXml());
                        XElement node = root.DescendantsAndSelf("{http://hl7.org/fhir}" + pathTest.Element).FirstOrDefault();
                        if (node != null)
                        {
                            node.Name = node.Name.LocalName;
                            category[property.Name]["SnippetXMLTest"] = node.ToString();
                        }
                        else
                        {
                            category[property.Name]["SnippetXMLTest"] = "";
                        }
                        Dictionary <string, dynamic> jsonRoot =
                            JsonConvert.DeserializeObject <Dictionary <string, dynamic> >(matchesTest.First().ToJson(),
                                                                                          new JsonSerializerSettings()
                        {
                            DateParseHandling = DateParseHandling.None
                        });
                        if (jsonRoot != null && jsonRoot.Keys.Contains(pathTest.Element))
                        {
                            category[property.Name]["SnippetJSONTest"] = "{" + $"\"{pathTest.Element}\": \"{jsonRoot[pathTest.Element]}\"" + "}";
                        }
                        else
                        {
                            category[property.Name]["SnippetJSONTest"] = "";
                        }
                    }
                    else
                    {
                        category[property.Name]["SnippetXMLTest"]  = matchesTest.First().ToXml();
                        category[property.Name]["SnippetJSONTest"] = matchesTest.First().ToJson();
                    }
                }
                else
                {
                    category[property.Name]["SnippetXMLTest"]  = "";
                    category[property.Name]["SnippetJSONTest"] = "";
                }

                // RecordCompare values
                if (info.Type == Property.Types.Dictionary)
                {
                    // Special case for Dictionary; we want to be able to describe what each key means
                    Dictionary <string, string> valueReference = (Dictionary <string, string>)property.GetValue(ReferenceRecord.GetRecord());

                    if (valueReference == null || valueReference.Values.All(vRV => String.IsNullOrWhiteSpace(vRV)))
                    {
                        continue;
                    }

                    Dictionary <string, string> valueTest = (Dictionary <string, string>)property.GetValue(TestRecord.GetRecord());
                    Dictionary <string, Dictionary <string, string> > moreInfo = new Dictionary <string, Dictionary <string, string> >();
                    bool match = true;
                    foreach (PropertyParam parameter in property.GetCustomAttributes().Reverse().Skip(1).Reverse().Skip(1))
                    {
                        moreInfo[parameter.Key] = new Dictionary <string, string>();
                        moreInfo[parameter.Key]["Description"] = parameter.Description;
                        if (valueReference != null && valueReference.ContainsKey(parameter.Key))
                        {
                            moreInfo[parameter.Key]["Value"] = valueReference[parameter.Key];
                        }
                        else
                        {
                            moreInfo[parameter.Key]["Value"] = null;
                        }
                        if (valueTest != null && valueTest.ContainsKey(parameter.Key))
                        {
                            moreInfo[parameter.Key]["FoundValue"] = valueTest[parameter.Key];
                        }
                        else
                        {
                            moreInfo[parameter.Key]["FoundValue"] = null;
                        }
                        // Check for match
                        if ((valueReference.ContainsKey(parameter.Key) && valueTest.ContainsKey(parameter.Key)) &&
                            (String.Equals((string)valueReference[parameter.Key], (string)valueTest[parameter.Key], StringComparison.OrdinalIgnoreCase)))
                        {
                            // Equal
                            Correct += 1;
                            moreInfo[parameter.Key]["Match"] = "true";
                        }
                        else if ((valueReference.ContainsKey(parameter.Key) && valueTest.ContainsKey(parameter.Key)) &&
                                 String.IsNullOrWhiteSpace((string)valueReference[parameter.Key]) &&
                                 String.IsNullOrWhiteSpace((string)valueTest[parameter.Key]))
                        {
                            // Equal
                            Correct += 1;
                            moreInfo[parameter.Key]["Match"] = "true";
                        }
                        else if (!valueReference.ContainsKey(parameter.Key) && !valueTest.ContainsKey(parameter.Key))
                        {
                            // Both null, equal
                            Correct += 1;
                            moreInfo[parameter.Key]["Match"] = "true";
                        }
                        else if (!valueReference.ContainsKey(parameter.Key) || (valueReference.ContainsKey(parameter.Key) && String.IsNullOrWhiteSpace(valueReference[parameter.Key])))
                        {
                            // Source is empty, so no need to punish test
                            Correct += 1;
                            moreInfo[parameter.Key]["Match"] = "true";
                        }
                        else if (parameter.Key == "display" && (!valueTest.ContainsKey(parameter.Key) || String.IsNullOrWhiteSpace(valueTest[parameter.Key])))
                        {
                            // Test record had nothing for display, equal
                            Correct += 1;
                            moreInfo[parameter.Key]["Match"] = "true";
                        }
                        else
                        {
                            // Not equal
                            Incorrect += 1;
                            moreInfo[parameter.Key]["Match"] = "false";
                            match = false;
                        }
                        Total += 1;
                    }
                    category[property.Name]["Match"] = match ? "true" : "false";
                    category[property.Name]["Value"] = moreInfo;
                }
                else
                {
                    category[property.Name]["Value"]      = property.GetValue(ReferenceRecord.GetRecord());
                    category[property.Name]["FoundValue"] = property.GetValue(TestRecord.GetRecord());
                    Total += 1;

                    // RecordCompare values
                    if (info.Type == Property.Types.String)
                    {
                        if (String.Equals((string)property.GetValue(ReferenceRecord.GetRecord()), (string)property.GetValue(TestRecord.GetRecord()), StringComparison.OrdinalIgnoreCase))
                        {
                            Correct += 1;
                            category[property.Name]["Match"] = "true";
                        }
                        else if (String.IsNullOrWhiteSpace((string)property.GetValue(ReferenceRecord.GetRecord())))
                        {
                            Correct += 1;
                            category[property.Name]["Match"] = "true";
                        }
                        else if (!String.IsNullOrWhiteSpace((string)property.GetValue(TestRecord.GetRecord())) &&
                                 !String.IsNullOrWhiteSpace((string)property.GetValue(ReferenceRecord.GetRecord())) &&
                                 ((string)property.GetValue(TestRecord.GetRecord())).ToLower().Contains(((string)property.GetValue(ReferenceRecord.GetRecord())).ToLower()))
                        {
                            Correct += 1;
                            category[property.Name]["Match"] = "true";
                        }
                        else
                        {
                            Incorrect += 1;
                            category[property.Name]["Match"] = "false";
                        }
                    }
                    else if (info.Type == Property.Types.StringDateTime)
                    {
                        DateTimeOffset referenceDateTime;
                        DateTimeOffset testDateTime;
                        if (property.GetValue(ReferenceRecord.GetRecord()) == null && property.GetValue(TestRecord.GetRecord()) == null)
                        {
                            Correct += 1;
                            category[property.Name]["Match"] = "true";
                        }
                        else if (String.IsNullOrWhiteSpace((string)property.GetValue(ReferenceRecord.GetRecord())))
                        {
                            Correct += 1;
                            category[property.Name]["Match"] = "true";
                        }
                        else if (DateTimeOffset.TryParse((string)property.GetValue(ReferenceRecord.GetRecord()), out referenceDateTime))
                        {
                            if (DateTimeOffset.TryParse((string)property.GetValue(TestRecord.GetRecord()), out testDateTime))
                            {
                                if (DateTimeOffset.Compare(referenceDateTime, testDateTime) == 0)
                                {
                                    Correct += 1;
                                    category[property.Name]["Match"] = "true";
                                }
                                else if (DateTimeOffset.Compare(new DateTimeOffset(referenceDateTime.Date, TimeSpan.Zero), new DateTimeOffset(testDateTime.Date, TimeSpan.Zero)) == 0)
                                {
                                    Correct += 1;
                                    category[property.Name]["Match"] = "true";
                                }
                                else
                                {
                                    Incorrect += 1;
                                    category[property.Name]["Match"] = "false";
                                }
                            }
                            else
                            {
                                Incorrect += 1;
                                category[property.Name]["Match"] = "false";
                            }
                        }
                        else
                        {
                            Incorrect += 1;
                            category[property.Name]["Match"] = "false";
                        }
                    }
                    else if (info.Type == Property.Types.StringArr)
                    {
                        string[] referenceArr = (string[])property.GetValue(ReferenceRecord.GetRecord());
                        string[] testArr      = (string[])property.GetValue(TestRecord.GetRecord());
                        if (referenceArr != null)
                        {
                            if (testArr != null)
                            {
                                if (String.Equals(String.Join(",", referenceArr.ToList().OrderBy(s => s).ToArray()), String.Join(",", testArr.ToList().OrderBy(s => s).ToArray()), StringComparison.OrdinalIgnoreCase))
                                {
                                    Correct += 1;
                                    category[property.Name]["Match"] = "true";
                                }
                                else if (referenceArr.ToList().Count == 0)
                                {
                                    Correct += 1;
                                    category[property.Name]["Match"] = "true";
                                }
                                else
                                {
                                    Incorrect += 1;
                                    category[property.Name]["Match"] = "false";
                                }
                            }
                            else
                            {
                                Incorrect += 1;
                                category[property.Name]["Match"] = "false";
                            }
                        }
                        else
                        {
                            Correct += 1;
                            category[property.Name]["Match"] = "true";
                        }
                    }
                    else if (info.Type == Property.Types.Bool)
                    {
                        if (bool.Equals(property.GetValue(ReferenceRecord.GetRecord()), property.GetValue(TestRecord.GetRecord())))
                        {
                            Correct += 1;
                            category[property.Name]["Match"] = "true";
                        }
                        else if (property.GetValue(ReferenceRecord.GetRecord()) == null)
                        {
                            Correct += 1;
                            category[property.Name]["Match"] = "true";
                        }
                        else
                        {
                            Incorrect += 1;
                            category[property.Name]["Match"] = "false";
                        }
                    }
                    else if (info.Type == Property.Types.TupleArr)
                    {
                        Tuple <string, string>[] referenceArr = (Tuple <string, string>[])property.GetValue(ReferenceRecord.GetRecord());
                        Tuple <string, string>[] testArr      = (Tuple <string, string>[])property.GetValue(TestRecord.GetRecord());
                        if (referenceArr != null)
                        {
                            if (testArr != null)
                            {
                                if (String.Equals(String.Join(",", referenceArr.ToList().OrderBy(s => s.Item1 + s.Item2)), String.Join(",", testArr.ToList().OrderBy(s => s.Item1 + s.Item2)), StringComparison.OrdinalIgnoreCase))
                                {
                                    Correct += 1;
                                    category[property.Name]["Match"] = "true";
                                }
                                else if (referenceArr.ToList().Count == 0)
                                {
                                    Correct += 1;
                                    category[property.Name]["Match"] = "true";
                                }
                                else
                                {
                                    Incorrect += 1;
                                    category[property.Name]["Match"] = "false";
                                }
                            }
                            else
                            {
                                Incorrect += 1;
                                category[property.Name]["Match"] = "false";
                            }
                        }
                        else if (testArr != null)
                        {
                            Incorrect += 1;
                            category[property.Name]["Match"] = "false";
                        }
                        else
                        {
                            Correct += 1;
                            category[property.Name]["Match"] = "true";
                        }
                    }
                    else if (info.Type == Property.Types.TupleCOD)
                    {
                        Tuple <string, string, Dictionary <string, string> >[] referenceArr = (Tuple <string, string, Dictionary <string, string> >[])property.GetValue(ReferenceRecord.GetRecord());
                        Tuple <string, string, Dictionary <string, string> >[] testArr      = (Tuple <string, string, Dictionary <string, string> >[])property.GetValue(TestRecord.GetRecord());
                        if (referenceArr != null)
                        {
                            if (testArr != null)
                            {
                                if (String.Equals(String.Join(",", referenceArr.ToList().OrderBy(s => s.Item1 + s.Item2)), String.Join(",", testArr.ToList().OrderBy(s => s.Item1 + s.Item2)), StringComparison.OrdinalIgnoreCase))
                                {
                                    Correct += 1;
                                    category[property.Name]["Match"] = "true";
                                }
                                else
                                {
                                    Incorrect += 1;
                                    category[property.Name]["Match"] = "false";
                                }
                            }
                            else
                            {
                                Incorrect += 1;
                                category[property.Name]["Match"] = "false";
                            }
                        }
                        else if (testArr != null)
                        {
                            Incorrect += 1;
                            category[property.Name]["Match"] = "false";
                        }
                        else
                        {
                            Correct += 1;
                            category[property.Name]["Match"] = "true";
                        }
                    }
                }
            }
            return(JsonConvert.SerializeObject(description));
        }
Пример #9
0
        public string MessageCompare()
        {
            Dictionary <string, Dictionary <string, dynamic> > description = new Dictionary <string, Dictionary <string, dynamic> >();
            BaseMessage bundle          = TestMessage.GetMessage();
            DeathRecord record          = ReferenceRecord.GetRecord();
            BaseMessage referenceBundle = new Message(ReferenceRecord, bundle.MessageType).GetMessage();
            // On the frontend this shares the same view as the RecordCompare below. This heading
            // is shown above the results in the app.
            string heading = "Message Validation Results";

            description.Add(heading, new Dictionary <string, dynamic>());
            Dictionary <string, dynamic> category = description[heading];

            foreach (PropertyInfo property in bundle.GetType().GetProperties())
            {
                Total += 1;
                // Add the new property to the category
                category[property.Name]                = new Dictionary <string, dynamic>();
                category[property.Name]["Name"]        = property.Name;
                category[property.Name]["Type"]        = (Nullable.GetUnderlyingType(property.PropertyType) ?? property.PropertyType).Name;
                category[property.Name]["Description"] = Message.GetDescriptionFor(property.Name);
                category[property.Name]["Value"]       = property.GetValue(referenceBundle);
                category[property.Name]["FoundValue"]  = property.GetValue(bundle);
                // The record should be valid since we check its validity elsewhere.
                // Here we just check to make sure the record is properly embedded
                // into the message bundle.
                if (property.PropertyType == typeof(DeathRecord))
                {
                    // Do not count DeathRecord base as part of the Total (since the Total is increased per-element in RecordCompare)
                    Total -= 1;
                    DeathRecord extracted = (DeathRecord)property.GetValue(bundle);
                    TestRecord = new Record(extracted);
                    int previousIncorrect = Incorrect;
                    category[property.Name]["SnippetJSON"] = RecordCompare();
                    // See if the value of Incorrect changed in 'RecordCompare' and use that to determine if the
                    // Record matches or not.
                    category[property.Name]["Match"] = previousIncorrect.Equals(Incorrect) ? "true" : "false";
                }
                else if (Message.validatePresenceOnly(property.Name))
                {
                    // Basic message validation ensures that these fields are present if they are required
                    // These values do not have to match our reference record and are just therefore
                    // set to valid here.
                    Correct += 1;
                    category[property.Name]["Match"] = "true";
                    // Override the displayed value to be equal to what the user provided on the UI
                    category[property.Name]["Value"] = category[property.Name]["FoundValue"];
                }
                else
                {
                    // Using == here seems to be checking ReferenceEquals and not Equals, causing the equality to return false.
                    // Calling Prop1.Equals(Prop2) here raises an error if the value is null in the ReferenceBundle.
                    // The best option here is to just use the object.Equals operator.
                    if (Equals(property.GetValue(referenceBundle), property.GetValue(bundle)))
                    {
                        Correct += 1;
                        category[property.Name]["Match"] = "true";
                    }
                    else
                    {
                        Incorrect += 1;
                        category[property.Name]["Match"] = "false";
                    }
                }
            }
            return(JsonConvert.SerializeObject(description));
        }
        /// <summary>
        /// Reads a CSV file using the specified delimiter.
        /// </summary>
        /// <param name="fileInfo">The CSV file to read.</param>
        /// <param name="readHeader"><c>true</c> if the first row contains the header.</param>
        /// <param name="delimiter">The delimiter for the values.</param>
        /// <returns>A reference dataset.</returns>
        public ReferenceDataset ReadFile(FileInfo fileInfo, bool readHeader, char delimiter)
        {
            var dataset = new ReferenceDataset();
            var header  = new List <string>();

            using (var reader = new StreamReader(fileInfo.OpenRead()))
            {
                string    line;
                const int ChunkSize = 1000000;
                while (null != (line = reader.ReadLine()))
                {
                    var splittedLine = line.Split(delimiter);
                    if (!header.Any())
                    {
                        // Read the header
                        if (readHeader)
                        {
                            foreach (var headerValue in splittedLine)
                            {
                                header.Add(headerValue);
                            }
                            continue;
                        }
                        else
                        {
                            for (var index = 0; index < splittedLine.Length; index++)
                            {
                                header.Add(string.Format(@"FIELD{0}", index + 1));
                            }
                        }
                    }

                    if (header.Count == splittedLine.Length)
                    {
                        try
                        {
                            // Read the values
                            IReferenceRecord record = new ReferenceRecord();
                            for (var index = 0; index < header.Count; index++)
                            {
                                var key   = header[index];
                                var value = splittedLine[index];
                                if (null != ValueFactory)
                                {
                                    value = ValueFactory.CreateValue(value);
                                }
                                if (null != RecordBuilder)
                                {
                                    record = RecordBuilder.BuildRecord(record, key, value);
                                }
                                else if (!record.Properties.ContainsKey(key))
                                {
                                    record.Properties.Add(key, value);
                                }
                            }

                            // Add the record
                            dataset.Records.Add(record);
                            if (0 == dataset.Records.Count % ChunkSize)
                            {
                                _logger.Info(@"{0} records created.", dataset.Records.Count);
                            }
                        }
                        catch (Exception ex)
                        {
                            _logger.Error(@"Error during record creation! '{0}'", ex.Message);
                        }
                    }
                }
            }
            return(dataset);
        }