/// <summary>
        /// Step through and calculate means+std devs for each voltage in .fullDATs
        /// </summary>
        public void GenerateStatsData()
        {
            StackTrace st = new StackTrace();

            Debug.WriteLine("GenerateStatsData() caller name: " + st.GetFrame(1).GetMethod().Name);
            StatsDataList = new List <LJVStatsDatum>();
            //remove scans where pixel did not light up
            for (int i = 0; i < LJVScanVMCollection.Count; i++)
            {
                if (!LJVScanVMCollection[i].TheLJVScan.PixelLitUp ?? false)
                {
                    Debug.WriteLine(LJVScanVMCollection[i].TheLJVScan.DeviceLJVScanSummary.Device.Label + LJVScanVMCollection[i].TheLJVScan.Pixel.Site + " did not light up");
                    LJVScanVMCollection.Remove(LJVScanVMCollection[i]);
                }
            }
            //first find the maximum voltage across all LJVScans
            decimal   maxVoltage = 0;
            LJVScanVM maxScan    = new LJVScanVM();

            foreach (LJVScanVM scan in LJVScanVMCollection)
            {
                try
                {
                    if (scan.FullLJVDataList.Count != scan.RawLJVDataList.Count)
                    {
                        scan.GenerateFullLJVData();
                    }
                    var scanMax = scan.FullLJVDataList.Max(x => x.Voltage);
                    if (scanMax > maxVoltage)
                    {
                        maxVoltage = scanMax;
                        maxScan    = scan;
                    }
                }
                catch (Exception e)
                {
                    Debug.WriteLine("LJVScanSummaryVM GenerateStatsData error: " + e.ToString());
                }
            }
            //assume that the StepSize is constant and loop through each step in maxScan
            for (int i = 0; i < maxScan.FullLJVDataList.Count; i++)
            {
                List <FullLJVDatum> dataAtVoltage = new List <FullLJVDatum>();
                foreach (LJVScanVM scan in LJVScanVMCollection)
                {
                    if (scan.FullLJVDataList.Count >= i)
                    {
                        FullLJVDatum datumAtIndex = scan.FullLJVDataList.Where(x => x.Voltage == maxScan.FullLJVDataList[i].Voltage).FirstOrDefault();
                        if (datumAtIndex != null)
                        {
                            dataAtVoltage.Add(datumAtIndex);
                        }
                    }
                }
                StatsDataList.Add(StatsDatumFromFullLJVList(dataAtVoltage));
            }
            SetStatsDataPath();
            DataProcessingService.WriteIENumberableToCSV(StatsDataList, _statsDataPath);
        }
        public bool GenerateNewDataEntitiesFromPath(string fp)
        {
            bool generatedNewEntities = false;

            _selectedRawDATFiles  = Directory.GetFiles(fp, "*.rawDAT", SearchOption.AllDirectories).ToList <string>();
            _selectedProcDATFiles = Directory.GetFiles(fp, "*.procDAT", SearchOption.AllDirectories).ToList <string>();
            _selectedELSpecFiles  = Directory.GetFiles(fp, "*.ELSpectrum", SearchOption.AllDirectories).ToList <string>();
            _selectedImageFiles   = Directory.GetFiles(fp, "*.jpg", SearchOption.AllDirectories).ToList <string>();
            _selectedImageFiles   = _selectedImageFiles.Where(x => !x.Contains("Cropped")).ToList();
            _selectedImageFiles   = _selectedImageFiles.Where(x => !x.Contains("Plots")).ToList();
            //generate a hashset for all Devices + TestConditions already added to the DeviceBatch
            var existingScanList = new List <Tuple <int, string> >();

            foreach (Device d in TheDeviceBatch.Devices)
            {
                foreach (DeviceLJVScanSummary ss in d.DeviceLJVScanSummaries)
                {
                    existingScanList.Add(new Tuple <int, string>(d.BatchIndex, ss.TestCondition));
                }
            }
            //get device BatchIndex and TestCondition from filepath
            var fileNameRawDATList  = IndexAListOfPaths(_selectedRawDATFiles);
            var fileNameProcDATList = IndexAListOfPaths(_selectedProcDATFiles);
            var fileNameELSpecList  = IndexAListOfPaths(_selectedELSpecFiles);
            var fileNameImageList   = IndexAListOfPaths(_selectedImageFiles);
            //select only filepaths that have not already been added to the DeviceBatch
            var newRawDatList  = ParseForNewData(existingScanList, fileNameRawDATList);
            var newProcDatList = ParseForNewData(existingScanList, fileNameProcDATList);
            var newELSpecList  = ParseForNewData(existingScanList, fileNameELSpecList);
            var newImageList   = ParseForNewData(existingScanList, fileNameImageList);
            //filter out duplicates using a hashset and remap the filepath (tuple item 3) for spreadsheet generation
            var testConditionsToAdd = new HashSet <Tuple <int, string, string> >();

            foreach (Tuple <int, string, string> t in newProcDatList)
            {
                //Debug.WriteLine("newProcDatList entry for device: " + t.Item1 + " testCondition: " + t.Item2);
                var index             = t.Item3.IndexOf("Processed Data");
                var testConditionPath = string.Concat(t.Item3.Remove(index), @"Scan Summaries\");
                Directory.CreateDirectory(testConditionPath);
                //Debug.WriteLine("testConditionPath is: " + testConditionPath);
                testConditionsToAdd.Add(new Tuple <int, string, string>(t.Item1, t.Item2, testConditionPath));
            }
            foreach (Tuple <int, string, string> t in testConditionsToAdd)
            {
                Debug.WriteLine("testConditionsToAdd entry for device: " + t.Item1 + " testCondition: " + t.Item2);
            }
            var testConditionsToAddList = testConditionsToAdd.ToList();

            //create new data entities for devices with data in 'new' lists
            //(this assumes that we will never encounter the case where we are adding ELSpec or Image but not a procDAT)
            foreach (Tuple <int, string, string> testcondition in testConditionsToAddList)
            {
                Debug.WriteLine("Device batchIndex: " + testcondition.Item1 + "testcondition: " + testcondition.Item2);
                foreach (Device d in TheDeviceBatch.Devices)
                {
                    //first check to make sure that there is actually data available to add to the new ScanSummary
                    bool dataIsAvailable = false;
                    foreach (Tuple <int, string, string> t in newProcDatList)
                    {
                        if (t.Item1 == testcondition.Item1 && t.Item1 == d.BatchIndex && t.Item2 == testcondition.Item2)
                        {
                            dataIsAvailable = true;
                            Debug.WriteLine("Data available for device with batchIndex: " + t.Item1 + " testcondition: " + t.Item2);
                        }
                    }
                    if (dataIsAvailable)
                    {
                        var ScanSummary = new LJVScanSummaryVM();
                        ctx.DeviceLJVScanSummaries.Add(ScanSummary.TheLJVScanSummary);
                        //ctx.Entry(ScanSummary.TheLJVScanSummary);
                        d.DeviceLJVScanSummaries.Add(ScanSummary.TheLJVScanSummary);
                        ScanSummary.TheLJVScanSummary.Device = d;
                        ScanSummary.TheLJVScanSummary.SpreadsheetReportPath = string.Concat(testcondition.Item3, d.Label, "_", testcondition.Item2, ".xlsx");
                        ScanSummary.TheLJVScanSummary.TestCondition         = testcondition.Item2;
                        foreach (Pixel p in d.Pixels)
                        {
                            LJVScan    newScan   = null;
                            ELSpectrum newELSpec = null;
                            EFDeviceBatchCodeFirst.Image newImage = null;
                            foreach (Tuple <int, string, string> t in newProcDatList)
                            {
                                if (t.Item1 == d.BatchIndex && t.Item2 == testcondition.Item2 && t.Item3.Substring(t.Item3.LastIndexOf("_") + 1, 5) == p.Site)
                                {
                                    Debug.WriteLine("Adding new LJVScan to device #" + d.BatchIndex);
                                    var scanVM = new LJVScanVM();
                                    scanVM.PopulatePropertiesFromPath(t.Item3);
                                    scanVM.TheLJVScan.Pixel = p;
                                    newScan = scanVM.TheLJVScan;
                                    newScan.ProcDATFilePath      = t.Item3;
                                    newScan.DeviceLJVScanSummary = ScanSummary.TheLJVScanSummary;
                                    string rawDatPath = t.Item3.Replace("Processed Data", "Raw Data");
                                    rawDatPath = rawDatPath.Replace(".procDAT", ".rawDAT");
                                    if (File.Exists(rawDatPath))
                                    {
                                        newScan.RawDATFilePath = rawDatPath;
                                        scanVM = new LJVScanVM(newScan);
                                    }
                                    ctx.LJVScans.Add(newScan);
                                }
                            }
                            foreach (Tuple <int, string, string> t in newELSpecList)
                            {
                                if (t.Item1 == d.BatchIndex && t.Item2 == testcondition.Item2 && t.Item3.Substring(t.Item3.LastIndexOf("_") + 1, 5) == p.Site)
                                {
                                    var ELSpecVM = new ELSpecVM();
                                    ELSpecVM.PopulatePropertiesFromPath(t.Item3);
                                    ELSpecVM.TheELSpectrum.QDBatch = d.QDBatch;
                                    ELSpecVM.TheELSpectrum.Pixel   = p;
                                    newELSpec = ELSpecVM.TheELSpectrum;
                                    ctx.ELSpectra.Add(newELSpec);
                                }
                            }
                            foreach (Tuple <int, string, string> t in newImageList)
                            {
                                if (t.Item1 == d.BatchIndex && t.Item2 == testcondition.Item2 && t.Item3.Substring(t.Item3.LastIndexOf("_") + 1, 5) == p.Site && !t.Item3.Contains("Cropped"))
                                {
                                    var ImageVM = new ImageVM();
                                    ImageVM.TheImage.FilePath = t.Item3;
                                    ImageVM.TheImage.Pixel    = p;
                                    newImage = ImageVM.TheImage;
                                    ctx.Images.Add(newImage);
                                }
                            }
                            if (newScan != null && newELSpec != null && newImage != null)//only generate entities if data exists
                            {
                                newELSpec.DeviceLJVScanSummary = ScanSummary.TheLJVScanSummary;
                                newScan.Image = newImage;
                                p.LJVScans.Add(newScan);
                                p.ELSpectrums.Add(newELSpec);
                                ScanSummary.TheLJVScanSummary.LJVScans.Add(newScan);
                                ScanSummary.TheLJVScanSummary.ELSpectrums.Add(newELSpec);
                                ScanSummary.TheLJVScanSummary.Images.Add(newImage);
                            }
                        }
                        ScanSummary.PopulateEntityPropertiesFromChildren();
                        SpreadsheetGenerator.GenerateLJVScanSummary(ScanSummary.TheLJVScanSummary);
                        generatedNewEntities = true;
                    }
                }
            }
            try
            {
                ctx.SaveChanges();
            }
            catch (DbEntityValidationException e)
            {
                Debug.WriteLine(e.ToString());
                foreach (var eve in e.EntityValidationErrors)
                {
                    Debug.WriteLine("Entity of type \"{0}\" in state \"{1}\" has the following validation errors: ", eve.Entry.Entity.GetType().Name, eve.Entry.State);
                    foreach (var ve in eve.ValidationErrors)
                    {
                        Debug.WriteLine("- Property: \"{0}\", Error: \"{1}\"", ve.PropertyName, ve.ErrorMessage);
                    }
                }
            }
            return(generatedNewEntities);
        }