public List <Tracklet> AnalyzeCCD(PipelineArguments Args)
        {
            Logger("Setting up pipeline");
            /* Deal with incorrect SWARP flux scaling */
            SWarpScaling.ApplyTransform = CorrectSWARP;

            string RunDir = Args.RunDir;

            if (!Directory.Exists(RunDir))
            {
                Directory.CreateDirectory(RunDir);
            }

            /* Read input images and preprocess for poisson noise */
            int ImageCount = Args.Inputs.Length;

            FitsImage[] FirstProcess = new FitsImage[ImageCount];
            double[]    PFW          = PipelineHelperFunctions.LinearizedPoissonKernel(PoissonRadius);

            Step.StepPipeline sp = new Step.StepPipeline(StandardBITPIX, RunDir, Args.Inputs.Length, MaxDetections);
            sp.LogHookImage     = LogImage;
            sp.LogHookDetection = LogDet;
            sp.LogMessage       = LogMessage;

            bool HasBadpix = Args.Badpixel != null;

            Logger("Begining to run the pipeline");
            var zpTask = System.Threading.Tasks.Task <Dictionary <IO.Image, double> > .Factory.StartNew(() => CalibrateZP(Args.Inputs));

            var skTask = System.Threading.Tasks.Task <bool> .Factory.StartNew(() => PrecacheSkyBot(Args.Inputs));

            BitArray[] map = PipelineHelperFunctions.ExtractBadpixel(Args.Badpixel, Logger);

            for (int i = 0; i < ImageCount; i++)
            {
                FitsImage Pipeline = Args.Inputs[i];
                Pipeline.GetProperty <ImageSource>().AddToSet(Pipeline, "Original");
                sp.SetModel(i, Pipeline, new List <IO.ImageProperties>()
                {
                    Pipeline.GetProperty <ObservationTime>()
                });

                if (!UseCoreFilter)
                {
                    sp.RunPipeline(RestrictedMean.RestrictedMeanFilter, "Poisson", i, ref Pipeline, PFW, RestrictedMean.Parameters(PoissonRadius));
                }
                else if (HasBadpix)
                {
                    sp.RunPipeline(CoreFilter.Filter, "Poisson", i, ref Pipeline, new CoreFilter.CoreFilterParameters(PFW, map), CoreFilter.Parameters(PoissonRadius));
                }
                else
                {
                    throw new ArgumentException("Must specify Badpixel files if trying to run with CoreFilter");
                }

                if (Operations.HasFlag(EnabledOperations.Normalization))
                {
                    if (!sp.EnsureImage("Normalized", i, out FitsImage Normalized))
                    {
                        Point4Distance p4d = new Point4Distance(Pipeline, Normalized, NormalizationMeshSize);
                        Logger("Generated Normalized image " + i);
                    }
                    else
                    {
                        Logger("Found Normalized image " + i);
                    }
                    Normalized.GetProperty <ImageSource>().AddToSet(Args.Inputs[i], "Normalized");
                    Pipeline = Normalized;
                }
                FirstProcess[i] = Pipeline;
            }

            /* Create the central median */
            string CentralPath = Path.Combine(RunDir, "Central.fits");

            if (!sp.EnsureCentralImage("Central", out FitsImage Central))
            {
                HardMedians.MultiImageMedian.Run(null, FirstProcess, Central, HardMedians.MultiImageMedianParameters);
                Logger("Generated Central image");
            }
            else
            {
                Logger("Found Central image");
            }

            Logger("Computed the multi-image median");

            /* Prepare the mask, slow object detector, trail detector, weights for second median filtering, etc. */
            ImageStatistics CentralStats = new ImageStatistics(Central);

            if (Args.Clipped)
            {
                CentralStats = new ImageStatistics(Central, CentralStats.ZeroLevel, 2 * CentralStats.StDev);
            }
            StarData StarList = new StarData();

            ComputeDetectorData(Central, CentralStats, StarList, out MaskByMedian.MaskProperties MaskProp, out DotDetector SlowDetector,
                                out LongTrailDetector.LongTrailData LTD);
            if (Args.Clipped)
            {
                SlowDetector.HighThresholdMultiplier *= 2;
                SlowDetector.LowThresholdMultiplier  *= 2;
            }

            DetectionReducer dr = new DetectionReducer()
            {
                PairingRadius = 0.7
            };

            if (Operations.HasFlag(EnabledOperations.SourceExtractor))
            {
                try
                {
                    dr.LoadStars(StarList.FixedStarList);
                }
                catch (Exception ex) { throw new ArgumentException("Could not read detections from SE catalog.", ex); }
                dr.GeneratePool();
            }


            Logger("Set up detectors");

            List <ImageDetection> FullDetectionsList = new List <ImageDetection>();

            double[] FMW2 = PipelineHelperFunctions.LinearizedMedianKernel();
            LTLimit  ltl  = new LTLimit()
            {
                MinPix = TrailMinPix
            };
            RipFilter rf = new RipFilter()
            {
                SigmaTop = 30
            };

            Logger("Ready for final image processing and detection");

            for (int i = 0; i < ImageCount; i++)
            {
                List <ImageDetection> LocalDetectionList = new List <ImageDetection>();

                FitsImage DetectionSource = FirstProcess[i];

                if (Operations.HasFlag(EnabledOperations.Masking))
                {
                    sp.RunPipeline(MaskByMedian.Masker, "Masked", i, ref DetectionSource, MaskProp, MaskByMedian.Parameters);
                }

                if (Operations.HasFlag(EnabledOperations.SecondMedian))
                {
                    sp.RunPipeline(HardMedians.WeightedMedian, "Second Median", i, ref DetectionSource, FMW2, HardMedians.WeightedMedianParameters(SecMedRadius));
                }

                ImageStatistics SecMedStat = new ImageStatistics(DetectionSource);

                if (Operations.HasFlag(EnabledOperations.LongTrailDetector))
                {
                    var Dets = sp.RunDetector((FitsImage img) =>
                    {
                        LongTrailDetector.PrepareAlgorithmForImage(img, SecMedStat, ref LTD);
                        LongTrailDetector.Algorithm.Run(LTD, DetectionSource, LongTrailDetector.Parameters);
                        return(LTD.Results);
                    }, DetectionSource, "Trail", DetectionAlgorithm.Trail);
                    LocalDetectionList.AddRange(Dets);
                }

                if (Operations.HasFlag(EnabledOperations.BlobDetector))
                {
                    var Dets = sp.RunDetector(SlowDetector.Detect, DetectionSource, "Blob", DetectionAlgorithm.Blob);
                    LocalDetectionList.AddRange(Dets);
                }

                if (Operations.HasFlag(EnabledOperations.SourceExtractor))
                {
                    var dts = sp.RunDetector((arg) =>
                    {
                        List <ImageDetection> Dets = ExtraIO.SourceExtractor.ParseSEFile(Args.CatalogData[i], Args.Inputs[i]);
                        Dets   = Dets.Where((x) => x.FetchProperty <ObjectPhotometry>().Flux > 300).ToList();
                        var ND = dr.Reduce(Dets);
                        return(ND);
                    }, DetectionSource, "SE", DetectionAlgorithm.SourceExtractor);
                    LocalDetectionList.AddRange(dts);
                }

                if (Operations.HasFlag(EnabledOperations.OutputDetectionMap))
                {
                    DetectionDebugMap(RunDir, i, LocalDetectionList, DetectionSource);
                }

                rf.ImgMean  = SecMedStat.ZeroLevel;
                rf.ImgSigma = SecMedStat.StDev;
                var NLDL = sp.RunFilters(LocalDetectionList, "LocalToGlobal", ltl, rf);
                Logger("Total " + NLDL.Count + " detections.");
                FullDetectionsList.AddRange(NLDL);
            }
            Logger("Filtering and pairing detections...");

            LinearityThresholdFilter LTF = new LinearityThresholdFilter()
            {
                MaxLineThickness = MaxLineThickness
            };
            List <ImageDetection> FilteredDetections = sp.RunFilters(FullDetectionsList, "MainFilter", LTF);

            StarList.MarkStarCrossed(FilteredDetections, StarCrossRadiusM, StarCrossMinFlux);
            if (Args.CCDBadzone != null)
            {
                FilteredDetections = sp.RunFilters(FilteredDetections, "Badzone", Args.CCDBadzone);
            }

            Logger("Before PrePair " + FilteredDetections.Count);
            PrePair.MatchDetections(FilteredDetections, MaxPairmatchDistance, MixMatch, SameArcSep);

            Logger("Left with " + FilteredDetections.Count + " detections");
            LinePoolSimple lps = new LinePoolSimple()
            {
                MaxLinErrorArcSec = MaxResidual, SearchExtraSmall = SmallExtraSearchRadius, SearchExtraBig = BigExtraSearchRadius
            };

            lps.LoadDetections(FilteredDetections);

            lps.GeneratePool();
            var Pairings = lps.FindTracklets();

            sp.NotePairings(FilteredDetections, Pairings);

            Logger("Found " + Pairings.Count + " raw tracklets");

            LinearityTest lintest = new LinearityTest();
            StaticFilter  stf     = new StaticFilter();
            TotalError    te      = new TotalError();
            var           TK2List = sp.RunFilters(Pairings, "Tracklet Filtering", stf, te);

            Logger("After filtering: " + TK2List.Count + " candidate objects found");

            sp.LogDetections(Path.Combine(RunDir, "detlog.txt"));

            Dictionary <IO.Image, double> ZP = zpTask.Result;

            skTask.Wait();

            var Recovered = RecoverTracklets(TK2List, Args.Inputs, Path.Combine(RunDir, "reclog.txt"), ZP);

            TrackletsDeduplication.Deduplicate(Recovered, 1.0);

            Logger("Recovered " + Recovered.Count + " candidate objects");

            PairSkyBot(Recovered, SkyBoTDistance, Args.FieldName, Args.CCDNumber, Args.Inputs, sp);

            return(Recovered);
        }
Exemple #2
0
        protected override void Execute(CodeActivityContext context)
        {
            string            message           = "";
            List <LogMessage> logList           = new List <LogMessage>();
            List <string>     nonInvoices       = new List <string>();
            List <string>     notConfidenceList = new List <string>();

            message = "Loading FlexiCapture Engine for Recognition ...";
            Console.WriteLine(message);
            logList.Add(new LogMessage(message, LogType.Information));

            string documentProjectId = DocumentProjectId.Get(context);

            message = "Get Document Project ID: " + documentProjectId;
            Console.WriteLine(message);
            logList.Add(new LogMessage(message, LogType.Information));

            int confidenceLevelLimitation = ConfidenceLevelLimitation.Get(context);

            message = string.Format("Confidence Level Limitation: {0}", confidenceLevelLimitation);
            Console.WriteLine(message);
            logList.Add(new LogMessage(message, LogType.Information));

            string sourceFolder = SourceFolder.Get(context);

            message = string.Format("Get PDF folder: {0}", sourceFolder);
            Console.WriteLine(message);
            logList.Add(new LogMessage(message, LogType.Information));

            string ocrTemplate = OCRTemplateFolder.Get(context);

            message = string.Format("OCR Template Folder: {0}", ocrTemplate);
            Console.WriteLine(message);
            logList.Add(new LogMessage(message, LogType.Information));

            string exportFolder = ExportFolder.Get(context);

            message = string.Format("Get Export Folder: {0}", exportFolder);
            Console.WriteLine(message);
            logList.Add(new LogMessage(message, LogType.Information));
            if (!Directory.Exists(exportFolder))
            {
                Directory.CreateDirectory(exportFolder);
                message = string.Format("Folder {0} is created completely", exportFolder);
                Console.WriteLine(message);
                logList.Add(new LogMessage(message, LogType.Information));
            }

            string recognizedFolder = RecognizedFolder.Get(context);

            Console.WriteLine("Get Recognize Folder: " + recognizedFolder);
            logList.Add(new LogMessage("Recognize Folder is:  " + recognizedFolder, LogType.Information));
            if (!Directory.Exists(recognizedFolder))
            {
                Directory.CreateDirectory(recognizedFolder);
                message = string.Format("Folder {0} is created completely", recognizedFolder);
                Console.WriteLine(message);
                logList.Add(new LogMessage(message, LogType.Information));
            }

            string notConfidenceFolder = NotConfidenceFolder.Get(context);

            Console.WriteLine("Get Not Confidence Folder: " + notConfidenceFolder);
            logList.Add(new LogMessage("Not Confidence Folder is:  " + notConfidenceFolder, LogType.Information));
            if (!Directory.Exists(notConfidenceFolder))
            {
                Directory.CreateDirectory(notConfidenceFolder);
                message = string.Format("Folder {0} is created completely", notConfidenceFolder);
                Console.WriteLine(message);
                logList.Add(new LogMessage(message, LogType.Information));
            }

            string rejectFolder = RejectFolder.Get(context);

            Console.WriteLine("Get Reject Folder: " + rejectFolder);
            logList.Add(new LogMessage("Reject Folder is:  " + rejectFolder, LogType.Information));
            if (!Directory.Exists(rejectFolder))
            {
                Directory.CreateDirectory(rejectFolder);
                message = string.Format("Folder {0} is created completely", rejectFolder);
                Console.WriteLine(message);
                logList.Add(new LogMessage(message, LogType.Information));
            }

            engine = LoadEngine(documentProjectId);

            Boolean isRecognized    = false;
            bool    isValidInvoice  = false;
            int     count           = 0;
            int     noError         = 0;
            int     noSuccess       = 0;
            int     noNotConfidence = 0;

            try
            {
                message = "Creating and configuring the FlexiCapture Processor...";
                Console.WriteLine(message);
                logList.Add(new LogMessage(message, LogType.Information));

                processor = engine.CreateFlexiCaptureProcessor();

                message = "Adding Document Definition to process...";
                Console.WriteLine(message);
                logList.Add(new LogMessage(message, LogType.Information));
                string[] TotalOCRFiles = Directory.GetFiles(ocrTemplate, "*.fcdot", SearchOption.AllDirectories);
                if (TotalOCRFiles.Length == 0)
                {
                    message = string.Format("OCR Template *.fcdot is not found in the folder {0}", ocrTemplate);
                    Console.WriteLine(message);
                    logList.Add(new LogMessage(message, LogType.Error));
                    throw new Exception(message);
                }
                foreach (string ocr in TotalOCRFiles)
                {
                    processor.AddDocumentDefinitionFile(ocr);
                    message = string.Format("OCR Temhplate {0} is added", ocr);
                    Console.WriteLine(message);
                    logList.Add(new LogMessage(message, LogType.Information));
                }

                message = string.Format("Adding images to process...");
                Console.WriteLine(message);
                logList.Add(new LogMessage(message, LogType.Information));
                string[] TotalFiles = Directory.GetFiles(sourceFolder, "*.pdf", SearchOption.AllDirectories);
                if (TotalFiles.Length == 0)
                {
                    message = string.Format("PDF Files *.pdf is not found in the folder {0}.", sourceFolder);
                    Console.WriteLine(message);
                    logList.Add(new LogMessage(message, LogType.Error));
                    throw new Exception(message);
                }
                foreach (string pdfFile in TotalFiles)
                {
                    processor.AddImageFile(pdfFile);
                    message = string.Format("PDF Files {0} is added", pdfFile);
                    Console.WriteLine(message);
                    logList.Add(new LogMessage(message, LogType.Information));
                }

                message = "Recognizing the images and exporting the results...";
                Console.WriteLine(message);
                logList.Add(new LogMessage(message, LogType.Information));

                while (true)
                {
                    // Recognize next document
                    confidenceLevel        = 0;
                    confidenceDetailLevel  = 0;
                    confidenceHeaderLevel  = 0;
                    totalConfidenceLevelHD = 0;
                    IDocument document = processor.RecognizeNextDocument();
                    if (document == null)
                    {
                        IProcessingError error = processor.GetLastProcessingError();
                        if (error != null)
                        {
                            // Processing error
                            message = string.Format("processing error because of {0}.", error.MessageText());
                            Console.WriteLine(message);
                            logList.Add(new LogMessage(message, LogType.Error));
                            continue;
                        }
                        else
                        {
                            // No more images
                            message = string.Format("all PDF Files has been executed or no PDF file in the folder {0}", sourceFolder);
                            Console.WriteLine(message);
                            logList.Add(new LogMessage(message, LogType.Information));
                            break;
                        }
                    }
                    else if (document.DocumentDefinition == null)
                    {
                        // Couldn't find matching template for the image. In this sample this is an error.
                        // In other scenarios this might be normal
                        message = string.Format("PDF file is not matched with existing OCR Templates.");
                        Console.WriteLine(message);
                        logList.Add(new LogMessage(message, LogType.Error));
                        //string tempPage = document.Pages[0].OriginalImagePath;
                        //if(tempPage != null)
                        //{
                        //    string movefile = Path.GetFileName(tempPage);
                        //    string tempfilename = Path.GetFileNameWithoutExtension(tempPage);

                        //    logList.Add(new LogMessage("Move to Reject Folder", LogType.Information));
                        //    if (File.Exists(tempPage))
                        //    {

                        //        MoveFileToDestinationFolder(sourceFolder, rejectFolder, tempPage, tempfilename);
                        //    }
                        //    else
                        //    {
                        //        MoveFileToDestinationFolder(notConfidenceFolder, rejectFolder, notConfidenceFolder+"\\"+movefile, tempfilename);
                        //    }

                        //}

                        continue;
                    }

                    string originalPath             = document.Pages[0].OriginalImagePath;
                    string file                     = Path.GetFileName(originalPath);
                    string filenamewithoutextention = Path.GetFileNameWithoutExtension(originalPath);

                    message = string.Format("Recognizing pdf {0} is started.", originalPath);
                    Console.WriteLine(message);
                    logList.Add(new LogMessage(message, LogType.Information));

                    //set confident level and status
                    message = string.Format("Extracting data from pdf {0} is started", originalPath);
                    Console.WriteLine(message);
                    logList.Add(new LogMessage(message, LogType.Information));
                    for (int i = 0; i < document.Sections.Count; i++)
                    { // extracing
                        var section = document.Sections[i];
                        if (object.ReferenceEquals(section, null))
                        {
                            continue;
                        }



                        for (int d = 0; d < section.Children.Count; d++)
                        {
                            var child = section.Children[d];
                            if (object.ReferenceEquals(child, null))
                            {
                                continue;
                            }

                            var field = ((IField)child);

                            message = string.Format("Extracting column {0} = {1}", field.Name, field.Value.AsText);
                            Console.WriteLine(message);

                            if (field.Name.ToUpper().Trim() == "INV_CONFIDENCE_LEVEL")
                            {
                                var value = TextFieldHelper.GetConfidenLevel(engine, document);
                                totalConfidenceLevelHD = value;
                                var data = engine.CreateText(value.ToString(), null);
                                field.Value.AsInteger = value;
                                confidenceLevel       = value;

                                message = string.Format("Confidence level of {0} is {1}", file, value);
                                Console.WriteLine(message);
                                logList.Add(new LogMessage(message, LogType.Information));
                            }

                            if (field.Name.ToUpper().Trim() == "INV_STATUS")
                            {
                                var value = "Recognized";
                                var data  = engine.CreateText(value, null);
                                field.Value.AsText.Delete(0, field.Value.AsText.Length);
                                field.Value.AsText.Insert(data, 0);
                            }


                            if (field.Name.ToUpper().Trim() == "FILE_NAME")
                            {
                                var data = engine.CreateText(file, null);
                                field.Value.AsText.Delete(0, field.Value.AsText.Length);
                                field.Value.AsText.Insert(data, 0);
                            }
                        }
                    } //end extracting
                    isValidInvoice = true;
                    if (isValidInvoice)
                    {
                        message = string.Format("Total Confidence is {0} ", totalConfidenceLevelHD);
                        Console.WriteLine(message);
                        logList.Add(new LogMessage(message, LogType.Information));
                        //check confidence level validation
                        if (totalConfidenceLevelHD >= confidenceLevelLimitation)
                        {
                            try
                            {
                                message = string.Format("Exporting process for pdf {0} is started ...", file);
                                Console.WriteLine(message);
                                logList.Add(new LogMessage(message, LogType.Information));

                                //IFileExportParams exportParams = engine.CreateFileExportParams();
                                //Console.WriteLine("XLS");
                                //exportParams.FileFormat = FileExportFormatEnum.FEF_XLS;


                                processor.ExportDocument(document, exportFolder);
                                Console.WriteLine("Exporting process is completed ...");

                                MoveFileToDestinationFolder(sourceFolder, exportFolder, originalPath, filenamewithoutextention);
                                Console.WriteLine(string.Format("Moving {0} to Export folder {1} is completed", file, exportFolder));

                                logList.Add(new LogMessage("Exporting process is ended ...", LogType.Information));
                                noSuccess++;
                            }
                            catch (Exception e)
                            {
                                noError++;
                                Console.WriteLine(string.Format("exporting is failed because of {0}.", e.Message));
                                logList.Add(new LogMessage(message, LogType.Error));

                                MoveFileToDestinationFolder(sourceFolder, rejectFolder, originalPath, filenamewithoutextention);
                                message = string.Format("Moving pdf {0} to Reject folder {1} is completed", file, rejectFolder);
                                Console.WriteLine(message);
                                logList.Add(new LogMessage(message, LogType.Information));
                                continue;
                            }
                        }
                        else
                        {
                            message = string.Format("Confidence of PDF {0} is {1} less than target confidence {2}", file, totalConfidenceLevelHD, confidenceLevelLimitation);
                            logList.Add(new LogMessage(message, LogType.Error));
                            notConfidenceList.Add(file);

                            logList.Add(new LogMessage(string.Format("Total number of not confidence is {0}", notConfidenceList.Count), LogType.Information));
                            MoveFileToDestinationFolder(sourceFolder, notConfidenceFolder, originalPath, filenamewithoutextention);
                            message = string.Format("Moving pdf {0} to Not Confidence folder {1} is completed", file, notConfidenceFolder);
                            Console.WriteLine(message);
                            logList.Add(new LogMessage(message, LogType.Information));
                            noNotConfidence++;
                        }
                    }
                    count++;
                }

                var msg = processor.GetLastProcessingError();
                if (msg != null)
                {
                    var msgError = string.Format("the processing error because of {0}.", msg.MessageText());
                    Console.WriteLine(msgError);
                    logList.Add(new LogMessage(msgError, LogType.Error));
                    noError++;
                }
                message = string.Format("No. of Not Confidence {0} and No. of Error {1} and No. of Exported to DB {2}", noNotConfidence, noError, noSuccess);
                logList.Add(new LogMessage(message, LogType.Information));
                isRecognized = true;
            }
            finally
            {
                UnloadEngine(ref engine);
                message = string.Format("Released FlexiCapture Engine for Recognition...");
                Console.WriteLine(message);
                logList.Add(new LogMessage(message, LogType.Information));

                IsRecognized.Set(context, isRecognized);
                NonInvoiceList.Set(context, nonInvoices);
                Messages.Set(context, logList);
                TotalExported.Set(context, noSuccess);
                TotalError.Set(context, noError);
                TotalNotConfidence.Set(context, noNotConfidence);
                NotConfidenceList.Set(context, notConfidenceList);
            }
        }