public List <Tracklet> AnalyzeCCD(PipelineArguments Args)
{
Logger("Setting up pipeline");
/* Deal with incorrect SWARP flux scaling */
SWarpScaling.ApplyTransform = CorrectSWARP;
string RunDir = Args.RunDir;
if (!Directory.Exists(RunDir))
{
Directory.CreateDirectory(RunDir);
}
/* Read input images and preprocess for poisson noise */
int ImageCount = Args.Inputs.Length;
FitsImage[] FirstProcess = new FitsImage[ImageCount];
double[] PFW = PipelineHelperFunctions.LinearizedPoissonKernel(PoissonRadius);
Step.StepPipeline sp = new Step.StepPipeline(StandardBITPIX, RunDir, Args.Inputs.Length, MaxDetections);
sp.LogHookImage = LogImage;
sp.LogHookDetection = LogDet;
sp.LogMessage = LogMessage;
bool HasBadpix = Args.Badpixel != null;
Logger("Begining to run the pipeline");
var zpTask = System.Threading.Tasks.Task <Dictionary <IO.Image, double> > .Factory.StartNew(() => CalibrateZP(Args.Inputs));
var skTask = System.Threading.Tasks.Task <bool> .Factory.StartNew(() => PrecacheSkyBot(Args.Inputs));
BitArray[] map = PipelineHelperFunctions.ExtractBadpixel(Args.Badpixel, Logger);
for (int i = 0; i < ImageCount; i++)
{
FitsImage Pipeline = Args.Inputs[i];
Pipeline.GetProperty <ImageSource>().AddToSet(Pipeline, "Original");
sp.SetModel(i, Pipeline, new List <IO.ImageProperties>()
{
Pipeline.GetProperty <ObservationTime>()
});
if (!UseCoreFilter)
{
sp.RunPipeline(RestrictedMean.RestrictedMeanFilter, "Poisson", i, ref Pipeline, PFW, RestrictedMean.Parameters(PoissonRadius));
}
else if (HasBadpix)
{
sp.RunPipeline(CoreFilter.Filter, "Poisson", i, ref Pipeline, new CoreFilter.CoreFilterParameters(PFW, map), CoreFilter.Parameters(PoissonRadius));
}
else
{
throw new ArgumentException("Must specify Badpixel files if trying to run with CoreFilter");
}
if (Operations.HasFlag(EnabledOperations.Normalization))
{
if (!sp.EnsureImage("Normalized", i, out FitsImage Normalized))
{
Point4Distance p4d = new Point4Distance(Pipeline, Normalized, NormalizationMeshSize);
Logger("Generated Normalized image " + i);
}
else
{
Logger("Found Normalized image " + i);
}
Normalized.GetProperty <ImageSource>().AddToSet(Args.Inputs[i], "Normalized");
Pipeline = Normalized;
}
FirstProcess[i] = Pipeline;
}
/* Create the central median */
string CentralPath = Path.Combine(RunDir, "Central.fits");
if (!sp.EnsureCentralImage("Central", out FitsImage Central))
{
HardMedians.MultiImageMedian.Run(null, FirstProcess, Central, HardMedians.MultiImageMedianParameters);
Logger("Generated Central image");
}
else
{
Logger("Found Central image");
}
Logger("Computed the multi-image median");
/* Prepare the mask, slow object detector, trail detector, weights for second median filtering, etc. */
ImageStatistics CentralStats = new ImageStatistics(Central);
if (Args.Clipped)
{
CentralStats = new ImageStatistics(Central, CentralStats.ZeroLevel, 2 * CentralStats.StDev);
}
StarData StarList = new StarData();
ComputeDetectorData(Central, CentralStats, StarList, out MaskByMedian.MaskProperties MaskProp, out DotDetector SlowDetector,
out LongTrailDetector.LongTrailData LTD);
if (Args.Clipped)
{
SlowDetector.HighThresholdMultiplier *= 2;
SlowDetector.LowThresholdMultiplier *= 2;
}
DetectionReducer dr = new DetectionReducer()
{
PairingRadius = 0.7
};
if (Operations.HasFlag(EnabledOperations.SourceExtractor))
{
try
{
dr.LoadStars(StarList.FixedStarList);
}
catch (Exception ex) { throw new ArgumentException("Could not read detections from SE catalog.", ex); }
dr.GeneratePool();
}
Logger("Set up detectors");
List <ImageDetection> FullDetectionsList = new List <ImageDetection>();
double[] FMW2 = PipelineHelperFunctions.LinearizedMedianKernel();
LTLimit ltl = new LTLimit()
{
MinPix = TrailMinPix
};
RipFilter rf = new RipFilter()
{
SigmaTop = 30
};
Logger("Ready for final image processing and detection");
for (int i = 0; i < ImageCount; i++)
{
List <ImageDetection> LocalDetectionList = new List <ImageDetection>();
FitsImage DetectionSource = FirstProcess[i];
if (Operations.HasFlag(EnabledOperations.Masking))
{
sp.RunPipeline(MaskByMedian.Masker, "Masked", i, ref DetectionSource, MaskProp, MaskByMedian.Parameters);
}
if (Operations.HasFlag(EnabledOperations.SecondMedian))
{
sp.RunPipeline(HardMedians.WeightedMedian, "Second Median", i, ref DetectionSource, FMW2, HardMedians.WeightedMedianParameters(SecMedRadius));
}
ImageStatistics SecMedStat = new ImageStatistics(DetectionSource);
if (Operations.HasFlag(EnabledOperations.LongTrailDetector))
{
var Dets = sp.RunDetector((FitsImage img) =>
{
LongTrailDetector.PrepareAlgorithmForImage(img, SecMedStat, ref LTD);
LongTrailDetector.Algorithm.Run(LTD, DetectionSource, LongTrailDetector.Parameters);
return(LTD.Results);
}, DetectionSource, "Trail", DetectionAlgorithm.Trail);
LocalDetectionList.AddRange(Dets);
}
if (Operations.HasFlag(EnabledOperations.BlobDetector))
{
var Dets = sp.RunDetector(SlowDetector.Detect, DetectionSource, "Blob", DetectionAlgorithm.Blob);
LocalDetectionList.AddRange(Dets);
}
if (Operations.HasFlag(EnabledOperations.SourceExtractor))
{
var dts = sp.RunDetector((arg) =>
{
List <ImageDetection> Dets = ExtraIO.SourceExtractor.ParseSEFile(Args.CatalogData[i], Args.Inputs[i]);
Dets = Dets.Where((x) => x.FetchProperty <ObjectPhotometry>().Flux > 300).ToList();
var ND = dr.Reduce(Dets);
return(ND);
}, DetectionSource, "SE", DetectionAlgorithm.SourceExtractor);
LocalDetectionList.AddRange(dts);
}
if (Operations.HasFlag(EnabledOperations.OutputDetectionMap))
{
DetectionDebugMap(RunDir, i, LocalDetectionList, DetectionSource);
}
rf.ImgMean = SecMedStat.ZeroLevel;
rf.ImgSigma = SecMedStat.StDev;
var NLDL = sp.RunFilters(LocalDetectionList, "LocalToGlobal", ltl, rf);
Logger("Total " + NLDL.Count + " detections.");
FullDetectionsList.AddRange(NLDL);
}
Logger("Filtering and pairing detections...");
LinearityThresholdFilter LTF = new LinearityThresholdFilter()
{
MaxLineThickness = MaxLineThickness
};
List <ImageDetection> FilteredDetections = sp.RunFilters(FullDetectionsList, "MainFilter", LTF);
StarList.MarkStarCrossed(FilteredDetections, StarCrossRadiusM, StarCrossMinFlux);
if (Args.CCDBadzone != null)
{
FilteredDetections = sp.RunFilters(FilteredDetections, "Badzone", Args.CCDBadzone);
}
Logger("Before PrePair " + FilteredDetections.Count);
PrePair.MatchDetections(FilteredDetections, MaxPairmatchDistance, MixMatch, SameArcSep);
Logger("Left with " + FilteredDetections.Count + " detections");
LinePoolSimple lps = new LinePoolSimple()
{
MaxLinErrorArcSec = MaxResidual, SearchExtraSmall = SmallExtraSearchRadius, SearchExtraBig = BigExtraSearchRadius
};
lps.LoadDetections(FilteredDetections);
lps.GeneratePool();
var Pairings = lps.FindTracklets();
sp.NotePairings(FilteredDetections, Pairings);
Logger("Found " + Pairings.Count + " raw tracklets");
LinearityTest lintest = new LinearityTest();
StaticFilter stf = new StaticFilter();
TotalError te = new TotalError();
var TK2List = sp.RunFilters(Pairings, "Tracklet Filtering", stf, te);
Logger("After filtering: " + TK2List.Count + " candidate objects found");
sp.LogDetections(Path.Combine(RunDir, "detlog.txt"));
Dictionary <IO.Image, double> ZP = zpTask.Result;
skTask.Wait();
var Recovered = RecoverTracklets(TK2List, Args.Inputs, Path.Combine(RunDir, "reclog.txt"), ZP);
TrackletsDeduplication.Deduplicate(Recovered, 1.0);
Logger("Recovered " + Recovered.Count + " candidate objects");
PairSkyBot(Recovered, SkyBoTDistance, Args.FieldName, Args.CCDNumber, Args.Inputs, sp);
return(Recovered);
}
protected override void Execute(CodeActivityContext context)
{
string message = "";
List <LogMessage> logList = new List <LogMessage>();
List <string> nonInvoices = new List <string>();
List <string> notConfidenceList = new List <string>();
message = "Loading FlexiCapture Engine for Recognition ...";
Console.WriteLine(message);
logList.Add(new LogMessage(message, LogType.Information));
string documentProjectId = DocumentProjectId.Get(context);
message = "Get Document Project ID: " + documentProjectId;
Console.WriteLine(message);
logList.Add(new LogMessage(message, LogType.Information));
int confidenceLevelLimitation = ConfidenceLevelLimitation.Get(context);
message = string.Format("Confidence Level Limitation: {0}", confidenceLevelLimitation);
Console.WriteLine(message);
logList.Add(new LogMessage(message, LogType.Information));
string sourceFolder = SourceFolder.Get(context);
message = string.Format("Get PDF folder: {0}", sourceFolder);
Console.WriteLine(message);
logList.Add(new LogMessage(message, LogType.Information));
string ocrTemplate = OCRTemplateFolder.Get(context);
message = string.Format("OCR Template Folder: {0}", ocrTemplate);
Console.WriteLine(message);
logList.Add(new LogMessage(message, LogType.Information));
string exportFolder = ExportFolder.Get(context);
message = string.Format("Get Export Folder: {0}", exportFolder);
Console.WriteLine(message);
logList.Add(new LogMessage(message, LogType.Information));
if (!Directory.Exists(exportFolder))
{
Directory.CreateDirectory(exportFolder);
message = string.Format("Folder {0} is created completely", exportFolder);
Console.WriteLine(message);
logList.Add(new LogMessage(message, LogType.Information));
}
string recognizedFolder = RecognizedFolder.Get(context);
Console.WriteLine("Get Recognize Folder: " + recognizedFolder);
logList.Add(new LogMessage("Recognize Folder is: " + recognizedFolder, LogType.Information));
if (!Directory.Exists(recognizedFolder))
{
Directory.CreateDirectory(recognizedFolder);
message = string.Format("Folder {0} is created completely", recognizedFolder);
Console.WriteLine(message);
logList.Add(new LogMessage(message, LogType.Information));
}
string notConfidenceFolder = NotConfidenceFolder.Get(context);
Console.WriteLine("Get Not Confidence Folder: " + notConfidenceFolder);
logList.Add(new LogMessage("Not Confidence Folder is: " + notConfidenceFolder, LogType.Information));
if (!Directory.Exists(notConfidenceFolder))
{
Directory.CreateDirectory(notConfidenceFolder);
message = string.Format("Folder {0} is created completely", notConfidenceFolder);
Console.WriteLine(message);
logList.Add(new LogMessage(message, LogType.Information));
}
string rejectFolder = RejectFolder.Get(context);
Console.WriteLine("Get Reject Folder: " + rejectFolder);
logList.Add(new LogMessage("Reject Folder is: " + rejectFolder, LogType.Information));
if (!Directory.Exists(rejectFolder))
{
Directory.CreateDirectory(rejectFolder);
message = string.Format("Folder {0} is created completely", rejectFolder);
Console.WriteLine(message);
logList.Add(new LogMessage(message, LogType.Information));
}
engine = LoadEngine(documentProjectId);
Boolean isRecognized = false;
bool isValidInvoice = false;
int count = 0;
int noError = 0;
int noSuccess = 0;
int noNotConfidence = 0;
try
{
message = "Creating and configuring the FlexiCapture Processor...";
Console.WriteLine(message);
logList.Add(new LogMessage(message, LogType.Information));
processor = engine.CreateFlexiCaptureProcessor();
message = "Adding Document Definition to process...";
Console.WriteLine(message);
logList.Add(new LogMessage(message, LogType.Information));
string[] TotalOCRFiles = Directory.GetFiles(ocrTemplate, "*.fcdot", SearchOption.AllDirectories);
if (TotalOCRFiles.Length == 0)
{
message = string.Format("OCR Template *.fcdot is not found in the folder {0}", ocrTemplate);
Console.WriteLine(message);
logList.Add(new LogMessage(message, LogType.Error));
throw new Exception(message);
}
foreach (string ocr in TotalOCRFiles)
{
processor.AddDocumentDefinitionFile(ocr);
message = string.Format("OCR Temhplate {0} is added", ocr);
Console.WriteLine(message);
logList.Add(new LogMessage(message, LogType.Information));
}
message = string.Format("Adding images to process...");
Console.WriteLine(message);
logList.Add(new LogMessage(message, LogType.Information));
string[] TotalFiles = Directory.GetFiles(sourceFolder, "*.pdf", SearchOption.AllDirectories);
if (TotalFiles.Length == 0)
{
message = string.Format("PDF Files *.pdf is not found in the folder {0}.", sourceFolder);
Console.WriteLine(message);
logList.Add(new LogMessage(message, LogType.Error));
throw new Exception(message);
}
foreach (string pdfFile in TotalFiles)
{
processor.AddImageFile(pdfFile);
message = string.Format("PDF Files {0} is added", pdfFile);
Console.WriteLine(message);
logList.Add(new LogMessage(message, LogType.Information));
}
message = "Recognizing the images and exporting the results...";
Console.WriteLine(message);
logList.Add(new LogMessage(message, LogType.Information));
while (true)
{
// Recognize next document
confidenceLevel = 0;
confidenceDetailLevel = 0;
confidenceHeaderLevel = 0;
totalConfidenceLevelHD = 0;
IDocument document = processor.RecognizeNextDocument();
if (document == null)
{
IProcessingError error = processor.GetLastProcessingError();
if (error != null)
{
// Processing error
message = string.Format("processing error because of {0}.", error.MessageText());
Console.WriteLine(message);
logList.Add(new LogMessage(message, LogType.Error));
continue;
}
else
{
// No more images
message = string.Format("all PDF Files has been executed or no PDF file in the folder {0}", sourceFolder);
Console.WriteLine(message);
logList.Add(new LogMessage(message, LogType.Information));
break;
}
}
else if (document.DocumentDefinition == null)
{
// Couldn't find matching template for the image. In this sample this is an error.
// In other scenarios this might be normal
message = string.Format("PDF file is not matched with existing OCR Templates.");
Console.WriteLine(message);
logList.Add(new LogMessage(message, LogType.Error));
//string tempPage = document.Pages[0].OriginalImagePath;
//if(tempPage != null)
//{
// string movefile = Path.GetFileName(tempPage);
// string tempfilename = Path.GetFileNameWithoutExtension(tempPage);
// logList.Add(new LogMessage("Move to Reject Folder", LogType.Information));
// if (File.Exists(tempPage))
// {
// MoveFileToDestinationFolder(sourceFolder, rejectFolder, tempPage, tempfilename);
// }
// else
// {
// MoveFileToDestinationFolder(notConfidenceFolder, rejectFolder, notConfidenceFolder+"\\"+movefile, tempfilename);
// }
//}
continue;
}
string originalPath = document.Pages[0].OriginalImagePath;
string file = Path.GetFileName(originalPath);
string filenamewithoutextention = Path.GetFileNameWithoutExtension(originalPath);
message = string.Format("Recognizing pdf {0} is started.", originalPath);
Console.WriteLine(message);
logList.Add(new LogMessage(message, LogType.Information));
//set confident level and status
message = string.Format("Extracting data from pdf {0} is started", originalPath);
Console.WriteLine(message);
logList.Add(new LogMessage(message, LogType.Information));
for (int i = 0; i < document.Sections.Count; i++)
{ // extracing
var section = document.Sections[i];
if (object.ReferenceEquals(section, null))
{
continue;
}
for (int d = 0; d < section.Children.Count; d++)
{
var child = section.Children[d];
if (object.ReferenceEquals(child, null))
{
continue;
}
var field = ((IField)child);
message = string.Format("Extracting column {0} = {1}", field.Name, field.Value.AsText);
Console.WriteLine(message);
if (field.Name.ToUpper().Trim() == "INV_CONFIDENCE_LEVEL")
{
var value = TextFieldHelper.GetConfidenLevel(engine, document);
totalConfidenceLevelHD = value;
var data = engine.CreateText(value.ToString(), null);
field.Value.AsInteger = value;
confidenceLevel = value;
message = string.Format("Confidence level of {0} is {1}", file, value);
Console.WriteLine(message);
logList.Add(new LogMessage(message, LogType.Information));
}
if (field.Name.ToUpper().Trim() == "INV_STATUS")
{
var value = "Recognized";
var data = engine.CreateText(value, null);
field.Value.AsText.Delete(0, field.Value.AsText.Length);
field.Value.AsText.Insert(data, 0);
}
if (field.Name.ToUpper().Trim() == "FILE_NAME")
{
var data = engine.CreateText(file, null);
field.Value.AsText.Delete(0, field.Value.AsText.Length);
field.Value.AsText.Insert(data, 0);
}
}
} //end extracting
isValidInvoice = true;
if (isValidInvoice)
{
message = string.Format("Total Confidence is {0} ", totalConfidenceLevelHD);
Console.WriteLine(message);
logList.Add(new LogMessage(message, LogType.Information));
//check confidence level validation
if (totalConfidenceLevelHD >= confidenceLevelLimitation)
{
try
{
message = string.Format("Exporting process for pdf {0} is started ...", file);
Console.WriteLine(message);
logList.Add(new LogMessage(message, LogType.Information));
//IFileExportParams exportParams = engine.CreateFileExportParams();
//Console.WriteLine("XLS");
//exportParams.FileFormat = FileExportFormatEnum.FEF_XLS;
processor.ExportDocument(document, exportFolder);
Console.WriteLine("Exporting process is completed ...");
MoveFileToDestinationFolder(sourceFolder, exportFolder, originalPath, filenamewithoutextention);
Console.WriteLine(string.Format("Moving {0} to Export folder {1} is completed", file, exportFolder));
logList.Add(new LogMessage("Exporting process is ended ...", LogType.Information));
noSuccess++;
}
catch (Exception e)
{
noError++;
Console.WriteLine(string.Format("exporting is failed because of {0}.", e.Message));
logList.Add(new LogMessage(message, LogType.Error));
MoveFileToDestinationFolder(sourceFolder, rejectFolder, originalPath, filenamewithoutextention);
message = string.Format("Moving pdf {0} to Reject folder {1} is completed", file, rejectFolder);
Console.WriteLine(message);
logList.Add(new LogMessage(message, LogType.Information));
continue;
}
}
else
{
message = string.Format("Confidence of PDF {0} is {1} less than target confidence {2}", file, totalConfidenceLevelHD, confidenceLevelLimitation);
logList.Add(new LogMessage(message, LogType.Error));
notConfidenceList.Add(file);
logList.Add(new LogMessage(string.Format("Total number of not confidence is {0}", notConfidenceList.Count), LogType.Information));
MoveFileToDestinationFolder(sourceFolder, notConfidenceFolder, originalPath, filenamewithoutextention);
message = string.Format("Moving pdf {0} to Not Confidence folder {1} is completed", file, notConfidenceFolder);
Console.WriteLine(message);
logList.Add(new LogMessage(message, LogType.Information));
noNotConfidence++;
}
}
count++;
}
var msg = processor.GetLastProcessingError();
if (msg != null)
{
var msgError = string.Format("the processing error because of {0}.", msg.MessageText());
Console.WriteLine(msgError);
logList.Add(new LogMessage(msgError, LogType.Error));
noError++;
}
message = string.Format("No. of Not Confidence {0} and No. of Error {1} and No. of Exported to DB {2}", noNotConfidence, noError, noSuccess);
logList.Add(new LogMessage(message, LogType.Information));
isRecognized = true;
}
finally
{
UnloadEngine(ref engine);
message = string.Format("Released FlexiCapture Engine for Recognition...");
Console.WriteLine(message);
logList.Add(new LogMessage(message, LogType.Information));
IsRecognized.Set(context, isRecognized);
NonInvoiceList.Set(context, nonInvoices);
Messages.Set(context, logList);
TotalExported.Set(context, noSuccess);
TotalError.Set(context, noError);
TotalNotConfidence.Set(context, noNotConfidence);
NotConfidenceList.Set(context, notConfidenceList);
}
}
