Ejemplo n.º 1
0
        public ExtractParseResult(string batchFileText, Dictionary <string, Type> dataObjectMap)
            : this()
        {
            var wrappers = CompositeWrapper.Parse(batchFileText, dataObjectMap);

            AddLines(wrappers);
        }
Ejemplo n.º 2
0
        public static void ImportFileInSteps(Stream dataStream, string path, int batchSize, Encoding encoding)
        {
            var allPnrs = new List <string>();

            using (var file = new StreamReader(dataStream, encoding))
            {
                var extractResult = new ExtractParseResult();

                long totalReadLinesCount = 0;
                using (var conn = new SqlConnection(CprBroker.Config.Properties.Settings.Default.CprBrokerConnectionString))
                {
                    conn.Open();
                    using (var dataContext = new ExtractDataContext(conn))
                    {
                        // Find existing extract or create a new one
                        var extract = dataContext.Extracts.Where(e => e.Filename == path && e.ProcessedLines != null && !e.Ready).OrderByDescending(e => e.ImportDate).FirstOrDefault();
                        if (extract == null)
                        {
                            extract = extractResult.ToExtract(path, false, 0);
                            Admin.LogFormattedSuccess("Creating new extract <{0}>", extract.ExtractId);
                            dataContext.Extracts.InsertOnSubmit(extract);
                            dataContext.SubmitChanges();
                        }
                        else
                        {
                            Admin.LogFormattedSuccess("Incomplete extract found <{0}>, resuming", extract.ExtractId);
                        }

                        // Start reading the file
                        while (!file.EndOfStream)
                        {
                            var wrappers            = CompositeWrapper.Parse(file, Constants.DataObjectMap, batchSize);
                            var batchReadLinesCount = wrappers.Count;
                            totalReadLinesCount += batchReadLinesCount;

                            Admin.LogFormattedSuccess("Batch read, records found <{0}>, total so far <{1}>", batchReadLinesCount, totalReadLinesCount);

                            using (var trans = conn.BeginTransaction(System.Data.IsolationLevel.ReadUncommitted))
                            {
                                dataContext.Transaction = trans;
                                extractResult.ClearArrays();
                                var uninsertedLinesCount = totalReadLinesCount - extract.ProcessedLines.Value;

                                if (uninsertedLinesCount > 0)
                                {
                                    var linesToSkip = wrappers.Count - (int)uninsertedLinesCount;
                                    if (linesToSkip > 0)
                                    {
                                        Admin.LogFormattedSuccess("Unaligned batch sizes, skipping <{0}> lines", linesToSkip);
                                        wrappers = wrappers.Skip(linesToSkip).ToList();
                                    }

                                    extractResult.AddLines(wrappers);

                                    // Set start record
                                    if (string.IsNullOrEmpty(extract.StartRecord) && extractResult.StartWrapper != null)
                                    {
                                        extract.StartRecord = extractResult.StartWrapper.Contents;
                                        extract.ExtractDate = extractResult.StartWrapper.ProductionDate.Value;
                                    }

                                    // Child records
                                    conn.BulkInsertAll <ExtractItem>(extractResult.ToExtractItems(extract.ExtractId, Constants.DataObjectMap, Constants.RelationshipMap, Constants.MultiRelationshipMap), trans);
                                    conn.BulkInsertAll <ExtractError>(extractResult.ToExtractErrors(extract.ExtractId), trans);
                                    // TODO: (Extract) In case some records have been skipped in a previous import attempt, make sure that allPnrs contains their PNR's
                                    conn.BulkInsertAll <ExtractPersonStaging>(extractResult.ToExtractPersonStagings(extract.ExtractId, allPnrs), trans);

                                    // Update counts
                                    extract.ProcessedLines = totalReadLinesCount;

                                    // End record and mark as ready
                                    if (extractResult.EndLine != null)
                                    {
                                        extract.EndRecord = extractResult.EndLine.Contents;
                                        extract.Ready     = true;
                                        Admin.LogFormattedSuccess("End record added");
                                    }
                                    dataContext.SubmitChanges();
                                    Admin.LogFormattedSuccess("Batch committed");
                                }
                                else
                                {
                                    Admin.LogFormattedSuccess("Batch already inserted, skipping");
                                }
                                trans.Commit();
                            }
                        }
                    }
                }
            }
        }