public FiltreretOejebliksbilledeType ReadPeriod(DateTime fromDate, DateTime toDate, PersonIdentifier pId, Func <string, Guid> cpr2UuidFunc) { using (var dataContext = new ExtractDataContext()) { var loadOptions = new System.Data.Linq.DataLoadOptions(); loadOptions.LoadWith <ExtractItem>(ei => ei.Extract); dataContext.LoadOptions = loadOptions; var individualResponses = Extract.GetPersonFromAllExtracts(pId.CprNumber, dataContext.ExtractItems, Constants.DataObjectMap); var fullResp = new IndividualHistoryResponseType(pId, individualResponses); var pnrs = fullResp.AllRelationPNRs; var uuids = CprBroker.Data.Part.PersonMapping.AssignGuids(pnrs); Func <string, Guid> uuidGetter = (pnr) => { Guid?uuid = null; var pnrIndex = Array.IndexOf <string>(pnrs, pnr); if (pnrIndex != -1) { uuid = uuids[pnrIndex]; } return(uuid.HasValue ? uuid.Value : cpr2UuidFunc(pnr)); }; return(fullResp .ToFiltreretOejebliksbilledeType(uuidGetter) .Filter(VirkningType.Create(fromDate, toDate))); } }
public static void FillExtractItems(ExtractConversion[] extractConversions, ExtractDataContext dataContext) { foreach (var extractConversion in extractConversions) { extractConversion.FillExtractItems(dataContext); } }
public static void ImportText(string text, string sourceFileName) { var parseResult = new ExtractParseResult(text, Constants.DataObjectMap); var extract = parseResult.ToExtract(sourceFileName); var extractItems = parseResult.ToExtractItems(extract.ExtractId, Constants.DataObjectMap, Constants.RelationshipMap, Constants.MultiRelationshipMap); var extractStaging = parseResult.ToExtractPersonStagings(extract.ExtractId); var extractErrors = parseResult.ToExtractErrors(extract.ExtractId); using (var conn = new SqlConnection(CprBroker.Config.Properties.Settings.Default.CprBrokerConnectionString)) { conn.Open(); using (var trans = conn.BeginTransaction(System.Data.IsolationLevel.ReadUncommitted)) { conn.BulkInsertAll <Extract>(new Extract[] { extract }, trans); conn.BulkInsertAll <ExtractItem>(extractItems, trans); conn.BulkInsertAll <ExtractPersonStaging>(extractStaging, trans); conn.BulkInsertAll <ExtractError>(extractErrors, trans); trans.Commit(); } using (var dataContext = new ExtractDataContext()) { extract = dataContext.Extracts.Where(ex => ex.ExtractId == extract.ExtractId).First(); extract.Ready = true; dataContext.SubmitChanges(); } } }
public static IndividualResponseType GetPerson(string pnr) { using (var dataContext = new ExtractDataContext()) { return(Extract.GetPersonFromLatestExtract(pnr, dataContext.ExtractItems, Constants.DataObjectMap)); } }
public static ExtractPersonStaging[] SelectTop(ExtractDataContext dataContext, int batchSize) { return(dataContext.ExtractPersonStagings .Where(eps => eps.Extract.Ready) .OrderBy(ep => ep.Extract.ExtractDate) .Take(batchSize) .ToArray()); }
public void FillExtractItems(ExtractDataContext dataContext) { var groupPnrs = this.Persons.Select(p => p.ExtractPersonStaging.PNR).ToArray(); var groupExtractItems = dataContext.ExtractItems.Where(ei => groupPnrs.Contains(ei.PNR) && ei.ExtractId == this.Extract.ExtractId).ToArray(); foreach (var person in this.Persons) { person.ExtractItems = groupExtractItems.Where(ei => ei.PNR == person.ExtractPersonStaging.PNR).ToArray(); } }
public static void ConvertPersons(int batchSize) { Admin.LogFormattedSuccess("ExtractManager.ConvertPersons() started, batch size <{0}>", batchSize); List <Guid> succeeded = new List <Guid>(), failed = new List <Guid>(); using (var dataContext = new ExtractDataContext()) { var dataLoadOptions = new System.Data.Linq.DataLoadOptions(); dataLoadOptions.LoadWith <ExtractItem>(ei => ei.Extract); dataContext.LoadOptions = dataLoadOptions; var persons = ExtractPersonStaging.SelectTop(dataContext, batchSize); Admin.LogFormattedSuccess("ExtractManager.ConvertPersons() - <{0}> persons found", persons.Length); var conversionExtracts = ExtractConversion.CreateFromPersonStagings(persons); ExtractConversion.FillExtractItems(conversionExtracts, dataContext); // Preload UUIDs var pnrs = ExtractConversion.AllPNRs(conversionExtracts); var cache = new UuidCache(); cache.FillCache(pnrs); int personIndex = 0; foreach (var personGroup in conversionExtracts) { Admin.LogFormattedSuccess("ExtractManager.ConvertPersons() - converting persons from extract <{0}>", personGroup.Extract.ExtractId); foreach (var person in personGroup.Persons) { personIndex++; try { Admin.LogFormattedSuccess("ExtractManager.ConvertPersons() - processing PNR <{0}>, person <{1}> of <{2}>", person.ExtractPersonStaging.PNR, personIndex, persons.Length); var uuid = cache.GetUuid(person.ExtractPersonStaging.PNR); var response = Extract.ToIndividualResponseType(person.ExtractPersonStaging.Extract, person.ExtractItems.AsQueryable(), Constants.DataObjectMap); var oioPerson = response.ToRegistreringType1(cache.GetUuid); var personIdentifier = new Schemas.PersonIdentifier() { CprNumber = person.ExtractPersonStaging.PNR, UUID = uuid }; UpdateDatabase.UpdatePersonRegistration(personIdentifier, oioPerson); succeeded.Add(person.ExtractPersonStaging.ExtractPersonStagingId); Admin.LogFormattedSuccess("ExtractManager.ConvertPersons() - finished PNR <{0}>, person <{1}> of <{2}>", person.ExtractPersonStaging.PNR, personIndex, persons.Length); } catch (Exception ex) { failed.Add(person.ExtractPersonStaging.ExtractPersonStagingId); Admin.LogException(ex); } } } } // Delete the staging tables from a new data context to maximize performance using (var dataContext = new ExtractDataContext()) { var persons = dataContext.ExtractPersonStagings.Where(ep => succeeded.Contains(ep.ExtractPersonStagingId)); dataContext.ExtractPersonStagings.DeleteAllOnSubmit(persons); dataContext.SubmitChanges(); } Admin.LogFormattedSuccess("ExtractManager.ConvertPersons() ending, batch size: <{0}>, succeeded: <{1}>, failed: <{2}>", batchSize, succeeded.Count, failed.Count); }
public static void ImportFileInSteps(Stream dataStream, string path, int batchSize, Encoding encoding) { var allPnrs = new List <string>(); using (var file = new StreamReader(dataStream, encoding)) { var extractResult = new ExtractParseResult(); long totalReadLinesCount = 0; using (var conn = new SqlConnection(CprBroker.Config.Properties.Settings.Default.CprBrokerConnectionString)) { conn.Open(); using (var dataContext = new ExtractDataContext(conn)) { // Find existing extract or create a new one var extract = dataContext.Extracts.Where(e => e.Filename == path && e.ProcessedLines != null && !e.Ready).OrderByDescending(e => e.ImportDate).FirstOrDefault(); if (extract == null) { extract = extractResult.ToExtract(path, false, 0); Admin.LogFormattedSuccess("Creating new extract <{0}>", extract.ExtractId); dataContext.Extracts.InsertOnSubmit(extract); dataContext.SubmitChanges(); } else { Admin.LogFormattedSuccess("Incomplete extract found <{0}>, resuming", extract.ExtractId); } // Start reading the file while (!file.EndOfStream) { var wrappers = CompositeWrapper.Parse(file, Constants.DataObjectMap, batchSize); var batchReadLinesCount = wrappers.Count; totalReadLinesCount += batchReadLinesCount; Admin.LogFormattedSuccess("Batch read, records found <{0}>, total so far <{1}>", batchReadLinesCount, totalReadLinesCount); using (var trans = conn.BeginTransaction(System.Data.IsolationLevel.ReadUncommitted)) { dataContext.Transaction = trans; extractResult.ClearArrays(); var uninsertedLinesCount = totalReadLinesCount - extract.ProcessedLines.Value; if (uninsertedLinesCount > 0) { var linesToSkip = wrappers.Count - (int)uninsertedLinesCount; if (linesToSkip > 0) { Admin.LogFormattedSuccess("Unaligned batch sizes, skipping <{0}> lines", linesToSkip); wrappers = wrappers.Skip(linesToSkip).ToList(); } extractResult.AddLines(wrappers); // Set start record if (string.IsNullOrEmpty(extract.StartRecord) && extractResult.StartWrapper != null) { extract.StartRecord = extractResult.StartWrapper.Contents; extract.ExtractDate = extractResult.StartWrapper.ProductionDate.Value; } // Child records conn.BulkInsertAll <ExtractItem>(extractResult.ToExtractItems(extract.ExtractId, Constants.DataObjectMap, Constants.RelationshipMap, Constants.MultiRelationshipMap), trans); conn.BulkInsertAll <ExtractError>(extractResult.ToExtractErrors(extract.ExtractId), trans); // TODO: (Extract) In case some records have been skipped in a previous import attempt, make sure that allPnrs contains their PNR's conn.BulkInsertAll <ExtractPersonStaging>(extractResult.ToExtractPersonStagings(extract.ExtractId, allPnrs), trans); // Update counts extract.ProcessedLines = totalReadLinesCount; // End record and mark as ready if (extractResult.EndLine != null) { extract.EndRecord = extractResult.EndLine.Contents; extract.Ready = true; Admin.LogFormattedSuccess("End record added"); } dataContext.SubmitChanges(); Admin.LogFormattedSuccess("Batch committed"); } else { Admin.LogFormattedSuccess("Batch already inserted, skipping"); } trans.Commit(); } } } } } }