public static void ImportText(string text, string sourceFileName)
{
var parseResult = new ExtractParseResult(text, Constants.DataObjectMap);
var extract = parseResult.ToExtract(sourceFileName);
var extractItems = parseResult.ToExtractItems(extract.ExtractId, Constants.DataObjectMap, Constants.RelationshipMap, Constants.MultiRelationshipMap);
var extractStaging = parseResult.ToExtractPersonStagings(extract.ExtractId);
var extractErrors = parseResult.ToExtractErrors(extract.ExtractId);
using (var conn = new SqlConnection(CprBroker.Config.Properties.Settings.Default.CprBrokerConnectionString))
{
conn.Open();
using (var trans = conn.BeginTransaction(System.Data.IsolationLevel.ReadUncommitted))
{
conn.BulkInsertAll <Extract>(new Extract[] { extract }, trans);
conn.BulkInsertAll <ExtractItem>(extractItems, trans);
conn.BulkInsertAll <ExtractPersonStaging>(extractStaging, trans);
conn.BulkInsertAll <ExtractError>(extractErrors, trans);
trans.Commit();
}
using (var dataContext = new ExtractDataContext())
{
extract = dataContext.Extracts.Where(ex => ex.ExtractId == extract.ExtractId).First();
extract.Ready = true;
dataContext.SubmitChanges();
}
}
}
public static void ConvertPersons(int batchSize)
{
Admin.LogFormattedSuccess("ExtractManager.ConvertPersons() started, batch size <{0}>", batchSize);
List <Guid> succeeded = new List <Guid>(), failed = new List <Guid>();
using (var dataContext = new ExtractDataContext())
{
var dataLoadOptions = new System.Data.Linq.DataLoadOptions();
dataLoadOptions.LoadWith <ExtractItem>(ei => ei.Extract);
dataContext.LoadOptions = dataLoadOptions;
var persons = ExtractPersonStaging.SelectTop(dataContext, batchSize);
Admin.LogFormattedSuccess("ExtractManager.ConvertPersons() - <{0}> persons found", persons.Length);
var conversionExtracts = ExtractConversion.CreateFromPersonStagings(persons);
ExtractConversion.FillExtractItems(conversionExtracts, dataContext);
// Preload UUIDs
var pnrs = ExtractConversion.AllPNRs(conversionExtracts);
var cache = new UuidCache();
cache.FillCache(pnrs);
int personIndex = 0;
foreach (var personGroup in conversionExtracts)
{
Admin.LogFormattedSuccess("ExtractManager.ConvertPersons() - converting persons from extract <{0}>", personGroup.Extract.ExtractId);
foreach (var person in personGroup.Persons)
{
personIndex++;
try
{
Admin.LogFormattedSuccess("ExtractManager.ConvertPersons() - processing PNR <{0}>, person <{1}> of <{2}>", person.ExtractPersonStaging.PNR, personIndex, persons.Length);
var uuid = cache.GetUuid(person.ExtractPersonStaging.PNR);
var response = Extract.ToIndividualResponseType(person.ExtractPersonStaging.Extract, person.ExtractItems.AsQueryable(), Constants.DataObjectMap);
var oioPerson = response.ToRegistreringType1(cache.GetUuid);
var personIdentifier = new Schemas.PersonIdentifier()
{
CprNumber = person.ExtractPersonStaging.PNR, UUID = uuid
};
UpdateDatabase.UpdatePersonRegistration(personIdentifier, oioPerson);
succeeded.Add(person.ExtractPersonStaging.ExtractPersonStagingId);
Admin.LogFormattedSuccess("ExtractManager.ConvertPersons() - finished PNR <{0}>, person <{1}> of <{2}>", person.ExtractPersonStaging.PNR, personIndex, persons.Length);
}
catch (Exception ex)
{
failed.Add(person.ExtractPersonStaging.ExtractPersonStagingId);
Admin.LogException(ex);
}
}
}
}
// Delete the staging tables from a new data context to maximize performance
using (var dataContext = new ExtractDataContext())
{
var persons = dataContext.ExtractPersonStagings.Where(ep => succeeded.Contains(ep.ExtractPersonStagingId));
dataContext.ExtractPersonStagings.DeleteAllOnSubmit(persons);
dataContext.SubmitChanges();
}
Admin.LogFormattedSuccess("ExtractManager.ConvertPersons() ending, batch size: <{0}>, succeeded: <{1}>, failed: <{2}>", batchSize, succeeded.Count, failed.Count);
}
public static void ImportFileInSteps(Stream dataStream, string path, int batchSize, Encoding encoding)
{
var allPnrs = new List <string>();
using (var file = new StreamReader(dataStream, encoding))
{
var extractResult = new ExtractParseResult();
long totalReadLinesCount = 0;
using (var conn = new SqlConnection(CprBroker.Config.Properties.Settings.Default.CprBrokerConnectionString))
{
conn.Open();
using (var dataContext = new ExtractDataContext(conn))
{
// Find existing extract or create a new one
var extract = dataContext.Extracts.Where(e => e.Filename == path && e.ProcessedLines != null && !e.Ready).OrderByDescending(e => e.ImportDate).FirstOrDefault();
if (extract == null)
{
extract = extractResult.ToExtract(path, false, 0);
Admin.LogFormattedSuccess("Creating new extract <{0}>", extract.ExtractId);
dataContext.Extracts.InsertOnSubmit(extract);
dataContext.SubmitChanges();
}
else
{
Admin.LogFormattedSuccess("Incomplete extract found <{0}>, resuming", extract.ExtractId);
}
// Start reading the file
while (!file.EndOfStream)
{
var wrappers = CompositeWrapper.Parse(file, Constants.DataObjectMap, batchSize);
var batchReadLinesCount = wrappers.Count;
totalReadLinesCount += batchReadLinesCount;
Admin.LogFormattedSuccess("Batch read, records found <{0}>, total so far <{1}>", batchReadLinesCount, totalReadLinesCount);
using (var trans = conn.BeginTransaction(System.Data.IsolationLevel.ReadUncommitted))
{
dataContext.Transaction = trans;
extractResult.ClearArrays();
var uninsertedLinesCount = totalReadLinesCount - extract.ProcessedLines.Value;
if (uninsertedLinesCount > 0)
{
var linesToSkip = wrappers.Count - (int)uninsertedLinesCount;
if (linesToSkip > 0)
{
Admin.LogFormattedSuccess("Unaligned batch sizes, skipping <{0}> lines", linesToSkip);
wrappers = wrappers.Skip(linesToSkip).ToList();
}
extractResult.AddLines(wrappers);
// Set start record
if (string.IsNullOrEmpty(extract.StartRecord) && extractResult.StartWrapper != null)
{
extract.StartRecord = extractResult.StartWrapper.Contents;
extract.ExtractDate = extractResult.StartWrapper.ProductionDate.Value;
}
// Child records
conn.BulkInsertAll <ExtractItem>(extractResult.ToExtractItems(extract.ExtractId, Constants.DataObjectMap, Constants.RelationshipMap, Constants.MultiRelationshipMap), trans);
conn.BulkInsertAll <ExtractError>(extractResult.ToExtractErrors(extract.ExtractId), trans);
// TODO: (Extract) In case some records have been skipped in a previous import attempt, make sure that allPnrs contains their PNR's
conn.BulkInsertAll <ExtractPersonStaging>(extractResult.ToExtractPersonStagings(extract.ExtractId, allPnrs), trans);
// Update counts
extract.ProcessedLines = totalReadLinesCount;
// End record and mark as ready
if (extractResult.EndLine != null)
{
extract.EndRecord = extractResult.EndLine.Contents;
extract.Ready = true;
Admin.LogFormattedSuccess("End record added");
}
dataContext.SubmitChanges();
Admin.LogFormattedSuccess("Batch committed");
}
else
{
Admin.LogFormattedSuccess("Batch already inserted, skipping");
}
trans.Commit();
}
}
}
}
}
}