public static void ImportText(string text, string sourceFileName)
{
var parseResult = new ExtractParseResult(text, Constants.DataObjectMap);
var extract = parseResult.ToExtract(sourceFileName);
var extractItems = parseResult.ToExtractItems(extract.ExtractId, Constants.DataObjectMap, Constants.RelationshipMap, Constants.MultiRelationshipMap);
var extractStaging = parseResult.ToExtractPersonStagings(extract.ExtractId);
var extractErrors = parseResult.ToExtractErrors(extract.ExtractId);
using (var conn = new SqlConnection(CprBroker.Config.Properties.Settings.Default.CprBrokerConnectionString))
{
conn.Open();
using (var trans = conn.BeginTransaction(System.Data.IsolationLevel.ReadUncommitted))
{
conn.BulkInsertAll <Extract>(new Extract[] { extract }, trans);
conn.BulkInsertAll <ExtractItem>(extractItems, trans);
conn.BulkInsertAll <ExtractPersonStaging>(extractStaging, trans);
conn.BulkInsertAll <ExtractError>(extractErrors, trans);
trans.Commit();
}
using (var dataContext = new ExtractDataContext())
{
extract = dataContext.Extracts.Where(ex => ex.ExtractId == extract.ExtractId).First();
extract.Ready = true;
dataContext.SubmitChanges();
}
}
}
public static void ImportFileInSteps(Stream dataStream, string path, int batchSize, Encoding encoding)
{
var allPnrs = new List <string>();
using (var file = new StreamReader(dataStream, encoding))
{
var extractResult = new ExtractParseResult();
long totalReadLinesCount = 0;
using (var conn = new SqlConnection(CprBroker.Config.Properties.Settings.Default.CprBrokerConnectionString))
{
conn.Open();
using (var dataContext = new ExtractDataContext(conn))
{
// Find existing extract or create a new one
var extract = dataContext.Extracts.Where(e => e.Filename == path && e.ProcessedLines != null && !e.Ready).OrderByDescending(e => e.ImportDate).FirstOrDefault();
if (extract == null)
{
extract = extractResult.ToExtract(path, false, 0);
Admin.LogFormattedSuccess("Creating new extract <{0}>", extract.ExtractId);
dataContext.Extracts.InsertOnSubmit(extract);
dataContext.SubmitChanges();
}
else
{
Admin.LogFormattedSuccess("Incomplete extract found <{0}>, resuming", extract.ExtractId);
}
// Start reading the file
while (!file.EndOfStream)
{
var wrappers = CompositeWrapper.Parse(file, Constants.DataObjectMap, batchSize);
var batchReadLinesCount = wrappers.Count;
totalReadLinesCount += batchReadLinesCount;
Admin.LogFormattedSuccess("Batch read, records found <{0}>, total so far <{1}>", batchReadLinesCount, totalReadLinesCount);
using (var trans = conn.BeginTransaction(System.Data.IsolationLevel.ReadUncommitted))
{
dataContext.Transaction = trans;
extractResult.ClearArrays();
var uninsertedLinesCount = totalReadLinesCount - extract.ProcessedLines.Value;
if (uninsertedLinesCount > 0)
{
var linesToSkip = wrappers.Count - (int)uninsertedLinesCount;
if (linesToSkip > 0)
{
Admin.LogFormattedSuccess("Unaligned batch sizes, skipping <{0}> lines", linesToSkip);
wrappers = wrappers.Skip(linesToSkip).ToList();
}
extractResult.AddLines(wrappers);
// Set start record
if (string.IsNullOrEmpty(extract.StartRecord) && extractResult.StartWrapper != null)
{
extract.StartRecord = extractResult.StartWrapper.Contents;
extract.ExtractDate = extractResult.StartWrapper.ProductionDate.Value;
}
// Child records
conn.BulkInsertAll <ExtractItem>(extractResult.ToExtractItems(extract.ExtractId, Constants.DataObjectMap, Constants.RelationshipMap, Constants.MultiRelationshipMap), trans);
conn.BulkInsertAll <ExtractError>(extractResult.ToExtractErrors(extract.ExtractId), trans);
// TODO: (Extract) In case some records have been skipped in a previous import attempt, make sure that allPnrs contains their PNR's
conn.BulkInsertAll <ExtractPersonStaging>(extractResult.ToExtractPersonStagings(extract.ExtractId, allPnrs), trans);
// Update counts
extract.ProcessedLines = totalReadLinesCount;
// End record and mark as ready
if (extractResult.EndLine != null)
{
extract.EndRecord = extractResult.EndLine.Contents;
extract.Ready = true;
Admin.LogFormattedSuccess("End record added");
}
dataContext.SubmitChanges();
Admin.LogFormattedSuccess("Batch committed");
}
else
{
Admin.LogFormattedSuccess("Batch already inserted, skipping");
}
trans.Commit();
}
}
}
}
}
}
