public static void CreateRecordLogEntry <RecordLogType>(this DCDContext dc, int recordId, ImportLog importLog, RecordImportResult result, RecordImportOperation operation, string notes, Exception e) where RecordLogType : IDCDRecordImportLog, new()
{
try
{
RecordLogType log = new RecordLogType();
log.RecordId = recordId;
log.ImportLog = dc.ImportLogs.FirstOrDefault(w => w.Id == importLog.Id);
log.Result = result.ToString();
log.Operation = operation.ToString();
log.Notes = notes + (e != null ? $"[{e.ToString()}]" : string.Empty);
log.TimeStamp = DateTime.Now;
log.InsertRecordOnSubmit(dc);
dc.SubmitChanges();
}
catch (Exception ex)
{
DCDImportLogger.Error("Error writing log message to database", ex);
}
if (result == RecordImportResult.Failure)
{
if (e != null)
{
DCDImportLogger.Error("Error processing record " + recordId + ". " + notes, e);
}
else
{
DCDImportLogger.Error("Error processing record " + recordId + ". " + notes);
}
}
else
{
DCDImportLogger.Info("Successfully processed record " + recordId + ".");
}
}
public static void UpdateImportLogEntry(this DCDContext dc, ImportLog log)
{
try
{
var existingLog = dc.ImportLogs.FirstOrDefault(l => l.Id == log.Id);
if (existingLog == null)
{
dc.ImportLogs.InsertOnSubmit(log);
}
else
{
existingLog.FileName = log.FileName;
existingLog.ImportEnd = log.ImportEnd;
existingLog.ImportStart = log.ImportStart;
existingLog.Notes = log.Notes;
existingLog.Result = log.Result;
}
dc.SubmitChanges();
}
catch (Exception ex)
{
DCDImportLogger.Error("Error writing log message to database", ex);
}
}
public static void ThrowCriticalImportError(string error, Exception e, string processingFileName, ImportLog importLog, StringBuilder importLogNotes)
{
try
{
//Log the error in the database and the log
importLogNotes.AppendLine(error);
importLog.ImportEnd = DateTime.Now;
importLog.Result = ImportResult.Failure.ToString();
importLog.Notes = importLogNotes.ToString();
using (DCDContext dc = new DCDContext())
dc.UpdateImportLogEntry(importLog);
}
catch (Exception ex)
{
DCDImportLogger.Error("Error writing log message to database", ex);
}
if (e != null)
{
DCDImportLogger.Error(error, e);
}
else
{
DCDImportLogger.Error(error);
}
string subject = "Error processing Deals, Companies, Drugs import file. ";
StringBuilder sb = new StringBuilder();
sb.AppendLine("There was an error processing the import file: " + processingFileName);
sb.AppendLine();
sb.AppendLine("ERROR:");
sb.AppendLine();
sb.AppendLine(error);
if (e != null)
{
sb.AppendLine(e.Message);
if (e.InnerException != null)
{
sb.AppendLine(e.InnerException.Message);
}
}
if (!string.IsNullOrEmpty(importLogNotes.ToString()))
{
sb.AppendLine(importLogNotes.ToString());
}
SendNotification(subject, sb.ToString());
}
public void StartIfStartable()
{
try
{
if (EnableWatcher)
{
//This is to make sure, the DCD DB has started and the sitecore context is ready.
Thread t = new Thread(() => { Thread.Sleep(30000); CreateWatcher(); });
t.Start();
}
}
catch (Exception ex)
{
DCDImportLogger.Error("Error Starting Watcher", ex);
}
}
private void processFile(string path)
{
DCDImportLogger.Debug("Inner file processing has started for [" + path + "]");
string name = path.Substring(path.LastIndexOf("\\") + 1);
DCDImporter importer = new DCDImporter(path, name);
bool success = false;
success = importer.ProcessFile();
string oldArchivePath = DestinationDirectory + name;
string newArchiveName = name.Replace("Processing_", "");
newArchiveName = (success) ? "Valid_" + newArchiveName : "Invalid_" + newArchiveName;
string newArchivePath = DestinationDirectory + newArchiveName;
//Delete the processed file file
File.Delete(path);
//Rename the old archive file
File.Move(oldArchivePath, newArchivePath);
}
private void CreateWatcher()
{
DCDImportLogger.Debug("Attempting to start FileWatcher.");
//Create a new FileSystemWatcher.
FileSystemWatcher watcher = new FileSystemWatcher();
//If import directory is invalid do not continue, because the watcher depends on it and the ProcessingDirectory depends on it too.
if (Directory.Exists(ImportDirectory) == false)
{
return;
}
//Set the filter to only catch xml files.
watcher.Filter = "*.xml";
watcher.Changed += Watcher_FileChanged;
watcher.Path = ImportDirectory;
//Enable the FileSystemWatcher events.
watcher.EnableRaisingEvents = true;
watcher.NotifyFilter = NotifyFilters.LastWrite;
DCDImportLogger.Info("FileWatcher for DCD Import Started.");
//Process any files that are in the "processing" folder
//This is to re-process anything that was aborted during an app pool recycle
DCDImportLogger.Debug("Checking path [" + ProcessingDirectory + "]");
if (Directory.Exists(ProcessingDirectory))
{
string[] files = Directory.GetFiles(ProcessingDirectory, "*.xml");
DCDImportLogger.Debug("Found [" + files.Length + "] files with filter *.xml");
//process each file
foreach (string file in files)
{
Action <string> processAction = path => processFile(path);
DCDImportLogger.Debug("Processing existing file [" + file + "]");
processAction.BeginInvoke(file, null, null);
}
}
}
public static void CreateImportLogEntry(this DCDContext dc, DateTime start, string fileName, out ImportLog log)
{
try
{
log = new ImportLog();
log.ImportStart = start;
log.ImportEnd = start;
log.Result = ImportResult.InProgress.ToString();
log.FileName = fileName;
dc.ImportLogs.InsertOnSubmit(log);
dc.SubmitChanges();
//return log;
}
catch (Exception ex)
{
DCDImportLogger.Error("Error writing log message to database", ex);
log = null;
}
//return null;
}
private void Watcher_FileChanged(object sender, FileSystemEventArgs e)
{
string fullPath = e.FullPath;
string name = e.Name;
DCDImportLogger.Debug("FileChanged event raised for [" + name + "] found at [" + fullPath + "]");
if (!Directory.Exists(DestinationDirectory))
{
Directory.CreateDirectory(DestinationDirectory);
}
if (!Directory.Exists(ProcessingDirectory))
{
Directory.CreateDirectory(ProcessingDirectory);
}
//If the file does not exist, exit out
//This event can be fired when a file is deleted, so we need this check
if (!File.Exists(fullPath))
{
return;
}
DCDImportLogger.Info(string.Format("Processing File: " + name));
// check number one to make sure file is done uploading
if (XMLFileUtilities.IsOpen(fullPath))
{
bool bFileOpen = true;
while (bFileOpen)
{
if (XMLFileUtilities.IsOpen(fullPath))
{
DCDImportLogger.Info(string.Format("File is open, Waiting 2 seconds and looping..."));
Thread.Sleep(2000);
}
else
{
bFileOpen = false;
DCDImportLogger.Info(string.Format("File is available... continue processing."));
}
}
}
//clean bad characters from the file
string processingName = "Processing_" + name.Replace(".xml", "") + "_" + DateTime.Now.ToString("yyyyMMddHHmmssfff") + ".xml";
string processingFullPath = ProcessingDirectory + processingName;
StreamReader reader = new StreamReader(fullPath, Encoding.GetEncoding("Windows-1252"));
StreamWriter writer = new StreamWriter(processingFullPath, false, Encoding.UTF8);
bool first = true;
while (true)
{
string line = reader.ReadLine();
if (line == null)
{
break;
}
if (first && line.Contains("<?xml version"))
{
writer.WriteLine("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
first = false;
}
else
{
writer.WriteLine(XMLFileUtilities.SanitizeXmlString(line));
}
}
reader.Close();
writer.Close();
//Copy the original file to the archive folder
string processingDestPath = DestinationDirectory + processingName;
File.Copy(fullPath, processingDestPath);
//Delete the original file
File.Delete(fullPath);
processFile(processingFullPath);
}
public bool ProcessFile()
{
using (DCDContext dcCntxt = new DCDContext())
{
//Initialize the database log entry for this import
dcCntxt.CreateImportLogEntry(DateTime.Now, _fileName, out _importingLogRecord);
}
if (_importingLogRecord == null)
{
return(false);
}
try
{
//Set the DTD validation options
XmlReaderSettings settings = new XmlReaderSettings();
settings.ProhibitDtd = false;
settings.ValidationType = ValidationType.DTD;
settings.ValidationEventHandler += new ValidationEventHandler(XMLValidationHandler);
settings.CheckCharacters = false;
XmlReader reader = XmlReader.Create(_documentPath, settings);
IBIContent content;
try
{
//Deserialize the xml file
XmlSerializer serializer = new XmlSerializer(typeof(IBIContent));
content = (IBIContent)serializer.Deserialize(reader);
}
catch (Exception e)
{
reader.Close();
DCDImportLogger.ThrowCriticalImportError("DCD XML doc de-serialization failed! ", e, _fileName, _importingLogRecord, _importLogNotes);
return(false);
}
reader.Close();
//If there was an error validating the XML file with the appropriate DTD file
if (!documentIsValid)
{
DCDImportLogger.ThrowCriticalImportError("DCD XML doc validation failed! " + _importLogNotes, _fileName, _importingLogRecord, _importLogNotes);
return(false);
}
//Our XML file is good. Quickly validate that the number of records is correct.
int numRecords;
if (!int.TryParse(content.Feed.NumberOfRecords, out numRecords))
{
DCDImportLogger.ThrowCriticalImportError("Error processing DCD XML doc. Number of records value is not a valid integer.", _fileName, _importingLogRecord, _importLogNotes);
return(false);
}
if (numRecords != content.RecordSet.Count)
{
DCDImportLogger.ThrowCriticalImportError("Error processing DCD XML doc. Number of records value does not equal the record set size.", _fileName, _importingLogRecord, _importLogNotes);
return(false);
}
//We're past the critical error point. Time to process each record
foreach (Record record in content.RecordSet)
{
using (DCDContext dc = new DCDContext())
{
bool success = false;
if (content.type == IBIContentType.deal)
{
//_importingLogRecord = new Model.DCD.DCDManager().GetImportLogById(_importingLogRecord.Id);
success = dc.ProcessRecord <Deal, DealRecordImportLog>(record, _importingLogRecord);
if (success)
{
if (record.Command == CommandType.Delete)
{
_dealDeletes.Add(record.Identification.RecordNumber);
}
else
{
_dealUpdates.Add(dc.GetRecord <Deal>(record.Identification.RecordId));
}
}
}
else if (content.type == IBIContentType.company)
{
success = dc.ProcessRecord <Informa.Models.DCD.Company, CompanyRecordImportLog>(record, _importingLogRecord);
if (success)
{
//TODO: This is a hack for the thin records. Replace this when full records are imported
if (!string.IsNullOrEmpty(record.Content.InnerXml))
{
XDocument contentDoc = XDocument.Parse(record.Content.InnerXml);
List <CompanyPath> paths = new List <CompanyPath>();
foreach (XElement node in contentDoc.Descendants("CompanyPath"))
{
CompanyPath path = new CompanyPath();
XAttribute id = node.Attribute("id");
if (id != null)
{
path.id = id.Value;
}
path.Value = node.Value;
paths.Add(path);
}
success = dc.AddRelatedCompanies(record.Identification.RecordId, paths);
}
if (record.Command == CommandType.Delete)
{
_companyDeletes.Add(record.Identification.RecordNumber);
}
else
{
_companyUpdates.Add(dc.GetRecord <Informa.Models.DCD.Company>(record.Identification.RecordId));
}
}
}
else if (content.type == IBIContentType.drug)
{
success = dc.ProcessRecord <Drug, DrugRecordImportLog>(record, _importingLogRecord);
if (success)
{
if (record.Command == CommandType.Delete)
{
_drugDeletes.Add(record.Identification.RecordNumber);
}
else
{
_drugUpdates.Add(dc.GetRecord <Drug>(record.Identification.RecordId));
}
}
}
if (!success)
{
_hasSomeFailures = true;
}
}
}
//COVEO INDEX REBUILDING REMOVED
using (DCDContext dc = new DCDContext())
{
//If it is an "all update" note any discrepancies and send them to the distro
if (content.Feed.type == FeedType.ALLRECORDS)
{
IEnumerable <IDCD> discrepancies = null;
IEnumerable <IDCD> olderItems = null;
IEnumerable <IDCD> noContentItems = null;
string type = string.Empty;
if (content.type == IBIContentType.deal)
{
var imported = dc.DealRecordImportLogs.Where(d => d.ImportLog == _importingLogRecord);
discrepancies = dc.Deals.Where(d => !imported.Any(dl => dl.RecordId == d.RecordId)).Select(i => (IDCD)i);
olderItems = imported.Where(r => r.Notes == "Existing record has a newer Last Modified Date.").Select(r => (IDCD)dc.Deals.FirstOrDefault(c => c.RecordId == r.RecordId.Value));
noContentItems = imported.Where(r => r.Notes == "Record does not have any content.").Select(r => (IDCD)dc.Deals.FirstOrDefault(c => c.RecordId == r.RecordId.Value));
type = "deals";
}
else if (content.type == IBIContentType.company)
{
var imported = dc.CompanyRecordImportLogs.Where(c => c.ImportLog == _importingLogRecord);
discrepancies = dc.Companies.Where(c => !imported.Any(co => co.RecordId == c.RecordId)).Select(i => (IDCD)i);
olderItems = imported.Where(r => r.Notes == "Existing record has a newer Last Modified Date.").Select(r => (IDCD)dc.Companies.FirstOrDefault(c => c.RecordId == r.RecordId.Value));
noContentItems = imported.Where(r => r.Notes == "Record does not have any content.").Select(r => (IDCD)dc.Companies.FirstOrDefault(c => c.RecordId == r.RecordId.Value));
type = "companies";
}
else if (content.type == IBIContentType.drug)
{
var imported = dc.DrugRecordImportLogs.Where(d => d.ImportLog == _importingLogRecord);
discrepancies = dc.Drugs.Where(d => !imported.Any(dl => dl.RecordId == d.RecordId)).Select(i => (IDCD)i);
olderItems = imported.Where(r => r.Notes == "Existing record has a newer Last Modified Date.").Select(r => (IDCD)dc.Drugs.FirstOrDefault(c => c.RecordId == r.RecordId.Value));
noContentItems = imported.Where(r => r.Notes == "Record does not have any content.").Select(r => (IDCD)dc.Drugs.FirstOrDefault(c => c.RecordId == r.RecordId.Value));
type = "drugs";
}
//Send email if there are any discrepancies
bool hasDiscrepancies = (discrepancies != null && discrepancies.Count() > 0);
bool hasOlderItems = (olderItems != null && olderItems.Count() > 0);
bool hasNoContentItems = noContentItems != null && noContentItems.Count() > 0;
if (hasDiscrepancies || hasOlderItems || hasNoContentItems)
{
string subject = DCDConstants.BusinessAcronym + " discrepancy report for " + type + " import: " + _fileName;
StringBuilder sb = new StringBuilder();
sb.AppendLine("There were discrepancies when processing the full import file: " + _fileName);
sb.AppendLine();
if (hasDiscrepancies)
{
sb.AppendLine("These existing " + type + " were not altered by the import.");
sb.AppendLine();
foreach (var discrepancy in discrepancies)
{
sb.AppendLine(discrepancy.RecordNumber + " - " + discrepancy.Title);
}
}
if (hasOlderItems)
{
sb.AppendLine("These existing " + type + " have a newer modified date in the database.");
sb.AppendLine();
foreach (var olderItem in olderItems)
{
sb.AppendLine(olderItem.RecordNumber + " - " + olderItem.Title);
}
}
if (hasNoContentItems)
{
sb.AppendLine("These existing " + type + " do not have a <Content> node in their record.");
sb.AppendLine();
foreach (var noContentItem in noContentItems)
{
sb.AppendLine(noContentItem.RecordNumber + " - " + noContentItem.Title);
}
}
DCDImportLogger.SendNotification(subject, sb.ToString());
}
}
}
Event.RaiseEvent("dcdimport:end", content.type);
Sitecore.Eventing.EventManager.QueueEvent <DCDImportEndRemoteEvent>(new DCDImportEndRemoteEvent(content.type));
}
catch (Exception e)
{
DCDImportLogger.ThrowCriticalImportError("Error processing DCD XML doc.", e, _fileName, _importingLogRecord, _importLogNotes);
return(false);
}
//If we get this far, the import was a success
//Log import completion
_importingLogRecord.ImportEnd = DateTime.Now;
_importingLogRecord.Result = (_hasSomeFailures) ? ImportResult.PartialSuccess.ToString() : ImportResult.Success.ToString();
_importingLogRecord.Notes = _importLogNotes.ToString();
using (DCDContext dcdContext = new DCDContext())
dcdContext.UpdateImportLogEntry(_importingLogRecord);
return(true);
}
