/// <summary>Runs the import job.</summary>
/// <param name="importJob">The import job.</param>
/// <param name="result">The result.</param>
/// <param name="scheduleHistoryItem">The schedule history item.</param>
public void Import(ExportImportJob importJob, ExportImportResult result, ScheduleHistoryItem scheduleHistoryItem)
{
scheduleHistoryItem.AddLogNote($"<br/><b>SITE IMPORT Started. JOB #{importJob.JobId}</b>");
this.timeoutSeconds = GetTimeoutPerSlot();
var importDto = JsonConvert.DeserializeObject <ImportDto>(importJob.JobObject);
if (importDto == null)
{
importJob.CompletedOnDate = DateUtils.GetDatabaseUtcTime();
importJob.JobStatus = JobStatus.Failed;
return;
}
var dbName = Path.Combine(ExportFolder, importJob.Directory, Constants.ExportDbName);
var finfo = new FileInfo(dbName);
if (!finfo.Exists)
{
DoUnPacking(importJob);
finfo = new FileInfo(dbName);
}
if (!finfo.Exists)
{
scheduleHistoryItem.AddLogNote("<br/>Import file not found. Name: " + dbName);
importJob.CompletedOnDate = DateUtils.GetDatabaseUtcTime();
importJob.JobStatus = JobStatus.Failed;
return;
}
using (var ctx = new ExportImportRepository(dbName))
{
var exportedDto = ctx.GetSingleItem <ExportDto>();
var exportVersion = new Version(exportedDto.SchemaVersion);
var importVersion = new Version(importDto.SchemaVersion);
if (importVersion < exportVersion)
{
importJob.CompletedOnDate = DateUtils.GetDatabaseUtcTime();
importJob.JobStatus = JobStatus.Failed;
scheduleHistoryItem.AddLogNote("Import NOT Possible");
var msg =
$"Exported version ({exportedDto.SchemaVersion}) is newer than import engine version ({importDto.SchemaVersion})";
result.AddSummary("Import NOT Possible", msg);
return;
}
var checkpoints = EntitiesController.Instance.GetJobChekpoints(importJob.JobId);
if (checkpoints.Count == 0)
{
result.AddSummary("Starting Importing Repository", finfo.Name);
result.AddSummary("Importing File Size", Util.FormatSize(finfo.Length));
CleanupDatabaseIfDirty(ctx);
}
else
{
result.AddSummary("Resuming Importing Repository", finfo.Name);
}
var implementors = Util.GetPortableImplementors().ToList();
var parentServices = implementors.Where(imp => string.IsNullOrEmpty(imp.ParentCategory)).ToList();
importJob.Name = exportedDto.ExportName;
importJob.Description = exportedDto.ExportDescription;
importJob.JobStatus = JobStatus.InProgress;
// there must be one parent implementor at least for this to work
implementors = implementors.Except(parentServices).ToList();
var nextLevelServices = new List <BasePortableService>();
var includedItems = GetAllCategoriesToInclude(exportedDto, implementors);
scheduleHistoryItem.AddLogNote($"<br/><b>SITE IMPORT Preparing Check Points. JOB #{importJob.JobId}: {importJob.Name}</b>");
this.PrepareCheckPoints(importJob.JobId, parentServices, implementors, includedItems, checkpoints);
var firstIteration = true;
AddJobToCache(importJob);
do
{
foreach (var service in parentServices.OrderBy(x => x.Priority))
{
if (importJob.IsCancelled)
{
importJob.JobStatus = JobStatus.Cancelled;
break;
}
if (implementors.Count > 0)
{
// collect children for next iteration
var children =
implementors.Where(imp => service.Category.Equals(imp.ParentCategory, IgnoreCaseComp));
nextLevelServices.AddRange(children);
implementors = implementors.Except(nextLevelServices).ToList();
}
if ((firstIteration && includedItems.Any(x => x.Equals(service.Category, IgnoreCaseComp))) ||
(!firstIteration && includedItems.Any(x => x.Equals(service.ParentCategory, IgnoreCaseComp))))
{
var serviceAssembly = service.GetType().Assembly.GetName().Name;
service.Result = result;
service.Repository = ctx;
service.CheckCancelled = CheckCancelledCallBack;
service.CheckPointStageCallback = this.CheckpointCallback;
service.CheckPoint = checkpoints.FirstOrDefault(cp => cp.Category == service.Category && cp.AssemblyName == serviceAssembly)
?? new ExportImportChekpoint
{
JobId = importJob.JobId,
AssemblyName = serviceAssembly,
Category = service.Category,
Progress = 0,
StartDate = DateUtils.GetDatabaseUtcTime(),
};
if (service.CheckPoint.StartDate == Null.NullDate)
{
service.CheckPoint.StartDate = DateUtils.GetDatabaseUtcTime();
}
this.CheckpointCallback(service);
try
{
service.ImportData(importJob, importDto);
}
finally
{
this.AddLogsToDatabase(importJob.JobId, result.CompleteLog);
}
scheduleHistoryItem.AddLogNote("<br/>Imported: " + service.Category);
}
}
firstIteration = false;
parentServices = new List <BasePortableService>(nextLevelServices);
nextLevelServices.Clear();
if (implementors.Count > 0 && parentServices.Count == 0)
{
// WARN: this is a case where there is a broken parent-children hierarchy
// and/or there are BasePortableService implementations without a known parent.
parentServices = implementors;
implementors.Clear();
scheduleHistoryItem.AddLogNote(
"<br/><b>Orphaned services:</b> " + string.Join(",", parentServices.Select(x => x.Category)));
}
}while (parentServices.Count > 0 && !this.TimeIsUp);
RemoveTokenFromCache(importJob);
if (this.TimeIsUp)
{
result.AddSummary(
$"Job time slot ({this.timeoutSeconds} sec) expired",
"Job will resume in the next scheduler iteration");
}
else if (importJob.JobStatus == JobStatus.InProgress)
{
importJob.JobStatus = JobStatus.Successful;
if (importDto.ExportDto.IncludeContent)
{
PagesExportService.ResetContentsFlag(ctx);
}
}
}
}
/// <summary>Runs the export job.</summary>
/// <param name="exportJob">The export job.</param>
/// <param name="result">The result.</param>
/// <param name="scheduleHistoryItem">The schedule history item.</param>
public void Export(ExportImportJob exportJob, ExportImportResult result, ScheduleHistoryItem scheduleHistoryItem)
{
var exportDto = JsonConvert.DeserializeObject <ExportDto>(exportJob.JobObject);
if (exportDto == null)
{
exportJob.CompletedOnDate = DateUtils.GetDatabaseUtcTime();
exportJob.JobStatus = JobStatus.Failed;
return;
}
this.timeoutSeconds = GetTimeoutPerSlot();
var dbName = Path.Combine(ExportFolder, exportJob.Directory, Constants.ExportDbName);
var finfo = new FileInfo(dbName);
dbName = finfo.FullName;
var checkpoints = EntitiesController.Instance.GetJobChekpoints(exportJob.JobId);
// Delete so we start a fresh export database; only if there is no previous checkpoint exists
if (checkpoints.Count == 0)
{
if (finfo.Directory != null && finfo.Directory.Exists)
{
finfo.Directory.Delete(true);
}
// Clear all the files in finfo.Directory. Create if doesn't exists.
finfo.Directory?.Create();
result.AddSummary("Starting Exporting Repository", finfo.Name);
}
else
{
if (finfo.Directory != null && finfo.Directory.Exists)
{
result.AddSummary("Resuming Exporting Repository", finfo.Name);
}
else
{
scheduleHistoryItem.AddLogNote("Resuming data not found.");
result.AddSummary("Resuming data not found.", finfo.Name);
return;
}
}
exportJob.JobStatus = JobStatus.InProgress;
// there must be one parent implementor at least for this to work
var implementors = Util.GetPortableImplementors().ToList();
var parentServices = implementors.Where(imp => string.IsNullOrEmpty(imp.ParentCategory)).ToList();
implementors = implementors.Except(parentServices).ToList();
var nextLevelServices = new List <BasePortableService>();
var includedItems = GetAllCategoriesToInclude(exportDto, implementors);
if (includedItems.Count == 0)
{
scheduleHistoryItem.AddLogNote("Export NOT Possible");
scheduleHistoryItem.AddLogNote("<br/>No items selected for exporting");
result.AddSummary("Export NOT Possible", "No items selected for exporting");
exportJob.CompletedOnDate = DateUtils.GetDatabaseUtcTime();
exportJob.JobStatus = JobStatus.Failed;
return;
}
scheduleHistoryItem.AddLogNote($"<br/><b>SITE EXPORT Preparing Check Points. JOB #{exportJob.JobId}: {exportJob.Name}</b>");
this.PrepareCheckPoints(exportJob.JobId, parentServices, implementors, includedItems, checkpoints);
scheduleHistoryItem.AddLogNote($"<br/><b>SITE EXPORT Started. JOB #{exportJob.JobId}: {exportJob.Name}</b>");
scheduleHistoryItem.AddLogNote($"<br/>Between [{exportDto.FromDateUtc ?? Constants.MinDbTime}] and [{exportDto.ToDateUtc:g}]");
var firstIteration = true;
AddJobToCache(exportJob);
using (var ctx = new ExportImportRepository(dbName))
{
ctx.AddSingleItem(exportDto);
do
{
foreach (var service in parentServices.OrderBy(x => x.Priority))
{
if (exportJob.IsCancelled)
{
exportJob.JobStatus = JobStatus.Cancelled;
break;
}
if (implementors.Count > 0)
{
// collect children for next iteration
var children =
implementors.Where(imp => service.Category.Equals(imp.ParentCategory, IgnoreCaseComp));
nextLevelServices.AddRange(children);
implementors = implementors.Except(nextLevelServices).ToList();
}
if ((firstIteration && includedItems.Any(x => x.Equals(service.Category, IgnoreCaseComp))) ||
(!firstIteration && includedItems.Any(x => x.Equals(service.ParentCategory, IgnoreCaseComp))))
{
var serviceAssembly = service.GetType().Assembly.GetName().Name;
service.Result = result;
service.Repository = ctx;
service.CheckCancelled = CheckCancelledCallBack;
service.CheckPointStageCallback = this.CheckpointCallback;
service.CheckPoint = checkpoints.FirstOrDefault(cp => cp.Category == service.Category && cp.AssemblyName == serviceAssembly);
if (service.CheckPoint == null)
{
service.CheckPoint = new ExportImportChekpoint
{
JobId = exportJob.JobId,
Category = service.Category,
AssemblyName = serviceAssembly,
StartDate = DateUtils.GetDatabaseUtcTime(),
};
// persist the record in db
this.CheckpointCallback(service);
}
else if (service.CheckPoint.StartDate == Null.NullDate)
{
service.CheckPoint.StartDate = DateUtils.GetDatabaseUtcTime();
}
try
{
service.ExportData(exportJob, exportDto);
}
finally
{
this.AddLogsToDatabase(exportJob.JobId, result.CompleteLog);
}
scheduleHistoryItem.AddLogNote("<br/>Exported: " + service.Category);
}
}
firstIteration = false;
parentServices = new List <BasePortableService>(nextLevelServices);
nextLevelServices.Clear();
if (implementors.Count > 0 && parentServices.Count == 0)
{
// WARN: this is a case where there is a broken parent-children hierarchy
// and/or there are BasePortableService implementations without a known parent.
parentServices = implementors;
implementors.Clear();
scheduleHistoryItem.AddLogNote(
"<br/><b>Orphaned services:</b> " + string.Join(",", parentServices.Select(x => x.Category)));
}
}while (parentServices.Count > 0 && !this.TimeIsUp);
RemoveTokenFromCache(exportJob);
}
if (this.TimeIsUp)
{
result.AddSummary(
$"Job time slot ({this.timeoutSeconds} sec) expired",
"Job will resume in the next scheduler iteration");
}
else if (exportJob.JobStatus == JobStatus.InProgress)
{
// Create Export Summary for manifest file.
var summary = new ImportExportSummary();
using (var ctx = new ExportImportRepository(dbName))
{
BaseController.BuildJobSummary(exportJob.Directory, ctx, summary);
}
DoPacking(exportJob, dbName);
// Complete the job.
exportJob.JobStatus = JobStatus.Successful;
SetLastJobStartTime(scheduleHistoryItem.ScheduleID, exportJob.CreatedOnDate);
var exportController = new ExportController();
var exportFileInfo = new ExportFileInfo
{
ExportPath = exportJob.Directory,
ExportSize = Util.FormatSize(GetExportSize(Path.Combine(ExportFolder, exportJob.Directory))),
};
summary.ExportFileInfo = exportFileInfo;
exportController.CreatePackageManifest(exportJob, exportFileInfo, summary);
}
}