static void Main(string[] args) { bool isAutomaticMode = false; bool isAPIImportMode = false; LogEvent("Starting Import:", true); if (args.Length > 0) { LogEvent("Arguments supplied: "); foreach (var arg in args) { LogEvent("arg: " + arg); if (arg == "-auto") isAutomaticMode = true; if (arg == "-api") isAPIImportMode = true; } } else { LogEvent("No Arguments supplied."); } if (isAutomaticMode) { bool actionPerformed = false; ActionType mode = ActionType.DataImport; if (mode == ActionType.DataImport && isAPIImportMode == true) { ExportType exportType = ExportType.API; ImportManager importManager = new ImportManager(Settings.Default.Import_DataFolder); LogEvent("Performing Import, Publishing via API: "+DateTime.UtcNow.ToShortTimeString()); importManager.PerformImportProcessing(exportType, Settings.Default.Import_DataFolder, Settings.Default.OCM_API_Identitifer, Settings.Default.OCM_API_SessionToken, true); LogEvent("Import Processed. Exiting. " + DateTime.UtcNow.ToShortTimeString()); actionPerformed = true; } if (mode == ActionType.NetworkServices) { //network service polling //OCM.API.NetworkServices.ServiceManager serviceManager = new OCM.API.NetworkServices.ServiceManager(); //serviceManager.Test(OCM.API.NetworkServices.ServiceProvider.CoulombChargePoint); } if (!actionPerformed) { LogEvent("Nothing to do. Exiting. " + DateTime.UtcNow.ToShortTimeString()); } } else { //show UI Application.Run(new AppMain()); } }
public void ExportImportTest() { // Arrange var exportContext = new ExportImportContext(); var autocompletesCount = 25; var relatedQueriesCount = 49; exportContext.SetupAutocompletes(autocompletesCount); exportContext.SetupRelatedQueries(relatedQueriesCount); var exporters = new List<IExporter> { exportContext.AutocompleteExporter, exportContext.RelatedQueryExporter }; var exportManager = new ExportManager(exporters, exportContext.SiteIdentityLoaderMock.Object, exportContext.Settings); var importContext = new ExportImportContext(); importContext.SetupAutocompletes(0); importContext.SetupRelatedQueries(0); var importers = new List<IImporter> { importContext.AutocompleteImporter, importContext.RelatedQueryImporter }; var importManager = new ImportManager(importers); // Act MemoryStream exportStream = new MemoryStream(); exportManager.ExportToStream(new List<string> { exportContext.AutocompleteExporter.EntityKey, exportContext.RelatedQueryExporter.EntityKey }, Helpers.AllSitesId, Helpers.AllLanguages, exportStream); var importStream = new MemoryStream(exportStream.ToArray()); importManager.ImportFromStream("MySite", importStream); // Assert Assert.Equal(autocompletesCount, importContext.Autocompletes.Count); Assert.Equal(relatedQueriesCount, importContext.RelatedQueries.Count); for (int i = 0; i < autocompletesCount; i++) { Assert.Equal(exportContext.Autocompletes[i].Id, importContext.Autocompletes[i].Id); Assert.Equal(exportContext.Autocompletes[i].Query, importContext.Autocompletes[i].Query); Assert.Equal(exportContext.Autocompletes[i].Priority, importContext.Autocompletes[i].Priority); Assert.NotEqual(exportContext.Autocompletes[i].Tags, importContext.Autocompletes[i].Tags); Assert.Equal("siteid:MySite", importContext.Autocompletes[i].Tags.First()); } for (int i = 0; i < relatedQueriesCount; i++) { Assert.Equal(exportContext.RelatedQueries[i].Id, importContext.RelatedQueries[i].Id); Assert.Equal(exportContext.RelatedQueries[i].Query, importContext.RelatedQueries[i].Query); Assert.Equal(exportContext.RelatedQueries[i].Priority, importContext.RelatedQueries[i].Priority); Assert.Equal(exportContext.RelatedQueries[i].Suggestion, importContext.RelatedQueries[i].Suggestion); Assert.NotEqual(exportContext.RelatedQueries[i].Tags, importContext.RelatedQueries[i].Tags); Assert.Equal("siteid:MySite", importContext.RelatedQueries[i].Tags.First()); } }
private ImportDataManager(string sourceFolder, string importedFolder, string importErrorsFolder, string logsFolder, ICountryService countryService, IRegionService regionService, IStoreService storeService, IEmployeeService employeeService) { _SourceFolder = sourceFolder; _ImportedFolder = importedFolder; _ImportErrorsFolder = importErrorsFolder; _LogsFolder = logsFolder; _CountryService = countryService; _RegionService = regionService; _StoreService = storeService; _EmployeeService = employeeService; _CIComparer = new CIComparer(); _LogFileNames = GetImportLogFileNames(); _ImportManager = CreateImportManager(); }
/// <summary> /// Gets the import manager. /// </summary> /// <returns></returns> private ImportManager GetImportManager() { ImportManager importManager = Page.Session["importManager"] as ImportManager; if (importManager == null) importManager = new ImportManager(typeof(ILead)); if (importManager != null) { ImportCSVOptions options = ((ImportCSVReader)importManager.SourceReader).Options; if (rdbOther.Checked) { if (!String.IsNullOrEmpty(txtOtherDelimiter.Text)) options.Delimiter = Convert.ToChar(txtOtherDelimiter.Text); } else { options.Delimiter = GetSelectedDelimiter(); } options.Qualifier = !lbxQualifier.SelectedValue.Equals("None") ? Convert.ToChar(lbxQualifier.SelectedValue) : '\x000'; options.FirstRowColHeader = chkFirstRow.Checked; } return importManager; }
/// <summary> /// Determines whether a match filter is mapped. /// </summary> /// <param name="importManager">The import manager.</param> /// <param name="matchFilter">The match filter.</param> /// <returns> /// <c>true</c> if [is filter mapped] [the specified import manager]; otherwise, <c>false</c>. /// </returns> private Boolean IsFilterMapped(ImportManager importManager, string matchFilter) { IList<ImportMap> importMaps = importManager.ImportMaps; foreach (ImportMap map in importMaps) { if (matchFilter.Equals(map.TargetProperty.PropertyId)) return true; } return false; }
public bool ImportV3(string payload, SaveStatus status) { DateTime started = DateTime.Now; var saveDuration = new TimeSpan(); InputEntityV3 input = new InputEntityV3(); var bnodes = new List <BNodeV3>(); var mainEntity = new Dictionary <string, object>(); Dictionary <string, object> dictionary = RegistryServices.JsonToDictionary(payload); object graph = dictionary["@graph"]; //serialize the graph object var glist = JsonConvert.SerializeObject(graph); //parse graph in to list of objects JArray graphList = JArray.Parse(glist); int cntr = 0; foreach (var item in graphList) { cntr++; if (cntr == 1) { var main = item.ToString(); //may not use this. Could add a trace method mainEntity = RegistryServices.JsonToDictionary(main); input = JsonConvert.DeserializeObject <InputEntityV3>(main); } else //is this too much of an assumption? { var bn = item.ToString(); bnodes.Add(JsonConvert.DeserializeObject <BNodeV3>(bn)); } } List <string> messages = new List <string>(); bool importSuccessfull = false; EntityServices mgr = new EntityServices(); MappingHelperV3 helper = new MappingHelperV3(2) { entityBlankNodes = bnodes }; helper.entityBlankNodes = bnodes; helper.CurrentEntityCTID = input.CTID; helper.CurrentEntityName = input.Name.ToString(); try { string ctid = input.CTID; string referencedAtId = input.CtdlId; LoggingHelper.DoTrace(5, " name: "+ input.Name.ToString()); LoggingHelper.DoTrace(6, " url: "+ input.SubjectWebpage); LoggingHelper.DoTrace(5, " ctid: "+ input.CTID); LoggingHelper.DoTrace(5, " @Id: "+ input.CtdlId); status.Ctid = ctid; if (status.DoingDownloadOnly) { return(true); } if (!DoesEntityExist(input.CTID, ref output)) { //set the rowid now, so that can be referenced as needed output.RowId = Guid.NewGuid(); LoggingHelper.DoTrace(1, string.Format(thisClassName + ".ImportV3(). Record was NOT found using CTID: '{0}'", input.CTID)); } else { LoggingHelper.DoTrace(1, string.Format(thisClassName + ".ImportV3(). Found record: '{0}' using CTID: '{1}'", input.Name, input.CTID)); } helper.currentBaseObject = output; helper.CurrentOwningAgentUid = output.RowId; output.AgentDomainType = input.Type; output.Name = helper.HandleLanguageMap(input.Name, output, "Name"); output.Description = helper.HandleLanguageMap(input.Description, output, "Description"); //map from idProperty to url output.SubjectWebpage = input.SubjectWebpage; output.CTID = input.CTID; //TBD handling if (!string.IsNullOrWhiteSpace(status.DocumentPublishedBy)) { //output.PublishedByOrganizationCTID = status.DocumentPublishedBy; var porg = OrganizationManager.GetSummaryByCtid(status.DocumentPublishedBy); if (porg != null && porg.Id > 0) { //TODO - store this in a json blob?????????? //output.PublishedByOrganizationId = porg.Id; //output.PublishedByOrganizationName = porg.Name; //this will result in being added to Entity.AgentRelationship output.PublishedBy = new List <Guid>() { porg.RowId }; } else { //if publisher not imported yet, all publishee stuff will be orphaned var entityUid = Guid.NewGuid(); var statusMsg = ""; var resPos = referencedAtId.IndexOf("/resources/"); var swp = referencedAtId.Substring(0, (resPos + "/resources/".Length)) + status.DocumentPublishedBy; int orgId = new OrganizationManager().AddPendingRecord(entityUid, status.DocumentPublishedBy, swp, ref statusMsg); } } else { //may need a check for existing published by to ensure not lost if (output.Id > 0) { if (output.OrganizationRole_Recipient != null && output.OrganizationRole_Recipient.Any()) { var publishedByList = output.OrganizationRole_Recipient.Where(s => s.RoleTypeId == 30).ToList(); if (publishedByList != null && publishedByList.Any()) { var pby = publishedByList[0].ActingAgentUid; output.PublishedBy = new List <Guid>() { publishedByList[0].ActingAgentUid }; } } } } output.CredentialRegistryId = status.EnvelopeId;; output.AlternateNames = helper.MapToTextValueProfile(input.AlternateName, output, "AlternateName"); output.Image = input.Image; output.AgentPurpose = input.AgentPurpose; output.AgentPurposeDescription = helper.HandleLanguageMap(input.AgentPurposeDescription, output, "AgentPurposeDescription"); output.FoundingDate = input.FoundingDate; //output.AvailabilityListing = helper.MapListToString( input.AvailabilityListing ); //future prep output.AvailabilityListings = input.AvailabilityListing; output.MissionAndGoalsStatement = input.MissionAndGoalsStatement; output.MissionAndGoalsStatementDescription = helper.HandleLanguageMap(input.MissionAndGoalsStatementDescription, output, "MissionAndGoalsStatementDescription"); output.Addresses = helper.FormatAvailableAtAddresses(input.Address, ref status); if (output.Addresses != null && output.Addresses.Any()) { output.AddressesJson = JsonConvert.SerializeObject(output.Addresses, MappingHelperV3.GetJsonSettings()); } // if (UtilityManager.GetAppKeyValue("skipOppImportIfNoShortRegions", false)) { if (output.Addresses.Count == 0) { //skip LoggingHelper.DoTrace(2, string.Format(" ***Skipping org# {0}, {1} as it has no addresses and this is a special run.", output.Id, output.Name)); return(true); } else if (output.HasAnyShortRegions == false) { //skip LoggingHelper.DoTrace(2, string.Format(" ***Skipping org# {0}, {1} as it has no addresses with short regions and this is a special run.", output.Id, output.Name)); return(true); } } //agent type, map to enumeration output.AgentType = helper.MapCAOListToEnumermation(input.AgentType); output.TransferValueStatement = input.TransferValueStatement; output.TransferValueStatementDescription = helper.HandleLanguageMap(input.TransferValueStatementDescription, output, "TransferValueStatementDescription"); //Manifests output.ConditionManifestIds = helper.MapEntityReferences(input.HasConditionManifest, CodesManager.ENTITY_TYPE_CONDITION_MANIFEST, CodesManager.ENTITY_TYPE_ORGANIZATION, ref status); output.CostManifestIds = helper.MapEntityReferences(input.HasCostManifest, CodesManager.ENTITY_TYPE_COST_MANIFEST, CodesManager.ENTITY_TYPE_ORGANIZATION, ref status); //hasVerificationService output.VerificationServiceProfiles = helper.MapVerificationServiceProfiles(input.VerificationServiceProfiles, ref status); // output.targetc //other enumerations // serviceType, AgentSectorType output.ServiceType = helper.MapCAOListToEnumermation(input.ServiceType); output.AgentSectorType = helper.MapCAOListToEnumermation(input.AgentSectorType); //Industries //output.Industry = helper.MapCAOListToEnumermation( input.IndustryType ); output.Industries = helper.MapCAOListToCAOProfileList(input.IndustryType); //naics output.Naics = input.Naics; //keywords output.Keyword = helper.MapToTextValueProfile(input.Keyword, output, "Keyword"); //duns, Fein. IpedsID, opeID if (!string.IsNullOrWhiteSpace(input.DUNS)) { output.IdentificationCodes.Add(new workIT.Models.ProfileModels.TextValueProfile { CodeSchema = "ceterms:duns", TextValue = input.DUNS }); } if (!string.IsNullOrWhiteSpace(input.FEIN)) { output.IdentificationCodes.Add(new workIT.Models.ProfileModels.TextValueProfile { CodeSchema = "ceterms:fein", TextValue = input.FEIN }); } if (!string.IsNullOrWhiteSpace(input.IpedsID)) { output.IdentificationCodes.Add(new workIT.Models.ProfileModels.TextValueProfile { CodeSchema = "ceterms:ipedsID", TextValue = input.IpedsID }); } if (!string.IsNullOrWhiteSpace(input.OPEID)) { output.IdentificationCodes.Add(new workIT.Models.ProfileModels.TextValueProfile { CodeSchema = "ceterms:opeID", TextValue = input.OPEID }); } if (!string.IsNullOrWhiteSpace(input.LEICode)) { output.IdentificationCodes.Add(new workIT.Models.ProfileModels.TextValueProfile { CodeSchema = "ceterms:leiCode", TextValue = input.LEICode }); } // if (!string.IsNullOrWhiteSpace(input.NcesID)) { output.IdentificationCodes.Add(new workIT.Models.ProfileModels.TextValueProfile { CodeSchema = "ceterms:ncesID", TextValue = input.NcesID }); } // if (!string.IsNullOrWhiteSpace(input.ISICV4)) { output.IdentificationCodes.Add(new workIT.Models.ProfileModels.TextValueProfile { CodeSchema = "ceterms:isicV4", TextValue = input.ISICV4 }); } //alternativeidentifier - should just be added to IdentificationCodes //output.AlternativeIdentifier = helper.MapIdentifierValueListToString( input.AlternativeIdentifier ); //output.AlternativeIdentifierList = helper.MapIdentifierValueList( input.AlternativeIdentifier ); //20-10-31 - replacing AlternativeIdentifier with Identifier output.Identifier = helper.MapIdentifierValueList(input.Identifier); if (output.Identifier != null && output.Identifier.Count() > 0) { output.IdentifierJson = JsonConvert.SerializeObject(output.Identifier, MappingHelperV3.GetJsonSettings()); } //email output.Emails = helper.MapToTextValueProfile(input.Email); //contact point //output.ContactPoint = helper.FormatContactPoints( input.ContactPoint, ref status ); //Jurisdiction output.Jurisdiction = helper.MapToJurisdiction(input.Jurisdiction, ref status); //SameAs URI output.SameAs = helper.MapToTextValueProfile(input.SameAs); //Social media output.SocialMediaPages = helper.MapToTextValueProfile(input.SocialMedia); //departments //not sure - MP - want to change how depts, and subs are handled output.ParentOrganization = helper.MapOrganizationReferenceGuids("Organization.ParentOrganization", input.ParentOrganization, ref status); output.Departments = helper.MapOrganizationReferenceGuids("Organization.Department", input.Department, ref status); output.SubOrganizations = helper.MapOrganizationReferenceGuids("Organization.SubOrganization", input.SubOrganization, ref status); //output.OrganizationRole_Subsidiary = helper.FormatOrganizationReferences( input.SubOrganization ); //Process profiles output.AdministrationProcess = helper.FormatProcessProfile(input.AdministrationProcess, ref status); output.MaintenanceProcess = helper.FormatProcessProfile(input.MaintenanceProcess, ref status); output.ComplaintProcess = helper.FormatProcessProfile(input.ComplaintProcess, ref status); output.DevelopmentProcess = helper.FormatProcessProfile(input.DevelopmentProcess, ref status); output.RevocationProcess = helper.FormatProcessProfile(input.RevocationProcess, ref status); output.ReviewProcess = helper.FormatProcessProfile(input.ReviewProcess, ref status); output.AppealProcess = helper.FormatProcessProfile(input.AppealProcess, ref status); //BYs output.AccreditedBy = helper.MapOrganizationReferenceGuids("Organization.AccreditedBy", input.AccreditedBy, ref status); output.ApprovedBy = helper.MapOrganizationReferenceGuids("Organization.ApprovedBy", input.ApprovedBy, ref status); output.RecognizedBy = helper.MapOrganizationReferenceGuids("Organization.RecognizedBy", input.RecognizedBy, ref status); output.RegulatedBy = helper.MapOrganizationReferenceGuids("Organization.RegulatedBy", input.RegulatedBy, ref status); //INs output.AccreditedIn = helper.MapToJurisdiction("Organization.AccreditedIn", input.AccreditedIn, ref status); output.ApprovedIn = helper.MapToJurisdiction("Organization.ApprovedIn", input.ApprovedIn, ref status); output.RecognizedIn = helper.MapToJurisdiction("Organization.RecognizedIn", input.RecognizedIn, ref status); output.RegulatedIn = helper.MapToJurisdiction("Organization.RegulatedIn", input.RegulatedIn, ref status); //Asserts //the entity type is not known output.Accredits = helper.MapEntityReferenceGuids("Organization.Accredits", input.Accredits, 0, ref status); output.Approves = helper.MapEntityReferenceGuids("Organization.Approves", input.Approves, 0, ref status); if (output.Approves.Count > 0) { } output.Offers = helper.MapEntityReferenceGuids("Organization.Offers", input.Offers, 0, ref status); output.Owns = helper.MapEntityReferenceGuids("Organization.Owns", input.Owns, 0, ref status); output.Renews = helper.MapEntityReferenceGuids("Organization.Renews", input.Renews, 0, ref status); output.Revokes = helper.MapEntityReferenceGuids("Organization.Revokes", input.Revokes, 0, ref status); output.Recognizes = helper.MapEntityReferenceGuids("Organization.Recognizes", input.Recognizes, 0, ref status); output.Regulates = helper.MapEntityReferenceGuids("Organization.Regulates", input.Regulates, 0, ref status); // TimeSpan duration = DateTime.Now.Subtract(started); if (duration.TotalSeconds > 5) { LoggingHelper.DoTrace(6, string.Format(" WARNING Mapping Duration: {0:N2} seconds ", duration.TotalSeconds)); } DateTime saveStarted = DateTime.Now; //=== if any messages were encountered treat as warnings for now if (messages.Count > 0) { status.SetMessages(messages, true); } //just in case check if entity added since start if (output.Id == 0) { ThisEntity entity = EntityServices.GetSummaryByCtid(ctid); if (entity != null && entity.Id > 0) { output.Id = entity.Id; output.RowId = entity.RowId; } } importSuccessfull = mgr.Import(output, ref status); saveDuration = DateTime.Now.Subtract(saveStarted); if (saveDuration.TotalSeconds > 5) { LoggingHelper.DoTrace(6, string.Format(" WARNING SAVE Duration: {0:N2} seconds ", saveDuration.TotalSeconds)); } // status.DocumentId = output.Id; status.DetailPageUrl = string.Format("~/organization/{0}", output.Id); status.DocumentRowId = output.RowId; //just in case if (status.HasErrors) { importSuccessfull = false; } //if record was added to db, add to/or set EntityResolution as resolved int ierId = new ImportManager().Import_EntityResolutionAdd(referencedAtId, ctid, CodesManager.ENTITY_TYPE_ORGANIZATION, output.RowId, output.Id, false, ref messages, output.Id > 0); } catch (Exception ex) { LoggingHelper.LogError(ex, string.Format("OrganizationImportV3. Exception encountered in CTID: {0}", input.CTID), false, "Organization Import exception"); } finally { var totalDuration = DateTime.Now.Subtract(started); if (totalDuration.TotalSeconds > 9 && (totalDuration.TotalSeconds - saveDuration.TotalSeconds > 3)) { LoggingHelper.DoTrace(5, string.Format(" WARNING Total Duration: {0:N2} seconds ", totalDuration.TotalSeconds)); } } return(importSuccessfull); }
static void Main(string[] args) { bool isAutomaticMode = false; bool isAPIImportMode = false; LogEvent("Starting Import:", true); if (args.Length > 0) { LogEvent("Arguments supplied: "); foreach (var arg in args) { LogEvent("arg: " + arg); if (arg == "-auto") { isAutomaticMode = true; } if (arg == "-api") { isAPIImportMode = true; } } } else { LogEvent("No Arguments supplied."); } if (isAutomaticMode) { bool actionPerformed = false; ActionType mode = ActionType.DataImport; if (mode == ActionType.DataImport && isAPIImportMode == true) { ExportType exportType = ExportType.API; var settings = new ImportSettings { GeolocationShapefilePath = ConfigurationManager.AppSettings["GeolocationShapefilePath"], ImportUserAPIKey = ConfigurationManager.AppSettings["APIKey"], MasterAPIBaseUrl = ConfigurationManager.AppSettings["APIBaseUrl"], TempFolderPath = Settings.Default.Import_DataFolder }; ImportManager importManager = new ImportManager(settings); LogEvent("Performing Import, Publishing via API: " + DateTime.UtcNow.ToShortTimeString()); importManager.PerformImportProcessing(new ImportProcessSettings { ExportType = exportType, DefaultDataPath = Settings.Default.Import_DataFolder, ApiIdentifier = Settings.Default.OCM_API_Identitifer, ApiSessionToken = Settings.Default.OCM_API_SessionToken }); LogEvent("Import Processed. Exiting. " + DateTime.UtcNow.ToShortTimeString()); actionPerformed = true; } if (mode == ActionType.NetworkServices) { //network service polling //OCM.API.NetworkServices.ServiceManager serviceManager = new OCM.API.NetworkServices.ServiceManager(); //serviceManager.Test(OCM.API.NetworkServices.ServiceProvider.CoulombChargePoint); } if (!actionPerformed) { LogEvent("Nothing to do. Exiting. " + DateTime.UtcNow.ToShortTimeString()); } } else { //show UI Application.Run(new AppMain()); } }
public bool Import(InputEntity input, string envelopeIdentifier, SaveStatus status) { List <string> messages = new List <string>(); bool importSuccessfull = false; EntityServices mgr = new EntityServices(); string ctid = input.Ctid; string referencedAtId = input.CtdlId; LoggingHelper.DoTrace(5, " name: "+ input.Name); LoggingHelper.DoTrace(6, " url: "+ input.SubjectWebpage); LoggingHelper.DoTrace(5, " ctid: "+ input.Ctid); LoggingHelper.DoTrace(5, " @Id: "+ input.CtdlId); status.Ctid = ctid; if (status.DoingDownloadOnly) { return(true); } if (!DoesEntityExist(input.Ctid, ref output)) { //set the rowid now, so that can be referenced as needed output.RowId = Guid.NewGuid(); } //re:messages - currently passed to mapping but no errors are trapped?? // - should use SaveStatus and skip import if errors encountered (vs warnings) output.AgentDomainType = input.Type; output.Name = input.Name; output.Description = input.Description; //map from idProperty to url output.SubjectWebpage = input.SubjectWebpage; output.CTID = input.Ctid; output.CredentialRegistryId = envelopeIdentifier; //output.AlternateNames = input.AlternateName; output.AlternateNames = MappingHelper.MapToTextValueProfile(input.AlternateName); output.ImageUrl = input.Image; output.AgentPurpose = input.AgentPurpose; output.AgentPurposeDescription = input.AgentPurposeDescription; output.FoundingDate = input.FoundingDate; if ((output.FoundingDate ?? "").Length > 20) { status.AddError("Organization Founding Date error - exceeds 20 characters: " + output.FoundingDate); output.FoundingDate = output.FoundingDate.Substring(0, 10); } output.AvailabilityListing = MappingHelper.MapListToString(input.AvailabilityListing); //future prep output.AvailabilityListings = input.AvailabilityListing; output.MissionAndGoalsStatement = input.MissionAndGoalsStatement; output.MissionAndGoalsStatementDescription = input.MissionAndGoalsStatementDescription; output.Addresses = MappingHelper.FormatAddresses(input.Address, ref status); //agent type, map to enumeration output.AgentType = MappingHelper.MapCAOListToEnumermation(input.AgentType); //Manifests output.ConditionManifestIds = MappingHelper.MapEntityReferences(input.HasConditionManifest, CodesManager.ENTITY_TYPE_CONDITION_MANIFEST, CodesManager.ENTITY_TYPE_ORGANIZATION, ref status); output.CostManifestIds = MappingHelper.MapEntityReferences(input.HasCostManifest, CodesManager.ENTITY_TYPE_COST_MANIFEST, CodesManager.ENTITY_TYPE_ORGANIZATION, ref status); //hasVerificationService output.VerificationServiceProfiles = MappingHelper.MapVerificationServiceProfiles(input.VerificationServiceProfiles, ref status); // output.targetc //other enumerations // serviceType, AgentSectorType output.ServiceType = MappingHelper.MapCAOListToEnumermation(input.ServiceType); output.AgentSectorType = MappingHelper.MapCAOListToEnumermation(input.AgentSectorType); //Industries output.Industry = MappingHelper.MapCAOListToEnumermation(input.IndustryType); output.Industries = MappingHelper.MapCAOListToFramework(input.IndustryType); //naics output.Naics = input.Naics; //keywords output.Keyword = MappingHelper.MapToTextValueProfile(input.Keyword); //duns, Fein. IpedsID, opeID if (!string.IsNullOrWhiteSpace(input.DUNS)) { output.IdentificationCodes.Add(new workIT.Models.ProfileModels.TextValueProfile { CodeSchema = "ceterms:duns", TextValue = input.DUNS }); } if (!string.IsNullOrWhiteSpace(input.FEIN)) { output.IdentificationCodes.Add(new workIT.Models.ProfileModels.TextValueProfile { CodeSchema = "ceterms:fein", TextValue = input.FEIN }); } if (!string.IsNullOrWhiteSpace(input.IpedsID)) { output.IdentificationCodes.Add(new workIT.Models.ProfileModels.TextValueProfile { CodeSchema = "ceterms:ipedsID", TextValue = input.IpedsID }); } if (!string.IsNullOrWhiteSpace(input.OPEID)) { output.IdentificationCodes.Add(new workIT.Models.ProfileModels.TextValueProfile { CodeSchema = "ceterms:opeID", TextValue = input.OPEID }); } if (!string.IsNullOrWhiteSpace(input.LEICode)) { output.IdentificationCodes.Add(new workIT.Models.ProfileModels.TextValueProfile { CodeSchema = "ceterms:leiCode", TextValue = input.LEICode }); } //alternativeidentifier - should just be added to IdentificationCodes output.AlternativeIdentifier = MappingHelper.MapIdentifierValueListToString(input.AlternativeIdentifier); output.AlternativeIdentifierList = MappingHelper.MapIdentifierValueList(input.AlternativeIdentifier); //email output.Emails = MappingHelper.MapToTextValueProfile(input.Email); //contact point - now in address //output.ContactPoint = MappingHelper.FormatContactPoints( input.ContactPoint, ref status ); //Jurisdiction output.Jurisdiction = MappingHelper.MapToJurisdiction(input.Jurisdiction, ref status); //SameAs output.SameAs = MappingHelper.MapToTextValueProfile(input.SameAs); //Social media output.SocialMediaPages = MappingHelper.MapToTextValueProfile(input.SocialMedia); //departments //not sure - MP - want to change how depts, and subs are handled //output.ParentOrganization = MappingHelper.MapOrganizationReferenceGuids( input.ParentOrganization, ref status ); output.Departments = MappingHelper.MapOrganizationReferenceGuids(input.Department, ref status); output.SubOrganizations = MappingHelper.MapOrganizationReferenceGuids(input.SubOrganization, ref status); //output.OrganizationRole_Subsidiary = MappingHelper.FormatOrganizationReferences( input.SubOrganization ); //Process profiles output.AdministrationProcess = MappingHelper.FormatProcessProfile(input.AdministrationProcess, ref status); output.MaintenanceProcess = MappingHelper.FormatProcessProfile(input.MaintenanceProcess, ref status); output.ComplaintProcess = MappingHelper.FormatProcessProfile(input.ComplaintProcess, ref status); output.DevelopmentProcess = MappingHelper.FormatProcessProfile(input.DevelopmentProcess, ref status); output.RevocationProcess = MappingHelper.FormatProcessProfile(input.RevocationProcess, ref status); output.ReviewProcess = MappingHelper.FormatProcessProfile(input.ReviewProcess, ref status); output.AppealProcess = MappingHelper.FormatProcessProfile(input.AppealProcess, ref status); //BYs output.AccreditedBy = MappingHelper.MapOrganizationReferenceGuids(input.AccreditedBy, ref status); output.ApprovedBy = MappingHelper.MapOrganizationReferenceGuids(input.ApprovedBy, ref status); output.RecognizedBy = MappingHelper.MapOrganizationReferenceGuids(input.RecognizedBy, ref status); output.RegulatedBy = MappingHelper.MapOrganizationReferenceGuids(input.RegulatedBy, ref status); //INs output.AccreditedIn = MappingHelper.MapToJurisdiction(input.AccreditedIn, ref status); output.ApprovedIn = MappingHelper.MapToJurisdiction(input.ApprovedIn, ref status); output.RecognizedIn = MappingHelper.MapToJurisdiction(input.RecognizedIn, ref status); output.RegulatedIn = MappingHelper.MapToJurisdiction(input.RegulatedIn, ref status); //Asserts //the entity type is not known output.Accredits = MappingHelper.MapEntityReferenceGuids(input.Accredits, 0, ref status); output.Approves = MappingHelper.MapEntityReferenceGuids(input.Approves, 0, ref status); output.Offers = MappingHelper.MapEntityReferenceGuids(input.Offers, 0, ref status); output.Owns = MappingHelper.MapEntityReferenceGuids(input.Owns, 0, ref status); output.Renews = MappingHelper.MapEntityReferenceGuids(input.Renews, 0, ref status); output.Revokes = MappingHelper.MapEntityReferenceGuids(input.Revokes, 0, ref status); output.Recognizes = MappingHelper.MapEntityReferenceGuids(input.Recognizes, 0, ref status); output.Regulates = MappingHelper.MapEntityReferenceGuids(input.Regulates, 0, ref status); //Ins - defer to later //=== if any messages were encountered treat as warnings for now if (messages.Count > 0) { status.SetMessages(messages, true); } //just in case check if entity added since start if (output.Id == 0) { ThisEntity entity = EntityServices.GetByCtid(ctid); if (entity != null && entity.Id > 0) { output.Id = entity.Id; output.RowId = entity.RowId; } } importSuccessfull = mgr.Import(output, ref status); status.DocumentId = output.Id; status.DetailPageUrl = string.Format("~/organization/{0}", output.Id); status.DocumentRowId = output.RowId; //just in case if (status.HasErrors) { importSuccessfull = false; } //if record was added to db, add to/or set EntityResolution as resolved int ierId = new ImportManager().Import_EntityResolutionAdd(referencedAtId, ctid, CodesManager.ENTITY_TYPE_ORGANIZATION, output.RowId, output.Id, (output.Id > 0), ref messages, output.Id > 0); return(importSuccessfull); }
public static async Task ProcessAssessmentImportLegacyAsync(string csetFilePath, string token, string processPath, string apiURL, PerformContext context) { var logger = new HangfireLogger(context); ImportManager manager = new ImportManager(); await manager.LaunchLegacyCSETProcess(csetFilePath, token, processPath, apiURL); }
private void btnImport_Click(object sender, EventArgs e) { const string OUT_TABLENAME = @"CSVImportTest"; const int MAX_BUFFER = 100; var start = DateTime.Now; try { var dci = csfSaveAs.DataConnectionInfo; if (dci.InputSource != InputSource.Database) { throw new ArgumentException("Please choose a database"); } var df = new DataFactory(dci.Provider, dci.ConnectionString); df.TestConnection(); if (df.Database.GetTables(OUT_TABLENAME, QueryFilter.Exact).Count >= 1) { df.ExecuteNonQuery("drop table " + OUT_TABLENAME); } int nr = 0; using (DataTable dt = new DataTable(OUT_TABLENAME)) { var options = new DelimiterImportOptions(","); foreach (var item in ImportManager.ImportRow(txtFilename.Value, options)) { nr++; #region Init Columns Definition if (nr == 1) { DataUtil.CopyColumns(item.Table.Columns, dt.Columns); } #endregion var dr = dt.NewRow(); DataUtil.CopyRow(item, dr); dt.Rows.Add(dr); if (nr % MAX_BUFFER == 0) { df.AppendTable(OUT_TABLENAME, dt); dt.Rows.Clear(); } } if (dt.Rows.Count > 0) { df.AppendTable(OUT_TABLENAME, dt); dt.Rows.Clear(); } } MessageBox.Show("Done Elapsed = " + DateUtil.ElapsedTime(start, DateTime.Now)); } catch (Exception ex) { FormUtil.WinException(ex); } }
/// <summary> /// Sets the default value for the EntityProperty specified. /// </summary> /// <param name="property">The property.</param> /// <param name="value">The value.</param> private void SetDefaultTargetPropertyValue(string property, string value, ImportManager importManager) { ImportTargetProperty prop = importManager.EntityManager.GetEntityProperty(property); if (prop != null) { prop.DefaultValue = value; Page.Session["importManager"] = importManager; } }
private void RemoveJob(ImportManager importManager) { IDictionary<string, string> jobs = Page.Session["importJobs"] as Dictionary<string, string>; if (jobs != null) { jobs.Remove(importManager.ToString()); } Page.Session["importJobs"] = jobs; }
private void AddJob(ImportManager importManager) { IDictionary<string, string> jobs = Page.Session["importJobs"] as Dictionary<string, string> ?? new Dictionary<string, string>(); if (!jobs.ContainsKey(importManager.ToString())) { jobs.Add(importManager.ToString(), importManager.SourceFileName); } Page.Session["importJobs"] = jobs; }
private void AddCrossReferenceMananager(ImportManager importManager) { IImportTransformationProvider transformationProvider = importManager.TransformationProvider; if (transformationProvider.CrossRefManager == null || transformationProvider.CrossRefManager.Sets.Count == 0) { ImportCrossRefManager crossRefManager = new ImportCrossRefManager(); ImportCrossRefSet crossRefSet = new ImportCrossRefSet(); crossRefSet.AddCrossReference("T", true); crossRefSet.AddCrossReference("F", false); crossRefSet.AddCrossReference("Y", true); crossRefSet.AddCrossReference("N", true); crossRefSet.AddCrossReference("Yes", true); crossRefSet.AddCrossReference("No", false); crossRefSet.AddCrossReference("True", true); crossRefSet.AddCrossReference("False", false); crossRefManager.Sets.Add("TrueFalse_Set", crossRefSet); crossRefSet = new ImportCrossRefSet(); crossRefSet.AddCrossReference("M", "Male"); crossRefSet.AddCrossReference("F", "Female"); crossRefSet.AddCrossReference("", "Unknown"); crossRefSet.AddCrossReference("U", "Unknown"); crossRefManager.Sets.Add("Gender_Set", crossRefSet); transformationProvider.CrossRefManager = crossRefManager; } }
public void Run(string budgetId, string userId, string file) { var excel = new ExcelQueryFactory(file); var anni = new[] { 2013, 2014 }; var movements = new List <Movimento>(); foreach (var anno in anni) { //movements.AddRange(excel.Worksheet<Movement>(anno + "") // .Where(r => r.Data != DateTime.MinValue)); var laura = excel.WorksheetRange <Movimento>("B6", "E10000", anno + "") .Where(r => r.Data != DateTime.MinValue) .ToList(); laura.ForEach(m => m.DistributionKey = "Laura"); var valerio = excel.WorksheetRange <Movimento>("G6", "J10000", anno + "") .Where(r => r.Data != DateTime.MinValue) .ToList(); valerio.ForEach(m => m.DistributionKey = "Valerio"); var comune = excel.WorksheetRange <Movimento>("L6", "O10000", anno + "") .Where(r => r.Data != DateTime.MinValue) .ToList(); movements.AddRange(laura); movements.AddRange(valerio); movements.AddRange(comune); } //var tasse = movements.GroupBy(g => g.Categoria).ToList(); Console.WriteLine("Read {0} movements from {1}", movements.Count, file); movements = movements.Where(r => r.Categoria != "Arancio").OrderBy(d => d.Data).ToList(); var importer = new ImportManager(_cm, _pm); var categorie = movements.Select(s => s.Categoria).ToArray(); importer.ImportCategoriesByName(categorie, budgetId, userId); var categories = _pm.GetCategories().GetBudgetsCategories(budgetId); var createLine = _cm.Create <CreateLine>(); //foreach (var m in movements.Where(r=> r.Categoria != "Arancio")) // createLine(m.ToCreateLine(new BudgetId(budgetId), userId, categories)); DateTime last = DateTime.MinValue; foreach (var m in movements) { last = DateTime.Now; createLine(m.ToCreateLine(last, new BudgetId(budgetId), userId, categories)); } var bp = _pm.GetBudgetLinesProjection(budgetId); var galt = bp.GetAllLines(last); galt.Wait(); var lines = galt.Result; Console.WriteLine("Loaded {0} movements into {1}", lines.Count(), budgetId); }
public IEnumerable <object> GetData(Item accountItem) { string[] idList = { }; if (accountItem.TemplateID == Templates.AccountPublic.TemplateID) { string ids = accountItem[Templates.AccountPublic.PlaylistIDs]; if (!string.IsNullOrEmpty(ids)) { idList = ids.Split(new[] { '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries); } } else { var data = ImportManager.ImportList <Playlist>("import_youtube_playlists", accountItem); if (data != null) { idList = data.Select(d => d.UniqueId).ToArray(); } } var authenticator = new YouTubeAuthenticator(accountItem); var context = new RestSharp.RestContext(Constants.SitecoreRestSharpService, authenticator); var parameters = new List <Parameter>(); var playlistIdParameter = new Parameter { Type = ParameterType.UrlSegment, Name = "playlistId" }; parameters.Add(playlistIdParameter); string nextPageToken = string.Empty; var pageTokenParameter = new Parameter { Type = ParameterType.UrlSegment, Name = "pageToken", Value = nextPageToken }; parameters.Add(pageTokenParameter); foreach (string id in idList) { { playlistIdParameter.Value = id; do { pageTokenParameter.Value = nextPageToken; IRestResponse <PagedCollection <PlaylistItem> > pagedEntities = context.Read <PagedCollection <PlaylistItem> >(this.RequestName, parameters); if (pagedEntities == null || pagedEntities.Data == null || pagedEntities.Data.Items == null) { LogHelper.Warn("Null Result during importing", this); throw new HttpException("Http null result"); } nextPageToken = (pagedEntities.Data.NextPageToken != null && pagedEntities.Data.NextPageToken != nextPageToken) ? pagedEntities.Data.NextPageToken : string.Empty; foreach (PlaylistItem entity in pagedEntities.Data.Items) { yield return(entity); } }while (!string.IsNullOrEmpty(nextPageToken)); } } }
public bool ImportV3(string payload, string envelopeIdentifier, SaveStatus status) { InputEntityV3 input = new InputEntityV3(); var bnodes = new List <BNodeV3>(); var mainEntity = new Dictionary <string, object>(); Dictionary <string, object> dictionary = RegistryServices.JsonToDictionary(payload); object graph = dictionary["@graph"]; //serialize the graph object var glist = JsonConvert.SerializeObject(graph); //parse graph in to list of objects JArray graphList = JArray.Parse(glist); int cntr = 0; foreach (var item in graphList) { cntr++; if (cntr == 1) { var main = item.ToString(); //may not use this. Could add a trace method mainEntity = RegistryServices.JsonToDictionary(main); input = JsonConvert.DeserializeObject <InputEntityV3>(main); } else //is this too much of an assumption? { var bn = item.ToString(); bnodes.Add(JsonConvert.DeserializeObject <BNodeV3>(bn)); } } List <string> messages = new List <string>(); bool importSuccessfull = false; EntityServices mgr = new EntityServices(); MappingHelperV3 helper = new MappingHelperV3 { entityBlankNodes = bnodes }; string ctid = input.Ctid; string referencedAtId = input.CtdlId; LoggingHelper.DoTrace(5, " name: "+ input.Name.ToString()); LoggingHelper.DoTrace(6, " url: "+ input.SubjectWebpage); LoggingHelper.DoTrace(5, " ctid: "+ input.Ctid); LoggingHelper.DoTrace(5, " @Id: "+ input.CtdlId); status.Ctid = ctid; if (status.DoingDownloadOnly) { return(true); } if (!DoesEntityExist(input.Ctid, ref output)) { //set the rowid now, so that can be referenced as needed output.RowId = Guid.NewGuid(); } helper.currentBaseObject = output; output.AgentDomainType = input.Type; output.Name = helper.HandleLanguageMap(input.Name, output, "Name"); output.Description = helper.HandleLanguageMap(input.Description, output, "Description"); //map from idProperty to url output.SubjectWebpage = input.SubjectWebpage; output.CTID = input.Ctid; output.CredentialRegistryId = envelopeIdentifier; output.AlternateNames = helper.MapToTextValueProfile(input.AlternateName, output, "AlternateName"); output.ImageUrl = input.Image; output.AgentPurpose = input.AgentPurpose; output.AgentPurposeDescription = helper.HandleLanguageMap(input.AgentPurposeDescription, output, "AgentPurposeDescription"); output.FoundingDate = input.FoundingDate; output.AvailabilityListing = helper.MapListToString(input.AvailabilityListing); //future prep output.AvailabilityListings = input.AvailabilityListing; output.MissionAndGoalsStatement = input.MissionAndGoalsStatement; //output.MissionAndGoalsStatementDescription = input.MissionAndGoalsStatementDescription; output.MissionAndGoalsStatementDescription = helper.HandleLanguageMap(input.MissionAndGoalsStatementDescription, output, "MissionAndGoalsStatementDescription"); output.Addresses = helper.FormatAvailableAtAddresses(input.Address, ref status); if (UtilityManager.GetAppKeyValue("skipOppImportIfNoShortRegions", false)) { if (output.Addresses.Count == 0) { //skip LoggingHelper.DoTrace(2, string.Format(" ***Skipping org# {0}, {1} as it has no addresses and this is a special run.", output.Id, output.Name)); return(true); } else if (output.HasAnyShortRegions == false) { //skip LoggingHelper.DoTrace(2, string.Format(" ***Skipping org# {0}, {1} as it has no addresses with short regions and this is a special run.", output.Id, output.Name)); return(true); } } //agent type, map to enumeration output.AgentType = helper.MapCAOListToEnumermation(input.AgentType); //Manifests output.ConditionManifestIds = helper.MapEntityReferences(input.HasConditionManifest, CodesManager.ENTITY_TYPE_CONDITION_MANIFEST, CodesManager.ENTITY_TYPE_ORGANIZATION, ref status); output.CostManifestIds = helper.MapEntityReferences(input.HasCostManifest, CodesManager.ENTITY_TYPE_COST_MANIFEST, CodesManager.ENTITY_TYPE_ORGANIZATION, ref status); //hasVerificationService output.VerificationServiceProfiles = helper.MapVerificationServiceProfiles(input.VerificationServiceProfiles, ref status); // output.targetc //other enumerations // serviceType, AgentSectorType output.ServiceType = helper.MapCAOListToEnumermation(input.ServiceType); output.AgentSectorType = helper.MapCAOListToEnumermation(input.AgentSectorType); //Industries //output.Industry = helper.MapCAOListToEnumermation( input.IndustryType ); output.Industries = helper.MapCAOListToCAOProfileList(input.IndustryType); //naics output.Naics = input.Naics; //keywords output.Keyword = helper.MapToTextValueProfile(input.Keyword, output, "Keyword"); //duns, Fein. IpedsID, opeID if (!string.IsNullOrWhiteSpace(input.DUNS)) { output.IdentificationCodes.Add(new workIT.Models.ProfileModels.TextValueProfile { CodeSchema = "ceterms:duns", TextValue = input.DUNS }); } if (!string.IsNullOrWhiteSpace(input.FEIN)) { output.IdentificationCodes.Add(new workIT.Models.ProfileModels.TextValueProfile { CodeSchema = "ceterms:fein", TextValue = input.FEIN }); } if (!string.IsNullOrWhiteSpace(input.IpedsID)) { output.IdentificationCodes.Add(new workIT.Models.ProfileModels.TextValueProfile { CodeSchema = "ceterms:ipedsID", TextValue = input.IpedsID }); } if (!string.IsNullOrWhiteSpace(input.OPEID)) { output.IdentificationCodes.Add(new workIT.Models.ProfileModels.TextValueProfile { CodeSchema = "ceterms:opeID", TextValue = input.OPEID }); } if (!string.IsNullOrWhiteSpace(input.LEICode)) { output.IdentificationCodes.Add(new workIT.Models.ProfileModels.TextValueProfile { CodeSchema = "ceterms:leiCode", TextValue = input.LEICode }); } // if (!string.IsNullOrWhiteSpace(input.NcesID)) { output.IdentificationCodes.Add(new workIT.Models.ProfileModels.TextValueProfile { CodeSchema = "ceterms:ncesID", TextValue = input.NcesID }); } // if (!string.IsNullOrWhiteSpace(input.ISICV4)) { output.IdentificationCodes.Add(new workIT.Models.ProfileModels.TextValueProfile { CodeSchema = "ceterms:isicV4", TextValue = input.ISICV4 }); } //alternativeidentifier - should just be added to IdentificationCodes output.AlternativeIdentifier = helper.MapIdentifierValueListToString(input.AlternativeIdentifier); output.AlternativeIdentifierList = helper.MapIdentifierValueList(input.AlternativeIdentifier); //email output.Emails = helper.MapToTextValueProfile(input.Email); //contact point //output.ContactPoint = helper.FormatContactPoints( input.ContactPoint, ref status ); //Jurisdiction output.Jurisdiction = helper.MapToJurisdiction(input.Jurisdiction, ref status); //SameAs output.SameAs = helper.MapToTextValueProfile(input.SameAs); //Social media output.SocialMediaPages = helper.MapToTextValueProfile(input.SocialMedia); //departments //not sure - MP - want to change how depts, and subs are handled output.ParentOrganization = helper.MapOrganizationReferenceGuids(input.ParentOrganization, ref status); output.Departments = helper.MapOrganizationReferenceGuids(input.Department, ref status); output.SubOrganizations = helper.MapOrganizationReferenceGuids(input.SubOrganization, ref status); //output.OrganizationRole_Subsidiary = helper.FormatOrganizationReferences( input.SubOrganization ); //Process profiles output.AdministrationProcess = helper.FormatProcessProfile(input.AdministrationProcess, ref status); output.MaintenanceProcess = helper.FormatProcessProfile(input.MaintenanceProcess, ref status); output.ComplaintProcess = helper.FormatProcessProfile(input.ComplaintProcess, ref status); output.DevelopmentProcess = helper.FormatProcessProfile(input.DevelopmentProcess, ref status); output.RevocationProcess = helper.FormatProcessProfile(input.RevocationProcess, ref status); output.ReviewProcess = helper.FormatProcessProfile(input.ReviewProcess, ref status); output.AppealProcess = helper.FormatProcessProfile(input.AppealProcess, ref status); //BYs output.AccreditedBy = helper.MapOrganizationReferenceGuids("Organization.AccreditedBy", input.AccreditedBy, ref status); output.ApprovedBy = helper.MapOrganizationReferenceGuids("Organization.ApprovedBy", input.ApprovedBy, ref status); output.RecognizedBy = helper.MapOrganizationReferenceGuids("Organization.RecognizedBy", input.RecognizedBy, ref status); output.RegulatedBy = helper.MapOrganizationReferenceGuids("Organization.RegulatedBy", input.RegulatedBy, ref status); //INs output.AccreditedIn = helper.MapToJurisdiction(input.AccreditedIn, ref status); output.ApprovedIn = helper.MapToJurisdiction(input.ApprovedIn, ref status); output.RecognizedIn = helper.MapToJurisdiction(input.RecognizedIn, ref status); output.RegulatedIn = helper.MapToJurisdiction(input.RegulatedIn, ref status); //Asserts //the entity type is not known output.Accredits = helper.MapEntityReferenceGuids(input.Accredits, 0, ref status); output.Approves = helper.MapEntityReferenceGuids(input.Approves, 0, ref status); if (output.Approves.Count > 0) { } output.Offers = helper.MapEntityReferenceGuids(input.Offers, 0, ref status); output.Owns = helper.MapEntityReferenceGuids(input.Owns, 0, ref status); output.Renews = helper.MapEntityReferenceGuids(input.Renews, 0, ref status); output.Revokes = helper.MapEntityReferenceGuids(input.Revokes, 0, ref status); output.Recognizes = helper.MapEntityReferenceGuids(input.Recognizes, 0, ref status); output.Regulates = helper.MapEntityReferenceGuids(input.Regulates, 0, ref status); //Ins - defer to later //=== if any messages were encountered treat as warnings for now if (messages.Count > 0) { status.SetMessages(messages, true); } //just in case check if entity added since start if (output.Id == 0) { ThisEntity entity = EntityServices.GetByCtid(ctid); if (entity != null && entity.Id > 0) { output.Id = entity.Id; output.RowId = entity.RowId; } } importSuccessfull = mgr.Import(output, ref status); status.DocumentId = output.Id; status.DetailPageUrl = string.Format("~/organization/{0}", output.Id); status.DocumentRowId = output.RowId; //just in case if (status.HasErrors) { importSuccessfull = false; } //if record was added to db, add to/or set EntityResolution as resolved int ierId = new ImportManager().Import_EntityResolutionAdd(referencedAtId, ctid, CodesManager.ENTITY_TYPE_ORGANIZATION, output.RowId, output.Id, false, ref messages, output.Id > 0); return(importSuccessfull); }
/// <summary> /// Determines whether a match filter is mapped. /// </summary> /// <param name="importManager">The import manager.</param> /// <param name="matchFilter">The match filter.</param> /// <returns> /// <c>true</c> if [is filter mapped] [the specified import manager]; otherwise, <c>false</c>. /// </returns> private static Boolean IsFilterMapped(ImportManager importManager, string matchFilter) { IList <ImportMap> importMaps = importManager.ImportMaps; return(importMaps.Any(map => matchFilter.Equals(map.TargetProperty.PropertyId))); }
public MainImportManager(ImportManager importManager, IMapper mapper) { m_importManager = importManager; m_mapper = mapper; }
} // public bool Import(string payload, string envelopeIdentifier, SaveStatus status) { LoggingHelper.DoTrace(6, "ImportPathwaySets - entered."); List <string> messages = new List <string>(); bool importSuccessfull = false; EntityServices mgr = new EntityServices(); // InputEntity input = new InputEntity(); var mainEntity = new Dictionary <string, object>(); // Dictionary <string, object> dictionary = RegistryServices.JsonToDictionary(payload); object graph = dictionary["@graph"]; //serialize the graph object var glist = JsonConvert.SerializeObject(graph); //parse graph in to list of objects JArray graphList = JArray.Parse(glist); var bnodes = new List <BNode>(); int cntr = 0; foreach (var item in graphList) { cntr++; if (cntr == 1) { var main = item.ToString(); //may not use this. Could add a trace method mainEntity = RegistryServices.JsonToDictionary(main); input = JsonConvert.DeserializeObject <InputEntity>(main); } else { //may have blank nodes? var bn = item.ToString(); bnodes.Add(JsonConvert.DeserializeObject <BNode>(bn)); var child = item.ToString(); if (child.IndexOf("_:") > -1) { bnodes.Add(JsonConvert.DeserializeObject <BNode>(child)); } else { //unexpected Dictionary <string, object> unexpected = RegistryServices.JsonToDictionary(child); object unexpectedType = unexpected["@type"]; status.AddError("Unexpected document type"); } } } MappingHelperV3 helper = new MappingHelperV3(CodesManager.ENTITY_TYPE_PATHWAY_SET); helper.entityBlankNodes = bnodes; helper.CurrentEntityCTID = input.CTID; helper.CurrentEntityName = input.Name.ToString(); status.EnvelopeId = envelopeIdentifier; try { string ctid = input.CTID; string referencedAtId = input.CtdlId; LoggingHelper.DoTrace(5, " name: "+ input.Name.ToString()); LoggingHelper.DoTrace(6, " url: "+ input.SubjectWebpage); LoggingHelper.DoTrace(5, " ctid: "+ input.CTID); LoggingHelper.DoTrace(6, " @Id: "+ input.CtdlId); status.Ctid = ctid; if (status.DoingDownloadOnly) { return(true); } //add/updating PathwaySet if (!DoesEntityExist(input.CTID, ref output)) { //set the rowid now, so that can be referenced as needed output.RowId = Guid.NewGuid(); } helper.currentBaseObject = output; output.Name = helper.HandleLanguageMap(input.Name, output, "Name"); output.Description = helper.HandleLanguageMap(input.Description, output, "Description"); output.SubjectWebpage = input.SubjectWebpage; output.CTID = input.CTID; //TBD handling of referencing third party publisher if (!string.IsNullOrWhiteSpace(status.DocumentPublishedBy)) { //output.PublishedByOrganizationCTID = status.DocumentPublishedBy; var porg = OrganizationManager.GetSummaryByCtid(status.DocumentPublishedBy); if (porg != null && porg.Id > 0) { //TODO - store this in a json blob?????????? //this will result in being added to Entity.AgentRelationship output.PublishedBy = new List <Guid>() { porg.RowId }; } else { //if publisher not imported yet, all publishee stuff will be orphaned var entityUid = Guid.NewGuid(); var statusMsg = ""; var resPos = referencedAtId.IndexOf("/resources/"); var swp = referencedAtId.Substring(0, (resPos + "/resources/".Length)) + status.DocumentPublishedBy; int orgId = new OrganizationManager().AddPendingRecord(entityUid, status.DocumentPublishedBy, swp, ref statusMsg); } } else { //may need a check for existing published by to ensure not lost if (output.Id > 0) { if (output.OrganizationRole != null && output.OrganizationRole.Any()) { var publishedByList = output.OrganizationRole.Where(s => s.RoleTypeId == 30).ToList(); if (publishedByList != null && publishedByList.Any()) { var pby = publishedByList[0].ActingAgentUid; output.PublishedBy = new List <Guid>() { publishedByList[0].ActingAgentUid }; } } } } //warning this gets set to blank if doing a manual import by ctid output.CredentialRegistryId = envelopeIdentifier; //BYs - do owned and offered first output.OfferedBy = helper.MapOrganizationReferenceGuids("PathwaySet.OfferedBy", input.OfferedBy, ref status); //note need to set output.OwningAgentUid to the first entry output.OwnedBy = helper.MapOrganizationReferenceGuids("PathwaySet.OwnedBy", input.OwnedBy, ref status); if (output.OwnedBy != null && output.OwnedBy.Count > 0) { output.OwningAgentUid = output.OwnedBy[0]; helper.CurrentOwningAgentUid = output.OwnedBy[0]; } else { //add warning? if (output.OfferedBy == null && output.OfferedBy.Count == 0) { status.AddWarning("document doesn't have an owning or offering organization."); } } //HasPathway is required, so must have data output.HasPathwayList = helper.MapEntityReferenceGuids("PathwaySet.HasPathway", input.HasPathway, thisEntityTypeId, ref status); // //need to check if pathways have been imported. Normally would have as pathway is published before the set //=== if any messages were encountered treat as warnings for now if (messages.Count > 0) { status.SetMessages(messages, true); } //adding common import pattern importSuccessfull = mgr.PathwaySetImport(output, ref status); status.DocumentId = output.Id; status.DetailPageUrl = string.Format("~/pathwayset/{0}", output.Id); status.DocumentRowId = output.RowId; //if record was added to db, add to/or set EntityResolution as resolved int ierId = new ImportManager().Import_EntityResolutionAdd(referencedAtId, ctid, CodesManager.ENTITY_TYPE_PATHWAY_SET, output.RowId, output.Id, (output.Id > 0), ref messages, output.Id > 0); //just in case - not sure if applicable, as will want to do components if the pathway exists if (status.HasErrors) { importSuccessfull = false; } // } catch (Exception ex) { LoggingHelper.LogError(ex, string.Format("Exception encountered in envelopeId: {0}", envelopeIdentifier), false, "PathwaySet Import exception"); } return(importSuccessfull); }
//public bool Import( InputEntity input, string envelopeIdentifier, SaveStatus status ) //{ // List<string> messages = new List<string>(); // bool importSuccessfull = false; // EntityServices mgr = new EntityServices(); // //try // //{ // //input = JsonConvert.DeserializeObject<InputEntity>( item.DecodedResource.ToString() ); // string ctid = input.Ctid; // string referencedAtId = input.CtdlId; // LoggingHelper.DoTrace( 5, " name: " + input.Name ); // LoggingHelper.DoTrace( 6, " url: " + input.SubjectWebpage ); // LoggingHelper.DoTrace( 5, " ctid: " + input.Ctid ); // LoggingHelper.DoTrace( 5, " @Id: " + input.CtdlId ); // status.Ctid = ctid; // if ( status.DoingDownloadOnly ) // return true; // if ( !DoesEntityExist( input.Ctid, ref output ) ) // { // output.RowId = Guid.NewGuid(); // } // //re:messages - currently passed to mapping but no errors are trapped?? // // - should use SaveStatus and skip import if errors encountered (vs warnings) // output.Name = input.Name; // output.Description = input.Description; // output.CTID = input.Ctid; // output.CredentialRegistryId = envelopeIdentifier; // output.SubjectWebpage = input.SubjectWebpage; // output.OwningAgentUid = MappingHelper.MapOrganizationReferencesGuid( input.ConditionManifestOf, ref status ); // output.Requires = MappingHelper.FormatConditionProfile( input.Requires, ref status ); // output.Recommends = MappingHelper.FormatConditionProfile( input.Recommends, ref status ); // output.EntryCondition = MappingHelper.FormatConditionProfile( input.EntryConditions, ref status ); // output.Corequisite = MappingHelper.FormatConditionProfile( input.Corequisite, ref status ); // output.Renewal = MappingHelper.FormatConditionProfile( input.Renewal, ref status ); // status.DocumentId = output.Id; // status.DocumentRowId = output.RowId; // //=== if any messages were encountered treat as warnings for now // if ( messages.Count > 0 ) // status.SetMessages( messages, true ); // importSuccessfull = mgr.Import( output, ref status ); // //just in case // if ( status.HasErrors ) // importSuccessfull = false; // //if record was added to db, add to/or set EntityResolution as resolved // int ierId = new ImportManager().Import_EntityResolutionAdd( referencedAtId, // ctid, // CodesManager.ENTITY_TYPE_CONDITION_MANIFEST, // output.RowId, // output.Id, // false, // ref messages, // output.Id > 0 ); // return importSuccessfull; //} public bool ImportV3(string payload, string envelopeIdentifier, SaveStatus status) { InputEntityV3 input = new InputEntityV3(); var bnodes = new List <BNodeV3>(); var mainEntity = new Dictionary <string, object>(); //status.AddWarning( "The resource uses @graph and is not handled yet" ); Dictionary <string, object> dictionary = RegistryServices.JsonToDictionary(payload); object graph = dictionary["@graph"]; //serialize the graph object var glist = JsonConvert.SerializeObject(graph); //parse graph in to list of objects JArray graphList = JArray.Parse(glist); int cntr = 0; foreach (var item in graphList) { cntr++; if (cntr == 1) { var main = item.ToString(); //may not use this. Could add a trace method mainEntity = RegistryServices.JsonToDictionary(main); input = JsonConvert.DeserializeObject <InputEntityV3>(main); } else { var bn = item.ToString(); bnodes.Add(JsonConvert.DeserializeObject <BNodeV3>(bn)); } } List <string> messages = new List <string>(); bool importSuccessfull = false; EntityServices mgr = new EntityServices(); MappingHelperV3 helper = new MappingHelperV3(19); helper.entityBlankNodes = bnodes; helper.CurrentEntityCTID = input.CTID; helper.CurrentEntityName = input.Name.ToString(); //try //{ //input = JsonConvert.DeserializeObject<InputEntity>( item.DecodedResource.ToString() ); string ctid = input.CTID; string referencedAtId = input.CtdlId; LoggingHelper.DoTrace(5, " name: "+ input.Name.ToString()); LoggingHelper.DoTrace(6, " url: "+ input.SubjectWebpage); LoggingHelper.DoTrace(5, " ctid: "+ input.CTID); LoggingHelper.DoTrace(5, " @Id: "+ input.CtdlId); status.Ctid = ctid; if (status.DoingDownloadOnly) { return(true); } try { if (!DoesEntityExist(input.CTID, ref output)) { output.RowId = Guid.NewGuid(); LoggingHelper.DoTrace(1, string.Format(thisClassName + ".ImportV3(). Record was NOT found using CTID: '{0}'", input.CTID)); } else { LoggingHelper.DoTrace(1, string.Format(thisClassName + ".ImportV3(). Found record: '{0}' using CTID: '{1}'", input.Name, input.CTID)); } helper.currentBaseObject = output; //re:messages - currently passed to mapping but no errors are trapped?? // - should use SaveStatus and skip import if errors encountered (vs warnings) output.Name = helper.HandleLanguageMap(input.Name, output, "Name"); output.Description = helper.HandleLanguageMap(input.Description, output, "Description"); output.CTID = input.CTID; output.CredentialRegistryId = envelopeIdentifier; output.SubjectWebpage = input.SubjectWebpage; output.OwningAgentUid = helper.MapOrganizationReferencesGuid("ConditionManifest.OwningAgentUid", input.ConditionManifestOf, ref status); helper.CurrentOwningAgentUid = output.OwningAgentUid; output.Requires = helper.FormatConditionProfile(input.Requires, ref status); output.Recommends = helper.FormatConditionProfile(input.Recommends, ref status); output.EntryCondition = helper.FormatConditionProfile(input.EntryConditions, ref status); output.Corequisite = helper.FormatConditionProfile(input.Corequisite, ref status); output.Renewal = helper.FormatConditionProfile(input.Renewal, ref status); status.DocumentId = output.Id; status.DocumentRowId = output.RowId; //=== if any messages were encountered treat as warnings for now if (messages.Count > 0) { status.SetMessages(messages, true); } importSuccessfull = mgr.Import(output, ref status); //just in case if (status.HasErrors) { importSuccessfull = false; } //if record was added to db, add to/or set EntityResolution as resolved int ierId = new ImportManager().Import_EntityResolutionAdd(referencedAtId, ctid, CodesManager.ENTITY_TYPE_CONDITION_MANIFEST, output.RowId, output.Id, false, ref messages, output.Id > 0); } catch (Exception ex) { LoggingHelper.LogError(ex, string.Format("CostManifest ImportV3. Exception encountered for CTID: {0}", ctid), false, "CostManifest Import exception"); } finally { } return(importSuccessfull); }
protected override void OnInit(EventArgs e) { _importManager = new ImportManager(); _exportManager = new ExportManager(); _exportersCheckBoxes = CreateCheckBoxes(exportersPanel, _exportManager.Exporters.Select(exporter => new cbLinks { Id = exporter.EntityKey, Text = Helpers.GetEntityName(exporter.EntityKey), Link = exporter.UiUrl }), true); _deletersCheckBoxes = CreateCheckBoxes(deletersPanel, _exportManager.Exporters.Select(exporter => new cbLinks { Id = exporter.EntityKey, Text = Helpers.GetEntityName(exporter.EntityKey), Link = exporter.UiUrl }), false); _exportManager.GetSites().ForEach(s => exportSite.Items.Add(new ListItem(s.Name, s.Id))); _exportManager.GetSites().ForEach(s => importSite.Items.Add(new ListItem(s.Name, s.Id))); _exportManager.GetSites().ForEach(s => deleteSite.Items.Add(new ListItem(s.Name, s.Id))); _exportManager.GetLanguages().ForEach(l => exportLanguage.Items.Add(new ListItem(l.Name, l.Id))); _exportManager.GetLanguages().ForEach(l => deleteLanguage.Items.Add(new ListItem(l.Name, l.Id))); base.OnInit(e); }
private ImportManager CreateImportManager() { ImportManager importManager = new ImportManager(); return importManager; }
static void Main(string[] args) { bool isAutomaticMode = true; bool isAPIImportMode = false; string importFolder = ""; string OCM_API_Identifier = null; string OCM_API_SessionToken = null; LogEvent("Starting Import:", true); if (args.Length > 0) { LogEvent("Arguments supplied: "); for (int i = 0; i < args.Length; i++) { string arg = args[i]; LogEvent("arg: " + arg); if (arg == "-auto") { isAutomaticMode = true; } if (arg == "-api") { isAPIImportMode = true; } //if (arg == "-log") EnableLogging = true; try { if (arg == "-api_identifier") { OCM_API_Identifier = args[i + 1]; } if (arg == "-api_sessiontoken") { OCM_API_SessionToken = args[i + 1]; } } catch (Exception) { LogEvent("Invalid parameter supplied."); } } } else { LogEvent("No Arguments supplied."); } if (isAutomaticMode) { bool actionPerformed = false; ActionType mode = ActionType.ClientTest; if (mode == ActionType.DataImport && isAPIImportMode == true && OCM_API_Identifier != null && OCM_API_SessionToken != null) { ExportType exportType = ExportType.API; ImportManager importManager = new ImportManager(new ImportSettings { TempFolderPath = importFolder, MasterAPIBaseUrl = "https://api.openchargemap.io/v3" }); LogEvent("Performing Import, Publishing via API (" + OCM_API_Identifier + ":" + OCM_API_SessionToken + "): " + DateTime.UtcNow.ToShortTimeString()); Task <bool> processing = importManager.PerformImportProcessing(new ImportProcessSettings { ExportType = exportType, DefaultDataPath = importFolder, ApiIdentifier = OCM_API_Identifier, ApiSessionToken = OCM_API_SessionToken }); processing.Wait(); LogEvent("Import Processed. Exiting. " + DateTime.UtcNow.ToShortTimeString()); actionPerformed = true; } if (mode == ActionType.NetworkServices) { //network service polling //OCM.API.NetworkServices.ServiceManager serviceManager = new OCM.API.NetworkServices.ServiceManager(); //serviceManager.Test(OCM.API.NetworkServices.ServiceProvider.CoulombChargePoint); } if (!actionPerformed) { LogEvent("Nothing to do. Exiting. " + DateTime.UtcNow.ToShortTimeString()); } } }
} // public bool Import(string payload, string envelopeIdentifier, SaveStatus status) { LoggingHelper.DoTrace(7, "ImportCompetencyFramesworks - entered."); List <string> messages = new List <string>(); MappingHelperV3 helper = new MappingHelperV3(10); bool importSuccessfull = true; InputEntity input = new InputEntity(); InputCompetency comp = new InputCompetency(); var mainEntity = new Dictionary <string, object>(); List <InputCompetency> competencies = new List <InputCompetency>(); Dictionary <string, object> dictionary = RegistryServices.JsonToDictionary(payload); object graph = dictionary["@graph"]; //serialize the graph object var glist = JsonConvert.SerializeObject(graph); //parse graph in to list of objects JArray graphList = JArray.Parse(glist); var bnodes = new List <BNode>(); int cntr = 0; foreach (var item in graphList) { cntr++; //note older frameworks will not be in the priority order var main = item.ToString(); if (cntr == 1 || main.IndexOf("ceasn:CompetencyFramework") > -1) { //HACK if (main.IndexOf("ceasn:CompetencyFramework") > -1) { input = JsonConvert.DeserializeObject <InputEntity>(main); } } else { //Error converting value "https://credentialengineregistry.org/resources/ce-949fcaba-45ed-44d9-88bf-43677277eb84" to type 'System.Collections.Generic.List`1[System.String]'. Path 'ceasn:isPartOf', line 11, position 108. //not set up to handle issues //comp = JsonConvert.DeserializeObject<InputCompetency>( item.ToString() ); //competencies.Add( comp ); //should just have competencies, but should check for bnodes var child = item.ToString(); if (child.IndexOf("_:") > -1) { bnodes.Add(JsonConvert.DeserializeObject <BNode>(child)); //ceasn:Competency } else if (child.IndexOf("ceasn:Competency") > -1) { competencies.Add(JsonConvert.DeserializeObject <InputCompetency>(child)); } else { //unexpected } } } //try //{ //input = JsonConvert.DeserializeObject<InputGraph>( item.DecodedResource.ToString() ); string ctid = input.CTID; status.Ctid = ctid; string referencedAtId = input.CtdlId; LoggingHelper.DoTrace(5, " ctid: "+ ctid); LoggingHelper.DoTrace(5, " @Id: "+ input.CtdlId); LoggingHelper.DoTrace(5, " name: "+ input.name.ToString()); string framework = input.name.ToString(); var org = new MC.Organization(); string orgCTID = ""; string orgName = ""; List <string> publisher = input.publisher; //20-06-11 - need to get creator, publisher, owner where possible // include an org reference with name, swp, and?? //should check creator first? Or will publisher be more likely to have an account Ctid? if (publisher != null && publisher.Count() > 0) { orgCTID = ResolutionServices.ExtractCtid(publisher[0]); //look up org name org = OrganizationManager.GetSummaryByCtid(orgCTID); } else { //try creator List <string> creator = input.creator; if (creator != null && creator.Count() > 0) { orgCTID = ResolutionServices.ExtractCtid(creator[0]); //look up org name org = OrganizationManager.GetSummaryByCtid(orgCTID); } } if (status.DoingDownloadOnly) { return(true); } //add/updating CompetencyFramework //21-02-22 HUH - WHY ARE WE USING ef here instead of output Framework ef = new Framework(); if (!DoesEntityExist(input.CTID, ref ef)) { //set the rowid now, so that can be referenced as needed //output.RowId = Guid.NewGuid(); ef.RowId = Guid.NewGuid(); LoggingHelper.DoTrace(1, string.Format(thisClassName + ".Import(). Record was NOT found using CTID: '{0}'", input.CTID)); } else { LoggingHelper.DoTrace(1, string.Format(thisClassName + ".Import(). Found record: '{0}' using CTID: '{1}'", input.name, input.CTID)); } helper.currentBaseObject = ef; ef.ExistsInRegistry = true; //store graph ef.CompetencyFrameworkGraph = glist; ef.TotalCompetencies = competencies.Count(); ef.Name = helper.HandleLanguageMap(input.name, ef, "Name"); ef.Description = helper.HandleLanguageMap(input.description, ef, "description"); ef.CTID = input.CTID; ef.OrganizationCTID = orgCTID; if (org != null && org.Id > 0) { orgName = org.Name; ef.OrganizationId = org.Id; helper.CurrentOwningAgentUid = org.RowId; } helper.MapInLanguageToTextValueProfile(input.inLanguage, "CompetencyFramework.InLanguage.CTID: " + ctid); //foreach ( var l in input.InLanguage ) //{ // if ( !string.IsNullOrWhiteSpace( l ) ) // { // var language = CodesManager.GetLanguage( l ); // output.InLanguageCodeList.Add( new TextValueProfile() // { // CodeId = language.CodeId, // TextTitle = language.Name, // TextValue = language.Value // } ); // } //} //TBD handling of referencing third party publisher if (!string.IsNullOrWhiteSpace(status.DocumentPublishedBy)) { //output.PublishedByOrganizationCTID = status.DocumentPublishedBy; var porg = OrganizationManager.GetSummaryByCtid(status.DocumentPublishedBy); if (porg != null && porg.Id > 0) { //TODO - store this in a json blob?????????? //this will result in being added to Entity.AgentRelationship ef.PublishedBy = new List <Guid>() { porg.RowId }; } else { //if publisher not imported yet, all publishee stuff will be orphaned var entityUid = Guid.NewGuid(); var statusMsg = ""; var resPos = referencedAtId.IndexOf("/resources/"); var swp = referencedAtId.Substring(0, (resPos + "/resources/".Length)) + status.DocumentPublishedBy; int orgId = new OrganizationManager().AddPendingRecord(entityUid, status.DocumentPublishedBy, swp, ref statusMsg); } } else { //may need a check for existing published by to ensure not lost if (ef.Id > 0) { //if ( ef.OrganizationRole != null && ef.OrganizationRole.Any() ) //{ // var publishedByList = ef.OrganizationRole.Where( s => s.RoleTypeId == 30 ).ToList(); // if ( publishedByList != null && publishedByList.Any() ) // { // var pby = publishedByList[ 0 ].ActingAgentUid; // ef.PublishedBy = new List<Guid>() { publishedByList[ 0 ].ActingAgentUid }; // } //} } } ef.CredentialRegistryId = envelopeIdentifier; //additions //ef.ind //can only handle one source int pcnt = 0; if (input.source != null) { foreach (var url in input.source) { pcnt++; ef.SourceUrl = url; break; } } ef.FrameworkUri = input.CtdlId; ef.hasTopChild = input.hasTopChild; // ApiEntity apiFramework = new ApiEntity() { Name = ef.Name, CTID = ef.CTID, Source = ef.SourceUrl, HasTopChild = ef.hasTopChild, Meta_LastUpdated = status.LocalUpdatedDate }; //?store competencies in string? if (competencies != null && competencies.Count > 0) { ef.TotalCompetencies = competencies.Count(); //TODO - should we limit this if 1000+ //do we use competencies in elastic? if not pause this usage cntr = 0; foreach (var c in competencies) { cntr++; var comments = helper.HandleLanguageMapList(c.comment, ef); ef.Competencies.Add( new workIT.Models.Elastic.IndexCompetency() { Name = c.competencyText.ToString(), //CTID = c.CTID, //Description = comments != null && comments.Count() > 0 ? comments[0].ToString() : "" } ); if (cntr >= 1000) { break; } } } //20-07-02 just storing the index ready competencies //ef.CompentenciesJson = JsonConvert.SerializeObject( competencies, MappingHelperV3.GetJsonSettings() ); ef.CompentenciesStore = JsonConvert.SerializeObject(ef.Competencies, MappingHelperV3.GetJsonSettings()); //test //ElasticManager.LoadCompetencies( ef.Name, ef.CompentenciesStore ); FormatCompetenciesHierarchy(apiFramework, competencies, helper); //TODO store whole framework or just the competencies? apiFramework.HasTopChild = null; ef.APIFramework = JsonConvert.SerializeObject(apiFramework, MappingHelperV3.GetJsonSettings()); // if (ef.TotalCompetencies == input.hasTopChild.Count()) { //flat list - use for simple display } else { foreach (var item in input.hasTopChild) { } } //adding using common import pattern new CompetencyFrameworkServices().Import(ef, ref status); status.DocumentId = ef.Id; status.DetailPageUrl = string.Format("~/competencyframework/{0}", ef.Id); status.DocumentRowId = ef.RowId; // //just in case if (status.HasErrors) { importSuccessfull = false; } //if record was added to db, add to/or set EntityResolution as resolved int ierId = new ImportManager().Import_EntityResolutionAdd(referencedAtId, ctid, CodesManager.ENTITY_TYPE_COMPETENCY_FRAMEWORK, output.RowId, output.Id, (output.Id > 0), ref messages, output.Id > 0); // //framework checks if (input.inLanguage == null || input.inLanguage.Count() == 0) { //document for followup //LoggingHelper.DoTrace( 5, " Framework missing inLanguage: " + input.name.ToString() ); } //output.Name = helper.HandleLanguageMap( input.name, output, "Name" ); //output.description = helper.HandleLanguageMap( input.description, output, "Description" ); //output.CTID = input.CTID; return(importSuccessfull); }
protected virtual List <TEntity> GetServiceData(Item accountItem) { return(ImportManager.ImportList <TEntity>(this.ImportName, accountItem)); }
private ImportManager CreateImportManager() { ImportManager importManager = new ImportManager(); return(importManager); }
private async void PerformTasks(object?state) { if (!_isImportInProgress) { _isImportInProgress = true; _logger.LogInformation("Checking import status.."); var tempPath = Path.GetTempPath(); var importManager = new ImportManager(_settings, _logger); try { var status = new ImportStatus { DateLastImport = DateTime.UtcNow, LastImportedProvider = "", LastImportStatus = "Started" }; var statusPath = Path.Combine(tempPath, "import_status.json"); if (File.Exists(statusPath)) { status = System.Text.Json.JsonSerializer.Deserialize <ImportStatus>(File.ReadAllText(statusPath)); } var allProviders = _settings.EnabledImports; var indexOfLastProvider = allProviders.IndexOf(status.LastImportedProvider ?? ""); var indexOfNextProvider = 0; // if there is a next provider use that, otherwise use first provider if (indexOfLastProvider > -1) { if (indexOfLastProvider < allProviders.Count - 1) { // next provider indexOfNextProvider = indexOfLastProvider + 1; } } status.LastImportedProvider = allProviders[indexOfNextProvider]; status.DateLastImport = DateTime.UtcNow; _logger.LogInformation($"Performing Import [{status.LastImportedProvider}], Publishing via API: {status.DateLastImport.Value.ToShortTimeString()}"); try { var stopwatch = Stopwatch.StartNew(); var importedOK = await importManager.PerformImportProcessing( new ImportProcessSettings { ExportType = Providers.ExportType.JSONAPI, DefaultDataPath = tempPath, FetchLiveData = true, FetchExistingFromAPI = true, PerformDeduplication = true, ProviderName = status.LastImportedProvider // leave blank to do all }); status.LastImportStatus = importedOK ? "Imported" : "Failed"; status.DateLastImport = DateTime.UtcNow; status.ProcessingTimeSeconds = stopwatch.Elapsed.TotalSeconds; // update import date on provider var providers = importManager.GetImportProviders(new List <API.Common.Model.DataProvider>()); var providerId = (providers.FirstOrDefault(p => p.GetProviderName() == status.LastImportedProvider) as BaseImportProvider)?.DataProviderID; if (providerId != null) { // notify API of last date of import for each provider await importManager.UpdateLastImportDate((int)providerId); } } catch (Exception exp) { _logger.LogError("Import failed: " + exp); status.LastImportStatus = "Failed with unknown exception"; } File.WriteAllText(statusPath, System.Text.Json.JsonSerializer.Serialize <ImportStatus>(status)); } catch (Exception exp) { _logger.LogError("Import failed: " + exp.ToString()); } _isImportInProgress = false; } else { _logger.LogInformation("Import already in progress.."); } }
public bool Import(InputEntity input, string envelopeIdentifier, SaveStatus status) { List <string> messages = new List <string>(); bool importSuccessfull = false; EntityServices mgr = new EntityServices(); string ctid = input.Ctid; string referencedAtId = input.CtdlId; LoggingHelper.DoTrace(5, " name: "+ input.Name); LoggingHelper.DoTrace(6, " url: "+ input.SubjectWebpage); LoggingHelper.DoTrace(5, " ctid: "+ input.Ctid); LoggingHelper.DoTrace(5, " @Id: "+ input.CtdlId); status.Ctid = ctid; if (status.DoingDownloadOnly) { return(true); } if (!DoesEntityExist(input.Ctid, ref output)) { output.RowId = Guid.NewGuid(); } //re:messages - currently passed to mapping but no errors are trapped?? // - should use SaveStatus and skip import if errors encountered (vs warnings) output.Name = input.Name; output.Description = input.Description; output.CTID = input.Ctid; output.CredentialRegistryId = envelopeIdentifier; output.DateEffective = input.DateEffective; output.SubjectWebpage = input.SubjectWebpage; output.Subject = MappingHelper.MapCAOListToTextValueProfile(input.Subject, CodesManager.PROPERTY_CATEGORY_SUBJECT); output.Keyword = MappingHelper.MapToTextValueProfile(input.Keyword); output.AvailabilityListing = MappingHelper.MapListToString(input.AvailabilityListing); output.AvailableOnlineAt = MappingHelper.MapListToString(input.AvailableOnlineAt); output.AssessmentExample = input.AssessmentExample; output.ExternalResearch = MappingHelper.MapListToString(input.ExternalResearch); output.AssessmentExampleDescription = input.AssessmentExampleDescription; output.AssessmentMethodType = MappingHelper.MapCAOListToEnumermation(input.AssessmentMethodType); output.AudienceType = MappingHelper.MapCAOListToEnumermation(input.AudienceType); output.VersionIdentifier = MappingHelper.MapIdentifierValueListToString(input.VersionIdentifier); output.VersionIdentifierList = MappingHelper.MapIdentifierValueList(input.VersionIdentifier); //To be looked output.CodedNotation = input.CodedNotation; output.AssessmentOutput = input.AssessmentOutput; output.AssessmentUseType = MappingHelper.MapCAOListToEnumermation(input.AssessmentUseType); output.DeliveryType = MappingHelper.MapCAOListToEnumermation(input.DeliveryType); output.DeliveryTypeDescription = input.DeliveryTypeDescription; output.IsProctored = input.IsProctored; output.HasGroupEvaluation = input.HasGroupEvaluation; output.HasGroupParticipation = input.HasGroupParticipation; foreach (var l in input.InLanguage) { if (!string.IsNullOrWhiteSpace(l)) { var language = CodesManager.GetLanguage(l); output.InLanguageCodeList.Add(new TextValueProfile() { CodeId = language.CodeId, TextTitle = language.Name, TextValue = language.Value }); } } output.ProcessStandards = input.ProcessStandards; output.ProcessStandardsDescription = input.ProcessStandardsDescription; output.ScoringMethodDescription = input.ScoringMethodDescription; output.ScoringMethodExample = input.ScoringMethodExample; output.ScoringMethodExampleDescription = input.ScoringMethodExampleDescription; output.ScoringMethodType = MappingHelper.MapCAOListToEnumermation(input.ScoringMethodType); //TBD - a custom version //output.InstructionalProgramType = MappingHelper.MapCAOListToEnumermation( input.InstructionalProgramType ); output.InstructionalProgramTypes = MappingHelper.MapCAOListToFramework(input.InstructionalProgramType); //output.CreditHourType = input.CreditHourType; //output.CreditHourValue = input.CreditHourValue; output.CreditUnitType = MappingHelper.MapCAOToEnumermation(input.CreditUnitType); output.CreditUnitValue = input.CreditUnitValue; output.CreditUnitTypeDescription = input.CreditUnitTypeDescription; output.Jurisdiction = MappingHelper.MapToJurisdiction(input.Jurisdiction, ref status); //EstimatedCost //will need to format, all populate Entity.RelatedCosts (for bubble up) - actually this would be for asmts, and lopps output.EstimatedCost = MappingHelper.FormatCosts(input.EstimatedCost, ref status); //common conditions output.ConditionManifestIds = MappingHelper.MapEntityReferences(input.CommonConditions, CodesManager.ENTITY_TYPE_CONDITION_MANIFEST, CodesManager.ENTITY_TYPE_ASSESSMENT_PROFILE, ref status); //common costs output.CostManifestIds = MappingHelper.MapEntityReferences(input.CommonCosts, CodesManager.ENTITY_TYPE_COST_MANIFEST, CodesManager.ENTITY_TYPE_ASSESSMENT_PROFILE, ref status); //connections output.AdvancedStandingFrom = MappingHelper.FormatConditionProfile(input.AdvancedStandingFrom, ref status); output.AdvancedStandingFor = MappingHelper.FormatConditionProfile(input.IsAdvancedStandingFor, ref status); output.PreparationFrom = MappingHelper.FormatConditionProfile(input.PreparationFrom, ref status); output.IsPreparationFor = MappingHelper.FormatConditionProfile(input.IsPreparationFor, ref status); output.IsRequiredFor = MappingHelper.FormatConditionProfile(input.IsRequiredFor, ref status); output.IsRecommendedFor = MappingHelper.FormatConditionProfile(input.IsRecommendedFor, ref status); //EstimatedDuration ============================== output.EstimatedDuration = MappingHelper.FormatDuration(input.EstimatedDuration, ref status); //conditions ====================================== output.Requires = MappingHelper.FormatConditionProfile(input.Requires, ref status); output.Recommends = MappingHelper.FormatConditionProfile(input.Recommends, ref status); output.EntryCondition = MappingHelper.FormatConditionProfile(input.EntryCondition, ref status); output.Corequisite = MappingHelper.FormatConditionProfile(input.Corequisite, ref status); //Process profiles ============================== output.AdministrationProcess = MappingHelper.FormatProcessProfile(input.AdministrationProcess, ref status); output.DevelopmentProcess = MappingHelper.FormatProcessProfile(input.DevelopmentProcess, ref status); output.MaintenanceProcess = MappingHelper.FormatProcessProfile(input.MaintenanceProcess, ref status); // output.Addresses = MappingHelper.FormatAvailableAtAddresses(input.AvailableAt, ref status); //BYs output.AccreditedBy = MappingHelper.MapOrganizationReferenceGuids(input.AccreditedBy, ref status); output.ApprovedBy = MappingHelper.MapOrganizationReferenceGuids(input.ApprovedBy, ref status); output.OfferedBy = MappingHelper.MapOrganizationReferenceGuids(input.OfferedBy, ref status); output.OwnedBy = MappingHelper.MapOrganizationReferenceGuids(input.OwnedBy, ref status); if (output.OwnedBy != null && output.OwnedBy.Count > 0) { output.OwningAgentUid = output.OwnedBy[0]; } else { //add warning? if (output.OfferedBy == null && output.OfferedBy.Count == 0) { status.AddWarning("document doesn't have an owning or offering organization."); } } output.RecognizedBy = MappingHelper.MapOrganizationReferenceGuids(input.RecognizedBy, ref status); output.RegulatedBy = MappingHelper.MapOrganizationReferenceGuids(input.RegulatedBy, ref status); //INs output.AccreditedIn = MappingHelper.MapToJurisdiction(input.AccreditedIn, ref status); output.ApprovedIn = MappingHelper.MapToJurisdiction(input.ApprovedIn, ref status); output.OfferedIn = MappingHelper.MapToJurisdiction(input.OfferedIn, ref status); output.RecognizedIn = MappingHelper.MapToJurisdiction(input.RecognizedIn, ref status); output.RegulatedIn = MappingHelper.MapToJurisdiction(input.RegulatedIn, ref status); //FinancialAssistance ============================ output.FinancialAssistanceOLD = MappingHelper.FormatFinancialAssistance(input.FinancialAssistance, ref status); //assesses compentencies output.AssessesCompetencies = MappingHelper.MapCAOListToCompetencies(input.Assesses); //=== if any messages were encountered treat as warnings for now if (messages.Count > 0) { status.SetMessages(messages, true); } //just in case check if entity added since start if (output.Id == 0) { ThisEntity entity = EntityServices.GetByCtid(ctid); if (entity != null && entity.Id > 0) { output.Id = entity.Id; output.RowId = entity.RowId; } } importSuccessfull = mgr.Import(output, ref status); status.DocumentId = output.Id; status.DetailPageUrl = string.Format("~/assessment/{0}", output.Id); status.DocumentRowId = output.RowId; //just in case if (status.HasErrors) { importSuccessfull = false; } //if record was added to db, add to/or set EntityResolution as resolved int ierId = new ImportManager().Import_EntityResolutionAdd(referencedAtId, ctid, CodesManager.ENTITY_TYPE_ASSESSMENT_PROFILE, output.RowId, output.Id, false, ref messages, output.Id > 0); return(importSuccessfull); }
public bool ImportV3(string payload, string envelopeIdentifier, SaveStatus status) { InputEntityV3 input = new InputEntityV3(); var bnodes = new List <BNode>(); var mainEntity = new Dictionary <string, object>(); //status.AddWarning( "The resource uses @graph and is not handled yet" ); Dictionary <string, object> dictionary = RegistryServices.JsonToDictionary(payload); object graph = dictionary["@graph"]; //serialize the graph object var glist = JsonConvert.SerializeObject(graph); //parse graph in to list of objects JArray graphList = JArray.Parse(glist); int cntr = 0; foreach (var item in graphList) { cntr++; if (cntr == 1) { var main = item.ToString(); //may not use this. Could add a trace method mainEntity = RegistryServices.JsonToDictionary(main); input = JsonConvert.DeserializeObject <InputEntityV3>(main); } else { var bn = item.ToString(); bnodes.Add(JsonConvert.DeserializeObject <BNode>(bn)); } } List <string> messages = new List <string>(); bool importSuccessfull = false; EntityServices mgr = new EntityServices(); MappingHelperV3 helper = new MappingHelperV3(); helper.entityBlankNodes = bnodes; string ctid = input.CTID; string referencedAtId = input.CtdlId; LoggingHelper.DoTrace(5, " name: "+ input.Name.ToString()); LoggingHelper.DoTrace(6, " url: "+ input.SubjectWebpage); LoggingHelper.DoTrace(5, " ctid: "+ input.CTID); LoggingHelper.DoTrace(5, " @Id: "+ input.CtdlId); status.Ctid = ctid; if (status.DoingDownloadOnly) { return(true); } if (!DoesEntityExist(input.CTID, ref output)) { //set the rowid now, so that can be referenced as needed output.RowId = Guid.NewGuid(); } helper.currentBaseObject = output; //start with language and may use with language maps foreach (var l in input.InLanguage) { if (!string.IsNullOrWhiteSpace(l)) { var language = CodesManager.GetLanguage(l); output.InLanguageCodeList.Add(new TextValueProfile() { CodeId = language.CodeId, TextTitle = language.Name, TextValue = language.Value }); } } if (input.InLanguage.Count > 0) { //could use to alter helper.DefaultLanguage } output.Name = helper.HandleLanguageMap(input.Name, output, "Name"); output.Description = helper.HandleLanguageMap(input.Description, output, "Description"); output.CTID = input.CTID; output.CredentialRegistryId = envelopeIdentifier; output.DateEffective = input.DateEffective; output.SubjectWebpage = input.SubjectWebpage; output.Subject = helper.MapCAOListToTextValueProfile(input.Subject, CodesManager.PROPERTY_CATEGORY_SUBJECT); output.Keyword = helper.MapToTextValueProfile(input.Keyword, output, "Keyword"); output.AvailabilityListing = helper.MapListToString(input.AvailabilityListing); output.AvailableOnlineAt = helper.MapListToString(input.AvailableOnlineAt); output.AssessmentExample = input.AssessmentExample; output.ExternalResearch = helper.MapListToString(input.ExternalResearch); output.AssessmentExampleDescription = helper.HandleLanguageMap(input.AssessmentExampleDescription, output, "AssessmentExampleDescription"); output.AssessmentMethodType = helper.MapCAOListToEnumermation(input.AssessmentMethodType); output.AudienceType = helper.MapCAOListToEnumermation(input.AudienceType); //CAO output.AudienceLevelType = helper.MapCAOListToEnumermation(input.AudienceLevelType); output.VersionIdentifier = helper.MapIdentifierValueListToString(input.VersionIdentifier); output.VersionIdentifierList = helper.MapIdentifierValueList(input.VersionIdentifier); //To be looked output.CodedNotation = input.CodedNotation; output.AssessmentOutput = helper.HandleLanguageMap(input.AssessmentOutput, output, "AssessmentOutput"); output.AssessmentUseType = helper.MapCAOListToEnumermation(input.AssessmentUseType); output.DeliveryType = helper.MapCAOListToEnumermation(input.DeliveryType); output.DeliveryTypeDescription = helper.HandleLanguageMap(input.DeliveryTypeDescription, output, "DeliveryTypeDescription"); output.IsProctored = input.IsProctored; output.HasGroupEvaluation = input.HasGroupEvaluation; output.HasGroupParticipation = input.HasGroupParticipation; output.ProcessStandards = input.ProcessStandards; output.ProcessStandardsDescription = helper.HandleLanguageMap(input.ProcessStandardsDescription, output, "ProcessStandardsDescription"); output.ScoringMethodDescription = helper.HandleLanguageMap(input.ScoringMethodDescription, output, "ScoringMethodDescription"); output.ScoringMethodExample = input.ScoringMethodExample; output.ScoringMethodExampleDescription = helper.HandleLanguageMap(input.ScoringMethodExampleDescription, output, "ScoringMethodExampleDescription"); output.ScoringMethodType = helper.MapCAOListToEnumermation(input.ScoringMethodType); //TBD - a custom version //output.InstructionalProgramType = helper.MapCAOListToEnumermation( input.InstructionalProgramType ); //occupations output.Occupations = helper.MapCAOListToCAOProfileList(input.OccupationType); //just append alternative items. Ensure empty lists are ignored output.Occupations.AddRange(helper.AppendLanguageMapListToCAOProfileList(input.AlternativeOccupationType)); //skip if no occupations if (output.Occupations.Count() == 0 && UtilityManager.GetAppKeyValue("skipCredImportIfNoOccupations", false)) { //LoggingHelper.DoTrace( 2, string.Format( " ***Skipping Credential# {0}, {1} as it has no occupations and this is a special run.", output.Id, output.Name ) ); //return true; } //Industries output.Industries = helper.MapCAOListToCAOProfileList(input.IndustryType); output.Industries.AddRange(helper.AppendLanguageMapListToCAOProfileList(input.AlternativeIndustryType)); //naics //output.Naics = input.Naics; output.InstructionalProgramTypes = helper.MapCAOListToCAOProfileList(input.InstructionalProgramType); output.InstructionalProgramTypes.AddRange(helper.AppendLanguageMapListToCAOProfileList(input.AlternativeInstructionalProgramType)); if (output.InstructionalProgramTypes.Count() == 0 && UtilityManager.GetAppKeyValue("skipAsmtImportIfNoCIP", false)) { //skip LoggingHelper.DoTrace(2, string.Format(" ***Skipping asmt# {0}, {1} as it has no InstructionalProgramTypes and this is a special run.", output.Id, output.Name)); return(true); } //handle QuantitativeValue output.CreditValue = helper.HandleQuantitiveValue(input.CreditValue, output, "CreditValue"); // if (!output.CreditValue.HasData()) { //if ( UtilityManager.GetAppKeyValue( "usingQuantitiveValue", false ) ) //{ //will not handle ranges //output.CreditValue = new workIT.Models.Common.QuantitativeValue //{ // Value = input.CreditHourValue, // CreditUnitType = helper.MapCAOToEnumermation( input.CreditUnitType ), // Description = helper.HandleLanguageMap( input.CreditUnitTypeDescription, output, "CreditUnitTypeDescription" ) //}; ////what about hours? ////output.CreditHourType = helper.HandleLanguageMap( input.CreditHourType, output, "CreditHourType" ); ////output.CreditHourValue = input.CreditHourValue; ////if there is hour data, can't be unit data, so assign //if ( input.CreditHourValue > 0 ) //{ // output.CreditValue.Value = input.CreditHourValue; // output.CreditValue.Description = helper.HandleLanguageMap( input.CreditHourType, output, "CreditHourType" ); //} //} //else //{ // output.CreditHourType = helper.HandleLanguageMap( input.CreditHourType, output, "CreditHourType" ); // output.CreditHourValue = input.CreditHourValue; // output.CreditUnitType = helper.MapCAOToEnumermation( input.CreditUnitType ); // output.CreditUnitValue = input.CreditUnitValue; // output.CreditUnitTypeDescription = helper.HandleLanguageMap( input.CreditUnitTypeDescription, output, "CreditUnitTypeDescription" ); //} } output.Jurisdiction = helper.MapToJurisdiction(input.Jurisdiction, ref status); //EstimatedCost //will need to format, all populate Entity.RelatedCosts (for bubble up) - actually this would be for asmts, and lopps output.EstimatedCost = helper.FormatCosts(input.EstimatedCost, ref status); //assesses compentencies output.AssessesCompetencies = helper.MapCAOListToCAOProfileList(input.Assesses); if (output.AssessesCompetencies.Count() == 0 && UtilityManager.GetAppKeyValue("skipAsmtImportIfNoCompetencies", false)) { //skip LoggingHelper.DoTrace(2, string.Format(" ***Skipping asmt# {0}, {1} as it has no competencies and this is a special run.", output.Id, output.Name)); return(true); } //common conditions output.ConditionManifestIds = helper.MapEntityReferences(input.CommonConditions, CodesManager.ENTITY_TYPE_CONDITION_MANIFEST, CodesManager.ENTITY_TYPE_ASSESSMENT_PROFILE, ref status); //common costs output.CostManifestIds = helper.MapEntityReferences(input.CommonCosts, CodesManager.ENTITY_TYPE_COST_MANIFEST, CodesManager.ENTITY_TYPE_ASSESSMENT_PROFILE, ref status); //connections output.AdvancedStandingFrom = helper.FormatConditionProfile(input.AdvancedStandingFrom, ref status); output.AdvancedStandingFor = helper.FormatConditionProfile(input.IsAdvancedStandingFor, ref status); output.PreparationFrom = helper.FormatConditionProfile(input.PreparationFrom, ref status); output.IsPreparationFor = helper.FormatConditionProfile(input.IsPreparationFor, ref status); output.IsRequiredFor = helper.FormatConditionProfile(input.IsRequiredFor, ref status); output.IsRecommendedFor = helper.FormatConditionProfile(input.IsRecommendedFor, ref status); //EstimatedDuration ============================== output.EstimatedDuration = helper.FormatDuration(input.EstimatedDuration, ref status); //conditions ====================================== output.Requires = helper.FormatConditionProfile(input.Requires, ref status); output.Recommends = helper.FormatConditionProfile(input.Recommends, ref status); output.EntryCondition = helper.FormatConditionProfile(input.EntryCondition, ref status); output.Corequisite = helper.FormatConditionProfile(input.Corequisite, ref status); //Process profiles ============================== output.AdministrationProcess = helper.FormatProcessProfile(input.AdministrationProcess, ref status); output.DevelopmentProcess = helper.FormatProcessProfile(input.DevelopmentProcess, ref status); output.MaintenanceProcess = helper.FormatProcessProfile(input.MaintenanceProcess, ref status); // output.Addresses = helper.FormatAvailableAtAddresses(input.AvailableAt, ref status); //BYs output.AccreditedBy = helper.MapOrganizationReferenceGuids(input.AccreditedBy, ref status); output.ApprovedBy = helper.MapOrganizationReferenceGuids(input.ApprovedBy, ref status); output.OfferedBy = helper.MapOrganizationReferenceGuids(input.OfferedBy, ref status); output.OwnedBy = helper.MapOrganizationReferenceGuids(input.OwnedBy, ref status); if (output.OwnedBy != null && output.OwnedBy.Count > 0) { output.OwningAgentUid = output.OwnedBy[0]; } else { //add warning? if (output.OfferedBy == null && output.OfferedBy.Count == 0) { status.AddWarning("document doesn't have an owning or offering organization."); } } output.RecognizedBy = helper.MapOrganizationReferenceGuids(input.RecognizedBy, ref status); output.RegulatedBy = helper.MapOrganizationReferenceGuids(input.RegulatedBy, ref status); //INs output.AccreditedIn = helper.MapToJurisdiction(input.AccreditedIn, ref status); output.ApprovedIn = helper.MapToJurisdiction(input.ApprovedIn, ref status); output.OfferedIn = helper.MapToJurisdiction(input.OfferedIn, ref status); output.RecognizedIn = helper.MapToJurisdiction(input.RecognizedIn, ref status); output.RegulatedIn = helper.MapToJurisdiction(input.RegulatedIn, ref status); //FinancialAssistance ============================ //output.FinancialAssistanceOLD = helper.FormatFinancialAssistance( input.FinancialAssistance, ref status ); output.FinancialAssistance = helper.FormatFinancialAssistance(input.FinancialAssistance, ref status); //=== if any messages were encountered treat as warnings for now if (messages.Count > 0) { status.SetMessages(messages, true); } //just in case check if entity added since start if (output.Id == 0) { ThisEntity entity = EntityServices.GetByCtid(ctid); if (entity != null && entity.Id > 0) { output.Id = entity.Id; output.RowId = entity.RowId; } } importSuccessfull = mgr.Import(output, ref status); status.DocumentId = output.Id; status.DetailPageUrl = string.Format("~/assessment/{0}", output.Id); status.DocumentRowId = output.RowId; //just in case if (status.HasErrors) { importSuccessfull = false; } //if record was added to db, add to/or set EntityResolution as resolved int ierId = new ImportManager().Import_EntityResolutionAdd(referencedAtId, ctid, CodesManager.ENTITY_TYPE_ASSESSMENT_PROFILE, output.RowId, output.Id, (output.Id > 0), ref messages, output.Id > 0); return(importSuccessfull); }
private void txtOptions_ButtonClicked(object sender, EventArgs e) { try { #region Sanity Checks if (string.IsNullOrEmpty(Filename)) { throw new ArgumentNullException("Filename"); } string ext = Path.GetExtension(Filename).Substring(1).ToUpper(); if (!ImportManager.IsSupportedExtension(ext)) { throw new ArgumentException("File extension not supported: " + ext); } #endregion var t = ImportManager.GetOptionType(ext); if (t == null) { throw new ArgumentException("Option type not found for extension: " + ext); } #region Create DataTable var dt = new DataTable("Options"); dt.Columns.Add("Key", typeof(string)).ReadOnly = true; dt.Columns.Add("Value", typeof(string)); dt.Columns.Add("DataType", typeof(string)).ReadOnly = true; var piList = t.GetProperties(); foreach (var item in piList) { var dr = dt.NewRow(); dr["Key"] = item.Name; dr["Value"] = DBNull.Value; dr["DataType"] = item.PropertyType.Name; dt.Rows.Add(dr); } #endregion using (var d = new DynamicEditorForm()) { d.Caption = "Import Option(s) Editor"; d.DataSource = dt; if (d.ShowDialog() != DialogResult.OK) { return; } } var sb = new StringBuilder(); foreach (DataRow dr in dt.Rows) { if (DataConvert.IsNull(dr["Value"])) { continue; } var key = dr["Key"].ToString(); var val = dr["Value"].ToString(); var type = dr["DataType"].ToString(); if (type.Equals("String", StringComparison.InvariantCultureIgnoreCase)) { val = string.Format("\"{0}\"", val); } if (sb.Length > 0) { sb.Append("; "); } sb.AppendFormat("{0}={1}", key, val); } Options = sb.ToString(); } catch (Exception ex) { FormUtil.WinException(ex); } }
} // public bool Import(string payload, string envelopeIdentifier, SaveStatus status) { List <string> messages = new List <string>(); bool importSuccessfull = false; EntityServices mgr = new EntityServices(); // InputEntity input = new InputEntity(); var bnodes = new List <BNode>(); var mainEntity = new Dictionary <string, object>(); // Dictionary <string, object> dictionary = RegistryServices.JsonToDictionary(payload); object graph = dictionary["@graph"]; //serialize the graph object var glist = JsonConvert.SerializeObject(graph); //parse graph in to list of objects JArray graphList = JArray.Parse(glist); int cntr = 0; foreach (var item in graphList) { cntr++; if (cntr == 1) { var main = item.ToString(); //may not use this. Could add a trace method mainEntity = RegistryServices.JsonToDictionary(main); input = JsonConvert.DeserializeObject <InputEntity>(main); } else { var bn = item.ToString(); //20-07-02 need to handle the enhanced bnodes bnodes.Add(JsonConvert.DeserializeObject <BNode>(bn)); } } ///============= process ============================= MappingHelperV3 helper = new MappingHelperV3(3); helper.entityBlankNodes = bnodes; helper.CurrentEntityCTID = input.CTID; helper.CurrentEntityName = input.Name.ToString(); string ctid = input.CTID; status.Ctid = ctid; string referencedAtId = input.CtdlId; LoggingHelper.DoTrace(5, " ctid: "+ ctid); LoggingHelper.DoTrace(5, " @Id: "+ input.CtdlId); LoggingHelper.DoTrace(5, " name: "+ input.Name.ToString()); if (status.DoingDownloadOnly) { return(true); } //add/updating DataSetProfile if (!DoesEntityExist(input.CTID, ref output, ref status)) { //set the rowid now, so that can be referenced as needed output.RowId = Guid.NewGuid(); } helper.currentBaseObject = output; output.Name = helper.HandleLanguageMap(input.Name, output, "Name"); output.Description = helper.HandleLanguageMap(input.Description, output, "Description"); output.CTID = input.CTID; //TBD handling of referencing third party publisher if (!string.IsNullOrWhiteSpace(status.DocumentPublishedBy)) { //output.PublishedByOrganizationCTID = status.DocumentPublishedBy; var porg = OrganizationManager.GetSummaryByCtid(status.DocumentPublishedBy); if (porg != null && porg.Id > 0) { //TODO - store this in a json blob?????????? //this will result in being added to Entity.AgentRelationship output.PublishedBy = new List <Guid>() { porg.RowId }; } else { //if publisher not imported yet, all publishee stuff will be orphaned var entityUid = Guid.NewGuid(); var statusMsg = ""; var resPos = referencedAtId.IndexOf("/resources/"); var swp = referencedAtId.Substring(0, (resPos + "/resources/".Length)) + status.DocumentPublishedBy; int orgId = new OrganizationManager().AddPendingRecord(entityUid, status.DocumentPublishedBy, swp, ref statusMsg); } } else { //may need a check for existing published by to ensure not lost if (output.Id > 0) { //TBD //if ( output.DataProvider != null && output.DataProvider.Any() ) //{ // var publishedByList = output.OrganizationRole.Where( s => s.RoleTypeId == 30 ).ToList(); // if ( publishedByList != null && publishedByList.Any() ) // { // var pby = publishedByList[ 0 ].ActingAgentUid; // output.PublishedBy = new List<Guid>() { publishedByList[ 0 ].ActingAgentUid }; // } //} } } output.Source = input.Source; //**** TBD DataProvider, or is the owner based on envelope? //output.OwnedBy = helper.MapOrganizationReferenceGuids( "DataSetProfile.OwnedBy", input.DataProvider, ref status ); //if ( output.OwnedBy != null && output.OwnedBy.Count > 0 ) //{ // output.OwningAgentUid = output.OwnedBy[ 0 ]; // helper.CurrentOwningAgentUid = output.OwnedBy[ 0 ]; //} // importSuccessfull = new DataSetProfileServices().Import(output, ref status); // status.DocumentId = output.Id; status.DetailPageUrl = string.Format("~/DataSetProfile/{0}", output.Id); status.DocumentRowId = output.RowId; //just in case if (status.HasErrors) { importSuccessfull = false; } //if record was added to db, add to/or set EntityResolution as resolved int ierId = new ImportManager().Import_EntityResolutionAdd(referencedAtId, ctid, entityTypeId, output.RowId, output.Id, false, ref messages, output.Id > 0); // return(importSuccessfull); }
internal static void LoadPlugins(string pluginPath) { loadedPlugins = new Dictionary <string, HMConPlugin>(); var dllFiles = Directory.GetFiles(pluginPath, "*.dll"); foreach (var path in dllFiles) { try { var assembly = Assembly.LoadFrom(path); foreach (var t in assembly.GetTypes()) { if (t.BaseType != null && t.BaseType.Name == typeof(HMConPlugin).Name && !t.IsAbstract) { var plugin = (HMConPlugin)Activator.CreateInstance(t); string info = ""; var i = plugin.GetImportHandler(); if (i != null) { ImportManager.RegisterHandler(i); info += info.Length > 0 ? "+I" : "I"; } var e = plugin.GetExportHandler(); if (e != null) { ExportUtility.RegisterHandler(e); info += info.Length > 0 ? "+E" : "E"; } var c = plugin.GetCommandHandler(); if (c != null) { CommandHandler.commandHandlers.Add(c); info += info.Length > 0 ? "+C" : "C"; } var attribute = t.GetCustomAttribute <PluginInfoAttribute>(); string pluginID; string pluginAttr; if (attribute != null) { pluginAttr = attribute.Name; pluginID = attribute.ID.ToUpper(); } else { ConsoleOutput.WriteWarning($"Plugin with class '{t.FullName}' does not specify a Plugin name!"); pluginID = "[" + t.Name + "]"; pluginAttr = t.Name; } if (loadedPlugins.ContainsKey(pluginID)) { continue; } loadedPlugins.Add(pluginID, plugin); ConsoleOutput.WriteLine($"Loaded Plugin '{pluginAttr}' [{info}]"); } } //ConsoleOutput.WriteWarning("Not a plugin dll: " + path); } catch { //ConsoleOutput.WriteWarning("Failed to load dll: " + path); } } }
public bool ImportV3(string payload, string envelopeIdentifier, SaveStatus status) { //instantiate the input document (credential) InputEntityV3 input = new InputEntityV3(); var bnodes = new List <BNode>(); var mainEntity = new Dictionary <string, object>(); //map payload to a dictionary Dictionary <string, object> dictionary = RegistryServices.JsonToDictionary(payload); //get the @graph property object graph = dictionary["@graph"]; //serialize the graph object var glist = JsonConvert.SerializeObject(graph); //parse graph in to list of objects JArray graphList = JArray.Parse(glist); int cntr = 0; foreach (var item in graphList) { cntr++; if (cntr == 1) { var main = item.ToString(); //may not use this. Could add a trace method //mainEntity = RegistryServices.JsonToDictionary( main ); input = JsonConvert.DeserializeObject <InputEntityV3>(main); } else { //save blank nodes - typically reference organizations var bn = item.ToString(); bnodes.Add(JsonConvert.DeserializeObject <BNode>(bn)); } } List <string> messages = new List <string>(); bool importSuccessfull = false; EntityServices mgr = new EntityServices(); MappingHelperV3 helper = new MappingHelperV3(); helper.entityBlankNodes = bnodes; status.EnvelopeId = envelopeIdentifier; try { //input = JsonConvert.DeserializeObject<InputEntity>( item.DecodedResource.ToString() ); string ctid = input.Ctid; string referencedAtId = input.CtdlId; LoggingHelper.DoTrace(5, " name: "+ input.Name.ToString()); LoggingHelper.DoTrace(6, " url: "+ input.SubjectWebpage); LoggingHelper.DoTrace(5, " ctid: "+ input.Ctid); LoggingHelper.DoTrace(6, " @Id: "+ input.CtdlId); status.Ctid = ctid; if (status.DoingDownloadOnly) { return(true); } if (!DoesEntityExist(input.Ctid, ref output)) { //set the rowid now, so that can be referenced as needed output.RowId = Guid.NewGuid(); } helper.currentBaseObject = output; //start with language and may use with language maps foreach (var l in input.InLanguage) { if (!string.IsNullOrWhiteSpace(l)) { var language = CodesManager.GetLanguage(l); output.InLanguageCodeList.Add(new TextValueProfile() { CodeId = language.CodeId, TextTitle = language.Name, TextValue = language.Value }); } } if (input.InLanguage.Count > 0) { //could use to alter helper.DefaultLanguage } output.Name = helper.HandleLanguageMap(input.Name, output, "Name"); output.Description = helper.HandleLanguageMap(input.Description, output, "Description"); output.SubjectWebpage = input.SubjectWebpage; output.CTID = input.Ctid; //warning this gets set to blank if doing a manual import by ctid output.CredentialRegistryId = envelopeIdentifier; output.CredentialStatusType = helper.MapCAOToEnumermation(input.CredentialStatusType); output.DateEffective = input.DateEffective; output.AlternateNames = helper.MapToTextValueProfile(input.AlternateName, output, "AlternateName"); output.ImageUrl = input.Image; output.CredentialTypeSchema = input.CredentialType; output.AvailabilityListing = helper.MapListToString(input.AvailabilityListing); output.AvailableOnlineAt = helper.MapListToString(input.AvailableOnlineAt); output.CredentialId = input.CredentialId; //TODO - develope entity for IdentitifierValue output.VersionIdentifier = helper.MapIdentifierValueListToString(input.VersionIdentifier); output.VersionIdentifierList = helper.MapIdentifierValueList(input.VersionIdentifier); output.CodedNotation = input.CodedNotation; output.ISICV4 = input.IsicV4; output.ProcessStandards = input.ProcessStandards; output.ProcessStandardsDescription = helper.HandleLanguageMap(input.ProcessStandardsDescription, output, "ProcessStandardsDescription"); output.LatestVersion = input.LatestVersion; output.PreviousVersion = input.PreviousVersion; output.NextVersion = input.NextVersion; output.SupersededBy = input.SupersededBy; output.Supersedes = input.Supersedes; output.Subject = helper.MapCAOListToTextValueProfile(input.Subject, CodesManager.PROPERTY_CATEGORY_SUBJECT); //occupations //output.Occupation = helper.MapCAOListToEnumermation( input.OccupationType ); //actually used by import output.Occupations = helper.MapCAOListToCAOProfileList(input.OccupationType); //just append alternative items. Ensure empty lists are ignored output.Occupations.AddRange(helper.AppendLanguageMapListToCAOProfileList(input.AlternativeOccupationType)); //skip if no occupations if (output.Occupations.Count() == 0 && UtilityManager.GetAppKeyValue("skipCredImportIfNoOccupations", false)) { //LoggingHelper.DoTrace( 2, string.Format( " ***Skipping Credential# {0}, {1} as it has no occupations and this is a special run.", output.Id, output.Name ) ); //return true; } //Industries output.Industries = helper.MapCAOListToCAOProfileList(input.IndustryType); output.Industries.AddRange(helper.AppendLanguageMapListToCAOProfileList(input.AlternativeIndustryType)); //naics output.Naics = input.Naics; output.InstructionalProgramTypes = helper.MapCAOListToCAOProfileList(input.InstructionalProgramType); output.InstructionalProgramTypes.AddRange(helper.AppendLanguageMapListToCAOProfileList(input.AlternativeInstructionalProgramType)); // //will want a custom method to lookup the rating NavyServices nsrvs = new NavyServices(); output.NavyRating = NavyServices.MapRatingsListToEnumermation(input.HasRating); output.NavyRatingType.AddRange(nsrvs.MapCAOListToCAOProfileList(input.HasRating, ref messages)); // output.Keyword = helper.MapToTextValueProfile(input.Keyword, output, "Keyword"); output.Jurisdiction = helper.MapToJurisdiction(input.Jurisdiction, ref status); //CopyrightHolder - expecting single; will need to expand output.CopyrightHolder = helper.MapOrganizationReferencesGuid("Credential.CopyrightHolder", input.CopyrightHolder, ref status); //CAO output.AudienceLevelType = helper.MapCAOListToEnumermation(input.AudienceLevelType); // output.AudienceType = helper.MapCAOListToEnumermation(input.AudienceType); output.DegreeConcentration = helper.MapCAOListToTextValueProfile(input.DegreeConcentration, CodesManager.PROPERTY_CATEGORY_DEGREE_CONCENTRATION); output.DegreeMajor = helper.MapCAOListToTextValueProfile(input.DegreeMajor, CodesManager.PROPERTY_CATEGORY_DEGREE_MAJOR); output.DegreeMinor = helper.MapCAOListToTextValueProfile(input.DegreeMinor, CodesManager.PROPERTY_CATEGORY_DEGREE_MINOR); output.AssessmentDeliveryType = helper.MapCAOListToEnumermation(input.AssessmentDeliveryType); output.LearningDeliveryType = helper.MapCAOListToEnumermation(input.LearningDeliveryType); //EstimatedCost //will need to format, all populate Entity.RelatedCosts (for bubble up) - actually this would be for asmts, and lopps output.EstimatedCost = helper.FormatCosts(input.EstimatedCost, ref status); //EstimatedDuration output.EstimatedDuration = helper.FormatDuration(input.EstimatedDuration, ref status); output.RenewalFrequency = helper.FormatDurationItem(input.RenewalFrequency); //conditions output.Requires = helper.FormatConditionProfile(input.Requires, ref status); output.Recommends = helper.FormatConditionProfile(input.Recommends, ref status); output.Renewal = helper.FormatConditionProfile(input.Renewal, ref status); output.Corequisite = helper.FormatConditionProfile(input.Corequisite, ref status); output.Revocation = helper.FormatRevocationProfile(input.Revocation, ref status); //connections output.AdvancedStandingFrom = helper.FormatConditionProfile(input.AdvancedStandingFrom, ref status); output.AdvancedStandingFor = helper.FormatConditionProfile(input.IsAdvancedStandingFor, ref status); output.PreparationFrom = helper.FormatConditionProfile(input.PreparationFrom, ref status); output.IsPreparationFor = helper.FormatConditionProfile(input.IsPreparationFor, ref status); output.IsRequiredFor = helper.FormatConditionProfile(input.IsRequiredFor, ref status); output.IsRecommendedFor = helper.FormatConditionProfile(input.IsRecommendedFor, ref status); //common conditions output.ConditionManifestIds = helper.MapEntityReferences(input.CommonConditions, CodesManager.ENTITY_TYPE_CONDITION_MANIFEST, CodesManager.ENTITY_TYPE_CREDENTIAL, ref status); //common costs output.CostManifestIds = helper.MapEntityReferences(input.CommonCosts, CodesManager.ENTITY_TYPE_COST_MANIFEST, CodesManager.ENTITY_TYPE_CREDENTIAL, ref status); //HasPart/IsPart //WARNING - these methods assume all parts are the same type - the provided thisEntityTypeId. AT THIS TIME, THE PARTS SHOULD ALL BE CREDENTIALS output.HasPartIds = helper.MapEntityReferences(input.HasPart, thisEntityTypeId, ref status); output.IsPartOfIds = helper.MapEntityReferences(input.IsPartOf, thisEntityTypeId, ref status); //Process profiles output.AdministrationProcess = helper.FormatProcessProfile(input.AdministrationProcess, ref status); output.DevelopmentProcess = helper.FormatProcessProfile(input.DevelopmentProcess, ref status); output.MaintenanceProcess = helper.FormatProcessProfile(input.MaintenanceProcess, ref status); output.AppealProcess = helper.FormatProcessProfile(input.AppealProcess, ref status); output.ComplaintProcess = helper.FormatProcessProfile(input.ComplaintProcess, ref status); output.ReviewProcess = helper.FormatProcessProfile(input.ReviewProcess, ref status); output.RevocationProcess = helper.FormatProcessProfile(input.RevocationProcess, ref status); //FinancialAssistance output.FinancialAssistance = helper.FormatFinancialAssistance(input.FinancialAssistance, ref status); output.Addresses = helper.FormatAvailableAtAddresses(input.AvailableAt, ref status); //BYs output.AccreditedBy = helper.MapOrganizationReferenceGuids("Credential.AccreditedBy", input.AccreditedBy, ref status); output.ApprovedBy = helper.MapOrganizationReferenceGuids("Credential.ApprovedBy", input.ApprovedBy, ref status); output.OfferedBy = helper.MapOrganizationReferenceGuids("Credential.OfferedBy", input.OfferedBy, ref status); //note need to set output.OwningAgentUid to the first entry output.OwnedBy = helper.MapOrganizationReferenceGuids("Credential.OwnedBy", input.OwnedBy, ref status); if (output.OwnedBy != null && output.OwnedBy.Count > 0) { output.OwningAgentUid = output.OwnedBy[0]; } else { //add warning? if (output.OfferedBy == null && output.OfferedBy.Count == 0) { status.AddWarning("document doesn't have an owning or offering organization."); } } output.RecognizedBy = helper.MapOrganizationReferenceGuids("Credential.RecognizedBy", input.RecognizedBy, ref status); output.RegulatedBy = helper.MapOrganizationReferenceGuids("Credential.RegulatedBy", input.RegulatedBy, ref status); output.RevokedBy = helper.MapOrganizationReferenceGuids("Credential.RevokedBy", input.RevokedBy, ref status); output.RenewedBy = helper.MapOrganizationReferenceGuids("Credential.RenewedBy", input.RenewedBy, ref status); //INs output.AccreditedIn = helper.MapToJurisdiction(input.AccreditedIn, ref status); output.ApprovedIn = helper.MapToJurisdiction(input.ApprovedIn, ref status); output.ApprovedIn = helper.MapToJurisdiction(input.ApprovedIn, ref status); output.RecognizedIn = helper.MapToJurisdiction(input.RecognizedIn, ref status); output.RegulatedIn = helper.MapToJurisdiction(input.RegulatedIn, ref status); output.RevokedIn = helper.MapToJurisdiction(input.RevokedIn, ref status); output.RenewedIn = helper.MapToJurisdiction(input.RenewedIn, ref status); //=== if any messages were encountered treat as warnings for now if (messages.Count > 0) { status.SetMessages(messages, true); } //just in case check if entity added since start if (output.Id == 0) { ThisEntity entity = EntityServices.GetByCtid(ctid); if (entity != null && entity.Id > 0) { output.Id = entity.Id; output.RowId = entity.RowId; } } //save the data importSuccessfull = mgr.Import(output, ref status); status.DocumentId = output.Id; status.DetailPageUrl = string.Format("~/credential/{0}", output.Id); status.DocumentRowId = output.RowId; //if record was added to db, add to/or set EntityResolution as resolved int ierId = new ImportManager().Import_EntityResolutionAdd(referencedAtId, ctid, CodesManager.ENTITY_TYPE_CREDENTIAL, output.RowId, output.Id, (output.Id > 0), ref messages, output.Id > 0); //just in case if (status.HasErrors) { importSuccessfull = false; } } catch (Exception ex) { LoggingHelper.LogError(ex, string.Format("Exception encountered in envelopeId: {0}", envelopeIdentifier), false, "workIT Import exception"); } return(importSuccessfull); }
private IEnumerator <IYield> UpdatePackages_Helper(Plug api, string wikiId, string apikey, bool force, bool init, Result <XDoc> result) { try { _status = PackageUpdaterStatus.Importing; yield return(Coroutine.Invoke(AuthenticateImportUser, api, wikiId, apikey, new Result <Plug>()).Set(x => api = x)); XDoc license = null; yield return(api.At("license").With("apikey", apikey).Get(new Result <XDoc>()).Set(x => license = x)); var importReport = new XDoc("packages"); var first = true; foreach (var directory in Directory.GetDirectories(_templatePackagePath)) { var directoryName = Path.GetFileName(directory); string restriction = null; if (directoryName.EqualsInvariantIgnoreCase("public")) { restriction = "Public"; } else if (directoryName.EqualsInvariantIgnoreCase("semi-public")) { restriction = "Semi-Public"; } else if (directoryName.EqualsInvariantIgnoreCase("private")) { restriction = "Private"; } foreach (var package in Directory.GetFiles(directory, "*.mt*").OrderBy(x => x)) { var ext = Path.GetExtension(package); if (!(ext.EqualsInvariantIgnoreCase(".mtarc") || ext.EqualsInvariantIgnoreCase(".mtapp"))) { continue; } if (!first) { importReport.End(); } first = false; importReport.Start("package").Elem("path", package); ArchivePackageReader packageReader; _log.DebugFormat("contemplating import of '{0}' for '{1}'", package, wikiId); try { packageReader = new ArchivePackageReader(package); } catch (Exception e) { SetError(importReport, wikiId, "error", e, "Unable to open package."); continue; } Result <XDoc> manifestResult; yield return(manifestResult = packageReader.ReadManifest(new Result <XDoc>()).Catch()); if (manifestResult.HasException) { SetError(importReport, wikiId, "error", manifestResult.Exception, "Unable to read package manifest."); continue; } var manifest = manifestResult.Value; // check for required capabilities var capabilitiesSatisfied = true; foreach (var capability in manifest["capability"]) { var capabilityName = capability["@name"].AsText; var capabilityValue = capability["@value"].AsText.IfNullOrEmpty("enabled"); if (string.IsNullOrEmpty(capabilityName)) { continue; } var licenseCapabilityValue = DekiLicense.GetCapability(license, capabilityName) ?? ""; if (!string.IsNullOrEmpty(capabilityName) && licenseCapabilityValue.EqualsInvariant(capabilityValue)) { continue; } capabilitiesSatisfied = false; SetError(importReport, wikiId, "error", null, "Missing capability '{0}' or incorrect capability value, expected '{1}' and got '{2}'.", capabilityName, capabilityValue, licenseCapabilityValue); } if (!capabilitiesSatisfied) { continue; } // add security xml if we are in a restriction enforcing path if (!string.IsNullOrEmpty(restriction)) { manifest.Start("security") .Start("permissions.page") .Elem("restriction", restriction) .End() .End(); } // if package predates @date.created, take file modified var dateCreated = manifest["@date.created"].AsDate ?? new FileInfo(package).LastWriteTime; // check whether we should import this package var filename = Path.GetFileName(package); var importOnce = manifest["@import-once"].AsBool ?? false; var initPackage = manifest["@init-only"].AsBool ?? false; importReport.Elem("name", filename).Attr("date.created", dateCreated).Attr("preserve-local", manifest["@preserve-local"].AsBool ?? false); var importPropertyName = XUri.EncodeSegment(PACKAGE_PROPERTY_NS + filename); if (initPackage && !init && !force) { SetError(importReport, wikiId, "skipped", null, "package '{0}' is an init package and neither init or force flags were set, skipping", package); continue; } if (force && !importOnce) { _log.DebugFormat("force is set and package '{0}' is not marked as oneTimeOnly, proceeding with import", package); } else { DreamMessage propertyResponse = null; yield return(api.At("site", "properties", importPropertyName).Get(new Result <DreamMessage>()).Set(x => propertyResponse = x)); if (propertyResponse.IsSuccessful) { var importedPackage = propertyResponse.ToDocument(); var importedPackageDate = importedPackage["date.created"].AsDate; if (!importedPackageDate.HasValue) { _log.WarnFormat("unable to retrieve imported package date.created for '{0}', treating package as new", package); } else if (dateCreated > importedPackageDate.Value) { _log.DebugFormat("package '{0}' is newer, proceed with import ({1} > {2}", package, dateCreated, importedPackageDate.Value); } else { SetError(importReport, wikiId, "skipped", null, "package '{0}' is not newer, skip import ({1} <= {2}.)", package, dateCreated, importedPackageDate.Value); continue; } } else if (propertyResponse.Status == DreamStatus.Unauthorized) { _log.WarnFormat("apiuri has lost its authentication, dropping out"); throw new UnauthorizedAccessException("Authentication for was lost"); } else { _log.DebugFormat("package '{0}' has not previously been imported, proceeding with import", package); } } // import package Result <Importer> importerResult; yield return(importerResult = Importer.CreateAsync(api, manifest, "/", new Result <Importer>()).Catch()); if (importerResult.HasException) { SetError(importReport, wikiId, "error", importerResult.Exception, "Unable to create importer for package."); continue; } var importer = importerResult.Value; var importManager = new ImportManager(importer, packageReader); Result importResult; yield return(importResult = importManager.ImportAsync(new Result()).Catch()); if (importResult.HasException) { SetError(importReport, wikiId, "error", importResult.Exception, "Import did not complete successfully."); continue; } // write import data as site property yield return(api.At("site", "properties", importPropertyName) .With("abort", "never") .With("description", string.Format("Import of package '{0}'", filename)) .Put(new XDoc("package").Elem("date.created", dateCreated), new Result <DreamMessage>())); importReport.Start("status").Attr("code", "ok").End(); _log.DebugFormat("sucessfully imported package '{0}'", package); } } if (!first) { importReport.End(); } result.Return(importReport); yield break; } finally { _status = PackageUpdaterStatus.Idle; } }
public ImportForwardCallAttribute(string nameSpace, string id, ImportCallOptions options) { Identifier = ImportManager.Combine(nameSpace, id); Options = IdentifierOptions.FullyQualified; CallOptions = options; }
/// <summary> /// Sets the state of the action. /// </summary> private void SetActionState() { ImportManager importManager = GetImportManager(); Action = importManager.ActionManager.GetAction("AddResponse") as ActionAddResponse; Action.HydrateChanges(); if (Action != null) { if (!Action.Initialized) { dtpResponseDate.DateTimeValue = DateTime.UtcNow; pklResponseStatus.PickListValue = GetLocalResourceObject("DefaultResponseStatus").ToString(); //if (String.IsNullOrEmpty(Action.Response.LeadSource)) //{ ImportTargetProperty tpLeadSource = importManager.EntityManager.GetEntityProperty("LeadSource"); if (tpLeadSource != null && !String.IsNullOrEmpty(tpLeadSource.DefaultValue.ToString())) { ILeadSource ls = EntityFactory.GetById <ILeadSource>(tpLeadSource.DefaultValue); if (ls != null) { lueResponseLeadSource.LookupResultValue = ls; lueResponseLeadSource.Text = ls.Description; } } //} txtComments.Text = Action.Response.Comments; pklResponseMethod.PickListValue = Action.Response.ResponseMethod; lueCampaign.LookupResultValue = Action.Response.Campaign; lbxStages.SelectedValue = Action.Response.Stage; pklInterest.PickListValue = Action.Response.Interest; pklInterestLevel.PickListValue = Action.Response.InterestLevel; if (Mode.Value == "") { Action.Initialized = true; Mode.Value = "ADD"; } } else { if (!String.IsNullOrEmpty(Action.Response.LeadSource)) { ILeadSource ls = EntityFactory.GetById <ILeadSource>(Action.Response.LeadSource); if (ls != null) { lueResponseLeadSource.LookupResultValue = ls; lueResponseLeadSource.Text = ls.Description; } } txtComments.Text = Action.Response.Comments; //lueResponseLeadSource.LookupResultValue = Action.Response.LeadSource; dtpResponseDate.DateTimeValue = Action.Response.ResponseDate; pklResponseStatus.PickListValue = Action.Response.Status; pklResponseMethod.PickListValue = Action.Response.ResponseMethod; lueCampaign.LookupResultValue = Action.Response.Campaign; lbxStages.SelectedValue = Action.Response.Stage; pklInterest.PickListValue = Action.Response.Interest; pklInterestLevel.PickListValue = Action.Response.InterestLevel; } } }
private void InitializeView() { ImportManager importManager = Page.Session["importManager"] as ImportManager; if (importManager == null) { return; } importManager.Options.CheckForDuplicates = chkCheckForDups.Checked; if (Mode.Value == "") { Mode.Value = "Initialized"; chkFindDupsInFile.Checked = importManager.Configuration.AdvancedOptions.IndexAfterInsert; } else { importManager.Configuration.AdvancedOptions.IndexAfterInsert = chkFindDupsInFile.Checked; } if (chklstFilters.Items.Count <= 0) { if (importManager.DuplicateProvider == null) { importManager.DuplicateProvider = new LeadDuplicateProvider(); } LeadDuplicateProvider duplicateProvider = (LeadDuplicateProvider)importManager.DuplicateProvider; foreach (MatchPropertyFilterMap propertyFilter in duplicateProvider.GetFilters()) { ListItem item = new ListItem(); //If resource does not exist then use the xml value. Item is prefixed with "Filter" to better identify resource items var filterResource = GetLocalResourceObject("Filter." + propertyFilter.PropertyName); item.Text = filterResource != null && filterResource.ToString() != "" ? filterResource.ToString() : propertyFilter.DisplayName; item.Value = propertyFilter.PropertyName; item.Selected = propertyFilter.Enabled; item.Enabled = IsFilterMapped(importManager, propertyFilter.PropertyName); chklstFilters.Items.Add(item); } } else { LeadDuplicateProvider duplicateProvider = (LeadDuplicateProvider)importManager.DuplicateProvider; if (duplicateProvider != null) { foreach (ListItem item in chklstFilters.Items) { item.Enabled = IsFilterMapped(importManager, item.Value); MatchPropertyFilterMap propertyFilter = duplicateProvider.GetPropertyFilter(item.Value); if (propertyFilter != null && ((!item.Enabled) || (!propertyFilter.Enabled))) { item.Selected = false; } } } else { foreach (ListItem item in chklstFilters.Items) { item.Enabled = false; } } } }
/// <summary> /// Raises the <see cref="E:System.Web.UI.Control.PreRender"/> event. /// </summary> /// <param name="e">An <see cref="T:System.EventArgs"/> object that contains the event data.</param> protected override void OnPreRender(EventArgs e) { ImportManager importManager = Page.Session["importManager"] as ImportManager; IImportHistory importHistory = null; string historyId = Page.Session["importHistoryId"] as string; importHistory = Sage.Platform.EntityFactory.GetById <IImportHistory>(historyId); if (importHistory == null) { if (importManager != null) { //importHistory = importManager.ImportHistory; } } if (importHistory != null) { try { lnkImportNumber.Text = string.Format("{0}-{1}", importHistory.Alternatekeyprefix, importHistory.Alternatekeysuffix); ImportProcessState processState = (ImportProcessState)Enum.Parse(typeof(ImportProcessState), importHistory.ProcessState); if (processState.Equals(ImportProcessState.Abort)) { //lblHeader.Text = string.Format(GetLocalResourceObject("AbortMsg").ToString()); if (importHistory.ErrorCount > 0) { lblHeader2.Text = string.Format("{0} {1}", GetLocalResourceObject("ViewImportHistoryMsgWithErrors").ToString(), GetLocalResourceObject("ViewImportHistoryMsg").ToString()); } else { lblHeader2.Text = string.Format("{0}", GetLocalResourceObject("ViewImportHistoryMsg").ToString()); } cmdAbort.Visible = false; } else { if (processState.Equals(ImportProcessState.Completed)) { cmdAbort.Visible = false; lblHeader.Text = string.Format(GetLocalResourceObject("CompletedMsg").ToString()); if (importHistory.ErrorCount > 0) { lblHeader2.Text = string.Format("{0} {1}", GetLocalResourceObject("ViewImportHistoryMsgWithErrors").ToString(), GetLocalResourceObject("ViewImportHistoryMsg").ToString()); } else { lblHeader2.Text = string.Format("{0}", GetLocalResourceObject("ViewImportHistoryMsg").ToString()); } importManager = null; Page.Session["importManager"] = null; //Page.Session["importHistoryId"] = null; radImportProcessArea.Visible = false; } else { cmdAbort.Visible = true; lblHeader.Text = string.Format(GetLocalResourceObject("lblPrimary_Progress.Caption").ToString()); lblHeader2.Text = string.Format(GetLocalResourceObject("ProcessingMsg").ToString()); } } } catch (Exception) { } } else { } base.OnPreRender(e); }
/// <summary> /// Gets and Sets the default target properties. /// </summary> /// <param name="importManager">The import manager.</param> private void SetDefaultTargetProperties(ImportManager importManager) { if (importManager.TargetPropertyDefaults == null || importManager.TargetPropertyDefaults.Count == 0) { ImportTargetProperty tpOwner = importManager.EntityManager.GetEntityProperty("Owner"); ImportTargetProperty tpLeadSource = importManager.EntityManager.GetEntityProperty("LeadSource"); ImportTargetProperty tpImportSource = importManager.EntityManager.GetEntityProperty("ImportSource"); if (tpOwner != null) { string ownwerId = "SYST00000001"; IOwner owner = ImportRules.GetDefaultOwner(); if (owner != null) { ownwerId = owner.Id.ToString(); } tpOwner.DefaultValue = ownwerId; importManager.TargetPropertyDefaults.Add(tpOwner); ownDefaultOwner.LookupResultValue = ownwerId; } if (tpLeadSource != null) { tpLeadSource.DefaultValue = String.Empty; importManager.TargetPropertyDefaults.Add(tpLeadSource); } } else { foreach (ImportTargetProperty tp in importManager.TargetPropertyDefaults) { if (tp.PropertyId.Equals("Owner")) ownDefaultOwner.LookupResultValue = EntityFactory.GetById<IOwner>(tp.DefaultValue); if (tp.PropertyId.Equals("LeadSource")) lueLeadSource.LookupResultValue = tp.DefaultValue.ToString(); } } }
public static void Main(string[] args) { ProductsManager.StartApi(false); ImportManager.Import(typeof(DescendantOfAdamAndEveImporter)); }
private IList<SourceFieldMap> GetSourceList(ImportManager importManager) { List<SourceFieldMap> list = new List<SourceFieldMap>(); if (importManager != null) { foreach (ImportSourceProperty sp in importManager.GetSourcePropertyDisplayList(true)) { SourceFieldMap sfm = new SourceFieldMap(); sfm.FieldIndex = sp.FieldIndex; sfm.FieldName = sp.FieldName; foreach (ImportMap map in importManager.ImportMaps) { if (map.SourceProperty.FieldName.Equals(sp.FieldName)) { sfm.SLXTargetProperty = map.TargetProperty.FullDisplayName; break; } } list.Add(sfm); } } return list; }
private static void Import(Opts opts) { bool createdTempFile = false; if (opts.FilePath.StartsWith("http:") || opts.FilePath.StartsWith("https:")) { DreamMessage packageMessage = Plug.New(opts.FilePath).InvokeEx("GET", DreamMessage.Ok(), new Result <DreamMessage>()).Wait(); if (!packageMessage.IsSuccessful) { throw new Exception(string.Format("Unable to download package from '{0}'", opts.FilePath)); } opts.FilePath = Path.GetTempFileName(); opts.Archive = true; createdTempFile = true; using (Stream tempStream = File.Create(opts.FilePath)) { packageMessage.ToStream().CopyTo(tempStream, packageMessage.ContentLength, new Result <long>()).Wait(); } } IPackageReader packageReader; if (opts.Archive) { if (!File.Exists(opts.FilePath)) { throw new ConfigurationException("No such file: {0}", opts.FilePath); } if (opts.Test) { return; } packageReader = new ArchivePackageReader(opts.FilePath); } else { if (!Directory.Exists(opts.FilePath)) { throw new ConfigurationException("No such directory: {0}", opts.FilePath); } if (opts.Test) { return; } packageReader = new FilePackageReader(opts.FilePath); } ImportManager manager; try { var manifest = packageReader.ReadManifest(new Result <XDoc>()).Wait(); FixupManifest(manifest, opts); var forceOverwrite = !(opts.PreserveLocalChanges ?? true); var importer = opts.ImportRelto.HasValue ? Importer.CreateAsync(opts.DekiApi, manifest, opts.ImportRelto.Value, forceOverwrite, new Result <Importer>()).Wait() : Importer.CreateAsync(opts.DekiApi, manifest, opts.ImportReltoPath, forceOverwrite, new Result <Importer>()).Wait(); manager = new ImportManager(importer, packageReader); } catch (Exception e) { if (createdTempFile) { File.Delete(opts.FilePath); } throw new Exception(string.Format("Import failed: {0}", e.Message), e); } manager.MaxRetries = opts.Retries; Result result = manager.ImportAsync(new Result()); int completed = 0; Console.WriteLine("Importing:"); while (!result.HasFinished) { Thread.Sleep(200); if (manager.CompletedItems <= completed) { continue; } if (SysUtil.IsUnix) { Console.WriteLine(" {0} of {1} files ({2:0}%)", manager.CompletedItems, manager.TotalItems, 100.0 * manager.CompletedItems / manager.TotalItems); } else { Console.Write(" {0} of {1} files ({2:0}%) \r", manager.CompletedItems, manager.TotalItems, 100.0 * manager.CompletedItems / manager.TotalItems); } completed = manager.CompletedItems; } if (!SysUtil.IsUnix) { Console.WriteLine(); } if (createdTempFile) { File.Delete(opts.FilePath); } if (result.HasException) { ImportException importException = result.Exception as ImportException; if (importException != null) { Console.WriteLine("Import failed on Item:\r\n{0}", importException.ManifestItem.ToPrettyString()); } throw new Exception(string.Format("Import failed: {0}", result.Exception.Message), result.Exception); } }
/// <summary> /// Determines whether a match filter is mapped. /// </summary> /// <param name="importManager">The import manager.</param> /// <param name="matchFilter">The match filter.</param> /// <returns> /// <c>true</c> if [is filter mapped] [the specified import manager]; otherwise, <c>false</c>. /// </returns> private Boolean IsFilterMapped(ImportManager importManager, string matchFilter) { IList<ImportMap> importMaps = importManager.ImportMaps; return importMaps.Any(map => matchFilter.Equals(map.TargetProperty.PropertyId)); }
private ImportManager CreateManager() { var settings = new SiteImportSettings(ImportUser); settings.IsWebTemplate = true; // Import all, but only add new data settings.ImportType = ImportTypeEnum.AllNonConflicting; settings.ImportOnlyNewObjects = true; settings.CopyFiles = false; // Allow bulk inserts for faster import, web templates must be consistent enough to allow this without collisions settings.AllowBulkInsert = true; settings.EnableSearchTasks = false; ImportManager im = new ImportManager(settings); return im; }