}//_getPKColumnForTable( string SourceTable ) /// <summary> /// Post the new import definitions created in this ImportMgr to the database. /// </summary> public void finalize() { if (null == _NbtImporter) { throw new CswDniException(CswEnumErrorType.Error, "UpdateImportMgr tried to store definition, but NbtImporter is not instanced.", ""); } _NbtImporter.storeDefinition(_importOrderTable, _importBindingsTable, _importRelationshipsTable); _NbtImporter.Finish(); }//finalize()
public static void uploadImportDefinition(ICswResources CswResources, CswWebSvcReturn ret, CswNbtImportWcf.ImportFileParams parms) { CswNbtResources CswNbtResources = (CswNbtResources)CswResources; CswNbtImporter Importer = new CswNbtImporter(CswNbtResources.AccessId, CswEnumSetupMode.NbtWeb); // Write uploaded file to temp dir CswTempFile myTempFile = new CswTempFile(CswResources); string path = myTempFile.saveToTempFile(parms.PostedFile.InputStream, DateTime.Now.Ticks + "_" + parms.PostedFile.FileName); Importer.storeDefinition(path, parms.ImportDefName); Importer.Finish(); }
public void threadCallBack(ICswResources CswResources) { _LogicRunStatus = CswEnumScheduleLogicRunStatus.Running; CswNbtResources CswNbtResources = (CswNbtResources)CswResources; CswNbtResources.AuditContext = "Scheduler Task: " + RuleName; if (CswEnumScheduleLogicRunStatus.Stopping != _LogicRunStatus) { try { Int32 ImportLimit = CswConvert.ToInt32(CswNbtResources.ConfigVbls.getConfigVariableValue(CswEnumConfigurationVariableNames.NodesProcessedPerCycle)); if (Int32.MinValue == ImportLimit) { ImportLimit = 10; // Default } if (_DataTableNames.Count > 0) { CswNbtImporter Importer = new CswNbtImporter(CswNbtResources.AccessId, CswEnumSetupMode.NbtExe); Int32 RowsProcessed; bool MoreToDo = Importer.ImportRows(ImportLimit, _DataTableNames[0], out RowsProcessed); if (false == MoreToDo) { _DataTableNames.RemoveAt(0); } Importer.Finish(); } else { _CswScheduleLogicDetail.LoadCount = 0; } _CswScheduleLogicDetail.StatusMessage = "Completed without error"; _LogicRunStatus = CswEnumScheduleLogicRunStatus.Succeeded; //last line }//try catch (Exception Exception) { _CswScheduleLogicDetail.StatusMessage = "CswScheduleLogicNbtImport::threadCallBack() exception: " + Exception.Message + "; " + Exception.StackTrace; CswNbtResources.logError(new CswDniException(_CswScheduleLogicDetail.StatusMessage)); _LogicRunStatus = CswEnumScheduleLogicRunStatus.Failed; } //catch } //if we're not shutting down } //threadCallBack()
public void threadCallBack(ICswResources CswResources) { _LogicRunStatus = CswEnumScheduleLogicRunStatus.Running; if (CswEnumScheduleLogicRunStatus.Stopping != _LogicRunStatus) { CswNbtResources _CswNbtResources = (CswNbtResources)CswResources; try { const Int32 MaxRowsToProcessPerThread = 1000; //This number must not exceed the amount of importing we expect to be able to do in _CswScheduleLogicDetail.MaxRunTimeMs const Int32 MinNumberToProcess = 100; //CIS-53123 - we should process at least this many rows at a time (because committing is expensive) - this number must not exceed 1000 (because we're using an IN clause) Int32 NumberToProcess = CswConvert.ToInt32(_CswNbtResources.ConfigVbls.getConfigVariableValue(CswEnumConfigurationVariableNames.NodesProcessedPerCycle)); NumberToProcess = NumberToProcess < MinNumberToProcess ? MinNumberToProcess : NumberToProcess; Int32 NumberOfCommits = MaxRowsToProcessPerThread / NumberToProcess;//CIS-53123 - process MaxRowsToProcessPerThread rows per rule iteration, committing every NumberToProcess rows const string QueueTableName = "nbtimportqueue"; const string QueuePkName = "nbtimportqueueid"; for (int c = 0; c < NumberOfCommits; c++) { string Sql = "select nbtimportqueueid, state, itempk, pkcolumnname, sheetname, priority, importorder, tablename, coalesce(viewname, tablename) as sourcename, nodetypename from " + QueueTableName + "@" + CAFDbLink + " iq" + " join " + CswNbtImportTables.ImportDefOrder.TableName + " io on ( coalesce(viewname, tablename) = iq.sheetname )" //TODO - remove ignoring Delete rows when we support Importing deleted nodes + " where state != '" + State.E + "' and state != '" + State.D //Much faster than (I or U) + "' order by decode (state, '" + State.I + "', 1, '" + State.U + "', 2) asc, priority desc, importorder asc, nbtimportqueueid asc"; CswArbitrarySelect QueueSelect = _CswNbtResources.makeCswArbitrarySelect("cafimport_queue_select", Sql); DataTable QueueTable = QueueSelect.getTable(0, NumberToProcess, false); if (QueueTable.Rows.Count > 0) { Collection <String> ImportQueuePKs = new Collection <string>(); CswCommaDelimitedString ItemPKs = new CswCommaDelimitedString(); string ImportOrder = QueueTable.Rows[0]["importorder"].ToString(); DataRow QueueRowDef = QueueTable.Rows[0]; CswNbtImporter Importer = new CswNbtImporter(_CswNbtResources.AccessId, CswEnumSetupMode.NbtExe); foreach (DataRow QueueRow in QueueTable.Rows) { string CurrentTblNamePkCol = CswConvert.ToString(QueueRow["pkcolumnname"]); if (string.IsNullOrEmpty(CurrentTblNamePkCol)) { throw new Exception("Could not find pkcolumn in data_dictionary for table " + QueueRow["tablename"]); } if (QueueRow["importorder"].ToString() != ImportOrder) { break;//We've changed NodeTypes - we'll pick them up next time around } ImportQueuePKs.Add(QueueRow[QueuePkName].ToString()); ItemPKs.Add("'" + QueueRow["itempk"] + "'"); } string ItemSql = "select * from " + QueueRowDef["sourcename"] + "@" + CAFDbLink + " where " + QueueRowDef["pkcolumnname"] + " in(" + ItemPKs + ")"; CswArbitrarySelect ItemSelect = _CswNbtResources.makeCswArbitrarySelect("cafimport_queue_select", ItemSql); DataTable ItemTable = ItemSelect.getTable(); for (int i = 0; i < ItemTable.Rows.Count; i++) { DataRow ItemRow = ItemTable.Rows[i]; string NodetypeName = QueueRowDef["nodetypename"].ToString(); bool Overwrite = QueueRowDef["state"].ToString().Equals("U"); string Error = Importer.ImportRow(ItemRow, DefinitionName, NodetypeName, Overwrite); if (string.IsNullOrEmpty(Error)) { // record success - delete the record _CswNbtResources.execArbitraryPlatformNeutralSql("delete from " + QueueTableName + "@" + CAFDbLink + " where " + QueuePkName + " = " + ImportQueuePKs[i]); } else { // truncate error to 2000 chars string SafeError = CswTools.SafeSqlParam(Error); if (SafeError.Length > 2000) { SafeError = SafeError.Substring(0, 2000); } // record failure - record the error on nbtimportqueue _CswNbtResources.execArbitraryPlatformNeutralSql("update " + QueueTableName + "@" + CAFDbLink + " set state = '" + State.E + "', " + " errorlog = '" + SafeError + "' " + " where " + QueuePkName + " = " + ImportQueuePKs[i]); } _CswScheduleLogicDetail.LoadCount = _CswScheduleLogicDetail.LoadCount - 1; } Importer.Finish(); //CIS-53123 - Commit every NumberToProcess rows (performes better than spawning a new thread every NumberToProcess rows) _CswNbtResources.finalize(); } else { c = NumberOfCommits;//We're done importing! } } _CswScheduleLogicDetail.StatusMessage = "Completed without error"; _LogicRunStatus = CswEnumScheduleLogicRunStatus.Succeeded; //last line }//try catch (Exception Exception) { _CswScheduleLogicDetail.StatusMessage = "CswScheduleLogicNbtCAFImport::ImportItems() exception: " + Exception.Message + "; " + Exception.StackTrace; _CswNbtResources.logError(new CswDniException(_CswScheduleLogicDetail.StatusMessage)); _LogicRunStatus = CswEnumScheduleLogicRunStatus.Failed; } //catch } //if we're not shutting down } //threadCallBack()