// 导出数据 void menu_export(object sender, System.EventArgs e) { string strError = ""; string strTimeMessage = ""; long lTotalCount = 0; // 总命中数 long lExportCount = 0; // 已经传出的数量 if (this.SelectedNode == null) { strError = "尚未选择要要导出数据的数据库节点"; goto ERROR1; } if (this.SelectedNode.ImageIndex != RESTYPE_DB) { strError = "所选择的节点不是数据库类型。请选择要导出数据的数据库节点。"; goto ERROR1; } ResPath respath = new ResPath(this.SelectedNode); // 询问导出数据的范围 ExportDataDialog data_range_dlg = new ExportDataDialog(); data_range_dlg.DbPath = respath.Path; data_range_dlg.AllRecords = true; data_range_dlg.StartPosition = FormStartPosition.CenterScreen; data_range_dlg.ShowDialog(this); if (data_range_dlg.DialogResult != DialogResult.OK) return; string strRange = "0-9999999999"; if (data_range_dlg.AllRecords == false) strRange = data_range_dlg.StartID + "-" + data_range_dlg.EndID; // 获得输出文件名 SaveFileDialog dlg = new SaveFileDialog(); dlg.Title = "请指定要保存的数据文件名"; dlg.CreatePrompt = false; dlg.OverwritePrompt = false; dlg.FileName = ""; dlg.FilterIndex = 1; dlg.Filter = "备份文件 (*.dp2bak)|*.dp2bak|XML文件 (*.xml)|*.xml|ISO2709文件 (*.iso;*.mrc)|*.iso;*.mrc|All files (*.*)|*.*"; dlg.RestoreDirectory = true; if (dlg.ShowDialog(this) != DialogResult.OK) return; ExportUtil export_util = new ExportUtil(); string strQueryXml = "<target list='" + respath.Path + ":" + "__id'><item><word>"+strRange+"</word><match>exact</match><relation>range</relation><dataType>number</dataType><maxCount>-1</maxCount></item><lang>chi</lang></target>"; RmsChannel cur_channel = Channels.CreateTempChannel(respath.Url); Debug.Assert(cur_channel != null, "Channels.GetChannel() 异常"); #if NO DigitalPlatform.Stop stop = null; if (stopManager != null) { stop = new DigitalPlatform.Stop(); stop.Register(this.stopManager, true); // 和容器关联 stop.OnStop += new StopEventHandler(this.DoStop); stop.Initial("正在导出数据 " + respath.FullPath); stop.BeginLoop(); } #endif DigitalPlatform.Stop stop = PrepareStop("正在导出数据 " + respath.FullPath); stop.OnStop -= new StopEventHandler(this.DoStop); // 去掉缺省的回调函数 stop.OnStop += (sender1, e1) => { if (cur_channel != null) cur_channel.Abort(); }; try { long lRet = cur_channel.DoSearch(strQueryXml, "default", out strError); if (lRet == -1) { strError = "检索数据库 '"+respath.Path+"' 时出错: " + strError; goto ERROR1; } if (lRet == 0) { strError = "数据库 '" + respath.Path + "' 中没有任何数据记录"; goto ERROR1; // not found } stop.Style = StopStyle.EnableHalfStop; // API的间隙才让中断。避免获取结果集的中途,因为中断而导致 Session 失效,结果集丢失,进而无法 Retry 获取 lTotalCount = lRet; // 总命中数 long lThisCount = lTotalCount; long lStart = 0; ProgressEstimate estimate = new ProgressEstimate(); estimate.SetRange(0, lTotalCount); estimate.StartEstimate(); stop.SetProgressRange(0, lTotalCount); int nRet = export_util.Begin(this, dlg.FileName, out strError); if (nRet == -1) goto ERROR1; DialogResult last_one_result = DialogResult.Yes; // 前一次对话框选择的方式 bool bDontAskOne = false; int nRedoOneCount = 0; DialogResult last_get_result = DialogResult.Retry; // 前一次对话框选择的方式 bool bDontAskGet = false; int nRedoGetCount = 0; for (; ; ) { Application.DoEvents(); // 出让界面控制权 if (stop.State != 0) { DialogResult result = MessageBox.Show(this, "确实要中断当前批处理操作?", "导出数据", MessageBoxButtons.YesNo, MessageBoxIcon.Question, MessageBoxDefaultButton.Button2); if (result == DialogResult.Yes) { strError = "用户中断"; goto ERROR1; } else { stop.Continue(); } } string strStyle = "id,xml,timestamp"; if (export_util.FileType == ExportFileType.BackupFile) strStyle = "id,xml,timestamp,metadata"; nRedoGetCount = 0; REDO_GET: Record[] searchresults = null; lRet = cur_channel.DoGetSearchResult( "default", lStart, lThisCount, strStyle, this.Lang, stop, out searchresults, out strError); if (lRet == -1) { if (stop.State != 0) // 已经中断 goto ERROR1; // 自动重试有次数限制,避免进入死循环 if (bDontAskGet == true && last_get_result == DialogResult.Retry && nRedoGetCount < 3) { nRedoGetCount++; goto REDO_GET; } DialogResult result = MessageDlg.Show(this, "获取检索结果时 (偏移量 " + lStart + ") 出错:\r\n---\r\n" + strError + "\r\n---\r\n\r\n是否重试获取操作?\r\n\r\n注:\r\n[重试] 重新获取\r\n[中断] 中断整个批处理", "导出数据", MessageBoxButtons.RetryCancel, MessageBoxDefaultButton.Button1, ref bDontAskOne, new string[] { "重试", "中断" }); last_get_result = result; if (result == DialogResult.Retry) { nRedoGetCount = 0; goto REDO_GET; } Debug.Assert(result == DialogResult.Cancel, ""); strError = "获取检索结果时出错: " + strError; goto ERROR1; } for (int i = 0; i < searchresults.Length; i++) { Record record = searchresults[i]; if (i == 0) { stop.SetMessage("正在输出记录 " + record.Path + ",已输出 " + lExportCount.ToString() + " 条。" + "剩余时间 " + ProgressEstimate.Format(estimate.Estimate(lExportCount)) + " 已经过时间 " + ProgressEstimate.Format(estimate.delta_passed)); } nRedoOneCount = 0; REDO_ONE: nRet = export_util.ExportOneRecord( cur_channel, stop, respath.Url, record.Path, record.RecordBody.Xml, record.RecordBody.Metadata, record.RecordBody.Timestamp, out strError); if (nRet == -1) { if (stop.State != 0) // 已经中断 goto ERROR1; // 重试、跳过、中断? // 重试的时候,注意保持文件最后位置,不要留下残余的尾部 // MessageBoxButtons.AbortRetryIgnore YesNoCancel if (bDontAskOne == true && last_one_result == DialogResult.No) continue; // TODO: 最好在日志文件中记载跳过的记录。或者批处理结束后显示出来 // 自动重试有次数限制,避免进入死循环 if (bDontAskOne == true && last_one_result == DialogResult.Yes && nRedoOneCount < 3) { nRedoOneCount++; goto REDO_ONE; } DialogResult result = MessageDlg.Show(this, "导出记录 '" + record.Path + "' 时出错:\r\n---\r\n" + strError + "\r\n---\r\n\r\n是否重试导出操作?\r\n\r\n注:\r\n[重试] 重新导出这条记录\r\n[跳过] 忽略导出这条记录,但继续后面的处理\r\n[中断] 中断整个批处理", "导出数据", MessageBoxButtons.YesNoCancel, MessageBoxDefaultButton.Button1, ref bDontAskOne, new string [] {"重试","跳过","中断"}); last_one_result = result; if (result == DialogResult.Yes) { nRedoOneCount = 0; goto REDO_ONE; } if (result == DialogResult.No) continue; Debug.Assert(result == DialogResult.Cancel, ""); goto ERROR1; } stop.SetProgressValue(lExportCount + 1); lExportCount++; } if (lStart + searchresults.Length >= lTotalCount) break; lStart += searchresults.Length; lThisCount -= searchresults.Length; } strTimeMessage = "总共耗费时间: " + estimate.GetTotalTime().ToString(); } finally { EndStop(stop); #if NO if (stopManager != null) { stop.EndLoop(); stop.OnStop -= new StopEventHandler(this.DoStop); stop.Initial(""); stop.Unregister(); // 和容器脱离关联 } #endif cur_channel.Close(); cur_channel = null; export_util.End(); } MessageBox.Show(this, "位于服务器 '" + respath.Url + "' 上的数据库 '" + respath.Path + "' 内共有记录 " + lTotalCount.ToString() + " 条,本次导出 " + lExportCount.ToString() + " 条。" + strTimeMessage); return; ERROR1: MessageBox.Show(this, strError); if (lExportCount > 0) MessageBox.Show(this, "数据库内共有记录 " + lTotalCount.ToString() + " 条,本次导出 " + lExportCount.ToString() + " 条"); }
public IEnumerator GetEnumerator() { string strError = ""; string strRange = "0-9999999999"; long lTotalCount = 0; // 总命中数 long lExportCount = 0; string strTimeMessage = ""; DigitalPlatform.Stop stop = this.Stop; StopStyle old_style = StopStyle.None; if (stop != null) { old_style = stop.Style; stop.Style = StopStyle.EnableHalfStop; // API的间隙才让中断。避免获取结果集的中途,因为中断而导致 Session 失效,结果集丢失,进而无法 Retry 获取 stop.OnStop += stop_OnStop; } ProgressEstimate estimate = new ProgressEstimate(); try { int i_path = 0; foreach (string path in this.Paths) { ResPath respath = new ResPath(path); string strQueryXml = "<target list='" + respath.Path + ":" + "__id'><item><word>" + strRange + "</word><match>exact</match><relation>range</relation><dataType>number</dataType><maxCount>-1</maxCount></item><lang>chi</lang></target>"; cur_channel = Channels.CreateTempChannel(respath.Url); Debug.Assert(cur_channel != null, "Channels.GetChannel() 异常"); try { long lRet = cur_channel.DoSearch(strQueryXml, "default", out strError); if (lRet == -1) { strError = "检索数据库 '" + respath.Path + "' 时出错: " + strError; throw new Exception(strError); } if (lRet == 0) { strError = "数据库 '" + respath.Path + "' 中没有任何数据记录"; continue; } lTotalCount += lRet; // 总命中数 estimate.SetRange(0, lTotalCount); if (i_path == 0) { estimate.StartEstimate(); } if (stop != null) { stop.SetProgressRange(0, lTotalCount); } SearchResultLoader loader = new SearchResultLoader(cur_channel, stop, this.ResultSetName, this.FormatList, this.Lang); loader.BatchSize = this.BatchSize; foreach (KernelRecord record in loader) { if (stop != null) { stop.SetProgressValue(lExportCount + 1); } lExportCount++; yield return(record); } strTimeMessage = "总共耗费时间: " + estimate.GetTotalTime().ToString(); } finally { cur_channel.Close(); cur_channel = null; } // MessageBox.Show(this, "位于服务器 '" + respath.Url + "' 上的数据库 '" + respath.Path + "' 内共有记录 " + lTotalCount.ToString() + " 条,本次导出 " + lExportCount.ToString() + " 条。" + strTimeMessage); i_path++; } } finally { if (stop != null) { stop.Style = old_style; stop.OnStop -= stop_OnStop; } } }
// 导入数据 int ImportData(bool bFastMode = false) { string strError = ""; string strTimeMessage = ""; int CHUNK_SIZE = 150 * 1024; // 70 if (this.SelectedNode == null) { strError = "尚未选择要要导入数据的数据库节点"; goto ERROR0; } if (this.SelectedNode.ImageIndex != RESTYPE_DB) { strError = "所选择的节点不是数据库类型。请选择要导入数据的数据库节点。"; goto ERROR0; } if (bFastMode == true) { DialogResult result = MessageBox.Show(this, "警告:\r\n在快速导入期间,相关数据库会进入一种锁定状态,对数据库的其他检索和修改操作暂时会被禁止,直到处理完成。\r\n\r\n请问确实要进行快速导入么?", "导入数据", MessageBoxButtons.YesNo, MessageBoxIcon.Question, MessageBoxDefaultButton.Button2); if (result == DialogResult.No) return 0; } OpenFileDialog dlg = new OpenFileDialog(); dlg.Title = "请指定要导入的数据文件"; dlg.FileName = ""; dlg.Filter = "备份文件 (*.dp2bak)|*.dp2bak|XML文件 (*.xml)|*.xml|ISO2709文件 (*.iso;*.mrc)|*.iso;*.mrc|All files (*.*)|*.*"; dlg.RestoreDirectory = true; if (dlg.ShowDialog() != DialogResult.OK) { return 0; } long lTotalCount = 0; ImportUtil import_util = new ImportUtil(); int nRet = import_util.Begin(this, this.AppInfo, dlg.FileName, out strError); if (nRet == -1 || nRet == 1) goto ERROR0; #if NO ResPath respath = new ResPath(this.SelectedNode); this.channel = Channels.GetChannel(respath.Url); Debug.Assert(channel != null, "Channels.GetChannel() 异常"); #endif // 缺省的目标数据库路径 ResPath default_target_respath = new ResPath(this.SelectedNode); RmsChannel cur_channel = Channels.CreateTempChannel(default_target_respath.Url); Debug.Assert(cur_channel != null, "Channels.GetChannel() 异常"); List<string> target_dburls = new List<string>(); #if NO DigitalPlatform.Stop stop = null; if (stopManager != null) { stop = new DigitalPlatform.Stop(); stop.Register(this.stopManager, true); // 和容器关联 stop.OnStop += new StopEventHandler(this.DoStop); stop.Initial("正在导入数据 " + respath.FullPath); stop.BeginLoop(); } #endif DigitalPlatform.Stop stop = PrepareStop("正在导入数据 ..."); // + default_target_respath.FullPath); stop.OnStop -= new StopEventHandler(this.DoStop); // 去掉缺省的回调函数 stop.OnStop += (sender1, e1) => { if (cur_channel != null) cur_channel.Abort(); }; stop.Style = StopStyle.EnableHalfStop; // API的间隙才让中断。避免获取结果集的中途,因为中断而导致 Session 失效,结果集丢失,进而无法 Retry 获取 ProgressEstimate estimate = new ProgressEstimate(); try // open import util { bool bDontPromptTimestampMismatchWhenOverwrite = false; DbNameMap map = new DbNameMap(); long lSaveOffs = -1; estimate.SetRange(0, import_util.Stream.Length); estimate.StartEstimate(); stop.SetProgressRange(0, import_util.Stream.Length); List<UploadRecord> records = new List<UploadRecord>(); int nBatchSize = 0; for (int index = 0; ; index++) { Application.DoEvents(); // 出让界面控制权 if (stop.State != 0) { DialogResult result = MessageBox.Show(this, "确实要中断当前批处理操作?", "导入数据", MessageBoxButtons.YesNo, MessageBoxIcon.Question, MessageBoxDefaultButton.Button2); if (result == DialogResult.Yes) { strError = "用户中断"; goto ERROR1; } else { stop.Continue(); } } //string strXml = ""; //string strResPath = ""; //string strTimeStamp = ""; UploadRecord record = null; if (import_util.FileType == ExportFileType.BackupFile) { if (lSaveOffs != -1) import_util.Stream.Seek(lSaveOffs, SeekOrigin.Begin); } nRet = import_util.ReadOneRecord(out record, out strError); if (nRet == -1) goto ERROR1; if (nRet == 1) break; if (import_util.FileType == ExportFileType.BackupFile) { // 保存每次读取后的文件指针位置 lSaveOffs = import_util.Stream.Position; } Debug.Assert(record != null, ""); #if NO XmlDocument dom = new XmlDocument(); try { dom.LoadXml(strXml); } catch (Exception ex) { strError = "XML装入DOM时出错: " + ex.Message; goto ERROR1; } string strResPath = DomUtil.GetAttr(DpNs.dprms, dom.DocumentElement, "path"); string strTimeStamp = DomUtil.GetAttr(DpNs.dprms, dom.DocumentElement, "timestamp"); #endif // 准备目标路径 { string strLongPath = record.Url + "?" + record.RecordBody.Path; #if NO // 根据原始路径准备即将写入的路径 // return: // -1 出错 // 0 用户放弃 // 1 成功 nRet = ImportUtil.PrepareOverwritePath( this.Servers, this.Channels, this, ref map, ref strLongPath, out strError); if (nRet == 0 || nRet == -1) goto ERROR1; #endif // 根据原始路径准备即将写入的路径 // return: // -1 出错 // 0 用户放弃 // 1 成功 // 2 要跳过本条 nRet = ImportUtil.PrepareOverwritePath( this, this.Servers, this.Channels, this.AppInfo, index, default_target_respath.FullPath, ref map, ref strLongPath, out strError); if (nRet == 0 || nRet == -1) goto ERROR1; if (nRet == 2) continue; ResPath respath = new ResPath(strLongPath); record.Url = respath.Url; record.RecordBody.Path = respath.Path; // 记载每个数据库的 URL string strDbUrl = GetDbUrl(strLongPath); if (target_dburls.IndexOf(strDbUrl) == -1) { // 每个数据库要进行一次快速模式的准备操作 if (bFastMode == true) { nRet = ManageKeysIndex(strDbUrl, "beginfastappend", "正在对数据库 "+strDbUrl+" 进行快速导入模式的准备工作 ...", out strError); if (nRet == -1) goto ERROR1; } target_dburls.Add(strDbUrl); } } bool bNeedPush = false; // 是否要把积累的记录推送出去进行写入? // 要进行以下检查: // 1) 当前记录和前一条记录之间,更换了服务器 // 2) 累积的记录尺寸超过要求 // 3) 当前记录是一条超大的记录 (这是因为要保持从文件中读出的顺序来写入(例如追加时候的号码增量顺序),就必须在单条写入本条前,先写入积累的那些记录) if (records.Count > 0) { if (record.TooLarge() == true) bNeedPush = true; else if (nBatchSize + record.RecordBody.Xml.Length > CHUNK_SIZE) bNeedPush = true; else { if (LastUrl(records) != record.Url) bNeedPush = true; } } if (bNeedPush == true) { // 准备 Channel Debug.Assert(records.Count > 0, ""); cur_channel = ImportUtil.GetChannel(this.Channels, stop, LastUrl(records), cur_channel); List<UploadRecord> save_records = new List<UploadRecord>(); save_records.AddRange(records); while (records.Count > 0) { // 将 XML 记录成批写入数据库 // return: // -1 出错 // >=0 本次已经写入的记录个数。本函数返回时 records 集合的元素数没有变化(但元素的Path和Timestamp会有变化),如果必要调主可截取records集合中后面未处理的部分再次调用本函数 nRet = ImportUtil.WriteRecords( this, stop, cur_channel, bFastMode, records, ref bDontPromptTimestampMismatchWhenOverwrite, out strError); if (nRet == -1) goto ERROR1; if (nRet == 0) { // TODO: 或可以改为单条写入 strError = "WriteRecords() error :" + strError; goto ERROR1; } Debug.Assert(nRet <= records.Count, ""); records.RemoveRange(0, nRet); lTotalCount += nRet; } // 上载对象 // return: // -1 出错 // 0 成功 nRet = import_util.UploadObjects( stop, cur_channel, save_records, ref bDontPromptTimestampMismatchWhenOverwrite, out strError); if (nRet == -1) goto ERROR1; nBatchSize = 0; stop.SetProgressValue(import_util.Stream.Position); stop.SetMessage("已经写入记录 " + lTotalCount.ToString() + " 条。" + "剩余时间 " + ProgressEstimate.Format(estimate.Estimate(import_util.Stream.Position)) + " 已经过时间 " + ProgressEstimate.Format(estimate.delta_passed)); } // 如果 记录的 XML 尺寸太大不便于成批上载,需要在单独直接上载 if (record.TooLarge() == true) { // 准备 Channel // ResPath respath = new ResPath(record.RecordBody.Path); cur_channel = ImportUtil.GetChannel(this.Channels, stop, record.Url, cur_channel); // 写入一条 XML 记录 // return: // -1 出错 // 0 邀请中断整个处理 // 1 成功 // 2 跳过本条,继续处理后面的 nRet = ImportUtil.WriteOneXmlRecord( this, stop, cur_channel, record, ref bDontPromptTimestampMismatchWhenOverwrite, out strError); if (nRet == -1) goto ERROR1; if (nRet == 0) goto ERROR1; List<UploadRecord> temp = new List<UploadRecord>(); temp.Add(record); // 上载对象 // return: // -1 出错 // 0 成功 nRet = import_util.UploadObjects( stop, cur_channel, temp, ref bDontPromptTimestampMismatchWhenOverwrite, out strError); if (nRet == -1) goto ERROR1; lTotalCount += 1; continue; } records.Add(record); if (record.RecordBody != null && record.RecordBody.Xml != null) nBatchSize += record.RecordBody.Xml.Length; } // 最后提交一次 if (records.Count > 0) { // 准备 Channel Debug.Assert(records.Count > 0, ""); cur_channel = ImportUtil.GetChannel(this.Channels, stop, LastUrl(records), cur_channel); List<UploadRecord> save_records = new List<UploadRecord>(); save_records.AddRange(records); while (records.Count > 0) { // 将 XML 记录成批写入数据库 // return: // -1 出错 // >=0 本次已经写入的记录个数。本函数返回时 records 集合的元素数没有变化(但元素的Path和Timestamp会有变化),如果必要调主可截取records集合中后面未处理的部分再次调用本函数 nRet = ImportUtil.WriteRecords( this, stop, cur_channel, bFastMode, records, ref bDontPromptTimestampMismatchWhenOverwrite, out strError); if (nRet == -1) goto ERROR1; if (nRet == 0) { strError = "WriteRecords() error :" + strError; goto ERROR1; } Debug.Assert(nRet <= records.Count, ""); records.RemoveRange(0, nRet); lTotalCount += nRet; } // 上载对象 // return: // -1 出错 // 0 成功 nRet = import_util.UploadObjects( stop, cur_channel, save_records, ref bDontPromptTimestampMismatchWhenOverwrite, out strError); if (nRet == -1) goto ERROR1; nBatchSize = 0; stop.SetProgressValue(import_util.Stream.Position); stop.SetMessage("已经写入记录 " + lTotalCount.ToString() + " 条。" + "剩余时间 " + ProgressEstimate.Format(estimate.Estimate(import_util.Stream.Position)) + " 已经过时间 " + ProgressEstimate.Format(estimate.delta_passed)); records.Clear(); nBatchSize = 0; } }// close import util finally { if (bFastMode == true) { foreach (string url in target_dburls) { string strQuickModeError = ""; nRet = ManageKeysIndex(url, "endfastappend", "正在对数据库 " + url + " 进行快速导入模式的收尾工作,请耐心等待 ...", out strQuickModeError); if (nRet == -1) MessageBox.Show(this, strQuickModeError); } } EndStop(stop); #if NO if (stopManager != null) { stop.EndLoop(); stop.OnStop -= new StopEventHandler(this.DoStop); stop.Initial(""); stop.Unregister(); // 和容器脱离关联 } #endif cur_channel.Close(); cur_channel = null; import_util.End(); } strTimeMessage = "总共耗费时间: " + estimate.GetTotalTime().ToString(); MessageBox.Show(this, "文件 " + dlg.FileName + " 内的数据已经成功导入下列数据库:\r\n\r\n" + StringUtil.MakePathList(target_dburls, "\r\n") + "\r\n\r\n共导入记录 " + lTotalCount.ToString() + " 条。\r\n\r\n" + strTimeMessage); return 0; ERROR0: MessageBox.Show(this, strError); return -1; ERROR1: MessageBox.Show(this, strError); // 使用了 lTotalCount 和 estimate 以后的报错 if (lTotalCount > 0) { strTimeMessage = "总共耗费时间: " + estimate.GetTotalTime().ToString(); MessageBox.Show(this, "文件 " + dlg.FileName + " 内的部分数据已经成功导入下列数据库:\r\n\r\n" + StringUtil.MakePathList(target_dburls, "\r\n") + "\r\n\r\n共导入记录 " + lTotalCount.ToString() + " 条。\r\n\r\n" + strTimeMessage); } return -1; }