public JsonResult EpdateEsData()
{
string EsNodeUrl = ConfigurationManager.AppSettings["EsNodeUrl"];
var esHelper = EsBaseHelper.CreateEsHelper(EsNodeUrl); //查询
esHelper.DefaultIndex = EsConst.defaultEsFileIndex;
esHelper.UpdateByQuery();
return(Json(new { }));
}
public JsonResult GetList(QueryBuilder qb)
{
var pi = qb.PageIndex;
var ps = qb.PageSize;
var qbItem = qb.Items.FirstOrDefault(a => a.Field == "QueryText");
var queryValue = string.Empty;
var spaceID = string.Empty;
if (qbItem != null)
{
queryValue = qbItem.Value.ToString();
}
qbItem = qb.Items.FirstOrDefault(a => a.Field == "SpaceID");
if (qbItem != null)
{
spaceID = qbItem.Value.ToString();
}
var dataList = new List <Dictionary <string, object> >();
var esHelper = EsBaseHelper.CreateEsHelper(EsNodeUrl); //查询
esHelper.DefaultIndex = EsConst.defaultEsFileIndex;
var queryResult = esHelper.QueryDocument(queryValue, spaceID, pi, ps);//查询结果
var hitsList = queryResult.Hits;
foreach (var item in hitsList)
{
var dic = new Dictionary <string, object>();
var hitSource = item.Source;
if (hitSource != null)
{
dic = FormulaHelper.ModelToDic(hitSource);
}
//获得高亮数据
var content = string.Empty; //文档内容
string title = string.Empty; //标题
string propertyStr = string.Empty; //属性
if (item.Highlight.Count > 0)
{
foreach (var highlight in item.Highlight)
{
if (highlight.Key == "title")
{
title += string.Join("", highlight.Value.ToArray());
}
else if (highlight.Key == "content")
{
content += string.Join(" ", highlight.Value.ToArray()).Replace("\r\n", "").Replace("\r", "").Replace("\n", "");
}
else if (highlight.Key == "propertyJson")
{
propertyStr += string.Join("", highlight.Value.ToArray());
}
}
}
if (string.IsNullOrEmpty(title) && hitSource != null)
{
title = hitSource.Title;
}
if (string.IsNullOrEmpty(propertyStr) && hitSource != null)
{
propertyStr = hitSource.PropertyJson;
}
dic.SetValue("title", title);
dic.SetValue("propertyStr", propertyStr);
dic.SetValue("content", content);
dataList.Add(dic);
}
var gridData = new GridData(dataList);
var total = 0;
if (queryResult.Total != null)
{
total = Convert.ToInt32(queryResult.Total.Value);
}
gridData.total = total;
return(Json(gridData));
}
public void Convert()
{
#region 创建esHelper、configHelper
var esUrl = ConfigurationManager.AppSettings["EsUrl"];
if (string.IsNullOrEmpty(esUrl))
{
LogWriter.Error(string.Format("文字提取程序异常：{0}，错误：{1}", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"), "配置文件缺少EsUrl"));
return;
}
var esHelper = EsBaseHelper.CreateEsHelper(esUrl);
try
{
if (!esHelper.ExistsIndex(EsConst.defaultEsFileIndex))
{
esHelper.CreateIndex <EsFile>(EsConst.defaultEsFileIndex);//创建es索引
}
}
catch (Exception ex)
{
LogWriter.Error(ex, string.Format("文字提取程序异常：{0}，错误：{1}", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"), ex.Message));
return;
}
esHelper.DefaultIndex = EsConst.defaultEsFileIndex;//设置es默认索引
SQLHelper configHelper = SQLHelper.CreateSqlHelper("DocConfigCntString");
#endregion
string spaceSql = "select * from dbo.S_DOC_Space ";
var spaceDt = configHelper.ExecuteDataTable(spaceSql);
string nodeConfigSQL = @"select NodeID ConfigID,SpaceID,AttrField Field,DataType,InputType,EnumKey,FulltextProp,AttrSort from S_DOC_NodeAttr
where FulltextProp='True'
union
select FileID ConfigID,SpaceID,FileAttrField Field,DataType,InputType,EnumKey,FulltextProp,AttrSort from S_DOC_FileAttr
where FulltextProp='True'
order by AttrSort ";
var propFieldDt = configHelper.ExecuteDataTable(nodeConfigSQL);
var enumFieldDt = propFieldDt.Select("EnumKey is not null");
var enumDic = new Dictionary <string, DataTable>();
var enumService = Formula.FormulaHelper.GetService <Formula.IEnumService>();
foreach (var item in enumFieldDt)
{
var enumKey = item["EnumKey"].ToString();
if (enumDic.ContainsKey(enumKey))
{
continue;
}
var enumDt = enumService.GetEnumTable(enumKey);
enumDic.Add(enumKey, enumDt);
}
foreach (DataRow space in spaceDt.Rows)
{
try
{
string constr = String.Format(conStrTemplate, space["Server"].ToString()
, space["UserName"].ToString(), space["Pwd"].ToString(), space["DbName"].ToString());
SQLHelper sqlHepler = new SQLHelper(constr);
string sql = @"select S_Attachment.*,S_FileInfo.ConfigID from S_Attachment
left join S_FileInfo on S_FileInfo.ID = S_Attachment.FileID
where S_Attachment.State='Normal' and CurrentVersion='True' and S_FileInfo.State='Published'
and ((MainFile is not null and MainFile!='' ) or (PDFFile is not null and PDFFile!='' ))
order by ID";
DataTable dt = sqlHepler.ExecuteDataTable(sql);
var SpaceID = space["ID"].ToString();
int i = 1;
foreach (DataRow attItem in dt.Rows)
{
string logSql = string.Empty;
var Id = SQLHelper.CreateGuid();
string mainFile = string.Empty;
if (attItem["PDFFile"] != DBNull.Value && attItem["PDFFile"] != null && !string.IsNullOrEmpty(attItem["PDFFile"].ToString()))
{
mainFile = attItem["PDFFile"].ToString();
}
if (string.IsNullOrEmpty(mainFile))
{
if (attItem["MainFile"] != DBNull.Value && attItem["MainFile"] != null && !string.IsNullOrEmpty(attItem["MainFile"].ToString()))
{
mainFile = attItem["MainFile"].ToString();
}
}
var FileID = attItem["FileID"].ToString();
var NodeID = attItem["NodeID"].ToString();
var ConfigID = attItem["ConfigID"].ToString();
var AttrID = attItem["ID"].ToString();
string FormatLogSql = @"INSERT INTO S_DOC_FulltextSearchConvertLog
([ID],[FsFileID],[AttrID] ,[FileID] ,[NodeID] ,[SpaceID] ,[CreateDate] ,[ConvertState],[ErrorMeesage])
VALUES ('" + Id + "' ,'" + mainFile.Replace("'", "''") + "','" + AttrID + "','" + FileID + "' ,'" + NodeID + "' ,'" + SpaceID + "' ,'" + DateTime.Now + "','{0}','{1}')";
string updateSql = "update S_Attachment Set State='Finish' where ID='" + attItem["ID"].ToString() + "'";
Console.WriteLine("文件信息：" + mainFile + " " + DateTime.Now.ToString());
var ext = GetFileExt(mainFile).ToLower();
try
{
string content = string.Empty;
//string fullPath = service.GetFileFullPath(mainFile);
var file = FileStoreHelper.GetFile(mainFile);
//Console.WriteLine("文件路径：" + fullPath);
#region 提取文字
switch (ext)
{
case "docx":
case "doc": content = ExtractHelper.GetWordText(file); break;
case "xls":
case "xlsx": content = ExtractHelper.GetExcelText(file); break;
case "pdf": content = ExtractHelper.GetPdfText_Itextsharp(file); break;
case "txt": content = ExtractHelper.GetTxtText(file); break;
default:
{
Console.WriteLine(ext + "文件跳过");
sqlHepler.ExecuteNonQuery(updateSql);
Console.Write(ext + " 格式不对跳过" + i.ToString() + "/" + dt.Rows.Count);
i++;
continue;
}
}
Console.WriteLine("获取信息内容完成");
#endregion
#region 属性json、全路径json
var fileDt = sqlHepler.ExecuteDataTable(@"select * from S_FileInfo where id='" + FileID + "'");
if (fileDt.Rows.Count == 0)
{
throw new Exception(string.Format("没有找到ID为【{0}】的S_FileInfo的记录", FileID));
}
var nodeFullID = fileDt.Rows[0]["FullNodeID"].ToString();
var nodeDt = sqlHepler.ExecuteDataTable(@"select * from S_NodeInfo where id in ('" + nodeFullID.Replace(".", "','") + "')");
nodeDt.PrimaryKey = new DataColumn[] { nodeDt.Columns["ID"] };
string propertyJson = string.Empty; //属性，空格分隔的string，暂时不存Json
string nodePathJson = string.Empty; //全路径Json
var nodePathList = new List <Dictionary <string, string> >();
nodePathList.Add(new Dictionary <string, string>()
{
{ "id", SpaceID }, { "name", space["Name"].ToString() }, { "type", "space" }
});
foreach (var nid in nodeFullID.Split('.'))
{
var nodeRow = nodeDt.Rows.Find(nid);
if (nodeRow == null)
{
continue;
}
//目录属性
var propString = GetPropStrings(propFieldDt, nodeRow, enumDic);
propertyJson += propString;
//全路径
string nname = nodeRow["Name"].ToString();
nodePathList.Add(new Dictionary <string, string>()
{
{ "id", nid }, { "name", nname }, { "type", "node" }
});
}
//文件节点目录
nodePathList.Add(new Dictionary <string, string>()
{
{ "id", SpaceID }, { "name", fileDt.Rows[0]["Name"].ToString() }, { "type", "file" }
});
nodePathJson = JsonHelper.ToJson(nodePathList);
//文件属性
var filePropString = GetPropStrings(propFieldDt, fileDt.Rows[0], enumDic);
propertyJson += filePropString;
propertyJson = propertyJson.TrimEnd();
#endregion
#region 插入Es
var esFile = new EsFile();
esFile.Id = FileID;
esFile.SpaceID = SpaceID;
esFile.NodeID = NodeID;
esFile.FileID = FileID;
esFile.AttrID = AttrID;
esFile.ConfigID = ConfigID;
if (!string.IsNullOrEmpty(mainFile))
{
esFile.FsFileID = System.Convert.ToInt32(mainFile.Split('_')[0]);
}
esFile.Title = GetFileName(mainFile);
esFile.Content = content;
esFile.PropertyJson = propertyJson;
esFile.FullPathJson = nodePathJson;
if (attItem["CreateDate"] != null && attItem["CreateDate"] != DBNull.Value)
{
esFile.FileCreateDate = System.Convert.ToDateTime(attItem["CreateDate"]);
}
else
{
esFile.FileCreateDate = DateTime.Now; //归档日期
}
if (attItem["CreateUserName"] != null && attItem["CreateUserName"] != DBNull.Value)
{
esFile.FileCreateUser = attItem["CreateUserName"].ToString(); //归档人
}
esFile.SecretLevel = string.Empty; //密级
esHelper.AddDocument(esFile);
Console.WriteLine("更新Es数据完成");
#endregion
sqlHepler.ExecuteNonQuery(updateSql);
logSql = String.Format(FormatLogSql, "Success", "");
configHelper.ExecuteNonQuery(logSql);
Console.Write(" 成功" + i.ToString() + "/" + dt.Rows.Count);
i++;
}
catch (Exception ex)
{
logSql = String.Format(FormatLogSql, "Error", ex.Message);
configHelper.ExecuteNonQuery(logSql);
LogWriter.Error(ex, string.Format("文字提取程序异常：{0}，错误：{1}", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"), ex.Message));
Console.WriteLine("失败跳过" + i.ToString() + "/" + dt.Rows.Count);
i++;
continue;
}
}
}
catch (Exception exp)
{
LogWriter.Error(exp, string.Format("文字提取程序异常：{0}，错误：{1}", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"), exp.Message));
Console.WriteLine(exp.InnerException);
continue;
}
}
}
