public void Run()
{
List <Task> taskList = new List <Task>();
TaskFactory taskFactory = new TaskFactory();
CTS = new CancellationTokenSource();
bool isfirst = true;
ParallelLoopResult pl = new ParallelLoopResult();
for (int i = 1; i < Page; i++)
{
ParallelOptions po = new ParallelOptions();
po.MaxDegreeOfParallelism = 20;
var list = GetSkipPage <Commodity>(JdType, PageSize, i);
ILuceneBulid builder = new LuceneBulid();
var loopresult = Parallel.ForEach(list, po, p =>
{
builder.BuildIndex(list.ToList(), i.ToString("000"), isfirst);
});
//builder.BuildIndex(list.ToList(), i.ToString("000"), isfirst);
}
while (pl.IsCompleted)
{
Console.WriteLine("123123");
}
Console.ReadKey();
}
public void Process()
{
try
{
logger.Debug(string.Format("ThreadNum={0}开始创建", CurrentThreadNum));
CommodityRepository commodityRepository = new CommodityRepository();
ILuceneBulid builder = new LuceneBulid();
bool isFirst = true;
int pageIndex = 1;
while (!CTS.IsCancellationRequested)
{
List <User> commodityList = commodityRepository.QueryListByEF(CurrentThreadNum, pageIndex, 1000);
if (commodityList == null || commodityList.Count == 0)
{
break;
}
else
{
builder.BuildIndex(commodityList, PathSuffix, isFirst);
logger.Debug(string.Format("ThreadNum={0}完成{1}条的创建", CurrentThreadNum, 1000 * pageIndex++));
isFirst = false;
}
}
}
catch (Exception ex)
{
CTS.Cancel();
logger.Error(string.Format("ThreadNum={0}出现异常", CurrentThreadNum), ex);
}
finally
{
logger.Debug(string.Format("ThreadNum={0}完成创建", CurrentThreadNum));
}
}
public void Process()
{
try
{
logger.Debug(string.Format("ThreadNum={0}开始创建", CurrentThreadNum));
CommodityRepository commodityRepository = new CommodityRepository();
LuceneBulid builder = new LuceneBulid();
bool isFirst = true;
int pageIndex = 1;
if (service == null)
{
while (!CTS.IsCancellationRequested)
{
List <Commodity> commodityList;
commodityList = commodityRepository.QueryList(CurrentThreadNum, pageIndex, 1000);
if (commodityList == null || commodityList.Count == 0)
{
break;
}
else
{
builder.BuildIndex(commodityList, PathSuffix, isFirst);
logger.Debug(string.Format("ThreadNum={0}完成{1}条的创建", CurrentThreadNum, 1000 * pageIndex++));
isFirst = false;
}
}
}
else
{
while (!CTS.IsCancellationRequested)
{
lock (_lock)
{
//新增代码，从Redis逐行获取
var result = service.BlockingPopItemFromList("commodity", TimeSpan.FromSeconds(3));
if (string.IsNullOrEmpty(result))
{
break;
}
else
{
var commodity = JsonHelper.JsonToObj <Commodity>(result);
builder.InsertIndex(commodity);
}
}
}
}
}
catch (Exception ex)
{
CTS.Cancel();
logger.Error(string.Format("ThreadNum={0}出现异常", CurrentThreadNum), ex);
}
finally
{
logger.Debug(string.Format("ThreadNum={0}完成创建", CurrentThreadNum));
}
}
/// <summary>
/// 构建索引
/// </summary>
public void Build()
{
TableIndexProcessor processor = new TableIndexProcessor();
var taskDataList = processor.GetEachThreadTask(40);
TaskFactory taskFactory = new TaskFactory();
taskFactory.StartNew(() =>
{
List <Task> taskList = new List <Task>();
var threadNums = Enumerable.Range(1, StaticConst.CategorySheetCount).ToList();
Random random = new Random();
int index = 0, alltaskCount = taskDataList.Count;
foreach (var taskData in taskDataList)
{
index++;
var threadCode = CommodityDAL.GetTName(random.Next(1, threadNums.Count));
Task task = taskFactory.StartNew(() =>
{
try
{
LuceneBulid luceneBuild = new LuceneBulid();
if (!PathSuffixList.Any(u => u == threadCode))
{
PathSuffixList.Add(threadCode);
}
luceneBuild.BuildIndex(taskData, threadCode, true);
}
catch (Exception ex)
{
Console.WriteLine($"BuildIndexError\t{ex.Message}");
StringValueEventArgs e = new StringValueEventArgs()
{
Value = ex.Message
};
OnTaskError(e);//每完成一个任务触发事件
CTS.Cancel();
}
}, CTS.Token).ContinueWith(t =>
{
ValueEventArgs e = new ValueEventArgs()
{
Value = (int)(index * 1.0 / alltaskCount) * 100
};
OnValueChanged(e);//每完成一个任务触发事件
});
taskList.Add(task);
if (taskList.Count > 20)
{
taskList = taskList.Where(t => !t.IsCompleted && !t.IsCanceled && !t.IsFaulted).ToList();
Task.WaitAny(taskList.ToArray());
}
}
taskList.Add(taskFactory.ContinueWhenAll(taskList.ToArray(), MergeAllLuceneIndex));
});
}
/// <summary>
/// 合并索引可以迁入process内部实现
/// </summary>
/// <param name="tasks"></param>
private static void MergeIndex(Task[] tasks = null)
{
try
{
// if (CTS.IsCancellationRequested) return;
ILuceneBulid builder = new LuceneBulid();
builder.MergeIndex(PathSuffixList.ToArray());
}
catch (Exception ex)
{
// CTS.Cancel();
logger.Error("MergeIndex出现异常", ex);
}
}
private static void MergeIndex <T>(Task[] tasks) where T : class, new()
{
try
{
if (m_cts.IsCancellationRequested)
{
return;
}
ILuceneBulid <T> builder = new LuceneBulid <T>();
builder.MergeIndex(m_pathSuffixList.ToArray());
}
catch (Exception ex)
{
m_cts.Cancel();
m_logger.Error("MergeIndex出现异常", ex);
}
}
private static void MergeAllLuceneIndex(Task[] obj)
{
try
{
if (CTS.IsCancellationRequested)
{
return;
}
ILuceneBulid builder = new LuceneBulid();
builder.MergeAllLuceneIndex(PathSuffixList.ToArray());
}
catch (Exception ex)
{
CTS.Cancel();
LogHelper.WriteLog($"MergeAllLuceneIndex\t{ex.Message}{ex}");
}
}
private void MergeAllLuceneIndex(Task[] obj)
{
try
{
ILuceneBulid builder = new LuceneBulid();
builder.MergeAllLuceneIndex(PathSuffixList.ToArray());
OnTaskComplate(new EventArgs());//任务完成触发事件
}
catch (Exception ex)
{
StringValueEventArgs e = new StringValueEventArgs()
{
Value = ex.Message
};
OnTaskError(e);//每完成一个任务触发事件
Console.WriteLine($"MergeAllLuceneIndex\t{ex.Message}{ex}");
}
}
/// <summary>
/// 构建索引
/// </summary>
public static void Build()
{
//分页数必须大于20 后期考虑
//得到任务集合，分30页
var taskDataList = TableIndexProcessor.GetEachThreadTask(20);
Stopwatch sw = new Stopwatch();
sw.Start();
TaskFactory taskFactory = new TaskFactory();
List <Task> taskList = new List <Task>();
for (int i = 0; i < taskDataList.Count; i++)
{
var i1 = i;
Task task = taskFactory.StartNew(() =>
{
try
{
LogHelper.WriteLog($"***Build{i1.ToString()} 索引开始创建****", ConsoleColor.Blue);
LuceneBulid luceneBuild = new LuceneBulid();
PathSuffixList.Add((i1 + 1).ToString("000"));
luceneBuild.BuildIndex(taskDataList.ElementAt(i1), (i1 + 1).ToString("000"), true);
}
catch (Exception e)
{
LogHelper.WriteLog($"Build{i1.ToString()}\t{e.Message}", e);
CTS.Cancel();
}
}, CTS.Token).ContinueWith((t) =>
{
LogHelper.WriteLog($"***Build{i1.ToString()} 索引创建完成****", ConsoleColor.Blue);
});
taskList.Add(task);
}
taskList.Add(taskFactory.ContinueWhenAll(taskList.ToArray(), MergeAllLuceneIndex));
Task.WaitAll(taskList.ToArray());
sw.Stop();
LogHelper.WriteLog($"Build{(CTS.IsCancellationRequested ? "失败" : "成功")} 耗时{sw.ElapsedMilliseconds / 1000 / 60}分钟");
}
public void Process()
{
try
{
m_currentThreadNum = Thread.CurrentThread.ManagedThreadId;
//m_logger.Debug(string.Format("ThreadNum={0}开始创建", m_currentThreadNum));
ILuceneBulid <T> builder = new LuceneBulid <T>();
bool isFirst = true;
if (!m_cts.IsCancellationRequested)
{
IEnumerable <T> modelList = m_getDataListFunc != null?m_getDataListFunc() : null;
if (modelList == null)
{
modelList = m_dataList;
}
int count = modelList != null?modelList.Count() : 0;
if (count > 0)
{
builder.BuildIndex(modelList, m_fieldModelList, m_pathSuffix, isFirst);
//m_logger.Debug(string.Format("ThreadNum={0}完成{1}条的创建", m_currentThreadNum, count));
isFirst = false;
}
}
}
catch (Exception ex)
{
m_cts.Cancel();
//m_logger.Error(string.Format("ThreadNum={0}出现异常", m_currentThreadNum), ex);
}
finally
{
//m_logger.Debug(string.Format("ThreadNum={0}完成创建", m_currentThreadNum));
}
}
