public virtual bool Init(IEnumerable <IFreeDocument> datas)
{
mainstream =
processManager.CurrentProcessCollections.OfType <SmartETLTool>()
.FirstOrDefault(d => d.CurrentETLTools.Contains(this));
etl =
processManager.CurrentProcessCollections.FirstOrDefault(d => d.Name == ETLSelector) as SmartETLTool;
if (etl != null)
{
return(true);
}
var task = processManager.CurrentProject.Tasks.FirstOrDefault(d => d.Name == ETLSelector);
if (task == null)
{
return(false);
}
ControlExtended.UIInvoke(() => { task.Load(false); });
etl =
processManager.CurrentProcessCollections.FirstOrDefault(d => d.Name == ETLSelector) as SmartETLTool;
etl.InitProcess(true);
return(etl != null);
}
public IDataProcess GetOneInstance(string name, bool isAddToList = true, bool newOne = false,
bool isAddUI = false)
{
if (newOne)
{
var process = PluginProvider.GetObjectByType <IDataProcess>(name);
if (process != null)
{
if (isAddToList)
{
ProcessCollection.Add(process);
process.SysDataManager = dataManager;
process.SysProcessManager = this;
var rc4 = process as AbstractProcessMethod;
if (rc4 != null)
{
rc4.MainPluginLocation = MainFrmUI.MainPluginLocation;
rc4.MainFrm = MainFrmUI;
}
XLogSys.Print.Info("已经成功添加" + process.TypeName + "到当前列表");
}
if (isAddUI)
{
ControlExtended.UIInvoke(() => LoadProcessView(process));
ControlExtended.UIInvoke(() => ShowConfigUI(process));
}
return(process);
}
}
return(ProcessCollection.Get(name, isAddToList));
}
public IEnumerable <IFreeDocument> Execute(IEnumerable <IFreeDocument> documents)
{
foreach (var document in documents)
{
var doc = MappingDocument(document);
if (AddTask)
{
var name = doc[Column];
ControlExtended.UIInvoke(() =>
{
var task = TemporaryTask <FreeDocument> .AddTempTaskSimple("ETL" + name, func(new List <IFreeDocument> {
doc
}),
d => d.LastOrDefault());
processManager.CurrentProcessTasks.Add(task);
});
}
else
{
var r = func(new List <IFreeDocument> {
doc
}).ToList();
}
yield return(document);
}
}
public virtual bool Init(IEnumerable <IFreeDocument> datas)
{
if (string.IsNullOrEmpty(ETLSelector))
{
return(false);
}
etl =
processManager.CurrentProcessCollections.FirstOrDefault(d => d.Name == ETLSelector) as SmartETLTool;
if (etl != null)
{
return(true);
}
var task = processManager.CurrentProject.Tasks.FirstOrDefault(d => d.Name == ETLSelector);
if (task == null)
{
throw new NullReferenceException($"can't find a ETL Module named {ETLSelector}");
}
ControlExtended.UIInvoke(() => { task.Load(false); });
etl.InitProcess(true);
return(etl != null);
}
public async Task <IEnumerable <ProjectItem> > GetProjects(string url = null)
{
try
{
if (url == null)
{
url = this.MarketUrl;
}
string username = null;
string project = null;
string target = null;
if (!this.GetRepoInfo(url, out username, out project, out target))
{
XLogSys.Print.Error(GlobalHelper.Get("market_url_check"));
return(null);
}
IReadOnlyList <RepositoryContent> result = null;
result = await client.Repository.Content.GetAllContents(username, project, target);
var items = result?.Where(d => d.Type == ContentType.File && (d.Name.EndsWith(".xml", true, null) || d.Name.EndsWith(".hproj", true, null))).Select(
d =>
{
var projectItem = new ProjectItem()
{
IsRemote = true,
SavePath = d.DownloadUrl
};
var suffix = d.Name.Split('.').Last();
var name = d.Name.Replace("." + suffix, "");
projectItem.Name = name;
var meta = name + ".meta";
var metafile = result.FirstOrDefault(d2 => d2.Name == meta);
if (metafile != null)
{
Task.Factory.StartNew(() =>
{
var response = WebRequest.Create(metafile.DownloadUrl).GetResponse().GetResponseStream();
using (StreamReader reader = new StreamReader(response, Encoding.UTF8))
{
var item = reader.ReadToEnd();
var metainfo = ParameterItem.GetParameters(item);
ControlExtended.UIInvoke(() =>
{
projectItem.DictDeserialize(metainfo.ToDictionary(d2 => d2.Key, d2 => (object)d2.Value));
});
}
});
}
return(projectItem);
});
return(items);
}
catch (Exception ex)
{
XLogSys.Print.Error(ex.Message);
return(new List <ProjectItem>());
}
}
private bool Refresh()
{
//motherkeys = this.Mother.Generate(Mother.CurrentETLTools.Take(Mother.CurrentETLTools.IndexOf(this.ETLModule)), false).Take(3).GetKeys().ToList();
// subkeys =
// SubTask.Generate(
// SubTask.CurrentETLTools.Take(RangeStart),false)
// .Take(3).GetKeys().ToList();
var dict =
MappingPairs.Where(
d =>
string.IsNullOrEmpty(d.Target.SelectItem) == false &&
string.IsNullOrEmpty(d.Source.SelectItem) == false).Distinct().GroupBy(d => d.Target.SelectItem).Select(group => group.First())
.ToDictionary(d => d.Target.SelectItem, d => d.Source.SelectItem);
ControlExtended.UIInvoke(() =>
{
MappingPairs.Clear();
foreach (var key in subkeys)
{
var pair = new MappingPair(motherkeys, subkeys);
pair.Target.SelectItem = key;
string value = key;
dict.TryGetValue(key, out value);
pair.Source.SelectItem = value;
MappingPairs.Add(pair);
}
});
return(true);
//var index = Mother.CurrentETLTools.IndexOf(ETLModule);
//if (index == -1)
// return;
//Mother.Generate(Mother.CurrentETLTools.Take(index), false);
}
private void FiddlerApplicationAfterSessionComplete(Session oSession)
{
if (oSession.oRequest.headers == null)
{
return;
}
var httpitem = new HttpItem {
Parameters = oSession.oRequest.headers.ToString()
};
if ((oSession.BitFlags & SessionFlags.IsHTTPS) != 0)
{
httpitem.URL = "https://" + oSession.url;
}
else
{
httpitem.URL = "http://" + oSession.url;
}
httpitem.Postdata = Encoding.Default.GetString(oSession.RequestBody);
if (string.IsNullOrWhiteSpace(SelectText) == false)
{
var content = oSession.GetResponseBodyAsString();
if (content.Contains(SelectText) == false)
{
return;
}
}
IsSuperMode = true;
StopVisit();
httpitem.DictCopyTo(Http);
var post = "";
if (Http.Method == MethodType.POST)
{
post = "post请求的内容为:\n" + httpitem.Postdata + "\n";
}
var window = MainFrm as Window;
ControlExtended.UIInvoke(() => { if (window != null)
{
window.Topmost = true;
}
});
var info = $"已经成功获取嗅探字段！ 真实请求地址:\n{oSession.url}，\n已自动配置了网页采集器，请求类型为{Http.Method}\n {post}已经刷新了网页采集器的内容";
XLogSys.Print.Info(info);
ControlExtended.UIInvoke(() => { if (window != null)
{
window.Topmost = false;
}
});
URL = oSession.url;
}
public override bool Init(IEnumerable <IFreeDocument> datas)
{
OneOutput = false;
crawler =
processManager.CurrentProcessCollections.FirstOrDefault(d => d.Name == CrawlerSelector) as SmartCrawler;
if (crawler != null)
{
IsMultiYield = crawler?.IsMultiData == ListType.List;
}
else
{
var task = processManager.CurrentProject.Tasks.FirstOrDefault(d => d.Name == CrawlerSelector);
if (task == null)
{
return(false);
}
ControlExtended.UIInvoke(() => { task.Load(false); });
crawler =
processManager.CurrentProcessCollections.FirstOrDefault(d => d.Name == CrawlerSelector) as
SmartCrawler;
}
return(crawler != null && base.Init(datas));
}
public void AddErrorLog(IFreeDocument item, Exception ex, IColumnProcess process)
{
if (string.IsNullOrEmpty(errorLogName))
{
return;
}
var param = item.Clone() as FreeDocument;
param["__SysObjectID"] = process.ObjectID;
param["__SysETL"] = (process as ToolBase)?.Father.Name;
param["__SysERROR"] = ex.Message;
param["__SysTime"] = DateTime.Now.ToString();
ControlExtended.UIInvoke(() =>
{
if (ConfigFile.GetConfig <DataMiningConfig>().IsAddErrorCollection)
{
if (errorCollection == null)
{
errorCollection = new DataCollection()
{
Name = errorLogName
};
DataManager.AddDataCollection(errorCollection);
}
errorCollection?.ComputeData.Add(param);
errorCollection?.OnPropertyChanged("Count");
}
else
{
XLogSys.Print.Error(string.Format(GlobalHelper.Get("key_208"), process.Column, process.TypeName, ex));
}
});
}
public static T GetModule <T>(this IColumnProcess process, string name) where T : class
{
var moduleName = (typeof(T) == typeof(SmartETLTool)) ? "数据清洗" : "网页采集器";
if (string.IsNullOrEmpty(name))
{
XLogSys.Print.Error($"您没有填写“{process.TypeName}”的对应参数。");
return(default(T));
}
var processManager = MainDescription.MainFrm.PluginDictionary[_static_name] as IProcessManager;
var module =
processManager.CurrentProcessCollections.FirstOrDefault(d => d.Name == name) as T;
if (module != null)
{
return(module);
}
var task = processManager.CurrentProject.Tasks.FirstOrDefault(d => d.Name == name);
if (task == null)
{
XLogSys.Print.Error($"没有找到名称为'{name}'的{moduleName}，请检查“{process.TypeName}”是否填写错误");
throw new NullReferenceException($"can't find a ETL Module named {name}");
}
ControlExtended.UIInvoke(() => { task.Load(false); });
module =
processManager.CurrentProcessCollections.FirstOrDefault(d => d.Name == name) as T;
return(module);
}
protected SmartCrawler GetCrawler(string name)
{
var crawler =
processManager.CurrentProcessCollections.FirstOrDefault(d => d.Name == name) as SmartCrawler;
if (crawler != null)
{
IsMultiYield = crawler?.IsMultiData == ListType.List;
}
else
{
var task = processManager.CurrentProject.Tasks.FirstOrDefault(d => d.Name == name);
if (task != null)
{
ControlExtended.UIInvoke(() => { task.Load(false); });
crawler =
processManager.CurrentProcessCollections.FirstOrDefault(d => d.Name == name) as
SmartCrawler;
}
if (crawler == null)
{
if (string.IsNullOrEmpty(name))
{
XLogSys.Print.Error($"您没有填写“从爬虫转换”的“爬虫选择”。需要填写要调用的网页采集器的名称");
}
else
{
XLogSys.Print.Error($"没有找到名称为'{name}'的网页采集器，请检查“从爬虫转换”的“爬虫选择”是否填写错误");
}
}
}
return(crawler);
}
protected SmartCrawler GetCrawler(string name)
{
var crawler =
processManager.CurrentProcessCollections.FirstOrDefault(d => d.Name == name) as SmartCrawler;
if (crawler != null)
{
IsMultiYield = crawler?.IsMultiData == ListType.List;
}
else
{
var task = processManager.CurrentProject.Tasks.FirstOrDefault(d => d.Name == name);
if (task != null)
{
ControlExtended.UIInvoke(() => { task.Load(false); });
crawler =
processManager.CurrentProcessCollections.FirstOrDefault(d => d.Name == name) as
SmartCrawler;
}
if (crawler == null)
{
XLogSys.Print.Error($"没有找到名称为'{name}'的网页采集器，是否没有填写或填写错误?");
}
}
return(crawler);
}
public virtual void Remove()
{
ControlExtended.UIInvoke(() => ProcessManager.CurrentProcessTasks.Remove(this));
CancellationToken?.Cancel();
autoReset.Close();
IsStart = false;
IsCanceled = true;
}
public virtual void OnPropertyChanged(string propName)
{
//this.VerifyPropertyName(propName);
if (null != PropertyChanged)
{
ControlExtended.UIInvoke(() => PropertyChanged(this, new PropertyChangedEventArgs(propName)));
}
}
public override void Start()
{
base.Start();
if (TaskAction != null)
{
IsStart = true;
CurrentTask = new Task(() =>
{
try
{ //Capture the thread
var thread = Thread.CurrentThread;
using (CancellationToken.Token.Register(() =>
{
Task.Factory.StartNew(() =>
{
for (int i = 0; i < 10; i++)
{
Thread.Sleep(100);
if (WasCanceled == true)
{
break;
}
}
if (WasCanceled == false)
{
thread.Abort();
}
});
}))
{
TaskAction();
}
}
catch (ThreadAbortException)
{
XLogSys.Print.Warn("任务已经强行被终止");
IsStart = false;
ControlExtended.UIInvoke(() => this.ContinueAction?.Invoke(this.CurrentIndex));
WasAborted = true;
}
catch (Exception ex)
{
XLogSys.Print.Error("任务已经出错：" + ex);
IsStart = false;
}
IsStart = false;
if (AutoDelete)
{
Remove();
}
}, CancellationToken.Token);
CurrentTask.Start();
}
}
public virtual void ShowDialogAdvance()
{
this.Progress = 0;
var allmill = DisplayTime.TotalMilliseconds;
MaxProgress = (int)(allmill / 500.0);
this.DataContext = this;
if (DisplayTime != TimeSpan.Zero)
{
var task = new Task(
d =>
{
try
{
for (int i = 0; i < allmill / 500; i++)
{
Thread.Sleep(500);
ControlExtended.UIInvoke(() => { this.Progress++; });
if (token.IsCancellationRequested == true)
{
return;
}
}
if (token.IsCancellationRequested == true)
{
return;
}
ControlExtended.UIInvoke(
() =>
{
try
{
this.DialogResult = DefaultResult;
}
catch (Exception ex)
{
return;
}
this.Close();
});
}
catch (Exception ex)
{
}
},
this.token)
;
task.Start();
}
base.ShowDialog();
}
public static async Task <T> RunBusyWork <T>(this IMainFrm manager, Func <T> func, string title = "系统正忙", string message = "正在处理长时间操作")
{
var dock = manager as IDockableManager;
ControlExtended.UIInvoke(() => dock?.SetBusy(ProgressBarState.Indeterminate, title, message));
var item = await Task.Run(func);
ControlExtended.UIInvoke(() => dock?.SetBusy(ProgressBarState.NoProgress));
return(item);
}
public static async Task <T> RunBusyWork <T>(this IMainFrm manager, Func <T> func, string title = "系统正忙",
string message = "Processing long running task")
{
var dock = manager as IDockableManager;
ControlExtended.UIInvoke(() => dock.SetBusy(true, title, message));
T item = await Task.Run(func);
ControlExtended.UIInvoke(() => dock.SetBusy(false));
return(item);
}
private void FiddlerApplicationAfterSessionComplete(Session oSession)
{
if (string.IsNullOrEmpty(URLFilter) == false)
{
URLFilter = URLFilter.Replace("http://", "");
if (URLFilter.Split(' ').Any(item => oSession.url.Contains(item) == false))
{
return;
}
}
var httpitem = new HttpItem {
Parameters = oSession.oRequest.headers.ToString()
};
if ((oSession.BitFlags & SessionFlags.IsHTTPS) != 0)
{
httpitem.URL = "https://" + oSession.url;
}
else
{
httpitem.URL = "http://" + oSession.url;
}
httpitem.Postdata = Encoding.Default.GetString(oSession.RequestBody);
if (string.IsNullOrEmpty(httpitem.Postdata) == false)
{
httpitem.Method = MethodType.POST;
ControlExtended.UIInvoke(() => Documents.Add(httpitem));
}
if (string.IsNullOrEmpty(ContentFilter) == false)
{
if (ContentFilter.Split(' ').Any(item => oSession.GetResponseBodyAsString().Contains(item) == false))
{
return;
}
}
httpitem.DictCopyTo(Http);
XLogSys.Print.Info("已经成功获取嗅探字段" + oSession.url);
}
public override IEnumerable <IFreeDocument> Execute(IEnumerable <IFreeDocument> documents)
{
foreach (var computeable in documents)
{
if (collection != null)
{
ControlExtended.UIInvoke(() =>
{
var data = computeable.Clone();
collection.ComputeData.Add(data);
collection.OnPropertyChanged("Count");
});
}
yield return(computeable);
}
}
public override IEnumerable <IFreeDocument> Execute(IEnumerable <IFreeDocument> documents)
{
foreach (var document in documents)
{
var name = AppHelper.Query(Table, document);
Monitor.Enter(this);
collection = dataManager.DataCollections.FirstOrDefault(d => d.Name == name);
if (collection == null)
{
if (string.IsNullOrEmpty(name) == false)
{
collection = new DataCollection(new List <IFreeDocument>())
{
Name = name
};
dataManager.AddDataCollection(collection);
}
}
Monitor.Exit(this);
if (collection == null)
{
XLogSys.Print.Error(GlobalHelper.Get("create_collection_error"));
yield return(document);
continue;
}
ControlExtended.UIInvoke(() =>
{
var data = document.Clone();
Monitor.Enter(collection);
collection.ComputeData.Add(data);
collection.OnPropertyChanged("Count");
Monitor.Exit(collection);
});
yield return(document);
}
}
public IDataProcess GetOneInstance(string name, bool isAddToList = true, bool newOne = false,
bool isAddUI = false)
{
if (newOne)
{
var process = PluginProvider.GetObjectByType <IDataProcess>(name);
if (process != null)
{
if (isAddToList)
{
;
process.SysDataManager = dataManager;
process.SysProcessManager = this;
var rc4 = process as AbstractProcessMethod;
if (rc4 != null)
{
rc4.MainPluginLocation = MainFrmUI.MainPluginLocation;
rc4.MainFrm = MainFrmUI;
}
var names =
CurrentProcessCollections.Select(d => d.Name);
var count = names.Count(d => d.Contains(process.TypeName));
if (count > 0)
{
process.Name = process.TypeName + (count + 1);
}
CurrentProcessCollections.Add(process);
XLogSys.Print.Info(GlobalHelper.Get("key_319") + process.TypeName + GlobalHelper.Get("key_320"));
}
if (isAddUI)
{
ControlExtended.UIInvoke(() => LoadProcessView(process));
}
return(process);
}
}
return(ProcessCollection.Get(name, isAddToList));
}
public override bool Init(IEnumerable <IFreeDocument> datas)
{
if (generator == null)
{
var mainstream =
processManager.CurrentProcessCollections.OfType <SmartETLTool>()
.FirstOrDefault(d => d.CurrentETLTools.Contains(this));
generator = mainstream.CurrentETLTools.FirstOrDefault(d => d.Name == GEName) as BfsGE;
}
crawler =
processManager.CurrentProcessCollections.FirstOrDefault(d => d.Name == CrawlerSelector) as SmartCrawler;
if (crawler != null)
{
IsMultiYield = crawler?.IsMultiData == ListType.List;
}
else
{
var task = processManager.CurrentProject.Tasks.FirstOrDefault(d => d.Name == CrawlerSelector);
if (task == null)
{
return(false);
}
ControlExtended.UIInvoke(() => { task.Load(false); });
crawler =
processManager.CurrentProcessCollections.FirstOrDefault(d => d.Name == CrawlerSelector) as
SmartCrawler;
}
IsMultiYield = crawler?.IsMultiData == ListType.List;
isfirst = true;
OneOutput = false;
if (IsRegex)
{
regex = new Regex(Prefix);
}
return(crawler != null && base.Init(datas));
}
public override bool Init(IEnumerable <IFreeDocument> datas)
{
crawler =
processManager.CurrentProcessCollections.FirstOrDefault(d => d.Name == CrawlerSelector) as SmartCrawler;
if (crawler != null)
{
}
else
{
var task = processManager.CurrentProject.Tasks.FirstOrDefault(d => d.Name == CrawlerSelector);
if (task == null)
{
return(false);
}
ControlExtended.UIInvoke(() => { task.Load(false); });
crawler =
processManager.CurrentProcessCollections.FirstOrDefault(d => d.Name == CrawlerSelector) as
SmartCrawler;
}
helper = new HttpHelper();
return(base.Init(datas));
}
public virtual bool Init(IEnumerable <IFreeDocument> datas)
{
if (string.IsNullOrEmpty(ETLSelector))
{
return(false);
}
mainstream =
processManager.CurrentProcessCollections.OfType <SmartETLTool>()
.FirstOrDefault(d => d.CurrentETLTools.Contains(this));
etl =
processManager.CurrentProcessCollections.FirstOrDefault(d => d.Name == ETLSelector) as SmartETLTool;
if (mainstream != null && mainstream.Name == this.Name)
{
throw new Exception("子流程不能调用自身，否则会引起循环调用");
}
if (etl != null)
{
return(true);
}
var task = processManager.CurrentProject.Tasks.FirstOrDefault(d => d.Name == ETLSelector);
if (task == null)
{
throw new NullReferenceException($"can't find a ETL Module named {ETLSelector}");
}
ControlExtended.UIInvoke(() => { task.Load(false); });
etl =
processManager.CurrentProcessCollections.FirstOrDefault(d => d.Name == ETLSelector) as SmartETLTool;
etl.InitProcess(true);
return(etl != null);
}
public IEnumerable <IFreeDocument> Execute(IEnumerable <IFreeDocument> documents)
{
foreach (var document in documents)
{
IFreeDocument doc = null;
if (string.IsNullOrEmpty(NewColumn))
{
doc = document.Clone();
}
else
{
doc = new FreeDocument();
doc.MergeQuery(document, NewColumn + " " + Column);
}
if (AddTask)
{
var name = doc[Column];
ControlExtended.UIInvoke(() =>
{
var task = TemporaryTask.AddTempTask("ETL" + name, func(new List <IFreeDocument> {
doc
}),
d => d.ToList());
processManager.CurrentProcessTasks.Add(task);
});
}
else
{
var r = func(new List <IFreeDocument> {
doc
}).ToList();
}
yield return(document);
}
}
public void RefreshSamples(bool canGetDatas = true)
{
if (SysProcessManager == null)
{
return;
}
if (!mudoleHasInit)
{
return;
}
if (SysProcessManager.CurrentProcessTasks.Any(d => d.Publisher == this))
{
XLogSys.Print.WarnFormat("{0}已经有任务在执行，请在执行完毕后再刷新，或取消该任务", Name);
return;
}
if (dataView == null && MainDescription.IsUIForm && IsUISupport)
{
var dock = MainFrm as IDockableManager ?? ControlExtended.DockableManager;
var control = dock?.ViewDictionary.FirstOrDefault(d => d.Model == this);
if (control != null)
{
if (control.View is IRemoteInvoke)
{
var invoke = control.View as IRemoteInvoke;
invoke.RemoteFunc = DropAction;
}
dynamic dy = control.View;
dataView = dy.DataList;
scrollViewer = dy.ScrollViewer;
alltoolList = dy.ETLToolList;
alltoolList.MouseMove += (s, e) =>
{
if (e.LeftButton == MouseButtonState.Pressed)
{
var attr = alltoolList.SelectedItem as XFrmWorkAttribute;
if (attr == null)
{
return;
}
var data = new DataObject(typeof(XFrmWorkAttribute), attr);
try
{
DragDrop.DoDragDrop(control.View as UserControl, data, DragDropEffects.Move);
}
catch (Exception ex)
{
}
}
};
}
}
Documents.Clear();
var alltools = CurrentETLTools.Take(ETLMount).ToList();
var hasInit = false;
var func = Aggregate(d => d, alltools, false);
if (!canGetDatas)
{
return;
}
var temptask = TemporaryTask.AddTempTask(Name + "_转换",
func(new List <IFreeDocument>()).Take(SampleMount),
data =>
{
ControlExtended.UIInvoke(() =>
{
Documents.Add((data));
if (hasInit == false && Documents.Count > 2)
{
InitUI();
hasInit = true;
}
});
}, d =>
{
if (!hasInit)
{
InitUI();
hasInit = true;
}
}
, SampleMount);
temptask.Publisher = this;
SysProcessManager.CurrentProcessTasks.Add(temptask);
}
public void ExecuteDatas()
{
var etls = CurrentETLTools.Take(ETLMount).Where(d => d.Enabled).ToList();
EnumerableFunc func = d => d;
var index = 0;
if (GenerateMode == GenerateMode.串行模式)
{
var generator = etls.FirstOrDefault() as IColumnGenerator;
if (generator == null)
{
return;
}
var realfunc3 = Aggregate(func, etls.Skip(1), true);
var task = TemporaryTask.AddTempTask(Name + "串行任务", generator.Generate(),
d => { realfunc3(new List <IFreeDocument> {
d
}).ToList(); }, null, generator.GenerateCount() ?? (-1));
SysProcessManager.CurrentProcessTasks.Add(task);
}
else
{
var timer = new DispatcherTimer();
TemporaryTask paratask = null;
var tolistTransformer = etls.FirstOrDefault(d => d.TypeName == "启动并行") as ToListTF;
if (tolistTransformer != null)
{
index = etls.IndexOf(tolistTransformer);
var beforefunc = Aggregate(func, etls.Take(index), true);
List <IFreeDocument> taskbuff = new List <IFreeDocument>();
paratask = TemporaryTask.AddTempTask("清洗任务并行化", beforefunc(new List <IFreeDocument>())
,
d2 =>
{ //TODO:这种分组方式可能会丢数据！！
if (taskbuff.Count < tolistTransformer.GroupMount)
{
taskbuff.Add(d2);
return;
}
var newtaskbuff = taskbuff.ToList();
taskbuff.Clear();
if (paratask.IsPause == false &&
SysProcessManager.CurrentProcessTasks.Count > MaxThreadCount)
{
iswait = true;
paratask.IsPause = true;
}
var countstr = d2.Query(tolistTransformer.MountColumn);
var name = d2.Query(tolistTransformer.IDColumn);
if (name == null)
{
name = "清洗任务";
}
var rcount = -1;
int.TryParse(countstr, out rcount);
var afterfunc = Aggregate(func, etls.Skip(index + 1), true);
var task = TemporaryTask.AddTempTask(name, afterfunc(newtaskbuff), d => { },
null, rcount, false);
if (tolistTransformer.DisplayProgress)
{
ControlExtended.UIInvoke(() => SysProcessManager.CurrentProcessTasks.Add(task));
}
task.Start();
}, d => timer.Stop(), -1, false);
}
else
{
var generator = etls.FirstOrDefault() as IColumnGenerator;
if (generator == null)
{
return;
}
var realfunc3 = Aggregate(func, etls.Skip(1), true);
paratask = TemporaryTask.AddTempTask("并行清洗任务", generator.Generate(),
d =>
{
if (paratask.IsPause == false &&
SysProcessManager.CurrentProcessTasks.Count > MaxThreadCount)
{
iswait = true;
paratask.IsPause = true;
}
var task = TemporaryTask.AddTempTask("子任务", realfunc3(new List <IFreeDocument> {
d
}),
d2 => { },
null, 1, false);
ControlExtended.UIInvoke(() => SysProcessManager.CurrentProcessTasks.Add(task));
task.Start();
}, d => timer.Stop(), generator.GenerateCount() ?? (-1), false);
}
SysProcessManager.CurrentProcessTasks.Add(paratask);
timer.Interval = TimeSpan.FromSeconds(3);
timer.Tick += (s, e) =>
{
if (paratask.IsCanceled)
{
timer.Stop();
return;
}
if (paratask.IsStart == false)
{
paratask.Start();
return;
}
if (iswait && SysProcessManager.CurrentProcessTasks.Count < MaxThreadCount)
{
paratask.IsPause = false;
iswait = false;
}
};
timer.Start();
}
}
public override bool Init()
{
if (MainDescription.IsUIForm)
{
DataCollections = new SafeObservable <DataCollection>();
dockableManager = MainFrmUI as IDockableManager;
var view = PluginProvider.GetObjectInstance <ICustomView>("系统状态视图");
var userControl = view as UserControl;
if (userControl != null)
{
userControl.DataContext = MainFrmUI;
dockableManager.AddDockAbleContent(FrmState.Mini, view, "系统状态视图");
}
}
else
{
DataCollections = new ObservableCollection <DataCollection>();
}
processManager = MainFrmUI.PluginDictionary["模块管理"] as IProcessManager;
commands = new List <ICommand>();
var dbaction = new BindingAction();
dbaction.ChildActions.Add(new Command("配置连接",
obj =>
{
Window w = PropertyGridFactory.GetPropertyWindow(obj);
w.ShowDialog();
},
obj => obj != null));
dbaction.ChildActions.Add(
new Command("刷新", obj => RefreshConnect(obj as IDataBaseConnector), obj => obj != null));
dbaction.ChildActions.Add(
new Command("执行查询", obj =>
{
var query = new QueryEntity();
query.Connector = obj as IDataBaseConnector;
query.GetQueryFunc = d =>
{
if (d == null)
{
return;
}
if (d.Any() == false)
{
return;
}
AddDataCollection(d, GetNewName());
};
propertyGridWindow.SetObjectView(query);
}, obj => obj != null));
dbaction.ChildActions.Add(
new Command("删除连接", obj =>
{
if (MessageBox.Show("确定要删除该连接吗？", "提示信息", MessageBoxButton.YesNoCancel) == MessageBoxResult.Yes)
{
var con = obj as DBConnectorBase;
_dbConnections.Remove(con);
}
}, obj => obj != null));
var dataaction = new BindingAction();
var tableAction = new BindingAction();
tableAction.ChildActions.Add(new Command(
"查看",
async obj =>
{
var items = obj as TableInfo;
List <IFreeDocument> dataAll = null;
try
{
dataAll = await
GetDataFromDB(items.Connector, items.Name, true,
items.Connector is FileManager ? -1 : 200);
}
catch (Exception ex)
{
MessageBox.Show("文件打开失败" + ex.Message);
return;
}
if (dataAll == null || dataAll.Count == 0)
{
XLogSys.Print.Warn("没有在表中的发现可用的数据");
return;
}
if (items.Connector is FileManager)
{
string file = (items.Connector as FileManager).LastFileName;
var name = Path.GetFileNameWithoutExtension(file);
AddDataCollection(dataAll, name);
return;
}
var excel = PluginProvider.GetObjectInstance <IDataViewer>("可编辑列表");
if (excel == null)
{
return;
}
object view = excel.SetCurrentView(dataAll.Select(d => d as IFreeDocument).ToList());
if (ControlExtended.DockableManager != null)
{
ControlExtended.DockableManager.AddDockAbleContent(
FrmState.Custom, view, items.Name);
}
},
obj => obj != null));
tableAction.ChildActions.Add(new Command(
"添加到数据集",
async obj =>
{
var items = obj as TableInfo;
List <IFreeDocument> datas = await GetDataFromDB(items.Connector, items.Name, true);
if (datas == null)
{
return;
}
AddDataCollection(datas, items.Name);
},
obj => obj != null));
tableAction.ChildActions.Add(new Command(
"添加虚拟数据集",
obj =>
{
var con = obj as TableInfo;
ReadCollection(con.Connector, con.Name, true);
},
obj => obj != null));
tableAction.ChildActions.Add(new Command(
"删除表",
obj =>
{
var items = obj as TableInfo;
DropTable(items.Connector, items.Name);
},
obj => obj != null));
tableAction.ChildActions.Add(new Command(
"查看属性",
obj =>
{
Window w = PropertyGridFactory.GetPropertyWindow(obj);
w.ShowDialog();
},
obj => obj != null));
tableAction.ChildActions.Add(new Command(
"执行查询",
obj =>
{
var query = new QueryEntity();
query.TableInfo = obj as TableInfo;
query.Connector = query.TableInfo.Connector;
query.GetQueryFunc = d =>
{
if (d == null)
{
return;
}
if (d.Any() == false)
{
return;
}
AddDataCollection(d, GetNewName());
};
propertyGridWindow.SetObjectView(query);
}, obj => obj != null));
var visitData = new BindingAction("浏览方式");
IEnumerable <Command> visitCommands = PluginProvider.GetPluginCollection(typeof(IDataViewer)).Select(
d =>
{
var comm = new Command(d.Name);
comm.Execute = d2 =>
{
var data = d2 as DataCollection;
if (data.Count == 0)
{
MessageBox.Show("不存在任何数据", "提示信息");
return;
}
ControlExtended.UIInvoke(() =>
{
var view = PluginProvider.GetObjectInstance <IDataViewer>(d.Name);
object r = view.SetCurrentView(data.ComputeData);
if (ControlExtended.DockableManager != null)
{
ControlExtended.DockableManager.AddDockAbleContent(
FrmState.Float, r, data.Name + " " + d.Name);
}
});
};
return(comm);
});
visitData.Execute = obj => visitCommands.FirstOrDefault(d => d.Text == "可编辑列表").Execute(obj);
foreach (Command visitCommand in visitCommands)
{
visitData.ChildActions.Add(visitCommand);
}
dataaction.ChildActions.Add(new Command(
"数据清洗", obj =>
{
DataCollection collection = GetCollection(obj);
if (collection == null)
{
return;
}
var plugin = processManager.GetOneInstance("SmartETLTool", true, true) as SmartETLTool;
dynamic generator = PluginProvider.GetObjectByType <IColumnProcess>("TableGE");
generator.TableSelector.SelectItem = collection;
plugin.CurrentETLTools.Add(generator);
plugin.RefreshSamples(true);
ControlExtended.DockableManager.ActiveModelContent(plugin);
}, obj => true, "new"));
dataaction.ChildActions.Add(new Command(
"拷贝", obj =>
{
DataCollection collection = GetCollection(obj);
if (collection == null)
{
return;
}
DataCollection n = collection.Clone(true);
n.Name = GetNewName(collection.Name);
DataCollections.Add(n);
}, obj => true, "page_new"));
var saveData = new Command("另存为", d =>
{
DataCollection collection = GetCollection(d);
if (collection == null)
{
return;
}
var ofd = new SaveFileDialog {
Filter = FileConnector.GetDataFilter(), DefaultExt = "*"
};
ofd.FileName = collection.Name + ".xlsx";
if (ofd.ShowDialog() == true)
{
string filename = ofd.FileName;
SaveFile(collection.Name, filename);
}
}, obj => true, "save");
dataaction.ChildActions.Add(saveData);
dataaction.ChildActions.Add(visitData);
dataaction.ChildActions.Add(new Command(
"新建",
obj =>
DataCollections.Add(new DataCollection(new List <IFreeDocument>())
{
Name = GetNewName("新建数据集")
}), obj => true, "box"));
dataaction.ChildActions.Add(new Command(
"配置", obj =>
{
DataCollection collection = GetCollection(obj);
if (collection != null)
{
propertyGridWindow.SetObjectView(collection);
}
}, obj => true, "settings"));
dataaction.ChildActions.Add(new Command(
"删除", obj =>
{
DataCollection collection = GetCollection(obj);
if (collection != null)
{
DataCollections.Remove(collection);
}
}, obj => true, "delete"));
var convert = new BindingAction("转换表类型");
dataaction.ChildActions.Add(convert);
convert.ChildActions.Add(new Command("转为非虚拟数据集", obj =>
{
DataCollection coll = GetCollection(obj);
if (coll.Count > 500000)
{
if (
MessageBox.Show("本集合数据量较大，转换可能会占用较高的内存和导致程序崩溃，确定继续吗?", "提示信息", MessageBoxButton.YesNoCancel) !=
MessageBoxResult.Yes)
{
return;
}
}
var docuts = new List <IFreeDocument>();
var task = TemporaryTask.AddTempTask("转为非虚拟数据集", coll.ComputeData, d =>
{
if (d != null)
{
docuts.Add(d as IFreeDocument);
}
}, result =>
{
var collection = new DataCollection(docuts)
{
Name = coll.Name + '1'
};
AddDataCollection(collection);
this.DataCollections.Remove(coll);
});
processManager.CurrentProcessTasks.Add(task);
}));
var insertdb = new BindingAction("保存到数据库");
insertdb.SetChildActionSource(() =>
{
return(_dbConnections.Select(dataBaseConnector => new Command(dataBaseConnector.Name, obj =>
{
var data = obj as DataCollection;
processManager.CurrentProcessTasks.Add(TemporaryTask.AddTempTask(data.Name + "插入到数据库", dataBaseConnector.InserDataCollection(data), result => dataBaseConnector.RefreshTableNames(), count: data.Count / 1000));
})).Cast <ICommand>().ToList());
});
dataaction.ChildActions.Add(insertdb);
var otherDataAction = new BindingAction();
otherDataAction.ChildActions.Add(new Command("清空数据", obj => CleanData(), obj => DataCollections.Count > 0));
commands.Add(dbaction);
commands.Add(tableAction);
commands.Add(dataaction);
commands.Add(otherDataAction);
var dblistAction = new BindingAction("数据库管理");
var addnew = new BindingAction("增加新连接");
dblistAction.ChildActions.Add(addnew);
foreach (XFrmWorkAttribute item in PluginProvider.GetPluginCollection(typeof(IDataBaseConnector)))
{
addnew.ChildActions.Add(new Command(item.Name)
{
Execute = obj =>
{
var con = PluginProvider.GetObjectInstance(item.MyType) as DBConnectorBase;
con.Name = item.Name;
_dbConnections.Add(con);
}
});
}
commands.Add(dblistAction);
dockableManager = MainFrmUI as IDockableManager;
if (processManager?.CurrentProject != null)
{
LoadDataConnections();
}
processManager.OnCurrentProjectChanged += (s, e) => LoadDataConnections();
ConfigUI = PropertyGridFactory.GetInstance(_dbConnections.FirstOrDefault());
propertyGridWindow = MainFrmUI.PluginDictionary["属性配置器"] as XFrmWorkPropertyGrid;
var changed = DataCollections as INotifyCollectionChanged;
changed.CollectionChanged += (s, e) => OnDataSourceChanged(new EventArgs());
return(true);
}
private void FiddlerApplicationAfterSessionComplete(Session oSession)
{
if (oSession.oRequest.headers == null)
{
return;
}
var httpitem = new HttpItem {
Parameters = oSession.oRequest.headers.ToString()
};
XLogSys.Print.Debug("visiting... " + oSession.url);
if ((oSession.BitFlags & SessionFlags.IsHTTPS) != 0)
{
httpitem.URL = "https://" + oSession.url;
}
else
{
httpitem.URL = "http://" + oSession.url;
}
if (oSession.RequestMethod.ToLower() == "post")
{
httpitem.Method = MethodType.POST;
}
httpitem.Postdata = Encoding.Default.GetString(oSession.RequestBody);
if (string.IsNullOrWhiteSpace(SelectText) == false)
{
var content = oSession.GetResponseBodyAsString();
content = JavaScriptAnalyzer.Decode(content);
if (content.Contains(SelectText) == false)
{
return;
}
}
if (string.IsNullOrWhiteSpace(SelectText) == true)
{
return;
}
if (ConfigFile.Config.Get <bool>("AutoStartStopFiddler"))
{
StopVisit();
}
httpitem.DictCopyTo(Http);
var post = "";
if (Http.Method == MethodType.POST)
{
post = "POST content is:\n" + httpitem.Postdata + "\n";
}
var window = MainFrm as Window;
ControlExtended.UIInvoke(() => { if (window != null)
{
window.Topmost = true;
}
});
var info = GlobalHelper.FormatArgs("success_get", oSession.url, Http.Method, post);
XLogSys.Print.Info(info);
//IsSuperMode = false;
ControlExtended.UIInvoke(() => { if (window != null)
{
window.Topmost = false;
}
});
SniffSucceed?.Invoke(this, new EventArgs());
URL = oSession.url;
}
