public Task Invoke(IDictionary<string, object> environment)
{
if (!environment.Get<string>(OwinConstants.RequestMethodKey).EqualsIgnoreCase("GET"))
{
return _inner(environment);
}
var originalOutput = environment.Get<Stream>(OwinConstants.ResponseBodyKey);
var recordedStream = new MemoryStream();
environment.Set(OwinConstants.ResponseBodyKey, recordedStream);
return _inner(environment).ContinueWith(t => {
recordedStream.Position = 0;
environment[OwinConstants.ResponseBodyKey] = originalOutput;
var response = new OwinHttpResponse(environment);
if (IsGetHtmlRequest(environment) && response.StatusCode < 500)
{
injectContent(environment, recordedStream, response);
}
else
{
response.StreamContents(recordedStream);
}
});
}
public virtual void DictDeserialize(IDictionary <string, object> docu, Scenario scenario = Scenario.Database)
{
ProcessToDo.DictDeserialize(docu, scenario);
Name = docu.Set("Name", Name);
Description = docu.Set("Description", Description);
ScriptPath = docu.Set("ScriptPath", ScriptPath);
}
public void DictDeserialize(IDictionary <string, object> docu, Scenario scenario = Scenario.Database)
{
Name = docu.Set("Name", Name);
XPath = docu.Set("XPath", XPath);
IsHTML = docu.Set("IsHtml", IsHTML);
IsEnabled = docu.Set("IsEnabled", IsEnabled);
}
public virtual void DictDeserialize(IDictionary <string, object> docu, Scenario scenario = Scenario.Database)
{
Name = docu.Set("Name", Name);
Description = docu.Set("Description", Description);
Version = docu.Set("Version", Version);
SavePath = docu.Set("SavePath", SavePath);
}
public override void DictDeserialize(IDictionary <string, object> dicts, Scenario scenario = Scenario.Database)
{
shouldUpdate = false;
base.DictDeserialize(dicts, scenario);
MaxThreadCount = dicts.Set("MaxThreadCount", MaxThreadCount);
GenerateMode = dicts.Set("GenerateMode", GenerateMode);
SampleMount = dicts.Set("SampleMount", SampleMount);
var doc = dicts as FreeDocument;
if (doc != null && doc.Children != null)
{
foreach (var child in doc.Children)
{
var name = child["Type"].ToString();
var process = PluginProvider.GetObjectByType <IColumnProcess>(name);
if (process != null)
{
process.DictDeserialize(child);
CurrentETLTools.Add(process);
process.Father = this;
var tool = process as ToolBase;
if (tool != null)
{
tool.ColumnSelector.GetItems = () => all_columns;
}
}
}
}
ETLMount = CurrentETLTools.Count;
shouldUpdate = true;
}
public override void DictDeserialize(IDictionary <string, object> dicts, Scenario scenario = Scenario.Database)
{
base.DictDeserialize(dicts, scenario);
URL = dicts.Set("URL", URL);
RootXPath = dicts.Set("RootXPath", RootXPath);
ShareCookie = dicts.Set("ShareCookie", ShareCookie);
IsMultiData = dicts.Set("IsMultiData", IsMultiData);
IsSuperMode = dicts.Set("IsSuperMode", IsSuperMode);
if (dicts.ContainsKey("HttpSet"))
{
var doc2 = dicts["HttpSet"];
var p = doc2 as IDictionary <string, object>;
Http.UnsafeDictDeserialize(p);
}
if (dicts.ContainsKey("Generator"))
{
var doc2 = dicts["Generator"];
var p = doc2 as IDictionary <string, object>;
}
var doc = dicts as FreeDocument;
if (doc?.Children != null)
{
foreach (var child in doc.Children)
{
var item = new CrawlItem();
item.DictDeserialize(child);
CrawlItems.Add(item);
}
}
}
public void DictDeserialize(IDictionary <string, object> docu, Scenario scenario = Scenario.Database)
{
Name = docu.Set("Name", Name);
XPath = docu.Set("XPath", XPath);
CrawlType = docu.Set("CrawlType", CrawlType);
IsEnabled = docu.Set("IsEnabled", IsEnabled);
Format = docu.Set("Format", Format);
}
public virtual void DictDeserialize(IDictionary<string, object> docu, Scenario scenario = Scenario.Database)
{
ProcessToDo.DictDeserialize(docu, scenario);
Name = docu.Set("Name", Name);
Description = docu.Set("Description", Description);
CreateTime = docu.Set("CreateTime", CreateTime);
ScriptPath = docu.Set("ScriptPath", ScriptPath);
}
public virtual void DictDeserialize(IDictionary <string, object> docu, Scenario scenario = Scenario.Database)
{
DBName = docu.Set("DBName", DBName);
Name = docu.Set("Name", Name);
AutoConnect = docu.Set("AutoConnect", AutoConnect);
ConnectionString = docu.Set("ConnectString", ConnectionString);
ConnectStringToOtherInfos();
}
public void DictDeserialize(IDictionary <string, object> docu, Scenario scenario = Scenario.Database)
{
Login = docu.Set("Login", Login);
IsKeepPassword = docu.Set("IsKeepPassword", IsKeepPassword);
if (IsKeepPassword)
{
Password = docu.Set("Password", Password);
}
MarketUrl = docu.Set("MarketUrl", MarketUrl);
}
public static void ParseValues(IDictionary <string, string> result, string header)
{
while (header.Length > 0)
{
var eq = header.IndexOf('=');
if (eq < 0)
{
eq = header.Length;
}
var name = header.Substring(0, eq).Trim().Trim(new[] { ';', ',' }).Trim();
var value = header = header.Substring(Math.Min(header.Length, eq + 1)).Trim();
if (value.StartsWith("\""))
{
ProcessValue(1, ref header, ref value, '"');
}
else if (value.StartsWith("'"))
{
ProcessValue(1, ref header, ref value, '\'');
}
else
{
ProcessValue(0, ref header, ref value, ' ', ',', ';');
}
result.Set(name, value);
}
}
public Task Invoke(IDictionary <string, object> environment)
{
if (!environment.Get <string>(OwinConstants.RequestMethodKey).EqualsIgnoreCase("GET"))
{
return(_inner(environment));
}
var originalOutput = environment.Get <Stream>(OwinConstants.ResponseBodyKey);
var recordedStream = new MemoryStream();
environment.Set(OwinConstants.ResponseBodyKey, recordedStream);
return(_inner(environment).ContinueWith(t => {
recordedStream.Position = 0;
environment[OwinConstants.ResponseBodyKey] = originalOutput;
var response = new OwinHttpResponse(environment);
if (IsGetHtmlRequest(environment) && response.StatusCode < 500)
{
injectContent(environment, recordedStream, response);
}
else
{
response.StreamContents(recordedStream);
}
}));
}
public static void Set <TKey, TValue>(this IDictionary <TKey, HashSet <TValue> > dic, TKey key, TValue value)
{
var set = dic.GetOrNew(key);
set.Set(value);
dic.Set(key, set);
}
/// <summary>
/// 合并字典。
/// </summary>
/// <param name="instance">源字典。</param>
/// <param name="toAdd">要合并的字典。</param>
/// <param name="replaceExisting">如果为 <c>true</c>，将替换已存在的项，如果为 false, 跳过已存在的项.</param>
public static void AddRange <TKey, TValue>(this IDictionary <TKey, TValue> instance, IEnumerable <KeyValuePair <TKey, TValue> > toAdd, bool replaceExisting = true)
{
Guard.ArgumentNotNull(toAdd, nameof(toAdd));
foreach (KeyValuePair <TKey, TValue> pair in toAdd)
{
instance.Set(pair.Key, pair.Value);
}
}
public void DictDeserialize(IDictionary <string, object> docu, Scenario scenario = Scenario.Database)
{
Name = docu.Set("Name", Name);
Size = docu.Set("Size", Size);
Description = docu.Set("Description", Description);
var doc = docu as FreeDocument;
if (doc != null && doc.Children != null)
{
foreach (FreeDocument item in doc.Children)
{
var Column = new ColumnInfo();
Column.DictDeserialize(item);
ColumnInfos.Add(Column);
}
}
}
public override IDictionary <string, object> Execute(IDocumentSession session, ISerializer serializer, IDictionary <string, object> row)
{
var prevDocument = row.Get <byte[]>("Document_pre_v2");
if (prevDocument != null)
{
row.Set(DocumentTable.DocumentColumn, Encoding.UTF8.GetString(prevDocument));
}
var prevMetadata = row.Get <byte[]>("Metadata_pre_v2");
if (prevMetadata != null)
{
row.Set(DocumentTable.MetadataColumn, Encoding.UTF8.GetString(prevMetadata));
}
return(row);
}
public virtual void DictDeserialize(IDictionary <string, object> docu, Scenario scenario = Scenario.Database)
{
Name = docu.Set("Name", Name);
Description = docu.Set("Description", Description);
ShortDescription = docu.Set("ShortDescription", ShortDescription);
Version = docu.Set("Version", Version);
SavePath = docu.Set("SavePath", SavePath);
Author = docu.Set("Author", Author);
ImgURL = docu.Set("ImgURL", ImgURL);
PublishTime = docu.Set("PublishTime", PublishTime);
}
private static void SetQueryString(IDictionary <string, object> environment, string queryString)
{
if (!string.IsNullOrWhiteSpace(queryString) && queryString[0] == '?')
{
// We don't want to store the query string with a leading '?' in OWIN.
queryString = queryString.Substring(1);
}
environment.Set(Constants.RequestQueryString, queryString);
}
public static TBuilder Bind <TBuilder>(this TBuilder instance,
string bindingName,
string value) where TBuilder : IHtmlAttributes
{
IDictionary <string, string> bindings = instance.HtmlAttributes.Bindings();
bindings.Set(bindingName, value);
return(instance.Bind(bindings));
}
public static void Merge <TKey, TValue>(this IDictionary <TKey, TValue> src, IEnumerable <KeyValuePair <TKey, TValue> > dest)
{
// ReSharper disable PossibleMultipleEnumeration
foreach (var value in dest)
{
src.Set(value.Key, value.Value);
}
// ReSharper restore PossibleMultipleEnumeration
}
/// <summary>
/// Gets data from a dictionary or sets it using the supplied factory; either way, returns the value.
/// </summary>
/// <typeparam name="T1"></typeparam>
/// <typeparam name="T2"></typeparam>
/// <param name="data"></param>
/// <param name="key"></param>
/// <param name="factory"></param>
/// <returns></returns>
public static T2 Get <T1, T2>(this IDictionary <T1, T2> data, T1 key, Func <T2> factory)
{
if (data.ContainsKey(key))
{
return(data[key]);
}
else
{
return(data.Set(key, factory()));
}
}
public object Evaluate(IDictionary<string, object> context)
{
if (this.Expressions.Count == 0) return null;
if (this.Expressions[0] is Variable)
{
var functionName = (this.Expressions[0] as Variable).Name;
var func = context.Get(functionName) as IFunction;
var rslt = func.Execute(context, this.Expressions.Skip(1).ToArray());
context.Set("@", rslt);
return rslt;
/*Func<object> action = () => func.Execute(context, this.Expressions.Skip(1).ToArray());
var task = new Task<object>(action);
var handle = new ManualResetEvent(false);
task.ContinueWith(x =>
{
Console.WriteLine("setting wait handle");
handle.Set();
});
EventLoop.Instance.Enqueue(task);
Console.WriteLine("Awaiting wait handle");
handle.WaitOne();
Console.WriteLine("resuming wait handle");
context.Set("@", task.Result);
return task.Result;*/
}
// this is a number of lines of code
object result = null;
foreach (Statement statement in this.Expressions)
{
result = statement.Evaluate(context);
}
context.Set("@", result);
return result;
}
public void DictDeserialize(IDictionary <string, object> docu, Scenario scenario = Scenario.Database)
{
Name = docu.Set("Name", Name);
var doc = docu as FreeDocument;
var res = doc?.Children?.Select(d => d as IFreeDocument).ToList();
if (res != null)
{
RealData = res;
}
}
/// <summary>
/// 合并字典。
/// </summary>
/// <param name="instance">源字典。</param>
/// <param name="toAdd">要合并的字典。</param>
/// <param name="replaceExisting">如果为 <c>true</c>，将替换已存在的项，如果为 false, 跳过已存在的项.</param>
public static void AddRange <TKey, TValue>(this IDictionary <TKey, TValue> instance, IEnumerable <KeyValuePair <TKey, TValue> > toAdd, bool replaceExisting = true)
{
if (toAdd == null)
{
throw new ArgumentNullException(nameof(toAdd));
}
foreach (var pair in toAdd)
{
instance.Set(pair.Key, pair.Value);
}
}
public void DictDeserialize(IDictionary <string, object> docu, Scenario scenario = Scenario.Database)
{
Importance = docu.Set("Importance", Importance);
Name = docu.Set("Name", Name);
DataType = docu.Set("DataType", DataType);
IsKey = docu.Set("IsKey", IsKey);
IsVirtual = docu.Set("IsVirtual", IsVirtual);
IsVirtual = docu.Set("CanNull", CanNull);
Description = docu.Set("Desc", Description);
}
public void DictDeserialize(IDictionary <string, object> docu, Scenario scenario = Scenario.Database)
{
URL = docu.Set("URL", URL);
Allowautoredirect = docu.Set("Allowautoredirect", Allowautoredirect);
Postdata = docu.Set("Postdata", Postdata);
Encoding = docu.Set("Encoding", Encoding);
Method = docu.Set("Method", Method);
Timeout = docu.Set("Timeout", Timeout);
Parameters = docu.Set("Parameters", Parameters);
}
public override void DictDeserialize(IDictionary <string, object> dicts, Scenario scenario = Scenario.Database)
{
base.DictDeserialize(dicts, scenario);
MaxThreadCount = dicts.Set("MaxThreadCount", MaxThreadCount);
GenerateMode = dicts.Set("GenerateMode", GenerateMode);
var doc = dicts as FreeDocument;
if (doc != null && doc.Children != null)
{
foreach (var child in doc.Children)
{
var name = child["Type"].ToString();
var process = PluginProvider.GetObjectByType <IColumnProcess>(name);
if (process != null)
{
process.DictDeserialize(child);
CurrentETLTools.Add(process);
}
}
}
}
public static TValue GetOrAdd <TValue>(this IDictionary <string, object> dictionary, string key, Func <string, TValue> valueFactory)
{
Contract.Requires(dictionary != null);
lock (dictionary)
{
if (!dictionary.ContainsKey(key))
{
dictionary.Set(key, valueFactory.Invoke(key));
}
return((TValue)dictionary[key]);
}
}
public override IDictionary <string, object> Execute(IDocumentSession session, ISerializer serializer, IDictionary <string, object> row)
{
var prevDocument = row.Get <byte[]>("Document_pre_v2");
if (prevDocument != null)
{
using (var inStream = new MemoryStream(prevDocument))
using (var bsonReader = new BsonReader(inStream))
{
var jObj = new JsonSerializer().Deserialize(bsonReader, typeof(JObject));
row.Set(DocumentTable.DocumentColumn, JsonConvert.SerializeObject(jObj));
}
}
var prevMetadata = row.Get <byte[]>("Metadata_pre_v2");
if (prevMetadata != null)
{
row.Set(DocumentTable.MetadataColumn, Encoding.UTF8.GetString(prevMetadata));
}
return(row);
}
public static TValue GetOrAdd <TValue>(this IDictionary <string, object> dictionary,
string key,
TValue defaultValue)
{
Contract.Assume(dictionary != null, "dictionary is null.");
Contract.Assume(!String.IsNullOrEmpty(key), "key is null or empty.");
lock (dictionary)
{
if (!dictionary.ContainsKey(key))
{
dictionary.Set(key, defaultValue);
}
return((TValue)dictionary[key]);
}
}
public static IDictionary <string, object> AttachPrefiex(this IDictionary <string, object> values, string prefix)
{
if (prefix.IsNullOrWhiteSpace())
{
return(values);
}
var newValues = values.Select(kp => new KeyValuePair <string, object>($"{prefix}.{kp.Key}", kp.Value));
values.Clear();
foreach (var value in newValues)
{
values.Set(value.Key, value.Value);
}
return(values);
}
public static IDictionary <String, Object> AttachPrefiex(this IDictionary <String, Object> values, string prefix)
{
if (prefix.IsNullOrWhiteSpace())
{
return(values);
}
var newValues = values.Select(kp => new KeyValuePair <String, object>(String.Format("{0}.{1}", prefix, kp.Key), kp.Value)).ToArray();
values.Clear();
foreach (var value in newValues)
{
values.Set(value.Key, value.Value);
}
return(values);
}
public static void ParseValues(IDictionary<string, string> result, string header) {
while (header.Length > 0) {
var eq = header.IndexOf('=');
if (eq < 0) eq = header.Length;
var name = header.Substring(0, eq).Trim().Trim(new[] { ';', ',' }).Trim();
var value = header = header.Substring(Math.Min(header.Length, eq + 1)).Trim();
if (value.StartsWith("\"")) {
ProcessValue(1, ref header, ref value, '"');
} else if (value.StartsWith("'")) {
ProcessValue(1, ref header, ref value, '\'');
} else {
ProcessValue(0, ref header, ref value, ' ', ',', ';');
}
result.Set(name, value);
}
}
/// <summary>
/// Get or create value if does not exit
/// </summary>
/// <typeparam name="TValue">value's type</typeparam>
/// <typeparam name="TCreate">create type</typeparam>
/// <param name="self">dictionary</param>
/// <param name="create">delegate to create value</param>
/// <param name="name">name (optional, use's type name as key if not specified)</param>
/// <returns></returns>
public static TValue GetOrCreate <TValue, TCreate>(this IDictionary <object, object> self, Func <TCreate> create, string name = null)
where TValue : class
where TCreate : class, TValue
{
Verify.IsNotNull(nameof(self), self);
Verify.IsNotNull(nameof(create), create);
object value;
name = name ?? typeof(TValue).Name;
if (!self.TryGetValue(name, out value))
{
var result = create();
self.Set(result);
return(result);
}
return((TValue)value);
}
public override void DictDeserialize(IDictionary<string, object> dicts, Scenario scenario = Scenario.Database)
{
base.DictDeserialize(dicts, scenario);
MaxThreadCount = dicts.Set("MaxThreadCount", MaxThreadCount);
GenerateMode = dicts.Set("GenerateMode", GenerateMode);
SampleMount = dicts.Set("SampleMount", SampleMount);
var doc = dicts as FreeDocument;
if (doc != null && doc.Children != null)
{
foreach (var child in doc.Children)
{
var name = child["Type"].ToString();
var process = PluginProvider.GetObjectByType<IColumnProcess>(name);
if (process != null)
{
process.DictDeserialize(child);
CurrentETLTools.Add(process);
}
}
}
}
public void DictDeserialize(IDictionary<string, object> docu, Scenario scenario = Scenario.Database)
{
URL = docu.Set("URL", URL);
Allowautoredirect = docu.Set("Allowautoredirect", Allowautoredirect);
Postdata = docu.Set("Postdata", Postdata);
Encoding = docu.Set("Encoding", Encoding);
Method = docu.Set("Method", Method);
Parameters = docu.Set("Parameters", Parameters);
}
public virtual void DictDeserialize(IDictionary<string, object> dicts, Scenario scenario = Scenario.Database)
{
Name = dicts.Set("Name", Name);
}
public object Execute(IDictionary<string, object> context, params object[] parameters)
{
var newContext = new Dictionary<string, object>();
foreach (var item in context)
{
newContext.Add(item.Key, item.Value);
}
for (var i = 0; i < this.Args.Length; i++)
{
newContext.Add(this.Args[i], parameters[i]);
}
if (this.Expressions.Length > 1)
{
var output = (new Statement { Expressions = this.Expressions.ToList() }).Evaluate(newContext);
context.Set("@", output);
return output;
}
var output2 = this.Expressions[0].Evaluate(newContext);
context.Set("@", output2);
return output2;
}
public virtual void DictDeserialize(IDictionary<string, object> docu, Scenario scenario = Scenario.Database)
{
EncodingType = docu.Set("Encoding", EncodingType);
}
public override void DictDeserialize(IDictionary<string, object> docu, Scenario scenario = Scenario.Database)
{
base.DictDeserialize(docu, scenario);
ContainHeader = docu.Set("ContainHeader", ContainHeader);
SplitString = docu.Set("SplitString", SplitString);
}
private static void LoadDecimalValidationAttributes(IDictionary<string, object> attributes)
{
Throw.IfNullArgument(attributes, "attributes");
var numberFormat = System.Threading.Thread.CurrentThread.CurrentCulture.NumberFormat;
attributes.Set(DataCarcassValidation, "true");
attributes.Set(DataNumberDecimalSeparator, numberFormat.CurrencyDecimalSeparator);
attributes.Set(DataNumberGroupSeparator, numberFormat.CurrencyGroupSeparator);
}
public override void DictDeserialize(IDictionary<string, object> dicts, Scenario scenario = Scenario.Database)
{
base.DictDeserialize(dicts, scenario);
URL = dicts.Set("URL", URL);
RootXPath = dicts.Set("RootXPath", RootXPath);
IsMultiData = dicts.Set("IsMultiData", IsMultiData);
URLFilter = dicts.Set("URLFilter", URLFilter);
Crawler = dicts.Set("Crawler", Crawler);
ContentFilter = dicts.Set("ContentFilter", ContentFilter);
if (dicts.ContainsKey("HttpSet"))
{
var doc2 = dicts["HttpSet"];
var p = doc2 as IDictionary<string, object>;
Http.UnsafeDictDeserialize(p);
}
if (dicts.ContainsKey("Login"))
{
var doc2 = dicts["Login"];
var p = doc2 as IDictionary<string, object>;
var item = new HttpItem();
item.DictDeserialize(p);
Documents.Add(item);
}
if (dicts.ContainsKey("Generator"))
{
var doc2 = dicts["Generator"];
var p = doc2 as IDictionary<string, object>;
}
var doc = dicts as FreeDocument;
if (doc?.Children != null)
{
foreach (var child in doc.Children)
{
var item = new CrawlItem();
item.DictDeserialize(child);
CrawlItems.Add(item);
}
}
}
public override void DictDeserialize(IDictionary<string, object> docu, Scenario scenario = Scenario.Database)
{
base.DictDeserialize(docu, scenario);
Source.SelectItem = docu.Set("Source", Source.SelectItem);
Target.SelectItem = docu.Set("Target", Target.SelectItem);
}
public object Execute(IDictionary<string, object> context, params object[] parameters)
{
var value = parameters[1].Evaluate(context);
context.Set((parameters[0] as Variable).Name, value);
return value;
}
public void DictDeserialize(IDictionary<string, object> docu, Scenario scenario = Scenario.Database)
{
Name = docu.Set("Name", Name);
XPath = docu.Set("XPath", XPath);
IsHTML = docu.Set("IsHtml", IsHTML);
}
