public static StringDict LoadDictionary(this string dictPath, bool ignoreComment = false)
{
var dict = new StringDict();
LoadDictionary(dict, dictPath, ignoreComment);
return(dict);
}
public SecurityCache(Func <SecurityCache, IdentityReference, SecurityAccount> factory = null)
{
this.factory = (factory != null) ? factory : SecurityAccount.FactoryImpl;
contextCache = new StringDict <PrincipalContext>();
accountCache = new StringDict <SecurityAccount>();
_lock = new object();
}
public TikaAsyncWorker(TikaDS parent, IStreamProvider elt)
{
action = LoadUrl;
Parent = parent;
dbgStoreDir = parent.DbgStoreDir;
Attribs = new StringDict();
var coll = elt.ContextNode.Attributes;
for (int i = 0; i < coll.Count; i++)
{
var att = coll[i];
if (att.LocalName.Equals("url", StringComparison.InvariantCultureIgnoreCase))
{
continue;
}
if (att.LocalName.Equals("baseurl", StringComparison.InvariantCultureIgnoreCase))
{
continue;
}
Attribs[att.LocalName] = att.Value;
}
StreamElt = elt;
FileInfo info = new FileInfo(StreamElt.FullName);
LastModifiedUtc = info.LastWriteTimeUtc;
FileSize = info.Length;
}
static void splitAndAdd(StringDict <_TagType> dict, String tags, _TagType type)
{
int prev = -1;
for (int i = 0; i < tags.Length; i++)
{
switch (tags[i])
{
case ' ':
case ',':
case ';':
if (prev < 0)
{
continue;
}
dict[tags.Substring(prev, i - prev)] = type;
prev = -1;
continue;
default:
if (prev < 0)
{
prev = i;
}
continue;
}
}
if (prev >= 0)
{
dict[tags.Substring(prev)] = type;
}
}
public object Retrieve(string serviceName, XmlRpcStruct args)
{
Console.WriteLine("Service \"{0}\"", serviceName);
ProvidersManager providers = new ProvidersManager();
Extractable service = null;
try {
service = providers.FindService(serviceName);
}
catch (InvalidNameException) {
string msg = "The name of the service should be in the format NameSpace.ServiceName";
throw new XmlRpcFaultException(32602, msg);
}
catch (ServiceNotFoundException) {
string msg = String.Format("The service \"{0}\" does not exist on this server.", serviceName);
throw new XmlRpcFaultException(32602, msg);
}
Console.WriteLine("Arguments (as strings):");
StringDict arguments = new StringDict();
foreach (DictionaryEntry keyval in args) {
arguments.Add((string)keyval.Key, keyval.Value.ToString());
Console.WriteLine("{0}=>{1}", keyval.Key, keyval.Value);
}
service.Extract(arguments);
Console.WriteLine("Result: {0}", service.ToString());
return service;
}
private StringDict getAttributes(XmlNode node)
{
StringDict ret = new StringDict();
if (node == null)
{
return(ret);
}
var coll = node.Attributes;
for (int i = 0; i < coll.Count; i++)
{
var att = coll[i];
if (att.LocalName.Equals("url", StringComparison.InvariantCultureIgnoreCase))
{
continue;
}
if (att.LocalName.Equals("baseurl", StringComparison.InvariantCultureIgnoreCase))
{
continue;
}
ret[att.LocalName] = att.Value;
}
return(ret);
}
public void ClearCompilerParms()
{
_cp = new CompilerParameters();
_cp.CompilerOptions = "/debug:pdbonly";// TreatWarningsAsErrors="false"
_refs = new StringDict();
AddReference("system.dll");
AddReference("system.xml.dll");
AddReference("system.linq.dll");
}
public NamedAdminCollection(XmlNode collNode, String childrenNode, Func <XmlNode, T> factory, bool mandatory)
: base(collNode, childrenNode, factory, mandatory)
{
namedItems = new StringDict <T>(Count);
for (int i = 0; i < Count; i++)
{
T item = base[i];
namedItems.Add(item.Name, item);
}
}
private void Form1_Load(object sender, EventArgs e)
{
try
{
AppDomain.CurrentDomain.AssemblyResolve += onResolve;
AppDomain.CurrentDomain.AssemblyLoad += onLoad;
trySetIcon();
String dir = Assembly.GetExecutingAssembly().Location;
StringDict dirs = new StringDict();
dir = IOUtils.FindDirectoryToRoot(Path.GetDirectoryName(dir), "ImportDirs");
if (dir != null)
{
dirs.Add(dir, null);
}
StringDict files = new StringDict();
foreach (var f in History.LoadHistory(HISTORY_KEY))
{
files[f] = null;
dir = Path.GetDirectoryName(Path.GetDirectoryName(f));
if (!String.IsNullOrEmpty(dir))
{
dirs[dir] = null;
}
}
foreach (var kvp in dirs)
{
FileTree tree = new FileTree();
tree.AddFileFilter(@"\\import\.xml$", true);
tree.ReadFiles(kvp.Key);
if (tree.Files.Count != 0)
{
tree.Files.Sort();
foreach (var relfile in tree.Files)
{
files[tree.GetFullName(relfile)] = null;
}
}
}
ac = new DirectoryAutocompleter(comboBox1, files.Select(kvp => kvp.Key).ToList());
if (comboBox1.Items.Count > 0)
{
comboBox1.SelectedIndex = 0;
}
}
catch (Exception ex)
{
Logs.ErrorLog.Log(ex);
throw;
}
}
public PromptParmsNeedArgs(int iPromptPageType, int iBizObjID, StringDict dictParms)
{
miBizObjID = iBizObjID;
miPromptPageType = iPromptPageType;
mPageParms = dictParms;
if (mPageParms == null)
{
mPageParms = new StringDict();
}
}
private void clearDict <T>(ref StringDict <T> refDict) where T : class
{
StringDict <T> dict = refDict;
refDict = null;
Exception e = null;
foreach (var kvp in dict)
{
Utils.Free(kvp.Value, ref e);
}
}
public ScriptExpressionHolder(List <String> customUsings = null)
{
className = "_ScriptExpressions";
mem = new MemoryStream();
wtr = mem.CreateTextWriter();
StringDict usings = new StringDict();
usings.Add("System", null);
usings.Add("System.IO", null);
usings.Add("System.Linq", null);
usings.Add("System.Text", null);
usings.Add("System.Xml", null);
usings.Add("System.Collections.Generic", null);
usings.Add("Bitmanager.Core", null);
usings.Add("Bitmanager.IO", null);
usings.Add("Bitmanager.Json", null);
usings.Add("Bitmanager.Elastic", null);
usings.Add("Bitmanager.ImportPipeline", null);
usings.Add("Bitmanager.ImportPipeline.StreamProviders", null);
usings.Add("Newtonsoft.Json.Linq", null);
if (customUsings != null)
{
foreach (var u in customUsings)
{
String uu = u;
if (uu.StartsWith("using "))
{
uu = u.Substring(6);
}
if (uu.EndsWith(";"))
{
uu = uu.Substring(0, uu.Length - 1);
}
uu = uu.Trim();
usings[uu] = null;
}
}
foreach (var kvp in usings)
{
wtr.Write("using ");
wtr.Write(kvp.Key);
wtr.WriteLine(";");
}
wtr.WriteLine();
wtr.WriteLine("namespace Bitmanager.ImportPipeline");
wtr.WriteLine("{");
wtr.WriteLine(" public class _ScriptExpressions");
wtr.WriteLine(" {");
}
public string ToJsonTest()
{
var dict = new StringDict()
{
{ "test", "a" }, { "t", "b" }
};
if (dict.ToJson() != "{\"test\":\"a\",\"t\":\"b\"}")
{
return(Fail(dict.ToJson()));
}
return(Pass());
}
public string StringifyTest1()
{
var dict = new StringDict()
{
{ "test", "a" }
};
var str = dict.Stringify();
if (str != "{test:a}")
{
return("文字列への変換に失敗しています " + str);
}
return("");
}
public static _TagType GetTagType(StringDict <_TagType> tagDict, String tag)
{
if (String.IsNullOrEmpty(tag))
{
return(_TagType.Unknown);
}
_TagType ret;
if (tagDict.TryGetValue(tag.ToLowerInvariant(), out ret))
{
return(ret);
}
return(_TagType.Unknown);
}
public ESIndexDefinitions(XmlHelper xml, XmlNode node, OnLoadConfig onLoadConfig = null)
{
XmlNodeList nodes = node.SelectMandatoryNodes("index");
list = new List <ESIndexDefinition>(nodes.Count);
dict = new StringDict <ESIndexDefinition>(nodes.Count);
foreach (XmlNode x in nodes)
{
ESIndexDefinition def = new ESIndexDefinition(xml, x, onLoadConfig);
list.Add(def);
dict.Add(def.Name, def);
}
}
public static void OutputLocalizationText(StringDict text, bool newLine = true)
{
if (LOCALIZATION [text].ContainsKey(CURRENT_LANG))
{
Console.Write(LOCALIZATION [text] [CURRENT_LANG]);
}
else
{
Console.Write(LOCALIZATION [text] [DEFAULT_LANG]);
}
if (newLine)
{
Console.WriteLine();
}
}
static HtmlProcessor()
{
var dict = new StringDict <_TagType>();
splitAndAdd(dict, "b big i small tt", _TagType.Inline);
splitAndAdd(dict, "abbr acronym cite code dfn em kbd strong samp var", _TagType.Inline);
splitAndAdd(dict, "a, bdo, br, img, map, q, script, span, sub, sup", _TagType.Inline);
splitAndAdd(dict, "button, input, label, select, textarea", _TagType.Inline);
splitAndAdd(dict, "address article aside blockquote canvas dd div dl fieldset figcaption", _TagType.Block);
splitAndAdd(dict, "figure footer form h1 h2 h3 h4 h5 h6 header hgroup hr main nav", _TagType.Block);
splitAndAdd(dict, "ol output p pre section table tfoot ul video", _TagType.Block);
splitAndAdd(dict, "style script noscript object", _TagType.Inline | _TagType.Unwanted);
tagDict = dict;
}
public PostProcessors(ImportEngine engine, XmlNode collNode)
{
postProcessors = new StringDict <IPostProcessor>();
if (collNode == null)
{
return;
}
var nodes = collNode.SelectNodes("postprocessor");
for (int i = 0; i < nodes.Count; i++)
{
XmlNode c = nodes[i];
IPostProcessor p = ImportEngine.CreateObject <IPostProcessor> (c, engine, c);
postProcessors.Add(p.Name, p);
}
}
public static void LoadDictionary(StringDict dict, string dictPath, bool ignoreComment = false)
{
using var textReader = new StreamReader(dictPath, true);
foreach (var line in textReader.Lines())
{
if (ignoreComment == true && line.StartsWith("#"))
{
continue;
}
var tuple = line.Split('=');
if (tuple.Length == 2 && !dict.ContainsKey(tuple[0]))
{
dict.Add(tuple[0], tuple[1]);
}
}
}
private int keyToIndex(string key)
{
int ret;
if (lenient)
{
if (lenientIndexes == null)
{
lenientIndexes = new StringDict <int>(false);
}
if (lenientIndexes.TryGetValue(key, out ret))
{
return(ret);
}
ret = -1;
if (!selectiveExport)
{
foreach (var kvp in lenientIndexes)
{
if (kvp.Value > ret)
{
ret = kvp.Value;
}
}
ret++;
}
lenientIndexes.Add(key, ret);
return(ret);
}
String key2 = key;
switch (key[0])
{
case 'f':
case 'F':
key2 = key.Substring(1); break;
}
if (int.TryParse(key2, NumberStyles.Integer, Invariant.Culture, out ret))
{
return(ret);
}
throw new BMException("Fieldname '{0}' should be a number or an 'F' with a number, to make sure that the field is written on the correct place in the CSV file.", key);
}
public string KeyNotFoundTest()
{
var dict = new StringDict()
{
{ "test", "a" }, { "t", "b" }
};
//var str = dict.Stringify();
if (dict.KeyNotFound("test"))
{
return("発見できるはずの要素が見つかっていません");
}
if (dict.KeyNotFound("a"))
{
return("");
}
return("発見できないはずの要素が見つかっています");
}
private void addConversionError(String field, Exception e)
{
if (conversionErrors == null)
{
conversionErrors = new StringDict <ConversionError>();
}
ConversionError err;
if (conversionErrors.TryGetValue(field, out err))
{
err.Count++;
}
else
{
conversionErrors.Add(field, new ConversionError(field, e));
}
}
/* Initializes the AXEMAS application with a given root page and sidebar. */
public void makeApplicationRootController(StringDict data, StringDict sideBarData = null, Type mainPage = null)
{
var app = getApplication();
if (mainPage == null)
{
mainPage = typeof(Controls.SectionViewPage);
}
if (sideBarData != null)
{
JObject JOsideBarData = (JObject)JToken.FromObject(sideBarData);
getSidebarController().createWebViewSidebar(JOsideBarData);
}
JObject JOdata = (JObject)JToken.FromObject(data);
Controls.SectionViewPage.Navigate(app.RootFrame, mainPage, JOdata);
}
private void diagnose()
{
StringComparer cmp;
bool caseSens = checkBox1.Checked;
cmp = caseSens ? StringComparer.Ordinal : StringComparer.OrdinalIgnoreCase;
listBox1.Items.Clear();
var dict = new StringDict <int>(!caseSens);
int cntPerItem = -1;
for (int i = 0; i < lines.Count;)
{
String line = lines[i];
int existing;
if (dict.TryGetValue(line, out existing))
{
addMsg("Line {0} clashes with existing line {1}.", i, existing);
}
else
{
dict.Add(line, i);
}
int start = i;
for (i++; i < lines.Count; i++)
{
String s = lines[i];
if (!cmp.Equals(s, line))
{
break;
}
}
int cnt = i - start;
if (cntPerItem < 0)
{
cntPerItem = cnt;
}
else if (cnt != cntPerItem)
{
addMsg("Line {0}: unexpected count {1}, existing={2}.", start, cnt, cntPerItem);
}
}
addMsg("Lines={0}, unique={1}, perUnique={2}. ", lines.Count, dict.Count, lines.Count / (double)dict.Count);
}
public void AddMissed(String x, bool touched = false)
{
if (x == null)
{
return;
}
x = x.ToLowerInvariant();
if (dict == null)
{
dict = new StringDict <bool>();
dict.Add(x, touched);
return;
}
if (dict.ContainsKey(x))
{
return;
}
dict.Add(x, touched);
}
static void Main(string[] args)
{
if (args.Length == 0) {
Console.WriteLine("Either pass --server to run as XML-RPC server mode, or pass " +
"\"ServiceName [arg, arg...]\" to run directly.");
return;
}
if (args.Length > 0 && args[0] == "--server") {
int port = 9996;
BlockingXmlRpcServer server = new BlockingXmlRpcServer(port);
server.Start();
} else {
ProvidersManager providers = new ProvidersManager();
Extractable service = null;
try {
service = providers.FindService(args[0]);
}
catch (InvalidNameException) {
Console.WriteLine("The name of the service should be in the format NameSpace.ServiceName");
return;
}
catch (ServiceNotFoundException) {
Console.WriteLine("The service {0} is not currently supported.", args[0]);
return;
}
StringDict arguments = new StringDict();
if (args.Length > 1)
for (int i = 1; i < args.Length; i++) {
string[] name_val = args[i].Split("=".ToCharArray(), 2);
if (name_val.Length > 1)
arguments.Add(name_val[0], name_val[1]);
else
Console.WriteLine("Wrong argument format, ignored: {0}", args[i]);
}
service.Extract(arguments);
Console.WriteLine (service.ToString());
}
}
public void Start(PipelineContext ctx)
{
if (trace)
{
ctx.ImportFlags |= _ImportFlags.TraceValues;
}
missed = new StringDict();
if (ScriptTypeName != null) //NB: always create a new script object. Never reuse an existing one.
{
ScriptObject = Objects.CreateObject(ScriptTypeName, ctx);
logger.Log("Script({0})={1}", ScriptTypeName, ScriptObject);
}
if (ImportEngine.ScriptExpressions.Count > 0) //NB: always create a new script object. Never reuse an existing one.
{
String cls = ImportEngine.ScriptExpressions.FullClassName;
ScriptExprObject = Objects.CreateObject(cls, ctx);
logger.Log("ScriptExpr({0})={1}", cls, ScriptExprObject);
}
//Clone the list of actions and strat them
actions = new List <ActionAdmin>(definedActions.Count);
for (int i = 0; i < definedActions.Count; i++)
{
ActionAdmin act = definedActions[i];
act.Action.Start(ctx);
actions.Add(act);
}
prepareActions();
if (endPointCache != null)
{
foreach (var kvp in this.endPointCache)
{
kvp.Value.Endpoint.Start(ctx);
}
}
started = true;
}
/// <summary>
/// Gets an existing endpoint from the cache or creates and initializes a new one.
/// If a new endpoint is instantiated, its optionally wrapped by a list of postprocessors
/// </summary>
public IDataEndpoint CreateOrGetDataEndpoint(PipelineContext ctx, String name, String postProcessors)
{
String endpointName = getEndpointName(name, ctx.DatasourceAdmin);
EndpointCacheEntry epEntry;
if (endPointCache == null)
{
endPointCache = new StringDict <EndpointCacheEntry>();
}
if (endPointCache.TryGetValue(endpointName, out epEntry))
{
if (postProcessors == null || String.Equals(postProcessors, epEntry.PostProcessors, StringComparison.OrdinalIgnoreCase))
{
return(epEntry.Endpoint);
}
throw new BMException("Endpoint [{0}] is used with different post-processors [{1}] adn [{2}].", epEntry.Name, epEntry.PostProcessors, postProcessors);
}
IDataEndpoint ep = this.ImportEngine.Endpoints.GetDataEndpoint(ctx, endpointName);
if (postProcessors == null)
{
postProcessors = DefaultPostProcessors;
}
if (postProcessors != null)
{
ep = wrapPostProcessors(ctx, ep, postProcessors);
}
epEntry = new EndpointCacheEntry(endpointName, ep, postProcessors);
endPointCache.Add(endpointName, epEntry);
if (started)
{
epEntry.Endpoint.Start(ctx);
}
return(epEntry.Endpoint);
}
public void Start(string separator, StringDict sdict)
{
var szTrCode = _resModel.Name;
fi = new FileInfo(Path.Combine(Settings.Default.root_path, szTrCode + ".csv"));
if (fi.Directory != null && !fi.Directory.Exists)
{
System.IO.Directory.CreateDirectory(fi.DirectoryName);
}
using (var writer = fi.AppendText())
{
StringBuilder sb = new StringBuilder();
_resModel.Blocks["OutBlock"].Rows.ForEach(row =>
{
sb.Append(row.Name).Append(',');
});
sb.Remove(sb.Length - 1, 1);
writer.WriteLine(sb.ToString());
}
var block = _resModel.Blocks["InBlock"];
block.Rows.ForEach(delegate(Row row) { _ixa.SetFieldData(block.Name, row.Name, sdict[row.Name]); });
_ixa.AdviseRealData();
}
private void resolveAssemblies()
{
StringDict pathes = new StringDict();
var domain = AppDomain.CurrentDomain;
pathes[IOUtils.DelSlash(domain.BaseDirectory)] = null;
String relPath = domain.RelativeSearchPath;
if (relPath != null)
{
relPath = Path.Combine(domain.BaseDirectory, relPath);
pathes[IOUtils.DelSlash(relPath)] = null;
}
if (ExtraSearchPath != null)
{
pathes[ExtraSearchPath] = null;
}
var list = _cp.ReferencedAssemblies;
for (int i = 0; i < list.Count; i++)
{
String dir = Path.GetDirectoryName(list[i]);
if (String.IsNullOrEmpty(dir))
{
continue;
}
pathes[dir] = null;
}
foreach (var kvp in pathes)
{
resolveAssemblies(kvp.Key);
}
}
public ReplaceConverter(XmlNode node) : base(node)
{
ReplacerFlags def = ReplacerFlags.NoMatchReturnOriginal;
maxMissed = XmlUtils.ReadInt(node, "@dumpmissed", -1);
if (maxMissed > 0)
{
missed = new StringDict();
def = ReplacerFlags.NoMatchReturnNull;
}
flags = XmlUtils.ReadEnum(node, "@flags", def);
replacers = new List <ReplacerElt>();
XmlNodeList list = node.SelectNodes("replace");
for (int i = 0; i < list.Count; i++)
{
var r = new ReplacerElt(list[i]);
if (r == null)
{
continue;
}
replacers.Add(r);
}
}
public ExtractArguments(string name, string val)
{
args = new StringDict();
args.Add(name, val);
}
public bool Extract(StringDict args)
{
Extractor ex = new Extractor();
return ex.Extract(this, args);
}
public bool Extract(object info, StringDict args)
{
Type t = info.GetType();
SourceURLAttribute[] srcattrs = (SourceURLAttribute[]) t.GetCustomAttributes(typeof(SourceURLAttribute), true);
if (srcattrs.Length == 0) {
Console.WriteLine("Class {0} doesn't specify any source URL. Skipping.", t.Name);
return false;
}
string source_url = srcattrs[0].url;
URLReplacer replacer = new URLReplacer(source_url, args);
source_url = replacer.Replace();
string xml = GetPage(source_url);
IDocument doc = null;
switch (srcattrs[0].format) {
case SourceDocFormat.XML:
doc = new XmlExtractorDocument();
break;
case SourceDocFormat.HTML:
doc = new HtmlExtractorDocument();
break;
default:
Console.WriteLine("Source format {0} is not recognized.", srcattrs[0].format);
return false;
}
if (!doc.Parse(xml)) {
Console.WriteLine("Parsing failed !");
return false;
}
return Extract(info, doc);
}
