/// <summary>
/// Get the solr parameters for clustering
/// </summary>
/// <param name="options"></param>
/// <returns></returns>
public IEnumerable <KeyValuePair <string, string> > GetClusteringParameters(QueryOptions options)
{
if (options.Clustering == null)
{
yield break;
}
var clst = options.Clustering;
yield return(KV.Create("clustering", true.ToString().ToLowerInvariant()));
if (clst.Engine != null)
{
yield return(KV.Create("clustering.engine", clst.Engine));
}
if (clst.Results.HasValue)
{
yield return(KV.Create("clustering.results", clst.Results.ToString().ToLowerInvariant()));
}
if (clst.Collection.HasValue)
{
yield return(KV.Create("clustering.collection", clst.Collection.ToString().ToLowerInvariant()));
}
if (clst.Algorithm != null)
{
yield return(KV.Create("carrot.algorithm", clst.Algorithm));
}
if (clst.Title != null)
{
yield return(KV.Create("carrot.title", clst.Title));
}
if (clst.Snippet != null)
{
yield return(KV.Create("carrot.snippet", clst.Snippet));
}
if (clst.Url != null)
{
yield return(KV.Create("carrot.url", clst.Url));
}
if (clst.ProduceSummary.HasValue)
{
yield return(KV.Create("carrot.produceSummary", clst.ProduceSummary.ToString().ToLowerInvariant()));
}
if (clst.FragSize.HasValue)
{
yield return(KV.Create("carrot.fragSize", clst.FragSize.ToString()));
}
if (clst.NumDescriptions.HasValue)
{
yield return(KV.Create("carrot.numDescriptions", clst.NumDescriptions.ToString()));
}
if (clst.SubClusters.HasValue)
{
yield return(KV.Create("carrot.outputSubClusters", clst.SubClusters.ToString().ToLowerInvariant()));
}
if (clst.LexicalResources != null)
{
yield return(KV.Create("carrot.lexicalResourcesDir", clst.LexicalResources));
}
}
//作废
public List <KV> OldGetMonitList()
{
string sql = "";
KeyValuePair <long, string> _dicMonitId = new KeyValuePair <long, string>(); //初始化
KV kv = new KV();
List <KV> kvLs = new List <KV>(); //获取几条
lock (this)
{
int count = global.GetEffectMonitKVCount(); //获取有效的监控数量
if (count > 0) //&& count >= Main.EachSearchUploadCount/2
{
kvLs = global.OpMonitKVList("take", null, Main.EachUploadCount);
if (count < Main.EachSearchUploadCount / 2)
{
}
}
else
{
if (global.GetMonitKVCount() > 0)
{
global.DeleteMonitKv();
}
#region 获得待监控的列表+当前待上传的monitId
//lock (global.monitKVList)//锁定查询语句
//{
// //当内存中没有数量时，查询待添加的N条记录（来自配置文件的MaxUploadCount）
// var monitKVLists = global.OpMonitKVList("getall");
// if (monitKVLists == null || monitKVLists.Count == 0) //|| monitKVLists.Count<Main.EachSearchUploadCount / 2
// //if (global.monitFileIdList == null || global.monitFileIdList.Count == 0)
// {
var ipNotLists = global.OpIpNotList("getall");
log("输出未在线的ip：" + string.Join(",", ipNotLists.Select(p => p.V)));
#region 获取MaxUploadCount条待拷贝记录(排除未在线终端)
//采集待插入的文件列表
//采集未在线的终端列表
//lcz, 这个地方的sql可以只返回同一客户机ip的，便于下面的一个连接多个文件拷贝
//获取不返回一个ip的文件，在从monitKVList中获取5个一样ip的终端去处理
AddMonitKv();
//}
//_dicMonitId = global.monitFileIdList.Last();//从内存中获取元素
//global.monitFileIdList.Remove(_dicMonitId.Key);//移除元素
//var monitKVLists = global.OpMonitKVList("getall");
//if (monitKVLists != null && monitKVLists.Count > 0)
if (global.GetEffectMonitKVCount() > 0)
{
kvLs = global.OpMonitKVList("take", null, Main.EachUploadCount);
}
//}
#endregion
}
}
return(kvLs);
}
public void TestNuGetKV()
{
// Create debug log
var debugLog = new Log(LogDestination.Debug, Log.DefaultCategoryName);
// Create a KV with null values for its URL and client credentials. This should raise an exception
var kv = new KV(debugLog, null, null, null, StoreLocation.CurrentUser, null);
}
public bool ContainsKey(KEY key)
{
List <KV> bucket; int bestIndexInBucket;
KV kv = Kv(key);
FindEntry(kv, out bucket, out bestIndexInBucket);
return(bestIndexInBucket >= 0);
}
private void AddDependent(KV kv)
{
if (dependents == null)
{
dependents = new List <KV>();
}
dependents.Add(kv);
}
internal static int GetHashCode_KV <TKey, TValue>(KV <TKey, TValue> kv)
{
return(kv.Key.GetHashCode() +
kv.Value.GetHashCode() +
kv.State.GetHashCode() +
kv.HasKey.GetHashCode() +
kv.HasValue.GetHashCode() +
kv.HasState.GetHashCode());
}
public void Index()
{
var m = new MergeCommand("core0", new MergeCommand.IndexDir("/path/to/index1"), new MergeCommand.IndexDir("/path/to/index2"));
var parameters = m.GetParameters().ToList();
CollectionAssert.Contains(parameters, KV.Create("action", "mergeindexes"));
CollectionAssert.Contains(parameters, KV.Create("indexDir", "/path/to/index1"));
CollectionAssert.Contains(parameters, KV.Create("indexDir", "/path/to/index2"));
CollectionAssert.Contains(parameters, KV.Create("core", "core0"));
}
public static ConfigLookupError parseErrorFor <A>(
ConfigPath path, object node, string extraInfo = null
) =>
ConfigLookupError.wrongType(ImmutableArray.Create(
KV.a("path", path.pathStrWithBase),
KV.a("expected-type", typeof(A).FullName),
KV.a("actual-type", node.GetType().FullName),
KV.a("extraInfo", extraInfo ?? ""),
KV.a("node-contents", node.asDebugString())
));
public void Core()
{
var m = new MergeCommand("core0", new MergeCommand.SrcCore("core1"), new MergeCommand.SrcCore("core2"));
var parameters = m.GetParameters().ToList();
CollectionAssert.Contains(parameters, KV.Create("action", "mergeindexes"));
CollectionAssert.Contains(parameters, KV.Create("srcCore", "core1"));
CollectionAssert.Contains(parameters, KV.Create("srcCore", "core2"));
CollectionAssert.Contains(parameters, KV.Create("core", "core0"));
}
/// <summary>
/// Serializes <see cref="SolrFacetPivotQuery"/>
/// </summary>
/// <param name="q"></param>
/// <returns></returns>
public override IEnumerable <KeyValuePair <string, string> > Serialize(SolrFacetPivotQuery q)
{
foreach (var pivotQ in q.Fields)
{
yield return(KV.Create("facet.pivot", string.Join(",", pivotQ.ToArray())));
}
if (q.MinCount.HasValue)
{
yield return(KV.Create("facet.pivot.mincount", q.MinCount.ToString()));
}
}
public void SerializeWithStartUnbound()
{
var q = new SolrFacetIntervalQuery("state");
q.Sets.Add(new FacetIntervalSet(null, new FacetIntervalSetValue("pa", true)));
var r = Serialize(q);
Assert.Contains(KV.Create("facet.interval", "state"), r);
Assert.Contains(KV.Create("f.state.facet.interval.set", "[*,pa]"), r);
}
/// <summary>
/// Gets the Solr parameters for collapse queries
/// </summary>
/// <param name="options"></param>
/// <returns></returns>
public IEnumerable <KeyValuePair <string, string> > GetGroupingQueryOptions(QueryOptions options)
{
if (options.Grouping == null || options.Grouping.Fields.Count == 0)
{
yield break;
}
yield return(KV.Create("group", true.ToString().ToLowerInvariant()));
foreach (var groupfield in options.Grouping.Fields)
{
if (string.IsNullOrEmpty(groupfield))
{
continue;
}
yield return(KV.Create("group.field", groupfield));
}
if (options.Grouping.Limit.HasValue)
{
yield return(KV.Create("group.limit", options.Grouping.Limit.ToString()));
}
if (options.Grouping.Offset.HasValue)
{
yield return(KV.Create("group.offset", options.Grouping.Offset.ToString()));
}
if (options.Grouping.Main.HasValue)
{
yield return(KV.Create("group.main", options.Grouping.Main.ToString().ToLowerInvariant()));
}
if (!string.IsNullOrEmpty(options.Grouping.Query))
{
yield return(KV.Create("group.query", options.Grouping.Query));
}
if (!string.IsNullOrEmpty(options.Grouping.Func))
{
yield return(KV.Create("group.func", options.Grouping.Func));
}
if (options.Grouping.OrderBy != null && options.Grouping.OrderBy.Count > 0)
{
yield return(KV.Create("group.sort", string.Join(",", options.Grouping.OrderBy.Select(x => x.ToString()).ToArray())));
}
if (options.Grouping.Ngroups.HasValue)
{
yield return(KV.Create("group.ngroups", options.Grouping.Ngroups.ToString().ToLowerInvariant()));
}
yield return(KV.Create("group.format", options.Grouping.Format.ToString().ToLowerInvariant()));
}
/// <summary>
/// Gets Solr parameters for facet queries
/// </summary>
/// <param name="fp"></param>
/// <returns></returns>
public IEnumerable <KeyValuePair <string, string> > GetFacetFieldOptions(FacetParameters fp)
{
if (fp == null)
{
yield break;
}
if (fp.Queries == null || fp.Queries.Count == 0)
{
yield break;
}
yield return(KV.Create("facet", "true"));
foreach (var fq in fp.Queries)
{
foreach (var fqv in facetQuerySerializer.Serialize(fq))
{
yield return(fqv);
}
}
if (fp.Prefix != null)
{
yield return(KV.Create("facet.prefix", fp.Prefix));
}
if (fp.EnumCacheMinDf.HasValue)
{
yield return(KV.Create("facet.enum.cache.minDf", fp.EnumCacheMinDf.ToString()));
}
if (fp.Limit.HasValue)
{
yield return(KV.Create("facet.limit", fp.Limit.ToString()));
}
if (fp.MinCount.HasValue)
{
yield return(KV.Create("facet.mincount", fp.MinCount.ToString()));
}
if (fp.Missing.HasValue)
{
yield return(KV.Create("facet.missing", fp.Missing.ToString().ToLowerInvariant()));
}
if (fp.Offset.HasValue)
{
yield return(KV.Create("facet.offset", fp.Offset.ToString()));
}
if (fp.Sort.HasValue)
{
yield return(KV.Create("facet.sort", fp.Sort.ToString().ToLowerInvariant()));
}
if (fp.Threads.HasValue)
{
yield return(KV.Create("facet.threads", fp.Threads.ToString().ToLowerInvariant()));
}
}
public RangeFacetingResult ParseRangeFacetingNode(XElement node)
{
var r = new RangeFacetingResult();
var intParser = new IntFieldParser();
foreach (var rangeFacetingNode in node.Elements())
{
var name = rangeFacetingNode.Attribute("name").Value;
switch (name)
{
case "gap":
r.Gap = rangeFacetingNode.Value;
break;
case "start":
r.Start = rangeFacetingNode.Value;
break;
case "end":
r.End = rangeFacetingNode.Value;
break;
case "counts":
foreach (var item in rangeFacetingNode.Elements())
{
r.RangeResults.Add(KV.Create(item.Attribute("name").Value, (int)intParser.Parse(item, typeof(int))));
}
break;
default:
//collect FacetRangeOther items
if (rangeFacetingNode.Name != "int")
{
break;
}
var count = (int)intParser.Parse(rangeFacetingNode, typeof(int));
if (name == FacetDateOther.After.ToString())
{
r.OtherResults[FacetRangeOther.After] = count;
}
else if (name == FacetDateOther.Before.ToString())
{
r.OtherResults[FacetRangeOther.Before] = count;
}
else if (name == FacetDateOther.Between.ToString())
{
r.OtherResults[FacetRangeOther.Between] = count;
}
break;
}
}
return(r);
}
public void SerializeWithStringSpecialCharacters()
{
var q = new SolrFacetIntervalQuery("state");
q.Sets.Add(new FacetIntervalSet(new FacetIntervalSetValue("a,z", true), new FacetIntervalSetValue("p[]a", true)));
var r = Serialize(q);
Assert.Contains(KV.Create("facet.interval", "state"), r);
Assert.Contains(KV.Create("f.state.facet.interval.set", @"[a\,z,p\[\]a]"), r);
}
public void SerializeWithEndUnbound()
{
var q = new SolrFacetIntervalQuery("state");
q.Sets.Add(new FacetIntervalSet(new FacetIntervalSetValue("az", true), null));
var r = Serialize(q);
Assert.Contains(KV.Create("facet.interval", "state"), r);
Assert.Contains(KV.Create("f.state.facet.interval.set", "[az,*]"), r);
}
public void SerializeWithIntOneItem()
{
var q = new SolrFacetIntervalQuery("version");
q.Sets.Add(new FacetIntervalSet(new FacetIntervalSetValue(1, true), new FacetIntervalSetValue(5, true)));
var r = Serialize(q);
Assert.Contains(KV.Create("facet.interval", "version"), r);
Assert.Contains(KV.Create("f.version.facet.interval.set", "[1,5]"), r);
}
public void SerializeWithStringOneItem()
{
var q = new SolrFacetIntervalQuery("state");
q.Sets.Add(new FacetIntervalSet(new FacetIntervalSetValue("az", true), new FacetIntervalSetValue("pa", true)));
var r = Serialize(q);
Assert.Contains(KV.Create("facet.interval", "state"), r);
Assert.Contains(KV.Create("f.state.facet.interval.set", "[az,pa]"), r);
}
public void GetCollapseExpandParameters()
{
var querySerializer = new DefaultQuerySerializer(new DefaultFieldSerializer());
var e = new SolrQueryExecuter <TestDocument>(null, null, querySerializer, null, null);
var p = e.GetAllParameters(SolrQuery.All, new QueryOptions {
Rows = 1,
CollapseExpand = new CollapseExpandParameters("somefield", null, null, null),
}).ToList();
Assert.Contains(p, KV.Create("fq", "{!collapse field=somefield}"));
}
public void SerializeWithDateOneItem()
{
var q = new SolrFacetIntervalQuery("timestamp");
q.Sets.Add(new FacetIntervalSet(new FacetIntervalSetValue(new DateTime(2009, 1, 1, 0, 0, 0, DateTimeKind.Utc), true), new FacetIntervalSetValue(new DateTime(2010, 1, 1, 0, 0, 0, DateTimeKind.Utc), true)));
var r = Serialize(q);
Assert.Contains(KV.Create("facet.interval", "timestamp"), r);
Assert.Contains(KV.Create("f.timestamp.facet.interval.set", "[2009-01-01T00:00:00Z,2010-01-01T00:00:00Z]"), r);
}
public void SerializeWithExclusive()
{
var q = new SolrFacetIntervalQuery("state");
q.Sets.Add(new FacetIntervalSet(new FacetIntervalSetValue("az", false), new FacetIntervalSetValue("pa", false)));
var r = Serialize(q);
Assert.Contains(KV.Create("facet.interval", "state"), r);
Assert.Contains(KV.Create("f.state.facet.interval.set", "(az,pa)"), r);
}
private ImHashMap <K, V> ReplaceRemovedWithConflicted()
{
if (Conflicts.Length == 1)
{
return(new ImHashMap <K, V>(new Data(Hash, Conflicts[0].Key, Conflicts[0].Value), Left, Right));
}
var shrinkedConflicts = new KV <K, V> [Conflicts.Length - 1];
Array.Copy(Conflicts, 1, shrinkedConflicts, 0, shrinkedConflicts.Length);
return(new ImHashMap <K, V>(new Data(Hash, Conflicts[0].Key, Conflicts[0].Value, shrinkedConflicts), Left, Right));
}
public void SinglePivotTest()
{
var q = new SolrFacetPivotQuery {
Fields = new[] { new PivotFields("manu_exact", "inStock") },
MinCount = 1
};
var r = serializer.Serialize(q);
Assert.Contains(r, KV.Create("facet.pivot", "manu_exact,inStock"));
Assert.Contains(r, KV.Create("facet.pivot.mincount", "1"));
}
private KV[] PrepareKeyValuePairs(int count)
{
KV[] keyValue = new KV[count];
for (int i = 0; i < count; i++)
{
keyValue[i] = new KV {
Key = "Key" + i, Value = ByteString.CopyFromUtf8("Value of Key" + i)
};
}
return(keyValue);
}
public void ExecuteWithAllParameters()
{
var parameters = new ExtractParameters(null, "1", "text.doc");
var conn = new MSolrConnection();
conn.postStream += new Moroco.MFunc <string, string, System.IO.Stream, System.Collections.Generic.IEnumerable <System.Collections.Generic.KeyValuePair <string, string> >, string>((url, type, stream, kvs) => {
Assert.Equal("/update/extract", url);
Assert.Equal("application/word-document", type);
var p = new[] {
KV.Create("literal.id", parameters.Id),
KV.Create("resource.name", parameters.ResourceName),
KV.Create("literal.field1", "value1"),
KV.Create("literal.field2", "value2"),
KV.Create("stream.type", "application/word-document"),
KV.Create("commit", "true"),
KV.Create("uprefix", "pref"),
KV.Create("defaultField", "field1"),
KV.Create("extractOnly", "true"),
KV.Create("extractFormat", "text"),
KV.Create("capture", "html"),
KV.Create("captureAttr", "true"),
KV.Create("xpath", "body"),
KV.Create("lowernames", "true")
};
Assert.Equal(p, kvs); //ignore order should be added
return("");
});
var cmd = new ExtractCommand(new ExtractParameters(null, "1", "text.doc")
{
AutoCommit = true,
Capture = "html",
CaptureAttributes = true,
DefaultField = "field1",
ExtractOnly = true,
ExtractFormat = ExtractFormat.Text,
Fields = new[] {
new ExtractField("field1", "value1"),
new ExtractField("field2", "value2")
},
LowerNames = true,
XPath = "body",
Prefix = "pref",
StreamType = "application/word-document"
});
cmd.Execute(conn);
Assert.Equal(1, conn.postStream.Calls);
}
/// <summary>
/// Initializes static members of the <see cref="TestConstants"/> class.
/// </summary>
static TestConstants()
{
#if OBA_DEV_ALEC
string environmentName = "oba-dev-alec";
ConfigFileName = environmentName + ".config";
#endif
#if OBA_DEV_SHARAD
string environmentName = "oba-dev-sharad";
ConfigFileName = environmentName + ".config";
#endif
#if OBA_PPE
string environmentName = "oba-ppe";
ConfigFileName = environmentName + ".config";
#endif
#if OBA_PROD
string environmentName = "oba-prod";
ConfigFileName = environmentName + ".config";
#endif
// use the fsr to read values that are not secrets
var fsr = new FileSettingsReader(ConfigurationManager.AppSettings["ConfigRelativePath"] + Path.DirectorySeparatorChar + ConfigFileName);
EmbeddedSocialAdminUserHandle = fsr.ReadValue("EmbeddedSocialAdminUserHandle");
EmbeddedSocialAppKey = fsr.ReadValue("EmbeddedSocialAppKey");
EmbeddedSocialUri = new Uri(fsr.ReadValue("EmbeddedSocialUri"));
AADOBAAppId = fsr.ReadValue("AADOBAAppId");
AADOBAHomePage = fsr.ReadValue("AADOBAHomePage");
AADTenantId = fsr.ReadValue("AADTenantId");
KeyVaultUri = fsr.ReadValue("KeyVaultUri");
OBACertThumbprint = fsr.ReadValue("OBACertThumbprint");
OBARegionsListUri = fsr.ReadValue("OBARegionsListUri");
// use the kvr to read secrets
ICertificateHelper cert = new CertificateHelper(OBACertThumbprint, AADOBAAppId, StoreLocation.CurrentUser);
IKeyVaultClient kvClient = new AzureKeyVaultClient(cert);
var log = new Log(LogDestination.Debug, Log.DefaultCategoryName);
var kv = new KV(log, AADOBAAppId, KeyVaultUri, OBACertThumbprint, StoreLocation.CurrentUser, kvClient);
var kvr = new KVSettingsReader(fsr, kv);
AzureStorageConnectionString = kvr.ReadValueAsync("AzureStorageConnectionString").Result;
OBAApiKey = kvr.ReadValueAsync("OBAApiKey").Result;
SendGridEmailAddr = kvr.ReadValueAsync("SendGridEmailAddr").Result;
SendGridKey = kvr.ReadValueAsync("SendGridKey").Result;
ServiceBusConnectionString = kvr.ReadValueAsync("ServiceBusConnectionString").Result;
// Obtain an AAD token using a cert from the local store for the current user
AADSettings aadSettings = new AADSettings(TestConstants.AADTenantId, TestConstants.AADOBAAppId, TestConstants.AADOBAHomePage, TestConstants.OBACertThumbprint);
CertificateHelper certHelper = new CertificateHelper(TestConstants.OBACertThumbprint, TestConstants.AADOBAAppId, StoreLocation.CurrentUser);
EmbeddedSocialAADToken = certHelper.GetAccessToken(aadSettings.Authority, aadSettings.AppUri).Result;
}
KV Get(object value)
{
KV kv = null;
if (Dics.TryGetValue(value, out kv))
{
return(kv);
}
kv = new KV((System.Collections.IDictionary)value);
Dics.Add(value, kv);
return(kv);
}
public void SerializeWithLocalParams()
{
var q = new SolrFacetIntervalQuery("state");
var lp = new LocalParams();
lp.Add("key", "arizonatopen");
q.Sets.Add(new FacetIntervalSet(new FacetIntervalSetValue("az", false), new FacetIntervalSetValue("pa", false), lp));
var r = Serialize(q);
Assert.Contains(KV.Create("facet.interval", "state"), r);
Assert.Contains(KV.Create("f.state.facet.interval.set", "{!key=arizonatopen}(az,pa)"), r);
}
public void GetCollapseExpandParameters_min_policy()
{
var querySerializer = new DefaultQuerySerializer(new DefaultFieldSerializer());
var e = new SolrQueryExecuter <TestDocument>(null, null, querySerializer, null, null);
var max = new CollapseExpandParameters.MinOrMax.Max("maxfield");
var policy = CollapseExpandParameters.NullPolicyType.Collapse;
var p = e.GetAllParameters(SolrQuery.All, new QueryOptions {
Rows = 1,
CollapseExpand = new CollapseExpandParameters("somefield", null, max, policy),
}).ToList();
Assert.Contains(p, KV.Create("fq", "{!collapse field=somefield nullPolicy=collapse max=maxfield}"));
}
public void TermVector()
{
var p = SolrQueryExecuter <TestDocument> .GetTermVectorQueryOptions(new QueryOptions {
TermVector = new TermVectorParameters {
Fields = new[] { "text" },
Options = TermVectorParameterOptions.All,
},
}).ToList();
Assert.Contains(p, KV.Create("tv", "true"));
Assert.Contains(p, KV.Create("tv.all", "true"));
Assert.Contains(p, KV.Create("tv.fl", "text"));
}