public void Add(TItem item)
{
lock (this)
{
_standardCollection.Add(item);
_alternateCollection.Add(item);
}
}
public void Add(string key, object value)
{
keys.Add(key);
vals.Add(value);
}
// 创建指定记录的检索点集合
// parameters:
// domData 记录数据dom 不能为null
// strRecordID 记录id 不能为null或空
// strLang 语言版本
// strStyle 风格，暂没有用上
// nKeySize 检索点尺寸
// keys out参数，返回生成的检索点集合
// strError out参数，出错信息
// return:
// -1 出错
// 0 成功
public int BuildKeys(XmlDocument domData,
string strRecordID,
string strLang,
// string strStyle,
int nKeySize,
out KeyCollection keys,
out string strError)
{
strError = "";
keys = new KeyCollection();
if (this.dom == null)
return 0;
if (domData == null)
{
strError = "BuildKeys()调用错误，domData参数不能为null。";
Debug.Assert(false, strError);
return -1;
}
// Debug.Assert(strRecordID != null && strRecordID != "", "BuildKeys()调用错误，strRecordID参数不能为null或为空。");
if (String.IsNullOrEmpty(strLang) == true)
{
strError = "BuildKeys()调用错误，strLang参数不能为null。";
Debug.Assert(false, strError);
return -1;
}
/*
if (String.IsNullOrEmpty(strStyle) == true)
{
strError = "BuildKeys()调用错误，strStyle参数不能为null。";
Debug.Assert(false, strError);
return -1;
}
* */
if (nKeySize < 0)
{
strError = "BuildKeys()调用错误，nKeySize参数不能小于0。";
Debug.Assert(false, strError);
return -1;
}
int nRet = 0;
// 找到所有<key>节点
// TODO: <key> 是否有明确的位置？ 那样就可以避免 // 查找。或者预先缓存起来
XmlNodeList keyList = dom.SelectNodes("//key");
XPathNavigator nav = domData.CreateNavigator();
CREATE_CACHE:
// 创建Cache
if (m_exprCache.Count == 0 && keyList.Count > 0)
{
for (int i = 0; i < keyList.Count; i++)
{
XmlNode nodeKey = keyList[i];
XmlElement nodeXPath = (XmlElement)nodeKey.SelectSingleNode("xpath");
if (nodeXPath == null)
continue;
string strScriptAttr = nodeXPath.GetAttribute("scripting");
if (String.Compare(strScriptAttr, "on", true) == 0)
continue;
string strXPath = nodeXPath.InnerText.Trim();
if (string.IsNullOrEmpty(strXPath) == true)
continue;
// strNstableName 如果为 null 表示属性不存在
string strNstableName = DomUtil.GetAttrDiff(nodeXPath, "nstable");
XmlNamespaceManager nsmgr = (XmlNamespaceManager)this.tableNsClient[nodeXPath];
#if DEBUG
if (nsmgr != null)
{
Debug.Assert(strNstableName != null, "如果具备名字空间对象，表明<xpath>元素应该有 'nstable' 属性。");
}
else
{
Debug.Assert(strNstableName == null, "如果不具备名字空间对象，表明<xpath>元素必须没有定义 'nstable' 属性。");
}
#endif
XPathExpression expr = nav.Compile(strXPath);
if (nsmgr != null)
expr.SetContext(nsmgr);
m_exprCache[nodeXPath] = expr;
}
}
string strKey = "";
string strKeyNoProcess = "";
string strFromName = "";
string strFromValue = "";
string strSqlTableName = "";
string strNum = "";
for (int i = 0; i < keyList.Count; i++)
{
XmlElement nodeKey = (XmlElement)keyList[i];
strKey = "";
strKeyNoProcess = "";
strFromName = "";
strFromValue = "";
strSqlTableName = "";
strNum = "";
// TODO: 用 GetElementsByTagName 优化
XmlNode nodeFrom = nodeKey.SelectSingleNode("from");
if (nodeFrom != null)
strFromValue = nodeFrom.InnerText.Trim(); // 2012/2/16
// 找不到<key>下级的<table>节点,就应该报错
XmlNode nodeTable = nodeKey.SelectSingleNode("table");
if (nodeTable == null)
{
strError = "<key>下级未定义<table>节点。";
return -1;
}
TableInfo tableInfo = (TableInfo)this.tableTableInfoClient[nodeTable];
Debug.Assert(tableInfo != null, "从Hashtable里取出的tabInfo不可能为null。");
strSqlTableName = tableInfo.SqlTableName.Trim();
// 根据语言版本获得来源名称
strFromName = tableInfo.GetCaption(strLang);
// 所有的检索点字符串
List<string> aKey = new List<string>();
XmlNode nodeXpath = nodeKey.SelectSingleNode("xpath");
string strScriptAttr = "";
if (nodeXpath != null)
strScriptAttr = DomUtil.GetAttr(nodeXpath, "scripting");
if (String.Compare(strScriptAttr, "on", true) == 0)
{
// 执行脚本得到检索点
//aKey.Add("abc");
//string strOutputString = "";
List<String> OutputStrings = null;
string strFunctionName = nodeXpath.InnerText.Trim(); // 2012/2/16
nRet = this.DoScriptFunction(domData,
strFunctionName,
"", //strInputString
// out strOutputString,
out OutputStrings,
out strError);
if (nRet == -1)
return -1;
// 2007/1/23
if (OutputStrings != null)
{
for (int j = 0; j < OutputStrings.Count; j++)
{
if (String.IsNullOrEmpty(OutputStrings[j]) == false)
{
aKey.Add(OutputStrings[j]);
// nCount++;
}
}
}
}
else
{
string strXpath = "";
if (nodeXpath != null)
strXpath = nodeXpath.InnerText.Trim(); // 2012/2/16
string strNstableName = DomUtil.GetAttrDiff(nodeXpath, "nstable");
#if NO
XmlNamespaceManager nsmgr = (XmlNamespaceManager)this.tableNsClient[nodeXpath];
#if DEBUG
if (nsmgr != null)
{
Debug.Assert(strNstableName != null, "此时应该没有定义'nstable'属性。");
}
else
{
Debug.Assert(strNstableName == null, "此时必须没有定义'nstable'属性。");
}
#endif
XPathExpression expr = nav.Compile(strXpath); // TODO 可以优化
if (nsmgr != null)
expr.SetContext(nsmgr);
#endif
// 2012/7/20优化
XPathExpression expr = (XPathExpression)m_exprCache[nodeXpath];
if (expr == null)
{
this.m_exprCache.Clear();
goto CREATE_CACHE; // TODO: 如何预防死循环?
}
string strMyKey = "";
if (expr.ReturnType == XPathResultType.Number)
{
strMyKey = nav.Evaluate(expr).ToString();//Convert.ToString((int)(nav.Evaluate(expr)));
aKey.Add(strMyKey);
}
else if (expr.ReturnType == XPathResultType.Boolean)
{
strMyKey = Convert.ToString((bool)(nav.Evaluate(expr)));
aKey.Add(strMyKey);
}
else if (expr.ReturnType == XPathResultType.String)
{
strMyKey = (string)(nav.Evaluate(expr));
aKey.Add(strMyKey);
}
else if (expr.ReturnType == XPathResultType.NodeSet)
{
// ????????xpath命中多个节点时，是否创建多个key
XPathNodeIterator iterator = null;
try
{
iterator = nav.Select(expr);
}
catch (Exception ex)
{
string strTempNstableName = "";
if (strNstableName == null)
strTempNstableName = "null";
else
strTempNstableName = "'" + strNstableName + "'";
strError = "用路径'" + strXpath + "'选节点时出错，" + ex.Message + " \r\n使用的名字空间表名为" + strTempNstableName + "。";
return -1;
}
if (iterator != null)
{
while (iterator.MoveNext())
{
XPathNavigator navigator = iterator.Current;
strMyKey = navigator.Value;
if (strMyKey == "")
continue;
aKey.Add(strMyKey);
}
}
}
else
{
throw (new Exception("XPathExpression的ReturnType为'" + expr.ReturnType.ToString() + "'无效"));
}
}
for (int j = 0; j < aKey.Count; j++)
{
strKey = aKey[j];
//???????注意，如果key内容为空，是否也应该算作一个key呢?
if (strKey == "")
continue;
strKeyNoProcess = strKey;
strNum = "-1";
List<string> outputKeys = new List<string>();
if (tableInfo.nodeConvertKeyString != null)
{
nRet = ConvertKeyWithStringNode(domData,
strKey,
tableInfo.nodeConvertKeyString,
out outputKeys,
out strError);
if (nRet == -1)
return -1;
}
else
{
outputKeys = new List<string>();
outputKeys.Add(strKey);
}
for (int k = 0; k < outputKeys.Count; k++)
{
string strOneKey = outputKeys[k];
//根据自身的配置进行处理,得到num
if (tableInfo.nodeConvertKeyNumber != null)
{
nRet = ConvertKeyWithNumberNode(
domData,
strOneKey,
tableInfo.nodeConvertKeyNumber,
out strNum,
out strError);
if (nRet == -1)
return -1;
if (nRet == 1)
{
// 2010/9/27
strOneKey = strError + " -- " + strOneKey;
strNum = "-1";
}
// 2010/11/20
if (String.IsNullOrEmpty(strNum) == true)
continue;
}
if (strOneKey.Length > nKeySize)
strOneKey = strOneKey.Substring(0, nKeySize);
if (strNum.Length >= 20)
strNum = strNum.Substring(0, 19);
KeyItem keyItem = new KeyItem(strSqlTableName,
strOneKey,
strFromValue,
strRecordID,
strNum,
strKeyNoProcess,
strFromName);
keys.Add(keyItem);
}
}
}
return 0;
}
private void DecodeThread()
{
var random = new Random();
for (;;)
{
Thread.Sleep(1000 * 3);
if (this.DecodeState == ManagerState.Stop) return;
DownloadItem item = null;
try
{
lock (_thisLock)
{
if (_settings.DownloadItems.Count > 0)
{
item = _settings.DownloadItems
.Where(n => !_workingSeeds.Contains(n.Seed))
.Where(n => n.State == DownloadState.Decoding || n.State == DownloadState.ParityDecoding)
.Where(n => n.Priority != 0)
.OrderBy(n => (n.Depth != n.Seed.Metadata.Depth) ? 0 : 1)
.OrderBy(n => (n.State == DownloadState.Decoding) ? 0 : 1)
.FirstOrDefault();
if (item != null)
{
_workingSeeds.Add(item.Seed);
}
}
}
}
catch (Exception)
{
return;
}
if (item == null) continue;
try
{
{
if ((item.Depth == 1 && !_cacheManager.Contains(item.Seed.Metadata.Key))
|| (item.Depth > 1 && !item.Index.Groups.All(n => _existManager.GetCount(n) >= n.InformationLength)))
{
item.State = DownloadState.Downloading;
}
else
{
var keys = new KeyCollection();
var compressionAlgorithm = CompressionAlgorithm.None;
var cryptoAlgorithm = CryptoAlgorithm.None;
byte[] cryptoKey = null;
if (item.Depth == 1)
{
keys.Add(item.Seed.Metadata.Key);
compressionAlgorithm = item.Seed.Metadata.CompressionAlgorithm;
cryptoAlgorithm = item.Seed.Metadata.CryptoAlgorithm;
cryptoKey = item.Seed.Metadata.CryptoKey;
}
else
{
item.State = DownloadState.ParityDecoding;
item.DecodeOffset = 0;
item.DecodeLength = item.Index.Groups.Sum(n => n.Length);
try
{
foreach (var group in item.Index.Groups.ToArray())
{
using (var tokenSource = new CancellationTokenSource())
{
var task = _cacheManager.ParityDecoding(group, tokenSource.Token);
while (!task.IsCompleted)
{
if (this.DecodeState == ManagerState.Stop || !_settings.DownloadItems.Contains(item)) tokenSource.Cancel();
Thread.Sleep(1000);
}
keys.AddRange(task.Result);
}
item.DecodeOffset += group.Length;
}
}
catch (Exception)
{
continue;
}
compressionAlgorithm = item.Index.CompressionAlgorithm;
cryptoAlgorithm = item.Index.CryptoAlgorithm;
cryptoKey = item.Index.CryptoKey;
}
item.State = DownloadState.Decoding;
if (item.Depth < item.Seed.Metadata.Depth)
{
string fileName = null;
bool largeFlag = false;
try
{
item.DecodeOffset = 0;
item.DecodeLength = keys.Sum(n => (long)_cacheManager.GetLength(n));
using (var stream = DownloadManager.GetUniqueFileStream(Path.Combine(_workDirectory, "index")))
using (ProgressStream decodingProgressStream = new ProgressStream(stream, (object sender, long readSize, long writeSize, out bool isStop) =>
{
isStop = (this.DecodeState == ManagerState.Stop || !_settings.DownloadItems.Contains(item));
if (!isStop && (stream.Length > item.Seed.Length))
{
isStop = true;
largeFlag = true;
}
item.DecodeOffset = writeSize;
}, 1024 * 1024, true))
{
fileName = stream.Name;
_cacheManager.Decoding(decodingProgressStream, compressionAlgorithm, cryptoAlgorithm, cryptoKey, keys);
}
}
catch (StopIoException)
{
if (File.Exists(fileName))
{
File.Delete(fileName);
}
if (largeFlag)
{
throw new Exception("size too large.");
}
continue;
}
catch (Exception)
{
if (File.Exists(fileName))
{
File.Delete(fileName);
}
throw;
}
Index index;
using (var stream = new UnbufferedFileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.Read, FileOptions.None, _bufferManager))
{
index = Index.Import(stream, _bufferManager);
}
File.Delete(fileName);
lock (_thisLock)
{
if (!_settings.DownloadItems.Contains(item)) continue;
item.DecodeOffset = 0;
item.DecodeLength = 0;
this.UncheckState(item.Index);
item.Index = index;
this.CheckState(item.Index);
foreach (var group in item.Index.Groups)
{
foreach (var key in group.Keys)
{
_cacheManager.Lock(key);
}
}
item.Indexes.Add(index);
item.Depth++;
item.State = DownloadState.Downloading;
}
}
else
{
item.State = DownloadState.Decoding;
string fileName = null;
bool largeFlag = false;
string downloadDirectory;
if (item.Path == null)
{
downloadDirectory = this.BaseDirectory;
}
else
{
if (Path.IsPathRooted(item.Path))
{
downloadDirectory = item.Path;
}
else
{
downloadDirectory = Path.Combine(this.BaseDirectory, item.Path);
}
}
Directory.CreateDirectory(downloadDirectory);
try
{
item.DecodeOffset = 0;
item.DecodeLength = keys.Sum(n => (long)_cacheManager.GetLength(n));
using (var stream = DownloadManager.GetUniqueFileStream(Path.Combine(downloadDirectory, string.Format("{0}.tmp", DownloadManager.GetNormalizedPath(item.Seed.Name)))))
using (ProgressStream decodingProgressStream = new ProgressStream(stream, (object sender, long readSize, long writeSize, out bool isStop) =>
{
isStop = (this.DecodeState == ManagerState.Stop || !_settings.DownloadItems.Contains(item));
if (!isStop && (stream.Length > item.Seed.Length))
{
isStop = true;
largeFlag = true;
}
item.DecodeOffset = writeSize;
}, 1024 * 1024, true))
{
fileName = stream.Name;
_cacheManager.Decoding(decodingProgressStream, compressionAlgorithm, cryptoAlgorithm, cryptoKey, keys);
if (stream.Length != item.Seed.Length) throw new Exception("Stream.Length != Seed.Length");
}
}
catch (StopIoException)
{
if (File.Exists(fileName))
{
File.Delete(fileName);
}
if (largeFlag)
{
throw new Exception("size too large.");
}
continue;
}
catch (Exception)
{
if (File.Exists(fileName))
{
File.Delete(fileName);
}
throw;
}
File.Move(fileName, DownloadManager.GetUniqueFilePath(Path.Combine(downloadDirectory, DownloadManager.GetNormalizedPath(item.Seed.Name))));
lock (_thisLock)
{
if (!_settings.DownloadItems.Contains(item)) continue;
item.DecodeOffset = 0;
item.DecodeLength = 0;
{
var usingKeys = new HashSet<Key>();
foreach (var index in item.Indexes)
{
foreach (var group in index.Groups)
{
usingKeys.UnionWith(group.Keys
.Where(n => _cacheManager.Contains(n))
.Reverse()
.Take(group.InformationLength));
}
}
_cacheManager.SetSeed(item.Seed.Clone(), usingKeys.ToArray());
}
_settings.DownloadedSeeds.Add(item.Seed.Clone());
_cacheManager.Unlock(item.Seed.Metadata.Key);
foreach (var index in item.Indexes)
{
foreach (var group in index.Groups)
{
foreach (var key in group.Keys)
{
_cacheManager.Unlock(key);
}
}
}
item.Indexes.Clear();
item.State = DownloadState.Completed;
}
}
}
}
}
catch (Exception e)
{
// Check
{
var list = new List<Key>();
list.Add(item.Seed.Metadata.Key);
foreach (var index in item.Indexes)
{
foreach (var group in index.Groups)
{
foreach (var key in group.Keys)
{
list.Add(key);
}
}
}
foreach (var key in list)
{
if (this.DecodeState == ManagerState.Stop) return;
if (!_cacheManager.Contains(key)) continue;
var buffer = new ArraySegment<byte>();
try
{
buffer = _cacheManager[key];
}
catch (Exception)
{
}
finally
{
if (buffer.Array != null)
{
_bufferManager.ReturnBuffer(buffer.Array);
}
}
}
}
item.State = DownloadState.Error;
Log.Error(e);
}
finally
{
_workingSeeds.Remove(item.Seed);
}
}
}
// return:
// -1 出错
// 0 成功
public int Write(
string strDatabaseName,
KeyCollection keys,
delegate_getfilename getfilename,
out string strError)
{
strError = "";
// 确保 keys 里面的事项是排序过的。如果没有排序，本函数也能工作，只是效率略低
DelayTable table = null;
KeyCollection part_keys = new KeyCollection();
foreach (KeyItem item in keys)
{
if (table == null)
{
table = GetTable(strDatabaseName, item.SqlTableName);
if (string.IsNullOrEmpty(table.FileName) == true)
{
string strFilename = getfilename(strDatabaseName, item.SqlTableName);
int nRet = table.Create(strFilename, out strError);
if (nRet == -1)
return -1;
}
}
else
{
if (table.TableName != item.SqlTableName)
{
if (part_keys.Count > 0)
{
table.Write(part_keys);
part_keys.Clear();
}
table = GetTable(strDatabaseName, item.SqlTableName);
if (string.IsNullOrEmpty(table.FileName) == true)
{
string strFilename = getfilename(strDatabaseName, item.SqlTableName);
int nRet = table.Create(strFilename, out strError);
if (nRet == -1)
return -1;
}
}
}
part_keys.Add(item);
}
if (part_keys.Count > 0)
{
Debug.Assert(table != null, "");
table.Write(part_keys);
part_keys.Clear();
}
return 0;
}
/// <summary>
/// 添加指定键值及输出内容项。
/// </summary>
/// <param name="key">键值。</param>
/// <param name="value">关联输出内容项。</param>
public void Add(string key, OutputContentObject value)
{
KeyCollection.Add(key, value);
Output.Metadata.CollectionAdd(Content, value.Content);
}