/// <summary>
/// 根据参数创建签名信息(阿里云的算法)。
/// @param array 参数数组。
/// @return string 签名字符串。
/// </summary>
/// <param name="parameter">参数数组</param>
/// <returns>签名字符串</returns>
private string _makeSignAliyun(Dictionary <string, object> parameter, string method)
{
var q = string.Empty;
var sign_mode = string.Empty;
if (parameter != null)
{
if (parameter.ContainsKey("sign_mode"))
{
sign_mode = parameter["sign_mode"] as string;
}
parameter = Utilities.KeySort(parameter);
if (sign_mode == "1" && parameter.ContainsKey("items"))
{
parameter.Remove("items");
}
q = Utilities.http_build_query(parameter);
q = Regex.Replace(q, "(%[0-9a-f][0-9a-f])", c => c.Value.ToUpper());
q = percentEncode(q);
q = HttpUtility.UrlEncode(q);
q = percentEncode(q);
q = method.ToUpper() + "&%2F&" + q;
q = Regex.Replace(q, "(%[0-9a-f][0-9a-f])", c => c.Value.ToUpper());
}
//Console.WriteLine(q);
return(HmacSha1Sign(q, _clientSecret + "&"));
}
/// <summary>
/// 替换Uri中的转义字符。
/// </summary>
/// <param name="value">Uri中的字符串</param>
/// <returns>转义后的字符串</returns>
private static string UrlEncode(string value)
{
value = HttpUtility.UrlEncode(value);
value = Regex.Replace(value, "(%[0-9a-f][0-9a-f])", c => c.Value.ToUpper());
value = value.Replace("(", "%28")
.Replace(")", "%29")
.Replace("$", "%24")
.Replace("*", "%2A")
.Replace("'", "%26");
value = value.Replace("%7E", "~");
return(value);
}
private static string UrlEncode(string str)
{
if (str == null)
{
return(null);
}
var stringToEncode =
HttpUtility.UrlEncode(str)
.Replace("+", "%20")
.Replace("*", "%2A")
.Replace("(", "%28")
.Replace(")", "%29")
.Replace("!", "%21")
.Replace("~", "%7E");
return(stringToEncode);
}
/// <prototype>
/// public Dictionary<String, String> pushHADocFile(String fileName, String tableName, Int32 offset =
/// 1,Int32 maxSize = PUSH_MAX_SIZE, Int32 frequence = PUSH_FREQUENCE)
/// </prototype>
public Dictionary <string, string> PushHADocFile(string fileName, string tableName, int offset = 1,
int maxSize = PUSH_MAX_SIZE, int frequence = PUSH_FREQUENCE)
{
using (var reader = _connect(fileName))
{
// 默认doc初始结构。
var doc = new Dictionary <string, object>();
doc["cmd"] = "";
doc["fields"] = new Dictionary <string, object>();
//$doc = array('cmd' => '', 'fields' => array());
// 当前行号，用来记录当前已经解析到了第多少行。
var lineNumber = 1;
// 最新成功push数据的行号，用于如果在重新上传的时候设定offset偏移行号。
var lastLineNumber = 0;
// 最后更新的doc中的字段名，如果此行没有字段结束符，则下行的数据会被添加到这行的字段上。
// 有一些富文本，在当前行没有结束此字段，则要记录最后的字段名称。
// 例如：
// rich_text=鲜花
// 礼品专卖店^_
// other_field=xxx^_
var lastField = "";
// 当前还未上传的文档的大小。单位MB.
var totalSize = 0;
// 当前秒次已经发了多少次请求，用于限流。
var timeFrequence = 0;
// 开始遍历文件。
var line = string.Empty;
var buffer = new List <Dictionary <string, object> >();
var uTime = new DateTime(1970, 1, 1, 0, 0, 0);
double time = 0;
var key = "";
var value = "";
while ((line = reader.ReadLine()) != null)
{
// 如果当前的行号小于设定的offset行号时跳过。
if (lineNumber < offset)
{
continue;
}
// 获取结果当前行的最后两个字符。
var bytes = line.ToCharArray();
var separator = bytes[bytes.Length - 1];
// 如果当前结束符是文档的结束符^^\n，则当前doc解析结束。并计算buffer+当前doc文档的
// 大小，如果大于指定的文档大小，则push buffer到api，并清空buffer，同时把当前doc
// 文档扔到buffer中。
if (separator == HA_DOC_ITEM_SEPARATOR)
{
lastField = string.Empty;
// 获取当前文档生成json并urlencode之后的size大小。
var json = JsonConvert.SerializeObject(doc);
var currentSize = HttpUtility.UrlEncode(json).Length;
// 如果计算的大小+buffer的大小大于等于限定的阀值self::PUSH_MAX_SIZE，则push
// buffer数据。
if (currentSize + totalSize >= maxSize * 1024 * 1024)
{
// push 数据到api。
var rel = doAction(buffer, tableName);
if ("OK" != rel.Status)
{
//TODO what is this error?
throw new Exception("Api returns error: " +
". Latest successful posted line is" + lastLineNumber);
}
// 如果push成功，则计算每秒钟的push的频率并如果超过频率则sleep。
lastLineNumber = lineNumber;
var newTime = (DateTime.UtcNow - uTime).TotalSeconds;
timeFrequence++;
// 如果时间为上次的push时间且push频率超过设定的频率，则unsleep 剩余的毫秒数。
if (Math.Floor(newTime) == time && timeFrequence >= frequence)
{
var left = Math.Floor(newTime) + 1 - newTime;
Thread.Sleep(Convert.ToInt32(left));
timeFrequence = 0;
}
// 重新设定时间和频率。
newTime = (DateTime.UtcNow - uTime).TotalSeconds;
if (time != newTime)
{
time = newTime;
timeFrequence = 0;
}
// 重置buffer为空，并重新设定total size 为0；
buffer.Clear();
totalSize = 0;
}
// doc 添加到buffer中，并增加total size的大小。
buffer.Add(doc);
totalSize += currentSize;
doc = new Dictionary <string, object>();
// 初始化doc。
doc["cmd"] = "";
doc["fields"] = new Dictionary <string, object>();
}
else if (separator == HA_DOC_FIELD_SEPARATOR)
{
// 表示当前字段结束。
var detail = line.TrimEnd(HA_DOC_FIELD_SEPARATOR, '\n');
if (!string.IsNullOrEmpty(lastField))
{
// 表示当前行非第一行数据，则获取最后生成的字段名称并给其赋值。
var item = doc["fields"] as Dictionary <string, object>;
var fv = _extractFieldValue(item[lastField] + detail);
if (fv.Length == 1)
{
item[lastField] = fv[0];
}
else
{
item[lastField] = fv;
}
}
else
{
// 表示当前为第一行数据，则解析key 和value。
try
{
var data = _parseHADocField(detail);
key = data[0];
value = data[1];
}
catch (Exception e)
{
throw new Exception(e.Message +
". Latest successful posted line number is " + lastLineNumber);
}
if (key.ToUpper() == "CMD")
{
doc["cmd"] = value.ToUpper();
}
else
{
var item = doc["fields"] as Dictionary <string, object>;
var fv = _extractFieldValue(value);
if (fv.Length == 1)
{
item[key] = fv[0];
}
else
{
item[key] = fv;
}
}
}
// 设置字段名称为空。
lastField = "";
}
else
{
// 此else 表示富文本的非最后一行。
line = line + "\n";
// 表示富文本非第一行。
if (!string.IsNullOrEmpty(lastField))
{
var item = doc["fields"] as Dictionary <string, object>;
item[lastField] += line;
}
else
{
// 表示字段的第一行数据。
try
{
var data = _parseHADocField(line);
key = data[0];
value = data[1];
}
catch (Exception e)
{
throw new Exception(e.Message +
". Latest successful posted line number is " + lastLineNumber);
}
var item = doc["fields"] as Dictionary <string, object>;
item[key] = value;
lastField = key;
}
}
lineNumber++;
}
// 如果buffer 中还有数据则再push一次数据。
if (buffer.Count > 0)
{
// push 数据到api。
var rel1 = doAction(buffer, tableName);
if (PUSH_RETURN_STATUS_OK != rel1.Status)
{
throw new Exception("Api returns error: " +
". Latest successful posted line number is " + lastLineNumber);
}
}
return(new Dictionary <string, string>
{
{ "status", "OK" },
{ "message", "The data is posted successfully." }
});
}
}
/// <summary>
/// HttpWebRequest 版本
/// 使用方法：
/// var post_string = new NameValueCollection();;
/// post_string.Add(...);
/// requestByWebrequest("POST", "http://alicloud.com/",post_string);
/// </summary>
/// <param name="method">发送方式，POST/GET</param>
/// <param name="url">WEB API的请求URL</param>
/// <param name="parameters">参数数组。</param>
/// <param name="httpOptions">http option参数数组。</param>
/// <returns>返回decode后的HTTP response。</returns>
protected virtual JObject requestByWebrequest(string method, string url, Dictionary <string, object> parameters,
NameValueCollection httpOptions = null)
{
var args = BuildParams(parameters);
if (method == "GET")
{
url += "?" + args;
args = string.Empty;
//Console.WriteLine(url);
}
if (debug)
{
debugInfo.Add(string.Format("encodedQueryString: {0}", url));
debugInfo.Add(string.Format("decodedQueryString: {0}", HttpUtility.UrlDecode(url)));
}
var request = (HttpWebRequest)WebRequest.Create(url);
#if NETCORE
request.Headers = new WebHeaderCollection();
request.Headers[HttpRequestHeader.AcceptEncoding] = "gzip, deflate";
// Set request method: 'GET' / 'POST'
request.Method = method;
//request.Headers.Add("Expect:");
request.ContentType = "application/x-www-form-urlencoded";
var argsInBytes = Encoding.UTF8.GetBytes(args);
request.Headers["ContentLength"] = argsInBytes.Length.ToString();
request.Headers["UserAgent"] = string.Format("opensearch/.net sdk {0}", SDK_VERSION);
//request.Accept = "*/*";
// Write the post data for the POST request
if (method == "POST")
{
using (var sw = request.GetRequestStreamAsync().GetAwaiter().GetResult())
{
sw.Write(argsInBytes, 0, argsInBytes.Length);
}
}
var responseString = string.Empty;
using (var webResonse = (HttpWebResponse)request.GetResponseAsync().GetAwaiter().GetResult())
{
using (var sr = new StreamReader(webResonse.GetResponseStream()))
{
responseString = sr.ReadToEnd();
}
var resultJObject = new JObject();
resultJObject["httpCode"] = (int)webResonse.StatusCode;
//resultJObject["rawResponse"] = JObject.Parse(responseString);
resultJObject["rawResponse"] = responseString;
return(resultJObject);
}
#else
request.Headers.Clear();
request.AutomaticDecompression = DecompressionMethods.GZip;
//request.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip, deflate");
//set Timeout
request.Timeout = timeout;
request.ReadWriteTimeout = requestTimeout;
// set Expect100Continue to false, so it will send only one http package.
request.ServicePoint.Expect100Continue = false;
// Set request method: 'GET' / 'POST'
request.Method = method;
//request.Headers.Add("Expect:");
request.ContentType = "application/x-www-form-urlencoded";
var argsInBytes = Encoding.UTF8.GetBytes(args);
request.ContentLength = argsInBytes.Length;
request.UserAgent = string.Format("opensearch/.net sdk {0}", SDK_VERSION);
//request.Accept = "*/*";
// Write the post data for the POST request
if (method == "POST")
{
//using (var sw = request.GetRequestStream())
{
var sw = request.GetRequestStream();
sw.Write(argsInBytes, 0, argsInBytes.Length);
sw.Close();
}
}
var responseString = string.Empty;
var webResonse = (HttpWebResponse)request.GetResponse();
using (var sr = new StreamReader(webResonse.GetResponseStream()))
{
responseString = sr.ReadToEnd();
}
webResonse.Close();
var resultJObject = new JObject();
resultJObject["httpCode"] = (int)webResonse.StatusCode;
//resultJObject["rawResponse"] = JObject.Parse(responseString);
resultJObject["rawResponse"] = responseString;
return(resultJObject);
#endif
}