/// <summary>
/// 获取搜索结果
/// </summary>
/// <returns></returns>
public List <string> GetSearchResult(string keyword, int maxCount = 60)
{
var list = new List <string>();
var count = 20;
for (int offset = 0; offset < 3 * count; offset = (++offset) * count)
{
var url = string.Format("https://www.toutiao.com/search_content/?offset={0}&format=json&keyword={1}&autoload=true&count={2}&cur_tab=1&from=search_tab", offset, HttpUtility.UrlEncode(keyword), count);
var httpDownloader = new HTTP();
var headers = new Dictionary <string, string>();
headers.Add("Accept", "application/json, text/javascript");
headers.Add("Accept-Encoding", "gzip, deflate, br");
headers.Add("Accept-Language", "zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7");
headers.Add("Host", "www.toutiao.com");
headers.Add("Content-Type", "application/x-www-form-urlencoded");
headers.Add("Referer", string.Format("https://www.toutiao.com/search/?keyword={0}", HttpUtility.UrlEncode(keyword)));
headers.Add("User-Agent", CONST.UserAgent);
headers.Add("X-Requested-With", "XMLHttpRequest");
var strData = httpDownloader.GetGzip(url, Encoding.UTF8, headers);
//strData = Convertor.FromUnicodeToUTF8(strData);
list.Add(strData);
LOGGER.Log(string.Format("正在获取 头条搜索结果 {0}", offset));
//ThreadManager.Pause(seconds: 5);
}
return(list);
}
public static object Get(string url, string cmd, string method, object args)
{
var retryCount = 5;
while (0 < retryCount)
{
retryCount--;
LOGGER.Log(string.Format("调用NodeService 还剩余{0}次", retryCount));
var json = string.Empty;
var context = new
{
CMD = cmd,
Method = method,
Args = args
};
try
{
json = Convertor.FromObjectToJson(context);
var httpdownloader = new HTTP();
var resString = httpdownloader.Post(url, Encoding.UTF8, json);
var res = Convertor.FromJsonToDict2(resString)["RES"];
return(res);
}
catch (Exception e)
{
LOGGER.Log(string.Format("调用NodeService的时候,发生异常：{0} {1}", e.Message, DateTime.Now));
LOGGER.Log(string.Format("参数：", args));
//var db = DataStorage.GetInstance(DBType.MongoDB);
//db.Save(new { Args = Convertor.FromObjectToDictionary(args), CreateTime = DateTime.Now, Message = e.Message }, "Exception", "StockService");
LOGGER.Beep();
Thread.Sleep(new TimeSpan(0, 5, 0));
}
}
return(null);
}
// Loads an Asset Pack
private static AssetPack LoadPack(string name)
{
AssetPack pack = InternalLogger.HandleThrow(() => AssetLoader.LoadPack(Path.Combine(ASSETS_FOLDER, name)));
LOGGER.Log($"- Loaded '{Path.GetFileNameWithoutExtension(name)}'");
return(pack);
}
public void ProcHistory()
{
var folderPath = @"F:\Excel\";
var files = Directory.EnumerateFiles(folderPath);
foreach (var filePath in files)
{
var fileName = filePath.Replace(folderPath, string.Empty).Replace(".xls", string.Empty);
var lines = File.ReadAllLines(filePath, Encoding.Default);
for (var k = 1; k < lines.Length; k++)
{
var line = lines[k];
var fileNameLength = fileName.Length;
var stockCode = fileName.Substring(0, 6);
var dateString = fileName.Substring(fileName.Length - 8, 8).Insert(4, "-").Insert(7, "-");
var arr = line.Split(new char[] { '\t' }, StringSplitOptions.RemoveEmptyEntries);
if (6 == arr.Length && lines[0].Contains("成交时间"))
{
var tradingTime = DateTime.Parse(dateString + " " + arr[0]);
var price = float.Parse(arr[1]);
var priceChange = arr[2];
var volume = int.Parse(arr[3]) * 100;
var turnover = float.Parse(arr[4]);
var kind = arr[5];
LOGGER.Log(string.Format("{0}{1}{2}{3}{4}{5}", tradingTime, price, priceChange, volume, turnover, kind));
}
LOGGER.Log(line);
}
}
}
/// <summary>
/// 数据注册
/// </summary>
public static void Register()
{
if (DateTime.MinValue == DataStorage.initialTime)
{
var configJson = new HTTP().GetGzip2(string.Format("http://aifuwu.wang/API.ashx?c=WangJun.DB.YunConfig&m=Load&p0={0}", YunConfig.CurrentGroupID), Encoding.UTF8);
if (!string.IsNullOrWhiteSpace(configJson))
{
var configDict = Convertor.FromJsonToDict2(configJson); ///加载当前的配置信息
foreach (var item in configDict)
{
if (item.Value.ToString().Contains("mongodb://"))
{
MongoDB.Register(item.Key, item.Value.ToString());
}
else if (item.Value.ToString().Contains("Data Source="))
{
SQLServer.Register(item.Key, item.Value.ToString());
}
else if (item.Value.ToString().Contains("server="))
{
MySQL.Register(item.Key, item.Value.ToString());
}
}
DataStorage.initialTime = DateTime.Now;
LOGGER.Log("数据库组件配置信息初始化完毕 初始化为 " + YunConfig.CurrentGroupID);
}
}
}
/// <summary>
/// 遍历一个根目录下的文件
/// </summary>
/// <param name="rootPath"></param>
public void TraverseFiles()
{
LOGGER.Log("TraverseFiles遍历线程启动");
CollectionTools.AddToQueue(this.fileQueue, this.GetFiles(this.rootPath));///获取该目录下文件信息
var rootSubFolders = this.GetSubFolder(this.rootPath);
CollectionTools.AddToQueue <FolderFileInfo>(this.folderQueue, rootSubFolders);
while (0 < this.folderQueue.Count)
{
var folder = this.folderQueue.Dequeue();
if (null != folder)
{
var files = this.GetFiles(folder.Path);
CollectionTools.AddToQueue(this.fileQueue, files);
var subFolders = this.GetSubFolder(folder.Path);
CollectionTools.AddToQueue <FolderFileInfo>(this.folderQueue, subFolders);
LOGGER.Log(string.Format("TraverseFiles遍历线程 文件夹队列长度:{0}\t文件队列长度:{1}", this.FolderCount, this.FileCount));
}
if (0 < this.FileCount && null == this.ThreadOutput)
{
this.StartOutputThread();
}
}
LOGGER.Log("TraverseFiles遍历线程结束");
}
//Stop all processes
public void Stop()
{
LOGGER.Log($"Crawl terminated by {USER}", name);
isRunning = false;
StopTimer("RedditCrawler");
_client.Dispose();
_crawlLoop = null;
}
//Stop the downloads if karuta is closed
public void Close()
{
LOGGER.Log("Crawl terminated by Karuta", name);
if (!isRunning)
{
return;
}
isRunning = false;
_client.Dispose();
_crawlLoop.Dispose();
}
public void Handle(IDriverMessage value)
{
string message = string.Empty;
LOGGER.Log(Level.Verbose, "Received a message from driver, handling it with MessagingDriverMessageHandler");
if (value.Message.IsPresent())
{
message = ByteUtilities.ByteArraysToString(value.Message.Value);
}
_parentTask.DriverMessage(message);
}
/// <summary>
/// SINA融资融券
/// </summary>
public void SyncRZRQ()
{
var startTime = DateTime.Now;///开始运行时间
Console.Title = "SINA融资融券 更新进程 启动时间：" + startTime;
var exe = StockTaskExecutor.CreateInstance();
var mongo = DataStorage.GetInstance(DBType.MongoDB);
var dbName = CONST.DB.DBName_StockService;
var collectionNameRZRQ = CONST.DB.CollectionName_RZRQ;
var methodName = "SyncRZRQ";
while (true)
{
var q = this.PrepareData(methodName);
while (0 < q.Count)
{
var stockCode = q.Dequeue();
var stockName = this.stockCodeDict[stockCode];
var array = WebDataSource.GetInstance().GetRZRQ(stockCode);
foreach (Dictionary <string, object> arrayItem in array)
{
var svItem = new
{
StockCode = stockCode,
StockName = stockName,
TradingDate = DateTime.Parse(arrayItem["日期"].ToString()),
RZYE = arrayItem["融资余额"],
RZMRE = arrayItem["融资买入额"],
RZCHE = arrayItem["融资偿还额"],
RQYLJE = arrayItem["融券余量金额"],
RQYL = arrayItem["融券余量"],
RQMCL = arrayItem["融券卖出量"],
RQCHL = arrayItem["融券偿还量"],
RQYE = arrayItem["融券余额"],
CreateTime = DateTime.Now
};
LOGGER.Log(string.Format("SINA融资融券 {0} {1} {2} 保存完毕", stockCode, stockName, svItem.TradingDate));
var query = "{\"StockCode\":\"" + stockCode + "\",\"TradingDate\":new Date('" + string.Format("{0:yyyy/MM/dd}", svItem.TradingDate) + "')}";
mongo.Save3(dbName, collectionNameRZRQ, svItem, query);
}
ThreadManager.Pause(seconds: 2);
TaskStatusManager.Set(methodName, new { ID = methodName, StockCode = stockCode, StockName = stockName, Status = "已下载", CreateTime = DateTime.Now });
LOGGER.Log(string.Format("SINA融资融券 {0} {1} 保存完毕", stockCode, stockName));
}
LOGGER.Log(string.Format("本次SINA融资融券更新完毕，下一次一天以后更新 {0}", DateTime.Now));
TaskStatusManager.Set(methodName, new { ID = methodName, Status = "队列处理完毕", CreateTime = DateTime.Now });
ThreadManager.Pause(days: 1);
}
}
/// <summary>
/// 不断更新最新的新闻
/// </summary>
public void SyncStockNews()
{
var startTime = DateTime.Now;///开始运行时间
Console.Title = "同花顺财经要闻 更新进程 启动时间：" + startTime;
var exe = StockTaskExecutor.CreateInstance();
while (true)
{
try
{
if (CONST.IsSafeUpdateTime(1))
{
exe.GetNewsListCJYW(DateTime.Now.ToShortDateString());
///非交易时间
if (DateTime.Now.Hour < 6 || DateTime.Now.Hour > 23) ///6点前,23点后
{
exe.GetNewsListCJYW(DateTime.Now.AddDays(-1).ToShortDateString());
exe.GetNewsListCJYW(DateTime.Now.AddDays(-2).ToShortDateString());
exe.GetNewsListCJYW(DateTime.Now.AddDays(-3).ToShortDateString());
exe.GetNewsListCJYW(DateTime.Now.AddDays(-4).ToShortDateString());
exe.GetNewsListCJYW(DateTime.Now.AddDays(-5).ToShortDateString());
exe.GetNewsListCJYW(DateTime.Now.AddDays(-6).ToShortDateString());
exe.GetNewsListCJYW(DateTime.Now.AddDays(-7).ToShortDateString());
ThreadManager.Pause(hours: 1); ///凌晨一小时更新一次
}
else
{
ThreadManager.Pause(minutes: 10); ///10分钟更新一次新闻
}
}
else
{
exe.GetNewsListCJYW(DateTime.Now.ToShortDateString()); ///交易时间
ThreadManager.Pause(minutes: 3); ///3分钟更新一次新闻
}
LOGGER.Log(string.Format("自动新闻更新 已运行时间 {0}", DateTime.Now - startTime));
}
catch (Exception e)
{
LOGGER.Log(string.Format("异常：{0}", e.Message));
LOGGER.Log(string.Format("位置：{0}", e.StackTrace));
LOGGER.Beep();
ThreadManager.Pause(minutes: 1); ///停一分钟
}
}
}
//+ HELPERS
// Loads an assembly
private static Assembly LoadAssembly(string name, string prefix = null)
{
Assembly loaded = InternalLogger.HandleThrow(() => AssemblyUtils.LoadWithSymbols(Path.Combine(ASSEM_FOLDER, name + DLL_EXTENSION)));
if (loaded != null)
{
LOGGER.Log($"- {prefix ?? string.Empty}Injected '{name}'");
MAIN_ASSEMBLIES.Add(loaded);
}
else
{
LOGGER.Log($"- Failed to Inject '{name}'");
}
return(loaded);
}
/// <summary>
/// SINA 公司简介
/// </summary>
public void SyncGSJJ()
{
var startTime = DateTime.Now;///开始运行时间
Console.Title = "SINA公司简介 更新进程 启动时间：" + startTime;
var exe = StockTaskExecutor.CreateInstance();
var mongo = DataStorage.GetInstance(DBType.MongoDB);
var dbName = CONST.DB.DBName_StockService;
var collectionName = CONST.DB.CollectionName_GSJJ;
var methodName = "SyncGSJJ";
while (true)
{
///获取所有股票代码,遍历更新数据
var q = this.PrepareData(methodName);
while (0 < q.Count)
{
var stockCode = q.Dequeue();
var resObj = WebDataSource.GetInstance().GetGSJJ(stockCode);
var resDict = (resObj is Dictionary <string, object>) ? (resObj as Dictionary <string, object>)["PageData"] : new Dictionary <string, object>();
///保存到数据库中 二维化
var svItem = new
{
StockCode = stockCode,
StockName = this.stockCodeDict[stockCode],
Url = string.Format("http://vip.stock.finance.sina.com.cn/corp/go.php/vCI_CorpInfo/stockid/{0}.phtml", stockCode),
CreateTime = DateTime.Now,
PageData = resDict
};
///删除旧数据
///添加新数据
LOGGER.Log(string.Format("更新{0} {1}的公司简介 ", stockCode, svItem.StockName));
var filter = "{\"ContentType\":\"SINA公司简介\",\"StockCode\":\"" + stockCode + "\"}";
mongo.Save3(dbName, collectionName, svItem, filter);
TaskStatusManager.Set(methodName, new { ID = methodName, StockCode = stockCode, StockName = svItem.StockName, Status = "已下载", CreateTime = DateTime.Now });
ThreadManager.Pause(seconds: 5);
}
TaskStatusManager.Set(methodName, new { ID = methodName, Status = "队列处理完毕", CreateTime = DateTime.Now });
LOGGER.Log(string.Format("本次板块概念更新完毕，下一次一天以后更新 {0}", DateTime.Now));
ThreadManager.Pause(days: 2);
q = this.PrepareData(methodName);
}
}
public void SyncStockRardar()
{
var startTime = DateTime.Now;///开始运行时间
Console.Title = "SINA 股市雷达异动 更新进程 启动时间：" + startTime;
var exe = StockTaskExecutor.CreateInstance();
var mongo = DataStorage.GetInstance(DBType.MongoDB);
var dbName = CONST.DB.DBName_StockService;
var collectionName = CONST.DB.CollectionName_Radar;
var methodName = "SyncStockRardar";
while (true)
{
var filterCheck = "{\"TradingDate\":new Date('" + string.Format("{0:yyyy/MM/dd}", DateTime.Now) + "')}";
var checkRes = mongo.Count(dbName, collectionName, filterCheck);
if (0 == checkRes && CONST.IsSafeUpdateTime(1))
{
var listData = WebDataSource.GetInstance().GetStockRadar();
foreach (var item in listData)
{
var tradingTime = item["异动时间"].ToString().Replace("-", "/");
var filter = "{\"ContentType\":\"SINA 股市雷达\",\"TradingTime\":new Date('" + tradingTime + "')}";
var svItem = new
{
StockCode = item["股票代码"],
StockName = item["股票简称"],
AnomalyInfo = item["异动信息"],
TradingTime = item["异动时间"],
TradingDate = Convert.ToDateTime(tradingTime).Date
};
mongo.Save3(dbName, collectionName, svItem, filter);
}
TaskStatusManager.Set(methodName, new { ID = methodName, Status = "队列处理完毕", CreateTime = DateTime.Now });
LOGGER.Log(string.Format("本次股市雷达异动更新完毕，下一次一天以后更新 {0}", DateTime.Now));
}
else
{
LOGGER.Log(string.Format("今日股市雷达异动已经更新，下一次一天以后更新"));
}
ThreadManager.Pause(hours: 2);
}
}
public void SyncExcel()
{
var q = this.PrepareData();
//var date = DateTime.Now;
for (var date = DateTime.Now; new DateTime(2017, 1, 1) < date; date = date.AddDays(-1))
{
foreach (var stockCode in q)
{
var stockName = this.stockCodeDict[stockCode];
if (!(date.DayOfWeek == DayOfWeek.Sunday || date.DayOfWeek == DayOfWeek.Saturday))
{
DataSourceSINA.GetInstance().DownloadExcel(date, stockCode, stockName);
LOGGER.Log(string.Format("{0}{1}{2}", stockCode, stockName, date));
}
}
}
}
/// <summary>
/// 移动数据
/// </summary>
/// <param name="sourceKeyName"></param>
/// <param name="sourceDbName"></param>
/// <param name="sourceCollectionName"></param>
/// <param name="sourceFilter"></param>
/// <param name="targetKeyName"></param>
/// <param name="targetDbName"></param>
/// <param name="targetCollectionName"></param>
/// <param name="needDeleteSource"></param>
public static void MoveCollection(MongoDB sourceInst, string sourceDbName, string sourceCollectionName, string sourceFilter, MongoDB targetInst, string targetDbName, string targetCollectionName, bool needDeleteSource = false)
{
if (null != sourceInst && null != targetInst)
{
sourceInst.EventTraverse += (object sender, EventArgs e) =>
{
var ee = e as EventProcEventArgs;
var dict = ee.Default as Dictionary <string, object>;
var filter = string.Format("{{\"_id\":ObjectId('{0}')}}", dict["_id"]);
targetInst.Save3(targetDbName, targetCollectionName, dict, filter);
LOGGER.Log(string.Format("正在转移数据 {0}", dict["_id"]));
; if (true == needDeleteSource)
{
sourceInst.Delete(sourceDbName, sourceCollectionName, filter);
}
};
sourceInst.Find(sourceDbName, sourceCollectionName, sourceFilter);
}
}
//Download an image via url
void SaveImage(Post p, string file, Uri url)
{
try
{
data = _client.DownloadData(url);
string ext = Path.GetExtension(url.AbsolutePath);
using (FileStream image = new FileStream($"{file}{ext}", FileMode.Create, FileAccess.Write))
{
try
{
if (!Directory.GetParent(file).Exists)
{
Directory.GetParent(file).Create();
}
image.Write(data, 0, data.Length);
LOGGER.Log($"Saved {file}{ext}", $"/r/{p.SubredditName}", verbose);
image.Flush();
RinDB.AddImage(new ImageModel()
{
name = Uri.EscapeDataString(Path.GetFileNameWithoutExtension(file).Remove(0, p.CreatedUTC.ToEpoch().ToString().Length + 2)),
fileUri = Uri.EscapeDataString($"{file.Replace($"{baseDir}/", "")}{ext}"),
timeadded = p.CreatedUTC.ToEpoch(),
isnsfw = p.NSFW,
tags = _curTags?.ToList()
});
imgCount++;
}catch (Exception e)
{
LOGGER.LogError($"Failed to save \"{p.Title}\", {e.Message}", $"/r/{p.SubredditName}", verbose);
LOGGER.LogError(e.StackTrace, name, verbose);
}
}
}
catch (Exception e)
{
LOGGER.LogWarning($"Unable to Download {p.Title}, {e.Message}", $"/r/{p.SubredditName}", verbose);
LOGGER.LogError(e.StackTrace, name, verbose);
}
}
/// <summary>
/// SINA大单数据更新
/// </summary>
public void SyncSINADaDan()
{
var startTime = DateTime.Now;///开始运行时间
Console.Title = "SINA大单数据更新 更新进程 启动时间：" + startTime;
var exe = StockTaskExecutor.CreateInstance();
while (true)
{
if (CONST.IsSafeUpdateTime(1))
{
exe.GetDaDanData();
LOGGER.Log(string.Format("大单数据已更新,下一次更新在一天后 {0}", DateTime.Now));
ThreadManager.Pause(days: 1);
}
else
{
LOGGER.Log(string.Format("未在安全更新时间内 {0}", DateTime.Now));
ThreadManager.Pause(minutes: 5);
}
}
}
/// <summary>
/// SINA融资融券
/// </summary>
public void SyncTouTiao()
{
var startTime = DateTime.Now;///开始运行时间
Console.Title = "股票全网新闻 更新进程 启动时间：" + startTime;
var exe = StockTaskExecutor.CreateInstance();
var mongo = DataStorage.GetInstance(DBType.MongoDB);
var dbName = CONST.DB.DBName_StockService;
var collectionNameRZRQ = CONST.DB.CollectionName_RZRQ;
var methodName = "SyncTouTiao";
while (true)
{
var q = this.PrepareData(methodName);
while (0 < q.Count)
{
var stockCode = q.Dequeue();
var stockName = this.stockCodeDict[stockCode];
var array = WebDataSource.GetInstance().GetTouTiaoSearch(stockName);
foreach (Dictionary <string, object> arrayItem in array)
{
var doc = DocItem.Create(arrayItem);
doc.Save();
LOGGER.Log(string.Format("股票全网新闻 {0} {1} 保存完毕", stockCode, stockName));
}
ThreadManager.Pause(seconds: 1);
TaskStatusManager.Set(methodName, new { ID = methodName, StockCode = stockCode, StockName = stockName, Status = "已下载", CreateTime = DateTime.Now });
LOGGER.Log(string.Format("股票全网新闻 {0} {1} 保存完毕", stockCode, stockName));
}
LOGGER.Log(string.Format("股票全网新闻更新完毕，下一次一天以后更新 {0}", DateTime.Now));
TaskStatusManager.Set(methodName, new { ID = methodName, Status = "队列处理完毕", CreateTime = DateTime.Now });
ThreadManager.Pause(days: 1);
}
}
/// <summary>
/// 更新股票代码
/// </summary>
public void SyncStockCode()
{
var startTime = DateTime.Now;///开始运行时间
Console.Title = "股票代码更新进程 启动时间：" + startTime;
var inst = StockTaskExecutor.CreateInstance();
while (true)
{
if (CONST.IsSafeUpdateTime(1)) ///非交易时间,且交易前1小时
{
LOGGER.Log(string.Format("准备更新股票代码 {0}", DateTime.Now));
inst.UpdateAllStockCode();
LOGGER.Log(string.Format("股票代码更新完毕 {0} {1} 下一次更新在一天后后 已运行 ", DateTime.Now, DateTime.Now - startTime));
ThreadManager.Pause(days: 1); ///每日更新一次
}
else
{
Console.WriteLine("交易时间或非安全可更新时间 {0}", DateTime.Now);
ThreadManager.Pause(hours: 1);
}
}
}
// Patches an assembly
private static void PatchAssembly(Assembly assembly, bool late = false, Type[] ignoreLate = null)
{
InternalLogger.HandleThrow(() =>
{
if (ignoreLate != null)
{
foreach (Type type in ignoreLate)
{
harmony.PatchWrapper(type);
}
}
if (late)
{
harmony.LatePatchAll(assembly);
}
else
{
harmony.PatchAll(assembly);
}
});
LOGGER.Log($"- Patched '{assembly.GetName().Name}'");
}
/// <summary>
/// 股市雷达 用于发现异常股票
/// http://finance.sina.com.cn/stockradar/stockradar16.html
/// </summary>
/// <returns></returns>
public List <string> GetStockRadar()
{
var list = new List <string>();
for (int k = 1; k <= 16; k++)
{
var url = string.Format("http://finance.sina.com.cn/stockradar/stockradar{0}.html", k);
var httpDownloader = new HTTP();
var headers = new Dictionary <HttpRequestHeader, string>();
headers.Add(HttpRequestHeader.Accept, "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8");
headers.Add(HttpRequestHeader.AcceptEncoding, "gzip, deflate");
headers.Add(HttpRequestHeader.AcceptLanguage, "zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7");
headers.Add(HttpRequestHeader.Host, "finance.sina.com.cn");
headers.Add(HttpRequestHeader.Referer, string.Format("http://finance.sina.com.cn/stockradar/stockradar{0}.html", (k % 14) + 2));
headers.Add(HttpRequestHeader.UserAgent, CONST.UserAgent);
var strData = httpDownloader.GetGzip2(url, Encoding.GetEncoding("GBK"), headers);
list.Add(strData);
LOGGER.Log(string.Format("正在获取 股市雷达 {0}", k));
ThreadManager.Pause(seconds: 5);
}
return(list);
}
//Start find and download the images
void Crawl()
{
if (_reddit == null || _imgurClient == null)
{
Setup();
}
if (getFrom != null)
{
LOGGER.Log($"Starting crawl of {getFrom}", name, verbose);
}
else
{
LOGGER.Log($"Starting Crawl of {subreddits.Count} subreddits", name, verbose);
}
string curDir = "";
imgCount = 0;
string file = "";
Listing <Post> posts = default(Listing <Post>);
List <Subreddit> subs;
bool postGet = false;
subs = new List <Subreddit>();
_crawlLoop = StartTimer("RedditCrawler", info =>
{
try
{
imgCount = 0;
if (needsReBuild)
{
subs.Clear();
LOGGER.Log("Rebuilding subreddit list", name, verbose);
if (getFrom != null)
{
try
{
subs.Add(_reddit.GetSubreddit(getFrom));
LOGGER.Log($"Connected to {getFrom}", name, verbose);
}
catch (Exception e)
{
LOGGER.LogWarning($"Failed to connect to subreddit: {getFrom}, {e.Message}", name, verbose);
_crawlLoop.Dispose();
}
}
else
{
if (subreddits.Count == 0)
{
isRunning = false;
return;
}
subs.Clear();
foreach (string s in subreddits)
{
if (!isRunning)
{
break;
}
try
{
subs.Add(_reddit.GetSubreddit(s));
LOGGER.Log($"Connected to {s}", name, verbose);
}
catch (Exception e)
{
LOGGER.LogWarning($"Failed to connect to subreddit: {s}, {e.Message}", name, verbose);
LOGGER.LogWarning(e.StackTrace, name, verbose);
continue;
}
}
}
needsReBuild = false;
LOGGER.Log("Finished Building, starting crawl", name, verbose);
}
if (!isRunning)
{
_crawlLoop.Dispose();
}
foreach (Subreddit sub in subs)
{
if (!isRunning)
{
break;
}
SelectTags(sub.Name);
//Change the current directory and make sure it exists
curDir = baseDir + "/" + sub.Name;
if (!Directory.Exists(curDir))
{
Directory.CreateDirectory(curDir);
}
bool subCollected = false;
int retryCount = 0;
while (!subCollected)
{
if (!isRunning || retryCount >= 10)
{
if (retryCount >= 10)
{
Write("Retry timeout");
}
break;
}
try
{
//Get posts
switch (searchMode)
{
case SearchMode.Hot:
posts = sub.Hot;
break;
case SearchMode.New:
posts = sub.New;
break;
case SearchMode.Top:
posts = sub.GetTop(FromTime.All);
break;
default:
posts = sub.New;
break;
}
subCollected = true;
}
catch (Exception e)
{
LOGGER.LogWarning($"Failed to connect to reddit: {e.Message}, retrying...", name, verbose);
retryCount++;
}
}
postGet = false;
while (!postGet)
{
if (!isRunning)
{
break;
}
try
{
foreach (Post p in posts.Take(postsToGet))
{
postGet = true;
if (!isRunning)
{
break;
}
string log = (p.Title + " " + p.Url);
string ext = Path.GetExtension(p.Url.AbsolutePath);
if (p.NSFW && !Directory.Exists(curDir + "/NSFW"))
{
Directory.CreateDirectory(curDir + "/NSFW");
}
//Create file name
file = FormatFileName(p.Title);
if (allowedFiles.Contains(ext)) //Direct link to image file
{
file = "[" + p.CreatedUTC.ToEpoch() + "] " + file;
file = curDir + ((p.NSFW) ? "/NSFW" : "") + "/" + file;
if (File.Exists($"{file}{ext}"))
{
//logger.LogWarning("Skipping \"" + p.Title + "\", file already exsits", "/r/" + sub.Name, verbose);
continue;
}
//logger.Log("Saving: " + log, "/r/" + sub.Name, _verbose);
SaveImage(p, file, p.Url);
}
else if (p.Url.DnsSafeHost == "imgur.com") //Imgur in-direct link/album
{
string imgurID = Path.GetFileNameWithoutExtension(p.Url.AbsolutePath);
if (p.Url.AbsolutePath.Contains("/a/") || p.Url.AbsolutePath.Contains("/gallery/")) //Save Imgur Album
{
DownloadImgurAlbum(p.Url, file, p, curDir);
}
else
{
if (imgurID != "new") //Save Imgur in-drect link
{
file = "[" + p.CreatedUTC.ToEpoch() + "] " + file;
file = curDir + ((p.NSFW) ? "/NSFW" : "") + "/" + file;
try
{
string link = $"http://i.imgur.com/{imgurID}.png";
ext = Path.GetExtension(link);
if (File.Exists($"{file}{ext}"))
{
//logger.LogWarning("Skipping \"" + p.Title + "\", file already exsits", "/r/" + sub.Name, verbose);
continue;
}
//logger.Log("Saving: " + log, "/r/" + sub.Name, _verbose);
SaveImage(p, file, new Uri(link));
}
catch (Exception e)
{
LOGGER.LogWarning($"Unable to Download {p.Title}, {e.Message}", $"/r/{p.SubredditName}", verbose);
LOGGER.LogError(e.StackTrace, name, verbose);
}
}
}
}
file = "";
}
}
catch (Exception e)
{
LOGGER.LogWarning($"Failed to get posts: {e.Message}, retrying...", name, verbose);
postGet = false;
}
}
}
}
catch (Exception e)
{
LOGGER.LogError($"Crawl failed... {e.Message}, shutting down", name, verbose);
LOGGER.LogError(e.StackTrace, name, verbose);
Write("Crawl Ended");
isRunning = false;
}
if (!isRunning && _crawlLoop != null)
{
StopTimer("RedditCrawler");
}
if (!loop)
{
LOGGER.Log($"Finished Dowloading {imgCount} images... shutting down", name, verbose);
isRunning = false;
loop = true;
}
else
{
LOGGER.Log($"Dowloaded {imgCount} images...", name, verbose);
LOGGER.Log($"Sleeping for {updateRate}ms", name, verbose);
}
}, 0, updateRate);
}
//+ GUU INJECT
// Loads the main system of Guu
internal static void LoadGuu()
{
// Prevents the system from being loaded twice
if (isLoaded)
{
LOGGER.Log(new StackTrace().ToString());
return;
}
//& Removes Sentry SDK from the game
SentrySdk sentrySdk = Object.FindObjectOfType <SentrySdk>();
if (sentrySdk != null)
{
sentrySdk.Dsn = string.Empty;
sentrySdk.GetType().GetField("_instance", BindingFlags.NonPublic | BindingFlags.Static)?.SetValue(null, null);
sentrySdk.StopAllCoroutines();
Application.logMessageReceived -= sentrySdk.OnLogMessageReceived;
Object.Destroy(sentrySdk, 1);
LOGGER.Log("Game is modded! Disabling Sentry SDK");
}
//& Run required system functions
if (!CMD_ARGS.Contains("--guuLauncher") && !DEBUG)
{
Application.Quit();
}
// Generate the dynamic folders if they are not present
foreach (string folder in DYNAMIC_FOLDERS)
{
if (!Directory.Exists(folder))
{
Directory.CreateDirectory(folder);
}
}
//& Starts Eden before everything else
LogHandler.RegisterIfEmpty(LOGGER.Log, LOGGER.LogWarning, LOGGER.LogError, LOGGER.LogCritical);
EdenUnity.Entry.EdenUnity.Init();
// Add exceptions of types
EdenHarmony.RegisterMethodException("SteamDLCProvider", "*");
EdenHarmony.RegisterMethodException("EpicDLCProvider", "*");
EdenHarmony.RegisterMethodException("DLCProvider", "*");
EdenHarmony.RegisterMethodException("DLCDirector", "*");
// Add exceptions of methods
EdenHarmony.RegisterMethodException <GameContext>("Awake");
// Add Name Resolve
EdenHarmony.EnumNameResolve += EnumInjector.NameResolve;
//& Starts the loading process and times it
DateTime total = DateTime.Now;
LOGGER.Log(LOG_MSEPARATOR);
LOGGER.Log("GUU LOADING PROCESS STARTED");
LOGGER.Log(LOG_SEPARATOR);
//& Injects all assemblies, section is timed
DateTime section = DateTime.Now;
LOGGER.Log("[INJECTING ASSEMBLIES]");
core = LoadAssembly(CORE_ASSEM);
api = LoadAssembly(API_ASSEM);
save = LoadAssembly(SAVE_ASSEM);
world = LoadAssembly(WORLD_ASSEM);
devTools = LoadAssembly(DEV_TOOLS_ASSEM);
patches = LoadAssembly(PATCHES_ASSEM);
game = Assembly.Load("Assembly-CSharp");
// Load Bridge Assemblies
if (ExceptionUtils.IgnoreErrors(() => Assembly.Load(SRML_ASSEM)) != null)
{
srmlBridge = LoadAssembly(SRML_BRIDGE_ASSEM, "Found SRML! ");
Loader.ModLoader.srmlLoaderBridge = srmlBridge.ObtainTypeByForce("LoaderBridge");
}
LOGGER.Log($"[INJECTION COMPLETED] - {(DateTime.Now - section).TotalMilliseconds} ms");
LOGGER.Log(LOG_SEPARATOR);
//& Patches all assemblies, section is timed
section = DateTime.Now;
LOGGER.Log("[PATCHING ASSEMBLIES]");
harmony = new EdenHarmony("Guu");
if (patches != null)
{
PatchAssembly(patches);
}
if (srmlBridge != null)
{
PatchAssembly(srmlBridge);
}
LOGGER.Log($"[PATCHING COMPLETED] - {(DateTime.Now - section).TotalMilliseconds} ms");
LOGGER.Log(LOG_SEPARATOR);
//& Loads all asset packs, section is timed
section = DateTime.Now;
LOGGER.Log("[LOADING ASSET PACKS]");
guiPack = LoadPack(GUI_PACK);
LOGGER.Log($"[LOADING COMPLETED] - {(DateTime.Now - section).TotalMilliseconds} ms");
LOGGER.Log(LOG_SEPARATOR);
//& Loads all addon libraries
section = DateTime.Now;
LOGGER.Log("[LOADING ADDON LIBRARIES]");
bool hasAddons = false;
DirectoryInfo addons = new DirectoryInfo(LIBS_FOLDER);
foreach (FileInfo file in addons.GetFiles($"*{DLL_EXTENSION}"))
{
hasAddons = true;
ExceptionUtils.ThrowSuccessMessage(() =>
{
Assembly loadedLib = AssemblyUtils.LoadWithSymbols(file.FullName);
ADDON_ASSEMBLIES.Add(loadedLib);
Type mainType = loadedLib.GetTypes().Single(t => t.IsSubclassOf(typeof(IAddonLoad)));
if (mainType == null)
{
return;
}
IAddonLoad main = Activator.CreateInstance(mainType) as IAddonLoad;
main?.Initialize();
}, $"- Loaded '{file.Name}'");
}
if (!hasAddons)
{
LOGGER.Log("- No Addons were found");
}
LOGGER.Log($"[LOADING COMPLETED] - {(DateTime.Now - section).TotalMilliseconds} ms");
LOGGER.Log(LOG_SEPARATOR);
//& Initializes all internal services
section = DateTime.Now;
LOGGER.Log("[INITIALIZING INTERNAL SERVICES]");
GuuServices.CreateServiceObject();
GuuServices.InitInternalServices();
LOGGER.Log($"[INITIALIZATION COMPLETED] - {(DateTime.Now - section).TotalMilliseconds} ms");
LOGGER.Log(LOG_SEPARATOR);
//& Finalize the loading process
section = DateTime.Now;
LOGGER.Log("[FINALIZING]");
APIHandler.InitializeHandler();
LOGGER.Log($"[FINALIZING COMPLETED] - {(DateTime.Now - section).TotalMilliseconds} ms");
LOGGER.Log(LOG_SEPARATOR);
LOGGER.Log($"SYSTEM IS FULLY OPERATIONAL - {(DateTime.Now - total).TotalMilliseconds} ms");
LOGGER.Log(LOG_MSEPARATOR);
// Marks loading completed
isLoaded = true;
// Finalizes
LOGGER.Log("Starting the Mod Loader!");
InternalLogger.HandleThrow(Loader.ModLoader.GatherMods);
}
//Initialize
static Karuta()
{
//Prepare Console
try
{
Console.Title = "Karuta";
Console.BackgroundColor = ConsoleColor.DarkMagenta;
Console.ForegroundColor = ConsoleColor.White;
Console.Clear();
}catch (Exception e)
{
Write(e.StackTrace);
}
Write("Preparing Karuta...");
Stopwatch sw = new Stopwatch();
sw.Start();
//Init Vars
_threads = new Dictionary <string, Thread>();
_timers = new Dictionary <string, Timer>();
_interpretor = new CommandInterpreter <Command>();
_logger = new Logger();
_random = new Random();
//Init Event Manager
eventManager = new EventManager();
eventManager.Init();
//Init Registry
if (File.Exists($@"{DATA_DIR}/karuta.data"))
{
Write("Loading Registry...", false);
_registry = Registry.Load($@"{DATA_DIR}/karuta.data");
//_registry.Migrate();
Write(" Done!");
}
else
{
_registry = new Registry();
_registry.Init();
_registry.SetValue("user", "user");
}
try
{
//Register Commands
RegisterCommands();
LoadPlugins();
//Initalize commands
Write("Initializing processes...", false);
_interpretor.Init();
Write(" Done!");
}catch (Exception e)
{
Write($"An error occured while initializing commands: {e.Message}");
Write(e.StackTrace);
}
sw.Stop();
long elapsedT = sw.ElapsedTicks / (Stopwatch.Frequency / (1000L));
Write($"Karuta is ready. Finished in {elapsedT}ms");
LOGGER.Log($"Karuta started in {elapsedT}ms", "Karuta");
}
/// <summary>
/// 下载大单数据
/// </summary>
public void GetDaDanData()
{
///获取页码码数
///下载每一页数据
///查找最近一个交易日的数据是否存在,若存在则不下载
///
var mongo = DataStorage.GetInstance(DBType.MongoDB);
var webSource = DataSourceSINA.GetInstance();
var pageCount = 0;// webSource.GetDaDanPageCount();
if (0 == pageCount)
{
pageCount = int.Parse(WebDataSource.GetDataFromHttpAgent(string.Format("http://aifuwu.wang/API.ashx?c=WangJun.Stock.DataSourceSINA&m=GetDaDanPageCount")));
}
var dbName = CONST.DB.DBName_StockService;
var collectionName = CONST.DB.CollectionName_DaDan;
///数据检查
var lastTradingDate = Convertor.CalTradingDate(DateTime.Now, "00:00:00");
///删除根据日期创建的临时表格
///将数据写入表格
///将这个数据转移到大表中
#region 存储临时数据
var tempCollectionName = collectionName + string.Format("{0:yyyyMMdd}", DateTime.Now);
mongo.Remove(dbName, tempCollectionName, "{}");
#endregion
var methodName = "GetDaDanData";
var status = TaskStatusManager.Get(methodName);
var startIndex = (status.ContainsKey("PageIndex")) ? int.Parse(status["PageIndex"].ToString()) : 0;
var totalCount = 0;
for (int i = startIndex; i < pageCount; i++)
{
var html = "";// webSource.GetDaDan(i);
if (string.IsNullOrWhiteSpace(html))
{
html = WebDataSource.GetDataFromHttpAgent(string.Format("http://aifuwu.wang/API.ashx?c=WangJun.Stock.DataSourceSINA&m=GetDaDan&p={0}", i));
}
var res = NodeService.Get(CONST.NodeServiceUrl, "新浪", "GetDataFromHtml", new { ContentType = "SINA大单", Page = html });
if (res is Dictionary <string, object> )
{
var item = new
{
PageData = (res as Dictionary <string, object>)["PageData"],
ContentType = "SINA大单",
CreateTime = DateTime.Now,
TradingDate = Convertor.CalTradingDate(DateTime.Now, "00:00:00"),
PageIndex = i,
PageCount = pageCount,
};
LOGGER.Log(string.Format("SINA大单保存 {0} {1}", i, pageCount));
var arrayList = item.PageData as ArrayList;
///数据二维化，计算可以计算的，剩下的再想办法计算
for (int k = 0; k < arrayList.Count; k++)
{
totalCount++;
var arrItem = arrayList[k] as Dictionary <string, object>;
var svItem2D = new
{
StockCode = arrItem["StockCode"],
StockName = arrItem["StockName"],
TickTime = arrItem["交易时间"],
Price = arrItem["成交价"],
Turnover = arrItem["成交量"],
PrevPrice = arrItem["之前价格"],
Kind = arrItem["成交类型"],
TradingTime = Convertor.CalTradingDate(item.TradingDate, arrItem["交易时间"].ToString()),
TradingDate = item.TradingDate,
CreateTime = item.CreateTime,
PageIndex = i,
PageCount = pageCount,
RowIndex = k
};
mongo.Save3(dbName, tempCollectionName, svItem2D);
TaskStatusManager.Set(methodName, new { ID = methodName, PageIndex = i, PageCount = pageCount, Status = "已下载", CreateTime = DateTime.Now });
LOGGER.Log(string.Format("SINA大单2D保存 {0} {1} {2}", i, k, pageCount));
}
}
ThreadManager.Pause(seconds: 2);
}
TaskStatusManager.Set(methodName, new { ID = methodName, Status = "队列处理完毕", CreateTime = DateTime.Now });
var countInDB = mongo.Count(CONST.DB.DBName_StockService, tempCollectionName, "{}");
if (totalCount == countInDB)
{
LOGGER.Log(string.Format("准备转移数据"));
///数据检查是否完毕,完备后插入
//DataStorage.MoveCollection(mongo, CONST.DB.DBName_StockService, tempCollectionName, "{}", mongo, CONST.DB.DBName_StockService, CONST.DB.CollectionName_DaDan, false);
}
else
{
LOGGER.Log(string.Format("数据对不上 {0} {1}", totalCount, countInDB));
}
}
/// <summary>
/// 获取指定日期的新闻
/// </summary>
public void GetNewsListCJYW(string dateTime)
{
if (StringChecker.IsDateTime(dateTime))
{
var formatDate = string.Format("{0:yyyyMMdd}", Convert.ToDateTime(dateTime));
var mongo = DataStorage.GetInstance(DBType.MongoDB);
var webSrc = DataSourceTHS.CreateInstance();
var newNewsList = new Dictionary <string, object>();
var listHtml = webSrc.GetNewsListCJYW(Convert.ToDateTime(dateTime));
var dbName = CONST.DB.DBName_StockService;
var collectionNews = CONST.DB.CollectionName_News;
if (!string.IsNullOrWhiteSpace(listHtml))
{
var resDict = NodeService.Get(CONST.NodeServiceUrl, "同花顺", "GetDataFromHtml", new { ContentType = "THS财经要闻新闻列表", Page = listHtml }) as Dictionary <string, object>;
var resList = resDict["PageData"] as ArrayList;
if (null != resList)
{
foreach (Dictionary <string, object> listItem in resList)
{
var href = listItem["Href"].ToString().Trim();
var parentUrl = string.Format("http://news.10jqka.com.cn/today_list/{0}/", formatDate).Trim();
var newsHtml = webSrc.GetNewsArticle(href, parentUrl);
if (!string.IsNullOrWhiteSpace(newsHtml))
{
TaskStatusManager.Set("GetNewsListCJYW", new { ID = "GetNewsListCJYW", CreateTime = DateTime.Now, Status = "准备操作THS财经要闻新闻详细", Html = newsHtml });
var resItem = NodeService.Get(CONST.NodeServiceUrl, "同花顺", "GetDataFromHtml", new { ContentType = "THS财经要闻新闻详细", Page = newsHtml }) as Dictionary <string, object>;
var resDetail = resItem["PageData"] as Dictionary <string, object>;
if (null != resDetail)
{
var newsCreateTimeString = resDetail["CreateTime"].ToString().Trim();
var svItem = new
{
ContentType = "THS财经要闻",
Title = resDetail["Title"].ToString().Trim(),
Url = href,
ParentUrl = parentUrl,
SourceHref = resDetail["SourceHref"].ToString().Trim(),
SourceName = resDetail["SourceName"].ToString().Trim(),
NewsCreateTime = (StringChecker.IsDateTime(newsCreateTimeString))?Convert.ToDateTime(newsCreateTimeString):DateTime.MinValue,
Content = resDetail["Content"].ToString().Trim(),
Tag = Convert.ToInt32(formatDate),
CreateTime = DateTime.Now,
PageMD5 = "无"
};
newNewsList.Add(href, svItem);
LOGGER.Log(string.Format("获取一个新闻正文 {0}", svItem.Title));
ThreadManager.Pause(seconds: 5); ///5秒钟更新一次新闻
///分词
var task = Task.Factory.StartNew(() =>
{
var fcZStartTime = DateTime.Now;
this.SaveFenCi(svItem.Content, svItem.Url);
LOGGER.Log(string.Format("分词花费时间 开始时间：{0} 花费时间：{1}", fcZStartTime, DateTime.Now - fcZStartTime));
});
}
}
}
foreach (var svItem in newNewsList)
{
var filter = "{\"Url\":\"" + svItem.Key + "\"}";
mongo.Save3(dbName, collectionNews, svItem.Value, filter);
}
}
}
else
{
LOGGER.Log(string.Format("获取的新闻列表为空白"));
}
}
else
{
LOGGER.Log(string.Format("传入的时间字符串不对:{0}", dateTime));
ThreadManager.Pause(minutes: 5);
}
}
/// <summary>
/// THS龙虎榜
/// </summary>
public void SyncLHB()
{
var startTime = DateTime.Now;///开始运行时间
Console.Title = "THS龙虎榜 更新进程 启动时间：" + startTime;
var exe = StockTaskExecutor.CreateInstance();
var mongo = DataStorage.GetInstance(DBType.MongoDB);
var dbName = CONST.DB.DBName_StockService;
var collectionNameLHB = CONST.DB.CollectionName_LHB;
var collectionNameLHBMX = CONST.DB.CollectionName_LHBMX;
var methodName = "SyncLHB";
while (true)
{
var q = this.PrepareData(methodName);
while (0 < q.Count)
{
var stockCode = q.Dequeue();
var stockName = this.stockCodeDict[stockCode];
var listData = WebDataSource.GetInstance().GetLHB(stockCode);
var lhb = listData.First() as Dictionary <string, object>;
var lhbColumn = lhb["Column"] as Dictionary <string, object>;
var lhbData = lhb["Data"] as ArrayList;
var tradingDate = DateTime.MinValue;
foreach (Dictionary <string, object> itemLHB in lhbData)
{
tradingDate = DateTime.Parse(itemLHB["C1"].ToString());
var queryLHB = "{ \"StockCode\":\"" + stockCode + "\", \"TradingDate\":new Date('" + string.Format("{0:yyyy/MM/dd}", tradingDate) + "')}";
var svItem = new
{
StockCode = stockCode,
StockName = stockName,
TradingDate = tradingDate,
Data = itemLHB
};
mongo.Save3(dbName, collectionNameLHB, svItem, queryLHB);
}
listData.RemoveAt(0);
foreach (Dictionary <string, object> lhbmx in listData) ///第一个是龙虎榜,后面的是龙虎榜明细
{
var queryLHBMX = "{ \"StockCode\":\"" + stockCode + "\", \"TradingDate\":new Date('" + string.Format("{0:yyyy/MM/dd}", tradingDate) + "')}";
var svItem = new
{
StockCode = stockCode,
StockName = stockName,
TradingDate = tradingDate,
Data = lhbmx
};
mongo.Save3(dbName, collectionNameLHBMX, svItem, queryLHBMX);
}
TaskStatusManager.Set(methodName, new { ID = methodName, StockCode = stockCode, StockName = stockName, Status = "已下载", CreateTime = DateTime.Now });
LOGGER.Log(string.Format("本次THS龙虎榜更新完毕，下一次一天以后更新 {0}", DateTime.Now));
ThreadManager.Pause(days: 1);
}
TaskStatusManager.Set(methodName, new { ID = methodName, Status = "队列处理完毕", CreateTime = DateTime.Now });
}
}
/// <summary>
/// THS资金流向
/// </summary>
public void SyncZJLX()
{
var startTime = DateTime.Now;///开始运行时间
Console.Title = "THS资金流向 更新进程 启动时间：" + startTime;
var exe = StockTaskExecutor.CreateInstance();
var mongo = DataStorage.GetInstance(DBType.MongoDB);
var dbName = CONST.DB.DBName_StockService;
var collectionName = CONST.DB.CollectionName_ZJLX;
var methodName = "SyncZJLX";
while (true)
{
///获取所有股票代码,遍历更新数据
var q = this.PrepareData(methodName);
while (0 < q.Count)
{
var stockCode = q.Dequeue();
var resObj = WebDataSource.GetInstance().GetZJLX(stockCode);
var resList = (resObj is Dictionary <string, object>) ? (resObj as Dictionary <string, object>)["PageData"] : new ArrayList();
var stockName = this.stockCodeDict[stockCode];
///二维化
if (resList is ArrayList)
{
var rows = (resList as ArrayList);
foreach (Dictionary <string, object> row in rows)
{
var svItem2D = new
{
StockCode = stockCode,
StockName = stockName,
Url = string.Format("http://stockpage.10jqka.com.cn/{0}/funds/", stockCode),
CreateTime = DateTime.Now,
TradingDate = DateTime.Parse(row["日期"].ToString()),
Close = row["收盘价"],
Increase = row["涨跌幅"],
NetInflow = row["资金净流入"],
NetLarge5Day = row["5日主力净额"],
NetLarge = row["大单(主力)净额"],
NetLargeProportion = row["大单(主力)净占比"],
NetMedium = row["中单净额"],
NetMediumProportion = row["中单净占比"],
NetSmall = row["小单净额"],
NetSmallProportion = row["小单净占比"],
};
var filter = "{\"ContentType\":\"THS资金流向\",\"StockCode\":\"" + stockCode + "\",\"TradingDate\":new Date('" + string.Format("{0:yyyy/MM/dd}", svItem2D.TradingDate) + "')}";
mongo.Save3(dbName, collectionName, svItem2D, filter);
LOGGER.Log(string.Format("THS资金流向 正在更新 {0} {1} {2}", svItem2D.StockCode, svItem2D.StockName, svItem2D.TradingDate));
}
}
ThreadManager.Pause(seconds: 5);
TaskStatusManager.Set(methodName, new { ID = methodName, StockCode = stockCode, StockName = stockName, Status = "已下载", CreateTime = DateTime.Now });
///同步到SQL数据库中
}
TaskStatusManager.Set(methodName, new { ID = methodName, Status = "队列处理完毕", CreateTime = DateTime.Now });
ThreadManager.Pause(days: 1);
q = this.PrepareData();
}
}
/// <summary>
/// 同步日线数据
/// </summary>
public void SyncKLineDay()
{
var mongo = DataStorage.GetInstance(DBType.MongoDB);
var exe = WebDataSource.GetInstance();//
var dbName = CONST.DB.DBName_StockService;
var collectionName = CONST.DB.CollectionName_KLine;
var methodName = "SyncKLineDay";
while (true)
{
var q = this.PrepareData(methodName);
var year = DateTime.Now.Year;
var jidu = Convertor.GetJidu(DateTime.Now); ///获取当前季度数据
while (0 < q.Count)
{
var stockCode = q.Dequeue();
var stockName = this.stockCodeDict[stockCode];
var resDict = exe.GetSINAKLineDay(stockCode, year, jidu);
if (null != resDict && resDict.ContainsKey("Rows") && (resDict["Rows"] is ArrayList))
{
var rows = resDict["Rows"] as ArrayList;
if (rows is ArrayList)
{
for (var k = 0; k < rows.Count; k++)
{
var row = rows[k] as Dictionary <string, object>;
var prevClose = 0.0f;
if (1 <= k)
{
prevClose = Convert.ToSingle((rows[k - 1] as Dictionary <string, object>)["收盘价"]);
}
var svItem2D = new
{
StockCode = stockCode,
StockName = this.stockCodeDict[stockCode],
ContentType = "SINA历史交易",
Year = year,
CreateTime = DateTime.Now,
UpdateTime = DateTime.Now,
TradingDate = Convert.ToDateTime(row["日期"]),
Open = row["开盘价"],
High = row["最高价"],
Close = row["收盘价"],
Low = row["最低价"],
Volume = row["交易量(股)"],
Turnover = row["交易金额(元)"],
Amplitude = (k == 0) ? -1 : (Convert.ToSingle(row["最高价"]) - Convert.ToSingle(row["最低价"])) / prevClose, ///振幅
Increase = (k == 0) ? -1 : ((Convert.ToSingle(row["收盘价"]) - prevClose) / prevClose), ///涨幅
AveragePrice = Convert.ToSingle(row["交易金额(元)"]) / Convert.ToSingle(row["交易量(股)"]), ///成交均价
RowIndex = k,
RowCount = rows.Count
};
///状态保存
var filter = "{\"StockCode\":\"" + stockCode + "\",\"TradingDate\":new Date('" + string.Format("{0:yyyy/MM/dd}", svItem2D.TradingDate) + "')}";
mongo.Save3(dbName, collectionName, svItem2D, filter);
TaskStatusManager.Set(methodName, new { ID = methodName, StockCode = stockCode, StockName = svItem2D.StockName, Status = "已下载", CreateTime = DateTime.Now });
LOGGER.Log(string.Format("保存 {0} {1} SINA历史交易2D", stockCode, stockName));
}
}
}
ThreadManager.Pause(seconds: 5);
TaskStatusManager.Set(methodName, new { ID = methodName, Status = "队列处理完毕", CreateTime = DateTime.Now });
///本季度下载完毕之后再下上一季度数据。
if (0 == q.Count && jidu == Convertor.GetJidu(DateTime.Now) && 1 < jidu)//当年2,3,4季度 当季度
{
jidu = jidu - 1;
q = this.PrepareData(methodName);
}
else if (0 == q.Count && jidu == Convertor.GetJidu(DateTime.Now) && 1 == jidu)
{
jidu = 4; ///上一年第四季度
year = year - 1; ///上一年
q = this.PrepareData(methodName);
}
#region 补充计算缺失
#endregion
}
LOGGER.Log("历史交易获取完毕 ,下一次将在24小时后开始");
ThreadManager.Pause(days: 1);
}
}