private void GetDreamList(int targetUser, NianContext data)
{
int page = 1;
var userInfo = data.UserInfo.Single(v => v.UserId == targetUser);
for (page = 1;
page <= Math.Ceiling(
int.Parse(userInfo.Status["dream"]) / 20.0f);
page++)
{
var result = api.GetUserDreams(targetUser.ToString(), page);
foreach (var dreamStatus in (JArray)result["dreams"])
{
var dreamInfo = (JObject)dreamStatus;
var id = dreamInfo["id"].Value <int>();
var title = dreamInfo["title"].Value <string>();
var di = data.Dreams.SingleOrDefault(
info => info.DreamId == id);
if (di == null)
{
di = new Dream();
userInfo.Dreams.Add(di);
DiffDetected?.Invoke(
"http://nian.so/#!/dream/" + id,
uName + "新增了梦想" + title,
uName + "新增了梦想" + title,
"Nian." + targetUser + ".Dream." + id + ".Info");
di.Steps = new List <Step>();
}
foreach (var val in dreamInfo)
{
di.Status[val.Key] = val.Value.Value <string>();
}
}
}
foreach (var dataDream in userInfo.Dreams)
{
GetDreamExtendInfo(dataDream, data, currentPeroid % privatePeroid != 0);
}
}
/// <summary>
/// 对应数据项的增量抓取函数表
/// </summary>
private void GetDreamExtendInfo(Dream dream, NianContext context, bool Incremantal = true)
{
int lastupdated = 0;
{
string tmp;
dream.Status.TryGetValue("lastupdated", out tmp);
lastupdated = int.Parse(tmp ?? "0");
}
context.Entry(dream).Collection(d => d.Steps).Load();
int maxPage = 1;
//Origin Target
int unresolvedDiff = 0;
bool isAllResolved = false;
int pos = dream.Steps.FindLastIndex(info => info.IsRemoved == false);
bool isFoundHead = false;//是否找到最近出现的元素
List <Step> pendingInsert = new List <Step>();
int page;
for (page = 1; page <= maxPage; page++)
{
if (dream.Status.ContainsKey("private"))
{
if (dream.Status["private"] == "2")
{
continue; //被删除的记本
}
if (dream.Status["private"] == "1" && currentPeroid % privatePeroid != 0)
{
return;
}
//私密记本一天只能拉取1次
}
var result = api.GetDreamUpdate(dream.Status["id"], page);
//Get Extend Dream Info
var id = dream.Status["id"]; var title = dream.Status["title"];
if (page == 1)
{
var dreamInfo = (JObject)result["dream"];
foreach (var values in dreamInfo)
{
string valItem = "";
if (values.Value is JArray)
{
continue;
}
string targetValue = values.Value.Value <string>();
if (dream.Status.ContainsKey(values.Key))
{
valItem = dream.Status[values.Key];
}
if (values.Key == "step")
{
unresolvedDiff =
int.Parse(targetValue) - int.Parse("0" + valItem);
}
if (valItem != targetValue)
{
DiffDetected?.Invoke(
"http://nian.so/#!/dream/" + id,
uName + "修改了梦想" + title + "的信息",
"属性" + values.Key + "从" + valItem + "变为" + targetValue,
"Nian." + TargetUID + ".Dream." + id + ".Info." + values.Key);
dream.Status[values.Key] = targetValue;
}
}
maxPage = (int)Math.Ceiling(int.Parse(dream.Status["step"]) / 20.0f);
}
if (unresolvedDiff == 0 && Incremantal)//需要翻到下一页才能完全保证,因为有同时删除和增加的情况
{
if (isAllResolved)
{
break;
}
else
{
isAllResolved = true;
}
}
else
{
isAllResolved = false;
}
var stepsArray = (JArray)result["steps"];
foreach (var stepInfo in stepsArray)
{
Step si;
int unresolvedComment = 0;
var currentId = stepInfo["sid"].Value <int>();
int index = dream.Steps.FindIndex(item => item.StepId == currentId);
Step refsi = dream.Steps.ElementAtOrDefault(index);
if (index == -1)
{
if (isFoundHead)
{
throw new Exception("莫名的顺序,怀疑不对");
}
si = new Step
{
Comments = new List <Comment>()
};
pendingInsert.Add(si);
foreach (var val in (JObject)stepInfo)
{
if (val.Key == "images")
{
foreach (var imgPath in (JArray)val.Value)
{
si.Images.Add(imgPath["path"].Value <string>());
}
}
else
{
si.Status[val.Key] = val.Value.Value <string>();
}
}
DiffDetected?.Invoke(
"http://nian.so/m/step/" + currentId,
uName + "在记本" + title + "发布了一条足迹",
"足迹内容:" + si.Status["content"],
"Nian." + TargetUID + ".Dream." + id + ".Step." + currentId);
unresolvedComment = ((JObject)stepInfo)["comments"].Value <int>();
unresolvedDiff--;
si.Status["lastupdated"] = DateTime.Now.ToUnixTime().ToString();
lastupdated = (int)DateTime.Now.ToUnixTime();
}
else
{
isFoundHead = true;
if (refsi.IsRemoved == true)
{
refsi.IsRemoved = false; //修复之前因为转为private被误标记为已删除的内容
pos = index;
}
if (pos - index < 0)
{
throw new Exception("???WTF???");
}
for (int j = index + 1; j <= pos; j++)
{
if (!dream.Steps[j].IsRemoved)
{
dream.Steps[j].IsRemoved = true;
unresolvedDiff++;
DiffDetected?.Invoke(
"http://nian.so/m/dream/" + dream.DreamId,
uName + "删除了一条在" + title + "的足迹",
"足迹内容:" + dream.Steps[j].Status["content"],
"Nian." + TargetUID + ".Dream." + id + ".Step." + dream.Steps[j].StepId);
dream.Steps[j].Status["lastupdated"] = DateTime.Now.ToUnixTime().ToString();
lastupdated = (int)DateTime.Now.ToUnixTime();
}
}
pos = index - 1;
unresolvedComment =
int.Parse(refsi.Status["comments"]) -
((JObject)stepInfo)["comments"].Value <int>();
refsi.Status["comments"] = ((JObject)stepInfo)["comments"].Value <string>();
si = refsi;
}
//需要检测的只有comment
if (unresolvedComment != 0)
{
ResolveComment(si, context, unresolvedComment);
si.Status["lastupdated"] = DateTime.Now.ToUnixTime().ToString();
lastupdated = (int)DateTime.Now.ToUnixTime();
}
}
dream.Status["lastupdated"] = lastupdated.ToString();
}
page -= 1; //for跳出的那一轮需要减去
if ((page == maxPage || maxPage == 0) && pos >= 0) //最后一页时如果pos不等于0，说明有step被删除了
{
for (int j = pos; j >= 0; j--)
{
if (!dream.Steps[j].IsRemoved)
{
dream.Steps[j].IsRemoved = true;
unresolvedDiff++;
DiffDetected?.Invoke(
"http://nian.so/m/dream/" + dream.Status["id"],
uName + "删除了一条在" + dream.Status["title"] + "的足迹",
"足迹内容:" + dream.Steps[j].Status["content"],
"Nian." + TargetUID + ".Dream." + dream.Status["id"] + ".Step." + dream.Steps[j].Status["sid"]);
}
}
}
if (unresolvedDiff != 0)
{
Console.WriteLine("Still Have UnresolvedDiff!!!!" + " Offset:" + unresolvedDiff);
}
for (int j = pendingInsert.Count - 1; j >= 0; j--)
{
dream.Steps.Add(pendingInsert[j]);
}
}
private void ResolveComment(Step step, NianContext context, int unresolvedComments)
{
//不想写增量了!
//不过不用增量的方法怎么检测删除比较好呢?
//还是写增量逻辑吧Q Q
context.Entry(step).Collection(b => b.Comments);
bool isFoundHead = false;
List <Comment> pendingInsert = new List <Comment>();
bool isAllResolved = false;
int pos = step.Comments.FindLastIndex(info => info.IsRemoved == false);
int maxPage = (int)Math.Ceiling(int.Parse(step.Status["comments"]) / 15.0f);
string sid = step.StepId.ToString();
for (int page = 1; page <= maxPage; page++)
{
var result = api.GetComments(step.StepId.ToString(), page);
var commentArray = (JArray)result["comments"];
foreach (var comment in commentArray)
{
var cid = comment["id"].Value <int>();
int index = step.Comments.FindIndex(item => item.CommentId == cid);
if (index == -1)
{
if (isFoundHead)
{
throw new Exception("这里顺序不对吧");
}
//CreateNew
Comment cmt = new Comment
{
};
pendingInsert.Add(cmt);
foreach (var val in (JObject)comment)
{
cmt.Status[val.Key] = val.Value.Value <string>();
}
DiffDetected?.Invoke(
"http://nian.so/m/step/" + step.Status["sid"],
uName + "的足迹下出现了一条新评论!",
"足迹:" + step.Status["content"] + " 评论:" + cmt.Status["content"] + " - By " + cmt.Status["user"],
"Nian." + TargetUID + ".Dream." + step.Status["dream"] + ".Step." + sid + ".Comments");
unresolvedComments--;
}
else
{
isFoundHead = true;
if (pos - index < 0)
{
throw new Exception("??????WTF");
}
for (int j = index + 1; j <= pos; j++)
{
if (!step.Comments[j].IsRemoved)
{
step.Comments[j].IsRemoved = true;
unresolvedComments++;
DiffDetected?.Invoke(
"http://nian.so/m/step/" + step.Status["sid"],
uName + "的足迹下有一条评论被删除了!",
"足迹:" + step.Status["content"] + " 评论:" + step.Comments[j].Status["content"] + " - By " + step.Comments[j].Status["user"],
"Nian." + TargetUID + ".Dream." + step.Status["dream"] + ".Step." + sid + ".Comments");
}
}
pos = index - 1;
}
}
if (unresolvedComments == 0)
{
if (isAllResolved)
{
break;
}
else
{
isAllResolved = true;
}
}
else
{
isAllResolved = false;
}
}
for (int j = pendingInsert.Count - 1; j >= 0; j--)
{
step.Comments.Add(pendingInsert[j]);
}
}
static void Main(string[] args)
{
Console.WriteLine("Hello World!");
if (args.Length != 2)
{
return;
}
if (!File.Exists(args[0]))
{
return;
}
LiteDatabase db = new LiteDatabase(args[0]);
var oldData = db.GetCollection <Stalker.NianData>().FindOne(Query.All());
var newData = new STK.NianContext(args[1]);
newData.Database.EnsureCreated();
{
//UserInfo
var status = new STK.UserStatus();
status.Status.AddRange(oldData.ListItems);
status.UserId = Convert.ToInt32(oldData.ListItems["uid"]);
newData.UserInfo.Add(status);
}
{
//Dreams Steps Comments
foreach (var oldDataDream in oldData.Dreams)
{
var newDataDream = new STK.Dream();
newDataDream.DreamId = Convert.ToInt32(oldDataDream.Status["id"]);
newDataDream.Status.AddRange(oldDataDream.Status);
if (newDataDream.Steps == null)
{
newDataDream.Steps = new List <STK.Step>();
}
newDataDream.Steps.AddRange(oldDataDream.Steps.Select(steps =>
{
var newStep = new STK.Step();
newStep.StepId = Convert.ToInt32(steps.Status["sid"]);
newStep.Images.AddRange(steps.Images);
newStep.IsRemoved = steps.IsRemoved;
newStep.Status.AddRange(steps.Status);
newStep.Comments = new List <STK.Comment>(steps.Comments.Select(comment =>
{
STK.Comment newComment = new STK.Comment();
newComment.CommentId = Convert.ToInt32(comment.Status["id"]);
newComment.Status.AddRange(comment.Status);
return(newComment);
}));
return(newStep);
}));
newData.Dreams.Add(newDataDream);
}
}
newData.SaveChanges();
foreach (var steps in newData.Steps)
{
Console.WriteLine(steps.StepId);
}
}
