// 用依然可以使用的Home_Timeline获取最新的一大串微博
public static void fetchUserStatusInFile(string listPath, string toPath, int topMax, bool followFirst = false)
{
// 关注所有的list
Crawler crawler = new Crawler();
if (followFirst) crawler.followUserByUserList(listPath);
// 找到用户最新的微博ID
long max = long.MinValue;
Serializer rr = new Serializer(false);
if (File.Exists(toPath))
{
rr.ReadStream(toPath);
Status ts = (Status)rr.ReadNextObject();
while (ts != null)
{
if (max < long.Parse(ts.ID)) max = long.Parse(ts.ID);
ts = (Status)rr.ReadNextObject();
}
rr.CloseStream();
}
rr.WriteStream(toPath);
int count = 0, page = 1, numOfRecord = 0;
while(count < topMax){
int need = Math.Min(topMax - count, 100);
Console.WriteLine(need);
List<Status> ls = crawler.getUserStatusByHome(need, page);
Console.WriteLine(need + " " + page);
foreach (Status s in ls)
{
// 如果是最新的就写入到记录文件中
if (max < long.Parse(s.ID))
{
rr.WriteObject(s);
Console.WriteLine(s.ID);
numOfRecord++;
}
}
count += need;
page++;
}
rr.CloseStream();
Console.WriteLine(numOfRecord + " recorded");
}
public static void fetchUserStatus(string userName, int topMax)
{
// 找到用户名对应的文件
string filePath = pathUserList;
// 找到用户最新的微博ID
long max = long.MinValue;
Serializer rr = new Serializer(false);
if(File.Exists(filePath))
{
rr.ReadStream(filePath);
Status ts = (Status)rr.ReadNextObject();
while (ts != null)
{
if (max < long.Parse(ts.ID)) max = long.Parse(ts.ID);
ts = (Status)rr.ReadNextObject();
}
rr.CloseStream();
}
// 抓用户最新的微博
Crawler cr = new Crawler();
List<Status> ls = cr.getUserStatusByUserName(userName, topMax, 1);
rr.WriteStream(filePath);
foreach (Status s in ls)
{
// 如果是最新的就写入到记录文件中
if (max < long.Parse(s.ID))
{
rr.WriteObject(s);
Console.WriteLine(s.ID);
}
}
rr.CloseStream();
}
public static void Main()
{
Console.WriteLine("=== START ===");
if (unfollow)
{
Crawler c = new Crawler();
//c.unfollowUserByUserList("../_Data/UserList_BigV.txt");
//c.unfollowUserByUserList("../_Data/UserList_Normal.txt");
//c.unfollowUserByUserList("../_Data/UserList_Zombie.txt");
//c.unfollowUserByUserList("../_Data/UserList_Spammer.txt");
}
fetchUserStatusInFile(pathUserList, pathDatabase, numberOfStatus, follow);
Console.WriteLine("=== END ===");
Console.ReadKey();
}
public static void keepWatchingPhotos(string userName, int watchLength)
{
Crawler cr = new Crawler();
int i = 0;
// 每小时限制150次API，每24秒一次call
while (true)
{
Console.WriteLine(DateTime.Now + " Watching..." + ++i);
List<Status> ls = cr.getUserStatusByUserName(userName, watchLength);
foreach (Status s in ls) Downloader.getStatusPictures(s, "../_Data/Pictures");
Thread.Sleep(24000);
}
}
