private static void ExtractNames(StreamWriter writer, string firstSecondName, string relativeLink)
{
var logPrefix = string.Format("[{0}][{1}]", firstSecondName[0], firstSecondName[1]);
Console.WriteLine("=======================================================================", logPrefix);
var namesUrl = "http://renlifang.msra.cn/" + relativeLink;
Console.WriteLine("{0} Loading names from {1}...", logPrefix, namesUrl);
WebRobot robot = new WebRobot();
var nameRawHtml = robot.Get(namesUrl);
var matches = Regex.Matches(nameRawHtml, NamePattern, RegexOptions.Compiled | RegexOptions.IgnoreCase);
foreach (Match match in matches)
{
var nameText = match.Groups["Name"].Value;
Match nameMatch = Regex.Match(nameText, @"^(?<Name>[^(]+)\((?<Count>.+)\)$", RegexOptions.Compiled);
if (nameMatch.Success)
{
var name = nameMatch.Groups["Name"].Value;
var count = nameMatch.Groups["Count"].Value;
Console.WriteLine("{0} Name: {1}, Count: {2}", logPrefix, name, count);
writer.WriteLine(string.Format(
"{0},{1},{2},{3}", firstSecondName[0], firstSecondName[1], name, count));
}
else
{
Console.WriteLine("{0} Invalid name: {1}", logPrefix, nameText);
}
}
}
public CheckinEntryProvider()
{
this.webRobot = new WebRobot();
webRobot.Referer = "http://zhaokaifang.com/";
webRobot.Host = "zhaokaifang.com";
webRobot.Origin = "http://zhaokaifang.com/";
webRobot.IsAjax = true;
}
public void TestRobot()
{
WebRobot robot = new WebRobot();
robot.IsAjax = true;
var response = robot.Post("http://zhaokaifang.com/q.php", "application/x-www-form-urlencoded", "name=&idcard=1");
Console.WriteLine(response);
}
private static void CrawlNames(StreamWriter writer, char firstNameChar)
{
WebRobot robot = new WebRobot();
var secondNameUrl = "http://renlifang.msra.cn/namelist.aspx?f=" + HttpUtility.UrlEncode(firstNameChar.ToString());
Console.WriteLine("[{0}] Loading from {1}...", firstNameChar, secondNameUrl);
var response = robot.Get(secondNameUrl);
// <div class="name-list"><a href="namelist.aspx?f=%e9%98%bf&s=%e9%98%bf">阿阿</a></div>
var matches = Regex.Matches(response, NamePattern, RegexOptions.Compiled | RegexOptions.IgnoreCase);
foreach (Match match in matches)
{
var firstSecondName = match.Groups["Name"].Value;
var relativeLink = match.Groups["Link"].Value;
if (firstSecondName.Length != 2)
{
Console.WriteLine("[{0}] Invalid FirstSecondName: {1}", firstSecondName + firstSecondName);
continue;
}
ExtractNames(writer, firstSecondName, relativeLink);
}
writer.Flush();
}
static void Main(string[] args)
{
if (args.Length != 4)
{
Console.WriteLine("Usage: TTWebClientRobot.exe WebApiAddress WebApiId WebApiKey WebApiSecret");
return;
}
string webApiAddress = args[0];
string webApiId = args[1];
string webApiKey = args[2];
string webApiSecret = args[3];
// Optional: Force to ignore server certificate
TickTraderWebClient.IgnoreServerCertificate();
// Create instance of the TickTrader Web API client
var client = new TickTraderWebClient(webApiAddress, webApiId, webApiKey, webApiSecret);
// Create and run WebRobot
var robot = new WebRobot(client);
robot.RunInConsole();
}
