private void bgWorker_DoWork(object sender, DoWorkEventArgs e)
{
try
{
CustomWeb web = new CustomWeb();
bool loginResult = Login(email, pass, web);
if (loginResult)
{
int pageSize = 100;
int nextIndex = crawlerInfo.CurrentIndex;
int counter = 1;
string rootPath = Directory.GetCurrentDirectory() + "\\Links\\";
if (!Directory.Exists(rootPath))
{
Directory.CreateDirectory(rootPath);
}
List <Sunfrog> totalList = new List <Sunfrog>();
do
{
List <Sunfrog> dataList = new List <Sunfrog>();
bool hasNext = GetNextLinks(web, ref dataList, nextIndex);
if (dataList.Count > 0)
{
totalList.AddRange(dataList);
}
DateTime now = DateTime.Now;
UpdateText(now.ToString("dd/MM/yyyy HH:mm:ss") + " : current file size: " + totalList.Count);
if (hasNext)
{
nextIndex += pageSize;
crawlerInfo.CurrentIndex = nextIndex;
//SaveSetting();
if (totalList.Count >= Settings.Default.SIZE)
{
CsvUtil.WriteObjectsToCSV(totalList, rootPath, $"links_{counter++}.csv");
totalList.Clear();
SaveSetting();
UpdateText("clr");
}
Thread.Sleep(1000 * Settings.Default.DELAY);
}
else
{
if (totalList.Count > 0)
{
// string desPath = rootPath + $"links_{startIndex}.txt";
CsvUtil.WriteObjectsToCSV(totalList, rootPath, $"links_{counter++}.csv");
totalList.Clear();
SaveSetting();
}
break;
}
} while (true);
}
else
{
MessageBox.Show("Login failed!");
}
}
catch (Exception ex)
{
logger.ErrorFormat($"Error: {ex.Message}, Stack Track: {ex.StackTrace}");
}
}