public E_ERROR_CODE downloadIOSData(DateTime startDate, DateTime endDate)
{
E_ERROR_CODE errorCode = E_ERROR_CODE.ERROR;
string message = string.Empty;
string queryDataUrl = string.Empty;
string dataFile = string.Empty;
string iosDataStorePath = _AppsFlyerDataSpiderConfig.getIOSDataStorePath();
string[] iosApplicationIds = _AppsFlyerDataSpiderConfig.getIOSApplicationIds();
if (iosDataStorePath == null)
{
_SystemLog.writeLog2Console(LOG_LEVEL.ERR, "Get android data file store path failed.");
return(E_ERROR_CODE.ERROR_GET_ANDROID_DATA_FILE_STORE_PATH_FAILED);
}
foreach (string androidApplicationId in iosApplicationIds)
{
queryDataUrl = _AppsFlyerDataSpiderConfig.getQueryDataUrl(startDate, endDate, androidApplicationId);
dataFile = string.Format("{0}/{1}_{2}.dat", iosDataStorePath, CPublic.getDateString(startDate), CPublic.getDateString(endDate));
message = string.Format("Query data url <{0}>, dataFile <{1}>.", queryDataUrl, dataFile);
_SystemLog.writeLog2Console(LOG_LEVEL.DEBUG, message);
errorCode = downloadData(queryDataUrl, _WebDriver, dataFile);
if (errorCode != E_ERROR_CODE.OK)
{
message = string.Format("ios application <{0}> download data failed, error code <{1}>.", androidApplicationId, errorCode);
_SystemLog.writeLog2Console(LOG_LEVEL.ERR, message);
}
}
return(E_ERROR_CODE.OK);
}
static int Main(string[] args)
{
string message = string.Empty;
_SystemConfig = new CSystemConfig();
if (_SystemConfig.getErrorCode() != 0)
{
return(1);
}
E_ERROR_CODE errorCode = loadConfigData();
if (errorCode != E_ERROR_CODE.OK)
{
return((int)errorCode);
}
_DbConnection = new CDbConnection(_DbConnectStr, _SystemLog);
if (!_DbConnection.testConnect())
{
_SystemLog.disposeLogWriter();
return((int)E_ERROR_CODE.ERROR_CONNECT_DB_FAILED);
}
initLocalVaribles();
while (true)
{
_BeiDouDeviceManager.processBeiDevice();
message = string.Format("Run cycle <{0}>, Next run time <{1}>....", _RequestCycle, CPublic.getDateTimeString(DateTime.Now.AddSeconds(_RequestCycle)));
_SystemLog.writeLog(LOG_LEVEL.WARNING, message);
Thread.Sleep(TimeSpan.FromSeconds(_RequestCycle));
}
}
public E_ERROR_CODE initAppsFlyerSpiderConfig()
{
E_ERROR_CODE errorCode = E_ERROR_CODE.ERROR;
errorCode = _AppsFlyerDataSpiderConfig.initConfig();
if (errorCode != E_ERROR_CODE.OK)
{
return(errorCode);
}
_SystemLog = new CSystemLog(_AppsFlyerDataSpiderConfig.getIsWriteLog2Screen(), _AppsFlyerDataSpiderConfig.getLogLevel(), _AppsFlyerDataSpiderConfig.getLogPath());
return(E_ERROR_CODE.OK);
}
private E_ERROR_CODE writeDataToFile(string responseContent, string dataFileName)
{
string message = string.Empty;
E_ERROR_CODE errorCode = E_ERROR_CODE.ERROR;
List <List <object> > recordTable = new List <List <object> >();
StreamWriter dataStreamWriter = null;
errorCode = convertFormat(recordTable, responseContent);
if (errorCode != E_ERROR_CODE.OK)
{
message = string.Format("Covert json data to record failed <{0}>.", dataFileName);
_SystemLog.writeLog2Console(LOG_LEVEL.ERR, message);
return(E_ERROR_CODE.ERROR_OPEN_DATA_FILE_FAILED);
}
if (recordTable.Count == 0)
{
message = string.Format("No data need to save failed, <{0}>.", dataFileName);
_SystemLog.writeLog2Console(LOG_LEVEL.ERR, message);
return(E_ERROR_CODE.OK);
}
try
{
dataStreamWriter = new StreamWriter(dataFileName, false, Encoding.UTF8);
dataStreamWriter.AutoFlush = true;
foreach (List <object> record in recordTable)
{
dataStreamWriter.WriteLine(String.Join(",", record));
}
return(E_ERROR_CODE.OK);
}
catch (Exception ex)
{
message = string.Format("Open data stream writer failed <{0}>,error message<{1}>.", dataFileName, ex.Message);
_SystemLog.writeLog2Console(LOG_LEVEL.ERR, message);
return(E_ERROR_CODE.ERROR_OPEN_DATA_FILE_FAILED);
}
finally
{
if (dataStreamWriter != null)
{
dataStreamWriter.Flush();
dataStreamWriter.Close();
dataStreamWriter.Dispose();
dataStreamWriter = null;
}
}
}
public E_ERROR_CODE initConfig()
{
E_ERROR_CODE errorCode = E_ERROR_CODE.ERROR;
_Config = new CSystemConfig(_ConfigFile, _SystemLog);
errorCode = _Config.initConfig();
if (errorCode != E_ERROR_CODE.OK)
{
_Config = null;
return(errorCode);
}
return(errorCode);
}
static int Main(string[] args)
{
E_ERROR_CODE errorCode = E_ERROR_CODE.ERROR;
_SystemLog = new CSystemLog(true, LOG_LEVEL.DEBUG, string.Format("{0}/{1}", CPublic.getAppStartPath(), "system_log"));
_SystemConfig = new CSystemConfig(CPublic.SYSTEM_CONFIG_FILE, _SystemLog);
errorCode = _SystemConfig.initConfig();
if (errorCode != E_ERROR_CODE.OK)
{
_SystemLog.writeLog2Console(LOG_LEVEL.CRIT, string.Format("Init system config <{0}> failed, error code <{1}>.", CPublic.SYSTEM_CONFIG_FILE, errorCode));
return((int)errorCode);
}
_SystemConfig.printSystemConfig();
_AppsflyerConfigFilePath = string.Format("{0}", _SystemConfig.getConfigItem(CONFIG_ITEM_APPSFLYER_CONFIG_FILE)).Trim();
if (!File.Exists(_AppsflyerConfigFilePath))
{
_SystemLog.writeLog2Console(LOG_LEVEL.CRIT, string.Format("Appsflyer config file <{0}> not exits.", _AppsflyerConfigFilePath));
return((int)E_ERROR_CODE.ERROR_FILE_NOT_EXIST);
}
CAdSeeDataSpider appsFlyerDataSpider = new CAdSeeDataSpider(_AppsflyerConfigFilePath, _SystemLog);
errorCode = appsFlyerDataSpider.initAppsFlyerSpiderConfig();
if (errorCode != E_ERROR_CODE.OK)
{
appsFlyerDataSpider.cleanUp();
_SystemLog.writeLog2Console(LOG_LEVEL.CRIT, string.Format("Init appsflyer spider config failed, error code <{0}>.", errorCode));
return((int)errorCode);
}
errorCode = appsFlyerDataSpider.initWebDriver();
if (errorCode != E_ERROR_CODE.OK)
{
appsFlyerDataSpider.cleanUp();
_SystemLog.writeLog2Console(LOG_LEVEL.CRIT, string.Format("Init webdriver failed, error code <{0}>.", errorCode));
return((int)errorCode);
}
errorCode = appsFlyerDataSpider.openHomePage();
if ((errorCode != E_ERROR_CODE.OK) || (!appsFlyerDataSpider.IsLoginSuccessed))
{
_SystemLog.writeLog2Console(LOG_LEVEL.ERR, string.Format("Login appsflyer system failed, after <{0}> login again.", 60));
return((int)errorCode);
}
errorCode = appsFlyerDataSpider.downloadAndroidAppAdData();
return((int)errorCode);
}
private E_ERROR_CODE downloadAndroidAdsetPerformanceReportData(DateTime currentTime)
{
E_ERROR_CODE errorCode = E_ERROR_CODE.ERROR;
DateTime queryDate = currentTime;
int days = _AppsFlyerDataSpiderConfig.getAppsflyerDelayDay();
for (int index = 0; index < days; ++index)
{
queryDate = currentTime.Subtract(TimeSpan.FromDays(1));
errorCode = downloadAndroidAdsetPerformanceReportDataByDate(queryDate);
if (errorCode != E_ERROR_CODE.OK)
{
_SystemLog.writeLog2Console(LOG_LEVEL.ERR, string.Format("Download android adset performance report data for date <{0}> failed, error code <{1}>.", queryDate.ToString("yyyy-MM-dd HH:mm"), errorCode));
}
}
return(errorCode);
}
internal E_ERROR_CODE downloadAndroidPerformanceReportData(DateTime currentTime)
{
E_ERROR_CODE errorCode = E_ERROR_CODE.ERROR;
errorCode = downloadAndroidCampaignPerformanceReportData(currentTime);
if (errorCode != E_ERROR_CODE.OK)
{
_SystemLog.writeLog2Console(LOG_LEVEL.ERR, string.Format("Download android campaing performance report data <{0}> failed.", currentTime.ToString("yyyy-MM-dd HH:mm")));
}
errorCode = downloadAndroidAdsetPerformanceReportData(currentTime);
if (errorCode != E_ERROR_CODE.OK)
{
_SystemLog.writeLog2Console(LOG_LEVEL.ERR, string.Format("Download android adset performance report data <{0}> failed.", currentTime.ToString("yyyy-MM-dd HH:mm")));
}
return(errorCode);
}
public E_ERROR_CODE loginSystem()
{
E_ERROR_CODE errorCode = E_ERROR_CODE.ERROR;
string appsflyerHomePageUrl = _AppsFlyerDataSpiderConfig.getLoginPageUrl();
errorCode = openHomePage(appsflyerHomePageUrl);
if (errorCode != E_ERROR_CODE.OK)
{
return(errorCode);
}
string userName = _AppsFlyerDataSpiderConfig.getUserName();
string password = _AppsFlyerDataSpiderConfig.getPassword();
if ((userName == null) || (password == null))
{
_SystemLog.writeLog2Console(LOG_LEVEL.CRIT, string.Format("UserName <{0}> or password <{1}> invalid.", userName, password));
return(E_ERROR_CODE.ERROR_USERNAME_OR_PASSWORD_IS_NULL);
}
return(loginAppsflyerSystem(userName, password));
}
public E_ERROR_CODE processBeiDevice()
{
string message = string.Empty;
E_ERROR_CODE errorCode = E_ERROR_CODE.ERROR;
errorCode = loadBeiDeviceFromDb();
if (errorCode != E_ERROR_CODE.OK)
{
return(errorCode);
}
errorCode = queryBeiDouDeviceFromRemoteServer();
if (errorCode != E_ERROR_CODE.OK)
{
return(errorCode);
}
findInsertDevice();
findDeleteDevice();
findUpdateDevice();
errorCode = insertDeviceToDb();
if (errorCode != E_ERROR_CODE.OK)
{
message = "Insert devices to db faild.";
_SystemLog.writeLog(LOG_LEVEL.ERR, message);
}
errorCode = updateDeviceToDb();
if (errorCode != E_ERROR_CODE.OK)
{
message = "Update devices to db faild.";
_SystemLog.writeLog(LOG_LEVEL.ERR, message);
}
return(errorCode);
}
static int Main(string[] args)
{
E_ERROR_CODE errorCode = E_ERROR_CODE.ERROR;
_SystemLog = new CSystemLog(true, LOG_LEVEL.DEBUG, string.Format("{0}/{1}", CPublic.getAppStartPath(), "system_log"));
_SystemConfig = new CSystemConfig(CPublic.SYSTEM_CONFIG_FILE, _SystemLog);
errorCode = _SystemConfig.initConfig();
if (errorCode != E_ERROR_CODE.OK)
{
_SystemLog.writeLog2Console(LOG_LEVEL.CRIT, string.Format("Init system config <{0}> failed, error code <{1}>.", CPublic.SYSTEM_CONFIG_FILE, errorCode));
return((int)errorCode);
}
_SystemConfig.printSystemConfig();
_AppsflyerConfigFilePath = string.Format("{0}", _SystemConfig.getConfigItem(CONFIG_ITEM_APPSFLYER_CONFIG_FILE)).Trim();
if (!File.Exists(_AppsflyerConfigFilePath))
{
_SystemLog.writeLog2Console(LOG_LEVEL.CRIT, string.Format("Appsflyer config file <{0}> not exits.", _AppsflyerConfigFilePath));
return((int)E_ERROR_CODE.ERROR_FILE_NOT_EXIST);
}
CAppsFlyerDataSpider appsFlyerDataSpider = new CAppsFlyerDataSpider(_AppsflyerConfigFilePath, _SystemLog);
errorCode = appsFlyerDataSpider.initAppsFlyerSpiderConfig();
if (errorCode != E_ERROR_CODE.OK)
{
appsFlyerDataSpider.cleanUp();
_SystemLog.writeLog2Console(LOG_LEVEL.CRIT, string.Format("Init appsflyer spider config failed, error code <{0}>.", errorCode));
return((int)errorCode);
}
errorCode = appsFlyerDataSpider.initWebDriver();
if (errorCode != E_ERROR_CODE.OK)
{
appsFlyerDataSpider.cleanUp();
_SystemLog.writeLog2Console(LOG_LEVEL.CRIT, string.Format("Init webdriver failed, error code <{0}>.", errorCode));
return((int)errorCode);
}
int retentionPeriod = appsFlyerDataSpider.AppsFlyerDataSpiderConfig.getRetentionPeriod();
int appsflyerDelayDay = appsFlyerDataSpider.AppsFlyerDataSpiderConfig.getAppsflyerDelayDay();
int queryCycle = appsFlyerDataSpider.AppsFlyerDataSpiderConfig.getDownloadDataCycle();
while (true)
{
_CurrentTime = DateTime.Now;
_LastTime = _CurrentTime;
errorCode = appsFlyerDataSpider.loginSystem();
if ((errorCode != E_ERROR_CODE.OK) || (!appsFlyerDataSpider.IsLoginSuccessed))
{
_SystemLog.writeLog2Console(LOG_LEVEL.ERR, string.Format("Login appsflyer system failed, after <{0}> login again.", 60));
Thread.Sleep(TimeSpan.FromSeconds(30));
}
/* 每8小时取一个留存数据 */
if (_CurrentTime.Subtract(_LastTime).Hours >= queryCycle)
{
DateTime endDate = _CurrentTime.Date.Subtract(TimeSpan.FromDays(appsflyerDelayDay));
DateTime startDate = endDate.Subtract(TimeSpan.FromDays((retentionPeriod - 1)));
errorCode = appsFlyerDataSpider.downloadAndroidData(startDate, endDate);
if (errorCode != E_ERROR_CODE.OK)
{
_SystemLog.writeLog2Console(LOG_LEVEL.ERR, string.Format("Download android data <{0},{1}> failed .", CPublic.getDateString(startDate), CPublic.getDateString(endDate)));
}
errorCode = appsFlyerDataSpider.downloadIOSData(startDate, endDate);
if (errorCode != E_ERROR_CODE.OK)
{
_SystemLog.writeLog2Console(LOG_LEVEL.ERR, string.Format("Download ios data <{0},{1}> failed .", CPublic.getDateString(startDate), CPublic.getDateString(endDate)));
}
}
/* AppsFlyer汇总绩效报告 Performance report */
if (_CurrentTime.Hour != _LastTime.Hour)
{
errorCode = appsFlyerDataSpider.downloadAndroidPerformanceReportData(_CurrentTime);
if (errorCode != E_ERROR_CODE.OK)
{
_SystemLog.writeLog2Console(LOG_LEVEL.ERR, string.Format("Download android performance report data <{0}> failed .", CPublic.getDateString(_CurrentTime)));
}
errorCode = appsFlyerDataSpider.downloadIOSPerformanceReportData(_CurrentTime);
if (errorCode != E_ERROR_CODE.OK)
{
_SystemLog.writeLog2Console(LOG_LEVEL.ERR, string.Format("Download ios performance report data <{0}> failed .", CPublic.getDateString(_CurrentTime)));
}
}
_LastTime = _CurrentTime;
_SystemLog.writeLog2Console(LOG_LEVEL.INFO, string.Format("Query cycle <{0}>, Current time <{1}>, Next download data at <{2}> .", queryCycle, CPublic.getDateTimeString(_CurrentTime), CPublic.getDateTimeString(_CurrentTime.AddSeconds(queryCycle))));
Thread.Sleep(TimeSpan.FromSeconds(30 * 60));
}
}
