public override BuildResults Crawl()
{
if (CosmosStream.StreamExists(base.URI))
{
string randomFilePath = this.PrepareFolderAndRandomFile();
for (int count = 0; count < 3; count++)
{
BuildResults result = CrawlInternal(randomFilePath);
switch (result)
{
case BuildResults.Crawler_Succeed:
FileHelper.MoveFile(randomFilePath, this.LocalPath);
return(result);
case BuildResults.Crawler_NoNewFileFound:
return(result);
default:
System.Threading.Thread.Sleep(1000);
break;
}
}
return(BuildResults.Crawler_UnexpectedError);
}
else
{
return(BuildResults.Crawler_CannotCopyRemoteFile);
}
}
/// <summary>
/// copy file from remote share folder to local download folder with random file name
/// </summary>
/// <param name="uri"></param>
/// <param name="filename"></param>
/// <param name="localVersion"></param>
/// <param name="remoteVersion"></param>
/// <returns></returns>
private BuildResults CrawlInternal(string filename)
{
try
{
this.RemoteVersion = CosmosStream.GetLastModifiedTime(base.URI);
if (this.RemoteVersion <= this.LocalVersion)
{
return(BuildResults.Crawler_NoNewFileFound);
}
else
{
CosmosStream.DownloadStream(base.URI, filename);
return(BuildResults.Crawler_Succeed);
}
}
catch
{
return(BuildResults.Crawler_UnexpectedError);
}
}
public void ProcessImage(string imageUrlListFile, string sourceTargetUrlMappingFile)
{
this.WriteLog(EventType.Critical, "[Start] | [{0}]", "ProcessImage");
string logPath = Path.Combine(this.WorkingFolder, "prolog.txt");//ImageWorkingFile.ImageProcessLogFile);
TraceLog log = new TraceLog("ImageProcess", logPath);
string imageUrlListFilePath = Path.Combine(this.WorkingFolder, imageUrlListFile);
string imageDownloadFolder = Path.Combine(this.WorkingFolder, "Download");
if (!Directory.Exists(imageDownloadFolder))
{
Directory.CreateDirectory(imageDownloadFolder);
}
string imageScaleFolder = Path.Combine(this.WorkingFolder, "Scale");
if (!Directory.Exists(imageScaleFolder))
{
Directory.CreateDirectory(imageScaleFolder);
}
string imageUrlMappingFile = Path.Combine(this.WorkingFolder, sourceTargetUrlMappingFile);
string serverFile = CosmosStream.BuildUrlPath("https://cosmos08.osdinfra.net/cosmos/bingads.marketplace.VC2/local/RAACenter/Data/SnRAA/", sourceTargetUrlMappingFile);
if (CosmosStream.StreamExists(serverFile))
{
CosmosStream.DownloadStream(serverFile, imageUrlMappingFile);
}
int oriImageCount = 0;
if (File.Exists(imageUrlMappingFile))
{
List <string> lines = File.ReadAllLines(imageUrlMappingFile).ToList();
oriImageCount = lines.Where(l => !string.IsNullOrEmpty(l)).Count();
}
bool bSuccess = ImageProcessor.ProcessImage(imageUrlListFilePath, imageDownloadFolder, imageScaleFolder, imageUrlMappingFile, serverFile, log, this.feedWriteLog);
if (bSuccess)
{
this.Result = BuildResults.Image_Succeed;
this.WriteLog(EventType.Critical, "[{0}] Image succeeded.", "Process");
}
else
{
this.Result = BuildResults.Image_ProcessImageFailed;
this.WriteLog(EventType.Critical, "[{0}] Image failed.", "Process");
//this.addAttachFile(ImageLibrary.ImageWorkingFile.ImageProcessLogFile);
//this.WriteLog(EventType.Error, "Please refer the image log [{0}] for more details", ImageLibrary.ImageWorkingFile.ImageProcessLogFile);
}
int newImageCount = 0;
if (File.Exists(imageUrlMappingFile))
{
List <string> lines = File.ReadAllLines(imageUrlMappingFile).ToList();
newImageCount = lines.Where(l => !string.IsNullOrEmpty(l)).Count();
}
this.WriteLog(EventType.Critical, "[ProcessedImageCount] | [{0}]", newImageCount - oriImageCount);
log.Close();
log = null;
this.WriteLog(EventType.Critical, "[End] | [{0}]", "ProcessImage");
}
