public Crawler Fetch()
{
if (!IsExternalIPAddress(this.SourceUrl))
{
State = "INVALID_URL";
return(this);
}
var request = HttpWebRequest.Create(this.SourceUrl) as HttpWebRequest;
using (var response = request.GetResponseAsync().Result as HttpWebResponse)
{
if (response.StatusCode != HttpStatusCode.OK)
{
State = "Url returns " + response.StatusCode + ", " + response.StatusDescription;
return(this);
}
if (response.ContentType.IndexOf("image") == -1)
{
State = "Url is not an image";
return(this);
}
ServerUrl = PathFormatter.Format(Path.GetFileName(this.SourceUrl), Config.GetString("catcherPathFormat"));
//var savePath = Path.Combine(Config.WebRootPath, ServerUrl);
var savePath = Path.Combine(Config.GetString("catcherSaveAbsolutePath"), ServerUrl);
//
try
{
var stream = response.GetResponseStream();
//
if (Config.GetValue <bool>("catcherFtpUpload"))
{
var fileExt = ServerUrl.Substring(savePath.LastIndexOf('.')).TrimStart('.');
var key = Config.GetString("catcherPathFormat") + "." + fileExt;
//FtpUpload.UploadFile(stream, savePath, Consts.ImgFtpServer.ip,
// Consts.ImgFtpServer.account, Consts.ImgFtpServer.pwd);
AliyunOssUpload.UploadFile(Consts.AliyunOssServer.AccessEndpoint,
Consts.AliyunOssServer.AccessKeyId, Consts.AliyunOssServer.AccessKeySecret,
Consts.AliyunOssServer.BucketName, key, fileExt, stream);
}
else
{
if (!Directory.Exists(Path.GetDirectoryName(savePath)))
{
Directory.CreateDirectory(Path.GetDirectoryName(savePath));
}
var reader = new BinaryReader(stream);
byte[] bytes;
using (var ms = new MemoryStream())
{
byte[] buffer = new byte[4096];
int count;
while ((count = reader.Read(buffer, 0, buffer.Length)) != 0)
{
ms.Write(buffer, 0, count);
}
bytes = ms.ToArray();
}
File.WriteAllBytes(savePath, bytes);
}
State = "SUCCESS";
}
catch (Exception e)
{
State = "抓取错误：" + e.Message;
}
return(this);
}
}
public override void Process()
{
byte[] uploadFileBytes = null;
string uploadFileName = null;
Stream fileStream = null;
if (UploadConfig.Base64)
{
uploadFileName = UploadConfig.Base64Filename;
uploadFileBytes = Convert.FromBase64String(Request.Form[UploadConfig.UploadFieldName]);
}
else
{
var file = Request.Form.Files[UploadConfig.UploadFieldName];
uploadFileName = file.FileName;
if (!CheckFileType(uploadFileName))
{
Result.State = UploadState.TypeNotAllow;
WriteResult();
return;
}
if (!CheckFileSize((int)file.Length))
{
Result.State = UploadState.SizeLimitExceed;
WriteResult();
return;
}
uploadFileBytes = new byte[file.Length];
try
{
fileStream = file.OpenReadStream();
file.OpenReadStream().Read(uploadFileBytes, 0, (int)file.Length);
}
catch (Exception)
{
Result.State = UploadState.NetworkError;
WriteResult();
}
}
Result.OriginFileName = uploadFileName;
var savePath = PathFormatter.Format(uploadFileName, UploadConfig.PathFormat);
var localPath = "";
if (string.IsNullOrEmpty(UploadConfig.SaveAbsolutePath))
{
localPath = Path.Combine(Config.WebRootPath, savePath);
}
else
{
localPath = Path.Combine(UploadConfig.SaveAbsolutePath, savePath);
}
try
{
if (UploadConfig.FtpUpload)
{
var fileExt = Path.GetExtension(uploadFileName).TrimStart('.');
var key = savePath;//UploadConfig.PathFormat + "." + fileExt;
//FtpUpload.UploadFile(new MemoryStream(uploadFileBytes), localPath, UploadConfig.FtpIp,
// UploadConfig.FtpAccount, UploadConfig.FtpPwd);
AliyunOssUpload.UploadFile(Consts.AliyunOssServer.AccessEndpoint,
Consts.AliyunOssServer.AccessKeyId, Consts.AliyunOssServer.AccessKeySecret,
Consts.AliyunOssServer.BucketName, key, fileExt, new MemoryStream(uploadFileBytes));
}
else
{
if (!Directory.Exists(Path.GetDirectoryName(localPath)))
{
Directory.CreateDirectory(Path.GetDirectoryName(localPath));
}
File.WriteAllBytes(localPath, uploadFileBytes);
}
Result.Url = savePath;
Result.State = UploadState.Success;
}
catch (Exception e)
{
Result.State = UploadState.FileAccessError;
Result.ErrorMessage = e.Message;
}
finally
{
WriteResult();
}
}
