public void getHTML_MultiPage()
{
RequestAttribute attrs = new RequestAttribute
{
URI = "https://api.dealerrater.com/",
URL = "https://www.dealerrater.com/dealer/McKaig-Chevrolet-Buick-A-Dealer-For-The-People-dealer-reviews-23685",
API = false,
AccessToken = "ACCESSTOKENHERE",
DealerID = "23685",
PaginationOptions = new PaginationOptions
{
Pages = 5,
ResultsPerPage = 10,
Offset = 0
}
};
WebScraper scraper = new WebScraper(attrs);
string returnedHTML = scraper.getHTML();
int index, count = 0;
while ((index = returnedHTML.ToLower().IndexOf("<!doctype html>")) != -1)
{
returnedHTML = returnedHTML.Substring(index + 15, returnedHTML.Length - (index + 15));
count++;
}
Assert.AreEqual(attrs.PaginationOptions.Pages, count);
}
public void getJSON_NonDRPage()
{
RequestAttribute attrs = new RequestAttribute
{
URI = "https://api.dealerrater.com/",
URL = "https://www.google.com/",
API = false,
AccessToken = "ACCESSTOKENHERE",
DealerID = "23685",
PaginationOptions = new PaginationOptions
{
Pages = 1,
ResultsPerPage = 10,
Offset = 0
}
};
WebScraper scraper = new WebScraper(attrs);
string returnedJSON = scraper.getJSON();
ReviewCollection collection = JsonConvert.DeserializeObject <ReviewCollection>(returnedJSON);
Assert.AreEqual(0, collection.reviews.Count);
}
public void getJSON_StressTest()
{
RequestAttribute attrs = new RequestAttribute
{
URI = "https://api.dealerrater.com/",
URL = "https://www.dealerrater.com/dealer/McKaig-Chevrolet-Buick-A-Dealer-For-The-People-dealer-reviews-23685",
API = false,
AccessToken = "ACCESSTOKENHERE",
DealerID = "23685",
PaginationOptions = new PaginationOptions
{
Pages = 50,
ResultsPerPage = 10,
Offset = 0
}
};
WebScraper scraper = new WebScraper(attrs);
string returnedJSON = scraper.getJSON();
ReviewCollection collection = JsonConvert.DeserializeObject <ReviewCollection>(returnedJSON);
Assert.AreEqual(attrs.PaginationOptions.Pages * attrs.PaginationOptions.ResultsPerPage, collection.reviews.Count);
}
public async Task <bool> UpdateRequestAttributeRuleAsync(string id, RequestAttribute body = null, CancellationToken cancellationToken = default)
{
var response = await GetRequestAttributeRulesUrl()
.AppendPathSegment(id)
.PutJsonAsync(body, cancellationToken)
.ConfigureAwait(false);
return(response.IsSuccessStatusCode);
}
private async Task <object> RunMethod(Attribute[] attributes, Type retType, ParameterInfo[] parameterInfos, Parameters parameters)
{
RequestAttribute ra = attributes.FirstOrDefault(s => s is RequestAttribute) as RequestAttribute;
if (ra == null)
{
FileTransferAttribute sa = attributes.FirstOrDefault(s => s is FileTransferAttribute) as FileTransferAttribute;
if (sa == null)
{
throw new MessageException("特性不存在");
}
return(SendStream(sa, parameters.ParametersInfo));
}
HttpResponseMessage rd = await Request(ra, parameterInfos, parameters.ParametersInfo);
if (rd.IsSuccessStatusCode)
{
Type respType = retType;
if (retType.IsTask())
{
var args = retType.GenericTypeArguments;
if (args.Length > 0)
{
respType = args[0];
}
}
if (respType == typeof(HttpResponseMessage))
{
return(rd);
}
if (respType == typeof(Stream))
{
return(await rd.Content.ReadAsStreamAsync());
}
if (respType == typeof(byte[]))
{
return(await rd.Content.ReadAsByteArrayAsync());
}
if (respType == typeof(void))
{
return(Task.CompletedTask);
}
string value = await rd.Content.ReadAsStringAsync();
if (respType == typeof(string))
{
return(value);
}
if (!string.IsNullOrWhiteSpace(value))
{
var v = value.ToObject(respType);
return(v);
}
}
throw new MessageException(rd.ToString());
}
public void Insert(Guid Guid, Guid SeriesGUID, string RequestedProcedureId, string ScheduledProcedureStepId)
{
var item = new RequestAttribute();
item.Guid = Guid;
item.SeriesGUID = SeriesGUID;
item.RequestedProcedureId = RequestedProcedureId;
item.ScheduledProcedureStepId = ScheduledProcedureStepId;
item.Save(UserName);
}
/// <summary>
/// 请求数据
/// </summary>
/// <param name="channel"></param>
/// <param name="ra"></param>
/// <param name="parameters">参数属性</param>
/// <param name="objs">参数</param>
/// <returns></returns>
private Task <HttpResponseMessage> Request(RequestAttribute ra, ParameterInfo[] parameters, params object[] objs)
{
Task <HttpResponseMessage> rd = null;
string uri = HttpWebHelper.PathCombine(Uri, ra.Uri);
switch (ra.RequestType)
{
case RequestType.Get:
rd = GetRequest(uri, parameters, objs);
break;
default:
rd = PostRequest(uri, objs);
break;
}
return(rd);
}
public void getJSON_ThrowsNotImplemented()
{
RequestAttribute attrs = new RequestAttribute
{
URI = "https://api.dealerrater.com/",
URL = "https://www.dealerrater.com/dealer/McKaig-Chevrolet-Buick-A-Dealer-For-The-People-dealer-reviews-23685",
API = false,
AccessToken = "ACCESSTOKENHERE",
DealerID = "23685",
PaginationOptions = new PaginationOptions
{
Pages = 5,
ResultsPerPage = 10,
Offset = 0
}
};
APIConnection api = new APIConnection(attrs);
Assert.ThrowsException <NotImplementedException>(() => api.getJSON());
}
public void getHTML_BadURL()
{
RequestAttribute attrs = new RequestAttribute
{
URI = "https://api.dealerrater.com/",
URL = "https://www.whatisthisurlitdoesnotexist.com/",
API = false,
AccessToken = "ACCESSTOKENHERE",
DealerID = "23685",
PaginationOptions = new PaginationOptions
{
Pages = 3,
ResultsPerPage = 10,
Offset = 0
}
};
WebScraper scraper = new WebScraper(attrs);
Assert.ThrowsException <WebException>(() => scraper.getHTML());
}
public void getHTML_NonDRSinglePageAndOffset()
{
RequestAttribute attrs = new RequestAttribute
{
URI = "https://api.dealerrater.com/",
URL = "https://www.google.com/",
API = false,
AccessToken = "ACCESSTOKENHERE",
DealerID = "23685",
PaginationOptions = new PaginationOptions
{
Pages = 1,
ResultsPerPage = 10,
Offset = 1
}
};
WebScraper scraper = new WebScraper(attrs);
Assert.ThrowsException <Exception>(() => scraper.getHTML());
}
private void AddRequestInfoCreation(ILGenerator methodIlGenerator, FieldBuilder requesterField, RequestAttribute requestAttribute)
{
// Load 'this' onto the stack
// Stack: [this]
methodIlGenerator.Emit(OpCodes.Ldarg_0);
// Load 'this.requester' onto the stack
// Stack: [this.requester]
methodIlGenerator.Emit(OpCodes.Ldfld, requesterField);
// Start loading the ctor params for RequestInfo onto the stack
// 1. HttpMethod
// Stack: [this.requester, HttpMethod]
methodIlGenerator.Emit(OpCodes.Call, httpMethodProperties[requestAttribute.Method].GetGetMethod());
// 2. The Path
// Stack: [this.requester, HttpMethod, path]
methodIlGenerator.Emit(OpCodes.Ldstr, requestAttribute.Path);
// Ctor the RequestInfo
// Stack: [this.requester, requestInfo]
methodIlGenerator.Emit(OpCodes.Newobj, requestInfoCtor);
}
private void AddRequestInfoCreation(ILGenerator methodIlGenerator, FieldBuilder requesterField, RequestAttribute requestAttribute)
{
// Load 'this' onto the stack
// Stack: [this]
methodIlGenerator.Emit(OpCodes.Ldarg_0);
// Load 'this.requester' onto the stack
// Stack: [this.requester]
methodIlGenerator.Emit(OpCodes.Ldfld, requesterField);
// Start loading the ctor params for RequestInfo onto the stack
// 1. HttpMethod
// Stack: [this.requester, HttpMethod]
// For the standard HTTP methods, we can get a static instance. For others, we'll need to construct the HttpMethod
// ourselves
if (httpMethodProperties.TryGetValue(requestAttribute.Method, out PropertyInfo cachedPropertyInfo))
{
methodIlGenerator.Emit(OpCodes.Call, cachedPropertyInfo.GetMethod);
}
else
{
methodIlGenerator.Emit(OpCodes.Ldstr, requestAttribute.Method.Method);
methodIlGenerator.Emit(OpCodes.Newobj, httpMethodCtor);
}
// 2. The Path
// Stack: [this.requester, HttpMethod, path]
methodIlGenerator.Emit(OpCodes.Ldstr, requestAttribute.Path ?? String.Empty);
// Ctor the RequestInfo
// Stack: [this.requester, requestInfo]
methodIlGenerator.Emit(OpCodes.Newobj, requestInfoCtor);
}
public async Task <EntityShortRepresentation> CreateRequestAttributeRuleAsync(RequestAttribute body = null, CancellationToken cancellationToken = default)
{
var response = await GetRequestAttributeRulesUrl()
.PostJsonAsync(body, cancellationToken)
.ReceiveJsonWithErrorChecking <EntityShortRepresentation>()
.ConfigureAwait(false);
return(response);
}
//TODO: acceptNulls
private void ProcessAttribute(RequestAttribute a, bool acceptNulls)
{
if (a is RequireModeAttribute)
{
RequireModeAttribute ra = a as RequireModeAttribute;
if (_context.Request.QueryString["mode"] == ra.Mode)
{
foreach (RequestAttribute at in ra.Requirements)
{
ProcessAttribute(at, false);
}
_mode = ra.Mode;
}
}
if (a is RequireContestIDAttribute)
{
if (_context.Request.QueryString["contestID"] != null)
{
if (!int.TryParse(_context.Request.QueryString["contestID"], out _tid) || !Contest.ValidateID(_tid))
{
_tid = -1;
throw new NeJudgeInvalidParametersException("contestID");
}
}
else if (!acceptNulls)
{
throw new NeJudgeInvalidParametersException("contestID");
}
}
else if (a is RequireProblemIDAttribute)
{
if (_context.Request.QueryString["problemID"] != null)
{
if (!int.TryParse(_context.Request.QueryString["problemID"], out _pid) || !Problem.ValidateID(_pid))
{
_pid = -1;
throw new NeJudgeInvalidParametersException("problemID");
}
}
else if (!acceptNulls)
{
throw new NeJudgeInvalidParametersException("problemID");
}
}
else if (a is RequireEveryProblemIDAttribute)
{
if (_context.Request.QueryString["problemID"] != null)
{
if (!int.TryParse(_context.Request.QueryString["problemID"], out _pid) || (_pid != 0 && !Problem.ValidateID(_pid)))
{
_pid = -1;
throw new NeJudgeInvalidParametersException("problemID");
}
}
}
else if (a is RequireOutcomeAttribute)
{
if (_context.Request.QueryString["outcome"] != null)
{
try
{
outcome = _context.Request.QueryString["outcome"];
}
catch
{
outcome = null;
throw new NeJudgeInvalidParametersException("outcome");
}
}
}
else if (a is RequireUserIDAttribute)
{
if (_context.Request.QueryString["userID"] != null)
{
if (!User.ValidateID(_context.Request.QueryString["userID"]))
{
_uid = null;
throw new NeJudgeInvalidParametersException("userID");
}
_uid = _context.Request.QueryString["userID"];
}
}
else if (a is RequireSubmissionIDAttribute)
{
if (_context.Request.QueryString["submissionID"] != null)
{
if (!int.TryParse(_context.Request.QueryString["submissionID"], out _sid) ||
!Submission.ValidateID(_sid))
{
_sid = -1;
throw new NeJudgeInvalidParametersException("submissionID");
}
}
}
}
public void Update(Guid Guid, Guid SeriesGUID, string RequestedProcedureId, string ScheduledProcedureStepId)
{
var item = new RequestAttribute();
item.MarkOld();
item.IsLoaded = true;
item.Guid = Guid;
item.SeriesGUID = SeriesGUID;
item.RequestedProcedureId = RequestedProcedureId;
item.ScheduledProcedureStepId = ScheduledProcedureStepId;
item.Save(UserName);
}
