public List <Hotel> PostProcess(List <Hotel> hotels, RequestArgs args)
{
List <Hotel> newHotels = new List <Hotel>();
foreach (Hotel hotel in hotels)
{
hotel.ScrapeURL = Site.URL.Host;
hotel.AmtPeople = args.People;
hotel.AmtRooms = args.Rooms;
foreach (HotelReservation reservation in hotel.ReservationData.GetAllReservations())
{
try // e.g. Inputs 92 / 20 May 2020
{
Convert.ToInt32(reservation.Price);
reservation.Price = "£" + reservation.Price;
}
catch
{
// This catches the dates so we can ignore them
}
}
newHotels.Add(hotel);
}
return(newHotels);
}
public void RegisterPages(RequestArgs args)
{
if (Site != null)
{
UriBuilder uriBuilder = new UriBuilder(Site.URL);
var param = HttpUtility.ParseQueryString(String.Empty);
param["rooms"] = "1";
param["adults"] = "2";
param["city_id"] = "800013148"; // Travel.com "Londons" City ID
param["currency"] = "GBP";
param["check_in"] = "22/04/2019";
param["check_out"] = "23/04/2018";
param["page"] = "1";
for (int i = 1; i < 10; i++)
{
param["page"] = i.ToString();
uriBuilder.Path = "hotels/results/";
uriBuilder.Query = param.ToString();
var layout = Site.AddPage(uriBuilder.Uri.PathAndQuery.Substring(1), By.Id("rs_main_css"));
layout.AddNode(new NodeRequest {
Property = "Name", XPath = "//div/article/div[2]/div[1]/div[2]/a"
});
}
Scraper.AddSite(Site);
}
else
{
throw new Exception("Site was not initialised in class with a valid url");
}
}
public override void Process(RequestArgs arguments)
{
if (! _throttle.IsAllowed())
{
throw new UnauthorizedAccessException("Request limit exceeded.");
}
}
private void CreateVersionItems(RequestArgs args)
{
Assert.ArgumentNotNull(args, "args");
Item[] scope = args.Scope;
Assert.IsNotNull(scope, "The scope is null.");
string queryString = WebUtil.GetQueryString("items");
Item[] items = null;
if (queryString != null && queryString.Length > 0 && queryString.Contains('|'))
{
items = Common.Functions.GetItems(queryString.Split('|'));
}
else if (queryString != null && queryString.Length > 0)
{
items = new Item[1];
}
items[0] = Common.Functions.GetItem(queryString);
if (items == null)
{
throw new ArgumentException("Items not found.");
}
Item[] scope2 = (scope.Length > 0) ? new Item[]
{
scope.First <Item>()
} : new Item[0];
CreateVersionArgs createVersionArgs = new CreateVersionArgs(scope2)
{
Scope = items
};
CorePipeline.Run("itemWebApiCreateVersion", createVersionArgs);
args.Result = createVersionArgs.Result;
}
protected void StartBuild(string buildName, RequestContext context, RequestArgs args)
{
if (context != Context)
{
return;
}
Build build;
var builds = Server.FindBuilds();
if (builds == null)
{
Server.RaiseCommonError("StartBuild: Failed to load builds directory!", true);
return;
}
if (!builds.TryGetValue(buildName, out build))
{
Server.RaiseCommonError("StartBuild: Wrong build name!", false);
return;
}
if (args.Count < build.Args.Count)
{
Server.RaiseCommonError(
$"StartBuild: build required {build.Args.Count} args, but {args.Count - 1} args is provided!",
true);
return;
}
var buildArgs = args.ToArray();
if (Server.TryInitBuild(Context, build, buildArgs))
{
Server.StartBuild(buildArgs);
}
}
public override void Process(RequestArgs arguments)
{
Assert.ArgumentNotNull(arguments, "arguments");
string queryString = WebUtil.GetQueryString("sc_itemid", null);
if (queryString == null)
{
string text = arguments.Context.HttpContext.Request.QueryString[null];
if (text != null)
{
if (Array.Exists <string>(text.Split(new char[]
{
','
}), (string s) => s.ToLower() == "sc_itemid"))
{
arguments.Context.Item = null;
}
}
return;
}
ID itemId;
if (!ID.TryParse(queryString, out itemId))
{
arguments.Context.Item = null;
return;
}
if (Sitecore.Context.Database.GetItem(itemId) == null)
{
arguments.Context.Item = null;
}
}
public override void Process([Sitecore.NotNull] RequestArgs arguments)
{
if (System.Web.HttpContext.Current.Request.QueryString["type"] == "xml")
{
Context.Current.Serializer = new XmlSerializer();
}
}
public override bool Get(RequestArgs args)
{
var s = GetGuid();
args.Send("[" + s + "," + s + 1000 + "]");
return(true);
}
public ActionResult GetCookiePost(RequestArgs param)
{
ILog log = LogManager.GetLogger("cookielog");
CookieRequester.PostRequest(new CrawlSettings(), param);
return(View());
}
public void GenerateReport(RequestArgs args) // Function that is used to generate the HTML report
{
Program.App.Log($"[{DateTime.Now.ToShortTimeString()}] [CMD] Creating report..");
Program.App.Log($"[{DateTime.Now.ToShortTimeString()}] [CMD] Gathering data, querying database..");
productMatching.Start(args); // Starts the product matching process with the user defined filter
Program.App.Log($"[{DateTime.Now.ToShortTimeString()}] [CMD] Results gathered, report being created");
Creator reportCreator = new Creator();
Task <string> reportLocation = reportCreator.Create(args, productMatching.GetResult()); // Passes the results from product matching to the report generation
productMatching.Reset();
while (reportLocation.IsCompleted == false)
{
Thread.Sleep(10); // Waits for the async method to be completed before continuing
}
Program.App.Log($"[{DateTime.Now.ToShortTimeString()}] [CMD] Report created opening.. (${reportLocation.Result})");
var p = new Process();
p.StartInfo = new ProcessStartInfo(@reportLocation.Result)
{
UseShellExecute = true
};
p.Start(); // Opens the report created, uses the primary web browsers associated on the users machine
}
private void UpdateMediaItem(RequestArgs args)
{
Assert.ArgumentNotNull(args, "args");
Item item = args.Context.Item;
if (item == null)
{
throw new BadRequestException("The specified location not found.");
}
Database database = args.Context.Database;
Assert.IsNotNull(database, "Database not resolved.");
Sitecore.Globalization.Language currentLanguage = args.Context.Language;
if (!item.Access.CanCreate())
{
throw new AccessDeniedException(string.Format("Access denied (access right: 'item:create', item: '{0}')", item.Paths.ParentPath));
}
HttpFileCollection files = args.Context.HttpContext.Request.Files;
List <Item> list = new List <Item>();
MediaItem mediaItem = Common.Functions.UpdateMediaItem(item, args.Context.HttpContext);
if (mediaItem != null)
{
list.Add(mediaItem.InnerItem);
ReadArgs readArgs = new ReadArgs(list.ToArray());
CorePipeline.Run("itemWebApiRead", readArgs);
args.Result = readArgs.Result;
}
}
public override void Process(RequestArgs arguments)
{
if (AverageRequestIntervalLessThanMinimum())
{
throw new UnauthorizedAccessException("Request limit exceeded.");
}
}
public async Task <string> Create(RequestArgs args, List <Hotel> hotels)
{
var engine = new RazorLightEngineBuilder()
.UseMemoryCachingProvider()
.UseFilesystemProject(Environment.CurrentDirectory + "/Report/Templates/")
.Build(); // Creates the razor engine used for generation
ReportViewModel model = new ReportViewModel // Initialise the view model with our supplied data for the template
{
SearchLocation = args.City,
CheckIn = args.CheckIn.ToShortDateString(),
CheckOut = args.CheckOut.ToShortDateString(),
PersonAmount = args.People,
RoomAmount = args.Rooms,
Gathered = DateTime.Now,
Hotels = hotels
};
string result = await engine.CompileRenderAsync("Template.cshtml", model); // Generate the HTMl with the template and viewmodel
Directory.CreateDirectory(Environment.CurrentDirectory + "/Reports/");
string filePath = Environment.CurrentDirectory + $"\\Reports\\{args.City}-{DateTime.Now.Hour}-{DateTime.Now.Minute}-{DateTime.Now.Second}-{DateTime.Now.Day}-{DateTime.Now.Month}-{DateTime.Now.Year}.html";
File.WriteAllText(filePath, result);
return(filePath);
}
public override void Invoke(CommandArguments commandArgument)
{
RequestArgs args = new RequestArgs
{
City = "",
Name = "",
};
string site = "";
foreach (var argument in commandArgument.Arguments)
{
if (argument.Key == "-city")
{
args.City = argument.Value;
}
if (argument.Key == "-name")
{
args.Name = argument.Value;
}
if (argument.Key == "-site")
{
site = argument.Value;
}
}
Program.App.QueryDatabase(args, site);
}
private static Item[] GetItems(RequestArgs arguments)
{
Assert.ArgumentNotNull(arguments, "arguments");
Database database = arguments.Context.Database;
if (database == null)
{
database = Common.Functions.GetDatabase();
}
Item item = Common.Functions.GetItem(arguments);
string queryString = WebUtil.GetQueryString("sc_database");
if (!string.IsNullOrEmpty(queryString) && !string.Equals(queryString, database.Name, StringComparison.OrdinalIgnoreCase))
{
throw new AccessDeniedException(string.Format("Access to the '{0}' database is denied. Only members of the Sitecore Client Users role can switch databases.", queryString));
}
string text = WebUtil.GetQueryString("query", null);
if (text != null)
{
Item[] result;
if (text.StartsWith("fast:"))
{
text = text.Substring(5);
try
{
result = (database.SelectItems(text) ?? new Item[0]);
return(result);
}
catch
{
throw new BadRequestException(string.Format("Bad Sitecore fast query: ({0}).", text));
}
}
if (item == null)
{
Logger.Warn("Context item not resolved.");
return(new Item[0]);
}
try
{
result = (Query.SelectItems(text) ?? new Item[0]);
}
catch
{
throw new BadRequestException(string.Format("Bad Sitecore query ({0}).", text));
}
return(result);
}
if (item != null)
{
return(new Item[]
{
item
});
}
Logger.Warn("Context item not resolved.");
return(new Item[0]);
}
public RequestResult(RequestArgs requestArgs)
{
if (requestArgs == null)
{
throw new ArgumentNullException("requestArgs");
}
RequestArgs = requestArgs;
}
void import_ReportRequest(FeatureImport sender, RequestArgs args)
{
args.Result = MessageBox.Show(
args.Request,
"Warning",
args.Buttons,
MessageBoxIcon.Warning);
}
/// <summary>
/// Iterates over all the sites and calls the GetData function
/// </summary>
public List <List <Hotel> > GetAllData(RequestArgs args)
{
List <List <Hotel> > data = new List <List <Hotel> >();
foreach (ISite site in Sites)
{
data.Add(site.GetData(args));
}
return(data);
}
public override bool Get(RequestArgs args)
{
var data = new byte[32] {
234, 23, 196, 234, 69, 238, 92, 244, 50, 110, 70, 181, 109, 139, 252, 209, 146, 174, 40, 140, 129, 41, 58, 89, 102, 193, 99, 194, 178, 192, 239, 152
};
var r = args.context.Response;
r.Headers.Add("content-type", "application/octet-stream");
args.GZipSend(data);
return(true);
}
public void QueryDatabase(RequestArgs args, string site) // helper function which is used to query the databse and return results from the database
{
var hotels = Storage.GetHotelsFullRequest(args, site);
Program.App.Log($"Hotels matching City: '{args.City}' Name: '{args.Name}' Site: '{site}'");
foreach (Hotel hotel in hotels)
{
Program.App.Log($" -> {hotel.City} - {hotel.Name}, {hotel.Address} ({hotel.ScrapeURL})");
}
Program.App.Log($"Found {hotels.Count} matches!");
}
/// <summary>
/// calls the GetData function of a specific site
/// </summary>
public List <Hotel> GetData(ISite site, RequestArgs args)
{
foreach (ISite isite in Sites)
{
if (isite.Equals(site))
{
return(isite.GetData(args));
}
}
return(new List <Hotel>());
}
public void PostRequestTest()
{
CrawlSettings settings = new CrawlSettings();
RequestArgs args = new RequestArgs()
{
PostParameters = "a=1234", RequestUri = "http://www.wqii.com.cn"
};
ILog log = LogManager.GetLogger("requestLogger");
CookieRequester.PostRequest(settings, args, log);
Assert.AreEqual(null, settings.CookieContainer);
}
public List <Hotel> GetData(RequestArgs args)
{
RegisterPages(args);
Scraper.Run(Site);
List <Hotel> hotels = Hotel.Map(Scraper.GetRawResult()[Site], args);
hotels = PostProcess(hotels, args);
return(hotels);
}
public void GetAllSiteData(RequestArgs args)
{
var hotelsList = Sites.GetAllData(args); // Returns a list of hotels from the providers after they've ran
foreach (List <Hotel> hotels in hotelsList)
{
hotels.ForEach((hotel) => { Storage.AddHotel(hotel); }); // iterates over the results and adds them to the database
}
Log($"[{DateTime.Now.ToShortTimeString()}] [CMD] All Sites have finished!");
Sites.FlushData();
}
protected override void BeginExecute()
{
this.DisposeLocalFile();
if (string.IsNullOrEmpty(this._url) || !(this.Result is InboxImageDataProviderObject result))
{
return;
}
this.SetWorkerStatus(DataProviderQueryStatus.RequestingData);
RequestArgs requestArgs = new RequestArgs(this._url, result, Environment.TickCount);
WebRequestHelper.ConstructWebRequest(this._url, EPassportPolicyId.MBI_SSL, HttpRequestCachePolicy.Default, true, false).GetResponseAsync(new AsyncRequestComplete(this.OnRequestComplete), requestArgs);
}
private void CreateAdvanceItem(RequestArgs args)
{
Assert.ArgumentNotNull(args, "args");
Item[] scope = args.Scope;
Assert.IsNotNull(scope, "The scope is null.");
Item[] scope2 = (scope.Length > 0) ? new Item[]
{
scope.First <Item>()
} : new Item[0];
if (scope2 != null && scope2.Length > 0)
{
Mindtree.ItemWebApi.Pipelines.Advance.Serialize.Entities.LoadOptions loadOptions = null;
SyncItem syncItem = null;
try
{
if (args.Context.HttpContext != null && args.Context.HttpContext.Request != null && args.Context.HttpContext.Request.Form != null)
{
string _enkey = string.Empty;
if (args.Context.HttpContext.Request.Form[Settings.enKey] != null)
{
_enkey = args.Context.HttpContext.Request.Form[Settings.enKey];
}
if (args.Context.HttpContext.Request.Form[Settings.loadOptions] != null)
{
string data = args.Context.HttpContext.Request.Form[Settings.loadOptions];
loadOptions = Mindtree.ItemWebApi.Pipelines.Advance.Serialize.SerializeManager.DeSerializeLoadOptions(StringCipher.Decrypt(data, _enkey));
}
if (args.Context.HttpContext.Request.Form[Settings.syncItem] != null)
{
string dataSyncItem = args.Context.HttpContext.Request.Form[Settings.syncItem];
syncItem = Mindtree.ItemWebApi.Pipelines.Advance.Serialize.SerializeManager.DeSerializeItem(StringCipher.Decrypt(dataSyncItem, _enkey), loadOptions);
}
}
}
catch (Exception)
{ }
//args.Context.HttpContext.Request.Params[""]
Mindtree.ItemWebApi.Pipelines.Advance.Create.CreateArgs createArgs = new Mindtree.ItemWebApi.Pipelines.Advance.Create.CreateArgs(scope2, Mindtree.ItemWebApi.Pipelines.Advance.Serialize.SerializeManager._loadOption, syncItem)
{
Scope = scope2
};
CorePipeline.Run("itemWebApiCreateAdvance", createArgs);
args.Result = createArgs.Result;
}
else
{
throw new BadRequestException("Parent Item doesn't Exist");
}
}
public void Start(RequestArgs args)
{
if (args.Name != String.Empty)
{
preMatchHotels = connection.GetHotelsByName(args);
}
else
{
preMatchHotels = connection.GetHotelsByCity(args);
}
Loop();
AddSingleHotels();
}
void OnHistoryRequested(RequestContext context, RequestArgs args)
{
_logger.LogDebug($"OnHistoryRequested: ({args.Count})");
var sb = new StringBuilder();
string buildName = args.Count > 0 ? args[0] : null;
sb.Append(buildName == null ? "History:\n" : $"History ({buildName}):\n");
var table = new StatTable();
FormatHistoryHeader(table);
AppendHistoryStats(_container.Builds, buildName, table);
table.Append(sb);
_server.RaiseCommonMessage(context, sb.ToString());
}
public void Ctor_AssignsValues()
{
// Arrange
RequestType type = RequestType.Get;
byte[] data = Encoding.UTF8.GetBytes("lorem ipsum");
// Act
var message = new RequestArgs(type, data, data.Length, new Socket(SocketType.Stream, ProtocolType.IP));
// Assert
Assert.Equal(message.Type, type);
Assert.Equal(message.Request, data);
}
public List <Hotel> PostProcess(List <Hotel> hotels, RequestArgs args)
{
List <Hotel> newHotels = new List <Hotel>();
foreach (Hotel hotel in hotels)
{
hotel.ScrapeURL = Site.URL.Host;
hotel.AmtPeople = args.People;
hotel.AmtRooms = args.Rooms;
newHotels.Add(hotel);
}
return(newHotels);
}
protected virtual void ExecuteCreateVersionRequest(RequestArgs args)
{
Assert.ArgumentNotNull(args, "args");
if (Common.Functions.IsMediaItem(args.Context))
{
this.CreateMediaVersionItems(args);
return;
}
else
{
this.CreateVersionItems(args);
return;
}
throw new BadRequestException("The specified Content-Type is not supported.");
}
protected ServerRequest ConvertMessage(RequestContext context, string message)
{
message = message.Trim();
var allParts = message.SplitByWhitespaces();
if (allParts.Count <= 0)
{
return(ServerRequest.Empty);
}
var request = allParts[0];
var requestArgs = new RequestArgs(allParts.Skip(1));
_logger.LogDebug($"ConvertMessage: \"{message}\" => [\"{request}\", {requestArgs.Count}]");
return(new ServerRequest(request, context, requestArgs));
}
