public WebScraper(ProxyCfg proxy, YtCollectCfg collectCfg, ISimpleFileStore logStore)
{
Proxy = proxy;
CollectCfg = collectCfg;
LogStore = logStore;
Clients = new ResourceCycle <HttpClient, ProxyConnectionCfg>(proxy.DirectAndProxies(), p => Task.FromResult(CreateHttpClient(p)));
}
public YtWeb(ProxyCfg proxy, YtCollectCfg collectCfg, ISimpleFileStore logStore)
{
Proxy = proxy;
CollectCfg = collectCfg;
LogStore = logStore;
Clients = new(proxy.DirectAndProxies(), p => Task.FromResult(p.CreateHttpClient()));
}
protected override void Build()
{
// Load Material
ModelMat = CreateModelMat();
// Get GameObjects
Prefab = PrefabUtils.CopyPrefab(BaseItem);
Prefab.name = NamePrefix + Name;
Prefab.transform.localScale = Scale;
GameObject child = Prefab.FindChild("model_pogofruit");
child.transform.localScale = ModelScale;
Object.Destroy(Prefab.GetComponent <BoxCollider>());
// Load Components
SphereCollider col = Prefab.AddComponent <SphereCollider>();
MeshFilter filter = Prefab.GetComponent <MeshFilter>();
Rigidbody body = Prefab.GetComponent <Rigidbody>();
Vacuumable vac = Prefab.GetComponent <Vacuumable>();
Identifiable iden = Prefab.GetComponent <Identifiable>();
ResourceCycle cycle = IsPlantable ? Prefab.GetComponent <ResourceCycle>() : null;
MeshFilter model = child.GetComponent <MeshFilter>();
MeshRenderer render = child.GetComponent <MeshRenderer>();
// Setup Components
col.radius = ModelScale.x;
filter.sharedMesh = Mesh;
body.mass = Mass;
vac.size = Size;
iden.id = ID;
if (IsPlantable)
{
if (cycle != null)
{
cycle.unripeGameHours = UnripeGameHours;
cycle.ripeGameHours = RipeGameHours;
cycle.edibleGameHours = EdibleGameHours;
cycle.rottenGameHours = RottenGameHours;
cycle.rottenMat = RottenMat ? RottenMat : cycle.rottenMat;
cycle.vacuumableWhenRipe = VacuumableWhenRipe;
}
}
else
{
Object.Destroy(Prefab.GetComponent <ResourceCycle>());
}
model.sharedMesh = Mesh;
render.sharedMaterial = ModelMat;
}
//+ METHODS
/// <inheritdoc />
public override void Build(GameObject prefab)
{
//? In case of questions, tofu keeps the model object as 'model_pogofruit'
GameObject model = prefab.FindChild("model_pogofruit");
model.name = MODEL_BASE_NAME;
model.transform.localScale = Vector3.one * ModelScale;
// Manages Collisions
if (ApplyCollider != null)
{
ApplyCollider.Invoke(prefab, this);
}
else
{
Object.Destroy(prefab.GetComponent <BoxCollider>());
SphereCollider col = prefab.AddComponent <SphereCollider>();
col.radius = ModelScale;
}
// Loading components
MeshFilter filter = prefab.GetComponent <MeshFilter>();
Rigidbody body = prefab.GetComponent <Rigidbody>();
Vacuumable vac = prefab.GetComponent <Vacuumable>();
Identifiable iden = prefab.GetComponent <Identifiable>();
ResourceCycle cycle = prefab.GetComponent <ResourceCycle>();
MeshFilter mFilter = model.GetComponent <MeshFilter>();
MeshRenderer render = model.GetComponent <MeshRenderer>();
// Setting up components
filter.sharedMesh = Mesh;
body.mass = Mass;
vac.size = VacSize;
iden.id = ID;
cycle.unripeGameHours = UnripeGameHours;
cycle.ripeGameHours = RipeGameHours;
cycle.edibleGameHours = EdibleGameHours;
cycle.rottenGameHours = RottenGameHours;
cycle.rottenMat = RottenMat ?? cycle.rottenMat;
cycle.vacuumableWhenRipe = VacWhenRipe;
mFilter.sharedMesh = Mesh;
render.sharedMaterial = ModelMat;
}
int _lastUsedBrowserIdx; // re-use the same browser proxies across different calls to recs and extra
public async Task <IReadOnlyCollection <RecsAndExtra> > GetRecsAndExtra(IReadOnlyCollection <string> videos, ILogger log)
{
var sw = Stopwatch.StartNew();
log = log.ForContext("Module", nameof(ChromeScraper));
var proxies = ProxyCfg.DirectAndProxies().Take(2)
.ToArray(); // only use the first fallback (datacenter proxies). residential is too expensive for the volume of data we are loading.
var parallel = CollectCfg.ChromeParallel;
var videosPerBrowserPool = videos.Count / CollectCfg.ChromeParallel;
var res = await videos.Batch(videosPerBrowserPool).WithIndex().BlockFunc(async batch => {
var(b, i) = batch;
var requests = new ConcurrentBag <(Request req, bool aborted)>();
await using var browsers = new ResourceCycle <Browser, ProxyConnectionCfg>(proxies, p => CreateBrowser(log, p), _lastUsedBrowserIdx);
async Task <Page> Page(Browser browser, ProxyConnectionCfg proxy)
{
var page = await browser.NewPageAsync(); // create page inside retry loop. some transient errors seems to be caused by state in the page
if (proxy.Creds != null)
{
await page.AuthenticateAsync(proxy.Creds.AsCreds()); // workaround for chrome not supporting password proxies
}
await page.SetCookieAsync(); // clears cookies
await ConfigureRequests(page, (req, aborted) => requests.Add((req, aborted)));
return(page);
}
return(await b.BlockFunc(async v => {
//var context = await browser.CreateIncognitoBrowserContextAsync(); // opening windows is more reliable than tabs in practice
var videoAttempt = 0;
var videoLog = log.ForContext("Video", v);
while (true)
{
var(browser, proxy) = await browsers.Get();
videoAttempt++;
var lastAttempt = videoAttempt >= CollectCfg.ChromeAttempts;
videoLog.Debug("loading video {Video}. Proxy={Proxy}", v, proxy?.Url == null ? "Direct" : proxy.Url);
using var page = await Page(browser, proxy);
try {
var(video, notOkResponse) = await GetVideo(page, v, videoLog);
if (notOkResponse != null)
{
if (notOkResponse.Status == HttpStatusCode.TooManyRequests)
{
await browsers.NextResource(browser);
_lastUsedBrowserIdx = browsers.Idx;
videoAttempt = 0;
videoLog.Information("ChromeScraper - error response ({Status}) loading {Video}: using next proxy", notOkResponse.Status, v);
}
else
{
throw new InvalidOperationException($"Not OK response ({notOkResponse.Status})");
}
}
if (video == null)
{
continue;
}
videoLog.Debug("ChromeScraper - finished loading video {Video} with {Comments} comments and {Recommendations} recs in {Duration}",
v, video.Extra.Comments?.Length ?? 0, video.Recs?.Length ?? 0, sw.Elapsed.HumanizeShort());
return video;
}
catch (Exception ex) {
videoLog.Warning(ex, "ChromeScraper - failed to load video {Video} from {Url} (attempt {Attempt}): {Error}", v, page.Url, videoAttempt,
ex.Message);
if (lastAttempt)
{
throw;
}
}
}
}, parallel: 1, progressUpdate: p => log.Debug("ChromeScraper - browser pool {Pool} progress {Complete}/{Total}",
i, p.CompletedTotal, b.Count)));
}, parallel);
log.Information("ChromeScraper - finished loading all {Videos} videos in {Duration}", videos.Count, sw.HumanizeShort());
return(res.SelectMany().ToReadOnly());
}
