biglyderv/bigly-caret
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

do not recalculate milked pages

Browse source
This commit is contained in:
biglyderv 2025-02-01 12:01:59 -05:00
parent 552ac802b1
commit 27bce6cc63
1 changed files with 8 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
site.js
View file

@ -12,11 +12,12 @@ let greylist = (process.env.greylist + '').split(',') || [];
let hh;
let percent = 0;
let d = process.env.depth || 1;
let noWorry = [];
async function urlCollector(url, path, file, useLimit, data2) {
let urls = data2[url] ? (data2[url][path] || []) : [];
urls = [...urls];
if (path != 'following') return urls;
if (path != 'following' || noWorry.indexOf(url) != -1) return urls;
let data;
try {
@ -31,8 +32,10 @@ async function urlCollector(url, path, file, useLimit, data2) {
let body = cheerio.load(data);
let links = body('a');
let ll = urls.length;
let returnedFalse = false;
links.each(function (i, link) {
if (useLimit && urls.length >= pageLimit * 40 + ll) {
returnedFalse = true;
return false;
}
let h = body(link).attr('href');
@ -60,6 +63,10 @@ async function urlCollector(url, path, file, useLimit, data2) {
return true;
})
if (!returnedFalse) {
noWorry.push(url);
}
let h3;
try {
h3 = new URL(new URL(url).origin);