archive/bigly-caret
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

f

Browse source
This commit is contained in:
dervperson 2025-01-31 20:21:42 -05:00
parent e03a05d07f
commit 7cbf8e36a6
1 changed files with 20 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
site.js
View file

@ -24,6 +24,8 @@ async function urlCollector(url, path, file, useLimit) {
let links = body('a'); let links = body('a');
let urls = []; let urls = [];
links.each(function(i, link) { links.each(function(i, link) {
if (useLimit && urls.length >= pageLimit * 40) return;
let h = body(link).attr('href'); let h = body(link).attr('href');
if (!h) return; if (!h) return;
h = h.trim(); h = h.trim();
@ -34,8 +36,14 @@ async function urlCollector(url, path, file, useLimit) {
u.pathname = h; u.pathname = h;
h = u.toString(); h = u.toString();
} }
urls.push(h) let h2;
if (useLimit && urls.length >= pageLimit * 40) return urls; try {
h2 = new URL(h);
} catch(err) {
return;
}
urls.push(h2.toString())
urls.push(h2.origin)
}) })
return urls; return urls;
@ -178,23 +186,23 @@ async function siteCollector(user, path, site, useLimit) {
p = p.concat(congested); p = p.concat(congested);
await Promise.all(p); await Promise.all(p);
for (let uf in users) { for (let uf in data) {
let u = users[uf]; let u = data[uf];
if (!u) continue; if (!u) continue;
let { following, followers } = u; let { following, followers } = u;
if (!following || !followers) continue; if (!following || !followers) continue;
for (let f of followers) { for (let f of followers) {
if (!users[f]) users[f] = {}; if (!data[f]) data[f] = {followers: []};
if (!users[f].following) users[f].following = []; if (!data[f].following) data[f].following = [];
if (users[f].following.indexOf(uf) == -1) { if (data[f].following.indexOf(uf) == -1) {
users[f].following.push(uf); data[f].following.push(uf);
} }
} }
for (let f of following) { for (let f of following) {
if (!users[f]) users[f] = {}; if (!data[f]) data[f] = {following: []};
if (!users[f].followers) users[f].followers = []; if (!data[f].followers) data[f].followers = [];
if (users[f].followers.indexOf(uf) == -1) { if (data[f].followers.indexOf(uf) == -1) {
users[f].followers.push(uf); data[f].followers.push(uf);
} }
} }
} }