From fffb29820556ff9967906a31e96d65fc58dc2166 Mon Sep 17 00:00:00 2001 From: biglyderv Date: Fri, 31 Jan 2025 22:01:33 -0500 Subject: [PATCH] fix mess --- site.js | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/site.js b/site.js index b55ec4e..04479ad 100644 --- a/site.js +++ b/site.js @@ -25,12 +25,12 @@ async function urlCollector(url, path, file, useLimit) { let body = cheerio.load(data); let links = body('a'); let urls = []; - links.each(function(i, link) { + links.each(function (i, link) { if (useLimit && urls.length >= pageLimit * 40) return; console.log(`User ${url} has ${i} pages calculated`); let h = body(link).attr('href'); - if (!h) return; + if (!h) return; h = h.trim(); if (h.startsWith('./') || h.startsWith('../') || h.startsWith('/')) { let u = new URL(url); @@ -40,7 +40,7 @@ async function urlCollector(url, path, file, useLimit) { let h2; try { h2 = new URL(h); - } catch(err) { + } catch (err) { return; } urls.push(h2.toString()) @@ -50,7 +50,7 @@ async function urlCollector(url, path, file, useLimit) { let h3; try { h3 = new URL(new URL(url).origin); - } catch(err) { + } catch (err) { return [...new Set(urls)]; } urls.push(h3.toString()) @@ -155,8 +155,6 @@ async function siteCollector(user, path, site, useLimit) { let congested = []; for (let u of users) { - let inLegal = penv.indexOf(u) != -1;//legal.indexOf(u) != -1; - let it = 0; while (p.length >= (process.env.maxRate || 15)) { p = p.filter(x => x != 'hi'); @@ -171,23 +169,27 @@ async function siteCollector(user, path, site, useLimit) { } } data[u] = { followers: [], following: [] }; - p.push(async function (k) { + p.push(async function (k, u) { + let inLegal = penv.indexOf(u) != -1;//legal.indexOf(u) != -1; + let j1 = await siteCollector(u, 'followers', site, inLegal); data[u].followers = j1; console.log(`User ${u} followers fully calculated`); p[k] = 'hi'; - }(p.length)); + }(p.length, u)); + + p.push(async function (k, u) { + let inLegal = penv.indexOf(u) != -1;//legal.indexOf(u) != -1; - p.push(async function (k) { let j1 = await siteCollector(u, 'following', site, inLegal); data[u].following = j1; console.log(`User ${u} following fully calculated`); p[k] = 'hi'; - }(p.length)); + }(p.length,u)); } p = p.concat(congested); @@ -209,14 +211,14 @@ async function siteCollector(user, path, site, useLimit) { } for (let f of followers) { - if (!data[f]) data[f] = {followers: []}; + if (!data[f]) data[f] = { followers: [] }; if (!data[f].following) data[f].following = []; if (data[f].following.indexOf(uf) == -1) { data[f].following.push(uf); } } for (let f of following) { - if (!data[f]) data[f] = {following: []}; + if (!data[f]) data[f] = { following: [] }; if (!data[f].followers) data[f].followers = []; if (data[f].followers.indexOf(uf) == -1) { data[f].followers.push(uf);