archive/bigly-caret
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix mess

Browse source
This commit is contained in:
dervperson 2025-01-31 22:01:33 -05:00
parent 12cbe62765
commit fffb298205
1 changed files with 14 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
site.js
View file

@ -25,7 +25,7 @@ async function urlCollector(url, path, file, useLimit) {
let body = cheerio.load(data); let body = cheerio.load(data);
let links = body('a'); let links = body('a');
let urls = []; let urls = [];
links.each(function(i, link) { links.each(function (i, link) {
if (useLimit && urls.length >= pageLimit * 40) return; if (useLimit && urls.length >= pageLimit * 40) return;
console.log(`User ${url} has ${i} pages calculated`); console.log(`User ${url} has ${i} pages calculated`);
@ -40,7 +40,7 @@ async function urlCollector(url, path, file, useLimit) {
let h2; let h2;
try { try {
h2 = new URL(h); h2 = new URL(h);
} catch(err) { } catch (err) {
return; return;
} }
urls.push(h2.toString()) urls.push(h2.toString())
@ -50,7 +50,7 @@ async function urlCollector(url, path, file, useLimit) {
let h3; let h3;
try { try {
h3 = new URL(new URL(url).origin); h3 = new URL(new URL(url).origin);
} catch(err) { } catch (err) {
return [...new Set(urls)]; return [...new Set(urls)];
} }
urls.push(h3.toString()) urls.push(h3.toString())
@ -155,8 +155,6 @@ async function siteCollector(user, path, site, useLimit) {
let congested = []; let congested = [];
for (let u of users) { for (let u of users) {
let inLegal = penv.indexOf(u) != -1;//legal.indexOf(u) != -1;
let it = 0; let it = 0;
while (p.length >= (process.env.maxRate || 15)) { while (p.length >= (process.env.maxRate || 15)) {
p = p.filter(x => x != 'hi'); p = p.filter(x => x != 'hi');
@ -171,23 +169,27 @@ async function siteCollector(user, path, site, useLimit) {
} }
} }
data[u] = { followers: [], following: [] }; data[u] = { followers: [], following: [] };
p.push(async function (k) { p.push(async function (k, u) {
let inLegal = penv.indexOf(u) != -1;//legal.indexOf(u) != -1;
let j1 = await siteCollector(u, 'followers', site, inLegal); let j1 = await siteCollector(u, 'followers', site, inLegal);
data[u].followers = j1; data[u].followers = j1;
console.log(`User ${u} followers fully calculated`); console.log(`User ${u} followers fully calculated`);
p[k] = 'hi'; p[k] = 'hi';
}(p.length)); }(p.length, u));
p.push(async function (k, u) {
let inLegal = penv.indexOf(u) != -1;//legal.indexOf(u) != -1;
p.push(async function (k) {
let j1 = await siteCollector(u, 'following', site, inLegal); let j1 = await siteCollector(u, 'following', site, inLegal);
data[u].following = j1; data[u].following = j1;
console.log(`User ${u} following fully calculated`); console.log(`User ${u} following fully calculated`);
p[k] = 'hi'; p[k] = 'hi';
}(p.length)); }(p.length,u));
} }
p = p.concat(congested); p = p.concat(congested);
@ -209,14 +211,14 @@ async function siteCollector(user, path, site, useLimit) {
} }
for (let f of followers) { for (let f of followers) {
if (!data[f]) data[f] = {followers: []}; if (!data[f]) data[f] = { followers: [] };
if (!data[f].following) data[f].following = []; if (!data[f].following) data[f].following = [];
if (data[f].following.indexOf(uf) == -1) { if (data[f].following.indexOf(uf) == -1) {
data[f].following.push(uf); data[f].following.push(uf);
} }
} }
for (let f of following) { for (let f of following) {
if (!data[f]) data[f] = {following: []}; if (!data[f]) data[f] = { following: [] };
if (!data[f].followers) data[f].followers = []; if (!data[f].followers) data[f].followers = [];
if (data[f].followers.indexOf(uf) == -1) { if (data[f].followers.indexOf(uf) == -1) {
data[f].followers.push(uf); data[f].followers.push(uf);