biglyderv/bigly-caret
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix mess

Browse source
This commit is contained in:
biglyderv 2025-01-31 22:01:33 -05:00
parent 12cbe62765
commit fffb298205
1 changed files with 14 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
site.js
View file

@ -25,12 +25,12 @@ async function urlCollector(url, path, file, useLimit) {
let body = cheerio.load(data);
let links = body('a');
let urls = [];
links.each(function(i, link) {
links.each(function (i, link) {
if (useLimit && urls.length >= pageLimit * 40) return;
console.log(`User ${url} has ${i} pages calculated`);
let h = body(link).attr('href');
if (!h) return;
if (!h) return;
h = h.trim();
if (h.startsWith('./') || h.startsWith('../') || h.startsWith('/')) {
let u = new URL(url);
@ -40,7 +40,7 @@ async function urlCollector(url, path, file, useLimit) {
let h2;
try {
h2 = new URL(h);
} catch(err) {
} catch (err) {
return;
}
urls.push(h2.toString())
@ -50,7 +50,7 @@ async function urlCollector(url, path, file, useLimit) {
let h3;
try {
h3 = new URL(new URL(url).origin);
} catch(err) {
} catch (err) {
return [...new Set(urls)];
}
urls.push(h3.toString())
@ -155,8 +155,6 @@ async function siteCollector(user, path, site, useLimit) {
let congested = [];
for (let u of users) {
let inLegal = penv.indexOf(u) != -1;//legal.indexOf(u) != -1;
let it = 0;
while (p.length >= (process.env.maxRate || 15)) {
p = p.filter(x => x != 'hi');
@ -171,23 +169,27 @@ async function siteCollector(user, path, site, useLimit) {
}
}
data[u] = { followers: [], following: [] };
p.push(async function (k) {
p.push(async function (k, u) {
let inLegal = penv.indexOf(u) != -1;//legal.indexOf(u) != -1;
let j1 = await siteCollector(u, 'followers', site, inLegal);
data[u].followers = j1;
console.log(`User ${u} followers fully calculated`);
p[k] = 'hi';
}(p.length));
}(p.length, u));
p.push(async function (k, u) {
let inLegal = penv.indexOf(u) != -1;//legal.indexOf(u) != -1;
p.push(async function (k) {
let j1 = await siteCollector(u, 'following', site, inLegal);
data[u].following = j1;
console.log(`User ${u} following fully calculated`);
p[k] = 'hi';
}(p.length));
}(p.length,u));
}
p = p.concat(congested);
@ -209,14 +211,14 @@ async function siteCollector(user, path, site, useLimit) {
}
for (let f of followers) {
if (!data[f]) data[f] = {followers: []};
if (!data[f]) data[f] = { followers: [] };
if (!data[f].following) data[f].following = [];
if (data[f].following.indexOf(uf) == -1) {
data[f].following.push(uf);
}
}
for (let f of following) {
if (!data[f]) data[f] = {following: []};
if (!data[f]) data[f] = { following: [] };
if (!data[f].followers) data[f].followers = [];
if (data[f].followers.indexOf(uf) == -1) {
data[f].followers.push(uf);