archive/bigly-caret
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

pagelimit on folowing users only

Browse source
This commit is contained in:
biglyderv 2025-01-31 18:05:21 -05:00
parent 393b8a4e54
commit c1a08a71c2
1 changed files with 11 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
site.js
View file

@ -19,7 +19,7 @@ async function textCollector(word, path, file) {
return words; return words;
} }
async function siteCollector(user, path, site) { async function siteCollector(user, path, site, useLimit) {
let users = []; let users = [];
let i = 1; let i = 1;
let out = []; let out = [];
@ -50,7 +50,7 @@ async function siteCollector(user, path, site) {
} }
cache[p] = j1; cache[p] = j1;
if (!j1.map || i > pageLimit) break; if (!j1.map || (i > pageLimit && useLimit)) break;
let users2; let users2;
if (site == 'darflen') { if (site == 'darflen') {
@ -76,9 +76,12 @@ async function siteCollector(user, path, site) {
penv = penv.split(','); penv = penv.split(',');
let users = []; let users = [];
let legal = [];
for (let u of penv) { for (let u of penv) {
users = users.concat(await siteCollector(u, 'followers', site)); let ca = await siteCollector(u, 'followers', site);
users = users.concat(ca);
legal = legal.concat(ca);
users = users.concat(await siteCollector(u, 'following', site)); users = users.concat(await siteCollector(u, 'following', site));
} }
@ -89,6 +92,8 @@ async function siteCollector(user, path, site) {
let congested = []; let congested = [];
for (let u of users) { for (let u of users) {
let inLegal = legal.indexOf(u) != -1;
let it = 0; let it = 0;
while (p.length >= (process.env.maxRate || 15)) { while (p.length >= (process.env.maxRate || 15)) {
p = p.filter(x => x != 'hi'); p = p.filter(x => x != 'hi');
@ -104,7 +109,7 @@ async function siteCollector(user, path, site) {
} }
data[u] = { followers: [], following: [] }; data[u] = { followers: [], following: [] };
p.push(async function (k) { p.push(async function (k) {
let j1 = await siteCollector(u, 'followers', site); let j1 = await siteCollector(u, 'followers', site,inLegal);
data[u].followers = j1; data[u].followers = j1;
console.log(`User ${u} followers fully calculated`); console.log(`User ${u} followers fully calculated`);
@ -113,7 +118,7 @@ async function siteCollector(user, path, site) {
}(p.length)); }(p.length));
p.push(async function (k) { p.push(async function (k) {
let j1 = await siteCollector(u, 'following', site); let j1 = await siteCollector(u, 'following', site,inLegal);
data[u].following = j1; data[u].following = j1;
console.log(`User ${u} following fully calculated`); console.log(`User ${u} following fully calculated`);