diff --git a/site.js b/site.js index 0c2ef92..d124487 100644 --- a/site.js +++ b/site.js @@ -19,7 +19,7 @@ async function textCollector(word, path, file) { return words; } -async function siteCollector(user, path, site) { +async function siteCollector(user, path, site, useLimit) { let users = []; let i = 1; let out = []; @@ -50,7 +50,7 @@ async function siteCollector(user, path, site) { } cache[p] = j1; - if (!j1.map || i > pageLimit) break; + if (!j1.map || (i > pageLimit && useLimit)) break; let users2; if (site == 'darflen') { @@ -76,9 +76,12 @@ async function siteCollector(user, path, site) { penv = penv.split(','); let users = []; - + let legal = []; + for (let u of penv) { - users = users.concat(await siteCollector(u, 'followers', site)); + let ca = await siteCollector(u, 'followers', site); + users = users.concat(ca); + legal = legal.concat(ca); users = users.concat(await siteCollector(u, 'following', site)); } @@ -89,6 +92,8 @@ async function siteCollector(user, path, site) { let congested = []; for (let u of users) { + let inLegal = legal.indexOf(u) != -1; + let it = 0; while (p.length >= (process.env.maxRate || 15)) { p = p.filter(x => x != 'hi'); @@ -104,7 +109,7 @@ async function siteCollector(user, path, site) { } data[u] = { followers: [], following: [] }; p.push(async function (k) { - let j1 = await siteCollector(u, 'followers', site); + let j1 = await siteCollector(u, 'followers', site,inLegal); data[u].followers = j1; console.log(`User ${u} followers fully calculated`); @@ -113,7 +118,7 @@ async function siteCollector(user, path, site) { }(p.length)); p.push(async function (k) { - let j1 = await siteCollector(u, 'following', site); + let j1 = await siteCollector(u, 'following', site,inLegal); data[u].following = j1; console.log(`User ${u} following fully calculated`);