From 316d8d0dedd1698f0e8147567d29347d7aa40378 Mon Sep 17 00:00:00 2001 From: biglyderv Date: Thu, 30 Jan 2025 09:37:33 -0500 Subject: [PATCH] way faster --- site.js | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/site.js b/site.js index 04b3264..6c1594d 100644 --- a/site.js +++ b/site.js @@ -14,7 +14,7 @@ async function siteCollector(user, path, site) { if (site == 'darflen') { p = `https://api.darflen.com/users/${user}/${path}/${i}`; } else if (site == 'scratch') { - p = `https://api.scratch.mit.edu/users/${user}/${path}/?offset=${i * 20}`; + p = `https://api.scratch.mit.edu/users/${user}/${path}/?limit=40&offset=${i * 40}`; } else { throw 'That site is not supported.'; } @@ -37,7 +37,7 @@ async function siteCollector(user, path, site) { users2 = j1.map(x => x.username); } - users = [...users, ...users2]; + users = users.concat(users2); if (users2.length == 0) break; @@ -53,12 +53,25 @@ async function siteCollector(user, path, site) { let data = {}; let p = []; + let fails = users; + let congested = []; + for (let u of users) { - if (p.length >= 10) { - await Promise.all(p); - p = []; + let it = 0; + while (p.length >= 15) { + p = p.filter(x => x != 'hi'); + if (p == []) break; + let pv = await Promise.any(p); + it++; + if (it > 10) { + console.warn(`Promises got congested. Moving to another array...`) + congested = congested.concat(p); + p = []; + break; + } } - p.push((async function () { + let i = p.length; + p[i] = (async function (i) { let j1 = await siteCollector(u, 'followers', site); let j2 = await siteCollector(u, 'following', site); @@ -68,9 +81,12 @@ async function siteCollector(user, path, site) { following: j2, } console.log(`User ${u} fully calculated`); - })()) + + p[i] = 'hi'; + })(i); } + p = p.concat(congested); await Promise.all(p); let dat = Object.entries(rankCalc(data, 100));