101 lines
2.6 KiB
JavaScript
101 lines
2.6 KiB
JavaScript
|
|
import { writeFile } from "fs/promises";
|
|
import { rankCalc } from "./rank.js";
|
|
|
|
let cache = {};
|
|
let site = process.env.site || 'darflen';
|
|
|
|
async function siteCollector(user, path, site) {
|
|
let users = [];
|
|
let i = 1;
|
|
while (true) {
|
|
let p;
|
|
|
|
if (site == 'darflen') {
|
|
p = `https://api.darflen.com/users/${user}/${path}/${i}`;
|
|
} else if (site == 'scratch') {
|
|
p = `https://api.scratch.mit.edu/users/${user}/${path}/?limit=40&offset=${i * 40}`;
|
|
} else {
|
|
throw 'That site is not supported.';
|
|
}
|
|
|
|
let j1 = cache[p];
|
|
if (!j1) {
|
|
let h1 = await fetch(p);
|
|
try {
|
|
j1 = await h1.json();
|
|
} catch(err) {
|
|
j1 = [];
|
|
}
|
|
}
|
|
cache[p] = j1;
|
|
|
|
let users2;
|
|
if (site == 'darflen') {
|
|
users2 = j1[path].map(x => x.profile.username);
|
|
} else if (site == 'scratch') {
|
|
users2 = j1.map(x => x.username);
|
|
}
|
|
|
|
users = users.concat(users2);
|
|
|
|
if (users2.length == 0) break;
|
|
|
|
i++;
|
|
|
|
console.log(`User ${user} has ${i} pages calculated`);
|
|
}
|
|
return users;
|
|
}
|
|
|
|
(async function () {
|
|
let users = await siteCollector(process.env.user || 'paradock', 'followers', site);
|
|
|
|
let data = {};
|
|
let p = [];
|
|
let fails = users;
|
|
let congested = [];
|
|
|
|
for (let u of users) {
|
|
let it = 0;
|
|
while (p.length >= process.env.maxRate || 15) {
|
|
p = p.filter(x => x != 'hi');
|
|
if (p.length == 0) break;
|
|
let pv = await Promise.any(p);
|
|
it++;
|
|
if (it > 10) {
|
|
console.warn(`Promises got congested. Moving to another array...`)
|
|
congested = congested.concat(p);
|
|
p = [];
|
|
break;
|
|
}
|
|
}
|
|
let i = p.length;
|
|
p[i] = (async function (i) {
|
|
let j1 = await siteCollector(u, 'followers', site);
|
|
|
|
let j2 = await siteCollector(u, 'following', site);
|
|
|
|
data[u] = {
|
|
followers: j1,
|
|
following: j2,
|
|
}
|
|
console.log(`User ${u} fully calculated`);
|
|
|
|
p[i] = 'hi';
|
|
})(i);
|
|
}
|
|
|
|
p = p.concat(congested);
|
|
await Promise.all(p);
|
|
|
|
let dat = Object.entries(rankCalc(data, 100));
|
|
dat = dat.sort((a, b) => a[1] - b[1]);
|
|
let dat2 = {};
|
|
for (let d of dat) {
|
|
dat2[d[0]] = d[1] * 100 + "%";
|
|
}
|
|
|
|
let srz = JSON.stringify(dat2);
|
|
await writeFile(`./users.json`, srz, 'utf8');
|
|
})()
|