114 lines
2.8 KiB
JavaScript
114 lines
2.8 KiB
JavaScript
|
|
import { writeFile } from "fs/promises";
|
|
import { rankCalc } from "./rank.js";
|
|
|
|
let cache = {};
|
|
let site = process.env.site || 'darflen';
|
|
|
|
async function siteCollector(user, path, site) {
|
|
let users = [];
|
|
let i = 1;
|
|
let out = [];
|
|
while (true) {
|
|
let p;
|
|
|
|
if (site == 'darflen') {
|
|
p = `https://api.darflen.com/users/${user}/${path}/${i}`;
|
|
} else if (site == 'scratch') {
|
|
p = `https://api.scratch.mit.edu/users/${user}/${path}/?limit=40&offset=${i * 40}`;
|
|
} else {
|
|
throw 'That site is not supported.';
|
|
}
|
|
|
|
let j1 = cache[p];
|
|
if (!j1) {
|
|
let h1 = await fetch(p);
|
|
try {
|
|
j1 = await h1.json();
|
|
} catch (err) {
|
|
j1 = [];
|
|
}
|
|
}
|
|
cache[p] = j1;
|
|
|
|
let users2;
|
|
if (site == 'darflen') {
|
|
users2 = j1[path].map(x => x.profile.username);
|
|
} else if (site == 'scratch') {
|
|
users2 = j1.map(x => x.username);
|
|
}
|
|
|
|
users.push(users2);
|
|
|
|
if (users2.length == 0) break;
|
|
|
|
i++;
|
|
|
|
console.log(`User ${user} has ${i} pages calculated`);
|
|
}
|
|
out = out.concat(...users);
|
|
return out;
|
|
}
|
|
|
|
(async function () {
|
|
let penv = process.env.user || 'paradock';
|
|
penv = penv.split(',')
|
|
|
|
let users = [];
|
|
|
|
for (let u of penv) {
|
|
users = users.concat(await siteCollector(u, 'followers', site))
|
|
}
|
|
|
|
let data = {};
|
|
let p = [];
|
|
let fails = users;
|
|
let congested = [];
|
|
|
|
for (let u of users) {
|
|
let it = 0;
|
|
while (p.length >= (process.env.maxRate || 15)) {
|
|
p = p.filter(x => x != 'hi');
|
|
if (p.length == 0) break;
|
|
let pv = await Promise.any(p);
|
|
it++;
|
|
if (it > 10) {
|
|
console.warn(`Promises got congested. Moving to another array...`)
|
|
congested = congested.concat(p);
|
|
p = [];
|
|
break;
|
|
}
|
|
}
|
|
data[u] = {followers: [], following: []};
|
|
p.push(async function (k) {
|
|
let j1 = await siteCollector(u, 'followers', site);
|
|
|
|
data[u].followers = j1;
|
|
console.log(`User ${u} followers fully calculated`);
|
|
|
|
p[k] = 'hi';
|
|
}(p.length));
|
|
|
|
p.push(async function (k) {
|
|
let j1 = await siteCollector(u, 'following', site);
|
|
|
|
data[u].following = j1;
|
|
console.log(`User ${u} following fully calculated`);
|
|
|
|
p[k] = 'hi';
|
|
}(p.length));
|
|
}
|
|
|
|
p = p.concat(congested);
|
|
await Promise.all(p);
|
|
|
|
let dat = Object.entries(rankCalc(data, 100));
|
|
dat = dat.sort((a, b) => a[1] - b[1]);
|
|
let dat2 = {};
|
|
for (let d of dat) {
|
|
dat2[d[0]] = d[1] * 100 + "%";
|
|
}
|
|
|
|
let srz = JSON.stringify(dat2);
|
|
await writeFile(`./users.json`, srz, 'utf8');
|
|
})()
|