bigly-caret/site.js

import { writeFile } from "fs/promises";
import { rankCalc } from "./rank.js";

let cache = {};
let site = process.env.site || 'darflen';

async function siteCollector(user, path, site) {
    let users = [];
    let i = 1;
    while (true) {
        let p;

        if (site == 'darflen') {
            p = `https://api.darflen.com/users/${user}/${path}/${i}`;
        } else if (site == 'scratch') {
            p = `https://api.scratch.mit.edu/users/${user}/${path}/?offset=${i * 20}`;
        } else {
            throw 'That site is not supported.';
        }

        let j1 = cache[p];
        if (!j1) {
            let h1 = await fetch(p);
            try {
                j1 = await h1.json();
            } catch(err) {
                j1 = [];
            }
        }
        cache[p] = j1;

        let users2;
        if (site == 'darflen') {
            users2 = j1[path].map(x => x.profile.username);
        } else if (site == 'scratch') {
            users2 = j1.map(x => x.username);
        }

        users = [...users, ...users2];

        if (users2.length == 0) break;

        i++;

        console.log(`User ${user} has ${i} pages calculated`);
    }
    return users;
}

(async function () {
    let users = await siteCollector(process.env.user || 'paradock', 'followers', site);

    let data = {};
    let p = [];
    for (let u of users) {
        if (p.length >= 10) {
            await Promise.all(p);
            p = [];
        }
        p.push((async function () {
            let j1 = await siteCollector(u, 'followers', site);

            let j2 = await siteCollector(u, 'following', site);

            data[u] = {
                followers: j1,
                following: j2,
            }
            console.log(`User ${u} fully calculated`);
        })())
    }

    await Promise.all(p);

    let dat = Object.entries(rankCalc(data, 100));
    dat = dat.sort((a, b) => a[1] - b[1]);
    let dat2 = {};
    for (let d of dat) {
        dat2[d[0]] = d[1] * 100 + "%";
    }

    let srz = JSON.stringify(dat2);
    await writeFile(`./users.json`, srz, 'utf8');
})()