bigly-caret/site.js

import { writeFile, readFile } from "fs/promises";
import { rankCalc } from "./rank.js";
import * as cheerio from 'cheerio';

let cache = {};
let site = process.env.site || 'darflen';
let route = process.env.route || './test.txt';
let pageLimit = process.env.pageLimit || Infinity;
let hh;

async function urlCollector(url, path, file, useLimit) {
    if (path != 'following') return [];

    let data;
    try {
        data = await fetch(url);
        data = await data.text();
    } catch (err) {
        return [];
    }

    let body = cheerio.load(data);
    let links = body('a');
    let urls = [];
    links.each(function(i, link) {
        if (useLimit && urls.length >= pageLimit * 40) return;

        let h = body(link).attr('href');
        if (!h)  return;
        h = h.trim();
        if (h.startsWith('./') || h.startsWith('../') || h.startsWith('/')) {
            let u = new URL(url);
            u.pathname = h;
            h = u.toString();
        }
        let h2, h3;
        try {
            h2 = new URL(h);
            h3 = new URL(h2.origin);
        } catch(err) {
            return;
        }
        urls.push(h2.toString())
        urls.push(h3.toString())
    })

    return [...new Set(urls)];
}

async function textCollector(word, path, file) {
    if (!hh) hh = await readFile(file, 'utf8');

    hh = hh.toLowerCase();

    let words = hh.split(/[\n.]+/g).filter(x => word.length == 0 || ` ${x} `.includes(` ${word} `));
    words = words.join(' ').split(/[^a-zA-Z0-9']+/g);
    words = [...new Set(words)];
    return words;
}

async function siteCollector(user, path, site, useLimit) {
    let users = [];
    let i = 1;
    let out = [];

    if (site == 'file') {
        return await textCollector(user, path, route);
    }

    if (site == 'url') {
        return await urlCollector(user, path, route, useLimit);
    }

    while (true) {
        let p;

        if (site == 'darflen') {
            p = `https://api.darflen.com/users/${user}/${path}/${i}`;
        } else if (site == 'scratch') {
            p = `https://api.scratch.mit.edu/users/${user}/${path}/?limit=40&offset=${i * 40}`;
        } else {
            throw 'That site is not supported.';
        }

        let j1 = cache[p];
        if (!j1) {
            let h1 = await fetch(p);
            try {
                j1 = await h1.json();
            } catch (err) {
                j1 = [];
            }
        }
        cache[p] = j1;

        if (!j1.map || (i >= pageLimit && useLimit)) break;

        let users2;
        if (site == 'darflen') {
            users2 = j1[path].map(x => x.profile.username);
        } else if (site == 'scratch') {
            users2 = j1.map(x => x.username);
        }

        users.push(users2);

        if (users2.length == 0) break;

        i++;

        console.log(`User ${user} has ${i} pages calculated`);
    }
    out = out.concat(...users);
    return out;
}

(async function () {
    let penv = process.env.user || 'paradock';
    penv = penv.split(',');

    let users = [];
    let legal = [];

    for (let u of penv) {
        let ca = await siteCollector(u, 'followers', site);
        users = users.concat(ca);
        legal = legal.concat(ca);

        users = users.concat(await siteCollector(u, 'following', site));
    }

    users = [...new Set(users)];

    let d = process.env.depth || 1;
    for (let i = 1; i < d; i++) {
        users = [...new Set(users)];
        let tempSet = [...users];
        for (let u of tempSet) {
            users = users.concat(await siteCollector(u, 'followers', site, true));
            users = users.concat(await siteCollector(u, 'following', site, true));
        }
    }

    let data = {};
    let p = [];
    let congested = [];

    for (let u of users) {
        let inLegal = legal.indexOf(u) != -1;

        let it = 0;
        while (p.length >= (process.env.maxRate || 15)) {
            p = p.filter(x => x != 'hi');
            if (p.length == 0) break;
            let pv = await Promise.any(p);
            it++;
            if (it > 10) {
                console.warn(`Promises got congested. Moving to another array...`)
                congested = congested.concat(p);
                p = [];
                break;
            }
        }
        data[u] = { followers: [], following: [] };
        p.push(async function (k) {
            let j1 = await siteCollector(u, 'followers', site, inLegal);

            data[u].followers = j1;
            console.log(`User ${u} followers fully calculated`);

            p[k] = 'hi';
        }(p.length));

        p.push(async function (k) {
            let j1 = await siteCollector(u, 'following', site, inLegal);

            data[u].following = j1;
            console.log(`User ${u} following fully calculated`);

            p[k] = 'hi';
        }(p.length));
    }

    p = p.concat(congested);
    await Promise.all(p);

    for (let uf in data) {
        let u = data[uf];
        if (!u) continue;
        let { following, followers } = u;
        if (!following || !followers) continue;

        try {
            let o = new URL(uf);

            if (followers.indexOf(o.origin) == -1) {
                followers.push(o.origin)
            }
        } catch (err) {
        }

        for (let f of followers) {
            if (!data[f]) data[f] = {followers: []};
            if (!data[f].following) data[f].following = [];
            if (data[f].following.indexOf(uf) == -1) {
                data[f].following.push(uf);
            }
        }
        for (let f of following) {
            if (!data[f]) data[f] = {following: []};
            if (!data[f].followers) data[f].followers = [];
            if (data[f].followers.indexOf(uf) == -1) {
                data[f].followers.push(uf);
            }
        }
    }

    let dat = Object.entries(rankCalc(data, 100, penv));
    dat = dat.sort((a, b) => a[1] - b[1]);
    let dat2 = {};
    for (let d of dat) {
        dat2[d[0]] = d[1] * 100 + "%";
    }

    let srz = JSON.stringify(dat2);
    await writeFile(`./users.json`, srz, 'utf8');
})()