137 lines
3.5 KiB
JavaScript
137 lines
3.5 KiB
JavaScript
|
|
import { writeFile, readFile } from "fs/promises";
|
|
import { rankCalc } from "./rank.js";
|
|
|
|
let cache = {};
|
|
let site = process.env.site || 'darflen';
|
|
let route = process.env.route || './test.txt';
|
|
let pageLimit = process.env.pageLimit || Infinity;
|
|
let hh;
|
|
|
|
async function textCollector(word, path, file) {
|
|
if (!hh) hh = await readFile(file, 'utf8');
|
|
|
|
hh = hh.toLowerCase();
|
|
|
|
let words = hh.split(/[\n.]+/g).filter(x => word.length == 0 || ` ${x} `.includes(` ${word} `));
|
|
words = words.join(' ').split(/[^a-zA-Z0-9']+/g);
|
|
words = [...new Set(words)];
|
|
return words;
|
|
}
|
|
|
|
async function siteCollector(user, path, site) {
|
|
let users = [];
|
|
let i = 1;
|
|
let out = [];
|
|
|
|
if (site == 'file') {
|
|
return await textCollector(user, path, route);
|
|
}
|
|
|
|
while (true) {
|
|
let p;
|
|
|
|
if (site == 'darflen') {
|
|
p = `https://api.darflen.com/users/${user}/${path}/${i}`;
|
|
} else if (site == 'scratch') {
|
|
p = `https://api.scratch.mit.edu/users/${user}/${path}/?limit=40&offset=${i * 40}`;
|
|
} else {
|
|
throw 'That site is not supported.';
|
|
}
|
|
|
|
let j1 = cache[p];
|
|
if (!j1) {
|
|
let h1 = await fetch(p);
|
|
try {
|
|
j1 = await h1.json();
|
|
} catch (err) {
|
|
j1 = [];
|
|
}
|
|
}
|
|
cache[p] = j1;
|
|
|
|
if (!j1.map || i > pageLimit) break;
|
|
|
|
let users2;
|
|
if (site == 'darflen') {
|
|
users2 = j1[path].map(x => x.profile.username);
|
|
} else if (site == 'scratch') {
|
|
users2 = j1.map(x => x.username);
|
|
}
|
|
|
|
users.push(users2);
|
|
|
|
if (users2.length == 0) break;
|
|
|
|
i++;
|
|
|
|
console.log(`User ${user} has ${i} pages calculated`);
|
|
}
|
|
out = out.concat(...users);
|
|
return out;
|
|
}
|
|
|
|
(async function () {
|
|
let penv = process.env.user || 'paradock';
|
|
penv = penv.split(',');
|
|
|
|
let users = [];
|
|
|
|
for (let u of penv) {
|
|
users = users.concat(await siteCollector(u, 'followers', site));
|
|
users = users.concat(await siteCollector(u, 'following', site));
|
|
}
|
|
|
|
users = [...new Set(users)];
|
|
|
|
let data = {};
|
|
let p = [];
|
|
let congested = [];
|
|
|
|
for (let u of users) {
|
|
let it = 0;
|
|
while (p.length >= (process.env.maxRate || 15)) {
|
|
p = p.filter(x => x != 'hi');
|
|
if (p.length == 0) break;
|
|
let pv = await Promise.any(p);
|
|
it++;
|
|
if (it > 10) {
|
|
console.warn(`Promises got congested. Moving to another array...`)
|
|
congested = congested.concat(p);
|
|
p = [];
|
|
break;
|
|
}
|
|
}
|
|
data[u] = { followers: [], following: [] };
|
|
p.push(async function (k) {
|
|
let j1 = await siteCollector(u, 'followers', site);
|
|
|
|
data[u].followers = j1;
|
|
console.log(`User ${u} followers fully calculated`);
|
|
|
|
p[k] = 'hi';
|
|
}(p.length));
|
|
|
|
p.push(async function (k) {
|
|
let j1 = await siteCollector(u, 'following', site);
|
|
|
|
data[u].following = j1;
|
|
console.log(`User ${u} following fully calculated`);
|
|
|
|
p[k] = 'hi';
|
|
}(p.length));
|
|
}
|
|
|
|
p = p.concat(congested);
|
|
await Promise.all(p);
|
|
|
|
let dat = Object.entries(rankCalc(data, 100, penv));
|
|
dat = dat.sort((a, b) => a[1] - b[1]);
|
|
let dat2 = {};
|
|
for (let d of dat) {
|
|
dat2[d[0]] = d[1] * 100 + "%";
|
|
}
|
|
|
|
let srz = JSON.stringify(dat2);
|
|
await writeFile(`./users.json`, srz, 'utf8');
|
|
})()
|