archive/bigly-caret
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

test

Browse source
This commit is contained in:
dervperson 2025-02-01 00:43:00 -05:00
parent f0f70535d9
commit 0c070c5aec
1 changed files with 62 additions and 53 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

115
site.js
View file

@ -20,7 +20,7 @@ async function urlCollector(url, path, file, useLimit, data2) {
data = await data.text(); data = await data.text();
console.log(`User ${url} was fetched`); console.log(`User ${url} was fetched`);
} catch (err) { } catch (err) {
console.log(`User ${url} did not fetch`); console.warn(`User ${url} did not fetch`);
return []; return [];
} }
@ -67,6 +67,64 @@ async function urlCollector(url, path, file, useLimit, data2) {
return [...new Set(urls)]; return [...new Set(urls)];
} }
async function rounder(users, data, mode) {
let p = [];
let congested = [];
let timeout = false;
setTimeout(function() {
timeout = true;
},(process.env.delay*1000) || (60*1000));
for (let u of users) {
if (timeout) break;
let it = 0;
while (p.length >= (process.env.maxRate || 15)) {
p = p.filter(x => x != 'hi');
if (p.length == 0) break;
let pv = await Promise.any(p);
it++;
if (it > 10) {
console.warn(`Promises got congested. Moving to another array...`)
congested = congested.concat(p);
p = [];
break;
}
}
data[u] = { followers: [], following: [] };
p.push(async function (k) {
await siteCollector(u, 'followers', site, mode, data);
console.log(`User ${u} followers fully calculated`);
p[k] = 'hi';
}(p.length));
p.push(async function (k) {
await siteCollector(u, 'following', site, mode, data);
console.log(`User ${u} following fully calculated`);
p[k] = 'hi';
}(p.length));
}
if (!timeout) {
p = p.concat(congested);
await Promise.all(p);
} else {
console.warn('Took too long...')
}
let endn = [];
for (let h in data) {
let dh = data[h];
endn = endn.concat(dh.following,dh.followers);
endn = [...new Set(endn)];
}
return endn;
}
async function textCollector(word, path, file) { async function textCollector(word, path, file) {
if (!hh) hh = await readFile(file, 'utf8'); if (!hh) hh = await readFile(file, 'utf8');
@ -147,66 +205,17 @@ async function siteCollector(user, path, site, useLimit, data2) {
let users = []; let users = [];
let data = {}; let data = {};
for (let u of penv) { users = await rounder(penv,data,false);
let ca = await siteCollector(u, 'followers', site, false, data);
users = users.concat(ca);
users = users.concat(await siteCollector(u, 'following', site, false, data));
}
users = [...new Set(users)]; users = [...new Set(users)];
let d = process.env.depth || 1; let d = process.env.depth || 1;
for (let i = 1; i < d; i++) { for (let i = 1; i < d; i++) {
users = [...new Set(users)];
let tempSet = [...users]; let tempSet = [...users];
for (let u of tempSet) { users = await rounder(tempSet,data,true);
users = users.concat(await siteCollector(u, 'followers', site, true, data)); users = [...new Set(users)];
users = users.concat(await siteCollector(u, 'following', site, true, data));
}
} }
let p = [];
let congested = [];
for (let u of users) {
let it = 0;
while (p.length >= (process.env.maxRate || 15)) {
p = p.filter(x => x != 'hi');
if (p.length == 0) break;
let pv = await Promise.any(p);
it++;
if (it > 10) {
console.warn(`Promises got congested. Moving to another array...`)
congested = congested.concat(p);
p = [];
break;
}
}
data[u] = { followers: [], following: [] };
p.push(async function (k) {
await siteCollector(u, 'followers', site, true, data);
console.log(`User ${u} followers fully calculated`);
p[k] = 'hi';
}(p.length));
p.push(async function (k) {
await siteCollector(u, 'following', site, true, data);
console.log(`User ${u} following fully calculated`);
p[k] = 'hi';
}(p.length));
}
p = p.concat(congested);
await Promise.any([
new Promise(resolve => setTimeout(resolve, process.env.delay || 1000*60)),
Promise.all(p)
]);
for (let uf in data) { for (let uf in data) {
let u = data[uf]; let u = data[uf];
if (!u) continue; if (!u) continue;