biglyderv/bigly-caret
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

dumbest mistake ever

Browse source
This commit is contained in:
biglyderv 2025-02-01 11:43:39 -05:00
parent c8cba84c19
commit d2bb8524bc
2 changed files with 25 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
rank.js
View file

@ -89,7 +89,7 @@ function rankCalc(result, iterations = 10, main = [], domain_mode = false) {
for (let i = 0; i < mm; i++) {
let prold = pr;
pr = [];
pr = {};
let msum = 0;
console.log(`Completed ${i} iterations`)

48
site.js
View file

@ -15,7 +15,9 @@ let percent = 0;
let d = process.env.depth || 1;
async function urlCollector(url, path, file, useLimit, data2) {
if (path != 'following') return [];
let urls = data2[url] ? (data2[url][path] || []) : [];
urls = [...urls];
if (path != 'following') return urls;
let data;
try {
@ -24,17 +26,18 @@ async function urlCollector(url, path, file, useLimit, data2) {
console.log(`User ${url} was fetched`);
} catch (err) {
console.warn(`User ${url} did not fetch`);
return [];
return urls;
}
let body = cheerio.load(data);
let links = body('a');
let urls = data2[url] ? (data2[url][path] || []) : [];
let ll = urls.length;
links.each(function (i, link) {
if (useLimit && urls.length >= pageLimit * 40 + ll) return;
if (useLimit && urls.length >= pageLimit * 40 + ll) {
return false;
}
let h = body(link).attr('href');
if (!h) return;
if (!h) return true;
h = h.trim();
if (h.startsWith('./') || h.startsWith('../') || h.startsWith('/')) {
let u = new URL(url);
@ -45,15 +48,17 @@ async function urlCollector(url, path, file, useLimit, data2) {
try {
h2 = new URL(h);
} catch (err) {
return;
}
if (!h2) return true;
for (let g of greylist) {
if (h2.toString().includes(g) && g != '') return;
if (h2.toString().includes(g) && g != '') return true;
}
if (blacklist.indexOf(h2.toString()) != -1) return;
if (blacklist.indexOf(h2.toString()) != -1) return true;
urls.push(h2.toString());
urls = [...new Set(urls)];
console.log(`User ${url} has ${urls.length} pages calculated`);
return true;
})
let h3;
@ -64,9 +69,9 @@ async function urlCollector(url, path, file, useLimit, data2) {
}
urls.push(h3.toString())
if (!data2[url]) data2[url] = { following: [], followers: [] };
data2[url][path] = urls;
data2[url][path] = [...new Set(urls)];
return [...new Set(urls)];
return data2[url][path];
}
async function rounder(users, data, mode) {
@ -99,7 +104,7 @@ async function rounder(users, data, mode) {
}
}
if (timeout) break;
data[u] = { followers: [], following: [] };
if (!data[u]) data[u] = { followers: [], following: [] };
p.push(async function (k) {
await siteCollector(u, 'followers', site, mode, data);
@ -129,16 +134,8 @@ async function rounder(users, data, mode) {
let endn = [...users];
let oldLength = endn.length;
for (let h in data) {
let dh = data[h];
if (dh.following.length == 0 && dh.followers.length == 0) continue;
let ffing = [...dh.following];
let ffers = [...dh.followers];
if (mode) {
ffing.length = Math.min(ffing.length, userLimit);
ffers.length = Math.min(ffers.length, userLimit);
}
endn = endn.concat(ffing, ffers);
if (endn.length > oldLength * 1.25) {
endn.push(h);
if (endn.length > oldLength * 1.25 || endn.length - oldLength > userLimit) {
endn = [...new Set(endn)];
}
if (mode && endn.length - oldLength > userLimit) break;
@ -166,7 +163,8 @@ async function textCollector(word, path, file) {
async function siteCollector(user, path, site, useLimit, data2) {
let users = [];
let urls = data2[user] ? (data2[user][path] || []) : [];
let i = 1 + (urls.length || 0);
let ul = urls.length;
let i = 1 + (ul || 0);
let out = [];
if (site == 'file') {
@ -199,7 +197,7 @@ async function siteCollector(user, path, site, useLimit, data2) {
}
cache[p] = j1;
if (!j1 || ((i - urls.length - 1) >= pageLimit && useLimit)) break;
if (!j1 || ((i - ul - 1) >= pageLimit && useLimit)) break;
let users2;
try {
@ -250,7 +248,9 @@ async function siteCollector(user, path, site, useLimit, data2) {
}
for (let uf of users) {
let u = data[uf];
if (!u) continue;
if (!u) {
data[uf] = {following: [], followers: []}
}
let { following, followers } = u;
if (!following || !followers) continue;