archive/bigly-caret
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix threshold logic

Browse source
This commit is contained in:
biglyderv 2025-02-01 13:30:21 -05:00
parent 8531c671f4
commit 03b7527839
1 changed files with 16 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
site.js
View file

@ -52,7 +52,7 @@ async function urlCollector(url, path, file, useLimit, data2) {
try { try {
h2 = new URL(h); h2 = new URL(h);
} catch (err) { } catch (err) {
} }
if (!h2) return true; if (!h2) return true;
if (rel && h2.host != new URL(url).host) return true; if (rel && h2.host != new URL(url).host) return true;
@ -80,7 +80,7 @@ async function urlCollector(url, path, file, useLimit, data2) {
if (!data2[url]) data2[url] = { following: [], followers: [] }; if (!data2[url]) data2[url] = { following: [], followers: [] };
data2[url][path] = [...new Set(urls)]; data2[url][path] = [...new Set(urls)];
data2[url][path].sort((x,y) => ((new URL(x).host == new URL(url).host) ? 1 : 0) - ((new URL(y).host == new URL(url).host) ? 1 : 0)) data2[url][path].sort((x, y) => ((new URL(x).host == new URL(url).host) ? 1 : 0) - ((new URL(y).host == new URL(url).host) ? 1 : 0))
return data2[url][path]; return data2[url][path];
} }
@ -125,7 +125,7 @@ async function rounder(users, data, mode) {
} }
p.push(async function (k) { p.push(async function (k) {
await siteCollector(u, 'followers', site, mode, data); await siteCollector(u, 'followers', site, mode, data);
percent += 50 / d / users.length; percent += 50 / d / users.length;
console.log(`User ${u} followers is fully calculated (${percent}% total)`); console.log(`User ${u} followers is fully calculated (${percent}% total)`);
@ -153,7 +153,7 @@ async function rounder(users, data, mode) {
let oldLength = endn.length; let oldLength = endn.length;
for (let h in data) { for (let h in data) {
endn.push(h); endn.push(h);
if (endn.length > oldLength * 1.25 ) { if (endn.length > oldLength * 1.25) {
endn = [...new Set(endn)]; endn = [...new Set(endn)];
} }
} }
@ -260,18 +260,19 @@ async function siteCollector(user, path, site, useLimit, data2) {
for (let i = 0; i < d; i++) { for (let i = 0; i < d; i++) {
if (i != 0) { if (i != 0) {
let tempSet = dat.map(x => x[0]); let tempSet = dat.map(x => x[0]);
users = tempSet.concat(await rounder(tempSet, data, true)); let oldLength = Object.keys(data).length;
let theData = {};
for (let a = 0; a < oldLength * threshold && a < oldLength; a++) {
let key = Object.keys(data)[a];
theData[key] = data[key];
}
users = tempSet.concat(await rounder(tempSet, theData, true));
users = [...new Set(users)]; users = [...new Set(users)];
} }
let oldLength = Object.keys(data).length;
for (let uf of users) { for (let uf of users) {
if (Object.keys(data).length > oldLength * threshold) {
break;
}
let u = data[uf]; let u = data[uf];
if (!u) { if (!u) {
data[uf] = {following: [], followers: []} data[uf] = { following: [], followers: [] }
} }
let { following, followers } = u; let { following, followers } = u;
if (!following || !followers) continue; if (!following || !followers) continue;
@ -304,13 +305,13 @@ async function siteCollector(user, path, site, useLimit, data2) {
} }
console.log(`Graph is fully repaired`); console.log(`Graph is fully repaired`);
dat = Object.entries(rankCalc(data, (i == d - 1) ? process.env.matrixIterations : 3, penv, site == 'url')); dat = Object.entries(rankCalc(data, (i == d - 1) ? process.env.matrixIterations : 3, penv, site == 'url'));
dat = dat.sort((a, b) => b[1] - a[1]); dat = dat.sort((a, b) => b[1] - a[1]);
console.log(`Graph is calculated with ${dat.length} entries`); console.log(`Graph is calculated with ${dat.length} entries`);
let dat2 = {}; let dat2 = {};
for (let d of dat) { for (let d of dat) {
dat2[d[0]] = d[1] * 100 + "%"; dat2[d[0]] = d[1] * 100 + "%";
} }
let srz = JSON.stringify(dat2); let srz = JSON.stringify(dat2);
let ff = `./users_${i}.json`; let ff = `./users_${i}.json`;
await writeFile(ff, srz, 'utf8'); await writeFile(ff, srz, 'utf8');
@ -318,5 +319,5 @@ async function siteCollector(user, path, site, useLimit, data2) {
} }
console.log(`Graph is complete (${Object.keys(users).length} entries)`); console.log(`Graph is complete (${Object.keys(users).length} entries)`);
})() })()