archive/bigly-caret
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

better handling of large arrays

Browse source
This commit is contained in:
biglyderv 2025-02-06 16:38:50 -05:00
parent 8f29cd70fe
commit 97c440fd8b
2 changed files with 15 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
rank.js
View file

@ -15,7 +15,7 @@ function multiplyMatrix(a, b, c) {
} }
// derived from https://git.dervland.net/biglyderv/new-bigly-chat/src/branch/master/docs/stats.php // derived from https://git.dervland.net/biglyderv/new-bigly-chat/src/branch/master/docs/stats.php
function rankCalc(result, iterations = 10, main = [], domainMode = false, isGpu = false, arrayMax = 800) { function rankCalc(result, iterations = 10, main = [], domainMode = false, isGpu = false, arrayMax = 800, oldVals = false) {
let fng = {}; let fng = {};
let fnc = {}; let fnc = {};
@ -25,22 +25,28 @@ function rankCalc(result, iterations = 10, main = [], domainMode = false, isGpu
let keys = Object.keys(result); let keys = Object.keys(result);
let leftover = []; let leftover = [];
keys = keys.sort((a,b) => result[a].followers.length < result[b].followers.length);
if (oldVals) {
keys = keys.sort((a, b) => oldVals[b] - oldVals[a]);
} else {
keys = keys.sort((a, b) => result[b].followers.length - result[a].followers.length);
}
let kl2 = keys.length; let kl2 = keys.length;
if (kl2 > arrayMax) { if (kl2 > arrayMax) {
console.warn(`Array too big. Splitting into multiple arrays...`); console.warn(`Array too big. Splitting into multiple arrays...`);
let ll = {}; let ll = {};
let hh = keys.slice(arrayMax); let hh = keys.slice(arrayMax);
for (let g of hh) { for (let g of hh) {
ll[g] = result[g]; ll[g] = result[g];
} }
leftover = rankCalc(ll,iterations,main,domainMode,isGpu,arrayMax); leftover = rankCalc(ll, iterations, main, domainMode, isGpu, arrayMax);
for (let i in leftover) { for (let i in leftover) {
leftover[i] /= (kl2 / arrayMax) * 3; leftover[i] /= (kl2 / arrayMax);
} }
} }
keys.length = Math.min(keys.length,1000); keys.length = Math.min(keys.length, 1000);
let rl = keys.length; let rl = keys.length;
rls = rl ** 2; rls = rl ** 2;
@ -163,7 +169,7 @@ function rankCalc(result, iterations = 10, main = [], domainMode = false, isGpu
} }
} }
pr = Object.assign(pr,leftover); pr = Object.assign(pr, leftover);
let ov = Object.keys(pr); let ov = Object.keys(pr);
let an = ov.filter(i => !isNaN(pr[i]) && main.indexOf(i) != -1); let an = ov.filter(i => !isNaN(pr[i]) && main.indexOf(i) != -1);

3
site.js
View file

@ -249,6 +249,7 @@ async function siteCollector(user, path, site, useLimit, data2, settings) {
} }
async function main(settings) { async function main(settings) {
let calcedRank = {};
let { site, discardThreshold, depth, user, matrixIterations, useArchive, } = settings; let { site, discardThreshold, depth, user, matrixIterations, useArchive, } = settings;
user = user.split(','); user = user.split(',');
@ -317,7 +318,7 @@ async function main(settings) {
} }
} }
console.log(`Graph is fully repaired`); console.log(`Graph is fully repaired`);
let calcedRank = rankCalc(data, (i == depth - 1) ? matrixIterations : 3, user, site == 'url', settings.isGpu, settings.arrayMax) calcedRank = rankCalc(data, (i == depth - 1) ? matrixIterations : 3, user, site == 'url', settings.isGpu, settings.arrayMax, calcedRank)
dat = Object.entries(calcedRank); dat = Object.entries(calcedRank);
dat = dat.sort((a, b) => b[1] - a[1]); dat = dat.sort((a, b) => b[1] - a[1]);
console.log(`Graph is calculated with ${dat.length} entries`); console.log(`Graph is calculated with ${dat.length} entries`);