From 6639a1b0700677d609dbbef541be0b81b86bd107 Mon Sep 17 00:00:00 2001 From: biglyderv Date: Thu, 6 Feb 2025 13:54:37 -0500 Subject: [PATCH] dissolve huge arrays --- index.js | 8 +++--- rank.js | 82 +++++++++++++++++++++++++++++++++----------------------- site.js | 18 +++++-------- 3 files changed, 60 insertions(+), 48 deletions(-) diff --git a/index.js b/index.js index 7856819..c94470c 100644 --- a/index.js +++ b/index.js @@ -17,11 +17,13 @@ let { useArchive = true, isGpu = false, mode = 'cli', - port = 6952 + port = 6952, + arrayMax = 800 } = process.env; let settings = { site, route, pageLimit, blackList, greyList, discardThreshold, delay, - depth, isRelative, fetchRate, user, matrixIterations, useArchive, isGpu + depth, isRelative, fetchRate, user, matrixIterations, useArchive, isGpu, + arrayMax }; if (mode == 'cli') { @@ -52,7 +54,7 @@ if (mode == 'cli') { data[entry.username] = user; data[entry.target] = target; } - let calcedRank = rankCalc(data, matrixIterations, ref ? [ref] : [], false, settings.isGpu); + let calcedRank = rankCalc(data, matrixIterations, ref ? [ref] : [], false, settings.isGpu, settings.arrayMax); let dat = Object.entries(calcedRank); dat = dat.sort((a, b) => b[1] - a[1]); console.log(`Graph is calculated with ${dat.length} entries`); diff --git a/rank.js b/rank.js index c976c06..9e2c689 100644 --- a/rank.js +++ b/rank.js @@ -1,7 +1,21 @@ import { GPU, input, Input } from "gpu.js"; +var rls; + +function multiplyMatrix(a, b, c) { + let outMatrix = new Float32Array(rls); + for (let i in outMatrix) { + let sum = 0; + for (let j = 0; j < c; j++) { + sum += a[(i % c) * c + j] * b[j * c + Math.floor(i / c)]; + } + outMatrix[i] = sum; + } + return [outMatrix]; +} + // derived from https://git.dervland.net/biglyderv/new-bigly-chat/src/branch/master/docs/stats.php -function rankCalc(result, iterations = 10, main = [], domainMode = false, isGpu = false) { +function rankCalc(result, iterations = 10, main = [], domainMode = false, isGpu = false, arrayMax = 800) { let fng = {}; let fnc = {}; @@ -9,25 +23,37 @@ function rankCalc(result, iterations = 10, main = [], domainMode = false, isGpu let msum_old = 0.001; let pr = {}; - let rl = Object.keys(result).length; - let matrixe = new Float32Array(rl ** 2); + let keys = Object.keys(result); + let leftover = []; + keys = keys.sort((a,b) => result[a].followers.length < result[b].followers.length); + if (keys.length > arrayMax) { + console.warn(`Array too big. Splitting into multiple arrays...`); + let ll = {}; + let hh = keys.slice(arrayMax); + for (let g of hh) { + ll[g] = result[g]; + } + leftover = rankCalc(ll,iterations,main,domainMode,isGpu,arrayMax); + } - for (let i = 0; i < rl ** 2; i += (rl + 1)) { + keys.length = Math.min(keys.length,1000); + + let rl = keys.length; + rls = rl ** 2; + let matrixe = new Float32Array(rls); + + for (let i = 0; i < rls; i += (rl + 1)) { matrixe[i] = 1; } - let keys = Object.keys(result); for (let unn in result) { - let v = true; - if (domainMode) { - v = false; - try { - let test = new URL(unn); - v = true; - } catch (err) { - v = false; - } + let v = !domainMode; + try { + new URL(unn); + v = true; + } catch (err) { } + if (!v) { continue; } @@ -35,7 +61,7 @@ function rankCalc(result, iterations = 10, main = [], domainMode = false, isGpu frs[unn] = result[unn].followers || []; fng[unn] = result[unn].following || []; - let lf = Object.keys(fng[unn]).length; + let lf = fng[unn].length; if (domainMode) { let domains = []; for (let x of fng[unn]) { @@ -60,13 +86,13 @@ function rankCalc(result, iterations = 10, main = [], domainMode = false, isGpu for (let unn in result) { let fnu = frs[unn]; if (!pr[unn]) pr[unn] = 0; + let nb = keys.indexOf(unn); for (let follow of fnu) { if (follow == unn) continue; let dst = fnc[fnu] || 0; let na = keys.indexOf(follow); - let nb = keys.indexOf(unn); if (na == -1 || nb == -1) continue; - let n = (na) * (rl) + (nb) * 1; + let n = na * rl + (nb) * 1; matrixe[n] = 1.1 + 1 / (dst + 3); msum_old += matrixe[n]; } @@ -79,12 +105,12 @@ function rankCalc(result, iterations = 10, main = [], domainMode = false, isGpu if (!(h.search == '')) fail *= 0.4; if (main.indexOf(unn) != -1) fail = 10; } catch (err) { - + } } if (fail != 1) { - for (let ig = (keys.indexOf(unn) || 0) * (rl); ig < ((keys.indexOf(unn) || 0) + 1) * (rl); ig++) { + for (let ig = nb * rl; ig < (nb + 1) * rl; ig++) { matrixe[ig] *= fail; } } @@ -93,7 +119,6 @@ function rankCalc(result, iterations = 10, main = [], domainMode = false, isGpu let mm = (iterations); let gpu = new GPU(); - let multiplyMatrix; if (isGpu) { multiplyMatrix = gpu.createKernel(function (a, b, c) { @@ -102,20 +127,9 @@ function rankCalc(result, iterations = 10, main = [], domainMode = false, isGpu sum += a[(this.thread.x % c) * c + i] * b[i * c + this.thread.x / c]; } return sum; - }).setOutput([keys.length ** 2, 1]); + }).setOutput([rls, 1]); } else { console.warn(`GPU mode not enabled. Using CPU multiplication...`) - multiplyMatrix = function (a, b, c) { - let outMatrix = new Float32Array(keys.length ** 2); - for (let i in outMatrix) { - let sum = 0; - for (let j = 0; j < c; j++) { - sum += a[(i % c) * c + j] * b[j * c + Math.floor(i / c)]; - } - outMatrix[i] = sum; - } - return [outMatrix]; - } } for (let i = 0; i < mm; i++) { @@ -125,7 +139,7 @@ function rankCalc(result, iterations = 10, main = [], domainMode = false, isGpu let msum = 0; console.log(`Completed ${i} iterations`) - matrixe = multiplyMatrix(matrixe, matrixe, keys.length)[0]; + matrixe = multiplyMatrix(matrixe, matrixe, rl)[0]; for (let h in matrixe) { msum += matrixe[h]; @@ -145,6 +159,8 @@ function rankCalc(result, iterations = 10, main = [], domainMode = false, isGpu } } + pr = Object.assign(pr,leftover); + let ov = Object.keys(pr); let an = ov.filter(i => !isNaN(pr[i]) && main.indexOf(i) != -1); let new_sum = an.map(n => pr[n]).reduce((a, b) => a + b, 1e-9); diff --git a/site.js b/site.js index c4ca0c0..1e55f67 100644 --- a/site.js +++ b/site.js @@ -41,18 +41,12 @@ async function urlCollector(url, path, file, useLimit, data2, settings) { let h = body(link).attr('href'); if (!h) return true; h = h.trim(); - if (h.startsWith('./') || h.startsWith('../') || h.startsWith('/')) { - let u = new URL(url); - u.pathname = h; - h = u.toString(); - } - if (h.startsWith('?')) { - let u = new URL(url); - u.search = h; - h = u.toString(); - } - if (h.startsWith('#')) { + let isHash = h.startsWith('#'); + let isQuery = h.startsWith('?'); + if (h.startsWith('?') || isHash || h.startsWith('./') || h.startsWith('../') || h.startsWith('/')) { let u = new URL(url); + if (isHash) u.search = h; + if (!isHash && !isQuery) u.pathname = h; h = u.toString(); } let h2; @@ -323,7 +317,7 @@ async function main(settings) { } } console.log(`Graph is fully repaired`); - let calcedRank = rankCalc(data, (i == depth - 1) ? matrixIterations : 3, user, site == 'url', settings.isGpu) + let calcedRank = rankCalc(data, (i == depth - 1) ? matrixIterations : 3, user, site == 'url', settings.isGpu, settings.arrayMax) dat = Object.entries(calcedRank); dat = dat.sort((a, b) => b[1] - a[1]); console.log(`Graph is calculated with ${dat.length} entries`);