biglyderv/bigly-caret
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

dissolve huge arrays

Browse source
This commit is contained in:
biglyderv 2025-02-06 13:54:37 -05:00
parent 2b2fada6d2
commit 6639a1b070
3 changed files with 60 additions and 48 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
index.js
View file

@ -17,11 +17,13 @@ let {
useArchive = true,
isGpu = false,
mode = 'cli',
port = 6952
port = 6952,
arrayMax = 800
} = process.env;
let settings = {
site, route, pageLimit, blackList, greyList, discardThreshold, delay,
depth, isRelative, fetchRate, user, matrixIterations, useArchive, isGpu
depth, isRelative, fetchRate, user, matrixIterations, useArchive, isGpu,
arrayMax
};
if (mode == 'cli') {
@ -52,7 +54,7 @@ if (mode == 'cli') {
data[entry.username] = user;
data[entry.target] = target;
}
let calcedRank = rankCalc(data, matrixIterations, ref ? [ref] : [], false, settings.isGpu);
let calcedRank = rankCalc(data, matrixIterations, ref ? [ref] : [], false, settings.isGpu, settings.arrayMax);
let dat = Object.entries(calcedRank);
dat = dat.sort((a, b) => b[1] - a[1]);
console.log(`Graph is calculated with ${dat.length} entries`);

82
rank.js
View file

@ -1,7 +1,21 @@
import { GPU, input, Input } from "gpu.js";
var rls;
function multiplyMatrix(a, b, c) {
let outMatrix = new Float32Array(rls);
for (let i in outMatrix) {
let sum = 0;
for (let j = 0; j < c; j++) {
sum += a[(i % c) * c + j] * b[j * c + Math.floor(i / c)];
}
outMatrix[i] = sum;
}
return [outMatrix];
}
// derived from https://git.dervland.net/biglyderv/new-bigly-chat/src/branch/master/docs/stats.php
function rankCalc(result, iterations = 10, main = [], domainMode = false, isGpu = false) {
function rankCalc(result, iterations = 10, main = [], domainMode = false, isGpu = false, arrayMax = 800) {
let fng = {};
let fnc = {};
@ -9,25 +23,37 @@ function rankCalc(result, iterations = 10, main = [], domainMode = false, isGpu
let msum_old = 0.001;
let pr = {};
let rl = Object.keys(result).length;
let matrixe = new Float32Array(rl ** 2);
let keys = Object.keys(result);
let leftover = [];
keys = keys.sort((a,b) => result[a].followers.length < result[b].followers.length);
if (keys.length > arrayMax) {
console.warn(`Array too big. Splitting into multiple arrays...`);
let ll = {};
let hh = keys.slice(arrayMax);
for (let g of hh) {
ll[g] = result[g];
}
leftover = rankCalc(ll,iterations,main,domainMode,isGpu,arrayMax);
}
for (let i = 0; i < rl ** 2; i += (rl + 1)) {
keys.length = Math.min(keys.length,1000);
let rl = keys.length;
rls = rl ** 2;
let matrixe = new Float32Array(rls);
for (let i = 0; i < rls; i += (rl + 1)) {
matrixe[i] = 1;
}
let keys = Object.keys(result);
for (let unn in result) {
let v = true;
if (domainMode) {
v = false;
try {
let test = new URL(unn);
v = true;
} catch (err) {
v = false;
}
let v = !domainMode;
try {
new URL(unn);
v = true;
} catch (err) {
}
if (!v) {
continue;
}
@ -35,7 +61,7 @@ function rankCalc(result, iterations = 10, main = [], domainMode = false, isGpu
frs[unn] = result[unn].followers || [];
fng[unn] = result[unn].following || [];
let lf = Object.keys(fng[unn]).length;
let lf = fng[unn].length;
if (domainMode) {
let domains = [];
for (let x of fng[unn]) {
@ -60,13 +86,13 @@ function rankCalc(result, iterations = 10, main = [], domainMode = false, isGpu
for (let unn in result) {
let fnu = frs[unn];
if (!pr[unn]) pr[unn] = 0;
let nb = keys.indexOf(unn);
for (let follow of fnu) {
if (follow == unn) continue;
let dst = fnc[fnu] || 0;
let na = keys.indexOf(follow);
let nb = keys.indexOf(unn);
if (na == -1 || nb == -1) continue;
let n = (na) * (rl) + (nb) * 1;
let n = na * rl + (nb) * 1;
matrixe[n] = 1.1 + 1 / (dst + 3);
msum_old += matrixe[n];
}
@ -79,12 +105,12 @@ function rankCalc(result, iterations = 10, main = [], domainMode = false, isGpu
if (!(h.search == '')) fail *= 0.4;
if (main.indexOf(unn) != -1) fail = 10;
} catch (err) {
}
}
if (fail != 1) {
for (let ig = (keys.indexOf(unn) || 0) * (rl); ig < ((keys.indexOf(unn) || 0) + 1) * (rl); ig++) {
for (let ig = nb * rl; ig < (nb + 1) * rl; ig++) {
matrixe[ig] *= fail;
}
}
@ -93,7 +119,6 @@ function rankCalc(result, iterations = 10, main = [], domainMode = false, isGpu
let mm = (iterations);
let gpu = new GPU();
let multiplyMatrix;
if (isGpu) {
multiplyMatrix = gpu.createKernel(function (a, b, c) {
@ -102,20 +127,9 @@ function rankCalc(result, iterations = 10, main = [], domainMode = false, isGpu
sum += a[(this.thread.x % c) * c + i] * b[i * c + this.thread.x / c];
}
return sum;
}).setOutput([keys.length ** 2, 1]);
}).setOutput([rls, 1]);
} else {
console.warn(`GPU mode not enabled. Using CPU multiplication...`)
multiplyMatrix = function (a, b, c) {
let outMatrix = new Float32Array(keys.length ** 2);
for (let i in outMatrix) {
let sum = 0;
for (let j = 0; j < c; j++) {
sum += a[(i % c) * c + j] * b[j * c + Math.floor(i / c)];
}
outMatrix[i] = sum;
}
return [outMatrix];
}
}
for (let i = 0; i < mm; i++) {
@ -125,7 +139,7 @@ function rankCalc(result, iterations = 10, main = [], domainMode = false, isGpu
let msum = 0;
console.log(`Completed ${i} iterations`)
matrixe = multiplyMatrix(matrixe, matrixe, keys.length)[0];
matrixe = multiplyMatrix(matrixe, matrixe, rl)[0];
for (let h in matrixe) {
msum += matrixe[h];
@ -145,6 +159,8 @@ function rankCalc(result, iterations = 10, main = [], domainMode = false, isGpu
}
}
pr = Object.assign(pr,leftover);
let ov = Object.keys(pr);
let an = ov.filter(i => !isNaN(pr[i]) && main.indexOf(i) != -1);
let new_sum = an.map(n => pr[n]).reduce((a, b) => a + b, 1e-9);

18
site.js
View file

@ -41,18 +41,12 @@ async function urlCollector(url, path, file, useLimit, data2, settings) {
let h = body(link).attr('href');
if (!h) return true;
h = h.trim();
if (h.startsWith('./') || h.startsWith('../') || h.startsWith('/')) {
let u = new URL(url);
u.pathname = h;
h = u.toString();
}
if (h.startsWith('?')) {
let u = new URL(url);
u.search = h;
h = u.toString();
}
if (h.startsWith('#')) {
let isHash = h.startsWith('#');
let isQuery = h.startsWith('?');
if (h.startsWith('?') || isHash || h.startsWith('./') || h.startsWith('../') || h.startsWith('/')) {
let u = new URL(url);
if (isHash) u.search = h;
if (!isHash && !isQuery) u.pathname = h;
h = u.toString();
}
let h2;
@ -323,7 +317,7 @@ async function main(settings) {
}
}
console.log(`Graph is fully repaired`);
let calcedRank = rankCalc(data, (i == depth - 1) ? matrixIterations : 3, user, site == 'url', settings.isGpu)
let calcedRank = rankCalc(data, (i == depth - 1) ? matrixIterations : 3, user, site == 'url', settings.isGpu, settings.arrayMax)
dat = Object.entries(calcedRank);
dat = dat.sort((a, b) => b[1] - a[1]);
console.log(`Graph is calculated with ${dat.length} entries`);