dissolve huge arrays
This commit is contained in:
parent
2b2fada6d2
commit
6639a1b070
3 changed files with 60 additions and 48 deletions
8
index.js
8
index.js
|
|
@ -17,11 +17,13 @@ let {
|
|||
useArchive = true,
|
||||
isGpu = false,
|
||||
mode = 'cli',
|
||||
port = 6952
|
||||
port = 6952,
|
||||
arrayMax = 800
|
||||
} = process.env;
|
||||
let settings = {
|
||||
site, route, pageLimit, blackList, greyList, discardThreshold, delay,
|
||||
depth, isRelative, fetchRate, user, matrixIterations, useArchive, isGpu
|
||||
depth, isRelative, fetchRate, user, matrixIterations, useArchive, isGpu,
|
||||
arrayMax
|
||||
};
|
||||
|
||||
if (mode == 'cli') {
|
||||
|
|
@ -52,7 +54,7 @@ if (mode == 'cli') {
|
|||
data[entry.username] = user;
|
||||
data[entry.target] = target;
|
||||
}
|
||||
let calcedRank = rankCalc(data, matrixIterations, ref ? [ref] : [], false, settings.isGpu);
|
||||
let calcedRank = rankCalc(data, matrixIterations, ref ? [ref] : [], false, settings.isGpu, settings.arrayMax);
|
||||
let dat = Object.entries(calcedRank);
|
||||
dat = dat.sort((a, b) => b[1] - a[1]);
|
||||
console.log(`Graph is calculated with ${dat.length} entries`);
|
||||
|
|
|
|||
74
rank.js
74
rank.js
|
|
@ -1,7 +1,21 @@
|
|||
import { GPU, input, Input } from "gpu.js";
|
||||
|
||||
var rls;
|
||||
|
||||
function multiplyMatrix(a, b, c) {
|
||||
let outMatrix = new Float32Array(rls);
|
||||
for (let i in outMatrix) {
|
||||
let sum = 0;
|
||||
for (let j = 0; j < c; j++) {
|
||||
sum += a[(i % c) * c + j] * b[j * c + Math.floor(i / c)];
|
||||
}
|
||||
outMatrix[i] = sum;
|
||||
}
|
||||
return [outMatrix];
|
||||
}
|
||||
|
||||
// derived from https://git.dervland.net/biglyderv/new-bigly-chat/src/branch/master/docs/stats.php
|
||||
function rankCalc(result, iterations = 10, main = [], domainMode = false, isGpu = false) {
|
||||
function rankCalc(result, iterations = 10, main = [], domainMode = false, isGpu = false, arrayMax = 800) {
|
||||
|
||||
let fng = {};
|
||||
let fnc = {};
|
||||
|
|
@ -9,25 +23,37 @@ function rankCalc(result, iterations = 10, main = [], domainMode = false, isGpu
|
|||
let msum_old = 0.001;
|
||||
let pr = {};
|
||||
|
||||
let rl = Object.keys(result).length;
|
||||
let matrixe = new Float32Array(rl ** 2);
|
||||
let keys = Object.keys(result);
|
||||
let leftover = [];
|
||||
keys = keys.sort((a,b) => result[a].followers.length < result[b].followers.length);
|
||||
if (keys.length > arrayMax) {
|
||||
console.warn(`Array too big. Splitting into multiple arrays...`);
|
||||
let ll = {};
|
||||
let hh = keys.slice(arrayMax);
|
||||
for (let g of hh) {
|
||||
ll[g] = result[g];
|
||||
}
|
||||
leftover = rankCalc(ll,iterations,main,domainMode,isGpu,arrayMax);
|
||||
}
|
||||
|
||||
for (let i = 0; i < rl ** 2; i += (rl + 1)) {
|
||||
keys.length = Math.min(keys.length,1000);
|
||||
|
||||
let rl = keys.length;
|
||||
rls = rl ** 2;
|
||||
let matrixe = new Float32Array(rls);
|
||||
|
||||
for (let i = 0; i < rls; i += (rl + 1)) {
|
||||
matrixe[i] = 1;
|
||||
}
|
||||
|
||||
let keys = Object.keys(result);
|
||||
for (let unn in result) {
|
||||
let v = true;
|
||||
if (domainMode) {
|
||||
v = false;
|
||||
let v = !domainMode;
|
||||
try {
|
||||
let test = new URL(unn);
|
||||
new URL(unn);
|
||||
v = true;
|
||||
} catch (err) {
|
||||
v = false;
|
||||
}
|
||||
}
|
||||
|
||||
if (!v) {
|
||||
continue;
|
||||
}
|
||||
|
|
@ -35,7 +61,7 @@ function rankCalc(result, iterations = 10, main = [], domainMode = false, isGpu
|
|||
frs[unn] = result[unn].followers || [];
|
||||
fng[unn] = result[unn].following || [];
|
||||
|
||||
let lf = Object.keys(fng[unn]).length;
|
||||
let lf = fng[unn].length;
|
||||
if (domainMode) {
|
||||
let domains = [];
|
||||
for (let x of fng[unn]) {
|
||||
|
|
@ -60,13 +86,13 @@ function rankCalc(result, iterations = 10, main = [], domainMode = false, isGpu
|
|||
for (let unn in result) {
|
||||
let fnu = frs[unn];
|
||||
if (!pr[unn]) pr[unn] = 0;
|
||||
let nb = keys.indexOf(unn);
|
||||
for (let follow of fnu) {
|
||||
if (follow == unn) continue;
|
||||
let dst = fnc[fnu] || 0;
|
||||
let na = keys.indexOf(follow);
|
||||
let nb = keys.indexOf(unn);
|
||||
if (na == -1 || nb == -1) continue;
|
||||
let n = (na) * (rl) + (nb) * 1;
|
||||
let n = na * rl + (nb) * 1;
|
||||
matrixe[n] = 1.1 + 1 / (dst + 3);
|
||||
msum_old += matrixe[n];
|
||||
}
|
||||
|
|
@ -84,7 +110,7 @@ function rankCalc(result, iterations = 10, main = [], domainMode = false, isGpu
|
|||
}
|
||||
|
||||
if (fail != 1) {
|
||||
for (let ig = (keys.indexOf(unn) || 0) * (rl); ig < ((keys.indexOf(unn) || 0) + 1) * (rl); ig++) {
|
||||
for (let ig = nb * rl; ig < (nb + 1) * rl; ig++) {
|
||||
matrixe[ig] *= fail;
|
||||
}
|
||||
}
|
||||
|
|
@ -93,7 +119,6 @@ function rankCalc(result, iterations = 10, main = [], domainMode = false, isGpu
|
|||
let mm = (iterations);
|
||||
|
||||
let gpu = new GPU();
|
||||
let multiplyMatrix;
|
||||
|
||||
if (isGpu) {
|
||||
multiplyMatrix = gpu.createKernel(function (a, b, c) {
|
||||
|
|
@ -102,20 +127,9 @@ function rankCalc(result, iterations = 10, main = [], domainMode = false, isGpu
|
|||
sum += a[(this.thread.x % c) * c + i] * b[i * c + this.thread.x / c];
|
||||
}
|
||||
return sum;
|
||||
}).setOutput([keys.length ** 2, 1]);
|
||||
}).setOutput([rls, 1]);
|
||||
} else {
|
||||
console.warn(`GPU mode not enabled. Using CPU multiplication...`)
|
||||
multiplyMatrix = function (a, b, c) {
|
||||
let outMatrix = new Float32Array(keys.length ** 2);
|
||||
for (let i in outMatrix) {
|
||||
let sum = 0;
|
||||
for (let j = 0; j < c; j++) {
|
||||
sum += a[(i % c) * c + j] * b[j * c + Math.floor(i / c)];
|
||||
}
|
||||
outMatrix[i] = sum;
|
||||
}
|
||||
return [outMatrix];
|
||||
}
|
||||
}
|
||||
|
||||
for (let i = 0; i < mm; i++) {
|
||||
|
|
@ -125,7 +139,7 @@ function rankCalc(result, iterations = 10, main = [], domainMode = false, isGpu
|
|||
|
||||
let msum = 0;
|
||||
console.log(`Completed ${i} iterations`)
|
||||
matrixe = multiplyMatrix(matrixe, matrixe, keys.length)[0];
|
||||
matrixe = multiplyMatrix(matrixe, matrixe, rl)[0];
|
||||
|
||||
for (let h in matrixe) {
|
||||
msum += matrixe[h];
|
||||
|
|
@ -145,6 +159,8 @@ function rankCalc(result, iterations = 10, main = [], domainMode = false, isGpu
|
|||
}
|
||||
}
|
||||
|
||||
pr = Object.assign(pr,leftover);
|
||||
|
||||
let ov = Object.keys(pr);
|
||||
let an = ov.filter(i => !isNaN(pr[i]) && main.indexOf(i) != -1);
|
||||
let new_sum = an.map(n => pr[n]).reduce((a, b) => a + b, 1e-9);
|
||||
|
|
|
|||
18
site.js
18
site.js
|
|
@ -41,18 +41,12 @@ async function urlCollector(url, path, file, useLimit, data2, settings) {
|
|||
let h = body(link).attr('href');
|
||||
if (!h) return true;
|
||||
h = h.trim();
|
||||
if (h.startsWith('./') || h.startsWith('../') || h.startsWith('/')) {
|
||||
let u = new URL(url);
|
||||
u.pathname = h;
|
||||
h = u.toString();
|
||||
}
|
||||
if (h.startsWith('?')) {
|
||||
let u = new URL(url);
|
||||
u.search = h;
|
||||
h = u.toString();
|
||||
}
|
||||
if (h.startsWith('#')) {
|
||||
let isHash = h.startsWith('#');
|
||||
let isQuery = h.startsWith('?');
|
||||
if (h.startsWith('?') || isHash || h.startsWith('./') || h.startsWith('../') || h.startsWith('/')) {
|
||||
let u = new URL(url);
|
||||
if (isHash) u.search = h;
|
||||
if (!isHash && !isQuery) u.pathname = h;
|
||||
h = u.toString();
|
||||
}
|
||||
let h2;
|
||||
|
|
@ -323,7 +317,7 @@ async function main(settings) {
|
|||
}
|
||||
}
|
||||
console.log(`Graph is fully repaired`);
|
||||
let calcedRank = rankCalc(data, (i == depth - 1) ? matrixIterations : 3, user, site == 'url', settings.isGpu)
|
||||
let calcedRank = rankCalc(data, (i == depth - 1) ? matrixIterations : 3, user, site == 'url', settings.isGpu, settings.arrayMax)
|
||||
dat = Object.entries(calcedRank);
|
||||
dat = dat.sort((a, b) => b[1] - a[1]);
|
||||
console.log(`Graph is calculated with ${dat.length} entries`);
|
||||
|
|
|
|||
Loading…
Reference in a new issue