dissolve huge arrays

index.js

@@ -17,11 +17,13 @@ let {
     useArchive = true,
     isGpu = false,
     mode = 'cli',
-    port = 6952
+    port = 6952,
+    arrayMax = 800
 } = process.env;
 let settings = {
     site, route, pageLimit, blackList, greyList, discardThreshold, delay,
-    depth, isRelative, fetchRate, user, matrixIterations, useArchive, isGpu
+    depth, isRelative, fetchRate, user, matrixIterations, useArchive, isGpu,
+    arrayMax
 };
 
 if (mode == 'cli') {
@@ -52,7 +54,7 @@ if (mode == 'cli') {
             data[entry.username] = user;
             data[entry.target] = target;
         }
-        let calcedRank = rankCalc(data, matrixIterations, ref ? [ref] : [], false, settings.isGpu);
+        let calcedRank = rankCalc(data, matrixIterations, ref ? [ref] : [], false, settings.isGpu, settings.arrayMax);
         let dat = Object.entries(calcedRank);
         dat = dat.sort((a, b) => b[1] - a[1]);
         console.log(`Graph is calculated with ${dat.length} entries`);

rank.js

@@ -1,7 +1,21 @@
 import { GPU, input, Input } from "gpu.js";
 
+var rls;
+
+function multiplyMatrix(a, b, c) {
+    let outMatrix = new Float32Array(rls);
+    for (let i in outMatrix) {
+        let sum = 0;
+        for (let j = 0; j < c; j++) {
+            sum += a[(i % c) * c + j] * b[j * c + Math.floor(i / c)];
+        }
+        outMatrix[i] = sum;
+    }
+    return [outMatrix];
+}
+
 // derived from https://git.dervland.net/biglyderv/new-bigly-chat/src/branch/master/docs/stats.php
-function rankCalc(result, iterations = 10, main = [], domainMode = false, isGpu = false) {
+function rankCalc(result, iterations = 10, main = [], domainMode = false, isGpu = false, arrayMax = 800) {
 
     let fng = {};
     let fnc = {};
@@ -9,25 +23,37 @@ function rankCalc(result, iterations = 10, main = [], domainMode = false, isGpu
     let msum_old = 0.001;
     let pr = {};
 
-    let rl = Object.keys(result).length;
+    let keys = Object.keys(result);
-    let matrixe = new Float32Array(rl ** 2);
+    let leftover = [];
+    keys = keys.sort((a,b) => result[a].followers.length < result[b].followers.length);
+    if (keys.length > arrayMax) {
+        console.warn(`Array too big. Splitting into multiple arrays...`);
+        let ll = {};
+        let hh = keys.slice(arrayMax);
+        for (let g of  hh) {
+            ll[g] = result[g];
+        }
+        leftover = rankCalc(ll,iterations,main,domainMode,isGpu,arrayMax);
+    }
 
-    for (let i = 0; i < rl ** 2; i += (rl + 1)) {
+    keys.length = Math.min(keys.length,1000);
+
+    let rl = keys.length;
+    rls = rl ** 2;
+    let matrixe = new Float32Array(rls);
+
+    for (let i = 0; i < rls; i += (rl + 1)) {
         matrixe[i] = 1;
     }
 
-    let keys = Object.keys(result);
     for (let unn in result) {
-        let v = true;
+        let v = !domainMode;
-        if (domainMode) {
+        try {
-            v = false;
+            new URL(unn);
-            try {
+            v = true;
-                let test = new URL(unn);
+        } catch (err) {
-                v = true;
-            } catch (err) {
-                v = false;
-            }
         }
+
         if (!v) {
             continue;
         }
@@ -35,7 +61,7 @@ function rankCalc(result, iterations = 10, main = [], domainMode = false, isGpu
         frs[unn] = result[unn].followers || [];
         fng[unn] = result[unn].following || [];
 
-        let lf = Object.keys(fng[unn]).length;
+        let lf = fng[unn].length;
         if (domainMode) {
             let domains = [];
             for (let x of fng[unn]) {
@@ -60,13 +86,13 @@ function rankCalc(result, iterations = 10, main = [], domainMode = false, isGpu
     for (let unn in result) {
         let fnu = frs[unn];
         if (!pr[unn]) pr[unn] = 0;
+        let nb = keys.indexOf(unn);
         for (let follow of fnu) {
             if (follow == unn) continue;
             let dst = fnc[fnu] || 0;
             let na = keys.indexOf(follow);
-            let nb = keys.indexOf(unn);
             if (na == -1 || nb == -1) continue;
-            let n = (na) * (rl) + (nb) * 1;
+            let n = na * rl + (nb) * 1;
             matrixe[n] = 1.1 + 1 / (dst + 3);
             msum_old += matrixe[n];
         }
@@ -79,12 +105,12 @@ function rankCalc(result, iterations = 10, main = [], domainMode = false, isGpu
                 if (!(h.search == '')) fail *= 0.4;
                 if (main.indexOf(unn) != -1) fail = 10;
             } catch (err) {
-    
+
             }
         }
 
         if (fail != 1) {
-            for (let ig = (keys.indexOf(unn) || 0) * (rl); ig < ((keys.indexOf(unn) || 0) + 1) * (rl); ig++) {
+            for (let ig = nb * rl; ig < (nb + 1) * rl; ig++) {
                 matrixe[ig] *= fail;
             }
         }
@@ -93,7 +119,6 @@ function rankCalc(result, iterations = 10, main = [], domainMode = false, isGpu
     let mm = (iterations);
 
     let gpu = new GPU();
-    let multiplyMatrix;
 
     if (isGpu) {
         multiplyMatrix = gpu.createKernel(function (a, b, c) {
@@ -102,20 +127,9 @@ function rankCalc(result, iterations = 10, main = [], domainMode = false, isGpu
                 sum += a[(this.thread.x % c) * c + i] * b[i * c + this.thread.x / c];
             }
             return sum;
-        }).setOutput([keys.length ** 2, 1]);
+        }).setOutput([rls, 1]);
     } else {
         console.warn(`GPU mode not enabled. Using CPU multiplication...`)
-        multiplyMatrix = function (a, b, c) {
-            let outMatrix = new Float32Array(keys.length ** 2);
-            for (let i in outMatrix) {
-                let sum = 0;
-                for (let j = 0; j < c; j++) {
-                    sum += a[(i % c) * c + j] * b[j * c + Math.floor(i / c)];
-                }
-                outMatrix[i] = sum;
-            }
-            return [outMatrix];
-        }
     }
 
     for (let i = 0; i < mm; i++) {
@@ -125,7 +139,7 @@ function rankCalc(result, iterations = 10, main = [], domainMode = false, isGpu
 
         let msum = 0;
         console.log(`Completed ${i} iterations`)
-        matrixe = multiplyMatrix(matrixe, matrixe, keys.length)[0];
+        matrixe = multiplyMatrix(matrixe, matrixe, rl)[0];
 
         for (let h in matrixe) {
             msum += matrixe[h];
@@ -145,6 +159,8 @@ function rankCalc(result, iterations = 10, main = [], domainMode = false, isGpu
             }
         }
 
+        pr = Object.assign(pr,leftover);
+
         let ov = Object.keys(pr);
         let an = ov.filter(i => !isNaN(pr[i]) && main.indexOf(i) != -1);
         let new_sum = an.map(n => pr[n]).reduce((a, b) => a + b, 1e-9);

site.js

@@ -41,18 +41,12 @@ async function urlCollector(url, path, file, useLimit, data2, settings) {
         let h = body(link).attr('href');
         if (!h) return true;
         h = h.trim();
-        if (h.startsWith('./') || h.startsWith('../') || h.startsWith('/')) {
+        let isHash = h.startsWith('#');
-            let u = new URL(url);
+        let isQuery = h.startsWith('?');
-            u.pathname = h;
+        if (h.startsWith('?') || isHash || h.startsWith('./') || h.startsWith('../') || h.startsWith('/')) {
-            h = u.toString();
-        }
-        if (h.startsWith('?')) {
-            let u = new URL(url);
-            u.search = h;
-            h = u.toString();
-        }
-        if (h.startsWith('#')) {
             let u = new URL(url);
+            if (isHash) u.search = h;
+            if (!isHash && !isQuery) u.pathname = h;
             h = u.toString();
         }
         let h2;
@@ -323,7 +317,7 @@ async function main(settings) {
             }
         }
         console.log(`Graph is fully repaired`);
-        let calcedRank = rankCalc(data, (i == depth - 1) ? matrixIterations : 3, user, site == 'url', settings.isGpu)
+        let calcedRank = rankCalc(data, (i == depth - 1) ? matrixIterations : 3, user, site == 'url', settings.isGpu, settings.arrayMax)
         dat = Object.entries(calcedRank);
         dat = dat.sort((a, b) => b[1] - a[1]);
         console.log(`Graph is calculated with ${dat.length} entries`);