From 0c070c5aec21c83ff9ae69a1e6945b45d50e4ec3 Mon Sep 17 00:00:00 2001
From: biglyderv <eggseggsarecool+git@gmail.com>
Date: Sat, 1 Feb 2025 00:43:00 -0500
Subject: [PATCH] test

---
 site.js | 115 ++++++++++++++++++++++++++++++--------------------------
 1 file changed, 62 insertions(+), 53 deletions(-)

diff --git a/site.js b/site.js
index 0a9bc40..01b8917 100644
--- a/site.js
+++ b/site.js
@@ -20,7 +20,7 @@ async function urlCollector(url, path, file, useLimit, data2) {
         data = await data.text();
         console.log(`User ${url} was fetched`);
     } catch (err) {
-        console.log(`User ${url} did not fetch`);
+        console.warn(`User ${url} did not fetch`);
         return [];
     }
 
@@ -67,6 +67,64 @@ async function urlCollector(url, path, file, useLimit, data2) {
     return [...new Set(urls)];
 }
 
+async function rounder(users, data, mode) {
+    let p = [];
+    let congested = [];
+    let timeout = false;
+    setTimeout(function() {
+        timeout = true;
+    },(process.env.delay*1000) || (60*1000));
+
+    for (let u of users) {
+        if (timeout) break;
+        let it = 0;
+        while (p.length >= (process.env.maxRate || 15)) {
+            p = p.filter(x => x != 'hi');
+            if (p.length == 0) break;
+            let pv = await Promise.any(p);
+            it++;
+            if (it > 10) {
+                console.warn(`Promises got congested. Moving to another array...`)
+                congested = congested.concat(p);
+                p = [];
+                break;
+            }
+        }
+        data[u] = { followers: [], following: [] };
+        p.push(async function (k) {
+            await siteCollector(u, 'followers', site, mode, data);
+
+            console.log(`User ${u} followers fully calculated`);
+
+            p[k] = 'hi';
+        }(p.length));
+
+        p.push(async function (k) {
+            await siteCollector(u, 'following', site, mode, data);
+
+            console.log(`User ${u} following fully calculated`);
+
+            p[k] = 'hi';
+        }(p.length));
+    }
+
+    if (!timeout) {
+        p = p.concat(congested);
+        await Promise.all(p);
+    } else {
+        console.warn('Took too long...')
+    }
+
+    let endn = [];
+    for (let h in data) {
+        let dh = data[h];
+        endn = endn.concat(dh.following,dh.followers);
+        endn = [...new Set(endn)];
+    }
+
+    return endn;
+}
+
 async function textCollector(word, path, file) {
     if (!hh) hh = await readFile(file, 'utf8');
 
@@ -147,66 +205,17 @@ async function siteCollector(user, path, site, useLimit, data2) {
     let users = [];
     let data = {};
 
-    for (let u of penv) {
-        let ca = await siteCollector(u, 'followers', site, false, data);
-        users = users.concat(ca);
-
-        users = users.concat(await siteCollector(u, 'following', site, false, data));
-    }
+    users = await rounder(penv,data,false);
 
     users = [...new Set(users)];
 
     let d = process.env.depth || 1;
     for (let i = 1; i < d; i++) {
-        users = [...new Set(users)];
         let tempSet = [...users];
-        for (let u of tempSet) {
-            users = users.concat(await siteCollector(u, 'followers', site, true, data));
-            users = users.concat(await siteCollector(u, 'following', site, true, data));
-        }
+        users = await rounder(tempSet,data,true);
+        users = [...new Set(users)];
     }
 
-    let p = [];
-    let congested = [];
-
-    for (let u of users) {
-        let it = 0;
-        while (p.length >= (process.env.maxRate || 15)) {
-            p = p.filter(x => x != 'hi');
-            if (p.length == 0) break;
-            let pv = await Promise.any(p);
-            it++;
-            if (it > 10) {
-                console.warn(`Promises got congested. Moving to another array...`)
-                congested = congested.concat(p);
-                p = [];
-                break;
-            }
-        }
-        data[u] = { followers: [], following: [] };
-        p.push(async function (k) {
-            await siteCollector(u, 'followers', site, true, data);
-
-            console.log(`User ${u} followers fully calculated`);
-
-            p[k] = 'hi';
-        }(p.length));
-
-        p.push(async function (k) {
-            await siteCollector(u, 'following', site, true, data);
-
-            console.log(`User ${u} following fully calculated`);
-
-            p[k] = 'hi';
-        }(p.length));
-    }
-
-    p = p.concat(congested);
-    await Promise.any([
-        new Promise(resolve => setTimeout(resolve, process.env.delay || 1000*60)),
-        Promise.all(p)
-    ]);
-
     for (let uf in data) {
         let u = data[uf];
         if (!u) continue;