biglyderv/bigly-caret
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add blacklist and greylist

Browse source
This commit is contained in:
biglyderv 2025-01-31 22:52:19 -05:00
parent 11681c8bd6
commit 20f739177c
1 changed files with 10 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
site.js
View file

@ -7,6 +7,8 @@ let cache = {};
let site = process.env.site || 'darflen';
let route = process.env.route || './test.txt';
let pageLimit = process.env.pageLimit || Infinity;
let blacklist = (process.env.blacklist+'').split(',') || [];
let greylist = (process.env.greylist+'').split(',') || [];
let hh;
async function urlCollector(url, path, file, useLimit, data2) {
@ -43,6 +45,10 @@ async function urlCollector(url, path, file, useLimit, data2) {
} catch (err) {
return;
}
for (let g of greylist) {
if (h2.toString().includes(g)) return;
}
if (blacklist.indexOf(h2.toString()) != -1) return;
urls.push(h2.toString());
urls = [...new Set(urls)];
console.log(`User ${url} has ${urls.length} pages calculated`);
@ -52,12 +58,12 @@ async function urlCollector(url, path, file, useLimit, data2) {
try {
h3 = new URL(new URL(url).origin);
} catch (err) {
}
urls.push(h3.toString())
if (!data2[url]) data2[url] = {following: [], followers: []};
if (!data2[url]) data2[url] = { following: [], followers: [] };
data2[url][path] = urls;
return [...new Set(urls)];
}
@ -181,7 +187,7 @@ async function siteCollector(user, path, site, useLimit, dat) {
}(p.length));
p.push(async function (k) {
await siteCollector(u, 'following', site,true, data);
await siteCollector(u, 'following', site, true, data);
console.log(`User ${u} following fully calculated`);