diff --git a/site.js b/site.js index 1fcbf25..1501bb7 100644 --- a/site.js +++ b/site.js @@ -7,6 +7,8 @@ let cache = {}; let site = process.env.site || 'darflen'; let route = process.env.route || './test.txt'; let pageLimit = process.env.pageLimit || Infinity; +let blacklist = (process.env.blacklist+'').split(',') || []; +let greylist = (process.env.greylist+'').split(',') || []; let hh; async function urlCollector(url, path, file, useLimit, data2) { @@ -43,6 +45,10 @@ async function urlCollector(url, path, file, useLimit, data2) { } catch (err) { return; } + for (let g of greylist) { + if (h2.toString().includes(g)) return; + } + if (blacklist.indexOf(h2.toString()) != -1) return; urls.push(h2.toString()); urls = [...new Set(urls)]; console.log(`User ${url} has ${urls.length} pages calculated`); @@ -52,12 +58,12 @@ async function urlCollector(url, path, file, useLimit, data2) { try { h3 = new URL(new URL(url).origin); } catch (err) { - + } urls.push(h3.toString()) - if (!data2[url]) data2[url] = {following: [], followers: []}; + if (!data2[url]) data2[url] = { following: [], followers: [] }; data2[url][path] = urls; - + return [...new Set(urls)]; } @@ -181,7 +187,7 @@ async function siteCollector(user, path, site, useLimit, dat) { }(p.length)); p.push(async function (k) { - await siteCollector(u, 'following', site,true, data); + await siteCollector(u, 'following', site, true, data); console.log(`User ${u} following fully calculated`);