diff --git a/site.js b/site.js index 208b924..b96528a 100644 --- a/site.js +++ b/site.js @@ -24,6 +24,7 @@ async function urlCollector(url, path, file, useLimit, data2) { let data; try { data = await fetch(url); + if (!data.headers.get("content-type").startsWith('text/html')) return urls; data = await data.text(); console.log(`User ${url} was fetched`); } catch (err) {