From bae808473cf07e181c3f6d22ab0ddf8aabac9498 Mon Sep 17 00:00:00 2001
From: "brobert (aider)" <borja@brobert.net>
Date: Tue, 1 Apr 2025 15:53:39 +0200
Subject: [PATCH] feat: Improve blog URL discovery with better logging

---
 scrape-blog.ts | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/scrape-blog.ts b/scrape-blog.ts
index ba8d33b..3bd713c 100644
--- a/scrape-blog.ts
+++ b/scrape-blog.ts
@@ -77,7 +77,16 @@ async function discoverAllPostUrls(baseUrl: string, maxDepth = 3): Promise<strin
         postUrls.add(url);
         console.log(`✅ Found post URL: ${url}`);
       } else {
-        console.log(`🔍 Found internal link: ${url}`);
+        // Log what type of internal link we found
+        if (path.startsWith('/tag/')) {
+          console.log(`🏷️ Found tag page: ${url}`);
+        } else if (path.startsWith('/category/')) {
+          console.log(`🗂️ Found category page: ${url}`);
+        } else if (path.startsWith('/page/')) {
+          console.log(`📄 Found pagination page: ${url}`);
+        } else {
+          console.log(`🔍 Found internal link: ${url}`);
+        }
       }
 
       // Find and filter links on page
@@ -102,9 +111,11 @@ async function discoverAllPostUrls(baseUrl: string, maxDepth = 3): Promise<strin
             continue;
           }
 
-          // Only add pagination links and post URLs to queue
+          // Add pagination, tag, category and post URLs to queue
           const path = urlObj.pathname;
           if (path.startsWith('/page/') ||  // Pagination
+              path.startsWith('/tag/') ||   // Tag pages
+              path.startsWith('/category/') || // Category pages
               /^\/\d{4}\/\d{2}\/\d{2}\/[^/]+\/$/.test(path)) {  // Post URLs
             queue.push({url: normalizedUrl, depth: depth + 1});
           }