|
|
@ -14,7 +14,7 @@ const BLOG_URL = 'https://cronicasperiodisticas.wordpress.com';
|
|
|
|
|
|
|
|
|
|
|
|
// Configure scraping behavior
|
|
|
|
// Configure scraping behavior
|
|
|
|
const SCRAPE_CONFIG = {
|
|
|
|
const SCRAPE_CONFIG = {
|
|
|
|
delayBetweenRequests: 2000, // 2 seconds between requests
|
|
|
|
delayBetweenRequests: 4000, // 2 seconds between requests
|
|
|
|
maxConcurrentRequests: 2,
|
|
|
|
maxConcurrentRequests: 2,
|
|
|
|
userAgent: 'Mozilla/5.0 (compatible; BlogScraper/1.0; +https://github.com/your-repo)'
|
|
|
|
userAgent: 'Mozilla/5.0 (compatible; BlogScraper/1.0; +https://github.com/your-repo)'
|
|
|
|
};
|
|
|
|
};
|
|
|
|