use autothrottle only for LI

juniorguru · May 3, 2024 · 5d98f39 · 5d98f39
1 parent 4a0751b
commit 5d98f39
Show file tree

Hide file tree

Showing 2 changed files with 1 addition and 2 deletions.
diff --git a/jg/plucker/jobs_linkedin/spider.py b/jg/plucker/jobs_linkedin/spider.py
@@ -29,6 +29,7 @@ class Spider(BaseSpider):
     name = "jobs-linkedin"
     download_delay = 5
     custom_settings = {
+        "AUTOTHROTTLE_ENABLED": True,
         "DOWNLOAD_HANDLERS": {
             "http": "scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler",
             "https": "scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler",

diff --git a/jg/plucker/settings.py b/jg/plucker/settings.py
@@ -30,8 +30,6 @@
 
 CLOSESPIDER_ERRORCOUNT = 1
 
-AUTOTHROTTLE_ENABLED = True
-
 TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
 
 PLAYWRIGHT_BROWSER_TYPE = "firefox"