raise AUTOTHROTTLE_TARGET_CONCURRENCY for jobscz

juniorguru · May 3, 2024 · 985c6d7 · 985c6d7
1 parent 5d98f39
commit 985c6d7
Show file tree

Hide file tree

Showing 3 changed files with 5 additions and 1 deletion.
diff --git a/jg/plucker/jobs_jobscz/spider.py b/jg/plucker/jobs_jobscz/spider.py
@@ -67,6 +67,9 @@
 class Spider(BaseSpider):
     name = "jobs-jobscz"
 
+    custom_settings = {
+        "AUTOTHROTTLE_TARGET_CONCURRENCY": 2,
+    }
     start_urls = [
         "https://beta.www.jobs.cz/prace/programator/",
         "https://beta.www.jobs.cz/prace/tester/",

diff --git a/jg/plucker/jobs_linkedin/spider.py b/jg/plucker/jobs_linkedin/spider.py
@@ -29,7 +29,6 @@ class Spider(BaseSpider):
     name = "jobs-linkedin"
     download_delay = 5
     custom_settings = {
-        "AUTOTHROTTLE_ENABLED": True,
         "DOWNLOAD_HANDLERS": {
             "http": "scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler",
             "https": "scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler",

diff --git a/jg/plucker/settings.py b/jg/plucker/settings.py
@@ -30,6 +30,8 @@
 
 CLOSESPIDER_ERRORCOUNT = 1
 
+AUTOTHROTTLE_ENABLED = True
+
 TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
 
 PLAYWRIGHT_BROWSER_TYPE = "firefox"