{"payload":{"feedbackUrl":"https://github.com/orgs/community/discussions/53140","repo":{"id":66670819,"defaultBranch":"master","name":"crawlee","ownerLogin":"apify","currentUserCanPush":false,"isFork":false,"isEmpty":false,"createdAt":"2016-08-26T18:35:03.000Z","ownerAvatar":"https://avatars.githubusercontent.com/u/24586296?v=4","public":true,"private":false,"isOrgOwned":true},"refInfo":{"name":"","listCacheKey":"v0:1715609252.0","currentOid":""},"activityList":{"items":[{"before":"79d442cf191f3bc615bb2abacefacbafd22784b8","after":"62ea166cce0847fd0d2cb0ac81c782d5f131f5ba","ref":"refs/heads/vlad/make-rqv2-default","pushedAt":"2024-05-13T14:46:49.000Z","pushType":"push","commitsCount":110,"pusher":{"login":"B4nan","name":"Martin Adámek","path":"/B4nan","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/615580?s=80&v=4"},"commit":{"message":"Merge branch 'master' into vlad/make-rqv2-default","shortMessageHtmlLink":"Merge branch 'master' into vlad/make-rqv2-default"}},{"before":"0c8dd95c5ce1e5df607353aeaaa891313535f698","after":"79d442cf191f3bc615bb2abacefacbafd22784b8","ref":"refs/heads/vlad/make-rqv2-default","pushedAt":"2024-05-13T14:46:40.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"B4nan","name":"Martin Adámek","path":"/B4nan","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/615580?s=80&v=4"},"commit":{"message":"Update docs/experiments/request_locking.mdx","shortMessageHtmlLink":"Update docs/experiments/request_locking.mdx"}},{"before":"36394b3b7a46a60aeb2c83f08bdef4e59ec3a7bb","after":"0c8dd95c5ce1e5df607353aeaaa891313535f698","ref":"refs/heads/vlad/make-rqv2-default","pushedAt":"2024-05-13T14:12:57.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"drobnikj","name":"Jakub Drobník","path":"/drobnikj","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/11837643?s=80&v=4"},"commit":{"message":"fix: update release date and adding blogpost about new changes","shortMessageHtmlLink":"fix: update release date and adding blogpost about new changes"}},{"before":"f709f86b6a37fa493f809b2f5e5292c0fc42d919","after":"3243077b4c57964679e2114685fa0056ff0d66c6","ref":"refs/heads/renovate/all-non-major","pushedAt":"2024-05-13T14:07:28.000Z","pushType":"force_push","commitsCount":0,"pusher":{"login":"renovate[bot]","name":null,"path":"/apps/renovate","primaryAvatarUrl":"https://avatars.githubusercontent.com/in/2740?s=80&v=4"},"commit":{"message":"chore(deps): update patch/minor dependencies","shortMessageHtmlLink":"chore(deps): update patch/minor dependencies"}},{"before":"5a45895f235f6d005728225e7ea9709549141761","after":null,"ref":"refs/heads/improve-scaling-defaults","pushedAt":"2024-05-13T14:06:00.000Z","pushType":"branch_deletion","commitsCount":0,"pusher":{"login":"B4nan","name":"Martin Adámek","path":"/B4nan","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/615580?s=80&v=4"}},{"before":"6da86a85d848cd1cf860a28e5f077b8b14cdb213","after":"2d5d443da5fa701b21aec003d4d84797882bc175","ref":"refs/heads/master","pushedAt":"2024-05-13T14:05:59.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"B4nan","name":"Martin Adámek","path":"/B4nan","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/615580?s=80&v=4"},"commit":{"message":"perf: improve scaling based on memory (#2459)\n\nWe only allowed to use 70% of the available memory, this PR changes the\r\nlimit to 90%. Tested with a low memory options and it did not have any\r\neffect, while it allows to use more memory on the large memory setups -\r\nwhere the 30% could mean 2gb or so, we dont need such a huge buffer.\r\n\r\nAlso increases the scaling steps to 10% instead of 5% so speed up the\r\nscaling.\r\n\r\nRelated:\r\n[apify.slack.com/archives/C0L33UM7Z/p1715109984834079](https://apify.slack.com/archives/C0L33UM7Z/p1715109984834079)","shortMessageHtmlLink":"perf: improve scaling based on memory (<a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2292434921\" data-permission-text=\"Title is private\" data-url=\"https://github.com/apify/crawlee/issues/2459\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/apify/crawlee/pull/2459/hovercard\" href=\"https://github.com/apify/crawlee/pull/2459\">#2459</a>)"}},{"before":"bff13ecd71843d0463ff0b01242ab4f007033eee","after":"7d074cb62eb8ba578accd4fda5da81827d2752aa","ref":"refs/heads/feat/file-downloader","pushedAt":"2024-05-13T12:58:30.000Z","pushType":"push","commitsCount":28,"pusher":{"login":"barjin","name":"Jindřich Bär","path":"/barjin","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/61918049?s=80&v=4"},"commit":{"message":"chore: regenerate lockfile","shortMessageHtmlLink":"chore: regenerate lockfile"}},{"before":"2e7bdaaae615a25171043d4be4db8fa082826e0f","after":"bff13ecd71843d0463ff0b01242ab4f007033eee","ref":"refs/heads/feat/file-downloader","pushedAt":"2024-05-13T12:17:08.000Z","pushType":"push","commitsCount":2,"pusher":{"login":"barjin","name":"Jindřich Bär","path":"/barjin","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/61918049?s=80&v=4"},"commit":{"message":"chore: rewrite file download test with express","shortMessageHtmlLink":"chore: rewrite file download test with express"}},{"before":null,"after":"5a45895f235f6d005728225e7ea9709549141761","ref":"refs/heads/improve-scaling-defaults","pushedAt":"2024-05-13T10:32:36.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"B4nan","name":"Martin Adámek","path":"/B4nan","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/615580?s=80&v=4"},"commit":{"message":"perf: improve scaling based on memory\n\nWe only allowed to use 70% of the available memory, this PR changes the limit to 90%. Tested with a low memory options and it did not have any effect, while it allows to use more memory on the large memory setups - where the 30% could mean 2gb or so, we dont need such a huge buffer.\n\nAlso increases the scaling steps to 10% instead of 5% so speed up the scaling.","shortMessageHtmlLink":"perf: improve scaling based on memory"}},{"before":"eaccf9d405fb9f9a5f99520a96dacd97876b3d00","after":"f709f86b6a37fa493f809b2f5e5292c0fc42d919","ref":"refs/heads/renovate/all-non-major","pushedAt":"2024-05-13T10:30:14.000Z","pushType":"force_push","commitsCount":0,"pusher":{"login":"renovate[bot]","name":null,"path":"/apps/renovate","primaryAvatarUrl":"https://avatars.githubusercontent.com/in/2740?s=80&v=4"},"commit":{"message":"chore(deps): update patch/minor dependencies","shortMessageHtmlLink":"chore(deps): update patch/minor dependencies"}},{"before":"42860cd286eaf9ef1ce2089caec64296d1fc98b0","after":null,"ref":"refs/heads/perf-adding-requests","pushedAt":"2024-05-13T10:29:11.000Z","pushType":"branch_deletion","commitsCount":0,"pusher":{"login":"B4nan","name":"Martin Adámek","path":"/B4nan","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/615580?s=80&v=4"}},{"before":"3a847f6185e1af1a7343c7f9d20c415c1eddf9c5","after":"6da86a85d848cd1cf860a28e5f077b8b14cdb213","ref":"refs/heads/master","pushedAt":"2024-05-13T10:29:10.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"B4nan","name":"Martin Adámek","path":"/B4nan","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/615580?s=80&v=4"},"commit":{"message":"perf: optimize adding large amount of requests via `crawler.addRequests()` (#2456)\n\nThis PR resolves three main issues with adding large amount of requests\r\ninto the queue:\r\n- Every requests added to the queue was automatically added to the LRU\r\nrequests cache, which has a size of 1 million items. this makes sense\r\nfor enqueuing a few items, but if we try to add more than the limit, we\r\nend up with overloading the LRU cache for no reason. Now we only add the\r\nfirst 1000 requests to the cache (plus any requests added via separate\r\ncalls, e.g. when doing `enqueueLinks` from inside a request handler,\r\nagain with a limit of the first 1000 links).\r\n- We used to validate the whole requests array via `ow`, and since the\r\nshape can vary, it was very slow (e.g. 20s just for the `ow`\r\nvalidation). Now we use a tailored validation for the array that does\r\nthe same but resolves within 100ms or so.\r\n- We always created the `Request` objects out of everything, which had a\r\nsignificant impact on memory usage. Now we skip this completely and let\r\nthe objects be created later when needed (when calling\r\n`RQ.addRequests()` which only receives the actual batch and not the\r\nwhole array)\r\n\r\nRelated: https://apify.slack.com/archives/C0L33UM7Z/p1715109984834079","shortMessageHtmlLink":"perf: optimize adding large amount of requests via `crawler.addReques…"}},{"before":"420fbc7efeba4dce0fce2763b7815a7fb8677d21","after":"3a847f6185e1af1a7343c7f9d20c415c1eddf9c5","ref":"refs/heads/master","pushedAt":"2024-05-13T09:05:31.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"apify-service-account","name":"Apify Service Account","path":"/apify-service-account","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/64261774?s=80&v=4"},"commit":{"message":"chore(docker): update docker state [skip ci]","shortMessageHtmlLink":"chore(docker): update docker state [skip ci]"}},{"before":"5ead825e1c67af0aa8fbbf2c4bade18b533fb6c7","after":"eaccf9d405fb9f9a5f99520a96dacd97876b3d00","ref":"refs/heads/renovate/all-non-major","pushedAt":"2024-05-13T07:25:46.000Z","pushType":"force_push","commitsCount":0,"pusher":{"login":"renovate[bot]","name":null,"path":"/apps/renovate","primaryAvatarUrl":"https://avatars.githubusercontent.com/in/2740?s=80&v=4"},"commit":{"message":"chore(deps): update patch/minor dependencies","shortMessageHtmlLink":"chore(deps): update patch/minor dependencies"}},{"before":"5ae65f7fee21d31f5f6755d1e5db182c27ddb84d","after":null,"ref":"refs/heads/renovate/linkedom-0.x","pushedAt":"2024-05-13T07:24:13.000Z","pushType":"branch_deletion","commitsCount":0,"pusher":{"login":"B4nan","name":"Martin Adámek","path":"/B4nan","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/615580?s=80&v=4"}},{"before":"3ff17003e1c62d8ab3162e6e04e80d7cf7177710","after":"420fbc7efeba4dce0fce2763b7815a7fb8677d21","ref":"refs/heads/master","pushedAt":"2024-05-13T07:24:13.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"B4nan","name":"Martin Adámek","path":"/B4nan","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/615580?s=80&v=4"},"commit":{"message":"chore(deps): update dependency linkedom to ^0.18.0 (#2457)","shortMessageHtmlLink":"chore(deps): update dependency linkedom to ^0.18.0 (<a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2291569979\" data-permission-text=\"Title is private\" data-url=\"https://github.com/apify/crawlee/issues/2457\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/apify/crawlee/pull/2457/hovercard\" href=\"https://github.com/apify/crawlee/pull/2457\">#2457</a>)"}},{"before":"af945781f8ad805716b4295a9f8d732dce23c079","after":"5ead825e1c67af0aa8fbbf2c4bade18b533fb6c7","ref":"refs/heads/renovate/all-non-major","pushedAt":"2024-05-13T04:22:49.000Z","pushType":"force_push","commitsCount":0,"pusher":{"login":"renovate[bot]","name":null,"path":"/apps/renovate","primaryAvatarUrl":"https://avatars.githubusercontent.com/in/2740?s=80&v=4"},"commit":{"message":"chore(deps): update patch/minor dependencies","shortMessageHtmlLink":"chore(deps): update patch/minor dependencies"}},{"before":"3ff17003e1c62d8ab3162e6e04e80d7cf7177710","after":null,"ref":"refs/heads/renovate/lock-file-maintenance","pushedAt":"2024-05-13T04:21:34.000Z","pushType":"branch_deletion","commitsCount":0,"pusher":{"login":"renovate[bot]","name":null,"path":"/apps/renovate","primaryAvatarUrl":"https://avatars.githubusercontent.com/in/2740?s=80&v=4"}},{"before":"0a8a20589bca1fa2c179c5960a9152db18911ff1","after":"3ff17003e1c62d8ab3162e6e04e80d7cf7177710","ref":"refs/heads/master","pushedAt":"2024-05-13T04:21:33.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"renovate[bot]","name":null,"path":"/apps/renovate","primaryAvatarUrl":"https://avatars.githubusercontent.com/in/2740?s=80&v=4"},"commit":{"message":"chore(deps): lock file maintenance","shortMessageHtmlLink":"chore(deps): lock file maintenance"}},{"before":null,"after":"3ff17003e1c62d8ab3162e6e04e80d7cf7177710","ref":"refs/heads/renovate/lock-file-maintenance","pushedAt":"2024-05-13T00:20:46.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"renovate[bot]","name":null,"path":"/apps/renovate","primaryAvatarUrl":"https://avatars.githubusercontent.com/in/2740?s=80&v=4"},"commit":{"message":"chore(deps): lock file maintenance","shortMessageHtmlLink":"chore(deps): lock file maintenance"}},{"before":null,"after":"5ae65f7fee21d31f5f6755d1e5db182c27ddb84d","ref":"refs/heads/renovate/linkedom-0.x","pushedAt":"2024-05-13T00:18:20.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"renovate[bot]","name":null,"path":"/apps/renovate","primaryAvatarUrl":"https://avatars.githubusercontent.com/in/2740?s=80&v=4"},"commit":{"message":"chore(deps): update dependency linkedom to ^0.18.0","shortMessageHtmlLink":"chore(deps): update dependency linkedom to ^0.18.0"}},{"before":"fb164b6ff650f0824007a172a8f3031e54d7fa7d","after":"af945781f8ad805716b4295a9f8d732dce23c079","ref":"refs/heads/renovate/all-non-major","pushedAt":"2024-05-12T04:05:08.000Z","pushType":"force_push","commitsCount":0,"pusher":{"login":"renovate[bot]","name":null,"path":"/apps/renovate","primaryAvatarUrl":"https://avatars.githubusercontent.com/in/2740?s=80&v=4"},"commit":{"message":"chore(deps): update patch/minor dependencies","shortMessageHtmlLink":"chore(deps): update patch/minor dependencies"}},{"before":"0a8a20589bca1fa2c179c5960a9152db18911ff1","after":null,"ref":"refs/heads/renovate/lock-file-maintenance","pushedAt":"2024-05-12T04:03:47.000Z","pushType":"branch_deletion","commitsCount":0,"pusher":{"login":"renovate[bot]","name":null,"path":"/apps/renovate","primaryAvatarUrl":"https://avatars.githubusercontent.com/in/2740?s=80&v=4"}},{"before":"38a202f3be7bab807947a3ae8924fe997a0576b2","after":"0a8a20589bca1fa2c179c5960a9152db18911ff1","ref":"refs/heads/master","pushedAt":"2024-05-12T04:03:47.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"renovate[bot]","name":null,"path":"/apps/renovate","primaryAvatarUrl":"https://avatars.githubusercontent.com/in/2740?s=80&v=4"},"commit":{"message":"chore(deps): lock file maintenance","shortMessageHtmlLink":"chore(deps): lock file maintenance"}},{"before":null,"after":"0a8a20589bca1fa2c179c5960a9152db18911ff1","ref":"refs/heads/renovate/lock-file-maintenance","pushedAt":"2024-05-12T01:27:31.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"renovate[bot]","name":null,"path":"/apps/renovate","primaryAvatarUrl":"https://avatars.githubusercontent.com/in/2740?s=80&v=4"},"commit":{"message":"chore(deps): lock file maintenance","shortMessageHtmlLink":"chore(deps): lock file maintenance"}},{"before":"ac9070696355eb83ddc8d791c3a2e463dac05976","after":"fb164b6ff650f0824007a172a8f3031e54d7fa7d","ref":"refs/heads/renovate/all-non-major","pushedAt":"2024-05-11T05:01:46.000Z","pushType":"force_push","commitsCount":0,"pusher":{"login":"renovate[bot]","name":null,"path":"/apps/renovate","primaryAvatarUrl":"https://avatars.githubusercontent.com/in/2740?s=80&v=4"},"commit":{"message":"chore(deps): update patch/minor dependencies","shortMessageHtmlLink":"chore(deps): update patch/minor dependencies"}},{"before":"38a202f3be7bab807947a3ae8924fe997a0576b2","after":null,"ref":"refs/heads/renovate/lock-file-maintenance","pushedAt":"2024-05-11T05:00:31.000Z","pushType":"branch_deletion","commitsCount":0,"pusher":{"login":"renovate[bot]","name":null,"path":"/apps/renovate","primaryAvatarUrl":"https://avatars.githubusercontent.com/in/2740?s=80&v=4"}},{"before":"8c246178e540162563f7a1ae1d3cee758fb893b5","after":"38a202f3be7bab807947a3ae8924fe997a0576b2","ref":"refs/heads/master","pushedAt":"2024-05-11T05:00:30.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"renovate[bot]","name":null,"path":"/apps/renovate","primaryAvatarUrl":"https://avatars.githubusercontent.com/in/2740?s=80&v=4"},"commit":{"message":"chore(deps): lock file maintenance","shortMessageHtmlLink":"chore(deps): lock file maintenance"}},{"before":null,"after":"38a202f3be7bab807947a3ae8924fe997a0576b2","ref":"refs/heads/renovate/lock-file-maintenance","pushedAt":"2024-05-11T01:41:18.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"renovate[bot]","name":null,"path":"/apps/renovate","primaryAvatarUrl":"https://avatars.githubusercontent.com/in/2740?s=80&v=4"},"commit":{"message":"chore(deps): lock file maintenance","shortMessageHtmlLink":"chore(deps): lock file maintenance"}},{"before":"d18c05c697a9b7d8a249bd5b6a69ec3d8bd081d3","after":"42860cd286eaf9ef1ce2089caec64296d1fc98b0","ref":"refs/heads/perf-adding-requests","pushedAt":"2024-05-10T13:37:43.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"B4nan","name":"Martin Adámek","path":"/B4nan","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/615580?s=80&v=4"},"commit":{"message":"improve naming","shortMessageHtmlLink":"improve naming"}}],"hasNextPage":true,"hasPreviousPage":false,"activityType":"all","actor":null,"timePeriod":"all","sort":"DESC","perPage":30,"cursor":"djE6ks8AAAAESIX-pgA","startCursor":null,"endCursor":null}},"title":"Activity · apify/crawlee"}