apify
diff --git a/‎src/apify/scrapy/middlewares/apify_proxy.py
Copy file name to clipboardExpand all lines: src/apify/scrapy/middlewares/apify_proxy.py
+1-5Lines changed: 1 addition & 5 deletions b/‎src/apify/scrapy/middlewares/apify_proxy.py
Copy file name to clipboardExpand all lines: src/apify/scrapy/middlewares/apify_proxy.py
+1-5Lines changed: 1 addition & 5 deletions
@@ -78,11 +78,6 @@ async def process_request(self, request: Request, spider: Spider) -> None:
         Raises:
             ValueError: If username and password are not provided in the proxy URL.
         """
-        # Do not use proxy for robots.txt, as it causes 403 Forbidden.
-        if request.url.endswith('/robots.txt'):
-            request.meta.pop('proxy', None)
-            return
-
         Actor.log.debug(f'ApifyHttpProxyMiddleware.process_request: request={request}, spider={spider}')
         url = await self._get_new_proxy_url()
 
@@ -151,4 +146,5 @@ async def _get_new_proxy_url(self) -> ParseResult:
 
         # Get a new proxy URL and return it
         new_url = await proxy_cfg.new_url()
+
         return urlparse(str(new_url))