diff options
Diffstat (limited to 'dev-python/scrapy')
| -rw-r--r-- | dev-python/scrapy/Manifest | 2 | ||||
| -rw-r--r-- | dev-python/scrapy/files/scrapy-2.11.0-lift-twisted-restriction.patch | 99 | ||||
| -rw-r--r-- | dev-python/scrapy/scrapy-2.12.0.ebuild (renamed from dev-python/scrapy/scrapy-2.11.0-r1.ebuild) | 12 |
3 files changed, 8 insertions, 105 deletions
diff --git a/dev-python/scrapy/Manifest b/dev-python/scrapy/Manifest index a5485bafa6a3..37c0254c0e7e 100644 --- a/dev-python/scrapy/Manifest +++ b/dev-python/scrapy/Manifest @@ -1 +1 @@ -DIST scrapy-2.11.0.tar.gz 1373184 BLAKE2B 7fc758a729d9e9f88c89e30c503ff8bf4c2e111bc00cbd003d5f8785dc818e5ef69ae3774013c4fe27db623c1246a57d57db097cf5220b257a9632bdd29b7023 SHA512 08da9684f4312e4149929f3785cc516207070fdd0eb99d8b7ed3276095378cc71d18534d0535fd1107af442aed42d6c91792ad7749d631b5f54522c3912ce073 +DIST scrapy-2.12.0.tar.gz 1210998 BLAKE2B f0bd637ac4182de4573d37fb6db8ee26aac20eeb9ded6d255a9abe246014a100e1ba567e80f64279d382954e18bab7af165a69f2a7732619b2e4369693ea8d1d SHA512 3d733d2cb858e0f6201d08e5e6388667155a3e9746cca6887652b1ce514d803da8c6523d1bba27d451a5e4337fc191983de46c2ad8e6a6b0479f38ebfa1ad3b1 diff --git a/dev-python/scrapy/files/scrapy-2.11.0-lift-twisted-restriction.patch b/dev-python/scrapy/files/scrapy-2.11.0-lift-twisted-restriction.patch deleted file mode 100644 index c3ef23287087..000000000000 --- a/dev-python/scrapy/files/scrapy-2.11.0-lift-twisted-restriction.patch +++ /dev/null @@ -1,99 +0,0 @@ -Backport commits 720f351a3eea5e5bfa83a6eaf50210cd1fa43992^..0630e4aaa10c3fb8c79c2542a229f5c0632cddde. -At the time of writing, ::gentoo no longer has <dev-python/twisted-23.8.0. -These commits refactor the signal handling logic to support newer versions of Twisted. - -Author: Lucio Sauer <watermanpaint@posteo.net> ---- a/scrapy/crawler.py -+++ b/scrapy/crawler.py -@@ -404,8 +404,8 @@ class CrawlerProcess(CrawlerRunner): - :param bool stop_after_crawl: stop or not the reactor when all - crawlers have finished - -- :param bool install_signal_handlers: whether to install the shutdown -- handlers (default: True) -+ :param bool install_signal_handlers: whether to install the OS signal -+ handlers from Twisted and Scrapy (default: True) - """ - from twisted.internet import reactor - -@@ -416,15 +416,17 @@ class CrawlerProcess(CrawlerRunner): - return - d.addBoth(self._stop_reactor) - -- if install_signal_handlers: -- install_shutdown_handlers(self._signal_shutdown) - resolver_class = load_object(self.settings["DNS_RESOLVER"]) - resolver = create_instance(resolver_class, self.settings, self, reactor=reactor) - resolver.install_on_reactor() - tp = reactor.getThreadPool() - tp.adjustPoolsize(maxthreads=self.settings.getint("REACTOR_THREADPOOL_MAXSIZE")) - reactor.addSystemEventTrigger("before", "shutdown", self.stop) -- reactor.run(installSignalHandlers=False) # blocking call -+ if install_signal_handlers: -+ reactor.addSystemEventTrigger( -+ "after", "startup", install_shutdown_handlers, self._signal_shutdown -+ ) -+ reactor.run(installSignalHandlers=install_signal_handlers) # blocking call - - def _graceful_stop_reactor(self) -> Deferred: - d = self.stop() ---- a/scrapy/utils/ossignal.py -+++ b/scrapy/utils/ossignal.py -@@ -19,13 +19,10 @@ def install_shutdown_handlers( - function: SignalHandlerT, override_sigint: bool = True - ) -> None: - """Install the given function as a signal handler for all common shutdown -- signals (such as SIGINT, SIGTERM, etc). If override_sigint is ``False`` the -- SIGINT handler won't be install if there is already a handler in place -- (e.g. Pdb) -+ signals (such as SIGINT, SIGTERM, etc). If ``override_sigint`` is ``False`` the -+ SIGINT handler won't be installed if there is already a handler in place -+ (e.g. Pdb) - """ -- from twisted.internet import reactor -- -- reactor._handleSignals() - signal.signal(signal.SIGTERM, function) - if signal.getsignal(signal.SIGINT) == signal.default_int_handler or override_sigint: - signal.signal(signal.SIGINT, function) ---- a/scrapy/utils/testproc.py -+++ b/scrapy/utils/testproc.py -@@ -2,7 +2,7 @@ from __future__ import annotations - - import os - import sys --from typing import Iterable, Optional, Tuple, cast -+from typing import Iterable, List, Optional, Tuple, cast - - from twisted.internet.defer import Deferred - from twisted.internet.error import ProcessTerminated -@@ -26,14 +26,15 @@ class ProcessTest: - env = os.environ.copy() - if settings is not None: - env["SCRAPY_SETTINGS_MODULE"] = settings -+ assert self.command - cmd = self.prefix + [self.command] + list(args) - pp = TestProcessProtocol() -- pp.deferred.addBoth(self._process_finished, cmd, check_code) -+ pp.deferred.addCallback(self._process_finished, cmd, check_code) - reactor.spawnProcess(pp, cmd[0], cmd, env=env, path=self.cwd) - return pp.deferred - - def _process_finished( -- self, pp: TestProcessProtocol, cmd: str, check_code: bool -+ self, pp: TestProcessProtocol, cmd: List[str], check_code: bool - ) -> Tuple[int, bytes, bytes]: - if pp.exitcode and check_code: - msg = f"process {cmd} exit with code {pp.exitcode}" ---- a/setup.py -+++ b/setup.py -@@ -6,8 +6,7 @@ version = (Path(__file__).parent / "scrapy/VERSION").read_text("ascii").strip() - - - install_requires = [ -- # 23.8.0 incompatibility: https://github.com/scrapy/scrapy/issues/6024 -- "Twisted>=18.9.0,<23.8.0", -+ "Twisted>=18.9.0", - "cryptography>=36.0.0", - "cssselect>=0.9.1", - "itemloaders>=1.0.1", diff --git a/dev-python/scrapy/scrapy-2.11.0-r1.ebuild b/dev-python/scrapy/scrapy-2.12.0.ebuild index e909c7386e60..6fe9ee6264c3 100644 --- a/dev-python/scrapy/scrapy-2.11.0-r1.ebuild +++ b/dev-python/scrapy/scrapy-2.12.0.ebuild @@ -5,15 +5,14 @@ EAPI=8 PYTHON_COMPAT=( python3_{9,10,11,12,13} ) DISTUTILS_USE_PEP517=setuptools -inherit distutils-r1 +inherit distutils-r1 pypi DESCRIPTION="A high-level Web Crawling and Web Scraping framework" HOMEPAGE="https://scrapy.org/" -SRC_URI="https://github.com/scrapy/scrapy/archive/refs/tags/${PV}.tar.gz -> ${P}.tar.gz" LICENSE="BSD" SLOT=0 -KEYWORDS="~amd64" +KEYWORDS="~amd64 ~arm64" IUSE="test" RESTRICT="!test? ( test )" @@ -41,13 +40,14 @@ RDEPEND="dev-python/cssselect[${PYTHON_USEDEP}] BDEPEND=" test? ( ${RDEPEND} + dev-python/botocore[${PYTHON_USEDEP}] + dev-python/defusedxml[${PYTHON_USEDEP}] + dev-python/pexpect[${PYTHON_USEDEP}] dev-python/testfixtures[${PYTHON_USEDEP}] dev-python/uvloop[${PYTHON_USEDEP}] ) " -PATCHES="${FILESDIR}"/${P}-lift-twisted-restriction.patch - distutils_enable_tests pytest EPYTEST_DESELECT=( @@ -57,5 +57,7 @@ EPYTEST_DESELECT=( tests/test_pipeline_files.py::TestFTPFileStore::test_persist # Flaky test: https://github.com/scrapy/scrapy/issues/6193 tests/test_crawl.py::CrawlTestCase::test_start_requests_laziness + # Missing dependencies + tests/test_spidermiddleware_output_chain.py ) EPYTEST_IGNORE=( docs ) |