tjh.dev
1{ stdenv, buildPythonPackage, fetchPypi, glibcLocales, mock, pytest, botocore,
2 testfixtures, pillow, six, twisted, w3lib, lxml, queuelib, pyopenssl,
3 service-identity, parsel, pydispatcher, cssselect, lib }:
4buildPythonPackage rec {
5 version = "1.6.0";
6 pname = "Scrapy";
7
8 checkInputs = [ glibcLocales mock pytest botocore testfixtures pillow ];
9 propagatedBuildInputs = [
10 six twisted w3lib lxml cssselect queuelib pyopenssl service-identity parsel pydispatcher
11 ];
12
13 patches = [
14 # Scrapy is usually installed via pip where copying all
15 # permissions makes sense. In Nix the files copied are owned by
16 # root and readonly. As a consequence scrapy can't edit the
17 # project templates.
18 ./permissions-fix.patch
19 ];
20
21 LC_ALL="en_US.UTF-8";
22
23 # Disable doctest plugin—enabled in the shipped pytest.ini—because it causes pytest to hang
24 # Ignore proxy tests because requires mitmproxy
25 # Ignore test_retry_dns_error because tries to resolve an invalid dns and weirdly fails with "Reactor was unclean"
26 # Ignore xml encoding test on darwin because lxml can't find encodings https://bugs.launchpad.net/lxml/+bug/707396
27 checkPhase = ''
28 pytest -p no:doctest --ignore=tests/test_linkextractors_deprecated.py --ignore=tests/test_proxy_connect.py --deselect tests/test_crawl.py::CrawlTestCase::test_retry_dns_error ${lib.optionalString stdenv.isDarwin "--deselect tests/test_utils_iterators.py::LxmlXmliterTestCase::test_xmliter_encoding"}
29 '';
30
31 src = fetchPypi {
32 inherit pname version;
33 sha256 = "558dfd10ac53cb324ecd7eefd3eac412161c7507c082b01b0bcd2c6e2e9f0766";
34 };
35
36 postInstall = ''
37 install -m 644 -D extras/scrapy.1 $out/share/man/man1/scrapy.1
38 install -m 644 -D extras/scrapy_bash_completion $out/share/bash-completion/completions/scrapy
39 install -m 644 -D extras/scrapy_zsh_completion $out/share/zsh/site-functions/_scrapy
40 '';
41
42 meta = with lib; {
43 description = "A fast high-level web crawling and web scraping framework, used to crawl websites and extract structured data from their pages";
44 homepage = https://scrapy.org/;
45 license = licenses.bsd3;
46 maintainers = with maintainers; [ drewkett marsam ];
47 platforms = platforms.unix;
48 };
49}