tjh.dev
1{ stdenv, buildPythonPackage, fetchPypi, glibcLocales, mock, pytest, botocore,
2 testfixtures, pillow, six, twisted, w3lib, lxml, queuelib, pyopenssl,
3 service-identity, parsel, pydispatcher, cssselect, lib }:
4buildPythonPackage rec {
5 version = "1.7.3";
6 pname = "Scrapy";
7
8 checkInputs = [ glibcLocales mock pytest botocore testfixtures pillow ];
9 propagatedBuildInputs = [
10 six twisted w3lib lxml cssselect queuelib pyopenssl service-identity parsel pydispatcher
11 ];
12
13 patches = [
14 # Scrapy is usually installed via pip where copying all
15 # permissions makes sense. In Nix the files copied are owned by
16 # root and readonly. As a consequence scrapy can't edit the
17 # project templates.
18 ./permissions-fix.patch
19 ];
20
21 LC_ALL="en_US.UTF-8";
22
23 # Disable doctest plugin—enabled in the shipped pytest.ini—because it causes pytest to hang
24 # Ignore proxy tests because requires mitmproxy
25 # Ignore test_retry_dns_error because tries to resolve an invalid dns and weirdly fails with "Reactor was unclean"
26 # Ignore xml encoding test on darwin because lxml can't find encodings https://bugs.launchpad.net/lxml/+bug/707396
27 checkPhase = ''
28 substituteInPlace pytest.ini --replace "addopts = --doctest-modules" "addopts ="
29 pytest --ignore=tests/test_linkextractors_deprecated.py --ignore=tests/test_proxy_connect.py --deselect tests/test_crawl.py::CrawlTestCase::test_retry_dns_error ${lib.optionalString stdenv.isDarwin "--deselect tests/test_utils_iterators.py::LxmlXmliterTestCase::test_xmliter_encoding"}
30 '';
31
32 src = fetchPypi {
33 inherit pname version;
34 sha256 = "fe5a40177960e97d42d1c752a73edb40f76a85a24076dec8535cffa499eb08c8";
35 };
36
37 postInstall = ''
38 install -m 644 -D extras/scrapy.1 $out/share/man/man1/scrapy.1
39 install -m 644 -D extras/scrapy_bash_completion $out/share/bash-completion/completions/scrapy
40 install -m 644 -D extras/scrapy_zsh_completion $out/share/zsh/site-functions/_scrapy
41 '';
42
43 meta = with lib; {
44 description = "A fast high-level web crawling and web scraping framework, used to crawl websites and extract structured data from their pages";
45 homepage = https://scrapy.org/;
46 license = licenses.bsd3;
47 maintainers = with maintainers; [ drewkett marsam ];
48 platforms = platforms.unix;
49 };
50}