tjh.dev
1{ buildPythonPackage, fetchurl, glibcLocales, mock, pytest, botocore,
2 testfixtures, pillow, six, twisted, w3lib, lxml, queuelib, pyopenssl,
3 service-identity, parsel, pydispatcher, cssselect, lib }:
4buildPythonPackage rec {
5 version = "1.4.0";
6 pname = "Scrapy";
7 name = "${pname}-${version}";
8
9 buildInputs = [ glibcLocales mock pytest botocore testfixtures pillow ];
10 propagatedBuildInputs = [
11 six twisted w3lib lxml cssselect queuelib pyopenssl service-identity parsel pydispatcher
12 ];
13
14 # Scrapy is usually installed via pip where copying all
15 # permissions makes sense. In Nix the files copied are owned by
16 # root and readonly. As a consequence scrapy can't edit the
17 # project templates.
18 patches = [ ./permissions-fix.patch ];
19
20 LC_ALL="en_US.UTF-8";
21
22 checkPhase = ''
23 py.test --ignore=tests/test_linkextractors_deprecated.py --ignore=tests/test_proxy_connect.py
24 # The ignored tests require mitmproxy, which depends on protobuf, but it's disabled on Python3
25 '';
26
27 src = fetchurl {
28 url = "mirror://pypi/S/Scrapy/${name}.tar.gz";
29 sha256 = "04a08f027eef5d271342a016439533c81ba46f14bfcf230fecf602e99beaf233";
30 };
31
32 meta = with lib; {
33 description = "A fast high-level web crawling and web scraping framework, used to crawl websites and extract structured data from their pages";
34 homepage = http://scrapy.org/;
35 license = licenses.bsd3;
36 maintainers = with maintainers; [ drewkett ];
37 platforms = platforms.linux;
38 };
39}