tjh.dev
1{ lib, python38, fetchPypi, fetchFromGitHub }:
2let
3 python = python38.override {
4 self = python;
5 packageOverrides = self: super: {
6 sqlalchemy = super.sqlalchemy.overridePythonAttrs (oldAttrs: rec {
7 version = "1.3.24";
8 src = fetchPypi {
9 inherit (oldAttrs) pname;
10 inherit version;
11 hash = "sha256-67t3fL+TEjWbiXv4G6ANrg9ctp+6KhgmXcwYpvXvdRk=";
12 };
13 doCheck = false;
14 });
15 tornado = super.tornado_4;
16 };
17 };
18
19in
20with python.pkgs; buildPythonApplication rec {
21 pname = "grab-site";
22 version = "2.2.7";
23
24 src = fetchFromGitHub {
25 rev = version;
26 owner = "ArchiveTeam";
27 repo = "grab-site";
28 sha256 = "sha256-tf8GyFjya3+TVc2VjlY6ztfjCJgof6tg4an18pz+Ig8=";
29 };
30
31 postPatch = ''
32 substituteInPlace setup.py \
33 --replace '"wpull @ https://github.com/ArchiveTeam/ludios_wpull/tarball/master#egg=wpull-${ludios_wpull.version}"' '"wpull"'
34 '';
35
36 propagatedBuildInputs = [
37 click
38 ludios_wpull
39 manhole
40 lmdb
41 autobahn
42 fb-re2
43 websockets
44 faust-cchardet
45 ];
46
47 checkPhase = ''
48 export PATH=$PATH:$out/bin
49 bash ./tests/offline-tests
50 '';
51
52 meta = with lib; {
53 description = "Crawler for web archiving with WARC output";
54 homepage = "https://github.com/ArchiveTeam/grab-site";
55 license = licenses.mit;
56 maintainers = with maintainers; [ ivan ];
57 platforms = platforms.all;
58 };
59}