tjh.dev
Clone of https://github.com/NixOS/nixpkgs.git (to stress-test knotserver)
1{
2 fetchFromGitHub,
3 lib,
4 nix-update-script,
5 openssl,
6 pkg-config,
7 rust-jemalloc-sys,
8 rustPlatform,
9 sqlite,
10 stdenv,
11 versionCheckHook,
12 zstd,
13}:
14
15rustPlatform.buildRustPackage (finalAttrs: {
16 pname = "spider";
17 version = "2.37.120";
18
19 src = fetchFromGitHub {
20 owner = "spider-rs";
21 repo = "spider";
22 tag = "v${finalAttrs.version}";
23 hash = "sha256-Aa041j8BHJ6s+R5zy8LYBo7rVKY9Xp2L9q4GdPS5Bo4=";
24 };
25
26 cargoHash = "sha256-Xnd4vqs8GwIlmC6HYZtC7xehF6HNkjs35QocCIZmQe8=";
27
28 nativeBuildInputs = [
29 pkg-config
30 rustPlatform.bindgenHook
31 ];
32
33 buildInputs = [
34 openssl
35 rust-jemalloc-sys
36 sqlite
37 zstd
38 ];
39
40 env = {
41 OPENSSL_NO_VENDOR = true;
42 ZSTD_SYS_USE_PKG_CONFIG = true;
43 };
44
45 checkFlags = [
46 # Sandbox limitation: no network or io_uring
47 "--skip=website::crawl"
48 "--skip=website::scrape"
49 "--skip=website::test_crawl_subdomains"
50 "--skip=website::test_crawl_tld"
51 "--skip=website::test_respect_robots_txt"
52 "--skip=page::parse_links"
53 "--skip=page::test_status_code"
54 "--skip=pdl_is_fresh"
55 "--skip=verify_revision_available"
56 ]
57 ++ lib.optionals stdenv.hostPlatform.isDarwin [
58 # Sandbox limitation: attempted to create a NULL object
59 "--skip=website::test_link_duplicates"
60 "--skip=website::not_crawl_blacklist"
61 "--skip=website::test_crawl_budget"
62 "--skip=website::test_crawl_subscription"
63 "--skip=website::Website::subscribe_guard"
64 "--skip=website::Website::subscribe"
65 ];
66
67 doInstallCheck = true;
68 nativeInstallCheckInputs = [ versionCheckHook ];
69 versionCheckProgramArg = "--version";
70
71 passthru.updateScript = nix-update-script { };
72
73 meta = {
74 changelog = "https://github.com/spider-rs/spider/releases/tag/v${finalAttrs.version}";
75 description = "Web crawler and scraper, building blocks for data curation workloads";
76 homepage = "https://github.com/spider-rs/spider";
77 license = lib.licenses.mit;
78 mainProgram = "spider";
79 maintainers = with lib.maintainers; [
80 j-mendez
81 KSJ2000
82 ];
83 platforms = lib.platforms.unix;
84 };
85})