tjh.dev
1{
2 lib,
3 buildPythonPackage,
4 fetchurl,
5 protobuf,
6 pymorphy3,
7 pymorphy3-dicts-uk,
8 sentencepiece,
9 setuptools,
10 spacy,
11 spacy-pkuseg,
12 spacy-transformers,
13 writeScript,
14 stdenv,
15 jq,
16 nix,
17 moreutils,
18}:
19let
20 buildModelPackage =
21 {
22 pname,
23 version,
24 sha256,
25 license,
26 }:
27
28 let
29 lang = builtins.substring 0 2 pname;
30 requires-protobuf = pname == "fr_dep_news_trf" || pname == "uk_core_news_trf";
31 in
32 buildPythonPackage {
33 inherit pname version;
34 pyproject = true;
35
36 src = fetchurl {
37 url = "https://github.com/explosion/spacy-models/releases/download/${pname}-${version}/${pname}-${version}.tar.gz";
38 inherit sha256;
39 };
40
41 propagatedBuildInputs =
42 [ spacy ]
43 ++ lib.optionals (lib.hasSuffix "_trf" pname) [ spacy-transformers ]
44 ++ lib.optionals (lang == "ru") [ pymorphy3 ]
45 ++ lib.optionals (lang == "uk") [
46 pymorphy3
47 pymorphy3-dicts-uk
48 ]
49 ++ lib.optionals (lang == "zh") [ spacy-pkuseg ]
50 ++ lib.optionals (pname == "fr_dep_news_trf") [ sentencepiece ];
51
52 postPatch = lib.optionalString requires-protobuf ''
53 substituteInPlace meta.json \
54 --replace "protobuf<3.21.0" "protobuf"
55 '';
56
57 nativeBuildInputs = [ setuptools ] ++ lib.optionals requires-protobuf [ protobuf ];
58
59 pythonImportsCheck = [ pname ];
60
61 passthru.updateScript = writeScript "update-spacy-models" ''
62 #!${stdenv.shell}
63 set -eou pipefail
64 PATH=${
65 lib.makeBinPath [
66 jq
67 nix
68 moreutils
69 ]
70 }
71
72 IFS=. read -r major minor patch <<<"${spacy.version}"
73 spacyVersion="$(echo "$major.$minor.0")"
74
75 pushd pkgs/development/python-modules/spacy/ || exit
76
77 jq -r '.[] | .pname' models.json | while IFS= read -r pname; do
78 if [ "$(jq --arg pname "$pname" -r '.[] | select(.pname == $pname) | .version' models.json)" == "$spacyVersion" ]; then
79 continue
80 fi
81
82 newHash="$(nix-prefetch-url "https://github.com/explosion/spacy-models/releases/download/$pname-$spacyVersion/$pname-$spacyVersion.tar.gz")"
83 jq --arg newHash "$newHash" --arg pname "$pname" --arg spacyVersion "$spacyVersion" \
84 '[(.[] | select(.pname != $pname)), (.[] | select(.pname == $pname) | .sha256 = $newHash | .version = $spacyVersion)] | sort_by(.pname)' \
85 models.json | sponge models.json
86 done
87
88 popd || exit
89 '';
90
91 meta = with lib; {
92 description = "Models for the spaCy NLP library";
93 homepage = "https://github.com/explosion/spacy-models";
94 license = licenses.${license};
95 };
96 };
97
98 makeModelSet =
99 models: with lib; listToAttrs (map (m: nameValuePair m.pname (buildModelPackage m)) models);
100in
101makeModelSet (lib.importJSON ./models.json)