pyrox.dev
1{
2 lib,
3 stdenv,
4 buildPythonPackage,
5 fetchPypi,
6 pythonOlder,
7 rustPlatform,
8 cargo,
9 rustc,
10 setuptools,
11 setuptools-rust,
12 wheel,
13 libiconv,
14 requests,
15 regex,
16 blobfile,
17}:
18let
19 pname = "tiktoken";
20 version = "0.5.1";
21 src = fetchPypi {
22 inherit pname version;
23 hash = "sha256-J+dzVkIyAE9PgQ/R+FI2Zz7DpW7X8SBvye2GcOvtuXo=";
24 };
25 postPatch = ''
26 cp ${./Cargo.lock} Cargo.lock
27 '';
28in
29buildPythonPackage {
30 inherit
31 pname
32 version
33 src
34 postPatch
35 ;
36 format = "pyproject";
37
38 disabled = pythonOlder "3.8";
39
40 nativeBuildInput = [
41 setuptools
42 setuptools-rust
43 wheel
44 ];
45
46 cargoDeps = rustPlatform.fetchCargoTarball {
47 inherit src postPatch;
48 name = "${pname}-${version}";
49 hash = "sha256-Q7XO+auj4tKDAGbqNn9pmJg8EJvooN2ie0lWwZVrld4=";
50 };
51
52 nativeBuildInputs = [
53 rustPlatform.cargoSetupHook
54 setuptools-rust
55 cargo
56 rustc
57 ];
58
59 buildInputs = lib.optionals stdenv.isDarwin [ libiconv ];
60
61 propagatedBuildInputs = [
62 requests
63 regex
64 blobfile
65 ];
66
67 # almost all tests require network access
68 doCheck = false;
69
70 pythonImportsCheck = [ "tiktoken" ];
71
72 meta = with lib; {
73 description = "tiktoken is a fast BPE tokeniser for use with OpenAI's models.";
74 homepage = "https://github.com/openai/tiktoken";
75 license = licenses.mit;
76 maintainers = with maintainers; [ happysalada ];
77 };
78}