pyrox.dev
1{
2 lib,
3 stdenv,
4 buildPythonPackage,
5 fetchPypi,
6 pythonOlder,
7 rustPlatform,
8 cargo,
9 rustc,
10 setuptools,
11 setuptools-rust,
12 libiconv,
13 requests,
14 regex,
15 blobfile,
16}:
17let
18 pname = "tiktoken";
19 version = "0.9.0";
20 src = fetchPypi {
21 inherit pname version;
22 hash = "sha256-0Cpcpqk44EkOH/lXvEjIsHjIjLg5d74WJbH9iqx5LF0=";
23 };
24 postPatch = ''
25 cp ${./Cargo.lock} Cargo.lock
26 '';
27in
28buildPythonPackage {
29 inherit
30 pname
31 version
32 src
33 postPatch
34 ;
35 pyproject = true;
36
37 disabled = pythonOlder "3.8";
38
39 build-system = [
40 setuptools
41 setuptools-rust
42 ];
43
44 cargoDeps = rustPlatform.fetchCargoVendor {
45 inherit src postPatch;
46 name = "${pname}-${version}";
47 hash = "sha256-MfTTRbSM+KgrYrWHYlJkGDc1qn3oulalDJM+huTaJ0g=";
48 };
49
50 nativeBuildInputs = [
51 rustPlatform.cargoSetupHook
52 setuptools-rust
53 cargo
54 rustc
55 ];
56
57 buildInputs = lib.optionals stdenv.hostPlatform.isDarwin [ libiconv ];
58
59 dependencies = [
60 requests
61 regex
62 blobfile
63 ];
64
65 # almost all tests require network access
66 doCheck = false;
67
68 pythonImportsCheck = [ "tiktoken" ];
69
70 meta = with lib; {
71 description = "tiktoken is a fast BPE tokeniser for use with OpenAI's models";
72 homepage = "https://github.com/openai/tiktoken";
73 license = licenses.mit;
74 maintainers = with maintainers; [ happysalada ];
75 };
76}