pyrox.dev
1{ lib
2, stdenv
3, buildPythonPackage
4, fetchPypi
5, pythonOlder
6, rustPlatform
7, cargo
8, rustc
9, setuptools
10, setuptools-rust
11, wheel
12, libiconv
13, requests
14, regex
15, blobfile
16}:
17let
18 pname = "tiktoken";
19 version = "0.5.1";
20 src = fetchPypi {
21 inherit pname version;
22 hash = "sha256-J+dzVkIyAE9PgQ/R+FI2Zz7DpW7X8SBvye2GcOvtuXo=";
23 };
24 postPatch = ''
25 cp ${./Cargo.lock} Cargo.lock
26 '';
27in
28buildPythonPackage {
29 inherit pname version src postPatch;
30 format = "pyproject";
31
32 disabled = pythonOlder "3.8";
33
34 nativeBuildInput = [
35 setuptools
36 setuptools-rust
37 wheel
38 ];
39
40 cargoDeps = rustPlatform.fetchCargoTarball {
41 inherit src postPatch;
42 name = "${pname}-${version}";
43 hash = "sha256-Q7XO+auj4tKDAGbqNn9pmJg8EJvooN2ie0lWwZVrld4=";
44 };
45
46 nativeBuildInputs = [
47 rustPlatform.cargoSetupHook
48 setuptools-rust
49 cargo
50 rustc
51 ];
52
53 buildInputs = lib.optionals stdenv.isDarwin [ libiconv ];
54
55 propagatedBuildInputs = [
56 requests
57 regex
58 blobfile
59 ];
60
61 # almost all tests require network access
62 doCheck = false;
63
64 pythonImportsCheck = [
65 "tiktoken"
66 ];
67
68 meta = with lib; {
69 description = "tiktoken is a fast BPE tokeniser for use with OpenAI's models.";
70 homepage = "https://github.com/openai/tiktoken";
71 license = licenses.mit;
72 maintainers = with maintainers; [ happysalada ];
73 };
74}