tjh.dev
1{ lib
2, stdenv
3, buildPythonPackage
4, fetchPypi
5, pythonOlder
6, rustPlatform
7, cargo
8, rustc
9, setuptools-rust
10, libiconv
11, requests
12, regex
13, blobfile
14}:
15let
16 pname = "tiktoken";
17 version = "0.3.3";
18 src = fetchPypi {
19 inherit pname version;
20 hash = "sha256-l7WLe/2pRXkeyFXlPRZujsIMY3iUK5OFGmyRnd+dBJY=";
21 };
22 postPatch = ''
23 cp ${./Cargo.lock} Cargo.lock
24 '';
25in
26buildPythonPackage {
27 inherit pname version src postPatch;
28 format = "setuptools";
29
30 disabled = pythonOlder "3.8";
31
32 nativeBuildInput = [
33 setuptools-rust
34 ];
35
36 cargoDeps = rustPlatform.fetchCargoTarball {
37 inherit src postPatch;
38 name = "${pname}-${version}";
39 hash = "sha256-27xR7xVH/u40Xl4VbJW/yEbURf0UcGPG5QK/04igseA=";
40 };
41
42 nativeBuildInputs = [
43 rustPlatform.cargoSetupHook
44 setuptools-rust
45 cargo
46 rustc
47 ];
48
49 buildInputs = lib.optionals stdenv.isDarwin [ libiconv ];
50
51 propagatedBuildInputs = [
52 requests
53 regex
54 blobfile
55 ];
56
57 meta = with lib; {
58 description = "tiktoken is a fast BPE tokeniser for use with OpenAI's models.";
59 homepage = "https://github.com/openai/tiktoken";
60 license = licenses.mit;
61 maintainers = with maintainers; [ happysalada ];
62 };
63}