tjh.dev
nixpkgs mirror (for testing)
github.com/NixOS/nixpkgs
nix
1{
2 lib,
3 stdenv,
4 buildPythonPackage,
5 fetchPypi,
6 rustPlatform,
7 cargo,
8 rustc,
9 setuptools,
10 setuptools-rust,
11 libiconv,
12 requests,
13 regex,
14 blobfile,
15}:
16let
17 pname = "tiktoken";
18 version = "0.12.0";
19 src = fetchPypi {
20 inherit pname version;
21 hash = "sha256-sYun7isJOGOXj8sU90s3B83I1NTTg2hTzn7GB3ITmTE=";
22 };
23 postPatch = ''
24 cp ${./Cargo.lock} Cargo.lock
25 '';
26in
27buildPythonPackage {
28 inherit
29 pname
30 version
31 src
32 postPatch
33 ;
34 pyproject = true;
35
36 build-system = [
37 setuptools
38 setuptools-rust
39 ];
40
41 cargoDeps = rustPlatform.fetchCargoVendor {
42 inherit
43 pname
44 version
45 src
46 postPatch
47 ;
48 hash = "sha256-daIKasW/lwYwIqMs3KvCDJWAoMn1CkPRpNqhl1jKpYY=";
49 };
50
51 nativeBuildInputs = [
52 rustPlatform.cargoSetupHook
53 setuptools-rust
54 cargo
55 rustc
56 ];
57
58 buildInputs = lib.optionals stdenv.hostPlatform.isDarwin [ libiconv ];
59
60 dependencies = [
61 requests
62 regex
63 blobfile
64 ];
65
66 # almost all tests require network access
67 doCheck = false;
68
69 pythonImportsCheck = [ "tiktoken" ];
70
71 meta = {
72 description = "Fast BPE tokeniser for use with OpenAI's models";
73 homepage = "https://github.com/openai/tiktoken";
74 license = lib.licenses.mit;
75 maintainers = with lib.maintainers; [ happysalada ];
76 };
77}