keyboard stuff
auri.ee
1"""This script automates the generation of the QMK API data.
2"""
3from pathlib import Path
4import shutil
5import json
6
7from milc import cli
8
9import qmk.path
10from qmk.datetime import current_datetime
11from qmk.info import info_json
12from qmk.json_schema import json_load
13from qmk.keymap import list_keymaps
14from qmk.keyboard import find_readme, list_keyboards, keyboard_alias_definitions
15from qmk.keycodes import load_spec, list_versions, list_languages
16
17DATA_PATH = Path('data')
18TEMPLATE_PATH = DATA_PATH / 'templates/api/'
19BUILD_API_PATH = Path('.build/api_data/')
20
21
22def _list_constants(output_folder):
23 """Produce a map of available constants
24 """
25 ret = {}
26 for file in (output_folder / 'constants').glob('**/*_[0-9].[0-9].[0-9].json'):
27 name, version = file.stem.rsplit('_', 1)
28 if name not in ret:
29 ret[name] = []
30 ret[name].append(version)
31
32 # Ensure content is sorted
33 for name in ret:
34 ret[name] = sorted(ret[name])
35
36 return ret
37
38
39def _resolve_keycode_specs(output_folder):
40 """To make it easier for consumers, publish pre-merged spec files
41 """
42 for version in list_versions():
43 overall = load_spec(version)
44
45 output_file = output_folder / f'constants/keycodes_{version}.json'
46 output_file.write_text(json.dumps(overall, separators=(',', ':')), encoding='utf-8')
47
48 for lang in list_languages():
49 for version in list_versions(lang):
50 overall = load_spec(version, lang)
51
52 output_file = output_folder / f'constants/keycodes_{lang}_{version}.json'
53 output_file.write_text(json.dumps(overall, separators=(',', ':')), encoding='utf-8')
54
55 # Purge files consumed by 'load_spec'
56 shutil.rmtree(output_folder / 'constants/keycodes/')
57
58
59def _filtered_copy(src, dst):
60 src = Path(src)
61 dst = Path(dst)
62
63 if dst.suffix == '.hjson':
64 data = json_load(src)
65
66 dst = dst.with_suffix('.json')
67 dst.write_text(json.dumps(data, separators=(',', ':')), encoding='utf-8')
68 return dst
69
70 if dst.suffix == '.jsonschema':
71 data = json_load(src)
72
73 dst.write_text(json.dumps(data), encoding='utf-8')
74 return dst
75
76 return shutil.copy2(src, dst)
77
78
79def _filtered_keyboard_list():
80 """Perform basic filtering of list_keyboards
81 """
82 keyboard_list = list_keyboards()
83 if cli.args.filter:
84 kb_list = []
85 for keyboard_name in keyboard_list:
86 if any(i in keyboard_name for i in cli.args.filter):
87 kb_list.append(keyboard_name)
88 keyboard_list = kb_list
89 return keyboard_list
90
91
92@cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't write the data to disk.")
93@cli.argument('-f', '--filter', arg_only=True, action='append', default=[], help="Filter the list of keyboards based on partial name matches the supplied value. May be passed multiple times.")
94@cli.subcommand('Generate QMK API data', hidden=False if cli.config.user.developer else True)
95def generate_api(cli):
96 """Generates the QMK API data.
97 """
98 v1_dir = BUILD_API_PATH / 'v1'
99 keyboard_all_file = v1_dir / 'keyboards.json' # A massive JSON containing everything
100 keyboard_list_file = v1_dir / 'keyboard_list.json' # A simple list of keyboard targets
101 keyboard_aliases_file = v1_dir / 'keyboard_aliases.json' # A list of historical keyboard names and their new name
102 keyboard_metadata_file = v1_dir / 'keyboard_metadata.json' # All the data configurator/via needs for initialization
103 constants_metadata_file = v1_dir / 'constants_metadata.json' # Metadata for available constants
104 usb_file = v1_dir / 'usb.json' # A mapping of USB VID/PID -> keyboard target
105
106 if BUILD_API_PATH.exists():
107 shutil.rmtree(BUILD_API_PATH)
108
109 shutil.copytree(TEMPLATE_PATH, BUILD_API_PATH)
110 shutil.copytree(DATA_PATH, v1_dir, copy_function=_filtered_copy)
111
112 # Filter down when required
113 keyboard_list = _filtered_keyboard_list()
114
115 kb_all = {}
116 usb_list = {}
117
118 # Generate and write keyboard specific JSON files
119 for keyboard_name in keyboard_list:
120 kb_json = info_json(keyboard_name)
121 kb_all[keyboard_name] = kb_json
122
123 keyboard_dir = v1_dir / 'keyboards' / keyboard_name
124 keyboard_info = keyboard_dir / 'info.json'
125 keyboard_readme = keyboard_dir / 'readme.md'
126 keyboard_readme_src = find_readme(keyboard_name)
127
128 # Populate the list of JSON keymaps
129 for keymap in list_keymaps(keyboard_name, c=False, fullpath=True):
130 keymap_rel = qmk.path.under_qmk_firmware(keymap)
131 if keymap_rel is None:
132 cli.log.debug('Skipping keymap %s (not in qmk_firmware)', keymap)
133 continue
134
135 if (keymap_rel / 'keymap.c').exists():
136 cli.log.debug('Skipping keymap %s (not pure dd keymap)', keymap)
137 continue
138
139 kb_json['keymaps'][keymap.name] = {
140 # TODO: deprecate 'url' as consumer needs to know its potentially hjson
141 'url': f'https://raw.githubusercontent.com/qmk/qmk_firmware/master/{keymap_rel}/keymap.json',
142
143 # Instead consumer should grab from API and not repo directly
144 'path': (keymap_rel / 'keymap.json').as_posix(),
145 }
146
147 keyboard_dir.mkdir(parents=True, exist_ok=True)
148 keyboard_json = json.dumps({'last_updated': current_datetime(), 'keyboards': {keyboard_name: kb_json}}, separators=(',', ':'))
149 if not cli.args.dry_run:
150 keyboard_info.write_text(keyboard_json, encoding='utf-8')
151 cli.log.debug('Wrote file %s', keyboard_info)
152
153 if keyboard_readme_src:
154 shutil.copyfile(keyboard_readme_src, keyboard_readme)
155 cli.log.debug('Copied %s -> %s', keyboard_readme_src, keyboard_readme)
156
157 # resolve keymaps as json
158 for keymap in kb_json['keymaps']:
159 keymap_hjson = kb_json['keymaps'][keymap]['path']
160 keymap_json = v1_dir / keymap_hjson
161 keymap_json.parent.mkdir(parents=True, exist_ok=True)
162 keymap_json.write_text(json.dumps(json_load(Path(keymap_hjson)), separators=(',', ':')), encoding='utf-8')
163 cli.log.debug('Wrote keymap %s', keymap_json)
164
165 if 'usb' in kb_json:
166 usb = kb_json['usb']
167
168 if 'vid' in usb and usb['vid'] not in usb_list:
169 usb_list[usb['vid']] = {}
170
171 if 'pid' in usb and usb['pid'] not in usb_list[usb['vid']]:
172 usb_list[usb['vid']][usb['pid']] = {}
173
174 if 'vid' in usb and 'pid' in usb:
175 usb_list[usb['vid']][usb['pid']][keyboard_name] = usb
176
177 # Generate data for the global files
178 keyboard_list = sorted(kb_all)
179 keyboard_aliases = keyboard_alias_definitions()
180 keyboard_metadata = {
181 'last_updated': current_datetime(),
182 'keyboards': keyboard_list,
183 'keyboard_aliases': keyboard_aliases,
184 'usb': usb_list,
185 }
186
187 # Feature specific handling
188 _resolve_keycode_specs(v1_dir)
189
190 # Write the global JSON files
191 keyboard_all_json = json.dumps({'last_updated': current_datetime(), 'keyboards': kb_all}, separators=(',', ':'))
192 usb_json = json.dumps({'last_updated': current_datetime(), 'usb': usb_list}, separators=(',', ':'))
193 keyboard_list_json = json.dumps({'last_updated': current_datetime(), 'keyboards': keyboard_list}, separators=(',', ':'))
194 keyboard_aliases_json = json.dumps({'last_updated': current_datetime(), 'keyboard_aliases': keyboard_aliases}, separators=(',', ':'))
195 keyboard_metadata_json = json.dumps(keyboard_metadata, separators=(',', ':'))
196 constants_metadata_json = json.dumps({'last_updated': current_datetime(), 'constants': _list_constants(v1_dir)}, separators=(',', ':'))
197
198 if not cli.args.dry_run:
199 keyboard_all_file.write_text(keyboard_all_json, encoding='utf-8')
200 usb_file.write_text(usb_json, encoding='utf-8')
201 keyboard_list_file.write_text(keyboard_list_json, encoding='utf-8')
202 keyboard_aliases_file.write_text(keyboard_aliases_json, encoding='utf-8')
203 keyboard_metadata_file.write_text(keyboard_metadata_json, encoding='utf-8')
204 constants_metadata_file.write_text(constants_metadata_json, encoding='utf-8')