huwcampbell.com
Reactos
1#!/usr/bin/env python3
2
3import sys
4import os
5import posixpath
6import string
7import argparse
8import subprocess
9import fnmatch
10import pygit2
11import yaml
12
13def string_to_valid_file_name(to_convert):
14 valid_chars = '-_.()' + string.ascii_letters + string.digits
15 result = ''
16 for c in to_convert:
17 if c in valid_chars:
18 result += c
19 else:
20 result += '_'
21 # strip final dot, if any
22 if result.endswith('.'):
23 return result[:-1]
24 return result
25
26class wine_sync:
27 def __init__(self, module):
28 if os.path.isfile('winesync.cfg'):
29 with open('winesync.cfg', 'r') as file_input:
30 config = yaml.safe_load(file_input)
31 self.reactos_src = config['repos']['reactos']
32 self.wine_src = config['repos']['wine']
33 self.wine_staging_src = config['repos']['wine-staging']
34 else:
35 config = { }
36 self.reactos_src = input('Please enter the path to the reactos git tree: ')
37 self.wine_src = input('Please enter the path to the wine git tree: ')
38 self.wine_staging_src = input('Please enter the path to the wine-staging git tree: ')
39 config['repos'] = { 'reactos': self.reactos_src,
40 'wine': self.wine_src,
41 'wine-staging': self.wine_staging_src }
42 with open('winesync.cfg', 'w') as file_output:
43 yaml.dump(config, file_output)
44
45 self.wine_repo = pygit2.Repository(self.wine_src)
46 self.wine_staging_repo = pygit2.Repository(self.wine_staging_src)
47 self.reactos_repo = pygit2.Repository(self.reactos_src)
48
49 # the standard author signature we will use
50 self.winesync_author_signature = pygit2.Signature('winesync', 'ros-dev@reactos.org')
51
52 # read the index from the reactos tree
53 self.reactos_index = self.reactos_repo.index
54 self.reactos_index.read()
55
56 # get the actual state for the asked module
57 self.module = module
58 with open(module + '.cfg', 'r') as file_input:
59 self.module_cfg = yaml.safe_load(file_input)
60
61 self.staged_patch_dir = posixpath.join('sdk', 'tools', 'winesync', self.module + '_staging')
62
63 def create_or_checkout_wine_branch(self, wine_tag, wine_staging_tag):
64 # build the wine branch name
65 wine_branch_name = 'winesync-' + wine_tag
66 if wine_staging_tag:
67 wine_branch_name += '-' + wine_staging_tag
68
69 branch = self.wine_repo.lookup_branch(wine_branch_name)
70 if branch is None:
71 # get our target commits
72 wine_target_commit = self.wine_repo.revparse_single(wine_tag)
73 if isinstance(wine_target_commit, pygit2.Tag):
74 wine_target_commit = wine_target_commit.target
75 if isinstance(wine_target_commit, pygit2.Commit):
76 wine_target_commit = wine_target_commit.id
77
78 # do the same for the wine-staging tree
79 if wine_staging_tag:
80 wine_staging_target_commit = self.wine_staging_repo.revparse_single(wine_staging_tag)
81 if isinstance(wine_staging_target_commit, pygit2.Tag):
82 wine_staging_target_commit = wine_staging_target_commit.target
83 if isinstance(wine_staging_target_commit, pygit2.Commit):
84 wine_staging_target_commit = wine_staging_target_commit.id
85
86 self.wine_repo.branches.local.create(wine_branch_name, self.wine_repo.revparse_single('HEAD'))
87 self.wine_repo.checkout(self.wine_repo.lookup_branch(wine_branch_name))
88 self.wine_repo.reset(wine_target_commit, pygit2.GIT_RESET_HARD)
89
90 # do the same for the wine-staging tree
91 if wine_staging_tag:
92 self.wine_staging_repo.branches.local.create(wine_branch_name, self.wine_staging_repo.revparse_single('HEAD'))
93 self.wine_staging_repo.checkout(self.wine_staging_repo.lookup_branch(wine_branch_name))
94 self.wine_staging_repo.reset(wine_staging_target_commit, pygit2.GIT_RESET_HARD)
95
96 # run the wine-staging script
97 if subprocess.call(['python', self.wine_staging_src + '/staging/patchinstall.py', 'DESTDIR=' + self.wine_src, '--all', '--backend=git-am']):
98 # the new script failed (it doesn't exist?), try the old one
99 subprocess.call(['bash', '-c', self.wine_staging_src + '/patches/patchinstall.sh DESTDIR=' + self.wine_src + ' --all --backend=git-am'])
100
101 # delete the branch we created
102 self.wine_staging_repo.checkout(self.wine_staging_repo.lookup_branch('master'))
103 self.wine_staging_repo.branches.delete(wine_branch_name)
104 else:
105 self.wine_repo.checkout(self.wine_repo.lookup_branch(wine_branch_name))
106
107 return wine_branch_name
108
109 # Helper function for resolving wine tree path to reactos one
110 # Note: it doesn't care about the fact that the file actually exists or not
111 def wine_to_reactos_path(self, wine_path):
112 if self.module_cfg['files'] and (wine_path in self.module_cfg['files']):
113 # we have a direct mapping
114 return self.module_cfg['files'][wine_path]
115
116 if not '/' in wine_path:
117 # root files should have a direct mapping
118 return None
119
120 wine_dir, wine_file = os.path.split(wine_path)
121 if self.module_cfg['directories'] and (wine_dir in self.module_cfg['directories']):
122 # we have a mapping for the directory
123 return posixpath.join(self.module_cfg['directories'][wine_dir], wine_file)
124
125 # no match
126 return None
127
128 def sync_wine_commit(self, wine_commit, in_staging, staging_patch_index):
129 # Get the diff object
130 diff = self.wine_repo.diff(wine_commit.parents[0], wine_commit)
131
132 modified_files = False
133 ignored_files = []
134 warning_message = ''
135 complete_patch = ''
136
137 if in_staging:
138 # see if we already applied this patch
139 patch_file_name = f'{staging_patch_index:04}-{string_to_valid_file_name(wine_commit.message.splitlines()[0])}.diff'
140 patch_dir = os.path.join(self.reactos_src, self.staged_patch_dir)
141 patch_path = os.path.join(patch_dir, patch_file_name)
142 if os.path.isfile(patch_path):
143 print(f'Skipping patch as {patch_path} already exists')
144 return True, ''
145
146 for delta in diff.deltas:
147 if delta.status == pygit2.GIT_DELTA_ADDED:
148 # check if we should care
149 new_reactos_path = self.wine_to_reactos_path(delta.new_file.path)
150 if not new_reactos_path is None:
151 warning_message += 'file ' + delta.new_file.path + ' is added to the wine tree!\n'
152 old_reactos_path = '/dev/null'
153 else:
154 old_reactos_path = None
155 elif delta.status == pygit2.GIT_DELTA_DELETED:
156 # check if we should care
157 old_reactos_path = self.wine_to_reactos_path(delta.old_file.path)
158 if not old_reactos_path is None:
159 warning_message += 'file ' + delta.old_file.path + ' is removed from the wine tree!\n'
160 new_reactos_path = '/dev/null'
161 else:
162 new_reactos_path = None
163 elif delta.new_file.path.endswith('Makefile.in'):
164 warning_message += 'file ' + delta.new_file.path + ' was modified!\n'
165 # no need to warn that those are ignored, we just did.
166 continue
167 else:
168 new_reactos_path = self.wine_to_reactos_path(delta.new_file.path)
169 old_reactos_path = self.wine_to_reactos_path(delta.old_file.path)
170
171 if (new_reactos_path is not None) or (old_reactos_path is not None):
172 # print('Must apply diff: ' + old_reactos_path + ' --> ' + new_reactos_path)
173 if delta.status == pygit2.GIT_DELTA_ADDED:
174 new_blob = self.wine_repo.get(delta.new_file.id)
175 blob_patch = pygit2.Patch.create_from(
176 old=None,
177 new=new_blob,
178 new_as_path=new_reactos_path)
179 elif delta.status == pygit2.GIT_DELTA_DELETED:
180 old_blob = self.wine_repo.get(delta.old_file.id)
181 blob_patch = pygit2.Patch.create_from(
182 old=old_blob,
183 new=None,
184 old_as_path=old_reactos_path)
185 else:
186 new_blob = self.wine_repo.get(delta.new_file.id)
187 old_blob = self.wine_repo.get(delta.old_file.id)
188
189 blob_patch = pygit2.Patch.create_from(
190 old=old_blob,
191 new=new_blob,
192 old_as_path=old_reactos_path,
193 new_as_path=new_reactos_path)
194
195 # print(str(wine_commit.id))
196 # print(blob_patch.text)
197
198 # this doesn't work
199 # reactos_diff = pygit2.Diff.parse_diff(blob_patch.text)
200 # reactos_repo.apply(reactos_diff)
201 try:
202 subprocess.run(['git', '-C', self.reactos_src, 'apply', '--reject'], input=blob_patch.data, check=True)
203 except subprocess.CalledProcessError as err:
204 warning_message += 'Error while applying patch to ' + new_reactos_path + '\n'
205
206 if delta.status == pygit2.GIT_DELTA_DELETED:
207 try:
208 self.reactos_index.remove(old_reactos_path)
209 except IOError as err:
210 warning_message += 'Error while removing file ' + old_reactos_path + '\n'
211 # here we check if the file exists. We don't complain, because applying the patch already failed anyway
212 elif os.path.isfile(os.path.join(self.reactos_src, new_reactos_path)):
213 self.reactos_index.add(new_reactos_path)
214
215 complete_patch += blob_patch.text
216
217 modified_files = True
218 else:
219 ignored_files += [delta.old_file.path, delta.new_file.path]
220
221 if not modified_files:
222 # We applied nothing
223 return False, ''
224
225 print('Applied patches from wine commit ' + str(wine_commit.id))
226
227 if ignored_files:
228 warning_message += 'WARNING: some files were ignored: ' + ' '.join(ignored_files) + '\n'
229
230 if not in_staging:
231 self.module_cfg['tags']['wine'] = str(wine_commit.id)
232 with open(self.module + '.cfg', 'w') as file_output:
233 yaml.dump(self.module_cfg, file_output)
234 self.reactos_index.add(f'sdk/tools/winesync/{self.module}.cfg')
235 else:
236 # Add the staging patch
237 # do not save the wine commit ID in <module>.cfg, as it's a local one for staging patches
238 if not os.path.isdir(patch_dir):
239 os.mkdir(patch_dir)
240 with open(patch_path, 'w') as file_output:
241 file_output.write(complete_patch)
242 self.reactos_index.add(posixpath.join(self.staged_patch_dir, patch_file_name))
243
244 self.reactos_index.write()
245
246 commit_msg = f'[WINESYNC] {wine_commit.message}\n'
247 if (in_staging):
248 commit_msg += f'wine-staging patch by {wine_commit.author.name} <{wine_commit.author.email}>'
249 else:
250 commit_msg += f'wine commit id {str(wine_commit.id)} by {wine_commit.author.name} <{wine_commit.author.email}>'
251
252 self.reactos_repo.create_commit(
253 'HEAD',
254 self.winesync_author_signature,
255 self.reactos_repo.default_signature,
256 commit_msg,
257 self.reactos_index.write_tree(),
258 [self.reactos_repo.head.target])
259
260 if (warning_message != ''):
261 warning_message += 'If needed, amend the current commit in your reactos tree and start this script again'
262
263 if not in_staging:
264 warning_message += f'\n' \
265 f'You can see the details of the wine commit here:\n' \
266 f' https://source.winehq.org/git/wine.git/commit/{str(wine_commit.id)}\n'
267 else:
268 patch_file_path = posixpath.join(self.staged_patch_dir, patch_file_name)
269 warning_message += f'\n' \
270 f'Do not forget to run\n' \
271 f' git diff HEAD^ \':(exclude){patch_file_path}\' > {patch_file_path}\n' \
272 f'after your correction and then\n' \
273 f' git add {patch_file_path}\n' \
274 f'before running "git commit --amend"'
275
276 return True, warning_message
277
278 def revert_staged_patchset(self):
279 # revert all of this in one commit
280 staged_patch_dir_path = posixpath.join(self.reactos_src, self.staged_patch_dir)
281 if not os.path.isdir(staged_patch_dir_path):
282 return True
283
284 has_patches = False
285
286 for patch_file_name in sorted(os.listdir(staged_patch_dir_path), reverse=True):
287 patch_path = os.path.join(staged_patch_dir_path, patch_file_name)
288 if not os.path.isfile(patch_path):
289 continue
290
291 has_patches = True
292
293 with open(patch_path, 'rb') as patch_file:
294 try:
295 subprocess.run(['git', '-C', self.reactos_src, 'apply', '-R', '--ignore-whitespace', '--reject'], stdin=patch_file, check=True)
296 except subprocess.CalledProcessError as err:
297 print(f'Error while reverting patch {patch_file_name}')
298 print('Please check, remove the offending patch with git rm, and relaunch this script')
299 return False
300
301 self.reactos_index.remove(posixpath.join(self.staged_patch_dir, patch_file_name))
302 self.reactos_index.write()
303 os.remove(patch_path)
304
305 if not has_patches:
306 return True
307
308 # Note: these path lists may be empty or None, in which case
309 # we should not call index.add_all(), otherwise we would add
310 # any untracked file present in the repository.
311 if self.module_cfg['files']:
312 self.reactos_index.add_all([f for f in self.module_cfg['files'].values()])
313 if self.module_cfg['directories']:
314 self.reactos_index.add_all([f'{d}/*.*' for d in self.module_cfg['directories'].values()])
315 self.reactos_index.write()
316
317 self.reactos_repo.create_commit(
318 'HEAD',
319 self.winesync_author_signature,
320 self.reactos_repo.default_signature,
321 f'[WINESYNC]: revert wine-staging patchset for {self.module}',
322 self.reactos_index.write_tree(),
323 [self.reactos_repo.head.target])
324 return True
325
326 def sync_to_wine(self, wine_tag, wine_staging_tag):
327 # Get our target commit
328 wine_target_commit = self.wine_repo.revparse_single(wine_tag)
329 if isinstance(wine_target_commit, pygit2.Tag):
330 wine_target_commit = wine_target_commit.target
331 if isinstance(wine_target_commit, pygit2.Commit):
332 wine_target_commit = wine_target_commit.id
333 # print(f'wine target commit is {wine_target_commit}')
334
335 # get the wine commit id where we left
336 in_staging = False
337 wine_last_sync = self.wine_repo.revparse_single(self.module_cfg['tags']['wine'])
338 if isinstance(wine_last_sync, pygit2.Tag):
339 if not self.revert_staged_patchset():
340 return
341 wine_last_sync = wine_last_sync.target
342 if isinstance(wine_last_sync, pygit2.Commit):
343 wine_last_sync = wine_last_sync.id
344
345 # create a branch to keep things clean
346 wine_branch_name = self.create_or_checkout_wine_branch(wine_tag, wine_staging_tag)
347
348 finished_sync = True
349 staging_patch_index = 1
350
351 # walk each commit between last sync and the asked tag/revision
352 wine_commit_walker = self.wine_repo.walk(self.wine_repo.head.target, pygit2.GIT_SORT_TOPOLOGICAL | pygit2.GIT_SORT_REVERSE)
353 wine_commit_walker.hide(wine_last_sync)
354 for wine_commit in wine_commit_walker:
355 applied_patch, warning_message = self.sync_wine_commit(wine_commit, in_staging, staging_patch_index)
356
357 if str(wine_commit.id) == str(wine_target_commit):
358 print('We are now in staging territory')
359 in_staging = True
360
361 if not applied_patch:
362 continue
363
364 if in_staging:
365 staging_patch_index += 1
366
367 if warning_message != '':
368 print("THERE WERE SOME ISSUES WHEN APPLYING THE PATCH\n\n")
369 print(warning_message)
370 print("\n")
371 finished_sync = False
372 break
373
374 # we're done without error
375 if finished_sync:
376 # update wine tag and commit
377 self.module_cfg['tags']['wine'] = wine_tag
378 with open(self.module + '.cfg', 'w') as file_output:
379 yaml.dump(self.module_cfg, file_output)
380
381 self.reactos_index.add(f'sdk/tools/winesync/{self.module}.cfg')
382 self.reactos_index.write()
383 self.reactos_repo.create_commit(
384 'HEAD',
385 self.winesync_author_signature,
386 self.reactos_repo.default_signature,
387 f'[WINESYNC]: {self.module} is now in sync with wine-staging {wine_tag}',
388 self.reactos_index.write_tree(),
389 [self.reactos_repo.head.target])
390
391 print('The branch ' + wine_branch_name + ' was created in your wine repository. You might want to delete it, but you should keep it in case you want to sync more module up to this wine version')
392
393def main():
394 parser = argparse.ArgumentParser()
395 parser.add_argument('module', help='The module you want to sync. <module>.cfg must exist in the current directory.')
396 parser.add_argument('wine_tag', help='The wine tag or commit id to sync to.')
397 parser.add_argument('wine_staging_tag', nargs='?', default=None, help='The optional wine staging tag or commit id to pick wine staged patches from.')
398
399 args = parser.parse_args()
400
401 syncator = wine_sync(args.module)
402
403 return syncator.sync_to_wine(args.wine_tag, args.wine_staging_tag)
404
405
406if __name__ == '__main__':
407 main()