| ... |
... |
@@ -67,136 +67,227 @@ def git_lines(git_args: list[str]) -> list[str]: |
|
67
|
67
|
return [line for line in git_text(git_args).split("\n") if line]
|
|
68
|
68
|
|
|
69
|
69
|
|
|
70
|
|
-def git_file_paths(git_ref: str) -> list[str]:
|
|
71
|
|
- """Get the full list of file paths found under the given tree.
|
|
72
|
|
-
|
|
73
|
|
- :param git_ref: The git reference for the tree to search.
|
|
74
|
|
- :returns: The found file paths.
|
|
75
|
|
- """
|
|
76
|
|
- return git_lines(["ls-tree", "-r", "--format=%(path)", git_ref])
|
|
77
|
|
-
|
|
78
|
|
-
|
|
79
|
|
-def matching_path(search_paths: list[str], filename: str) -> str | None:
|
|
80
|
|
- """Get the matching file path with the given filename, if it exists.
|
|
81
|
|
-
|
|
82
|
|
- :param search_paths: The file paths to search through.
|
|
83
|
|
- :param filename: The file name to match.
|
|
84
|
|
- :returns: The unique file path with the matching name, or None if no such
|
|
85
|
|
- match was found.
|
|
86
|
|
- :throws Exception: If multiple paths shared the same file name.
|
|
87
|
|
- """
|
|
88
|
|
- matching = [path for path in search_paths if os.path.basename(path) == filename]
|
|
89
|
|
- if not matching:
|
|
90
|
|
- return None
|
|
91
|
|
- if len(matching) > 1:
|
|
92
|
|
- raise Exception("Multiple occurrences of {filename}")
|
|
93
|
|
- return matching[0]
|
|
94
|
|
-
|
|
95
|
|
-
|
|
96
|
|
-def git_file_content(git_ref: str, path: str | None) -> str | None:
|
|
97
|
|
- """Get the file content of the specified git blob object.
|
|
98
|
|
-
|
|
99
|
|
- :param git_ref: The reference for the tree to find the file under.
|
|
100
|
|
- :param path: The file path for the object, or None if there is no path.
|
|
101
|
|
- :returns: The file content, or None if no path was given.
|
|
102
|
|
- """
|
|
103
|
|
- if path is None:
|
|
104
|
|
- return None
|
|
105
|
|
- return git_text(["cat-file", "blob", f"{git_ref}:{path}"])
|
|
106
|
|
-
|
|
107
|
|
-
|
|
108
|
|
-def get_stable_branch(branch_prefix: str) -> str:
|
|
|
70
|
+class BrowserBranch:
|
|
|
71
|
+ """Represents a browser git branch."""
|
|
|
72
|
+
|
|
|
73
|
+ def __init__(self, branch_name: str, is_head: bool = False) -> None:
|
|
|
74
|
+ """Create a new instance.
|
|
|
75
|
+
|
|
|
76
|
+ :param branch_name: The branch's git name.
|
|
|
77
|
+ :param is_head: Whether the branch matches "HEAD".
|
|
|
78
|
+ """
|
|
|
79
|
+ version_match = re.match(
|
|
|
80
|
+ r"(?P<prefix>[a-z]+\-browser)\-"
|
|
|
81
|
+ r"(?P<firefox>[0-9]+(?:\.[0-9]+){1,2})esr\-"
|
|
|
82
|
+ r"(?P<browser>[0-9]+\.[05])\-"
|
|
|
83
|
+ r"(?P<number>[0-9]+)$",
|
|
|
84
|
+ branch_name,
|
|
|
85
|
+ )
|
|
|
86
|
+
|
|
|
87
|
+ if not version_match:
|
|
|
88
|
+ raise ValueError(f"Unable to parse the version from the ref {branch_name}")
|
|
|
89
|
+
|
|
|
90
|
+ self.name = branch_name
|
|
|
91
|
+ self.prefix = version_match.group("prefix")
|
|
|
92
|
+ self.browser_version = version_match.group("browser")
|
|
|
93
|
+ self._is_head = is_head
|
|
|
94
|
+ self._ref = "HEAD" if is_head else f"origin/{branch_name}"
|
|
|
95
|
+
|
|
|
96
|
+ firefox_nums = [int(n) for n in version_match.group("firefox").split(".")]
|
|
|
97
|
+ if len(firefox_nums) == 2:
|
|
|
98
|
+ firefox_nums.append(0)
|
|
|
99
|
+ browser_nums = [int(n) for n in self.browser_version.split(".")]
|
|
|
100
|
+ branch_number = int(version_match.group("number"))
|
|
|
101
|
+ # Prioritise the firefox ESR version, then the browser version then the
|
|
|
102
|
+ # branch number.
|
|
|
103
|
+ self._ordered = (
|
|
|
104
|
+ firefox_nums[0],
|
|
|
105
|
+ firefox_nums[1],
|
|
|
106
|
+ firefox_nums[2],
|
|
|
107
|
+ browser_nums[0],
|
|
|
108
|
+ browser_nums[1],
|
|
|
109
|
+ branch_number,
|
|
|
110
|
+ )
|
|
|
111
|
+
|
|
|
112
|
+ # Minor version for browser is only ever "0" or "5", so we can convert
|
|
|
113
|
+ # the version to an integer.
|
|
|
114
|
+ self._browser_int_version = int(2 * float(self.browser_version))
|
|
|
115
|
+
|
|
|
116
|
+ self._file_paths: list[str] | None = None
|
|
|
117
|
+
|
|
|
118
|
+ def release_below(self, other: "BrowserBranch", num: int) -> bool:
|
|
|
119
|
+ """Determine whether another branch is within range of a previous
|
|
|
120
|
+ browser release.
|
|
|
121
|
+
|
|
|
122
|
+ The browser versions are expected to increment by "0.5", and a previous
|
|
|
123
|
+ release branch's version is expected to be `num * 0.5` behind the
|
|
|
124
|
+ current one.
|
|
|
125
|
+
|
|
|
126
|
+ :param other: The branch to compare.
|
|
|
127
|
+ :param num: The number of "0.5" releases behind to test with.
|
|
|
128
|
+ """
|
|
|
129
|
+ return other._browser_int_version == self._browser_int_version - num
|
|
|
130
|
+
|
|
|
131
|
+ def __lt__(self, other: "BrowserBranch") -> bool:
|
|
|
132
|
+ return self._ordered < other._ordered
|
|
|
133
|
+
|
|
|
134
|
+ def __gt__(self, other: "BrowserBranch") -> bool:
|
|
|
135
|
+ return self._ordered > other._ordered
|
|
|
136
|
+
|
|
|
137
|
+ def get_file_content(self, filename: str) -> str | None:
|
|
|
138
|
+ """Fetch the file content for the named file in this branch.
|
|
|
139
|
+
|
|
|
140
|
+ :param filename: The name of the file to fetch the content for.
|
|
|
141
|
+ :returns: The file content, or `None` if no file could be found.
|
|
|
142
|
+ """
|
|
|
143
|
+ if self._file_paths is None:
|
|
|
144
|
+ if not self._is_head:
|
|
|
145
|
+ # Minimal fetch of non-HEAD branch to get the file paths.
|
|
|
146
|
+ # Individual file blobs will be downloaded as needed.
|
|
|
147
|
+ git_run(
|
|
|
148
|
+ ["fetch", "--depth=1", "--filter=blob:none", "origin", self._ref]
|
|
|
149
|
+ )
|
|
|
150
|
+ self._file_paths = git_lines(
|
|
|
151
|
+ ["ls-tree", "-r", "--format=%(path)", self._ref]
|
|
|
152
|
+ )
|
|
|
153
|
+
|
|
|
154
|
+ matching = [
|
|
|
155
|
+ path for path in self._file_paths if os.path.basename(path) == filename
|
|
|
156
|
+ ]
|
|
|
157
|
+ if not matching:
|
|
|
158
|
+ return None
|
|
|
159
|
+ if len(matching) > 1:
|
|
|
160
|
+ raise Exception(f"Multiple occurrences of {filename}")
|
|
|
161
|
+
|
|
|
162
|
+ path = matching[0]
|
|
|
163
|
+
|
|
|
164
|
+ return git_text(["cat-file", "blob", f"{self._ref}:{path}"])
|
|
|
165
|
+
|
|
|
166
|
+
|
|
|
167
|
+def get_stable_branch(
|
|
|
168
|
+ compare_version: BrowserBranch,
|
|
|
169
|
+) -> tuple[BrowserBranch, BrowserBranch | None]:
|
|
109
|
170
|
"""Find the most recent stable branch in the origin repository.
|
|
110
|
171
|
|
|
111
|
|
- :param branch_prefix: The prefix that the stable branch should have.
|
|
112
|
|
- :returns: The branch name.
|
|
|
172
|
+ :param compare_version: The development branch to compare against.
|
|
|
173
|
+ :returns: The stable and legacy branches. If no legacy branch is found,
|
|
|
174
|
+ `None` will be returned instead.
|
|
113
|
175
|
"""
|
|
114
|
|
- tag_glob = f"{branch_prefix}-*-build1"
|
|
|
176
|
+ # We search for build1 tags. These are added *after* the rebase of browser
|
|
|
177
|
+ # commits, so the corresponding branch should contain our strings.
|
|
|
178
|
+ # Moreover, we *assume* that the branch with the most recent ESR version
|
|
|
179
|
+ # with such a tag will be used in the *next* stable build in
|
|
|
180
|
+ # tor-browser-build.
|
|
|
181
|
+ tag_glob = f"{compare_version.prefix}-*esr-*-*-build1"
|
|
|
182
|
+
|
|
115
|
183
|
# To speed up, only fetch the tags without blobs.
|
|
116
|
184
|
git_run(
|
|
117
|
185
|
["fetch", "--depth=1", "--filter=object:type=tag", "origin", "tag", tag_glob]
|
|
118
|
186
|
)
|
|
119
|
|
- # Get most recent stable tag.
|
|
|
187
|
+ stable_branches = []
|
|
|
188
|
+ legacy_branches = []
|
|
|
189
|
+ stable_annotation_regex = re.compile(r"\bstable\b")
|
|
|
190
|
+ legacy_annotation_regex = re.compile(r"\blegacy\b")
|
|
|
191
|
+
|
|
120
|
192
|
for build_tag, annotation in (
|
|
121
|
|
- line.split(" ", 1)
|
|
122
|
|
- for line in git_lines(["tag", "-n1", "--list", tag_glob, "--sort=-taggerdate"])
|
|
|
193
|
+ line.split(" ", 1) for line in git_lines(["tag", "-n1", "--list", tag_glob])
|
|
123
|
194
|
):
|
|
124
|
|
- if "stable" in annotation:
|
|
|
195
|
+ is_stable = bool(stable_annotation_regex.search(annotation))
|
|
|
196
|
+ is_legacy = bool(legacy_annotation_regex.search(annotation))
|
|
|
197
|
+ if not is_stable and not is_legacy:
|
|
|
198
|
+ continue
|
|
|
199
|
+ try:
|
|
125
|
200
|
# Branch name is the same as the tag, minus "-build1".
|
|
126
|
|
- return re.sub(r"-build1$", "", build_tag)
|
|
127
|
|
- raise Exception("No stable build1 tag found")
|
|
128
|
|
-
|
|
129
|
|
-
|
|
130
|
|
-def get_version_from_branch_name(branch_name: str) -> tuple[str, float]:
|
|
131
|
|
- """Get the branch prefix and version from its name.
|
|
132
|
|
-
|
|
133
|
|
- :param branch_name: The branch to extract from.
|
|
134
|
|
- :returns: The branch prefix and its version number.
|
|
135
|
|
- """
|
|
136
|
|
- version_match = re.match(
|
|
137
|
|
- r"([a-z-]+)-[^-]*-([0-9]+\.[05])-",
|
|
138
|
|
- branch_name,
|
|
|
201
|
+ branch = BrowserBranch(re.sub(r"-build1$", "", build_tag))
|
|
|
202
|
+ except ValueError:
|
|
|
203
|
+ logger.warning(f"Could not read the version for {build_tag}")
|
|
|
204
|
+ continue
|
|
|
205
|
+ if branch.prefix != compare_version.prefix:
|
|
|
206
|
+ continue
|
|
|
207
|
+ if is_stable:
|
|
|
208
|
+ # Stable can be one release version behind.
|
|
|
209
|
+ # NOTE: In principle, when switching between versions there may be a
|
|
|
210
|
+ # window of time where the development branch has not yet progressed
|
|
|
211
|
+ # to the next "0.5" release, so has the same browser version as the
|
|
|
212
|
+ # stable branch. So we also allow for matching browser versions.
|
|
|
213
|
+ # NOTE:
|
|
|
214
|
+ # 1. The "Will be unused in" message will not make sense, but we do
|
|
|
215
|
+ # not expect string differences in this scenario.
|
|
|
216
|
+ # 2. We do not expect this scenario to last for long.
|
|
|
217
|
+ if not (
|
|
|
218
|
+ compare_version.release_below(branch, 1)
|
|
|
219
|
+ or compare_version.release_below(branch, 0)
|
|
|
220
|
+ ):
|
|
|
221
|
+ continue
|
|
|
222
|
+ stable_branches.append(branch)
|
|
|
223
|
+ elif is_legacy:
|
|
|
224
|
+ # Legacy can be two release versions behind.
|
|
|
225
|
+ # We also allow for being just one version behind.
|
|
|
226
|
+ if not (
|
|
|
227
|
+ compare_version.release_below(branch, 2)
|
|
|
228
|
+ or compare_version.release_below(branch, 1)
|
|
|
229
|
+ ):
|
|
|
230
|
+ continue
|
|
|
231
|
+ legacy_branches.append(branch)
|
|
|
232
|
+
|
|
|
233
|
+ if not stable_branches:
|
|
|
234
|
+ raise Exception("No stable build1 branch found")
|
|
|
235
|
+
|
|
|
236
|
+ return (
|
|
|
237
|
+ # Return the stable branch with the highest version.
|
|
|
238
|
+ max(stable_branches),
|
|
|
239
|
+ max(legacy_branches) if legacy_branches else None,
|
|
139
|
240
|
)
|
|
140
|
241
|
|
|
141
|
|
- if not version_match:
|
|
142
|
|
- raise ValueError(f"Unable to parse the version from the branch {branch_name}")
|
|
143
|
242
|
|
|
144
|
|
- return (version_match.group(1), float(version_match.group(2)))
|
|
|
243
|
+current_branch = BrowserBranch(args.current_branch, is_head=True)
|
|
145
|
244
|
|
|
|
245
|
+stable_branch, legacy_branch = get_stable_branch(current_branch)
|
|
146
|
246
|
|
|
147
|
|
-branch_prefix, current_version = get_version_from_branch_name(args.current_branch)
|
|
|
247
|
+if os.environ.get("TRANSLATION_INCLUDE_LEGACY", "") != "true":
|
|
|
248
|
+ legacy_branch = None
|
|
148
|
249
|
|
|
149
|
|
-stable_branch = get_stable_branch(branch_prefix)
|
|
150
|
|
-_, stable_version = get_version_from_branch_name(stable_branch)
|
|
151
|
|
-
|
|
152
|
|
-if stable_version > current_version or stable_version < current_version - 0.5:
|
|
153
|
|
- raise Exception(
|
|
154
|
|
- f"Version of stable branch {stable_branch} is not within 0.5 of the "
|
|
155
|
|
- f"current branch {args.current_branch}"
|
|
156
|
|
- )
|
|
157
|
|
-
|
|
158
|
|
-# Minimal fetch of stable_branch.
|
|
159
|
|
-# Individual file blobs will be downloaded as needed.
|
|
160
|
|
-git_run(["fetch", "--depth=1", "--filter=blob:none", "origin", stable_branch])
|
|
161
|
|
-
|
|
162
|
|
-current_file_paths = git_file_paths("HEAD")
|
|
163
|
|
-old_file_paths = git_file_paths(f"origin/{stable_branch}")
|
|
164
|
|
-
|
|
165
|
|
-ci_commit = os.environ.get("CI_COMMIT_SHA", "")
|
|
166
|
|
-ci_url_base = os.environ.get("CI_PROJECT_URL", "")
|
|
167
|
|
-
|
|
168
|
|
-json_data = {
|
|
169
|
|
- "commit": ci_commit,
|
|
170
|
|
- "commit-url": f"{ci_url_base}/-/commit/{ci_commit}"
|
|
171
|
|
- if (ci_commit and ci_url_base)
|
|
172
|
|
- else "",
|
|
173
|
|
- "project-path": os.environ.get("CI_PROJECT_PATH", ""),
|
|
174
|
|
- "current-branch": args.current_branch,
|
|
175
|
|
- "stable-branch": stable_branch,
|
|
176
|
|
- "files": [],
|
|
177
|
|
-}
|
|
|
250
|
+files_list = []
|
|
178
|
251
|
|
|
179
|
252
|
for translation_branch, name in (
|
|
180
|
253
|
part.strip().split(":", 1) for part in args.filenames.split(" ") if part.strip()
|
|
181
|
254
|
):
|
|
182
|
|
- current_path = matching_path(current_file_paths, name)
|
|
183
|
|
- old_path = matching_path(old_file_paths, name)
|
|
|
255
|
+ current_content = current_branch.get_file_content(name)
|
|
|
256
|
+ stable_content = stable_branch.get_file_content(name)
|
|
184
|
257
|
|
|
185
|
|
- if current_path is None and old_path is None:
|
|
|
258
|
+ if current_content is None and stable_content is None:
|
|
186
|
259
|
# No file in either branch.
|
|
187
|
260
|
logger.warning(f"{name} does not exist in either the current or stable branch")
|
|
188
|
|
- elif current_path is None:
|
|
|
261
|
+ elif current_content is None:
|
|
189
|
262
|
logger.warning(f"{name} deleted in the current branch")
|
|
190
|
|
- elif old_path is None:
|
|
|
263
|
+ elif stable_content is None:
|
|
191
|
264
|
logger.warning(f"{name} does not exist in the stable branch")
|
|
192
|
265
|
|
|
193
|
266
|
content = combine_files(
|
|
194
|
267
|
name,
|
|
195
|
|
- git_file_content("HEAD", current_path),
|
|
196
|
|
- git_file_content(f"origin/{stable_branch}", old_path),
|
|
197
|
|
- f"Will be unused in Tor Browser {current_version}!",
|
|
|
268
|
+ current_content,
|
|
|
269
|
+ stable_content,
|
|
|
270
|
+ f"Will be unused in Tor Browser {current_branch.browser_version}!",
|
|
198
|
271
|
)
|
|
199
|
|
- json_data["files"].append(
|
|
|
272
|
+
|
|
|
273
|
+ if legacy_branch:
|
|
|
274
|
+ legacy_content = legacy_branch.get_file_content(name)
|
|
|
275
|
+ if (
|
|
|
276
|
+ legacy_content is not None
|
|
|
277
|
+ and current_content is None
|
|
|
278
|
+ and stable_content is None
|
|
|
279
|
+ ):
|
|
|
280
|
+ logger.warning(f"{name} still exists in the legacy branch")
|
|
|
281
|
+ elif legacy_content is None:
|
|
|
282
|
+ logger.warning(f"{name} does not exist in the legacy branch")
|
|
|
283
|
+ content = combine_files(
|
|
|
284
|
+ name,
|
|
|
285
|
+ content,
|
|
|
286
|
+ legacy_content,
|
|
|
287
|
+ f"Unused in Tor Browser {stable_branch.browser_version}!",
|
|
|
288
|
+ )
|
|
|
289
|
+
|
|
|
290
|
+ files_list.append(
|
|
200
|
291
|
{
|
|
201
|
292
|
"name": name,
|
|
202
|
293
|
"branch": translation_branch,
|
| ... |
... |
@@ -204,5 +295,23 @@ for translation_branch, name in ( |
|
204
|
295
|
}
|
|
205
|
296
|
)
|
|
206
|
297
|
|
|
|
298
|
+
|
|
|
299
|
+ci_commit = os.environ.get("CI_COMMIT_SHA", "")
|
|
|
300
|
+ci_url_base = os.environ.get("CI_PROJECT_URL", "")
|
|
|
301
|
+
|
|
|
302
|
+json_data = {
|
|
|
303
|
+ "commit": ci_commit,
|
|
|
304
|
+ "commit-url": f"{ci_url_base}/-/commit/{ci_commit}"
|
|
|
305
|
+ if (ci_commit and ci_url_base)
|
|
|
306
|
+ else "",
|
|
|
307
|
+ "project-path": os.environ.get("CI_PROJECT_PATH", ""),
|
|
|
308
|
+ "current-branch": current_branch.name,
|
|
|
309
|
+ "stable-branch": stable_branch.name,
|
|
|
310
|
+ "files": files_list,
|
|
|
311
|
+}
|
|
|
312
|
+
|
|
|
313
|
+if legacy_branch:
|
|
|
314
|
+ json_data["legacy-branch"] = legacy_branch.name
|
|
|
315
|
+
|
|
207
|
316
|
with open(args.outname, "w") as file:
|
|
208
|
317
|
json.dump(json_data, file) |