Merge branch 'overhaul2025'
This commit is contained in:
104
.github/scripts/generate-build-matrices.py
vendored
104
.github/scripts/generate-build-matrices.py
vendored
@@ -1,113 +1,33 @@
|
||||
import itertools
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import NoReturn
|
||||
|
||||
EXTENSION_REGEX = re.compile(r"^src/(?P<lang>\w+)/(?P<extension>\w+)")
|
||||
MULTISRC_LIB_REGEX = re.compile(r"^lib-multisrc/(?P<multisrc>\w+)")
|
||||
LIB_REGEX = re.compile(r"^lib/(?P<lib>\w+)")
|
||||
MODULE_REGEX = re.compile(r"^:src:(?P<lang>\w+):(?P<extension>\w+)$")
|
||||
CORE_FILES_REGEX = re.compile(
|
||||
r"^(buildSrc/|core/|gradle/|build\.gradle\.kts|common\.gradle|gradle\.properties|settings\.gradle\.kts|utils/)"
|
||||
)
|
||||
|
||||
def run_command(command: str) -> str:
|
||||
result = subprocess.run(command, capture_output=True, text=True, shell=True)
|
||||
if result.returncode != 0:
|
||||
print(result.stderr.strip())
|
||||
sys.exit(result.returncode)
|
||||
return result.stdout.strip()
|
||||
|
||||
def get_module_list(ref: str) -> tuple[list[str], list[str]]:
|
||||
changed_files = run_command(f"git diff --name-only {ref}").splitlines()
|
||||
|
||||
modules = set()
|
||||
libs = set()
|
||||
deleted = set()
|
||||
core_files_changed = False
|
||||
|
||||
for file in map(lambda x: Path(x).as_posix(), changed_files):
|
||||
if CORE_FILES_REGEX.search(file):
|
||||
core_files_changed = True
|
||||
elif match := EXTENSION_REGEX.search(file):
|
||||
lang = match.group("lang")
|
||||
extension = match.group("extension")
|
||||
if Path("src", lang, extension).is_dir():
|
||||
modules.add(f':src:{lang}:{extension}')
|
||||
deleted.add(f"{lang}.{extension}")
|
||||
elif match := MULTISRC_LIB_REGEX.search(file):
|
||||
multisrc = match.group("multisrc")
|
||||
if Path("lib-multisrc", multisrc).is_dir():
|
||||
libs.add(f":lib-multisrc:{multisrc}:printDependentExtensions")
|
||||
elif match := LIB_REGEX.search(file):
|
||||
lib = match.group("lib")
|
||||
if Path("lib", lib).is_dir():
|
||||
libs.add(f":lib:{lib}:printDependentExtensions")
|
||||
|
||||
def is_extension_module(module: str) -> bool:
|
||||
if not (match := MODULE_REGEX.search(module)):
|
||||
return False
|
||||
lang = match.group("lang")
|
||||
extension = match.group("extension")
|
||||
deleted.add(f"{lang}.{extension}")
|
||||
return True
|
||||
|
||||
if len(libs) != 0 and not core_files_changed:
|
||||
modules.update([
|
||||
module for module in
|
||||
run_command("./gradlew -q " + " ".join(libs)).splitlines()
|
||||
if is_extension_module(module)
|
||||
])
|
||||
|
||||
if os.getenv("IS_PR_CHECK") != "true" and not core_files_changed:
|
||||
with Path(".github/always_build.json").open() as always_build_file:
|
||||
always_build = json.load(always_build_file)
|
||||
for extension in always_build:
|
||||
modules.add(":src:" + extension.replace(".", ":"))
|
||||
deleted.add(extension)
|
||||
|
||||
if core_files_changed:
|
||||
(all_modules, all_deleted) = get_all_modules()
|
||||
|
||||
modules.update(all_modules)
|
||||
deleted.update(all_deleted)
|
||||
|
||||
return list(modules), list(deleted)
|
||||
|
||||
def get_all_modules() -> tuple[list[str], list[str]]:
|
||||
modules = []
|
||||
deleted = []
|
||||
for lang in Path("src").iterdir():
|
||||
for extension in lang.iterdir():
|
||||
modules.append(f":src:{lang.name}:{extension.name}")
|
||||
deleted.append(f"{lang.name}.{extension.name}")
|
||||
return modules, deleted
|
||||
|
||||
def main() -> NoReturn:
|
||||
_, ref, build_type = sys.argv
|
||||
modules, deleted = get_module_list(ref)
|
||||
_, _, build_type = sys.argv
|
||||
|
||||
# Always build only src:all:kavita
|
||||
modules = [":src:all:kavita"]
|
||||
deleted = ["all.kavita"]
|
||||
|
||||
chunked = {
|
||||
"chunk": [
|
||||
{"number": i + 1, "modules": modules}
|
||||
for i, modules in
|
||||
enumerate(itertools.batched(
|
||||
map(lambda x: f"{x}:assemble{build_type}", modules),
|
||||
int(os.getenv("CI_CHUNK_SIZE", 65))
|
||||
))
|
||||
{
|
||||
"number": 1,
|
||||
"modules": [f":src:all:kavita:assemble{build_type}"]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
print(f"Module chunks to build:\n{json.dumps(chunked, indent=2)}\n\nModule to delete:\n{json.dumps(deleted, indent=2)}")
|
||||
print(
|
||||
f"Module chunks to build:\n{json.dumps(chunked, indent=2)}\n\nModule to delete:\n{json.dumps(deleted, indent=2)}")
|
||||
|
||||
if os.getenv("CI") == "true":
|
||||
with open(os.getenv("GITHUB_OUTPUT"), 'a') as out_file:
|
||||
out_file.write(f"matrix={json.dumps(chunked)}\n")
|
||||
out_file.write(f"delete={json.dumps(deleted)}\n")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
Reference in New Issue
Block a user