diff options
Diffstat (limited to 'meta')
| -rw-r--r-- | meta/common/java.py | 3 | ||||
| -rw-r--r-- | meta/model/java.py | 151 | ||||
| -rw-r--r-- | meta/run/generate_java.py | 75 | ||||
| -rw-r--r-- | meta/run/update_java.py | 82 |
4 files changed, 222 insertions, 89 deletions
diff --git a/meta/common/java.py b/meta/common/java.py index ec13b2a55f..8b83f65aa0 100644 --- a/meta/common/java.py +++ b/meta/common/java.py @@ -4,11 +4,14 @@ BASE_DIR = "java_runtime" RELEASE_FILE = join(BASE_DIR, "releases.json") ADOPTIUM_DIR = join(BASE_DIR, "adoptium") +OPENJ9_DIR = join(BASE_DIR, "ibm") AZUL_DIR = join(BASE_DIR, "azul") ADOPTIUM_VERSIONS_DIR = join(ADOPTIUM_DIR, "versions") +OPENJ9_VERSIONS_DIR = join(OPENJ9_DIR, "versions") AZUL_VERSIONS_DIR = join(AZUL_DIR, "versions") JAVA_MINECRAFT_COMPONENT = "net.minecraft.java" JAVA_ADOPTIUM_COMPONENT = "net.adoptium.java" +JAVA_OPENJ9_COMPONENT = "com.ibm.java" JAVA_AZUL_COMPONENT = "com.azul.java" diff --git a/meta/model/java.py b/meta/model/java.py index b941860726..4a7571e491 100644 --- a/meta/model/java.py +++ b/meta/model/java.py @@ -18,7 +18,7 @@ class JavaRuntimeOS(StrEnum): MacOsX64 = "mac-os-x64" MacOsX86 = "mac-os-x86" # rare MacOsArm64 = "mac-os-arm64" - # MacOsArm32 = "mac-os-arm32" # doesn't exsist + # MacOsArm32 = "mac-os-arm32" # doesn't exist LinuxX64 = "linux-x64" LinuxX86 = "linux-x86" LinuxArm64 = "linux-arm64" @@ -123,15 +123,18 @@ class APIQuery(MetaBase): return urlencode(set_parts, doseq=True) -class AdoptiumJvmImpl(StrEnum): - Hostspot = "hotspot" +# Adoptx refers to both Adoptium (Eclipse Temurin) and AdoptOpenJDK (IBM Semeru Runtime Open Edition) +class AdoptxJvmImpl(StrEnum): + Hotspot = "hotspot" + OpenJ9 = "openj9" -class AdoptiumVendor(StrEnum): +class AdoptxVendor(StrEnum): Eclipse = "eclipse" + Ibm = "ibm" -class AdoptiumArchitecture(StrEnum): +class AdoptxArchitecture(StrEnum): X64 = "x64" X86 = "x86" X32 = "x32" @@ -144,22 +147,22 @@ class AdoptiumArchitecture(StrEnum): Riscv64 = "riscv64" -class AdoptiumReleaseType(StrEnum): - GenralAccess = "ga" +class AdoptxReleaseType(StrEnum): + GeneralAccess = "ga" EarlyAccess = "ea" -class AdoptiumSortMethod(StrEnum): +class AdoptxSortMethod(StrEnum): Default = "DEFAULT" Date = "DATE" -class AdoptiumSortOrder(StrEnum): +class AdoptxSortOrder(StrEnum): Asc = "ASC" Desc = "DESC" -class AdoptiumImageType(StrEnum): +class AdoptxImageType(StrEnum): Jdk = "jdk" Jre = "jre" Testimage = "testimage" @@ -169,12 +172,12 @@ class AdoptiumImageType(StrEnum): Sbom = "sbom" -class AdoptiumHeapSize(StrEnum): +class AdoptxHeapSize(StrEnum): Normal = "normal" Large = "large" -class AdoptiumProject(StrEnum): +class AdoptxProject(StrEnum): Jdk = "jdk" Valhalla = "valhalla" Metropolis = "metropolis" @@ -182,12 +185,12 @@ class AdoptiumProject(StrEnum): Shenandoah = "shenandoah" -class AdoptiumCLib(StrEnum): +class AdoptxCLib(StrEnum): Musl = "musl" Glibc = "glibc" -class AdoptiumOs(StrEnum): +class AdoptxOs(StrEnum): Linux = "linux" Windows = "windows" Mac = "mac" @@ -196,43 +199,72 @@ class AdoptiumOs(StrEnum): AlpineLinux = "alpine-linux" -ADOPTIUM_API_BASE = " https://api.adoptium.net" -ADOPTIUM_API_FEATURE_RELEASES = f"{ADOPTIUM_API_BASE}/v3/assets/feature_releases/{{feature_version}}/{{release_type}}" +ADOPTIUM_API_BASE = "https://api.adoptium.net" +OPENJ9_API_BASE = " https://api.adoptopenjdk.net" +ADOPTX_API_FEATURE_RELEASES = f"{{base_url}}/v3/assets/feature_releases/{{feature_version}}/{{release_type}}" # ?image_type={{image_type}}&heap_size={{heap_size}}&project={{project}}&vendor={{vendor}}&page_size={{page_size}}&page={{page}}&sort_method={{sort_method}}&sort_order={{sort_order}} -ADOPTIUM_API_AVAILABLE_RELEASES = f"{ADOPTIUM_API_BASE}/v3/info/available_releases" +ADOPTX_API_AVAILABLE_RELEASES = f"{{base_url}}/v3/info/available_releases" -class AdoptiumAPIFeatureReleasesQuery(APIQuery): - architecture: Optional[AdoptiumArchitecture] = None +class AdoptxAPIFeatureReleasesQuery(APIQuery): + architecture: Optional[AdoptxArchitecture] = None before: Optional[datetime] = None - c_lib: Optional[AdoptiumCLib] = None - heap_size: Optional[AdoptiumHeapSize] = AdoptiumHeapSize.Normal - image_type: Optional[AdoptiumImageType] = None - jvm_impl: Optional[AdoptiumJvmImpl] = None - os: Optional[AdoptiumOs] = None + c_lib: Optional[AdoptxCLib] = None + heap_size: Optional[AdoptxHeapSize] = AdoptxHeapSize.Normal + image_type: Optional[AdoptxImageType] = None + jvm_impl: Optional[AdoptxJvmImpl] = None + os: Optional[AdoptxOs] = None + vendor: Optional[AdoptxVendor] = None page_size: int = 10 page: int = 0 - project: Optional[AdoptiumProject] = AdoptiumProject.Jdk - sort_method: Optional[AdoptiumSortMethod] = AdoptiumSortMethod.Default - sort_order: Optional[AdoptiumSortOrder] = AdoptiumSortOrder.Desc - vendor: Optional[AdoptiumVendor] = AdoptiumVendor.Eclipse + project: Optional[AdoptxProject] = AdoptxProject.Jdk + sort_method: Optional[AdoptxSortMethod] = AdoptxSortMethod.Default + sort_order: Optional[AdoptxSortOrder] = AdoptxSortOrder.Desc -def adoptiumAPIFeatureReleasesUrl( - feature: int, - release_type: AdoptiumReleaseType = AdoptiumReleaseType.GenralAccess, - query: AdoptiumAPIFeatureReleasesQuery = AdoptiumAPIFeatureReleasesQuery(), +def adoptxAPIFeatureReleasesUrl( + base_url: str, + feature_version: int, + release_type: AdoptxReleaseType = AdoptxReleaseType.GeneralAccess, + query: AdoptxAPIFeatureReleasesQuery = AdoptxAPIFeatureReleasesQuery(), ): url = urlparse( - ADOPTIUM_API_FEATURE_RELEASES.format( - feature_version=feature, + ADOPTX_API_FEATURE_RELEASES.format( + base_url=base_url, + feature_version=feature_version, release_type=release_type.value, ) ) return urlunparse(url._replace(query=query.to_query())) -class AdoptiumAvailableReleases(MetaBase): +def adoptiumAPIFeatureReleasesUrl( + feature_version: int, + release_type: AdoptxReleaseType = AdoptxReleaseType.GeneralAccess, + query: AdoptxAPIFeatureReleasesQuery = AdoptxAPIFeatureReleasesQuery(), +): + return adoptxAPIFeatureReleasesUrl( + feature_version=feature_version, + release_type=release_type, + query=query, + base_url=ADOPTIUM_API_BASE, + ) + + +def openj9APIFeatureReleasesUrl( + feature_version: int, + release_type: AdoptxReleaseType = AdoptxReleaseType.GeneralAccess, + query: AdoptxAPIFeatureReleasesQuery = AdoptxAPIFeatureReleasesQuery(), +): + return adoptxAPIFeatureReleasesUrl( + feature_version=feature_version, + release_type=release_type, + query=query, + base_url=OPENJ9_API_BASE, + ) + + +class AdoptxAvailableReleases(MetaBase): available_releases: list[int] available_lts_releases: list[int] most_recent_lts: Optional[int] @@ -241,13 +273,13 @@ class AdoptiumAvailableReleases(MetaBase): tip_version: Optional[int] -class AdoptiumFile(MetaBase): +class AdoptxFile(MetaBase): name: str link: str size: Optional[int] -class AdoptiumPackage(AdoptiumFile): +class AdoptxPackage(AdoptxFile): checksum: Optional[str] checksum_link: Optional[str] signature_link: Optional[str] @@ -255,22 +287,22 @@ class AdoptiumPackage(AdoptiumFile): # we intentionally omit download_count -class AdoptiumBinary(MetaBase): +class AdoptxBinary(MetaBase): os: str - architecture: AdoptiumArchitecture - image_type: AdoptiumImageType - c_lib: Optional[AdoptiumCLib] - jvm_impl: AdoptiumJvmImpl - package: Optional[AdoptiumPackage] - installer: Optional[AdoptiumPackage] - heap_size: AdoptiumHeapSize + architecture: AdoptxArchitecture + image_type: AdoptxImageType + c_lib: Optional[AdoptxCLib] + jvm_impl: AdoptxJvmImpl + package: Optional[AdoptxPackage] + installer: Optional[AdoptxPackage] + heap_size: AdoptxHeapSize updated_at: datetime scm_ref: Optional[str] - project: AdoptiumProject + project: AdoptxProject # we intentionally omit download_count -class AdoptiumVersion(MetaBase): +class AdoptxVersion(MetaBase): major: Optional[int] minor: Optional[int] security: Optional[int] @@ -283,31 +315,31 @@ class AdoptiumVersion(MetaBase): optional: Optional[str] -class AdoptiumRelease(MetaBase): +class AdoptxRelease(MetaBase): release_id: str = Field(alias="id") release_link: str release_name: str timestamp: datetime updated_at: datetime - binaries: list[AdoptiumBinary] + binaries: list[AdoptxBinary] release_type: str - vendor: AdoptiumVendor - version_data: AdoptiumVersion - source: Optional[AdoptiumFile] - release_notes: Optional[AdoptiumFile] + vendor: AdoptxVendor + version_data: AdoptxVersion + source: Optional[AdoptxFile] + release_notes: Optional[AdoptxFile] # we intentionally omit download_count -class AdoptiumReleases(MetaBase): - __root__: list[AdoptiumRelease] +class AdoptxReleases(MetaBase): + __root__: list[AdoptxRelease] - def __iter__(self) -> Generator[tuple[str, AdoptiumRelease], None, None]: + def __iter__(self) -> Generator[tuple[str, AdoptxRelease], None, None]: yield from ((str(i), val) for i, val in enumerate(self.__root__)) - def __getitem__(self, item: int) -> AdoptiumRelease: + def __getitem__(self, item: int) -> AdoptxRelease: return self.__root__[item] - def append(self, rls: AdoptiumRelease): + def append(self, rls: AdoptxRelease): self.__root__.append(rls) @@ -556,7 +588,8 @@ class ZuluPackagesDetail(MetaBase): MOJANG_OS_NAMES = ["mac-os", "linux", "windows"] MOJANG_OS_ARCHITECTURES = [ - "x64" "x86", + "x64", + "x86", "arm64", "arm32", ] diff --git a/meta/run/generate_java.py b/meta/run/generate_java.py index 13e01a0496..257326542a 100644 --- a/meta/run/generate_java.py +++ b/meta/run/generate_java.py @@ -9,9 +9,12 @@ from meta.common import ensure_component_dir, launcher_path, upstream_path from meta.common.java import ( JAVA_MINECRAFT_COMPONENT, JAVA_ADOPTIUM_COMPONENT, + JAVA_OPENJ9_COMPONENT, JAVA_AZUL_COMPONENT, ADOPTIUM_DIR, ADOPTIUM_VERSIONS_DIR, + OPENJ9_DIR, + OPENJ9_VERSIONS_DIR, AZUL_DIR, AZUL_VERSIONS_DIR, ) @@ -25,11 +28,11 @@ from meta.model.java import ( JavaChecksumMeta, JavaChecksumType, JavaRuntimeDownloadType, - AdoptiumAvailableReleases, - AdoptiumReleases, - AdoptiumRelease, - AdoptiumImageType, - AdoptiumBinary, + AdoptxAvailableReleases, + AdoptxReleases, + AdoptxRelease, + AdoptxImageType, + AdoptxBinary, ZuluPackageList, ZuluPackageDetail, AzulJavaPackageType, @@ -193,9 +196,9 @@ def mojang_runtime_to_java_runtime( ) -def adoptium_release_binary_to_java_runtime( - rls: AdoptiumRelease, - binary: AdoptiumBinary, +def adoptx_release_binary_to_java_runtime( + rls: AdoptxRelease, + binary: AdoptxBinary, runtime_os: JavaRuntimeOS, ) -> JavaRuntimeMeta: assert binary.package is not None @@ -216,7 +219,13 @@ def adoptium_release_binary_to_java_runtime( ), build=rls.version_data.build, ) - rls_name = f"{rls.vendor}_temurin_{binary.image_type}{version}" + + if rls.vendor == "eclipse": + rls_distribution = "temurin" + elif rls.vendor == "ibm": + rls_distribution = "semeru-open" + + rls_name = f"{rls.vendor}_{rls_distribution}_{binary.image_type}{version}" return JavaRuntimeMeta( name=rls_name, vendor=rls.vendor, @@ -315,7 +324,7 @@ def main(): def add_java_runtime(runtime: JavaRuntimeMeta, major: int): if major not in javas: javas[major] = list[JavaRuntimeMeta]() - print(f"Regestering runtime: {runtime.name} for Java {major}") + print(f"Registering runtime: {runtime.name} for Java {major}") javas[major].append(runtime) # only add specific versions to the list @@ -349,18 +358,18 @@ def main(): print("Processing Adoptium Releases") adoptium_path = os.path.join(UPSTREAM_DIR, ADOPTIUM_DIR, "available_releases.json") if os.path.exists(adoptium_path): - adoptium_available_releases = AdoptiumAvailableReleases.parse_file( + adoptium_available_releases = AdoptxAvailableReleases.parse_file( adoptium_path ) for major in adoptium_available_releases.available_releases: - adoptium_releases = AdoptiumReleases.parse_file( + adoptium_releases = AdoptxReleases.parse_file( os.path.join(UPSTREAM_DIR, ADOPTIUM_VERSIONS_DIR, f"java{major}.json") ) for _, rls in adoptium_releases: for binary in rls.binaries: if ( binary.package is None - or binary.image_type is not AdoptiumImageType.Jre + or binary.image_type is not AdoptxImageType.Jre ): continue binary_arch = translate_arch(str(binary.architecture)) @@ -370,13 +379,44 @@ def main(): continue java_os = JavaRuntimeOS(f"{binary_os}-{binary_arch}") - runtime = adoptium_release_binary_to_java_runtime( + runtime = adoptx_release_binary_to_java_runtime( rls, binary, java_os ) add_java_runtime(runtime, major) - writeJavas(javas=javas, uid=JAVA_ADOPTIUM_COMPONENT) javas = {} + + print("Processing OpenJ9 Releases") + openj9_path = os.path.join(UPSTREAM_DIR, OPENJ9_DIR, "available_releases.json") + if os.path.exists(openj9_path): + openj9_available_releases = AdoptxAvailableReleases.parse_file( + openj9_path + ) + for major in openj9_available_releases.available_releases: + openj9_releases = AdoptxReleases.parse_file( + os.path.join(UPSTREAM_DIR, OPENJ9_VERSIONS_DIR, f"java{major}.json") + ) + for _, rls in openj9_releases: + for binary in rls.binaries: + if ( + binary.package is None + or binary.image_type is not AdoptxImageType.Jre + ): + continue + binary_arch = translate_arch(str(binary.architecture)) + binary_os = translate_os(str(binary.os)) + if binary_arch is None or binary_os is None: + print(f"Ignoring release for {binary.os} {binary.architecture}") + continue + + java_os = JavaRuntimeOS(f"{binary_os}-{binary_arch}") + runtime = adoptx_release_binary_to_java_runtime( + rls, binary, java_os + ) + add_java_runtime(runtime, major) + writeJavas(javas=javas, uid=JAVA_OPENJ9_COMPONENT) + javas = {} + print("Processing Azul Packages") azul_path = os.path.join(UPSTREAM_DIR, AZUL_DIR, "packages.json") if os.path.exists(azul_path): @@ -410,7 +450,7 @@ def main(): writeJavas(javas=javas, uid=JAVA_AZUL_COMPONENT) javas = {} - # constructs the missing mojang javas based on adoptium or azul + # constructs the missing mojang javas based on adoptium or azul (do not consider openj9 since it is for more niche cases) def get_mojang_extra_java( mojang_component: MojangJavaComponent, java_os: JavaRuntimeOS ) -> JavaRuntimeMeta | None: @@ -434,6 +474,7 @@ def main(): mojang_java_manifest = JavaIndex.parse_file( os.path.join(UPSTREAM_DIR, JAVA_MANIFEST_FILE) ) + # print(mojang_java_manifest) for mojang_os_name in mojang_java_manifest: if mojang_os_name == MojangJavaOsName.Gamecore: continue # empty @@ -471,7 +512,7 @@ def main(): major = int(mojang_runtime.version.name.partition(".")[0]) runtime = mojang_runtime_to_java_runtime(mojang_runtime, comp, java_os) add_java_runtime(runtime, major) - # mojang doesn't provide any versions for the following systems so borrow info from adoptium/azul + # mojang doesn't provide any versions for the following systems so borrow info from adoptium/azul (do not consider openj9 since it is for more niche cases) for java_os in [ JavaRuntimeOS.WindowsArm32, JavaRuntimeOS.LinuxArm32, diff --git a/meta/run/update_java.py b/meta/run/update_java.py index 326fb5f924..bc9ad8db1f 100644 --- a/meta/run/update_java.py +++ b/meta/run/update_java.py @@ -4,18 +4,26 @@ from meta.common import upstream_path, ensure_upstream_dir, default_session from meta.common.java import ( BASE_DIR, ADOPTIUM_DIR, + OPENJ9_DIR, AZUL_DIR, ADOPTIUM_VERSIONS_DIR, + OPENJ9_VERSIONS_DIR, AZUL_VERSIONS_DIR, ) from meta.model.java import ( - ADOPTIUM_API_AVAILABLE_RELEASES, + ADOPTIUM_API_BASE, + OPENJ9_API_BASE, + ADOPTX_API_AVAILABLE_RELEASES, + adoptxAPIFeatureReleasesUrl, adoptiumAPIFeatureReleasesUrl, - AdoptiumImageType, - AdoptiumAPIFeatureReleasesQuery, - AdoptiumAvailableReleases, - AdoptiumRelease, - AdoptiumReleases, + openj9APIFeatureReleasesUrl, + AdoptxJvmImpl, + AdoptxVendor, + AdoptxImageType, + AdoptxAPIFeatureReleasesQuery, + AdoptxAvailableReleases, + AdoptxRelease, + AdoptxReleases, azulApiPackagesUrl, AzulApiPackagesQuery, ZuluPackage, @@ -33,8 +41,10 @@ UPSTREAM_DIR = upstream_path() ensure_upstream_dir(BASE_DIR) ensure_upstream_dir(ADOPTIUM_DIR) +ensure_upstream_dir(OPENJ9_DIR) ensure_upstream_dir(AZUL_DIR) ensure_upstream_dir(ADOPTIUM_VERSIONS_DIR) +ensure_upstream_dir(OPENJ9_VERSIONS_DIR) ensure_upstream_dir(AZUL_VERSIONS_DIR) @@ -43,10 +53,10 @@ sess = default_session() def main(): print("Getting Adoptium Release Manifests ") - r = sess.get(ADOPTIUM_API_AVAILABLE_RELEASES) + r = sess.get(ADOPTX_API_AVAILABLE_RELEASES.format(base_url=ADOPTIUM_API_BASE)) r.raise_for_status() - available = AdoptiumAvailableReleases(**r.json()) + available = AdoptxAvailableReleases(**r.json()) available_releases_file = os.path.join( UPSTREAM_DIR, ADOPTIUM_DIR, "available_releases.json" @@ -58,11 +68,11 @@ def main(): page_size = 10 - releases_for_feature: list[AdoptiumRelease] = [] + releases_for_feature: list[AdoptxRelease] = [] page = 0 while True: - query = AdoptiumAPIFeatureReleasesQuery( - image_type=AdoptiumImageType.Jre, page_size=page_size, page=page + query = AdoptxAPIFeatureReleasesQuery( + image_type=AdoptxImageType.Jre, page_size=page_size, page=page, jvm_impl=AdoptxJvmImpl.Hotspot, vendor=AdoptxVendor.Eclipse ) api_call = adoptiumAPIFeatureReleasesUrl(feature, query=query) print("Fetching JRE Page:", page, api_call) @@ -72,7 +82,7 @@ def main(): else: r_rls.raise_for_status() - releases = list(AdoptiumRelease(**rls) for rls in r_rls.json()) + releases = list(AdoptxRelease(**rls) for rls in r_rls.json()) releases_for_feature.extend(releases) if len(r_rls.json()) < page_size: @@ -80,12 +90,58 @@ def main(): page += 1 print("Total Adoptium releases for feature:", len(releases_for_feature)) - releases = AdoptiumReleases(__root__=releases_for_feature) + releases = AdoptxReleases(__root__=releases_for_feature) feature_file = os.path.join( UPSTREAM_DIR, ADOPTIUM_VERSIONS_DIR, f"java{feature}.json" ) releases.write(feature_file) + print("Getting OpenJ9 Release Manifests ") + r = sess.get(ADOPTX_API_AVAILABLE_RELEASES.format(base_url=OPENJ9_API_BASE)) + r.raise_for_status() + + available = AdoptxAvailableReleases(**r.json()) + + available_releases_file = os.path.join( + UPSTREAM_DIR, OPENJ9_DIR, "available_releases.json" + ) + available.write(available_releases_file) + + for feature in available.available_releases: + print("Getting Manifests for OpenJ9 feature release:", feature) + + page_size = 10 + + releases_for_feature: list[AdoptxRelease] = [] + page = 0 + while True: + query = AdoptxAPIFeatureReleasesQuery( + image_type=AdoptxImageType.Jre, page_size=page_size, page=page, jvm_impl=AdoptxJvmImpl.OpenJ9, vendor=AdoptxVendor.Ibm + ) + api_call = openj9APIFeatureReleasesUrl(feature, query=query) + print("Fetching JRE Page:", page, api_call) + r_rls = sess.get(api_call) + if r_rls.status_code == 404: + break + else: + r_rls.raise_for_status() + + releases = list(AdoptxRelease(**rls) for rls in r_rls.json()) + releases_for_feature.extend(releases) + + if len(r_rls.json()) < page_size: + break + page += 1 + + print("Total OpenJ9 releases for feature:", len(releases_for_feature)) + releases = AdoptxReleases(__root__=releases_for_feature) + if len(releases_for_feature) == 0: + continue + feature_file = os.path.join( + UPSTREAM_DIR, OPENJ9_VERSIONS_DIR, f"java{feature}.json" + ) + releases.write(feature_file) + print("Getting Azul Release Manifests") zulu_packages: list[ZuluPackage] = [] page = 1 |