diff options
| -rw-r--r-- | meta/common/java.py | 10 | ||||
| -rw-r--r-- | meta/common/mojang.py | 2 | ||||
| -rw-r--r-- | meta/model/__init__.py | 4 | ||||
| -rw-r--r-- | meta/model/java.py | 439 | ||||
| -rw-r--r-- | meta/model/mojang.py | 42 | ||||
| -rw-r--r-- | updateJava.py | 157 | ||||
| -rwxr-xr-x | updateMojang.py | 17 |
7 files changed, 669 insertions, 2 deletions
diff --git a/meta/common/java.py b/meta/common/java.py new file mode 100644 index 0000000000..8de3c47c4e --- /dev/null +++ b/meta/common/java.py @@ -0,0 +1,10 @@ +from os.path import join + +BASE_DIR = "java_runtime" + +RELEASE_FILE = join(BASE_DIR, "releases.json") +ADOPTIUM_DIR = join(BASE_DIR, "adoptium") +AZUL_DIR = join(BASE_DIR, "azul") + +ADOPTIUM_VERSIONS_DIR = join(ADOPTIUM_DIR, "versions") +AZUL_VERSIONS_DIR = join(AZUL_DIR, "versions")
\ No newline at end of file diff --git a/meta/common/mojang.py b/meta/common/mojang.py index dc80f44412..562ae594d7 100644 --- a/meta/common/mojang.py +++ b/meta/common/mojang.py @@ -14,3 +14,5 @@ LIBRARY_PATCHES_FILE = join(BASE_DIR, "library-patches.json") MINECRAFT_COMPONENT = "net.minecraft" LWJGL_COMPONENT = "org.lwjgl" LWJGL3_COMPONENT = "org.lwjgl3" + +JAVA_MANIFEST_FILE = join(BASE_DIR, "java_all.json")
\ No newline at end of file diff --git a/meta/model/__init__.py b/meta/model/__init__.py index 0246cdb3d1..0428b7f1fb 100644 --- a/meta/model/__init__.py +++ b/meta/model/__init__.py @@ -129,7 +129,7 @@ class GradleSpecifier: class MetaBase(pydantic.BaseModel): - def dict(self, **kwargs) -> Dict[str, Any]: + def dict(self, **kwargs: Any) -> Dict[str, Any]: for k in ["by_alias"]: if k in kwargs: del kwargs[k] @@ -145,7 +145,7 @@ class MetaBase(pydantic.BaseModel): exclude_none=True, sort_keys=True, by_alias=True, indent=4, **kwargs ) - def write(self, file_path): + def write(self, file_path: str): with open(file_path, "w") as f: f.write(self.json()) diff --git a/meta/model/java.py b/meta/model/java.py new file mode 100644 index 0000000000..d4d2e00cb9 --- /dev/null +++ b/meta/model/java.py @@ -0,0 +1,439 @@ +from . import ( + MetaBase, +) +from pydantic import Field +from datetime import datetime +from enum import Enum +from typing import Optional, List, Dict, Any, Iterator, Iterable, NamedTuple +from collections import namedtuple +from urllib.parse import urljoin, urlencode, urlparse, urlunparse + +# namedtuple to match the internal signature of urlunparse + + +class URLComponents(NamedTuple): + scheme: str + netloc: str + url: str + path: str + query: str + fragment: str + + +class APIQuery(MetaBase): + + def to_query(self): + set_parts: dict[str, Any] = {} + for key, value in self.dict().items(): + if value is not None: + if isinstance(value, Enum): + set_parts[key] = value.value + elif isinstance(value, list): + if len(value) > 0: + set_parts[key] = value + elif isinstance(value, datetime): + set_parts[key] = value.isoformat() + else: + set_parts[key] = value + return urlencode(set_parts, doseq=True) + + +class AdoptiumJvmImpl(Enum): + Hostspot = "hotspot" + + +class AdoptiumVendor(Enum): + Eclipse = "eclipse" + + +class AdoptiumArchitecture(Enum): + X64 = "x64" + X86 = "x86" + X32 = "x32" + Ppc64 = "ppc64" + Ppc64le = "ppc64le" + S390x = "s390x" + Aarch64 = "aarch64" + Arm = "arm" + Sparcv9 = "sparcv9" + Riscv64 = "riscv64" + + +class AdoptiumReleaseType(Enum): + GenralAccess = "ga" + EarlyAccess = "ea" + + +class AdoptiumSortMethod(Enum): + Default = "DEFAULT" + Date = "DATE" + + +class AdoptiumSortOrder(Enum): + Asc = "ASC" + Desc = "DESC" + + +class AdoptiumImageType(Enum): + Jdk = "jdk" + Jre = "jre" + Testimage = "testimage" + Debugimage = "debugimage" + Staticlibs = "staticlibs" + Sources = "sources" + Sbom = "sbom" + + +class AdoptiumHeapSize(Enum): + Normal = "normal" + Large = "large" + + +class AdoptiumProject(Enum): + Jdk = "jdk" + Valhalla = "valhalla" + Metropolis = "metropolis" + Jfr = "jfr" + Shenandoah = "shenandoah" + + +class AdoptiumCLib(Enum): + Musl = "musl" + Glibc = "Glibc" + + +class AdoptiumOs(Enum): + Linux = "linux" + Windows = "windows" + Mac = "mac" + Solaris = "solaris" + Aix = "aix" + AlpineLinux = "alpine-linux" + + +ADOPTIUM_API_BASE = " https://api.adoptium.net" +ADOPTIUM_API_FEATURE_RELEASES = f"{ADOPTIUM_API_BASE}/v3/assets/feature_releases/{{feature_version}}/{{release_type}}" +# ?image_type={{image_type}}&heap_size={{heap_size}}&project={{project}}&vender={{vender}}&page_size={{page_size}}&page={{page}}&sort_method={{sort_method}}&sort_order={{sort_order}} +ADOPTIUM_API_AVAILABLE_RELEASES = f"{ADOPTIUM_API_BASE}/v3/info/available_releases" + + +class AdoptiumAPIFeatureReleasesQuery(APIQuery): + architecture: Optional[AdoptiumArchitecture] = None + before: Optional[datetime] = None + c_lib: Optional[AdoptiumCLib] = None + heap_size: Optional[AdoptiumHeapSize] = AdoptiumHeapSize.Normal + image_type: Optional[AdoptiumImageType] = None + jvm_impl: Optional[AdoptiumJvmImpl] = None + os: Optional[AdoptiumOs] = None + page_size: int = 10 + page: int = 0 + project: Optional[AdoptiumProject] = AdoptiumProject.Jdk + sort_method: Optional[AdoptiumSortMethod] = AdoptiumSortMethod.Default + sort_order: Optional[AdoptiumSortOrder] = AdoptiumSortOrder.Desc + vender: Optional[AdoptiumVendor] = AdoptiumVendor.Eclipse + + +def adoptiumAPIFeatureReleases( + feature: int, + release_type: AdoptiumReleaseType = AdoptiumReleaseType.GenralAccess, + query: AdoptiumAPIFeatureReleasesQuery = AdoptiumAPIFeatureReleasesQuery() +): + url = urlparse(ADOPTIUM_API_FEATURE_RELEASES.format( + feature_version=feature, + release_type=release_type.value, + )) + return urlunparse(url._replace(query=query.to_query())) + + +class AdoptiumAvailableReleases(MetaBase): + available_releases: list[int] + available_lts_releases: list[int] + most_recent_lts: Optional[int] + most_recent_feature_release: Optional[int] + most_recent_feature_version: Optional[int] + tip_version: Optional[int] + + +class AdoptiumFile(MetaBase): + name: str + link: str + size: Optional[int] + + +class AdoptiumPackage(AdoptiumFile): + checksum: Optional[str] + checksum_link: Optional[str] + signature_link: Optional[str] + download_count: Optional[int] + metadata_link: Optional[str] + + +class AdoptiumBinary(MetaBase): + os: str + architecture: AdoptiumArchitecture + image_type: AdoptiumImageType + c_lib: Optional[AdoptiumCLib] + jvm_impl: AdoptiumJvmImpl + package: Optional[AdoptiumPackage] + installer: Optional[AdoptiumPackage] + heap_size: AdoptiumHeapSize + download_count: Optional[int] + updated_at: datetime + scm_ref: Optional[str] + project: AdoptiumProject + + +class AdoptiumVersion(MetaBase): + major: Optional[int] + minor: Optional[int] + security: Optional[int] + patch: Optional[int] + pre: Optional[str] + adopt_build_number: Optional[int] + semver: str + openjdk_version: str + build: Optional[int] + optional: Optional[str] + + +class AdoptiumRelease(MetaBase): + release_id: str = Field(alias="id") + release_link: str + release_name: str + timestamp: datetime + updated_at: datetime + binaries: list[AdoptiumBinary] + download_count: Optional[int] + release_type: str + vendor: AdoptiumVendor + version_data: AdoptiumVersion + source: Optional[AdoptiumFile] + release_notes: Optional[AdoptiumFile] + + +class AdoptiumReleasesWrap(MetaBase): + releases: list[AdoptiumRelease] + + +class AzulProduct(Enum): + Zulu = "zulu" + + +class AzulAvailabilityType(Enum): + SA = "SA" + CA = "CA" + NV = "NV" + _LA = "LA" + + +class AzulJavaPackageType(Enum): + Jdk = "jdk" + Jre = "jre" + + +class AzulReleaseType(Enum): + CPU = "CPU" + PSU = "PSU" + LU = "LU" + + +class AzulOs(Enum): + Linux = "linux" + Macos = "macos" + Qnx = "qnx" + Windows = "windows" + Solaris = "solaris" + + +class AzulLibCType(Enum): + Glibc = "glibc" + Uclibc = "uclibc" + Musl = "musl" + + +class AzulCPUGen(Enum): + V5 = "v5" + V6kV6kz = "v6k_v6kz" + V6t2 = "v6t2" + V7 = "v7" + V8 = "v8" + + +class AzulArch(Enum): + Arm = "arm" + X86 = "x86" + Mips = "mips" + Ppc = "ppc" + Sparcv9 = "sparcv9" + Sparc = "sparc" + + +class AzulHwBitness(Enum): + X32 = 32 + X64 = 64 + + +class AzulAbi(Enum): + HardFloat = "hard_float" + SoftFloat = "soft_float" + Spe = "spe" + Any = "any" + + +class AzulArchiveType(Enum): + Deb = "deb" + Rpm = "rpm" + Dmg = "dmg" + Targz = "tar.gz" + Zip = "zip" + Cab = "cab" + Msi = "msi" + + +class AzulReleaseStatus(Enum): + Eval = "eval" + Ea = "ea" + Ga = "ga" + Both = "both" + + +class AzulSupportTerm(Enum): + Sts = "sts" + Mts = "mts" + Lts = "lts" + + +class AzulCertifications(Enum): + Tck = "tck" + _Aqavit = "aqavit" + none = "none" + + +class AzulSignatureType(Enum): + Openpgp = "openpgp" + + +class AzulOsQueryParam(Enum): + Macos = "macos" + Windows = "windows" + Linux = "linux" + LinuxMusl = "linux-musl" + LinuxGlibc = "linux-glibc" + Qnx = "qnx" + Solaris = "solaris" + + +class AzulArchQueryParam(Enum): + X86 = "x86" + X64 = "x64" + Amd64 = "amd64" + I686 = "i686" + Arm = "arm" + Aarch64 = "aarch64" + Aarch32 = "aarch32" + Aarch32sf = "aarch32sf" + Aarch32hf = "aarch32hf" + Ppc = "ppc" + Ppc64 = "ppc64" + Ppc64hf = "ppc64hf" + Ppc32 = "ppc32" + Ppc32spe = "ppc32spe" + Ppc32hf = "ppc32hf" + Sparc = "sparc" + Sparc32 = "sparc32" + Sparcv9 = "sparcv9" + Sparcv9_64 = "sparcv9-64" + + +AZUL_API_BASE = "https://api.azul.com/metadata/v1" +AZUL_API_PACKAGES = f"{AZUL_API_BASE}/zulu/packages/" +AZUL_API_PACKAGE_DETAIL = f"{AZUL_API_BASE}/zulu/packages/{{package_uuid}}" + + +class AzulApiPackagesQuery(APIQuery): + java_version: Optional[str] = None + os: Optional[AzulOsQueryParam] = None + arch: Optional[AzulArchQueryParam] = None + archive_type: Optional[AzulArchiveType] = None + java_package_type: Optional[AzulJavaPackageType] = None + javafx_bundled: Optional[bool] = None + crac_supported: Optional[bool] = None + support_term: Optional[AzulSupportTerm] = None + release_type: Optional[AzulReleaseType] = None + latest: Optional[bool] = None + distro_version: Optional[str] = None + java_package_features: list[str] = [] + release_status: Optional[AzulReleaseStatus] = None + availability_types: list[AzulAvailabilityType] = None + certifications: list[AzulCertifications] = None + include_fields: list[str] = [] + page: int = 0 + page_size: int = 100 + + +def azulApiPackages(query: AzulApiPackagesQuery = AzulApiPackagesQuery()): + url = urlparse(AZUL_API_PACKAGES) + return urlunparse(url._replace(query=query.to_query())) + + +def azulApiPackageDetail(package_uuid: str): + return AZUL_API_PACKAGE_DETAIL.format(package_uuid=package_uuid) + + +class ZuluSignatureDetail(MetaBase): + type: AzulSignatureType + url: str + details: dict[str, Any] + signature_index: int + signature: str + + +class ZuluPackageDetail(MetaBase): + package_uuid: str + name: Optional[str] + md5_hash: Optional[str] + sha256_hash: Optional[str] + build_date: datetime + last_modified: datetime + download_url: str + product: AzulProduct + availability_type: AzulAvailabilityType + java_version: list[int] + openjdk_build_number: Optional[int] + java_package_type: AzulJavaPackageType + javafx_bundled: bool + release_type: AzulReleaseType + os: AzulOs + lib_c_type: Optional[AzulLibCType] + cpu_gen: Optional[list[AzulCPUGen]] + arch: AzulArch + hw_bitness: AzulHwBitness + abi: AzulAbi + archive_type: AzulArchiveType + release_status: AzulReleaseStatus + support_term: AzulSupportTerm + certifications: Optional[list[AzulCertifications]] + latest: Optional[bool] + size: int + distro_version: list[int] + signatures: list[ZuluSignatureDetail] + + +class ZuluPackageList(MetaBase): + package_uuid: str + name: Optional[str] + java_version: list[int] + openjdk_build_number: Optional[int] + latest: Optional[bool] + download_url: str + product: Optional[AzulProduct] + distro_version: list[int] + availability_type: Optional[AzulAvailabilityType] + + +class ZuluPackagesListWrap(MetaBase): + packages: list[ZuluPackageList] + + +class ZuluPackagesDetailListWrap(MetaBase): + packages: list[ZuluPackageDetail] diff --git a/meta/model/mojang.py b/meta/model/mojang.py index 2e35634008..ce178efe2e 100644 --- a/meta/model/mojang.py +++ b/meta/model/mojang.py @@ -190,6 +190,48 @@ class JavaVersion(MetaBase): major_version: int = Field(8, alias="majorVersion") +class JavaIndexAvailability(MetaBase): + group: int + progress: int + + +class JavaIndexManifest(MetaBase): + sha1: str + size: int + url: str + + +class JavaIndexVersion(MetaBase): + name: str + released: datetime + + +class JavaRuntime(MetaBase): + availability: JavaIndexAvailability + manifest: JavaIndexManifest + version: JavaIndexVersion + + +class JavaIndexEntry(MetaBase): + java_runtime_alpha: list[JavaRuntime] = Field(alias="java-runtime-alpha") + java_runtime_beta: list[JavaRuntime] = Field(alias="java-runtime-beta") + java_runtime_gamma: list[JavaRuntime] = Field(alias="java-runtime-gamma") + jre_legacy: list[JavaRuntime] = Field(alias="jre-legacy") + minecraft_java_exe: list[JavaRuntime] = Field(alias="minecraft-java-exe") + + +class JavaIndex(MetaBase): + gamecore: JavaIndexEntry + linux: JavaIndexEntry + linux_i386: JavaIndexEntry = Field(alias="linux-i386") + mac_os: JavaIndexEntry = Field(alias="mac-os") + mac_os_arm64: JavaIndexEntry = Field(alias="mac-os-arm64") + windows_arm64: JavaIndexEntry = Field(alias="windows-arm64") + windows_x64: JavaIndexEntry = Field(alias="windows-x64") + windows_x86: JavaIndexEntry = Field(alias="windows-x86") + + + class MojangVersion(MetaBase): @validator("minimum_launcher_version") def validate_minimum_launcher_version(cls, v): diff --git a/updateJava.py b/updateJava.py new file mode 100644 index 0000000000..4eb6ffc486 --- /dev/null +++ b/updateJava.py @@ -0,0 +1,157 @@ +import json +import os +import zipfile + +from meta.common import upstream_path, ensure_upstream_dir, static_path, default_session +from meta.common.java import ( + BASE_DIR, + ADOPTIUM_DIR, + AZUL_DIR, + ADOPTIUM_VERSIONS_DIR, + AZUL_VERSIONS_DIR, +) +from meta.model.java import ( + ADOPTIUM_API_AVAILABLE_RELEASES, + adoptiumAPIFeatureReleases, + AdoptiumImageType, + AdoptiumAPIFeatureReleasesQuery, + AdoptiumAvailableReleases, + AdoptiumRelease, + AdoptiumReleasesWrap, + azulApiPackages, + AzulApiPackagesQuery, + ZuluPackageList, + ZuluPackagesListWrap, + AzulArchiveType, + AzulReleaseStatus, + AzulAvailabilityType, + AzulJavaPackageType, + azulApiPackageDetail, + ZuluPackageDetail, + ZuluPackagesDetailListWrap, +) + +UPSTREAM_DIR = upstream_path() +STATIC_DIR = static_path() + +ensure_upstream_dir(BASE_DIR) +ensure_upstream_dir(ADOPTIUM_DIR) +ensure_upstream_dir(AZUL_DIR) +ensure_upstream_dir(ADOPTIUM_VERSIONS_DIR) +ensure_upstream_dir(AZUL_VERSIONS_DIR) + + +sess = default_session() + + +def main(): + print("Getting Adoptium Release Manifests ") + r = sess.get(ADOPTIUM_API_AVAILABLE_RELEASES) + r.raise_for_status() + + available = AdoptiumAvailableReleases(**r.json()) + + available_releases_file = os.path.join( + UPSTREAM_DIR, ADOPTIUM_DIR, "available_releases.json") + available.write(available_releases_file) + + for feature in available.available_releases: + print("Getting Manifests for Adoptium feature release:", feature) + page = 0 + page_size = 10 + + releases_for_feature: list[AdoptiumRelease] = [] + + while True: + query = AdoptiumAPIFeatureReleasesQuery( + image_type=AdoptiumImageType.Jre, page_size=page_size, page=page) + api_call = adoptiumAPIFeatureReleases(feature, query=query) + print("Fetching Page:", page, api_call) + r_rls = sess.get(api_call) + if r_rls.status_code == 404: + break + else: + r_rls.raise_for_status() + + releases = list(AdoptiumRelease(**rls) for rls in r_rls.json()) + releases_for_feature.extend(releases) + + if len(r_rls.json()) < page_size: + break + page += 1 + + print("Total Adoptium releases for feature:", len(releases_for_feature)) + releases = AdoptiumReleasesWrap(releases=releases_for_feature) + feature_file = os.path.join( + UPSTREAM_DIR, ADOPTIUM_VERSIONS_DIR, "{}.json".format(feature)) + releases.write(feature_file) + + print("Getting Azul Release Manifests") + zulu_packages: list[ZuluPackageList] = [] + page = 1 + page_size = 100 + while True: + + query = AzulApiPackagesQuery( + archive_type=AzulArchiveType.Zip, + release_status=AzulReleaseStatus.Ga, + availability_types=[AzulAvailabilityType.CA], + java_package_type=AzulJavaPackageType.Jre, + javafx_bundled=False, + page=page, + page_size=page_size) + api_call = azulApiPackages(query=query) + + print("Processing Page:", page, api_call) + + r = sess.get(api_call) + if r.status_code == 404: + break + else: + r.raise_for_status() + + packages = list(ZuluPackageList(**pkg) for pkg in r.json()) + zulu_packages.extend(packages) + if len(packages) < page_size: + break + page += 1 + + print("Total Azul Packages:", len(zulu_packages)) + packages = ZuluPackagesListWrap(packages=zulu_packages) + azul_manifest_file = os.path.join(UPSTREAM_DIR, AZUL_DIR, "packages.json") + packages.write(azul_manifest_file) + + azul_major_versions: dict[int, ZuluPackagesListWrap] = {} + + for pkg in packages.packages: + + major_version = pkg.java_version[0] + if major_version not in azul_major_versions: + azul_major_versions[major_version] = ZuluPackagesListWrap( + packages=[]) + + azul_major_versions[major_version].packages.append(pkg) + + pkg_file = os.path.join( + UPSTREAM_DIR, AZUL_VERSIONS_DIR, "{}.json".format(pkg.package_uuid)) + if os.path.exists(pkg_file) and os.path.isfile(pkg_file): + pass + else: + + api_call = azulApiPackageDetail(pkg.package_uuid) + print("Fetching Azul package manifest:", pkg.package_uuid) + r_pkg = sess.get(api_call) + r_pkg.raise_for_status() + + pkg_detail = ZuluPackageDetail(**r_pkg.json()) + pkg_detail.write(pkg_file) + + for major in azul_major_versions: + major_file = os.path.join( + UPSTREAM_DIR, AZUL_VERSIONS_DIR, "{}.json".format(major)) + azul_major_versions[major].write(major_file) + + +if __name__ == "__main__": + main() + # diff --git a/updateMojang.py b/updateMojang.py index 8d3a2296e0..91ca558cc9 100755 --- a/updateMojang.py +++ b/updateMojang.py @@ -11,6 +11,7 @@ from meta.common.mojang import ( ASSETS_DIR, STATIC_EXPERIMENTS_FILE, STATIC_OLD_SNAPSHOTS_FILE, + JAVA_MANIFEST_FILE, ) from meta.model.mojang import ( MojangIndexWrap, @@ -19,6 +20,7 @@ from meta.model.mojang import ( ExperimentIndexWrap, OldSnapshotIndexWrap, OldSnapshotIndex, + JavaIndex, ) UPSTREAM_DIR = upstream_path() @@ -81,6 +83,19 @@ def fetch_version(path, url): return version_json +MOJANG_JAVA_URL = "https://piston-meta.mojang.com/v1/products/java-runtime/2ec0cc96c44e5a76b9c8b7c39df7210883d12871/all.json" + +def update_javas(): + r = sess.get(MOJANG_JAVA_URL) + r.raise_for_status() + + remote_javas = JavaIndex(**r.json()) + + java_manifest_path = os.path.join(UPSTREAM_DIR, JAVA_MANIFEST_FILE) + + remote_javas.write(java_manifest_path) + + def main(): # get the remote version list r = sess.get("https://piston-meta.mojang.com/mc/game/version_manifest_v2.json") @@ -160,6 +175,8 @@ def main(): remote_versions.index.write(version_manifest_path) + print("Getting Mojang Java runtime manfest") + update_javas() if __name__ == "__main__": main() |
