diff options
| -rwxr-xr-x | generateForge.py | 478 | ||||
| -rw-r--r-- | meta/common/forge.py | 17 | ||||
| -rw-r--r-- | meta/forgeutil.py | 323 | ||||
| -rw-r--r-- | meta/model/forge.py | 306 | ||||
| -rw-r--r-- | static/forge/forge-legacyinfo.json (renamed from static/forge-legacyinfo.json) | 0 | ||||
| -rwxr-xr-x | updateForge.py | 517 |
6 files changed, 793 insertions, 848 deletions
diff --git a/generateForge.py b/generateForge.py index 222233023c..b161b29a3e 100755 --- a/generateForge.py +++ b/generateForge.py @@ -1,17 +1,24 @@ +import os import re import sys +from datetime import timezone from distutils.version import LooseVersion -from meta.forgeutil import * -from meta.metautil import * -from meta.common import ensure_component_dir, polymc_path, upstream_path +from meta.common import ensure_component_dir, polymc_path, upstream_path, static_path +from meta.common.forge import FORGE_COMPONENT, INSTALLER_MANIFEST_DIR, VERSION_MANIFEST_DIR, DERIVED_INDEX_FILE, \ + STATIC_LEGACYINFO_FILE, INSTALLER_INFO_DIR, BAD_VERSIONS, FORGEWRAPPER_MAVEN +from meta.common.mojang import MINECRAFT_COMPONENT +from meta.model import MetaVersion, Dependency, Library, GradleSpecifier, MojangLibraryDownloads, MojangArtifact, \ + MetaPackage +from meta.model.forge import ForgeVersion, ForgeInstallerProfile, ForgeLegacyInfo, fml_libs_for_version, \ + ForgeInstallerProfileV2, InstallerInfo, DerivedForgeIndex, ForgeLegacyInfoList +from meta.model.mojang import MojangVersion PMC_DIR = polymc_path() UPSTREAM_DIR = upstream_path() +STATIC_DIR = static_path() -ensure_component_dir("net.minecraftforge") - -FORGEWRAPPER_MAVEN = "https://polymc.github.io/files/maven/%s" +ensure_component_dir(FORGE_COMPONENT) def eprint(*args, **kwargs): @@ -26,12 +33,11 @@ def loadMcVersionFilter(version): if version in mcVersionCache: return mcVersionCache[version] libSet = set() - with open(PMC_DIR + "/net.minecraft/%s.json" % version, 'r', encoding='utf-8') as mcFile: - mcVersion = PolyMCVersionFile(json.load(mcFile)) - for lib in mcVersion.libraries: - libSet.add(lib.name) - mcVersionCache[version] = libSet - return libSet + mcVersion = MetaVersion.parse_file(os.path.join(PMC_DIR, MINECRAFT_COMPONENT, f"{version}.json")) + for lib in mcVersion.libraries: + libSet.add(lib.name) + mcVersionCache[version] = libSet + return libSet ''' @@ -58,30 +64,30 @@ def shouldIgnoreArtifact(libSet, match): return False -def versionFromProfile(profile, version): - result = PolyMCVersionFile({"name": "Forge", "version": version.rawVersion, "uid": "net.minecraftforge"}) +def versionFromProfile(profile: ForgeInstallerProfile, version): + result = MetaVersion(name="Forge", version=version.rawVersion, uid=FORGE_COMPONENT) mcversion = profile.install.minecraft - result.requires = [DependencyEntry(uid='net.minecraft', equals=mcversion)] - result.mainClass = profile.versionInfo.mainClass - args = profile.versionInfo.minecraftArguments + result.requires = [Dependency(uid=MINECRAFT_COMPONENT, equals=mcversion)] + result.main_class = profile.versionInfo.main_class + args = profile.versionInfo.minecraft_arguments tweakers = [] expression = re.compile("--tweakClass ([a-zA-Z0-9\\.]+)") match = expression.search(args) - while match != None: - tweakers.append(match.group(1)); + while match is not None: + tweakers.append(match.group(1)) args = args[:match.start()] + args[match.end():] - match = expression.search(args); + match = expression.search(args) if len(tweakers) > 0: args = args.strip() - result.addTweakers = tweakers; + result.additional_tweakers = tweakers # result.minecraftArguments = args - result.releaseTime = profile.versionInfo.time + result.release_time = profile.versionInfo.time libs = [] mcFilter = loadMcVersionFilter(mcversion) for forgeLib in profile.versionInfo.libraries: - if forgeLib.name.isLwjgl(): + if forgeLib.name.is_lwjgl(): continue - if forgeLib.name.isLog4j(): + if forgeLib.name.is_log4j(): continue if shouldIgnoreArtifact(mcFilter, forgeLib.name): continue @@ -93,7 +99,7 @@ def versionFromProfile(profile, version): fixedName.version = "%s-%s" % (mcversion, fixedName.version) elif fixedName.artifact == "forge": fixedName.classifier = "universal" - ourLib = PolyMCLibrary(name=fixedName) + ourLib = Library(name=fixedName) if forgeLib.url == "http://files.minecraftforge.net/maven/": ourLib.url = "https://maven.minecraftforge.net/" else: @@ -106,32 +112,32 @@ def versionFromProfile(profile, version): return result -def versionFromModernizedInstaller(installerVersion: MojangVersionFile, version: ForgeVersion): +def versionFromModernizedInstaller(installerVersion: MojangVersion, version: ForgeVersion): eprint("Generating Modernized Forge %s." % version.longVersion) - result = PolyMCVersionFile({"name": "Forge", "version": version.rawVersion, "uid": "net.minecraftforge"}) + result = MetaVersion(name="Forge", version=version.rawVersion, uid=FORGE_COMPONENT) mcversion = version.mcversion - result.requires = [DependencyEntry(uid='net.minecraft', equals=mcversion)] - result.mainClass = installerVersion.mainClass - args = installerVersion.minecraftArguments + result.requires = [Dependency(uid=MINECRAFT_COMPONENT, equals=mcversion)] + result.main_class = installerVersion.main_class + args = installerVersion.minecraft_arguments tweakers = [] expression = re.compile("--tweakClass ([a-zA-Z0-9\\.]+)") match = expression.search(args) - while match != None: - tweakers.append(match.group(1)); + while match is not None: + tweakers.append(match.group(1)) args = args[:match.start()] + args[match.end():] - match = expression.search(args); + match = expression.search(args) if len(tweakers) > 0: args = args.strip() - result.addTweakers = tweakers; + result.additional_tweakers = tweakers # result.minecraftArguments = args - result.releaseTime = installerVersion.releaseTime + result.release_time = installerVersion.release_time libs = [] mcFilter = loadMcVersionFilter(mcversion) for upstreamLib in installerVersion.libraries: - pmcLib = PolyMCLibrary(upstreamLib.to_json()) - if pmcLib.name.isLwjgl(): + pmcLib = Library.parse_obj(upstreamLib.dict()) + if pmcLib.name.is_lwjgl(): continue - if pmcLib.name.isLog4j(): + if pmcLib.name.is_log4j(): continue if shouldIgnoreArtifact(mcFilter, pmcLib.name): continue @@ -139,8 +145,8 @@ def versionFromModernizedInstaller(installerVersion: MojangVersionFile, version: if pmcLib.name.artifact == "forge": fixedName = pmcLib.name fixedName.classifier = "universal" - pmcLib.downloads.artifact.path = fixedName.getPath() - pmcLib.downloads.artifact.url = "https://files.minecraftforge.net/maven/%s" % fixedName.getPath() + pmcLib.downloads.artifact.path = fixedName.path() + pmcLib.downloads.artifact.url = "https://files.minecraftforge.net/maven/%s" % fixedName.path() pmcLib.name = fixedName libs.append(pmcLib) continue @@ -149,8 +155,8 @@ def versionFromModernizedInstaller(installerVersion: MojangVersionFile, version: fixedName.artifact = "forge" fixedName.classifier = "universal" fixedName.version = "%s-%s" % (mcversion, fixedName.version) - pmcLib.downloads.artifact.path = fixedName.getPath() - pmcLib.downloads.artifact.url = "https://files.minecraftforge.net/maven/%s" % fixedName.getPath() + pmcLib.downloads.artifact.path = fixedName.path() + pmcLib.downloads.artifact.url = "https://files.minecraftforge.net/maven/%s" % fixedName.path() pmcLib.name = fixedName libs.append(pmcLib) continue @@ -161,288 +167,228 @@ def versionFromModernizedInstaller(installerVersion: MojangVersionFile, version: return result -def versionFromLegacy(version, legacyinfo: ForgeLegacyInfo): - result = PolyMCVersionFile({"name": "Forge", "version": version.rawVersion, "uid": "net.minecraftforge"}) +def versionFromLegacy(version: ForgeVersion, legacyinfo: ForgeLegacyInfo): + result = MetaVersion(name="Forge", version=version.rawVersion, uid=FORGE_COMPONENT) mcversion = version.mcversion_sane - result.requires = [DependencyEntry(uid='net.minecraft', equals=mcversion)] - result.releaseTime = legacyinfo.releaseTime + result.requires = [Dependency(uid=MINECRAFT_COMPONENT, equals=mcversion)] + result.release_time = legacyinfo.releaseTime result.order = 5 - if mcversion in fmlLibsMapping: - result.addTraits = ["legacyFML"] + if fml_libs_for_version(mcversion): # WHY, WHY DID I WASTE MY TIME REWRITING FMLLIBSMAPPING + result.additional_traits = ["legacyFML"] url = version.url() - classifier = None if "universal" in url: classifier = "universal" else: classifier = "client" - coord = GradleSpecifier("net.minecraftforge:forge:%s:%s" % (version.longVersion, classifier)) - mainmod = PolyMCLibrary(name=coord) + coord = GradleSpecifier("net.minecraftforge", "forge", version.longVersion, classifier) + mainmod = Library(name=coord) mainmod.downloads = MojangLibraryDownloads() - mainmod.downloads.artifact = MojangArtifact() + mainmod.downloads.artifact = MojangArtifact(url=version.url(), sha1=legacyinfo.sha1, size=legacyinfo.size) mainmod.downloads.artifact.path = None - mainmod.downloads.artifact.url = version.url() - mainmod.downloads.artifact.sha1 = legacyinfo.sha1 - mainmod.downloads.artifact.size = legacyinfo.size - result.jarMods = [mainmod] + result.jar_mods = [mainmod] return result -def versionFromBuildSystemInstaller(installerVersion: MojangVersionFile, installerProfile: ForgeInstallerProfileV2, +def versionFromBuildSystemInstaller(installerVersion: MojangVersion, installerProfile: ForgeInstallerProfileV2, version: ForgeVersion): eprint("Generating Forge %s." % version.longVersion) - result = PolyMCVersionFile({"name": "Forge", "version": version.rawVersion, "uid": "net.minecraftforge"}) - result.requires = [DependencyEntry(uid='net.minecraft', equals=version.mcversion_sane)] - result.mainClass = "io.github.zekerzhayard.forgewrapper.installer.Main" + result = MetaVersion(name="Forge", version=version.rawVersion, uid=FORGE_COMPONENT) + result.requires = [Dependency(uid=MINECRAFT_COMPONENT, equals=version.mcversion_sane)] + result.main_class = "io.github.zekerzhayard.forgewrapper.installer.Main" # FIXME: Add the size and hash here mavenLibs = [] # load the locally cached installer file info and use it to add the installer entry in the json - with open(UPSTREAM_DIR + "/forge/installer_info/%s.json" % version.longVersion, 'r', encoding='utf-8') as f: - installerInfo = InstallerInfo(json.load(f)) - InstallerLib = PolyMCLibrary( - name=GradleSpecifier("net.minecraftforge:forge:%s:installer" % (version.longVersion))) - InstallerLib.downloads = MojangLibraryDownloads() - InstallerLib.downloads.artifact = MojangArtifact() - InstallerLib.downloads.artifact.url = "https://files.minecraftforge.net/maven/%s" % ( - InstallerLib.name.getPath()) - InstallerLib.downloads.artifact.sha1 = installerInfo.sha1hash - InstallerLib.downloads.artifact.size = installerInfo.size - mavenLibs.append(InstallerLib) + installerInfo = InstallerInfo.parse_file( + os.path.join(UPSTREAM_DIR, INSTALLER_INFO_DIR, f"{version.longVersion}.json")) + InstallerLib = Library( + name=GradleSpecifier("net.minecraftforge", "forge", version.longVersion, "installer")) + InstallerLib.downloads = MojangLibraryDownloads() + InstallerLib.downloads.artifact = MojangArtifact( + url="https://files.minecraftforge.net/maven/%s" % (InstallerLib.name.path()), + sha1=installerInfo.sha1hash, + size=installerInfo.size) + mavenLibs.append(InstallerLib) for upstreamLib in installerProfile.libraries: - pmcLib = PolyMCLibrary(upstreamLib.to_json()) + pmcLib = Library.parse_obj(upstreamLib.dict()) if pmcLib.name.group == "net.minecraftforge": if pmcLib.name.artifact == "forge": if pmcLib.name.classifier == "universal": - pmcLib.downloads.artifact.url = "https://files.minecraftforge.net/maven/%s" % pmcLib.name.getPath() + pmcLib.downloads.artifact.url = "https://files.minecraftforge.net/maven/%s" % pmcLib.name.path() mavenLibs.append(pmcLib) continue - if pmcLib.name.isLog4j(): + if pmcLib.name.is_log4j(): continue mavenLibs.append(pmcLib) - result.mavenFiles = mavenLibs + result.maven_files = mavenLibs libraries = [] - # wrapperLib = PolyMCLibrary(name=GradleSpecifier("io.github.zekerzhayard:ForgeWrapper:1.4.1")) - # wrapperLib.downloads = MojangLibraryDownloads() - # wrapperLib.downloads.artifact = MojangArtifact() - # wrapperLib.downloads.artifact.url = "https://meta.polymc.org/maven/%s" % (wrapperLib.name.getPath()) - # wrapperLib.downloads.artifact.sha1 = "82f01de97e29ba34be9fc628084b6d10ce2235c5" - # wrapperLib.downloads.artifact.size = 14351 - # libraries.append(wrapperLib) - - # wrapperLib = PolyMCLibrary(name=GradleSpecifier("io.github.zekerzhayard:ForgeWrapper:1.4.2")) - # wrapperLib.downloads = MojangLibraryDownloads() - # wrapperLib.downloads.artifact = MojangArtifact() - # wrapperLib.downloads.artifact.url = "https://meta.polymc.org/maven/%s" % (wrapperLib.name.getPath()) - # wrapperLib.downloads.artifact.sha1 = "79ff9c1530e8743450c5c3ebc6e07b535437aa6e" - # wrapperLib.downloads.artifact.size = 22346 - # libraries.append(wrapperLib) - - # wrapperLib = PolyMCLibrary(name=GradleSpecifier("io.github.zekerzhayard:ForgeWrapper:1.5.1")) - # wrapperLib.downloads = MojangLibraryDownloads() - # wrapperLib.downloads.artifact = MojangArtifact() - # wrapperLib.downloads.artifact.url = "https://meta.polymc.org/maven/%s" % (wrapperLib.name.getPath()) - # wrapperLib.downloads.artifact.sha1 = "90104e9aaa8fbedf6c3d1f6d0b90cabce080b5a9" - # wrapperLib.downloads.artifact.size = 29892 - # libraries.append(wrapperLib) - - # wrapperLib = PolyMCLibrary(name=GradleSpecifier("io.github.zekerzhayard:ForgeWrapper:1.5.3")) - # wrapperLib.downloads = MojangLibraryDownloads() - # wrapperLib.downloads.artifact = MojangArtifact() - # wrapperLib.downloads.artifact.url = "https://meta.polymc.org/maven/%s" % (wrapperLib.name.getPath()) - # wrapperLib.downloads.artifact.sha1 = "2b0e06937349a209dbb90dca6381258daa456ad7" - # wrapperLib.downloads.artifact.size = 30486 - # libraries.append(wrapperLib) - - # wrapperLib = PolyMCLibrary(name=GradleSpecifier("io.github.zekerzhayard:ForgeWrapper:1.5.4")) - # wrapperLib.downloads = MojangLibraryDownloads() - # wrapperLib.downloads.artifact = MojangArtifact() - # wrapperLib.downloads.artifact.url = "https://meta.polymc.org/maven/%s" % (wrapperLib.name.getPath()) - # wrapperLib.downloads.artifact.sha1 = "e97805af76d4c1cebb753132eadbabd92e67a17b" - # wrapperLib.downloads.artifact.size = 34299 - # libraries.append(wrapperLib) - - # wrapperLib = PolyMCLibrary(name=GradleSpecifier("io.github.zekerzhayard:ForgeWrapper:pmc1")) - # wrapperLib.downloads = MojangLibraryDownloads() - # wrapperLib.downloads.artifact = MojangArtifact() - # wrapperLib.downloads.artifact.url = "https://meta.polymc.org/maven/%s" % (wrapperLib.name.getPath()) - # wrapperLib.downloads.artifact.sha1 = "e8e0fe708742ecf15ab4af55ae8227fa4349362d" - # wrapperLib.downloads.artifact.size = 34628 - # libraries.append(wrapperLib) - - # wrapperLib = PolyMCLibrary(name=GradleSpecifier("io.github.zekerzhayard:ForgeWrapper:1.5.5")) - # wrapperLib.downloads = MojangLibraryDownloads() - # wrapperLib.downloads.artifact = MojangArtifact() - # wrapperLib.downloads.artifact.url = "https://meta.polymc.org/maven/%s" % (wrapperLib.name.getPath()) - # wrapperLib.downloads.artifact.sha1 = "566dfd60aacffaa02884614835f1151d36f1f985" - # wrapperLib.downloads.artifact.size = 34331 - # libraries.append(wrapperLib) - - wrapperLib = PolyMCLibrary(name=GradleSpecifier("io.github.zekerzhayard:ForgeWrapper:mmc2")) + + wrapperLib = Library(name=GradleSpecifier("io.github.zekerzhayard", "ForgeWrapper", "mmc2")) wrapperLib.downloads = MojangLibraryDownloads() - wrapperLib.downloads.artifact = MojangArtifact() - wrapperLib.downloads.artifact.url = FORGEWRAPPER_MAVEN % (wrapperLib.name.getPath()) - wrapperLib.downloads.artifact.sha1 = "4ee5f25cc9c7efbf54aff4c695da1054c1a1d7a3" - wrapperLib.downloads.artifact.size = 34444 + wrapperLib.downloads.artifact = MojangArtifact(url=FORGEWRAPPER_MAVEN % (wrapperLib.name.path()), + sha1="4ee5f25cc9c7efbf54aff4c695da1054c1a1d7a3", + size=34444) libraries.append(wrapperLib) for upstreamLib in installerVersion.libraries: - pmcLib = PolyMCLibrary(upstreamLib.to_json()) + pmcLib = Library.parse_obj(upstreamLib.dict()) if pmcLib.name.group == "net.minecraftforge": if pmcLib.name.artifact == "forge": fixedName = pmcLib.name fixedName.classifier = "launcher" - pmcLib.downloads.artifact.path = fixedName.getPath() - pmcLib.downloads.artifact.url = "https://files.minecraftforge.net/maven/%s" % fixedName.getPath() + pmcLib.downloads.artifact.path = fixedName.path() + pmcLib.downloads.artifact.url = "https://files.minecraftforge.net/maven/%s" % fixedName.path() pmcLib.name = fixedName libraries.append(pmcLib) continue - if pmcLib.name.isLog4j(): + if pmcLib.name.is_log4j(): continue libraries.append(pmcLib) result.libraries = libraries - result.releaseTime = installerVersion.releaseTime + result.release_time = installerVersion.release_time result.order = 5 mcArgs = "--username ${auth_player_name} --version ${version_name} --gameDir ${game_directory} --assetsDir ${assets_root} --assetIndex ${assets_index_name} --uuid ${auth_uuid} --accessToken ${auth_access_token} --userType ${user_type} --versionType ${version_type}" for arg in installerVersion.arguments.game: mcArgs += " %s" % arg - result.minecraftArguments = mcArgs + result.minecraft_arguments = mcArgs return result -# load the locally cached version list -with open(UPSTREAM_DIR + "/forge/derived_index.json", 'r', encoding='utf-8') as f: - main_json = json.load(f) - remoteVersionlist = DerivedForgeIndex(main_json) - -recommendedVersions = [] - -tsPath = "static/forge-legacyinfo.json" - -legacyinfolist = None -with open(tsPath, 'r', encoding='utf-8') as tsFile: - legacyinfolist = ForgeLegacyInfoList(json.load(tsFile)) - -legacyVersions = [ - "1.1", - "1.2.3", - "1.2.4", - "1.2.5", - "1.3.2", - "1.4.1", - "1.4.2", - "1.4.3", - "1.4.4", - "1.4.5", - "1.4.6", - "1.4.7", - "1.5", - "1.5.1", - "1.5.2", - "1.6.1", - "1.6.2", - "1.6.3", - "1.6.4", - "1.7.10", - "1.7.10-pre4", - "1.7.2", - "1.8", - "1.8.8", - "1.8.9", - "1.9", - "1.9.4", - "1.10", - "1.10.2", - "1.11", - "1.11.2", - "1.12", - "1.12.1", - "1.12.2", -] - -for id, entry in remoteVersionlist.versions.items(): - if entry.mcversion == None: - eprint("Skipping %s with invalid MC version" % id) - continue - - version = ForgeVersion(entry) - if version.url() == None: - eprint("Skipping %s with no valid files" % id) - continue - eprint("Processing Forge %s" % version.rawVersion) - versionElements = version.rawVersion.split('.') - if len(versionElements) < 1: - eprint("Skipping version %s with not enough version elements" % (id)) - continue - - majorVersionStr = versionElements[0] - if not majorVersionStr.isnumeric(): - eprint("Skipping version %s with non-numeric major version %s" % (id, majorVersionStr)) - continue - - majorVersion = int(majorVersionStr) - # if majorVersion >= 37: - # eprint ("Skipping unsupported major version %d (%s)" % (majorVersion, id)) - # continue - - if entry.recommended: - recommendedVersions.append(version.rawVersion) - - # If we do not have the corresponding Minecraft version, we ignore it - if not os.path.isfile(PMC_DIR + "/net.minecraft/%s.json" % version.mcversion_sane): - eprint("Skipping %s with no corresponding Minecraft version %s" % (id, version.mcversion_sane)) - continue - - outVersion = None - - # Path for new-style build system based installers - installerVersionFilepath = UPSTREAM_DIR + "/forge/version_manifests/%s.json" % version.longVersion - profileFilepath = UPSTREAM_DIR + "/forge/installer_manifests/%s.json" % version.longVersion - - eprint(installerVersionFilepath) - if os.path.isfile(installerVersionFilepath): - with open(installerVersionFilepath, 'r', encoding='utf-8') as installerVersionFile: - installerVersion = MojangVersionFile(json.load(installerVersionFile)) - if entry.mcversion in legacyVersions: - outVersion = versionFromModernizedInstaller(installerVersion, version) +def main(): + # load the locally cached version list + remoteVersionlist = DerivedForgeIndex.parse_file(os.path.join(UPSTREAM_DIR, DERIVED_INDEX_FILE)) + + recommendedVersions = [] + + legacyinfolist = ForgeLegacyInfoList.parse_file(os.path.join(STATIC_DIR, STATIC_LEGACYINFO_FILE)) + + legacyVersions = [ + "1.1", + "1.2.3", + "1.2.4", + "1.2.5", + "1.3.2", + "1.4.1", + "1.4.2", + "1.4.3", + "1.4.4", + "1.4.5", + "1.4.6", + "1.4.7", + "1.5", + "1.5.1", + "1.5.2", + "1.6.1", + "1.6.2", + "1.6.3", + "1.6.4", + "1.7.10", + "1.7.10-pre4", + "1.7.2", + "1.8", + "1.8.8", + "1.8.9", + "1.9", + "1.9.4", + "1.10", + "1.10.2", + "1.11", + "1.11.2", + "1.12", + "1.12.1", + "1.12.2", + ] + + for id, entry in remoteVersionlist.versions.items(): + if entry.mcversion is None: + eprint("Skipping %s with invalid MC version" % id) + continue + + version = ForgeVersion(entry) + + if version.longVersion in BAD_VERSIONS: + # Version 1.12.2-14.23.5.2851 is ultra cringe, I can't imagine why you would even spend one second on + # actually adding support for this version. + # It is cringe, because it's installer info is broken af + eprint(f"Skipping bad version {version.longVersion}") + continue + + if version.url() is None: + eprint("Skipping %s with no valid files" % id) + continue + eprint("Processing Forge %s" % version.rawVersion) + versionElements = version.rawVersion.split('.') + if len(versionElements) < 1: + eprint("Skipping version %s with not enough version elements" % (id)) + continue + + majorVersionStr = versionElements[0] + if not majorVersionStr.isnumeric(): + eprint("Skipping version %s with non-numeric major version %s" % (id, majorVersionStr)) + continue + + majorVersion = int(majorVersionStr) + # if majorVersion >= 37: + # eprint ("Skipping unsupported major version %d (%s)" % (majorVersion, id)) + # continue + + if entry.recommended: + recommendedVersions.append(version.rawVersion) + + # If we do not have the corresponding Minecraft version, we ignore it + if not os.path.isfile(os.path.join(PMC_DIR, MINECRAFT_COMPONENT, f"{version.mcversion_sane}.json")): + eprint("Skipping %s with no corresponding Minecraft version %s" % (id, version.mcversion_sane)) + continue + + outVersion = None + + # Path for new-style build system based installers + installerVersionFilepath = os.path.join(UPSTREAM_DIR, VERSION_MANIFEST_DIR, f"{version.longVersion}.json") + profileFilepath = os.path.join(UPSTREAM_DIR, INSTALLER_MANIFEST_DIR, f"{version.longVersion}.json") + + eprint(installerVersionFilepath) + if os.path.isfile(installerVersionFilepath): + installerVersion = MojangVersion.parse_file(installerVersionFilepath) + if entry.mcversion in legacyVersions: + outVersion = versionFromModernizedInstaller(installerVersion, version) + else: + installerProfile = ForgeInstallerProfileV2.parse_file(profileFilepath) + outVersion = versionFromBuildSystemInstaller(installerVersion, installerProfile, version) else: - with open(profileFilepath, 'r', encoding='utf-8') as profileFile: - installerProfile = ForgeInstallerProfileV2(json.load(profileFile)) - outVersion = versionFromBuildSystemInstaller(installerVersion, installerProfile, version) - else: - if version.usesInstaller(): + if version.uses_installer(): - # If we do not have the Forge json, we ignore this version - if not os.path.isfile(profileFilepath): - eprint("Skipping %s with missing profile json" % id) - continue - with open(profileFilepath, 'r', encoding='utf-8') as profileFile: - profile = ForgeInstallerProfile(json.load(profileFile)) + # If we do not have the Forge json, we ignore this version + if not os.path.isfile(profileFilepath): + eprint("Skipping %s with missing profile json" % id) + continue + profile = ForgeInstallerProfile.parse_file(profileFilepath) outVersion = versionFromProfile(profile, version) - else: - # Generate json for legacy here - if version.mcversion_sane == "1.6.1": - continue - build = version.build - if not str(build).encode('utf-8').decode('utf8') in legacyinfolist.number: - eprint("Legacy build %d is missing in legacy info. Ignoring." % build) - continue + else: + # Generate json for legacy here + if version.mcversion_sane == "1.6.1": + continue + build = version.build + if not str(build).encode('utf-8').decode('utf8') in legacyinfolist.number: + eprint("Legacy build %d is missing in legacy info. Ignoring." % build) + continue + + outVersion = versionFromLegacy(version, legacyinfolist.number[str(build)]) - outVersion = versionFromLegacy(version, legacyinfolist.number[build]) + outFilepath = os.path.join(PMC_DIR, FORGE_COMPONENT, f"{outVersion.version}.json") + outVersion.write(outFilepath) - outFilepath = PMC_DIR + "/net.minecraftforge/%s.json" % outVersion.version - with open(outFilepath, 'w') as outfile: - json.dump(outVersion.to_json(), outfile, sort_keys=True, indent=4) + recommendedVersions.sort() -recommendedVersions.sort() + print('Recommended versions:', recommendedVersions) -print('Recommended versions:', recommendedVersions) + package = MetaPackage(uid=FORGE_COMPONENT, name="Forge", project_url="https://www.minecraftforge.net/forum/") + package.recommended = recommendedVersions + package.write(os.path.join(PMC_DIR, FORGE_COMPONENT, "package.json")) -sharedData = PolyMCSharedPackageData(uid='net.minecraftforge', name="Forge") -sharedData.projectUrl = 'https://www.minecraftforge.net/forum/' -sharedData.recommended = recommendedVersions -sharedData.write() +if __name__ == '__main__': + main() diff --git a/meta/common/forge.py b/meta/common/forge.py new file mode 100644 index 0000000000..d39249cc0c --- /dev/null +++ b/meta/common/forge.py @@ -0,0 +1,17 @@ +from os.path import join + +BASE_DIR = "forge" + +JARS_DIR = join(BASE_DIR, "jars") +INSTALLER_INFO_DIR = join(BASE_DIR, "installer_info") +INSTALLER_MANIFEST_DIR = join(BASE_DIR, "installer_manifests") +VERSION_MANIFEST_DIR = join(BASE_DIR, "version_manifests") +FILE_MANIFEST_DIR = join(BASE_DIR, "files_manifests") +DERIVED_INDEX_FILE = join(BASE_DIR, "derived_index.json") + +STATIC_LEGACYINFO_FILE = join(BASE_DIR, "forge-legacyinfo.json") + +FORGE_COMPONENT = "net.minecraftforge" + +FORGEWRAPPER_MAVEN = "https://polymc.github.io/files/maven/%s" +BAD_VERSIONS = ["1.12.2-14.23.5.2851"] diff --git a/meta/forgeutil.py b/meta/forgeutil.py deleted file mode 100644 index 4c2615cb8d..0000000000 --- a/meta/forgeutil.py +++ /dev/null @@ -1,323 +0,0 @@ -from collections import namedtuple - -from .metautil import * - - -# A post-processed entry constructed from the reconstructed Forge version index -class ForgeVersion: - def __init__(self, entry): - self.build = entry.build - self.rawVersion = entry.version - self.mcversion = entry.mcversion - self.mcversion_sane = self.mcversion.replace("_pre", "-pre", 1) - self.branch = entry.branch - self.installer_filename = None - self.installer_url = None - self.universal_filename = None - self.universal_url = None - self.changelog_url = None - self.longVersion = "%s-%s" % (self.mcversion, self.rawVersion) - if self.branch != None: - self.longVersion = self.longVersion + "-%s" % (self.branch) - for classifier, fileentry in entry.files.items(): - extension = fileentry.extension - checksum = fileentry.hash - filename = fileentry.filename(self.longVersion) - url = fileentry.url(self.longVersion) - if (classifier == "installer") and (extension == "jar"): - self.installer_filename = filename - self.installer_url = url - if (classifier == "universal" or classifier == "client") and (extension == "jar" or extension == "zip"): - self.universal_filename = filename - self.universal_url = url - if (classifier == "changelog") and (extension == "txt"): - self.changelog_url = url - - def name(self): - return "Forge %d" % (self.build) - - def usesInstaller(self): - if self.installer_url == None: - return False - if self.mcversion == "1.5.2": - return False - return True - - def filename(self): - if self.usesInstaller(): - return self.installer_filename - else: - return self.universal_filename - - def url(self): - if self.usesInstaller(): - return self.installer_url - else: - return self.universal_url - - def isSupported(self): - if self.url() == None: - return False - - versionElements = self.rawVersion.split('.') - if len(versionElements) < 1: - return False - - majorVersionStr = versionElements[0] - if not majorVersionStr.isnumeric(): - return False - - # majorVersion = int(majorVersionStr) - # if majorVersion >= 37: - # return False - - return True - - -class ForgeFile(JsonObject): - classifier = StringProperty(required=True) - hash = StringProperty(required=True) - extension = StringProperty(required=True) - - def filename(self, longversion): - return "%s-%s-%s.%s" % ("forge", longversion, self.classifier, self.extension) - - def url(self, longversion): - return "https://files.minecraftforge.net/maven/net/minecraftforge/forge/%s/%s" % ( - longversion, self.filename(longversion)) - - -class ForgeEntry(JsonObject): - longversion = StringProperty(required=True) - mcversion = StringProperty(required=True) - version = StringProperty(required=True) - build = IntegerProperty(required=True) - branch = StringProperty() - latest = BooleanProperty() - recommended = BooleanProperty() - files = DictProperty(ForgeFile) - - -class ForgeMcVersionInfo(JsonObject): - latest = StringProperty() - recommended = StringProperty() - versions = ListProperty(StringProperty()) - - -class DerivedForgeIndex(JsonObject): - versions = DictProperty(ForgeEntry) - by_mcversion = DictProperty(ForgeMcVersionInfo) - - -''' -FML library mappings - these are added to legacy Forge versions because Forge no longer can download these -by itself - the locations have changed and some of this has to be rehosted on PolyMC servers. -''' - -FMLLib = namedtuple('FMLLib', ('filename', 'checksum', 'ours')) - -fmlLibsMapping = {} - -fmlLibsMapping["1.3.2"] = [ - FMLLib("argo-2.25.jar", "bb672829fde76cb163004752b86b0484bd0a7f4b", False), - FMLLib("guava-12.0.1.jar", "b8e78b9af7bf45900e14c6f958486b6ca682195f", False), - FMLLib("asm-all-4.0.jar", "98308890597acb64047f7e896638e0d98753ae82", False) -] - -fml14 = [ - FMLLib("argo-2.25.jar", "bb672829fde76cb163004752b86b0484bd0a7f4b", False), - FMLLib("guava-12.0.1.jar", "b8e78b9af7bf45900e14c6f958486b6ca682195f", False), - FMLLib("asm-all-4.0.jar", "98308890597acb64047f7e896638e0d98753ae82", False), - FMLLib("bcprov-jdk15on-147.jar", "b6f5d9926b0afbde9f4dbe3db88c5247be7794bb", False) -] -fmlLibsMapping["1.4"] = fml14; -fmlLibsMapping["1.4.1"] = fml14; -fmlLibsMapping["1.4.2"] = fml14; -fmlLibsMapping["1.4.3"] = fml14; -fmlLibsMapping["1.4.4"] = fml14; -fmlLibsMapping["1.4.5"] = fml14; -fmlLibsMapping["1.4.6"] = fml14; -fmlLibsMapping["1.4.7"] = fml14; - -fmlLibsMapping["1.5"] = [ - FMLLib("argo-small-3.2.jar", "58912ea2858d168c50781f956fa5b59f0f7c6b51", False), - FMLLib("guava-14.0-rc3.jar", "931ae21fa8014c3ce686aaa621eae565fefb1a6a", False), - FMLLib("asm-all-4.1.jar", "054986e962b88d8660ae4566475658469595ef58", False), - FMLLib("bcprov-jdk15on-148.jar", "960dea7c9181ba0b17e8bab0c06a43f0a5f04e65", True), - FMLLib("deobfuscation_data_1.5.zip", "5f7c142d53776f16304c0bbe10542014abad6af8", False), - FMLLib("scala-library.jar", "458d046151ad179c85429ed7420ffb1eaf6ddf85", True) -] - -fmlLibsMapping["1.5.1"] = [ - FMLLib("argo-small-3.2.jar", "58912ea2858d168c50781f956fa5b59f0f7c6b51", False), - FMLLib("guava-14.0-rc3.jar", "931ae21fa8014c3ce686aaa621eae565fefb1a6a", False), - FMLLib("asm-all-4.1.jar", "054986e962b88d8660ae4566475658469595ef58", False), - FMLLib("bcprov-jdk15on-148.jar", "960dea7c9181ba0b17e8bab0c06a43f0a5f04e65", True), - FMLLib("deobfuscation_data_1.5.1.zip", "22e221a0d89516c1f721d6cab056a7e37471d0a6", False), - FMLLib("scala-library.jar", "458d046151ad179c85429ed7420ffb1eaf6ddf85", True) -] - -fmlLibsMapping["1.5.2"] = [ - FMLLib("argo-small-3.2.jar", "58912ea2858d168c50781f956fa5b59f0f7c6b51", False), - FMLLib("guava-14.0-rc3.jar", "931ae21fa8014c3ce686aaa621eae565fefb1a6a", False), - FMLLib("asm-all-4.1.jar", "054986e962b88d8660ae4566475658469595ef58", False), - FMLLib("bcprov-jdk15on-148.jar", "960dea7c9181ba0b17e8bab0c06a43f0a5f04e65", True), - FMLLib("deobfuscation_data_1.5.2.zip", "446e55cd986582c70fcf12cb27bc00114c5adfd9", False), - FMLLib("scala-library.jar", "458d046151ad179c85429ed7420ffb1eaf6ddf85", True) -] - -''' -"install": { - "profileName": "Forge", - "target":"Forge8.9.0.753", - "path":"net.minecraftforge:minecraftforge:8.9.0.753", - "version":"Forge 8.9.0.753", - "filePath":"minecraftforge-universal-1.6.1-8.9.0.753.jar", - "welcome":"Welcome to the simple Forge installer.", - "minecraft":"1.6.1", - "logo":"/big_logo.png", - "mirrorList": "http://files.minecraftforge.net/mirror-brand.list" -}, -"install": { - "profileName": "forge", - "target":"1.11-forge1.11-13.19.0.2141", - "path":"net.minecraftforge:forge:1.11-13.19.0.2141", - "version":"forge 1.11-13.19.0.2141", - "filePath":"forge-1.11-13.19.0.2141-universal.jar", - "welcome":"Welcome to the simple forge installer.", - "minecraft":"1.11", - "mirrorList" : "http://files.minecraftforge.net/mirror-brand.list", - "logo":"/big_logo.png", - "modList":"none" -}, -''' - - -class ForgeInstallerProfileInstallSection(JsonObject): - profileName = StringProperty(required=True) - target = StringProperty(required=True) - path = GradleSpecifierProperty(required=True) - version = StringProperty(required=True) - filePath = StringProperty(required=True) - welcome = StringProperty(required=True) - minecraft = StringProperty(required=True) - logo = StringProperty(required=True) - mirrorList = StringProperty(required=True) - modList = StringProperty(exclude_if_none=True, default=None) - - -class ForgeLibrary(MojangLibrary): - url = StringProperty(exclude_if_none=True) - serverreq = BooleanProperty(exclude_if_none=True, default=None) - clientreq = BooleanProperty(exclude_if_none=True, default=None) - checksums = ListProperty(StringProperty) - comment = StringProperty() - - -class ForgeVersionFile(MojangVersionFile): - libraries = ListProperty(ForgeLibrary, exclude_if_none=True, default=None) # overrides Mojang libraries - inheritsFrom = StringProperty() - jar = StringProperty() - - -''' -"optionals": [ - { - "name": "Mercurius", - "client": true, - "server": true, - "default": true, - "inject": true, - "desc": "A mod that collects statistics about Minecraft and your system.<br>Useful for Forge to understand how Minecraft/Forge are used.", - "url": "http://www.minecraftforge.net/forum/index.php?topic=43278.0", - "artifact": "net.minecraftforge:MercuriusUpdater:1.11.2", - "maven": "http://files.minecraftforge.net/maven/" - } -] -''' - - -class ForgeOptional(JsonObject): - name = StringProperty() - client = BooleanProperty() - server = BooleanProperty() - default = BooleanProperty() - inject = BooleanProperty() - desc = StringProperty() - url = StringProperty() - artifact = GradleSpecifierProperty() - maven = StringProperty() - - -class ForgeInstallerProfile(JsonObject): - install = ObjectProperty(ForgeInstallerProfileInstallSection, required=True) - versionInfo = ObjectProperty(ForgeVersionFile, required=True) - optionals = ListProperty(ForgeOptional) - - -class ForgeLegacyInfo(JsonObject): - releaseTime = ISOTimestampProperty() - size = IntegerProperty() - sha256 = StringProperty() - sha1 = StringProperty() - - -class ForgeLegacyInfoList(JsonObject): - number = DictProperty(ForgeLegacyInfo) - - -class DataSpec(JsonObject): - client = StringProperty() - server = StringProperty() - - -class ProcessorSpec(JsonObject): - jar = StringProperty() - classpath = ListProperty(StringProperty) - args = ListProperty(StringProperty) - outputs = DictProperty(StringProperty) - sides = ListProperty(StringProperty, exclude_if_none=True, default=None) - - -# Note: This is only used in one version (1.12.2-14.23.5.2851) and we don't even use the installer profile in it. -# It's here just so it parses and we can continue... -class ForgeInstallerProfileV1_5(JsonObject): - _comment = ListProperty(StringProperty) - spec = IntegerProperty() - profile = StringProperty() - version = StringProperty() - icon = StringProperty() - json = StringProperty() - path = GradleSpecifierProperty() - logo = StringProperty() - minecraft = StringProperty() - welcome = StringProperty() - # We don't know what 'data' actually is in this one. It's an empty array - data = ListProperty(StringProperty) - processors = ListProperty(ProcessorSpec) - libraries = ListProperty(MojangLibrary) - mirrorList = StringProperty(exclude_if_none=True, default=None) - - -class ForgeInstallerProfileV2(JsonObject): - _comment = ListProperty(StringProperty) - spec = IntegerProperty() - profile = StringProperty() - version = StringProperty() - icon = StringProperty() - json = StringProperty() - path = GradleSpecifierProperty() - logo = StringProperty() - minecraft = StringProperty() - welcome = StringProperty() - data = DictProperty(DataSpec) - processors = ListProperty(ProcessorSpec) - libraries = ListProperty(MojangLibrary) - mirrorList = StringProperty(exclude_if_none=True, default=None) - serverJarPath = StringProperty(exclude_if_none=True, default=None) - - -class InstallerInfo(JsonObject): - sha1hash = StringProperty() - sha256hash = StringProperty() - size = IntegerProperty() diff --git a/meta/model/forge.py b/meta/model/forge.py new file mode 100644 index 0000000000..6290cc8dff --- /dev/null +++ b/meta/model/forge.py @@ -0,0 +1,306 @@ +from datetime import datetime +from typing import Optional, List, Dict + +from pydantic import Field + +from . import MetaBase, GradleSpecifier, MojangLibrary +from .mojang import MojangVersion + + +class ForgeFile(MetaBase): + classifier: str + hash: str + extension: str + + def filename(self, longversion): + return "%s-%s-%s.%s" % ("forge", longversion, self.classifier, self.extension) + + def url(self, longversion): + return "https://files.minecraftforge.net/maven/net/minecraftforge/forge/%s/%s" % ( + longversion, self.filename(longversion)) + + +class ForgeEntry(MetaBase): + longversion: str + mcversion: str + version: str + build: int + branch: Optional[str] + latest: Optional[bool] + recommended: Optional[bool] + files: Optional[Dict[str, ForgeFile]] + + +class ForgeMCVersionInfo(MetaBase): + latest: Optional[str] + recommended: Optional[str] + versions: List[str] = Field([]) + + +class DerivedForgeIndex(MetaBase): + versions: Dict[str, ForgeEntry] = Field({}) + by_mcversion: Dict[str, ForgeMCVersionInfo] = Field({}) + + +class FMLLib(MetaBase): # old ugly stuff. Maybe merge this with Library or MojangLibrary later + filename: str + checksum: str + ours: bool + + +class ForgeInstallerProfileInstallSection(MetaBase): + """ + "install": { + "profileName": "Forge", + "target":"Forge8.9.0.753", + "path":"net.minecraftforge:minecraftforge:8.9.0.753", + "version":"Forge 8.9.0.753", + "filePath":"minecraftforge-universal-1.6.1-8.9.0.753.jar", + "welcome":"Welcome to the simple Forge installer.", + "minecraft":"1.6.1", + "logo":"/big_logo.png", + "mirrorList": "http://files.minecraftforge.net/mirror-brand.list" + }, + "install": { + "profileName": "forge", + "target":"1.11-forge1.11-13.19.0.2141", + "path":"net.minecraftforge:forge:1.11-13.19.0.2141", + "version":"forge 1.11-13.19.0.2141", + "filePath":"forge-1.11-13.19.0.2141-universal.jar", + "welcome":"Welcome to the simple forge installer.", + "minecraft":"1.11", + "mirrorList" : "http://files.minecraftforge.net/mirror-brand.list", + "logo":"/big_logo.png", + "modList":"none" + }, + """ + profileName: str + target: str + path: GradleSpecifier + version: str + filePath: str + welcome: str + minecraft: str + logo: str + mirrorList: str + modList: Optional[str] + + +class ForgeLibrary(MojangLibrary): + url: Optional[str] + serverreq: Optional[bool] + clientreq: Optional[bool] + checksums: Optional[List[str]] + comment: Optional[str] + + +class ForgeVersionFile(MojangVersion): + libraries: Optional[List[ForgeLibrary]] # overrides Mojang libraries + inheritsFrom: Optional[str] + jar: Optional[str] + + +class ForgeOptional(MetaBase): + """ + "optionals": [ + { + "name": "Mercurius", + "client": true, + "server": true, + "default": true, + "inject": true, + "desc": "A mod that collects statistics about Minecraft and your system.<br>Useful for Forge to understand how Minecraft/Forge are used.", + "url": "http://www.minecraftforge.net/forum/index.php?topic=43278.0", + "artifact": "net.minecraftforge:MercuriusUpdater:1.11.2", + "maven": "http://files.minecraftforge.net/maven/" + } + ] + """ + name: Optional[str] + client: Optional[bool] + server: Optional[bool] + default: Optional[bool] + inject: Optional[bool] + desc: Optional[str] + url: Optional[str] + artifact: Optional[GradleSpecifier] + maven: Optional[str] + + +class ForgeInstallerProfile(MetaBase): + install: ForgeInstallerProfileInstallSection + versionInfo: ForgeVersionFile + optionals: Optional[List[ForgeOptional]] + + +class ForgeLegacyInfo(MetaBase): + releaseTime: Optional[datetime] + size: Optional[int] + sha256: Optional[str] + sha1: Optional[str] + + +class ForgeLegacyInfoList(MetaBase): + number: Dict[str, ForgeLegacyInfo] = Field({}) + + +class DataSpec(MetaBase): + client: Optional[str] + server: Optional[str] + + +class ProcessorSpec(MetaBase): + jar: Optional[str] + classpath: Optional[List[str]] + args: Optional[List[str]] + outputs: Optional[Dict[str, str]] + sides: Optional[List[str]] + + +class ForgeInstallerProfileV2(MetaBase): + _comment: Optional[List[str]] + spec: Optional[int] + profile: Optional[str] + version: Optional[str] + icon: Optional[str] + json_data: Optional[str] = Field(alias="json") + path: Optional[GradleSpecifier] + logo: Optional[str] + minecraft: Optional[str] + welcome: Optional[str] + data: Optional[Dict[str, DataSpec]] + processors: Optional[List[ProcessorSpec]] + libraries: Optional[List[MojangLibrary]] + mirrorList: Optional[str] + serverJarPath: Optional[str] + + +class InstallerInfo(MetaBase): + sha1hash: Optional[str] + sha256hash: Optional[str] + size: Optional[int] + + +def fml_libs_for_version(mc_version: str) -> List[FMLLib]: + argo_2_25 = FMLLib(filename="argo-2.25.jar", + checksum="bb672829fde76cb163004752b86b0484bd0a7f4b", + ours=False) + argo_small_3_2 = FMLLib(filename="argo-small-3.2.jar", + checksum="58912ea2858d168c50781f956fa5b59f0f7c6b51", + ours=False) + guava_12_0_1 = FMLLib(filename="guava-12.0.1.jar", + checksum="b8e78b9af7bf45900e14c6f958486b6ca682195f", + ours=False) + guava_14_0_rc3 = FMLLib(filename="guava-14.0-rc3.jar", + checksum="931ae21fa8014c3ce686aaa621eae565fefb1a6a", + ours=False) + asm_all_4_0 = FMLLib(filename="asm-all-4.0.jar", + checksum="98308890597acb64047f7e896638e0d98753ae82", + ours=False) + asm_all_4_1 = FMLLib(filename="asm-all-4.1.jar", + checksum="054986e962b88d8660ae4566475658469595ef58", + ours=False) + bcprov_jdk15on_147 = FMLLib(filename="bcprov-jdk15on-147.jar", + checksum="b6f5d9926b0afbde9f4dbe3db88c5247be7794bb", + ours=False) + bcprov_jdk15on_148 = FMLLib(filename="bcprov-jdk15on-148.jar", + checksum="960dea7c9181ba0b17e8bab0c06a43f0a5f04e65", + ours=True) + scala_library = FMLLib(filename="scala-library.jar", + checksum="458d046151ad179c85429ed7420ffb1eaf6ddf85", + ours=True) + + deobfuscation_data_1_5 = FMLLib(filename="deobfuscation_data_1.5.zip", + checksum="5f7c142d53776f16304c0bbe10542014abad6af8", + ours=False) + + deobfuscation_data_1_5_1 = FMLLib(filename="deobfuscation_data_1.5.1.zip", + checksum="22e221a0d89516c1f721d6cab056a7e37471d0a6", + ours=False) + deobfuscation_data_1_5_2 = FMLLib(filename="deobfuscation_data_1.5.2.zip", + checksum="446e55cd986582c70fcf12cb27bc00114c5adfd9", + ours=False) + if mc_version == "1.3.2": + return [argo_2_25, guava_12_0_1, asm_all_4_0] + elif mc_version in ["1.4", "1.4.1", "1.4.2", "1.4.3", "1.4.4", "1.4.5", "1.4.6", "1.4.7"]: + return [argo_2_25, guava_12_0_1, asm_all_4_0, bcprov_jdk15on_147] + elif mc_version == "1.5": + return [argo_small_3_2, guava_14_0_rc3, asm_all_4_1, bcprov_jdk15on_148, deobfuscation_data_1_5, + scala_library] + elif mc_version == "1.5.1": + return [argo_small_3_2, guava_14_0_rc3, asm_all_4_1, bcprov_jdk15on_148, deobfuscation_data_1_5_1, + scala_library] + elif mc_version == "1.5.2": + return [argo_small_3_2, guava_14_0_rc3, asm_all_4_1, bcprov_jdk15on_148, deobfuscation_data_1_5_2, + scala_library] + return [] + + +# A post-processed entry constructed from the reconstructed Forge version index +class ForgeVersion: + def __init__(self, entry: ForgeEntry): + self.build = entry.build + self.rawVersion = entry.version + self.mcversion = entry.mcversion + self.mcversion_sane = self.mcversion.replace("_pre", "-pre", 1) + self.branch = entry.branch + self.installer_filename = None + self.installer_url = None + self.universal_filename = None + self.universal_url = None + self.changelog_url = None + self.longVersion = "%s-%s" % (self.mcversion, self.rawVersion) + if self.branch is not None: + self.longVersion = self.longVersion + "-%s" % self.branch + + # this comment's whole purpose is to say this: cringe + for classifier, file in entry.files.items(): + extension = file.extension + filename = file.filename(self.longVersion) + url = file.url(self.longVersion) + if (classifier == "installer") and (extension == "jar"): + self.installer_filename = filename + self.installer_url = url + if (classifier == "universal" or classifier == "client") and (extension == "jar" or extension == "zip"): + self.universal_filename = filename + self.universal_url = url + if (classifier == "changelog") and (extension == "txt"): + self.changelog_url = url + + def name(self): + return "Forge %d" % self.build + + def uses_installer(self): + if self.installer_url is None: + return False + if self.mcversion == "1.5.2": + return False + return True + + def filename(self): + if self.uses_installer(): + return self.installer_filename + return self.universal_filename + + def url(self): + if self.uses_installer(): + return self.installer_url + return self.universal_url + + def is_supported(self): + if self.url() is None: + return False + + foo = self.rawVersion.split('.') + if len(foo) < 1: + return False + + major_version = foo[0] + if not major_version.isnumeric(): + return False + + # majorVersion = int(majorVersionStr) + # if majorVersion >= 37: + # return False + + return True diff --git a/static/forge-legacyinfo.json b/static/forge/forge-legacyinfo.json index 28e45244f8..28e45244f8 100644 --- a/static/forge-legacyinfo.json +++ b/static/forge/forge-legacyinfo.json diff --git a/updateForge.py b/updateForge.py index a2cc8e9a5d..0492339b83 100755 --- a/updateForge.py +++ b/updateForge.py @@ -3,20 +3,42 @@ ''' import copy import hashlib +import json +import os import re import sys import zipfile from contextlib import suppress +from datetime import datetime from pathlib import Path from pprint import pprint import requests from cachecontrol import CacheControl from cachecontrol.caches import FileCache -from meta.forgeutil import * -from meta.metautil import * +from pydantic import ValidationError -UPSTREAM_DIR = os.environ["UPSTREAM_DIR"] +from meta.common import upstream_path, ensure_upstream_dir, static_path +from meta.common.forge import JARS_DIR, INSTALLER_INFO_DIR, INSTALLER_MANIFEST_DIR, VERSION_MANIFEST_DIR, \ + FILE_MANIFEST_DIR, BAD_VERSIONS, STATIC_LEGACYINFO_FILE +from meta.model.forge import ForgeFile, ForgeEntry, ForgeMCVersionInfo, ForgeLegacyInfoList, DerivedForgeIndex, \ + ForgeVersion, ForgeInstallerProfile, ForgeInstallerProfileV2, InstallerInfo, \ + ForgeLegacyInfo +from meta.model.mojang import MojangVersion + +UPSTREAM_DIR = upstream_path() +STATIC_DIR = static_path() + +ensure_upstream_dir(JARS_DIR) +ensure_upstream_dir(INSTALLER_INFO_DIR) +ensure_upstream_dir(INSTALLER_MANIFEST_DIR) +ensure_upstream_dir(VERSION_MANIFEST_DIR) +ensure_upstream_dir(FILE_MANIFEST_DIR) + +LEGACYINFO_PATH = os.path.join(STATIC_DIR, STATIC_LEGACYINFO_FILE) + +forever_cache = FileCache('caches/http_cache', forever=True) +sess = CacheControl(requests.Session(), forever_cache) def eprint(*args, **kwargs): @@ -31,56 +53,8 @@ def filehash(filename, hashtype, blocksize=65536): return hash.hexdigest() -forever_cache = FileCache('caches/http_cache', forever=True) -sess = CacheControl(requests.Session(), forever_cache) - -# get the remote version list fragments -r = sess.get('https://files.minecraftforge.net/net/minecraftforge/forge/maven-metadata.json') -r.raise_for_status() -main_json = r.json() -assert type(main_json) == dict - -r = sess.get('https://files.minecraftforge.net/net/minecraftforge/forge/promotions_slim.json') -r.raise_for_status() -promotions_json = r.json() -assert type(promotions_json) == dict - -promotedKeyExpression = re.compile("(?P<mc>[^-]+)-(?P<promotion>(latest)|(recommended))(-(?P<branch>[a-zA-Z0-9\\.]+))?") - -recommendedSet = set() - -newIndex = DerivedForgeIndex() - -# FIXME: does not fully validate that the file has not changed format -# NOTE: For some insane reason, the format of the versions here is special. It having a branch at the end means it affects that particular branch -# We don't care about Forge having branches. -# Therefore we only use the short version part for later identification and filter out the branch-specific promotions (among other errors). -print("Processing promotions:") -for promoKey, shortversion in promotions_json.get('promos').items(): - match = promotedKeyExpression.match(promoKey) - if not match: - print('Skipping promotion %s, the key did not parse:' % promoKey) - pprint(promoKey) - assert match - if not match.group('mc'): - print('Skipping promotion %s, because it has no Minecraft version.' % promoKey) - continue - if match.group('branch'): - print('Skipping promotion %s, because it on a branch only.' % promoKey) - continue - elif match.group('promotion') == 'recommended': - recommendedSet.add(shortversion) - print('%s added to recommended set' % shortversion) - elif match.group('promotion') == 'latest': - pass - else: - assert False - -versionExpression = re.compile( - "^(?P<mc>[0-9a-zA-Z_\\.]+)-(?P<ver>[0-9\\.]+\\.(?P<build>[0-9]+))(-(?P<branch>[a-zA-Z0-9\\.]+))?$") - - -def getSingleForgeFilesManifest(longversion): +def get_single_forge_files_manifest(longversion): + print(f"Getting Forge manifest for {longversion}") pathThing = UPSTREAM_DIR + "/forge/files_manifests/%s.json" % longversion files_manifest_file = Path(pathThing) from_file = False @@ -144,218 +118,243 @@ def getSingleForgeFilesManifest(longversion): return retDict -print("") -print("Making dirs...") -os.makedirs(UPSTREAM_DIR + "/forge/jars/", exist_ok=True) -os.makedirs(UPSTREAM_DIR + "/forge/installer_info/", exist_ok=True) -os.makedirs(UPSTREAM_DIR + "/forge/installer_manifests/", exist_ok=True) -os.makedirs(UPSTREAM_DIR + "/forge/version_manifests/", exist_ok=True) -os.makedirs(UPSTREAM_DIR + "/forge/files_manifests/", exist_ok=True) - -print("") -print("Processing versions:") -for mcversion, value in main_json.items(): - assert type(mcversion) == str - assert type(value) == list - for longversion in value: - assert type(longversion) == str - match = versionExpression.match(longversion) - if not match: - pprint(longversion) - assert match - assert match.group('mc') == mcversion +def main(): + # get the remote version list fragments + r = sess.get('https://files.minecraftforge.net/net/minecraftforge/forge/maven-metadata.json') + r.raise_for_status() + main_json = r.json() + assert type(main_json) == dict - files = getSingleForgeFilesManifest(longversion) + r = sess.get('https://files.minecraftforge.net/net/minecraftforge/forge/promotions_slim.json') + r.raise_for_status() + promotions_json = r.json() + assert type(promotions_json) == dict - build = int(match.group('build')) - version = match.group('ver') - branch = match.group('branch') + promotedKeyExpression = re.compile( + "(?P<mc>[^-]+)-(?P<promotion>(latest)|(recommended))(-(?P<branch>[a-zA-Z0-9\\.]+))?") - isRecommended = (version in recommendedSet) + recommendedSet = set() - entry = ForgeEntry( - longversion=longversion, - mcversion=mcversion, - version=version, - build=build, - branch=branch, + newIndex = DerivedForgeIndex() + + # FIXME: does not fully validate that the file has not changed format + # NOTE: For some insane reason, the format of the versions here is special. It having a branch at the end means it affects that particular branch + # We don't care about Forge having branches. + # Therefore we only use the short version part for later identification and filter out the branch-specific promotions (among other errors). + print("Processing promotions:") + for promoKey, shortversion in promotions_json.get('promos').items(): + match = promotedKeyExpression.match(promoKey) + if not match: + print('Skipping promotion %s, the key did not parse:' % promoKey) + pprint(promoKey) + assert match + if not match.group('mc'): + print('Skipping promotion %s, because it has no Minecraft version.' % promoKey) + continue + if match.group('branch'): + print('Skipping promotion %s, because it on a branch only.' % promoKey) + continue + elif match.group('promotion') == 'recommended': + recommendedSet.add(shortversion) + print('%s added to recommended set' % shortversion) + elif match.group('promotion') == 'latest': + pass + else: + assert False + + versionExpression = re.compile( + "^(?P<mc>[0-9a-zA-Z_\\.]+)-(?P<ver>[0-9\\.]+\\.(?P<build>[0-9]+))(-(?P<branch>[a-zA-Z0-9\\.]+))?$") + + print("") + print("Processing versions:") + for mcversion, value in main_json.items(): + assert type(mcversion) == str + assert type(value) == list + for longversion in value: + assert type(longversion) == str + match = versionExpression.match(longversion) + if not match: + pprint(longversion) + assert match + assert match.group('mc') == mcversion + + files = get_single_forge_files_manifest(longversion) + + build = int(match.group('build')) + version = match.group('ver') + branch = match.group('branch') + + isRecommended = (version in recommendedSet) + + entry = ForgeEntry( + longversion=longversion, + mcversion=mcversion, + version=version, + build=build, + branch=branch, + # NOTE: we add this later after the fact. The forge promotions file lies about these. + latest=False, + recommended=isRecommended, + files=files + ) + newIndex.versions[longversion] = entry + if not newIndex.by_mcversion: + newIndex.by_mcversion = dict() + if not mcversion in newIndex.by_mcversion: + newIndex.by_mcversion.setdefault(mcversion, ForgeMCVersionInfo()) + newIndex.by_mcversion[mcversion].versions.append(longversion) # NOTE: we add this later after the fact. The forge promotions file lies about these. - latest=False, - recommended=isRecommended, - files=files - ) - newIndex.versions[longversion] = entry - if not newIndex.by_mcversion: - newIndex.by_mcversion = dict() - if not mcversion in newIndex.by_mcversion: - newIndex.by_mcversion.setdefault(mcversion, ForgeMcVersionInfo()) - newIndex.by_mcversion[mcversion].versions.append(longversion) - # NOTE: we add this later after the fact. The forge promotions file lies about these. - # if entry.latest: - # newIndex.by_mcversion[mcversion].latest = longversion - if entry.recommended: - newIndex.by_mcversion[mcversion].recommended = longversion - -print("") -print("Post processing promotions and adding missing 'latest':") -for mcversion, info in newIndex.by_mcversion.items(): - latestVersion = info.versions[-1] - info.latest = latestVersion - newIndex.versions[latestVersion].latest = True - print("Added %s as latest for %s" % (latestVersion, mcversion)) - -print("") -print("Dumping index files...") - -with open(UPSTREAM_DIR + "/forge/maven-metadata.json", 'w', encoding='utf-8') as f: - json.dump(main_json, f, sort_keys=True, indent=4) - -with open(UPSTREAM_DIR + "/forge/promotions_slim.json", 'w', encoding='utf-8') as f: - json.dump(promotions_json, f, sort_keys=True, indent=4) - -with open(UPSTREAM_DIR + "/forge/derived_index.json", 'w', encoding='utf-8') as f: - json.dump(newIndex.to_json(), f, sort_keys=True, indent=4) - -versions = [] -legacyinfolist = ForgeLegacyInfoList() -tsPath = "static/forge-legacyinfo.json" - -fuckedVersions = [] - -print("Grabbing installers and dumping installer profiles...") -# get the installer jars - if needed - and get the installer profiles out of them -for id, entry in newIndex.versions.items(): - eprint("Updating Forge %s" % id) - if entry.mcversion == None: - eprint("Skipping %d with invalid MC version" % entry.build) - continue - - version = ForgeVersion(entry) - if version.url() == None: - eprint("Skipping %d with no valid files" % version.build) - continue - - jarFilepath = UPSTREAM_DIR + "/forge/jars/%s" % version.filename() - - if version.usesInstaller(): - installerInfoFilepath = UPSTREAM_DIR + "/forge/installer_info/%s.json" % version.longVersion - profileFilepath = UPSTREAM_DIR + "/forge/installer_manifests/%s.json" % version.longVersion - versionJsonFilepath = UPSTREAM_DIR + "/forge/version_manifests/%s.json" % version.longVersion - installerRefreshRequired = False - if not os.path.isfile(profileFilepath): - installerRefreshRequired = True - if not os.path.isfile(installerInfoFilepath): - installerRefreshRequired = True - - if installerRefreshRequired: - # grab the installer if it's not there - if not os.path.isfile(jarFilepath): - eprint("Downloading %s" % version.url()) - rfile = sess.get(version.url(), stream=True) - rfile.raise_for_status() - with open(jarFilepath, 'wb') as f: - for chunk in rfile.iter_content(chunk_size=128): - f.write(chunk) - - eprint("Processing %s" % version.url()) - # harvestables from the installer - if not os.path.isfile(profileFilepath): - print(jarFilepath) - with zipfile.ZipFile(jarFilepath, 'r') as jar: - with suppress(KeyError): - with jar.open('version.json', 'r') as profileZipEntry: - versionJsonData = profileZipEntry.read(); - versionJsonJson = json.loads(versionJsonData) + # if entry.latest: + # newIndex.by_mcversion[mcversion].latest = longversion + if entry.recommended: + newIndex.by_mcversion[mcversion].recommended = longversion + + print("") + print("Post processing promotions and adding missing 'latest':") + for mcversion, info in newIndex.by_mcversion.items(): + latestVersion = info.versions[-1] + info.latest = latestVersion + newIndex.versions[latestVersion].latest = True + print("Added %s as latest for %s" % (latestVersion, mcversion)) + + print("") + print("Dumping index files...") + + with open(UPSTREAM_DIR + "/forge/maven-metadata.json", 'w', encoding='utf-8') as f: + json.dump(main_json, f, sort_keys=True, indent=4) + + with open(UPSTREAM_DIR + "/forge/promotions_slim.json", 'w', encoding='utf-8') as f: + json.dump(promotions_json, f, sort_keys=True, indent=4) + + newIndex.write(UPSTREAM_DIR + "/forge/derived_index.json") + + legacyinfolist = ForgeLegacyInfoList() + + print("Grabbing installers and dumping installer profiles...") + # get the installer jars - if needed - and get the installer profiles out of them + for id, entry in newIndex.versions.items(): + eprint("Updating Forge %s" % id) + if entry.mcversion is None: + eprint("Skipping %d with invalid MC version" % entry.build) + continue + + version = ForgeVersion(entry) + if version.url() is None: + eprint("Skipping %d with no valid files" % version.build) + continue + if version.longVersion in BAD_VERSIONS: + eprint(f"Skipping bad version {version.longVersion}") + continue + + jarFilepath = UPSTREAM_DIR + "/forge/jars/%s" % version.filename() + + if version.uses_installer(): + installerInfoFilepath = UPSTREAM_DIR + "/forge/installer_info/%s.json" % version.longVersion + profileFilepath = UPSTREAM_DIR + "/forge/installer_manifests/%s.json" % version.longVersion + versionJsonFilepath = UPSTREAM_DIR + "/forge/version_manifests/%s.json" % version.longVersion + installerRefreshRequired = False + if not os.path.isfile(profileFilepath): + installerRefreshRequired = True + if not os.path.isfile(installerInfoFilepath): + installerRefreshRequired = True + + if installerRefreshRequired: + # grab the installer if it's not there + if not os.path.isfile(jarFilepath): + eprint("Downloading %s" % version.url()) + rfile = sess.get(version.url(), stream=True) + rfile.raise_for_status() + with open(jarFilepath, 'wb') as f: + for chunk in rfile.iter_content(chunk_size=128): + f.write(chunk) + + eprint("Processing %s" % version.url()) + # harvestables from the installer + if not os.path.isfile(profileFilepath): + print(jarFilepath) + with zipfile.ZipFile(jarFilepath, 'r') as jar: + with suppress(KeyError): + with jar.open('version.json', 'r') as profileZipEntry: + versionJsonData = profileZipEntry.read() + profileZipEntry.close() + + # Process: does it parse? + doesItParse = MojangVersion.parse_raw(versionJsonData) + + with open(versionJsonFilepath, 'wb') as versionJsonFile: + versionJsonFile.write(versionJsonData) + versionJsonFile.close() + + with jar.open('install_profile.json', 'r') as profileZipEntry: + installProfileJsonData = profileZipEntry.read() profileZipEntry.close() # Process: does it parse? - doesItParse = MojangVersionFile(versionJsonJson) - - with open(versionJsonFilepath, 'wb') as versionJsonFile: - versionJsonFile.write(versionJsonData) - versionJsonFile.close() - - with jar.open('install_profile.json', 'r') as profileZipEntry: - installProfileJsonData = profileZipEntry.read() - profileZipEntry.close() - - # Process: does it parse? - installProfileJsonJson = json.loads(installProfileJsonData) - atLeastOneFormatWorked = False - exception = None - try: - doesItParseV1 = ForgeInstallerProfile(installProfileJsonJson) - atLeastOneFormatWorked = True - except BaseException as err: - exception = err - try: - doesItParseV2 = ForgeInstallerProfileV2(installProfileJsonJson) - atLeastOneFormatWorked = True - except BaseException as err: - exception = err - - # NOTE: Only here for 1.12.2-14.23.5.2851 - try: - doesItParseV1_5 = ForgeInstallerProfileV1_5(installProfileJsonJson) - atLeastOneFormatWorked = True - except BaseException as err: - exception = err - - if not atLeastOneFormatWorked: - if version.isSupported(): - raise exception - else: - eprint("Version %s is not supported and won't be generated later." % version.longVersion) - - with open(profileFilepath, 'wb') as profileFile: - profileFile.write(installProfileJsonData) - profileFile.close() - - # installer info v1 - if not os.path.isfile(installerInfoFilepath): - installerInfo = InstallerInfo() - eprint("SHA1 %s" % jarFilepath) - installerInfo.sha1hash = filehash(jarFilepath, hashlib.sha1) - eprint("SHA256 %s" % jarFilepath) - installerInfo.sha256hash = filehash(jarFilepath, hashlib.sha256) - eprint("SIZE %s" % jarFilepath) - installerInfo.size = os.path.getsize(jarFilepath) - eprint("DUMP %s" % jarFilepath) - with open(installerInfoFilepath, 'w', encoding='utf-8') as installerInfoFile: - json.dump(installerInfo.to_json(), installerInfoFile, sort_keys=True, indent=4) - installerInfoFile.close() - else: - pass - # ignore the two versions without install manifests and jar mod class files - # TODO: fix those versions? - if version.mcversion_sane == "1.6.1": - continue + atLeastOneFormatWorked = False + exception = None + try: + ForgeInstallerProfile.parse_raw(installProfileJsonData) + atLeastOneFormatWorked = True + except ValidationError as err: + exception = err + try: + ForgeInstallerProfileV2.parse_raw(installProfileJsonData) + atLeastOneFormatWorked = True + except ValidationError as err: + exception = err + + if not atLeastOneFormatWorked: + if version.is_supported(): + raise exception + else: + eprint( + "Version %s is not supported and won't be generated later." % version.longVersion) + + with open(profileFilepath, 'wb') as profileFile: + profileFile.write(installProfileJsonData) + profileFile.close() + + # installer info v1 + if not os.path.isfile(installerInfoFilepath): + installerInfo = InstallerInfo() + installerInfo.sha1hash = filehash(jarFilepath, hashlib.sha1) + installerInfo.sha256hash = filehash(jarFilepath, hashlib.sha256) + installerInfo.size = os.path.getsize(jarFilepath) + installerInfo.write(installerInfoFilepath) + else: + # ignore the two versions without install manifests and jar mod class files + # TODO: fix those versions? + if version.mcversion_sane == "1.6.1": + continue - # only gather legacy info if it's missing - if not os.path.isfile(tsPath): - # grab the jar/zip if it's not there - if not os.path.isfile(jarFilepath): - rfile = sess.get(version.url(), stream=True) - rfile.raise_for_status() - with open(jarFilepath, 'wb') as f: - for chunk in rfile.iter_content(chunk_size=128): - f.write(chunk) - # find the latest timestamp in the zip file - tstamp = datetime.datetime.fromtimestamp(0) - with zipfile.ZipFile(jarFilepath, 'r') as jar: - allinfo = jar.infolist() - for info in allinfo: - tstampNew = datetime.datetime(*info.date_time) - if tstampNew > tstamp: - tstamp = tstampNew - legacyInfo = ForgeLegacyInfo() - legacyInfo.releaseTime = tstamp - legacyInfo.sha1 = filehash(jarFilepath, hashlib.sha1) - legacyInfo.sha256 = filehash(jarFilepath, hashlib.sha256) - legacyInfo.size = os.path.getsize(jarFilepath) - legacyinfolist.number[id] = legacyInfo - -# only write legacy info if it's missing -if not os.path.isfile(tsPath): - with open(tsPath, 'w') as outfile: - json.dump(legacyinfolist.to_json(), outfile, sort_keys=True, indent=4) + # only gather legacy info if it's missing + if not os.path.isfile(LEGACYINFO_PATH): + # grab the jar/zip if it's not there + if not os.path.isfile(jarFilepath): + rfile = sess.get(version.url(), stream=True) + rfile.raise_for_status() + with open(jarFilepath, 'wb') as f: + for chunk in rfile.iter_content(chunk_size=128): + f.write(chunk) + # find the latest timestamp in the zip file + tstamp = datetime.fromtimestamp(0) + with zipfile.ZipFile(jarFilepath, 'r') as jar: + allinfo = jar.infolist() + for info in allinfo: + tstampNew = datetime(*info.date_time) + if tstampNew > tstamp: + tstamp = tstampNew + legacyInfo = ForgeLegacyInfo() + legacyInfo.releaseTime = tstamp + legacyInfo.sha1 = filehash(jarFilepath, hashlib.sha1) + legacyInfo.sha256 = filehash(jarFilepath, hashlib.sha256) + legacyInfo.size = os.path.getsize(jarFilepath) + legacyinfolist.number[id] = legacyInfo + + # only write legacy info if it's missing + if not os.path.isfile(LEGACYINFO_PATH): + legacyinfolist.write(LEGACYINFO_PATH) + + +if __name__ == '__main__': + main() |