diff options
| -rwxr-xr-x | meta/run/update_fabric.py | 45 | ||||
| -rwxr-xr-x | meta/run/update_mojang.py | 49 | ||||
| -rwxr-xr-x | meta/run/update_quilt.py | 51 |
3 files changed, 86 insertions, 59 deletions
diff --git a/meta/run/update_fabric.py b/meta/run/update_fabric.py index 132495b92c..850ec8eac1 100755 --- a/meta/run/update_fabric.py +++ b/meta/run/update_fabric.py @@ -1,3 +1,4 @@ +import concurrent.futures import json import os import zipfile @@ -93,6 +94,26 @@ def compute_jar_file(path, url): data.write(path + ".json") +def compute_jar_file_concurrent(component, it): + print(f"Processing {component} {it['version']} ") + jar_maven_url = get_maven_url(it["maven"], "https://maven.fabricmc.net/", ".jar") + compute_jar_file( + os.path.join(UPSTREAM_DIR, JARS_DIR, transform_maven_key(it["maven"])), + jar_maven_url, + ) + print(f"Processing {component} {it['version']} Done") + + +def get_json_file_concurrent(it): + print(f"Downloading JAR info for loader {it['version']} ") + maven_url = get_maven_url(it["maven"], "https://maven.fabricmc.net/", ".json") + get_json_file( + os.path.join(UPSTREAM_DIR, INSTALLER_INFO_DIR, f"{it['version']}.json"), + maven_url, + ) + print(f"Downloading JAR info for loader {it['version']} Done") + + def main(): # get the version list for each component we are interested in for component in ["intermediary", "loader"]: @@ -100,30 +121,18 @@ def main(): os.path.join(UPSTREAM_DIR, META_DIR, f"{component}.json"), "https://meta.fabricmc.net/v2/versions/" + component, ) - for it in index: - print(f"Processing {component} {it['version']} ") - jar_maven_url = get_maven_url( - it["maven"], "https://maven.fabricmc.net/", ".jar" - ) - compute_jar_file( - os.path.join(UPSTREAM_DIR, JARS_DIR, transform_maven_key(it["maven"])), - jar_maven_url, - ) + with concurrent.futures.ThreadPoolExecutor() as executor: + for it in index: + executor.submit(compute_jar_file_concurrent, component, it) # for each loader, download installer JSON file from maven with open( os.path.join(UPSTREAM_DIR, META_DIR, "loader.json"), "r", encoding="utf-8" ) as loaderVersionIndexFile: loader_version_index = json.load(loaderVersionIndexFile) - for it in loader_version_index: - print(f"Downloading JAR info for loader {it['version']} ") - maven_url = get_maven_url( - it["maven"], "https://maven.fabricmc.net/", ".json" - ) - get_json_file( - os.path.join(UPSTREAM_DIR, INSTALLER_INFO_DIR, f"{it['version']}.json"), - maven_url, - ) + with concurrent.futures.ThreadPoolExecutor() as executor: + for it in loader_version_index: + executor.submit(get_json_file_concurrent, it) if __name__ == "__main__": diff --git a/meta/run/update_mojang.py b/meta/run/update_mojang.py index 33b4e061ad..ddadc16e89 100755 --- a/meta/run/update_mojang.py +++ b/meta/run/update_mojang.py @@ -1,3 +1,4 @@ +import concurrent.futures import json import os import zipfile @@ -96,6 +97,28 @@ def update_javas(): remote_javas.write(java_manifest_path) +def fetch_version_concurrent(remote_versions, x): + version = remote_versions.versions[x] + print( + "Updating " + + version.id + + " to timestamp " + + version.release_time.strftime("%s") + ) + fetch_version(os.path.join(UPSTREAM_DIR, VERSIONS_DIR, f"{x}.json"), version.url) + + +def fetch_modified_version_concurrent(old_snapshots, x): + version = old_snapshots.versions[x] + old_snapshots_path = os.path.join(UPSTREAM_DIR, VERSIONS_DIR, f"{x}.json") + + print("Updating old snapshot " + version.id) + if not os.path.isfile(old_snapshots_path): + fetch_modified_version(old_snapshots_path, version) + else: + print("Already have old snapshot " + version.id) + + def main(): # get the remote version list r = sess.get("https://piston-meta.mojang.com/mc/game/version_manifest_v2.json") @@ -124,17 +147,9 @@ def main(): else: pending_ids = remote_ids - for x in pending_ids: - version = remote_versions.versions[x] - print( - "Updating " - + version.id - + " to timestamp " - + version.release_time.isoformat() - ) - fetch_version( - os.path.join(UPSTREAM_DIR, VERSIONS_DIR, f"{x}.json"), version.url - ) + with concurrent.futures.ThreadPoolExecutor() as executor: + for x in pending_ids: + executor.submit(fetch_version_concurrent, remote_versions, x) # deal with experimental snapshots separately if os.path.exists(STATIC_EXPERIMENTS_FILE): @@ -160,15 +175,9 @@ def main(): ) old_snapshots_ids = set(old_snapshots.versions.keys()) - for x in old_snapshots_ids: - version = old_snapshots.versions[x] - old_snapshots_path = os.path.join(UPSTREAM_DIR, VERSIONS_DIR, f"{x}.json") - - print("Updating old snapshot " + version.id) - if not os.path.isfile(old_snapshots_path): - fetch_modified_version(old_snapshots_path, version) - else: - print("Already have old snapshot " + version.id) + with concurrent.futures.ThreadPoolExecutor() as executor: + for x in old_snapshots_ids: + executor.submit(fetch_modified_version_concurrent, old_snapshots, x) remote_versions.index.write(version_manifest_path) diff --git a/meta/run/update_quilt.py b/meta/run/update_quilt.py index 5eaa8ed0d0..5f3c40caa8 100755 --- a/meta/run/update_quilt.py +++ b/meta/run/update_quilt.py @@ -1,10 +1,9 @@ +import concurrent.futures import json import os import zipfile from datetime import datetime -import requests - from meta.common import ( upstream_path, ensure_upstream_dir, @@ -12,7 +11,6 @@ from meta.common import ( default_session, ) from meta.common.quilt import JARS_DIR, INSTALLER_INFO_DIR, META_DIR, USE_QUILT_MAPPINGS -from meta.common.fabric import DATETIME_FORMAT_HTTP from meta.model.fabric import FabricJarInfo UPSTREAM_DIR = upstream_path() @@ -82,6 +80,29 @@ def compute_jar_file(path, url): data.write(path + ".json") +def compute_jar_file_concurrent(component, it): + print(f"Processing {component} {it['version']} ") + jar_maven_url = get_maven_url( + it["maven"], "https://maven.quiltmc.org/repository/release/", ".jar" + ) + compute_jar_file( + os.path.join(UPSTREAM_DIR, JARS_DIR, transform_maven_key(it["maven"])), + jar_maven_url, + ) + print(f"Processing {component} {it['version']} Done") + + +def get_json_file_concurrent(it): + print(f"Downloading JAR info for loader {it['version']} ") + maven_url = get_maven_url( + it["maven"], "https://maven.quiltmc.org/repository/release/", ".json" + ) + get_json_file( + os.path.join(UPSTREAM_DIR, INSTALLER_INFO_DIR, f"{it['version']}.json"), + maven_url, + ) + + def main(): # get the version list for each component we are interested in components = ["loader"] @@ -92,30 +113,18 @@ def main(): os.path.join(UPSTREAM_DIR, META_DIR, f"{component}.json"), "https://meta.quiltmc.org/v3/versions/" + component, ) - for it in index: - print(f"Processing {component} {it['version']} ") - jar_maven_url = get_maven_url( - it["maven"], "https://maven.quiltmc.org/repository/release/", ".jar" - ) - compute_jar_file( - os.path.join(UPSTREAM_DIR, JARS_DIR, transform_maven_key(it["maven"])), - jar_maven_url, - ) + with concurrent.futures.ThreadPoolExecutor() as executor: + for it in index: + executor.submit(compute_jar_file_concurrent, component, it) # for each loader, download installer JSON file from maven with open( os.path.join(UPSTREAM_DIR, META_DIR, "loader.json"), "r", encoding="utf-8" ) as loaderVersionIndexFile: loader_version_index = json.load(loaderVersionIndexFile) - for it in loader_version_index: - print(f"Downloading JAR info for loader {it['version']} ") - maven_url = get_maven_url( - it["maven"], "https://maven.quiltmc.org/repository/release/", ".json" - ) - get_json_file( - os.path.join(UPSTREAM_DIR, INSTALLER_INFO_DIR, f"{it['version']}.json"), - maven_url, - ) + with concurrent.futures.ThreadPoolExecutor() as executor: + for it in loader_version_index: + executor.submit(get_json_file_concurrent, it) if __name__ == "__main__": |