From 5895e5accfbb1268500f9dd35003abf207a548c8 Mon Sep 17 00:00:00 2001 From: Sefa Eyeoglu Date: Sat, 2 Apr 2022 16:49:31 +0200 Subject: feat: implement experiments updates --- meta/common/__init__.py | 6 ++ meta/common/http.py | 6 ++ meta/common/mojang.py | 2 + meta/model/mojang.py | 20 +++++- static/mojang/minecraft-experiments.json | 104 +++++++++++++++++++++++++++++++ updateMojang.py | 50 ++++++++++++--- 6 files changed, 180 insertions(+), 8 deletions(-) create mode 100644 meta/common/http.py create mode 100644 static/mojang/minecraft-experiments.json diff --git a/meta/common/__init__.py b/meta/common/__init__.py index f0f620f4e9..478404362a 100644 --- a/meta/common/__init__.py +++ b/meta/common/__init__.py @@ -20,6 +20,12 @@ def upstream_path(): return "upstream" +def static_path(): + if "STATIC_DIR" in os.environ: + return os.environ["STATIC_DIR"] + return "static" + + def ensure_upstream_dir(path): path = os.path.join(upstream_path(), path) if not os.path.exists(path): diff --git a/meta/common/http.py b/meta/common/http.py new file mode 100644 index 0000000000..c057e4b06f --- /dev/null +++ b/meta/common/http.py @@ -0,0 +1,6 @@ +def download_binary_file(sess, path, url): + with open(path, 'wb') as f: + r = sess.get(url) + r.raise_for_status() + for chunk in r.iter_content(chunk_size=128): + f.write(chunk) diff --git a/meta/common/mojang.py b/meta/common/mojang.py index 953e3d1d37..1c16afc885 100644 --- a/meta/common/mojang.py +++ b/meta/common/mojang.py @@ -6,5 +6,7 @@ VERSION_MANIFEST_FILE = join(BASE_DIR, "version_manifest_v2.json") VERSIONS_DIR = join(BASE_DIR, "versions") ASSETS_DIR = join(BASE_DIR, "assets") +STATIC_EXPERIMENTS_FILE = join(BASE_DIR, "minecraft-experiments.json") + MINECRAFT_COMPONENT = "" LWJGL_COMPONENT = "" diff --git a/meta/model/mojang.py b/meta/model/mojang.py index 37a14b4435..fa00071dd7 100644 --- a/meta/model/mojang.py +++ b/meta/model/mojang.py @@ -1,5 +1,7 @@ from datetime import datetime -from typing import Optional, List +from typing import Optional, List, Dict + +from pydantic import AnyHttpUrl from . import MetaBase @@ -50,3 +52,19 @@ class MojangIndexWrap: self.index = index self.latest = index.latest self.versions = dict((x.id, x) for x in index.versions) + + +class ExperimentEntry(MetaBase): + id: str + url: AnyHttpUrl + wiki: Optional[AnyHttpUrl] + + +class ExperimentIndex(MetaBase): + experiments: List[ExperimentEntry] + + +class ExperimentIndexWrap: + def __init__(self, index: ExperimentIndex): + self.index: ExperimentIndex = index + self.versions: Dict[str, ExperimentEntry] = dict((x.id, x) for x in index.experiments) diff --git a/static/mojang/minecraft-experiments.json b/static/mojang/minecraft-experiments.json new file mode 100644 index 0000000000..4b16b70601 --- /dev/null +++ b/static/mojang/minecraft-experiments.json @@ -0,0 +1,104 @@ +{ + "experiments": [ + { + "id": "1.19_deep_dark_experimental_snapshot-1", + "wiki": "https://minecraft.fandom.com/wiki/Java_Edition_Deep_Dark_Experimental_Snapshot_1", + "url": "https://launcher.mojang.com/v1/objects/b1e589c1d6ed73519797214bc796e53f5429ac46/1_19_deep_dark_experimental_snapshot-1.zip" + }, + { + "id": "1.18_experimental-snapshot-7", + "wiki": "https://minecraft.fandom.com/wiki/Java_Edition_1.18_Experimental_Snapshot_7", + "url": "https://launcher.mojang.com/v1/objects/ab4ecebb133f56dd4c4c4c3257f030a947ddea84/1_18_experimental-snapshot-7.zip" + }, + { + "id": "1.18_experimental-snapshot-6", + "wiki": "https://minecraft.fandom.com/wiki/Java_Edition_1.18_Experimental_Snapshot_6", + "url": "https://launcher.mojang.com/v1/objects/4697c84c6a347d0b8766759d5b00bc5a00b1b858/1_18_experimental-snapshot-6.zip" + }, + { + "id": "1.18_experimental-snapshot-5", + "wiki": "https://minecraft.fandom.com/wiki/Java_Edition_1.18_Experimental_Snapshot_5", + "url": "https://launcher.mojang.com/v1/objects/d9cb7f6fb4e440862adfb40a385d83e3f8d154db/1_18_experimental-snapshot-5.zip" + }, + { + "id": "1.18_experimental-snapshot-4", + "wiki": "https://minecraft.fandom.com/wiki/Java_Edition_1.18_Experimental_Snapshot_4", + "url": "https://launcher.mojang.com/v1/objects/b92a360cbae2eb896a62964ad8c06c3493b6c390/1_18_experimental-snapshot-4.zip" + }, + { + "id": "1.18_experimental-snapshot-3", + "wiki": "https://minecraft.fandom.com/wiki/Java_Edition_1.18_Experimental_Snapshot_3", + "url": "https://launcher.mojang.com/v1/objects/846648ff9fe60310d584061261de43010e5c722b/1_18_experimental-snapshot-3.zip" + }, + { + "id": "1.18_experimental-snapshot-2", + "wiki": "https://minecraft.fandom.com/wiki/Java_Edition_1.18_Experimental_Snapshot_2", + "url": "https://launcher.mojang.com/v1/objects/0adfe4f321aa45248fc88ac888bed5556633e7fb/1_18_experimental-snapshot-2.zip" + }, + { + "id": "1.18_experimental-snapshot-1", + "wiki": "https://minecraft.fandom.com/wiki/Java_Edition_1.18_Experimental_Snapshot_1", + "url": "https://launcher.mojang.com/v1/objects/231bba2a21e18b8c60976e1f6110c053b7b93226/1_18_experimental-snapshot-1.zip" + }, + { + "id": "1.16_combat-6", + "wiki": "https://minecraft.fandom.com/wiki/Java_Edition_Combat_Test_8c", + "url": "https://launcher.mojang.com/experiments/combat/ea08f7eb1f96cdc82464e27c0f95d23965083cfb/1_16_combat-6.zip" + }, + { + "id": "1.16_combat-5", + "wiki": "https://minecraft.fandom.com/wiki/Java_Edition_Combat_Test_8b", + "url": "https://launcher.mojang.com/experiments/combat/9b2b984d635d373564b50803807225c75d7fd447/1_16_combat-5.zip" + }, + { + "id": "1.16_combat-4", + "wiki": "https://minecraft.fandom.com/wiki/Java_Edition_Combat_Test_8", + "url": "https://cdn.discordapp.com/attachments/369990015096455168/947864881028272198/1_16_combat-4.zip" + }, + { + "id": "1.16_combat-3", + "wiki": "https://minecraft.fandom.com/wiki/Java_Edition_Combat_Test_7c", + "url": "https://launcher.mojang.com/experiments/combat/2557b99d95588505e988886220779087d7d6b1e9/1_16_combat-3.zip" + }, + { + "id": "1.16_combat-2", + "wiki": "https://minecraft.fandom.com/wiki/Java_Edition_Combat_Test_7b", + "url": "https://archive.org/download/Combat_Test_7ab/1_16_combat-2.zip" + }, + { + "id": "1.16_combat-1", + "wiki": "https://minecraft.fandom.com/wiki/Java_Edition_Combat_Test_7", + "url": "https://archive.org/download/Combat_Test_7ab/1_16_combat-1.zip" + }, + { + "id": "1.16_combat-0", + "wiki": "https://minecraft.fandom.com/wiki/Java_Edition_Combat_Test_6", + "url": "https://launcher.mojang.com/experiments/combat/5a8ceec8681ed96ab6ecb9607fb5d19c8a755559/1_16_combat-0.zip" + }, + { + "id": "1.15_combat-6", + "wiki": "https://minecraft.fandom.com/wiki/Java_Edition_Combat_Test_5", + "url": "https://launcher.mojang.com/experiments/combat/52263d42a626b40c947e523128f7a195ec5af76a/1_15_combat-6.zip" + }, + { + "id": "1.15_combat-1", + "wiki": "https://minecraft.fandom.com/wiki/Java_Edition_Combat_Test_4", + "url": "https://launcher.mojang.com/experiments/combat/ac11ea96f3bb2fa2b9b76ab1d20cacb1b1f7ef60/1_15_combat-1.zip" + }, + { + "id": "1.14_combat-3", + "wiki": "https://minecraft.fandom.com/wiki/Java_Edition_Combat_Test_3", + "url": "https://launcher.mojang.com/experiments/combat/0f209c9c84b81c7d4c88b4632155b9ae550beb89/1_14_combat-3.zip" + }, + { + "id": "1.14_combat-0", + "wiki": "https://minecraft.fandom.com/wiki/Java_Edition_Combat_Test_2", + "url": "https://launcher.mojang.com/experiments/combat/d164bb6ecc5fca9ac02878c85f11befae61ac1ca/1_14_combat-0.zip" + }, + { + "id": "1.14_combat-212796", + "wiki": "https://minecraft.fandom.com/wiki/Java_Edition_1.14.3_-_Combat_Test", + "url": "https://launcher.mojang.com/experiments/combat/610f5c9874ba8926d5ae1bcce647e5f0e6e7c889/1_14_combat-212796.zip" + } + ] +} \ No newline at end of file diff --git a/updateMojang.py b/updateMojang.py index 965ddb5b2f..54d5d739be 100755 --- a/updateMojang.py +++ b/updateMojang.py @@ -1,15 +1,18 @@ import json import os.path +import zipfile import requests from cachecontrol import CacheControl from cachecontrol.caches import FileCache -from meta.common import upstream_path, ensure_upstream_dir -from meta.common.mojang import BASE_DIR, VERSION_MANIFEST_FILE, VERSIONS_DIR, ASSETS_DIR -from meta.model.mojang import MojangIndexWrap, MojangIndex +from meta.common import upstream_path, ensure_upstream_dir, static_path +from meta.common.http import download_binary_file +from meta.common.mojang import BASE_DIR, VERSION_MANIFEST_FILE, VERSIONS_DIR, ASSETS_DIR, STATIC_EXPERIMENTS_FILE +from meta.model.mojang import MojangIndexWrap, MojangIndex, ExperimentIndex, ExperimentIndexWrap UPSTREAM_DIR = upstream_path() +STATIC_DIR = static_path() ensure_upstream_dir(BASE_DIR) ensure_upstream_dir(VERSIONS_DIR) @@ -26,6 +29,26 @@ def fetch_version_file(path, url): return asset_id, asset_url +def fetch_zipped_version_file(path, url): + zip_path = f"{path}.zip" + download_binary_file(sess, zip_path, url) + with zipfile.ZipFile(zip_path, 'r') as z: + for info in z.infolist(): + if info.filename.endswith(".json"): + print(f"Found {info.filename} as version json") + version_json = json.load(z.open(info)) + break + + assert version_json + + with open(path, 'w', encoding='utf-8') as f: + json.dump(version_json, f, sort_keys=True, indent=4) + + asset_id = version_json["assetIndex"]["id"] + asset_url = version_json["assetIndex"]["url"] + return asset_id, asset_url + + def fetch_file(path, url): r = sess.get(url) r.raise_for_status() @@ -45,11 +68,11 @@ def main(): remote_versions = MojangIndexWrap(MojangIndex(**r.json())) remote_ids = set(remote_versions.versions.keys()) - version_manifest = os.path.join(UPSTREAM_DIR, VERSION_MANIFEST_FILE) + version_manifest_path = os.path.join(UPSTREAM_DIR, VERSION_MANIFEST_FILE) - if os.path.exists(version_manifest): + if os.path.exists(version_manifest_path): # get the local version list - current_versions = MojangIndexWrap(MojangIndex.parse_file(version_manifest)) + current_versions = MojangIndexWrap(MojangIndex.parse_file(version_manifest_path)) local_ids = set(current_versions.versions.keys()) # versions not present locally but present remotely are new @@ -71,11 +94,24 @@ def main(): asset_id, asset_url = fetch_version_file(os.path.join(UPSTREAM_DIR, VERSIONS_DIR, f"{x}.json"), version.url) assets[asset_id] = asset_url + # deal with experimental snapshots separately + static_experiments_path = os.path.join(STATIC_DIR, STATIC_EXPERIMENTS_FILE) + if os.path.exists(static_experiments_path): + experiments = ExperimentIndexWrap(ExperimentIndex.parse_file(static_experiments_path)) + experiment_ids = set(experiments.versions.keys()) + + for x in experiment_ids: + version = experiments.versions[x] + print("Updating experiment " + version.id) + asset_id, asset_url = fetch_zipped_version_file(os.path.join(UPSTREAM_DIR, VERSIONS_DIR, f"{x}.json"), + version.url) + assets[asset_id] = asset_url + for asset_id, asset_url in assets.items(): print("assets", asset_id, asset_url) fetch_file(os.path.join(UPSTREAM_DIR, ASSETS_DIR, f"{asset_id}.json"), asset_url) - remote_versions.index.write(version_manifest) + remote_versions.index.write(version_manifest_path) if __name__ == '__main__': -- cgit 0.0.5-2-1-g0f52