feat: implement experiments updates

6 files changed, 180 insertions, 8 deletions

diff --git a/meta/common/__init__.py b/meta/common/__init__.py
index f0f620f4e9..478404362a 100644
--- a/meta/common/__init__.py
+++ b/meta/common/__init__.py
@@ -20,6 +20,12 @@ def upstream_path():
     return "upstream"
 
 
+def static_path():
+    if "STATIC_DIR" in os.environ:
+        return os.environ["STATIC_DIR"]
+    return "static"
+
+
 def ensure_upstream_dir(path):
     path = os.path.join(upstream_path(), path)
     if not os.path.exists(path):
diff --git a/meta/common/http.py b/meta/common/http.py
new file mode 100644
index 0000000000..c057e4b06f
--- /dev/null
+++ b/meta/common/http.py
@@ -0,0 +1,6 @@
+def download_binary_file(sess, path, url):
+    with open(path, 'wb') as f:
+        r = sess.get(url)
+        r.raise_for_status()
+        for chunk in r.iter_content(chunk_size=128):
+            f.write(chunk)
diff --git a/meta/common/mojang.py b/meta/common/mojang.py
index 953e3d1d37..1c16afc885 100644
--- a/meta/common/mojang.py
+++ b/meta/common/mojang.py
@@ -6,5 +6,7 @@ VERSION_MANIFEST_FILE = join(BASE_DIR, "version_manifest_v2.json")
 VERSIONS_DIR = join(BASE_DIR, "versions")
 ASSETS_DIR = join(BASE_DIR, "assets")
 
+STATIC_EXPERIMENTS_FILE = join(BASE_DIR, "minecraft-experiments.json")
+
 MINECRAFT_COMPONENT = ""
 LWJGL_COMPONENT = ""
diff --git a/meta/model/mojang.py b/meta/model/mojang.py
index 37a14b4435..fa00071dd7 100644
--- a/meta/model/mojang.py
+++ b/meta/model/mojang.py
@@ -1,5 +1,7 @@
 from datetime import datetime
-from typing import Optional, List
+from typing import Optional, List, Dict
+
+from pydantic import AnyHttpUrl
 
 from . import MetaBase
 
@@ -50,3 +52,19 @@ class MojangIndexWrap:
         self.index = index
         self.latest = index.latest
         self.versions = dict((x.id, x) for x in index.versions)
+
+
+class ExperimentEntry(MetaBase):
+    id: str
+    url: AnyHttpUrl
+    wiki: Optional[AnyHttpUrl]
+
+
+class ExperimentIndex(MetaBase):
+    experiments: List[ExperimentEntry]
+
+
+class ExperimentIndexWrap:
+    def __init__(self, index: ExperimentIndex):
+        self.index: ExperimentIndex = index
+        self.versions: Dict[str, ExperimentEntry] = dict((x.id, x) for x in index.experiments)
diff --git a/static/mojang/minecraft-experiments.json b/static/mojang/minecraft-experiments.json
new file mode 100644
index 0000000000..4b16b70601
--- /dev/null
+++ b/static/mojang/minecraft-experiments.json
@@ -0,0 +1,104 @@
+{
+  "experiments": [
+    {
+      "id": "1.19_deep_dark_experimental_snapshot-1",
+      "wiki": "https://minecraft.fandom.com/wiki/Java_Edition_Deep_Dark_Experimental_Snapshot_1",
+      "url": "https://launcher.mojang.com/v1/objects/b1e589c1d6ed73519797214bc796e53f5429ac46/1_19_deep_dark_experimental_snapshot-1.zip"
+    },
+    {
+      "id": "1.18_experimental-snapshot-7",
+      "wiki": "https://minecraft.fandom.com/wiki/Java_Edition_1.18_Experimental_Snapshot_7",
+      "url": "https://launcher.mojang.com/v1/objects/ab4ecebb133f56dd4c4c4c3257f030a947ddea84/1_18_experimental-snapshot-7.zip"
+    },
+    {
+      "id": "1.18_experimental-snapshot-6",
+      "wiki": "https://minecraft.fandom.com/wiki/Java_Edition_1.18_Experimental_Snapshot_6",
+      "url": "https://launcher.mojang.com/v1/objects/4697c84c6a347d0b8766759d5b00bc5a00b1b858/1_18_experimental-snapshot-6.zip"
+    },
+    {
+      "id": "1.18_experimental-snapshot-5",
+      "wiki": "https://minecraft.fandom.com/wiki/Java_Edition_1.18_Experimental_Snapshot_5",
+      "url": "https://launcher.mojang.com/v1/objects/d9cb7f6fb4e440862adfb40a385d83e3f8d154db/1_18_experimental-snapshot-5.zip"
+    },
+    {
+      "id": "1.18_experimental-snapshot-4",
+      "wiki": "https://minecraft.fandom.com/wiki/Java_Edition_1.18_Experimental_Snapshot_4",
+      "url": "https://launcher.mojang.com/v1/objects/b92a360cbae2eb896a62964ad8c06c3493b6c390/1_18_experimental-snapshot-4.zip"
+    },
+    {
+      "id": "1.18_experimental-snapshot-3",
+      "wiki": "https://minecraft.fandom.com/wiki/Java_Edition_1.18_Experimental_Snapshot_3",
+      "url": "https://launcher.mojang.com/v1/objects/846648ff9fe60310d584061261de43010e5c722b/1_18_experimental-snapshot-3.zip"
+    },
+    {
+      "id": "1.18_experimental-snapshot-2",
+      "wiki": "https://minecraft.fandom.com/wiki/Java_Edition_1.18_Experimental_Snapshot_2",
+      "url": "https://launcher.mojang.com/v1/objects/0adfe4f321aa45248fc88ac888bed5556633e7fb/1_18_experimental-snapshot-2.zip"
+    },
+    {
+      "id": "1.18_experimental-snapshot-1",
+      "wiki": "https://minecraft.fandom.com/wiki/Java_Edition_1.18_Experimental_Snapshot_1",
+      "url": "https://launcher.mojang.com/v1/objects/231bba2a21e18b8c60976e1f6110c053b7b93226/1_18_experimental-snapshot-1.zip"
+    },
+    {
+      "id": "1.16_combat-6",
+      "wiki": "https://minecraft.fandom.com/wiki/Java_Edition_Combat_Test_8c",
+      "url": "https://launcher.mojang.com/experiments/combat/ea08f7eb1f96cdc82464e27c0f95d23965083cfb/1_16_combat-6.zip"
+    },
+    {
+      "id": "1.16_combat-5",
+      "wiki": "https://minecraft.fandom.com/wiki/Java_Edition_Combat_Test_8b",
+      "url": "https://launcher.mojang.com/experiments/combat/9b2b984d635d373564b50803807225c75d7fd447/1_16_combat-5.zip"
+    },
+    {
+      "id": "1.16_combat-4",
+      "wiki": "https://minecraft.fandom.com/wiki/Java_Edition_Combat_Test_8",
+      "url": "https://cdn.discordapp.com/attachments/369990015096455168/947864881028272198/1_16_combat-4.zip"
+    },
+    {
+      "id": "1.16_combat-3",
+      "wiki": "https://minecraft.fandom.com/wiki/Java_Edition_Combat_Test_7c",
+      "url": "https://launcher.mojang.com/experiments/combat/2557b99d95588505e988886220779087d7d6b1e9/1_16_combat-3.zip"
+    },
+    {
+      "id": "1.16_combat-2",
+      "wiki": "https://minecraft.fandom.com/wiki/Java_Edition_Combat_Test_7b",
+      "url": "https://archive.org/download/Combat_Test_7ab/1_16_combat-2.zip"
+    },
+    {
+      "id": "1.16_combat-1",
+      "wiki": "https://minecraft.fandom.com/wiki/Java_Edition_Combat_Test_7",
+      "url": "https://archive.org/download/Combat_Test_7ab/1_16_combat-1.zip"
+    },
+    {
+      "id": "1.16_combat-0",
+      "wiki": "https://minecraft.fandom.com/wiki/Java_Edition_Combat_Test_6",
+      "url": "https://launcher.mojang.com/experiments/combat/5a8ceec8681ed96ab6ecb9607fb5d19c8a755559/1_16_combat-0.zip"
+    },
+    {
+      "id": "1.15_combat-6",
+      "wiki": "https://minecraft.fandom.com/wiki/Java_Edition_Combat_Test_5",
+      "url": "https://launcher.mojang.com/experiments/combat/52263d42a626b40c947e523128f7a195ec5af76a/1_15_combat-6.zip"
+    },
+    {
+      "id": "1.15_combat-1",
+      "wiki": "https://minecraft.fandom.com/wiki/Java_Edition_Combat_Test_4",
+      "url": "https://launcher.mojang.com/experiments/combat/ac11ea96f3bb2fa2b9b76ab1d20cacb1b1f7ef60/1_15_combat-1.zip"
+    },
+    {
+      "id": "1.14_combat-3",
+      "wiki": "https://minecraft.fandom.com/wiki/Java_Edition_Combat_Test_3",
+      "url": "https://launcher.mojang.com/experiments/combat/0f209c9c84b81c7d4c88b4632155b9ae550beb89/1_14_combat-3.zip"
+    },
+    {
+      "id": "1.14_combat-0",
+      "wiki": "https://minecraft.fandom.com/wiki/Java_Edition_Combat_Test_2",
+      "url": "https://launcher.mojang.com/experiments/combat/d164bb6ecc5fca9ac02878c85f11befae61ac1ca/1_14_combat-0.zip"
+    },
+    {
+      "id": "1.14_combat-212796",
+      "wiki": "https://minecraft.fandom.com/wiki/Java_Edition_1.14.3_-_Combat_Test",
+      "url": "https://launcher.mojang.com/experiments/combat/610f5c9874ba8926d5ae1bcce647e5f0e6e7c889/1_14_combat-212796.zip"
+    }
+  ]
+}
\ No newline at end of file
diff --git a/updateMojang.py b/updateMojang.py
index 965ddb5b2f..54d5d739be 100755
--- a/updateMojang.py
+++ b/updateMojang.py
@@ -1,15 +1,18 @@
 import json
 import os.path
+import zipfile
 
 import requests
 from cachecontrol import CacheControl
 from cachecontrol.caches import FileCache
 
-from meta.common import upstream_path, ensure_upstream_dir
-from meta.common.mojang import BASE_DIR, VERSION_MANIFEST_FILE, VERSIONS_DIR, ASSETS_DIR
-from meta.model.mojang import MojangIndexWrap, MojangIndex
+from meta.common import upstream_path, ensure_upstream_dir, static_path
+from meta.common.http import download_binary_file
+from meta.common.mojang import BASE_DIR, VERSION_MANIFEST_FILE, VERSIONS_DIR, ASSETS_DIR, STATIC_EXPERIMENTS_FILE
+from meta.model.mojang import MojangIndexWrap, MojangIndex, ExperimentIndex, ExperimentIndexWrap
 
 UPSTREAM_DIR = upstream_path()
+STATIC_DIR = static_path()
 
 ensure_upstream_dir(BASE_DIR)
 ensure_upstream_dir(VERSIONS_DIR)
@@ -26,6 +29,26 @@ def fetch_version_file(path, url):
     return asset_id, asset_url
 
 
+def fetch_zipped_version_file(path, url):
+    zip_path = f"{path}.zip"
+    download_binary_file(sess, zip_path, url)
+    with zipfile.ZipFile(zip_path, 'r') as z:
+        for info in z.infolist():
+            if info.filename.endswith(".json"):
+                print(f"Found {info.filename} as version json")
+                version_json = json.load(z.open(info))
+                break
+
+    assert version_json
+
+    with open(path, 'w', encoding='utf-8') as f:
+        json.dump(version_json, f, sort_keys=True, indent=4)
+
+    asset_id = version_json["assetIndex"]["id"]
+    asset_url = version_json["assetIndex"]["url"]
+    return asset_id, asset_url
+
+
 def fetch_file(path, url):
     r = sess.get(url)
     r.raise_for_status()
@@ -45,11 +68,11 @@ def main():
     remote_versions = MojangIndexWrap(MojangIndex(**r.json()))
     remote_ids = set(remote_versions.versions.keys())
 
-    version_manifest = os.path.join(UPSTREAM_DIR, VERSION_MANIFEST_FILE)
+    version_manifest_path = os.path.join(UPSTREAM_DIR, VERSION_MANIFEST_FILE)
 
-    if os.path.exists(version_manifest):
+    if os.path.exists(version_manifest_path):
         # get the local version list
-        current_versions = MojangIndexWrap(MojangIndex.parse_file(version_manifest))
+        current_versions = MojangIndexWrap(MojangIndex.parse_file(version_manifest_path))
         local_ids = set(current_versions.versions.keys())
 
         # versions not present locally but present remotely are new
@@ -71,11 +94,24 @@ def main():
         asset_id, asset_url = fetch_version_file(os.path.join(UPSTREAM_DIR, VERSIONS_DIR, f"{x}.json"), version.url)
         assets[asset_id] = asset_url
 
+    # deal with experimental snapshots separately
+    static_experiments_path = os.path.join(STATIC_DIR, STATIC_EXPERIMENTS_FILE)
+    if os.path.exists(static_experiments_path):
+        experiments = ExperimentIndexWrap(ExperimentIndex.parse_file(static_experiments_path))
+        experiment_ids = set(experiments.versions.keys())
+
+        for x in experiment_ids:
+            version = experiments.versions[x]
+            print("Updating experiment " + version.id)
+            asset_id, asset_url = fetch_zipped_version_file(os.path.join(UPSTREAM_DIR, VERSIONS_DIR, f"{x}.json"),
+                                                            version.url)
+            assets[asset_id] = asset_url
+
     for asset_id, asset_url in assets.items():
         print("assets", asset_id, asset_url)
         fetch_file(os.path.join(UPSTREAM_DIR, ASSETS_DIR, f"{asset_id}.json"), asset_url)
 
-    remote_versions.index.write(version_manifest)
+    remote_versions.index.write(version_manifest_path)
 
 
 if __name__ == '__main__':