Concurrency (#57)

Co-authored-by: bloodnighttw <emails@bntw.dev>

3 files changed, 86 insertions, 59 deletions

diff --git a/meta/run/update_fabric.py b/meta/run/update_fabric.py
index 132495b92c..850ec8eac1 100755
--- a/meta/run/update_fabric.py
+++ b/meta/run/update_fabric.py
@@ -1,3 +1,4 @@
+import concurrent.futures
 import json
 import os
 import zipfile
@@ -93,6 +94,26 @@ def compute_jar_file(path, url):
     data.write(path + ".json")
 
 
+def compute_jar_file_concurrent(component, it):
+    print(f"Processing {component} {it['version']} ")
+    jar_maven_url = get_maven_url(it["maven"], "https://maven.fabricmc.net/", ".jar")
+    compute_jar_file(
+        os.path.join(UPSTREAM_DIR, JARS_DIR, transform_maven_key(it["maven"])),
+        jar_maven_url,
+    )
+    print(f"Processing {component} {it['version']} Done")
+
+
+def get_json_file_concurrent(it):
+    print(f"Downloading JAR info for loader {it['version']} ")
+    maven_url = get_maven_url(it["maven"], "https://maven.fabricmc.net/", ".json")
+    get_json_file(
+        os.path.join(UPSTREAM_DIR, INSTALLER_INFO_DIR, f"{it['version']}.json"),
+        maven_url,
+    )
+    print(f"Downloading JAR info for loader {it['version']} Done")
+
+
 def main():
     # get the version list for each component we are interested in
     for component in ["intermediary", "loader"]:
@@ -100,30 +121,18 @@ def main():
             os.path.join(UPSTREAM_DIR, META_DIR, f"{component}.json"),
             "https://meta.fabricmc.net/v2/versions/" + component,
         )
-        for it in index:
-            print(f"Processing {component} {it['version']} ")
-            jar_maven_url = get_maven_url(
-                it["maven"], "https://maven.fabricmc.net/", ".jar"
-            )
-            compute_jar_file(
-                os.path.join(UPSTREAM_DIR, JARS_DIR, transform_maven_key(it["maven"])),
-                jar_maven_url,
-            )
+        with concurrent.futures.ThreadPoolExecutor() as executor:
+            for it in index:
+                executor.submit(compute_jar_file_concurrent, component, it)
 
     # for each loader, download installer JSON file from maven
     with open(
         os.path.join(UPSTREAM_DIR, META_DIR, "loader.json"), "r", encoding="utf-8"
     ) as loaderVersionIndexFile:
         loader_version_index = json.load(loaderVersionIndexFile)
-        for it in loader_version_index:
-            print(f"Downloading JAR info for loader {it['version']} ")
-            maven_url = get_maven_url(
-                it["maven"], "https://maven.fabricmc.net/", ".json"
-            )
-            get_json_file(
-                os.path.join(UPSTREAM_DIR, INSTALLER_INFO_DIR, f"{it['version']}.json"),
-                maven_url,
-            )
+        with concurrent.futures.ThreadPoolExecutor() as executor:
+            for it in loader_version_index:
+                executor.submit(get_json_file_concurrent, it)
 
 
 if __name__ == "__main__":
diff --git a/meta/run/update_mojang.py b/meta/run/update_mojang.py
index 52921f3e8a..defc371c1c 100755
--- a/meta/run/update_mojang.py
+++ b/meta/run/update_mojang.py
@@ -1,3 +1,4 @@
+import concurrent.futures
 import json
 import os
 import zipfile
@@ -80,6 +81,28 @@ def fetch_version(path, url):
     return version_json
 
 
+def fetch_version_concurrent(remote_versions, x):
+    version = remote_versions.versions[x]
+    print(
+        "Updating "
+        + version.id
+        + " to timestamp "
+        + version.release_time.strftime("%s")
+    )
+    fetch_version(os.path.join(UPSTREAM_DIR, VERSIONS_DIR, f"{x}.json"), version.url)
+
+
+def fetch_modified_version_concurrent(old_snapshots, x):
+    version = old_snapshots.versions[x]
+    old_snapshots_path = os.path.join(UPSTREAM_DIR, VERSIONS_DIR, f"{x}.json")
+
+    print("Updating old snapshot " + version.id)
+    if not os.path.isfile(old_snapshots_path):
+        fetch_modified_version(old_snapshots_path, version)
+    else:
+        print("Already have old snapshot " + version.id)
+
+
 def main():
     # get the remote version list
     r = sess.get("https://piston-meta.mojang.com/mc/game/version_manifest_v2.json")
@@ -108,17 +131,9 @@ def main():
     else:
         pending_ids = remote_ids
 
-    for x in pending_ids:
-        version = remote_versions.versions[x]
-        print(
-            "Updating "
-            + version.id
-            + " to timestamp "
-            + version.release_time.strftime("%s")
-        )
-        fetch_version(
-            os.path.join(UPSTREAM_DIR, VERSIONS_DIR, f"{x}.json"), version.url
-        )
+    with concurrent.futures.ThreadPoolExecutor() as executor:
+        for x in pending_ids:
+            executor.submit(fetch_version_concurrent, remote_versions, x)
 
     # deal with experimental snapshots separately
     if os.path.exists(STATIC_EXPERIMENTS_FILE):
@@ -144,15 +159,9 @@ def main():
         )
         old_snapshots_ids = set(old_snapshots.versions.keys())
 
-        for x in old_snapshots_ids:
-            version = old_snapshots.versions[x]
-            old_snapshots_path = os.path.join(UPSTREAM_DIR, VERSIONS_DIR, f"{x}.json")
-
-            print("Updating old snapshot " + version.id)
-            if not os.path.isfile(old_snapshots_path):
-                fetch_modified_version(old_snapshots_path, version)
-            else:
-                print("Already have old snapshot " + version.id)
+        with concurrent.futures.ThreadPoolExecutor() as executor:
+            for x in old_snapshots_ids:
+                executor.submit(fetch_modified_version_concurrent, old_snapshots, x)
 
     remote_versions.index.write(version_manifest_path)
 
diff --git a/meta/run/update_quilt.py b/meta/run/update_quilt.py
index 5eaa8ed0d0..5f3c40caa8 100755
--- a/meta/run/update_quilt.py
+++ b/meta/run/update_quilt.py
@@ -1,10 +1,9 @@
+import concurrent.futures
 import json
 import os
 import zipfile
 from datetime import datetime
 
-import requests
-
 from meta.common import (
     upstream_path,
     ensure_upstream_dir,
@@ -12,7 +11,6 @@ from meta.common import (
     default_session,
 )
 from meta.common.quilt import JARS_DIR, INSTALLER_INFO_DIR, META_DIR, USE_QUILT_MAPPINGS
-from meta.common.fabric import DATETIME_FORMAT_HTTP
 from meta.model.fabric import FabricJarInfo
 
 UPSTREAM_DIR = upstream_path()
@@ -82,6 +80,29 @@ def compute_jar_file(path, url):
     data.write(path + ".json")
 
 
+def compute_jar_file_concurrent(component, it):
+    print(f"Processing {component} {it['version']} ")
+    jar_maven_url = get_maven_url(
+        it["maven"], "https://maven.quiltmc.org/repository/release/", ".jar"
+    )
+    compute_jar_file(
+        os.path.join(UPSTREAM_DIR, JARS_DIR, transform_maven_key(it["maven"])),
+        jar_maven_url,
+    )
+    print(f"Processing {component} {it['version']} Done")
+
+
+def get_json_file_concurrent(it):
+    print(f"Downloading JAR info for loader {it['version']} ")
+    maven_url = get_maven_url(
+        it["maven"], "https://maven.quiltmc.org/repository/release/", ".json"
+    )
+    get_json_file(
+        os.path.join(UPSTREAM_DIR, INSTALLER_INFO_DIR, f"{it['version']}.json"),
+        maven_url,
+    )
+
+
 def main():
     # get the version list for each component we are interested in
     components = ["loader"]
@@ -92,30 +113,18 @@ def main():
             os.path.join(UPSTREAM_DIR, META_DIR, f"{component}.json"),
             "https://meta.quiltmc.org/v3/versions/" + component,
         )
-        for it in index:
-            print(f"Processing {component} {it['version']} ")
-            jar_maven_url = get_maven_url(
-                it["maven"], "https://maven.quiltmc.org/repository/release/", ".jar"
-            )
-            compute_jar_file(
-                os.path.join(UPSTREAM_DIR, JARS_DIR, transform_maven_key(it["maven"])),
-                jar_maven_url,
-            )
+        with concurrent.futures.ThreadPoolExecutor() as executor:
+            for it in index:
+                executor.submit(compute_jar_file_concurrent, component, it)
 
     # for each loader, download installer JSON file from maven
     with open(
         os.path.join(UPSTREAM_DIR, META_DIR, "loader.json"), "r", encoding="utf-8"
     ) as loaderVersionIndexFile:
         loader_version_index = json.load(loaderVersionIndexFile)
-        for it in loader_version_index:
-            print(f"Downloading JAR info for loader {it['version']} ")
-            maven_url = get_maven_url(
-                it["maven"], "https://maven.quiltmc.org/repository/release/", ".json"
-            )
-            get_json_file(
-                os.path.join(UPSTREAM_DIR, INSTALLER_INFO_DIR, f"{it['version']}.json"),
-                maven_url,
-            )
+        with concurrent.futures.ThreadPoolExecutor() as executor:
+            for it in loader_version_index:
+                executor.submit(get_json_file_concurrent, it)
 
 
 if __name__ == "__main__":