harden concurrent task execution

Signed-off-by: Trial97 <alexandru.tripon97@gmail.com>

4 files changed, 31 insertions, 9 deletions

diff --git a/meta/run/update_forge.py b/meta/run/update_forge.py
index 59bdbeb29e..23ad333955 100755
--- a/meta/run/update_forge.py
+++ b/meta/run/update_forge.py
@@ -365,6 +365,7 @@ def main():
     print("Grabbing installers and dumping installer profiles...")
     # get the installer jars - if needed - and get the installer profiles out of them
     with concurrent.futures.ThreadPoolExecutor() as executor:
+        futures = []
         for key, entry in new_index.versions.items():
             eprint("Updating Forge %s" % key)
             if entry.mc_version is None:
@@ -382,7 +383,9 @@ def main():
             jar_path = os.path.join(UPSTREAM_DIR, JARS_DIR, version.filename())
 
             if version.uses_installer():
-                executor.submit(process_forge_version, version, jar_path)
+                futures.append(
+                    executor.submit(process_forge_version, version, jar_path)
+                )
             else:
                 # ignore the two versions without install manifests and jar mod class files
                 # TODO: fix those versions?
@@ -407,6 +410,8 @@ def main():
                     legacy_info.sha256 = file_hash(jar_path, hashlib.sha256)
                     legacy_info.size = os.path.getsize(jar_path)
                     legacy_info_list.number[key] = legacy_info
+        for f in futures:
+            f.result()
 
     # only write legacy info if it's missing
     if not os.path.isfile(LEGACYINFO_PATH):
diff --git a/meta/run/update_mojang.py b/meta/run/update_mojang.py
index acdbec95a4..c05b15ce46 100755
--- a/meta/run/update_mojang.py
+++ b/meta/run/update_mojang.py
@@ -150,8 +150,12 @@ def main():
         pending_ids = remote_ids
 
     with concurrent.futures.ThreadPoolExecutor() as executor:
-        for x in pending_ids:
+        futures = [
             executor.submit(fetch_version_concurrent, remote_versions, x)
+            for x in pending_ids
+        ]
+        for f in futures:
+            f.result()
 
     # deal with experimental snapshots separately
     if os.path.exists(STATIC_EXPERIMENTS_FILE):
@@ -178,8 +182,12 @@ def main():
         old_snapshots_ids = set(old_snapshots.versions.keys())
 
         with concurrent.futures.ThreadPoolExecutor() as executor:
-            for x in old_snapshots_ids:
+            futures = [
                 executor.submit(fetch_modified_version_concurrent, old_snapshots, x)
+                for x in old_snapshots_ids
+            ]
+            for f in futures:
+                f.result()
 
     remote_versions.index.write(version_manifest_path)
 
diff --git a/meta/run/update_neoforge.py b/meta/run/update_neoforge.py
index 5530540173..242ad8a6b1 100644
--- a/meta/run/update_neoforge.py
+++ b/meta/run/update_neoforge.py
@@ -118,9 +118,6 @@ def get_single_forge_files_manifest(longversion, artifact: str):
 
 def process_neoforge_version(key, entry):
     eprint("Updating NeoForge %s" % key)
-    if entry.mc_version is None:
-        eprint("Skipping %d with invalid MC version" % entry.build)
-        return
 
     version = NeoForgeVersion(entry)
     if version.url() is None:
@@ -311,8 +308,12 @@ def main():
     print("Grabbing installers and dumping installer profiles...")
     # get the installer jars - if needed - and get the installer profiles out of them
     with concurrent.futures.ThreadPoolExecutor() as executor:
-        for key, entry in new_index.versions.items():
+        futures = [
             executor.submit(process_neoforge_version, key, entry)
+            for key, entry in new_index.versions.items()
+        ]
+        for f in futures:
+            f.result()
 
 
 if __name__ == "__main__":
diff --git a/meta/run/update_quilt.py b/meta/run/update_quilt.py
index 2f3160265d..e694fd7b46 100755
--- a/meta/run/update_quilt.py
+++ b/meta/run/update_quilt.py
@@ -106,8 +106,12 @@ def main():
             "https://meta.quiltmc.org/v3/versions/" + component,
         )
         with concurrent.futures.ThreadPoolExecutor() as executor:
-            for it in index:
+            futures = [
                 executor.submit(compute_jar_file_concurrent, component, it)
+                for it in index
+            ]
+            for f in futures:
+                f.result()
 
     # for each loader, download installer JSON file from maven
     with open(
@@ -115,8 +119,12 @@ def main():
     ) as loaderVersionIndexFile:
         loader_version_index = json.load(loaderVersionIndexFile)
         with concurrent.futures.ThreadPoolExecutor() as executor:
-            for it in loader_version_index:
+            futures = [
                 executor.submit(get_json_file_concurrent, it)
+                for it in loader_version_index
+            ]
+            for f in futures:
+                f.result()
 
 
 if __name__ == "__main__":