make forge and neoforge concurrent

Signed-off-by: Trial97 <alexandru.tripon97@gmail.com>

7 files changed, 299 insertions, 289 deletions

diff --git a/meta/model/java.py b/meta/model/java.py
index 4a7571e491..c3efde9292 100644
--- a/meta/model/java.py
+++ b/meta/model/java.py
@@ -201,7 +201,9 @@ class AdoptxOs(StrEnum):
 
 ADOPTIUM_API_BASE = "https://api.adoptium.net"
 OPENJ9_API_BASE = " https://api.adoptopenjdk.net"
-ADOPTX_API_FEATURE_RELEASES = f"{{base_url}}/v3/assets/feature_releases/{{feature_version}}/{{release_type}}"
+ADOPTX_API_FEATURE_RELEASES = (
+    f"{{base_url}}/v3/assets/feature_releases/{{feature_version}}/{{release_type}}"
+)
 # ?image_type={{image_type}}&heap_size={{heap_size}}&project={{project}}&vendor={{vendor}}&page_size={{page_size}}&page={{page}}&sort_method={{sort_method}}&sort_order={{sort_order}}
 ADOPTX_API_AVAILABLE_RELEASES = f"{{base_url}}/v3/info/available_releases"
 
diff --git a/meta/model/neoforge.py b/meta/model/neoforge.py
index d2e4fe8846..ccdb325a23 100644
--- a/meta/model/neoforge.py
+++ b/meta/model/neoforge.py
@@ -46,6 +46,7 @@ class NeoForgeMCVersionInfo(MetaBase):
 class DerivedNeoForgeIndex(MetaBase):
     versions: Dict[str, NeoForgeEntry] = Field({})
 
+
 class FMLLib(
     MetaBase
 ):  # old ugly stuff. Maybe merge this with Library or Library later
diff --git a/meta/run/generate_java.py b/meta/run/generate_java.py
index 257326542a..4c6c261719 100644
--- a/meta/run/generate_java.py
+++ b/meta/run/generate_java.py
@@ -344,12 +344,10 @@ def main():
                 and major in [8, 17, 21, 25]
             )
             or (
-                runtime.runtime_os 
-                in [
-                    JavaRuntimeOS.LinuxX86,
-                    JavaRuntimeOS.LinuxRiscv64
-                ]  
-                and major in [17, 21, 25])
+                runtime.runtime_os
+                in [JavaRuntimeOS.LinuxX86, JavaRuntimeOS.LinuxRiscv64]
+                and major in [17, 21, 25]
+            )
         ):
             if major not in extra_mojang_javas:
                 extra_mojang_javas[major] = list[JavaRuntimeMeta]()
@@ -358,9 +356,7 @@ def main():
     print("Processing Adoptium Releases")
     adoptium_path = os.path.join(UPSTREAM_DIR, ADOPTIUM_DIR, "available_releases.json")
     if os.path.exists(adoptium_path):
-        adoptium_available_releases = AdoptxAvailableReleases.parse_file(
-            adoptium_path
-        )
+        adoptium_available_releases = AdoptxAvailableReleases.parse_file(adoptium_path)
         for major in adoptium_available_releases.available_releases:
             adoptium_releases = AdoptxReleases.parse_file(
                 os.path.join(UPSTREAM_DIR, ADOPTIUM_VERSIONS_DIR, f"java{major}.json")
@@ -389,9 +385,7 @@ def main():
     print("Processing OpenJ9 Releases")
     openj9_path = os.path.join(UPSTREAM_DIR, OPENJ9_DIR, "available_releases.json")
     if os.path.exists(openj9_path):
-        openj9_available_releases = AdoptxAvailableReleases.parse_file(
-            openj9_path
-        )
+        openj9_available_releases = AdoptxAvailableReleases.parse_file(openj9_path)
         for major in openj9_available_releases.available_releases:
             openj9_releases = AdoptxReleases.parse_file(
                 os.path.join(UPSTREAM_DIR, OPENJ9_VERSIONS_DIR, f"java{major}.json")
diff --git a/meta/run/generate_neoforge.py b/meta/run/generate_neoforge.py
index 5c6f965435..de32256869 100644
--- a/meta/run/generate_neoforge.py
+++ b/meta/run/generate_neoforge.py
@@ -139,20 +139,18 @@ def main():
         installer = MojangVersion.parse_file(installer_version_filepath)
         profile = NeoForgeInstallerProfileV2.parse_file(profile_filepath)
         v = version_from_build_system_installer(installer, profile, version)
-        
-        #we can get the minecraft version from the profile json info, so let's just do that instead of hacky regex
-        v.requires = [Dependency(uid=MINECRAFT_COMPONENT, equals=profile.minecraft)] 
+
+        # we can get the minecraft version from the profile json info, so let's just do that instead of hacky regex
+        v.requires = [Dependency(uid=MINECRAFT_COMPONENT, equals=profile.minecraft)]
         # If we do not have the corresponding Minecraft version, we ignore it
         if not os.path.isfile(
-            os.path.join(
-                LAUNCHER_DIR, MINECRAFT_COMPONENT, f"{profile.minecraft}.json"
-            )
+            os.path.join(LAUNCHER_DIR, MINECRAFT_COMPONENT, f"{profile.minecraft}.json")
         ):
             eprint(
                 "Skipping %s with no corresponding Minecraft version %s"
                 % (key, profile.minecraft)
             )
-            continue     
+            continue
         v.write(os.path.join(LAUNCHER_DIR, NEOFORGE_COMPONENT, f"{v.version}.json"))
 
         recommended_versions.sort()
diff --git a/meta/run/update_forge.py b/meta/run/update_forge.py
index 7ec9e45023..59bdbeb29e 100755
--- a/meta/run/update_forge.py
+++ b/meta/run/update_forge.py
@@ -1,7 +1,8 @@
 """
- Get the source files necessary for generating Forge versions
+Get the source files necessary for generating Forge versions
 """
 
+import concurrent.futures
 import copy
 import hashlib
 import json
@@ -136,6 +137,109 @@ def get_single_forge_files_manifest(longversion):
     return ret_dict
 
 
+def process_forge_version(version, jar_path):
+    installer_info_path = (
+        UPSTREAM_DIR + "/forge/installer_info/%s.json" % version.long_version
+    )
+    profile_path = (
+        UPSTREAM_DIR + "/forge/installer_manifests/%s.json" % version.long_version
+    )
+    version_file_path = (
+        UPSTREAM_DIR + "/forge/version_manifests/%s.json" % version.long_version
+    )
+
+    new_sha1 = None
+    sha1_file = jar_path + ".sha1"
+    if not os.path.isfile(jar_path):
+        remove_files([profile_path, installer_info_path])
+    else:
+        fileSha1 = get_file_sha1_from_file(jar_path, sha1_file)
+        try:
+            rfile = sess.get(version.url() + ".sha1")
+            rfile.raise_for_status()
+            new_sha1 = rfile.text.strip()
+            if fileSha1 != new_sha1:
+                remove_files([jar_path, profile_path, installer_info_path, sha1_file])
+        except Exception as e:
+            eprint("Failed to check sha1 %s" % version.url())
+            eprint("Error is %s" % e)
+
+    installer_refresh_required = not os.path.isfile(profile_path) or not os.path.isfile(
+        installer_info_path
+    )
+
+    if installer_refresh_required:
+        # grab the installer if it's not there
+        if not os.path.isfile(jar_path):
+            eprint("Downloading %s" % version.url())
+            download_binary_file(sess, jar_path, version.url())
+            if new_sha1 is None:
+                try:
+                    rfile = sess.get(version.url() + ".sha1")
+                    rfile.raise_for_status()
+                    new_sha1 = rfile.text.strip()
+                except Exception as e:
+                    eprint("Failed to download new sha1 %s" % version.url())
+                    eprint("Error is %s" % e)
+            if new_sha1 is not None:  # this is in case the fetch failed
+                with open(sha1_file, "w") as file:
+                    file.write(new_sha1)
+
+    eprint("Processing %s" % version.url())
+    # harvestables from the installer
+    if not os.path.isfile(profile_path):
+        print(jar_path)
+        with zipfile.ZipFile(jar_path) as jar:
+            with suppress(KeyError):
+                with jar.open("version.json") as profile_zip_entry:
+                    version_data = profile_zip_entry.read()
+
+                    # Process: does it parse?
+                    MojangVersion.parse_raw(version_data)
+
+                    with open(version_file_path, "wb") as versionJsonFile:
+                        versionJsonFile.write(version_data)
+                        versionJsonFile.close()
+
+            with jar.open("install_profile.json") as profile_zip_entry:
+                install_profile_data = profile_zip_entry.read()
+
+                # Process: does it parse?
+                is_parsable = False
+                exception = None
+                try:
+                    ForgeInstallerProfile.parse_raw(install_profile_data)
+                    is_parsable = True
+                except ValidationError as err:
+                    exception = err
+                try:
+                    ForgeInstallerProfileV2.parse_raw(install_profile_data)
+                    is_parsable = True
+                except ValidationError as err:
+                    exception = err
+
+                if not is_parsable:
+                    if version.is_supported():
+                        raise exception
+                    else:
+                        eprint(
+                            "Version %s is not supported and won't be generated later."
+                            % version.long_version
+                        )
+
+                with open(profile_path, "wb") as profileFile:
+                    profileFile.write(install_profile_data)
+                    profileFile.close()
+
+    # installer info v1
+    if not os.path.isfile(installer_info_path):
+        installer_info = InstallerInfo()
+        installer_info.sha1hash = file_hash(jar_path, hashlib.sha1)
+        installer_info.sha256hash = file_hash(jar_path, hashlib.sha256)
+        installer_info.size = os.path.getsize(jar_path)
+        installer_info.write(installer_info_path)
+
+
 def main():
     # get the remote version list fragments
     r = sess.get(
@@ -260,150 +364,49 @@ def main():
 
     print("Grabbing installers and dumping installer profiles...")
     # get the installer jars - if needed - and get the installer profiles out of them
-    for key, entry in new_index.versions.items():
-        eprint("Updating Forge %s" % key)
-        if entry.mc_version is None:
-            eprint("Skipping %d with invalid MC version" % entry.build)
-            continue
-
-        version = ForgeVersion(entry)
-        if version.url() is None:
-            eprint("Skipping %d with no valid files" % version.build)
-            continue
-        if version.long_version in BAD_VERSIONS:
-            eprint(f"Skipping bad version {version.long_version}")
-            continue
+    with concurrent.futures.ThreadPoolExecutor() as executor:
+        for key, entry in new_index.versions.items():
+            eprint("Updating Forge %s" % key)
+            if entry.mc_version is None:
+                eprint("Skipping %d with invalid MC version" % entry.build)
+                continue
 
-        jar_path = os.path.join(UPSTREAM_DIR, JARS_DIR, version.filename())
+            version = ForgeVersion(entry)
+            if version.url() is None:
+                eprint("Skipping %d with no valid files" % version.build)
+                continue
+            if version.long_version in BAD_VERSIONS:
+                eprint(f"Skipping bad version {version.long_version}")
+                continue
 
-        if version.uses_installer():
-            installer_info_path = (
-                UPSTREAM_DIR + "/forge/installer_info/%s.json" % version.long_version
-            )
-            profile_path = (
-                UPSTREAM_DIR
-                + "/forge/installer_manifests/%s.json" % version.long_version
-            )
-            version_file_path = (
-                UPSTREAM_DIR + "/forge/version_manifests/%s.json" % version.long_version
-            )
+            jar_path = os.path.join(UPSTREAM_DIR, JARS_DIR, version.filename())
 
-            new_sha1 = None
-            sha1_file = jar_path + ".sha1"
-            if not os.path.isfile(jar_path):
-                remove_files([profile_path, installer_info_path])
+            if version.uses_installer():
+                executor.submit(process_forge_version, version, jar_path)
             else:
-                fileSha1 = get_file_sha1_from_file(jar_path, sha1_file)
-                try:
-                    rfile = sess.get(version.url() + ".sha1")
-                    rfile.raise_for_status()
-                    new_sha1 = rfile.text.strip()
-                    if fileSha1 != new_sha1:
-                        remove_files(
-                            [jar_path, profile_path, installer_info_path, sha1_file]
-                        )
-                except Exception as e:
-                    eprint("Failed to check sha1 %s" % version.url())
-                    eprint("Error is %s" % e)
-
-            installer_refresh_required = not os.path.isfile(
-                profile_path
-            ) or not os.path.isfile(installer_info_path)
-
-            if installer_refresh_required:
-                # grab the installer if it's not there
-                if not os.path.isfile(jar_path):
-                    eprint("Downloading %s" % version.url())
-                    download_binary_file(sess, jar_path, version.url())
-                    if new_sha1 is None:
-                        try:
-                            rfile = sess.get(version.url() + ".sha1")
-                            rfile.raise_for_status()
-                            new_sha1 = rfile.text.strip()
-                        except Exception as e:
-                            eprint("Failed to download new sha1 %s" % version.url())
-                            eprint("Error is %s" % e)
-                    if new_sha1 is not None:  # this is in case the fetch failed
-                        with open(sha1_file, "w") as file:
-                            file.write(new_sha1)
-
-            eprint("Processing %s" % version.url())
-            # harvestables from the installer
-            if not os.path.isfile(profile_path):
-                print(jar_path)
-                with zipfile.ZipFile(jar_path) as jar:
-                    with suppress(KeyError):
-                        with jar.open("version.json") as profile_zip_entry:
-                            version_data = profile_zip_entry.read()
-
-                            # Process: does it parse?
-                            MojangVersion.parse_raw(version_data)
-
-                            with open(version_file_path, "wb") as versionJsonFile:
-                                versionJsonFile.write(version_data)
-                                versionJsonFile.close()
-
-                    with jar.open("install_profile.json") as profile_zip_entry:
-                        install_profile_data = profile_zip_entry.read()
-
-                        # Process: does it parse?
-                        is_parsable = False
-                        exception = None
-                        try:
-                            ForgeInstallerProfile.parse_raw(install_profile_data)
-                            is_parsable = True
-                        except ValidationError as err:
-                            exception = err
-                        try:
-                            ForgeInstallerProfileV2.parse_raw(install_profile_data)
-                            is_parsable = True
-                        except ValidationError as err:
-                            exception = err
-
-                        if not is_parsable:
-                            if version.is_supported():
-                                raise exception
-                            else:
-                                eprint(
-                                    "Version %s is not supported and won't be generated later."
-                                    % version.long_version
-                                )
-
-                        with open(profile_path, "wb") as profileFile:
-                            profileFile.write(install_profile_data)
-                            profileFile.close()
-
-            # installer info v1
-            if not os.path.isfile(installer_info_path):
-                installer_info = InstallerInfo()
-                installer_info.sha1hash = file_hash(jar_path, hashlib.sha1)
-                installer_info.sha256hash = file_hash(jar_path, hashlib.sha256)
-                installer_info.size = os.path.getsize(jar_path)
-                installer_info.write(installer_info_path)
-        else:
-            # ignore the two versions without install manifests and jar mod class files
-            # TODO: fix those versions?
-            if version.mc_version_sane == "1.6.1":
-                continue
-
-            # only gather legacy info if it's missing
-            if not os.path.isfile(LEGACYINFO_PATH):
-                # grab the jar/zip if it's not there
-                if not os.path.isfile(jar_path):
-                    download_binary_file(sess, jar_path, version.url())
-                # find the latest timestamp in the zip file
-                tstamp = datetime.fromtimestamp(0)
-                with zipfile.ZipFile(jar_path) as jar:
-                    for info in jar.infolist():
-                        tstamp_new = datetime(*info.date_time)
-                        if tstamp_new > tstamp:
-                            tstamp = tstamp_new
-                legacy_info = ForgeLegacyInfo()
-                legacy_info.release_time = tstamp
-                legacy_info.sha1 = file_hash(jar_path, hashlib.sha1)
-                legacy_info.sha256 = file_hash(jar_path, hashlib.sha256)
-                legacy_info.size = os.path.getsize(jar_path)
-                legacy_info_list.number[key] = legacy_info
+                # ignore the two versions without install manifests and jar mod class files
+                # TODO: fix those versions?
+                if version.mc_version_sane == "1.6.1":
+                    continue
+
+                # only gather legacy info if it's missing
+                if not os.path.isfile(LEGACYINFO_PATH):
+                    # grab the jar/zip if it's not there
+                    if not os.path.isfile(jar_path):
+                        download_binary_file(sess, jar_path, version.url())
+                    # find the latest timestamp in the zip file
+                    tstamp = datetime.fromtimestamp(0)
+                    with zipfile.ZipFile(jar_path) as jar:
+                        for info in jar.infolist():
+                            tstamp_new = datetime(*info.date_time)
+                            if tstamp_new > tstamp:
+                                tstamp = tstamp_new
+                    legacy_info = ForgeLegacyInfo()
+                    legacy_info.release_time = tstamp
+                    legacy_info.sha1 = file_hash(jar_path, hashlib.sha1)
+                    legacy_info.sha256 = file_hash(jar_path, hashlib.sha256)
+                    legacy_info.size = os.path.getsize(jar_path)
+                    legacy_info_list.number[key] = legacy_info
 
     # only write legacy info if it's missing
     if not os.path.isfile(LEGACYINFO_PATH):
diff --git a/meta/run/update_java.py b/meta/run/update_java.py
index bc9ad8db1f..80ca7b4a70 100644
--- a/meta/run/update_java.py
+++ b/meta/run/update_java.py
@@ -11,8 +11,8 @@ from meta.common.java import (
     AZUL_VERSIONS_DIR,
 )
 from meta.model.java import (
-	ADOPTIUM_API_BASE,
-	OPENJ9_API_BASE,
+    ADOPTIUM_API_BASE,
+    OPENJ9_API_BASE,
     ADOPTX_API_AVAILABLE_RELEASES,
     adoptxAPIFeatureReleasesUrl,
     adoptiumAPIFeatureReleasesUrl,
@@ -72,7 +72,11 @@ def main():
         page = 0
         while True:
             query = AdoptxAPIFeatureReleasesQuery(
-                image_type=AdoptxImageType.Jre, page_size=page_size, page=page, jvm_impl=AdoptxJvmImpl.Hotspot, vendor=AdoptxVendor.Eclipse
+                image_type=AdoptxImageType.Jre,
+                page_size=page_size,
+                page=page,
+                jvm_impl=AdoptxJvmImpl.Hotspot,
+                vendor=AdoptxVendor.Eclipse,
             )
             api_call = adoptiumAPIFeatureReleasesUrl(feature, query=query)
             print("Fetching JRE Page:", page, api_call)
@@ -116,7 +120,11 @@ def main():
         page = 0
         while True:
             query = AdoptxAPIFeatureReleasesQuery(
-                image_type=AdoptxImageType.Jre, page_size=page_size, page=page, jvm_impl=AdoptxJvmImpl.OpenJ9, vendor=AdoptxVendor.Ibm
+                image_type=AdoptxImageType.Jre,
+                page_size=page_size,
+                page=page,
+                jvm_impl=AdoptxJvmImpl.OpenJ9,
+                vendor=AdoptxVendor.Ibm,
             )
             api_call = openj9APIFeatureReleasesUrl(feature, query=query)
             print("Fetching JRE Page:", page, api_call)
diff --git a/meta/run/update_neoforge.py b/meta/run/update_neoforge.py
index 186edb253c..5530540173 100644
--- a/meta/run/update_neoforge.py
+++ b/meta/run/update_neoforge.py
@@ -1,7 +1,8 @@
 """
- Get the source files necessary for generating Forge versions
+Get the source files necessary for generating Forge versions
 """
 
+import concurrent.futures
 import copy
 import hashlib
 import json
@@ -115,6 +116,127 @@ def get_single_forge_files_manifest(longversion, artifact: str):
     return ret_dict
 
 
+def process_neoforge_version(key, entry):
+    eprint("Updating NeoForge %s" % key)
+    if entry.mc_version is None:
+        eprint("Skipping %d with invalid MC version" % entry.build)
+        return
+
+    version = NeoForgeVersion(entry)
+    if version.url() is None:
+        eprint("Skipping %d with no valid files" % version.build)
+        return
+    if not version.uses_installer():
+        eprint(f"version {version.long_version} does not use installer")
+        return
+
+    jar_path = os.path.join(UPSTREAM_DIR, JARS_DIR, version.filename())
+
+    installer_info_path = (
+        UPSTREAM_DIR + "/neoforge/installer_info/%s.json" % version.long_version
+    )
+    profile_path = (
+        UPSTREAM_DIR + "/neoforge/installer_manifests/%s.json" % version.long_version
+    )
+    version_file_path = (
+        UPSTREAM_DIR + "/neoforge/version_manifests/%s.json" % version.long_version
+    )
+
+    new_sha1 = None
+    sha1_file = jar_path + ".sha1"
+    if not os.path.isfile(jar_path):
+        remove_files([profile_path, installer_info_path])
+    else:
+        fileSha1 = get_file_sha1_from_file(jar_path, sha1_file)
+        try:
+            rfile = sess.get(version.url() + ".sha1")
+            rfile.raise_for_status()
+            new_sha1 = rfile.text.strip()
+            if fileSha1 != new_sha1:
+                remove_files([jar_path, profile_path, installer_info_path, sha1_file])
+        except Exception as e:
+            eprint("Failed to check sha1 %s" % version.url())
+            eprint("Error is %s" % e)
+
+    installer_refresh_required = not os.path.isfile(profile_path) or not os.path.isfile(
+        installer_info_path
+    )
+
+    if installer_refresh_required:
+        # grab the installer if it's not there
+        if not os.path.isfile(jar_path):
+            eprint("Downloading %s" % version.url())
+            try:
+                Path(jar_path).parent.mkdir(parents=True, exist_ok=True)
+                download_binary_file(sess, jar_path, version.url())
+            except Exception as e:
+                eprint("Failed to download %s" % version.url())
+                eprint("Error is %s" % e)
+                return
+            if new_sha1 is None:
+                try:
+                    rfile = sess.get(version.url() + ".sha1")
+                    rfile.raise_for_status()
+                    new_sha1 = rfile.text.strip()
+                except Exception as e:
+                    eprint("Failed to download new sha1 %s" % version.url())
+                    eprint("Error is %s" % e)
+            if new_sha1 is not None:  # this is in case the fetch failed
+                with open(sha1_file, "w") as file:
+                    file.write(new_sha1)
+
+    eprint("Processing %s" % version.url())
+    # harvestables from the installer
+    if not os.path.isfile(profile_path):
+        print(jar_path)
+        with zipfile.ZipFile(jar_path) as jar:
+            with suppress(KeyError):
+                with jar.open("version.json") as profile_zip_entry:
+                    version_data = profile_zip_entry.read()
+
+                    # Process: does it parse?
+                    MojangVersion.parse_raw(version_data)
+
+                    Path(version_file_path).parent.mkdir(parents=True, exist_ok=True)
+                    with open(version_file_path, "wb") as versionJsonFile:
+                        versionJsonFile.write(version_data)
+                        versionJsonFile.close()
+
+            with jar.open("install_profile.json") as profile_zip_entry:
+                install_profile_data = profile_zip_entry.read()
+
+                # Process: does it parse?
+                is_parsable = False
+                exception = None
+                try:
+                    NeoForgeInstallerProfileV2.parse_raw(install_profile_data)
+                    is_parsable = True
+                except ValidationError as err:
+                    exception = err
+
+                if not is_parsable:
+                    if version.is_supported():
+                        raise exception
+                    else:
+                        eprint(
+                            "Version %s is not supported and won't be generated later."
+                            % version.long_version
+                        )
+
+                Path(profile_path).parent.mkdir(parents=True, exist_ok=True)
+                with open(profile_path, "wb") as profileFile:
+                    profileFile.write(install_profile_data)
+                    profileFile.close()
+
+    # installer info v1
+    if not os.path.isfile(installer_info_path):
+        installer_info = InstallerInfo()
+        installer_info.sha1hash = file_hash(jar_path, hashlib.sha1)
+        installer_info.sha256hash = file_hash(jar_path, hashlib.sha256)
+        installer_info.size = os.path.getsize(jar_path)
+        installer_info.write(installer_info_path)
+
+
 def main():
     # get the 1.20.1 remote version list fragments
     r = sess.get(
@@ -136,7 +258,7 @@ def main():
 
     new_index = DerivedNeoForgeIndex()
 
-    #let's keep the regex here to remove the 1.20.1-
+    # let's keep the regex here to remove the 1.20.1-
     version_expression = re.compile(
         r"^(?P<mc>[0-9a-zA-Z_\.]+)-(?P<ver>[0-9\.]+\.(?P<build>[0-9]+))(-(?P<branch>[a-zA-Z0-9\.]+))?$"
     )
@@ -172,7 +294,7 @@ def main():
             files=files,
         )
         new_index.versions[long_version] = entry
-        
+
         if entry.recommended:
             new_index.recommended = long_version
 
@@ -188,127 +310,9 @@ def main():
 
     print("Grabbing installers and dumping installer profiles...")
     # get the installer jars - if needed - and get the installer profiles out of them
-    for key, entry in new_index.versions.items():
-        eprint("Updating NeoForge %s" % key)
-
-        version = NeoForgeVersion(entry)
-        if version.url() is None:
-            eprint("Skipping %d with no valid files" % version.build)
-            continue
-        if not version.uses_installer():
-            eprint(f"version {version.long_version} does not use installer")
-            continue
-
-        jar_path = os.path.join(UPSTREAM_DIR, JARS_DIR, version.filename())
-
-        installer_info_path = (
-            UPSTREAM_DIR + "/neoforge/installer_info/%s.json" % version.long_version
-        )
-        profile_path = (
-            UPSTREAM_DIR
-            + "/neoforge/installer_manifests/%s.json" % version.long_version
-        )
-        version_file_path = (
-            UPSTREAM_DIR + "/neoforge/version_manifests/%s.json" % version.long_version
-        )
-
-        new_sha1 = None
-        sha1_file = jar_path + ".sha1"
-        if not os.path.isfile(jar_path):
-            remove_files([profile_path, installer_info_path])
-        else:
-            fileSha1 = get_file_sha1_from_file(jar_path, sha1_file)
-            try:
-                rfile = sess.get(version.url() + ".sha1")
-                rfile.raise_for_status()
-                new_sha1 = rfile.text.strip()
-                if fileSha1 != new_sha1:
-                    remove_files(
-                        [jar_path, profile_path, installer_info_path, sha1_file]
-                    )
-            except Exception as e:
-                eprint("Failed to check sha1 %s" % version.url())
-                eprint("Error is %s" % e)
-
-        installer_refresh_required = not os.path.isfile(
-            profile_path
-        ) or not os.path.isfile(installer_info_path)
-
-        if installer_refresh_required:
-            # grab the installer if it's not there
-            if not os.path.isfile(jar_path):
-                eprint("Downloading %s" % version.url())
-                try:
-                    Path(jar_path).parent.mkdir(parents=True, exist_ok=True)
-                    download_binary_file(sess, jar_path, version.url())
-                except Exception as e:
-                    eprint("Failed to download %s" % version.url())
-                    eprint("Error is %s" % e)
-                    continue
-                if new_sha1 is None:
-                    try:
-                        rfile = sess.get(version.url() + ".sha1")
-                        rfile.raise_for_status()
-                        new_sha1 = rfile.text.strip()
-                    except Exception as e:
-                        eprint("Failed to download new sha1 %s" % version.url())
-                        eprint("Error is %s" % e)
-                if new_sha1 is not None:  # this is in case the fetch failed
-                    with open(sha1_file, "w") as file:
-                        file.write(new_sha1)
-
-        eprint("Processing %s" % version.url())
-        # harvestables from the installer
-        if not os.path.isfile(profile_path):
-            print(jar_path)
-            with zipfile.ZipFile(jar_path) as jar:
-                with suppress(KeyError):
-                    with jar.open("version.json") as profile_zip_entry:
-                        version_data = profile_zip_entry.read()
-
-                        # Process: does it parse?
-                        MojangVersion.parse_raw(version_data)
-
-                        Path(version_file_path).parent.mkdir(
-                            parents=True, exist_ok=True
-                        )
-                        with open(version_file_path, "wb") as versionJsonFile:
-                            versionJsonFile.write(version_data)
-                            versionJsonFile.close()
-
-                with jar.open("install_profile.json") as profile_zip_entry:
-                    install_profile_data = profile_zip_entry.read()
-
-                    # Process: does it parse?
-                    is_parsable = False
-                    exception = None
-                    try:
-                        NeoForgeInstallerProfileV2.parse_raw(install_profile_data)
-                        is_parsable = True
-                    except ValidationError as err:
-                        exception = err
-
-                    if not is_parsable:
-                        if version.is_supported():
-                            raise exception
-                        else:
-                            eprint(
-                                "Version %s is not supported and won't be generated later."
-                                % version.long_version
-                            )
-
-                    Path(profile_path).parent.mkdir(parents=True, exist_ok=True)
-                    with open(profile_path, "wb") as profileFile:
-                        profileFile.write(install_profile_data)
-                        profileFile.close()
-
-        # installer info v1
-        if not os.path.isfile(installer_info_path):
-            installer_info = InstallerInfo()
-            installer_info.sha1hash = file_hash(jar_path, hashlib.sha1)
-            installer_info.sha256hash = file_hash(jar_path, hashlib.sha256)
-            installer_info.size = os.path.getsize(jar_path)
-            installer_info.write(installer_info_path)
+    with concurrent.futures.ThreadPoolExecutor() as executor:
+        for key, entry in new_index.versions.items():
+            executor.submit(process_neoforge_version, key, entry)
 
 
 if __name__ == "__main__":