Chache sha1 file

Signed-off-by: Trial97 <alexandru.tripon97@gmail.com>

5 files changed, 76 insertions, 59 deletions

diff --git a/meta/common/__init__.py b/meta/common/__init__.py
index 16a4f507bf..8e42933942 100644
--- a/meta/common/__init__.py
+++ b/meta/common/__init__.py
@@ -1,6 +1,8 @@
 import os
 import os.path
 import datetime
+import hashlib
+import sys
 from urllib.parse import urlparse
 from typing import Any, Optional
 
@@ -95,3 +97,26 @@ def remove_files(file_paths):
                 os.remove(file_path)
         except Exception as e:
             print(e)
+
+
+def eprint(*args, **kwargs):
+    print(*args, file=sys.stderr, **kwargs)
+
+
+def filehash(filename, hashtype, blocksize=65536):
+    hashtype = hashtype()
+    with open(filename, "rb") as f:
+        for block in iter(lambda: f.read(blocksize), b""):
+            hashtype.update(block)
+    return hashtype.hexdigest()
+
+
+def get_file_sha1_from_file(file_name, sha1_file):
+    if os.path.isfile(sha1_file):
+        with open(sha1_file, "r") as file:
+            return file.read()
+
+    new_sha1 = filehash(file_name, hashlib.sha1)
+    with open(sha1_file, "w") as file:
+        file.write(new_sha1)
+    return
diff --git a/meta/run/generate_forge.py b/meta/run/generate_forge.py
index 95b5ac39cb..a01f503ebe 100755
--- a/meta/run/generate_forge.py
+++ b/meta/run/generate_forge.py
@@ -1,11 +1,10 @@
 import os
 import re
-import sys
 from packaging import version as pversion
 from operator import attrgetter
 from typing import Collection
 
-from meta.common import ensure_component_dir, launcher_path, upstream_path
+from meta.common import ensure_component_dir, launcher_path, upstream_path, eprint
 from meta.common.forge import (
     FORGE_COMPONENT,
     INSTALLER_MANIFEST_DIR,
@@ -44,10 +43,6 @@ UPSTREAM_DIR = upstream_path()
 ensure_component_dir(FORGE_COMPONENT)
 
 
-def eprint(*args, **kwargs):
-    print(*args, file=sys.stderr, **kwargs)
-
-
 # Construct a set of libraries out of a Minecraft version file, for filtering.
 mc_version_cache = {}
 
diff --git a/meta/run/generate_neoforge.py b/meta/run/generate_neoforge.py
index eab7fb33e3..5c6f965435 100644
--- a/meta/run/generate_neoforge.py
+++ b/meta/run/generate_neoforge.py
@@ -1,11 +1,10 @@
 from copy import deepcopy
 import os
 import re
-import sys
 from operator import attrgetter
 from typing import Collection
 
-from meta.common import ensure_component_dir, launcher_path, upstream_path
+from meta.common import ensure_component_dir, launcher_path, upstream_path, eprint
 from meta.common.neoforge import (
     NEOFORGE_COMPONENT,
     INSTALLER_MANIFEST_DIR,
@@ -38,10 +37,6 @@ UPSTREAM_DIR = upstream_path()
 ensure_component_dir(NEOFORGE_COMPONENT)
 
 
-def eprint(*args, **kwargs):
-    print(*args, file=sys.stderr, **kwargs)
-
-
 def version_from_build_system_installer(
     installer: MojangVersion,
     profile: NeoForgeInstallerProfileV2,
diff --git a/meta/run/update_forge.py b/meta/run/update_forge.py
index 5a86727199..30c701fb32 100755
--- a/meta/run/update_forge.py
+++ b/meta/run/update_forge.py
@@ -7,7 +7,6 @@ import hashlib
 import json
 import os
 import re
-import sys
 import zipfile
 from contextlib import suppress
 from datetime import datetime
@@ -21,6 +20,9 @@ from meta.common import (
     ensure_upstream_dir,
     default_session,
     remove_files,
+    eprint,
+    filehash,
+    get_file_sha1_from_file,
 )
 from meta.common.forge import (
     JARS_DIR,
@@ -43,6 +45,7 @@ from meta.model.forge import (
     InstallerInfo,
     ForgeLegacyInfo,
 )
+from meta.common.http import download_binary_file
 from meta.model.mojang import MojangVersion
 
 UPSTREAM_DIR = upstream_path()
@@ -58,18 +61,6 @@ LEGACYINFO_PATH = os.path.join(UPSTREAM_DIR, LEGACYINFO_FILE)
 sess = default_session()
 
 
-def eprint(*args, **kwargs):
-    print(*args, file=sys.stderr, **kwargs)
-
-
-def filehash(filename, hashtype, blocksize=65536):
-    hashtype = hashtype()
-    with open(filename, "rb") as f:
-        for block in iter(lambda: f.read(blocksize), b""):
-            hashtype.update(block)
-    return hashtype.hexdigest()
-
-
 def get_single_forge_files_manifest(longversion):
     print(f"Getting Forge manifest for {longversion}")
     path_thing = UPSTREAM_DIR + "/forge/files_manifests/%s.json" % longversion
@@ -297,15 +288,20 @@ def main():
                 UPSTREAM_DIR + "/forge/version_manifests/%s.json" % version.long_version
             )
 
+            new_sha1 = None
+            sha1_file = jar_path + ".sha1"
             if not os.path.isfile(jar_path):
                 remove_files([profile_path, installer_info_path])
             else:
-                fileSha1 = filehash(jar_path, hashlib.sha1)
+                fileSha1 = get_file_sha1_from_file(jar_path, sha1_file)
                 try:
                     rfile = sess.get(version.url() + ".sha1")
                     rfile.raise_for_status()
-                    if fileSha1 != rfile.text.strip():
-                        remove_files([jar_path, profile_path, installer_info_path])
+                    new_sha1 = rfile.text.strip()
+                    if fileSha1 != new_sha1:
+                        remove_files(
+                            [jar_path, profile_path, installer_info_path, sha1_file]
+                        )
                 except Exception as e:
                     eprint("Failed to check sha1 %s" % version.url())
                     eprint("Error is %s" % e)
@@ -318,11 +314,18 @@ def main():
                 # grab the installer if it's not there
                 if not os.path.isfile(jar_path):
                     eprint("Downloading %s" % version.url())
-                    rfile = sess.get(version.url(), stream=True)
-                    rfile.raise_for_status()
-                    with open(jar_path, "wb") as f:
-                        for chunk in rfile.iter_content(chunk_size=128):
-                            f.write(chunk)
+                    download_binary_file(sess, jar_path, version.url())
+                    if new_sha1 is None:
+                        try:
+                            rfile = sess.get(version.url() + ".sha1")
+                            rfile.raise_for_status()
+                            new_sha1 = rfile.text.strip()
+                        except Exception as e:
+                            eprint("Failed to download new sha1 %s" % version.url())
+                            eprint("Error is %s" % e)
+                    if new_sha1 is not None:  # this is in case the fetch failed
+                        with open(sha1_file, "w") as file:
+                            file.write(new_sha1)
 
             eprint("Processing %s" % version.url())
             # harvestables from the installer
@@ -387,11 +390,7 @@ def main():
             if not os.path.isfile(LEGACYINFO_PATH):
                 # grab the jar/zip if it's not there
                 if not os.path.isfile(jar_path):
-                    rfile = sess.get(version.url(), stream=True)
-                    rfile.raise_for_status()
-                    with open(jar_path, "wb") as f:
-                        for chunk in rfile.iter_content(chunk_size=128):
-                            f.write(chunk)
+                    download_binary_file(sess, jar_path, version.url())
                 # find the latest timestamp in the zip file
                 tstamp = datetime.fromtimestamp(0)
                 with zipfile.ZipFile(jar_path) as jar:
diff --git a/meta/run/update_neoforge.py b/meta/run/update_neoforge.py
index ee8d6ff950..d9b7cc0f64 100644
--- a/meta/run/update_neoforge.py
+++ b/meta/run/update_neoforge.py
@@ -7,7 +7,6 @@ import hashlib
 import json
 import os
 import re
-import sys
 import zipfile
 from contextlib import suppress
 from datetime import datetime
@@ -22,7 +21,11 @@ from meta.common import (
     ensure_upstream_dir,
     default_session,
     remove_files,
+    eprint,
+    filehash,
+    get_file_sha1_from_file,
 )
+from meta.common.http import download_binary_file
 from meta.common.neoforge import (
     JARS_DIR,
     INSTALLER_INFO_DIR,
@@ -52,18 +55,6 @@ ensure_upstream_dir(FILE_MANIFEST_DIR)
 sess = default_session()
 
 
-def eprint(*args, **kwargs):
-    print(*args, file=sys.stderr, **kwargs)
-
-
-def filehash(filename, hashtype, blocksize=65536):
-    hashtype = hashtype()
-    with open(filename, "rb") as f:
-        for block in iter(lambda: f.read(blocksize), b""):
-            hashtype.update(block)
-    return hashtype.hexdigest()
-
-
 def find_nth(haystack, needle, n):
     start = haystack.find(needle)
     while start >= 0 and n > 1:
@@ -221,15 +212,20 @@ def main():
             UPSTREAM_DIR + "/neoforge/version_manifests/%s.json" % version.long_version
         )
 
+        new_sha1 = None
+        sha1_file = jar_path + ".sha1"
         if not os.path.isfile(jar_path):
             remove_files([profile_path, installer_info_path])
         else:
-            fileSha1 = filehash(jar_path, hashlib.sha1)
+            fileSha1 = get_file_sha1_from_file(jar_path, sha1_file)
             try:
                 rfile = sess.get(version.url() + ".sha1")
                 rfile.raise_for_status()
-                if fileSha1 != rfile.text.strip():
-                    remove_files([jar_path, profile_path, installer_info_path])
+                new_sha1 = rfile.text.strip()
+                if fileSha1 != new_sha1:
+                    remove_files(
+                        [jar_path, profile_path, installer_info_path, sha1_file]
+                    )
             except Exception as e:
                 eprint("Failed to check sha1 %s" % version.url())
                 eprint("Error is %s" % e)
@@ -243,16 +239,23 @@ def main():
             if not os.path.isfile(jar_path):
                 eprint("Downloading %s" % version.url())
                 try:
-                    rfile = sess.get(version.url(), stream=True)
-                    rfile.raise_for_status()
                     Path(jar_path).parent.mkdir(parents=True, exist_ok=True)
-                    with open(jar_path, "wb") as f:
-                        for chunk in rfile.iter_content(chunk_size=128):
-                            f.write(chunk)
+                    download_binary_file(sess, jar_path, version.url())
                 except Exception as e:
                     eprint("Failed to download %s" % version.url())
                     eprint("Error is %s" % e)
                     continue
+                if new_sha1 is None:
+                    try:
+                        rfile = sess.get(version.url() + ".sha1")
+                        rfile.raise_for_status()
+                        new_sha1 = rfile.text.strip()
+                    except Exception as e:
+                        eprint("Failed to download new sha1 %s" % version.url())
+                        eprint("Error is %s" % e)
+                if new_sha1 is not None:  # this is in case the fetch failed
+                    with open(sha1_file, "w") as file:
+                        file.write(new_sha1)
 
         eprint("Processing %s" % version.url())
         # harvestables from the installer