Factor more shared code to avoid duplicates

Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com>

Author: pombredanne

etc/scripts/7z.py

@@ -11,7 +11,6 @@
 from distutils.dir_util import copy_tree
 import os
 import shutil
-import subprocess
 import sys
 
 import shared_utils
@@ -23,15 +22,6 @@
 TRACE_DEEP = False
 
 
-def extract_7zip(location, target_dir):
-    """
-    Extract a 7z archive at `location` in the `target_dir` directory.
-    """
-    out = subprocess.check_output(['7z', 'x', location], cwd=target_dir)
-    if not b'Everything is Ok' in out:
-        raise Exception(out)
-
-
 def install_files(extracted_dir, install_dir, copies):
     """
     Install libraries and licenses from the extracted_dir
@@ -58,19 +48,6 @@ def install_files(extracted_dir, install_dir, copies):
                     os.makedirs(dst, exist_ok=True)
                 shutil.copy2(src, dst)
 
-def extract_in_place(location):
-    """
-    Extract an archive at `location` in a directory created side-by-side with
-    the archive.
-    Return the directory where the files are extracted
-    """
-    target_dir = location.replace('.exe', '')
-    if os.path.exists(target_dir):
-        shutil.rmtree(target_dir)
-    os.makedirs(target_dir, exist_ok=True)
-    extract_7zip(location, target_dir)
-    return target_dir
-
 
 def fetch_package(name, cache_dir):
     """
@@ -102,7 +79,7 @@ def fetch_package(name, cache_dir):
     fetched_binary_loc = shared_utils.fetch_file(url=bin_url, dir_location=bin_cache_dir)
     shared_utils.verify(fetched_binary_loc, bin_sha256)
 
-    extracted_dir = extract_in_place(fetched_binary_loc)
+    extracted_dir = shared_utils.extract_in_place(fetched_binary_loc)
     install_files(extracted_dir, install_dir, copies)
 
     # also fetch sources

etc/scripts/homebrew.py

@@ -615,7 +615,7 @@ def process_package(package, osarch, install_dir, copies, bin_cache_dir, src_cac
     # fetch the binary for the requested osarch
     package_binary_download = package.download_urls[osarch]
     fetched_binary_loc = package_binary_download.fetch(dir_location=bin_cache_dir)
-    extracted_dir, _extracted_locations = shared_utils.extract_in_place(location=fetched_binary_loc)
+    extracted_dir = shared_utils.extract_in_place(fetched_binary_loc)
 
     # fetch the upstream formula and collect extra sources/patches:
     # formula_loc = package.formula_download_url.fetch(dir_location=src_cache_dir)

etc/scripts/msys2.py

@@ -728,9 +728,11 @@ def update_package(name, version=None, repo='mingw64', cache_dir=None,
     except:
         fetched_binary_loc = shared_utils.fetch_file(url=root_package.download_url, dir_location=bin_cache_dir, force=True)
         shared_utils.verify(fetched_binary_loc, root_package.sha256)
-    extracted_dir, extracted_locations = shared_utils.extract_in_place(fetched_binary_loc)
+    extracted_dir = shared_utils.extract_in_place(fetched_binary_loc)
     install_files(extracted_dir, install_dir, package_short_name=root_package.short_name, copies=copies)
 
+    extracted_locs = [extracted_dir]
+
     # also fetch sources
     shared_utils.fetch_file(url=root_package.source_package.download_url, dir_location=src_cache_dir)
 
@@ -751,24 +753,19 @@ def update_package(name, version=None, repo='mingw64', cache_dir=None,
         except:
             fetched_binary_loc = shared_utils.fetch_file(url=dep.download_url, dir_location=bin_cache_dir, indent=3, force=True)
             shared_utils.verify(fetched_binary_loc, dep.sha256)
-        extracted_dir, dep_extracted_locations = shared_utils.extract_in_place(fetched_binary_loc)
+        extracted_dir = shared_utils.extract_in_place(fetched_binary_loc)
         install_files(extracted_dir, install_dir, package_short_name=dep.short_name, copies=copies)
 
         # also fetch sources
         shared_utils.fetch_file(url=dep.source_package.download_url, dir_location=src_cache_dir, indent=5)
 
-        extracted_locations.extend(dep_extracted_locations)
+        extracted_locs.append(extracted_dir)
 
     check_installed_files(install_dir, copies, root_package)
 
     # finally cleanup after thyself, removing extracted locations
-
-    for exloc in extracted_locations:
-        if os.path.exists(exloc):
-            if os.path.isdir(exloc):
-                shutil.rmtree(exloc, False)
-            else:
-                os.remove(exloc)
+    for loc in extracted_locs:
+        shutil.rmtree(loc, False)
 
 
 def main(argv):

etc/scripts/shared_utils.py

@@ -1,11 +1,13 @@
+# Copyright (c) 2020 nexB Inc.
 
+import hashlib
 import os
+import shutil
 import subprocess
 import tarfile
+import zipfile
 
 import requests
-import shutil
-import hashlib
 
 REQUEST_TIMEOUT = 60
 
@@ -14,8 +16,6 @@
 TRACE_INSTALL = False
 
 
-
-
 def fetch_file(url, dir_location, file_name=None, force=False, indent=1):
     """
     Fetch the file at `url` and save it in `dir_location`.
@@ -36,46 +36,74 @@ def fetch_file(url, dir_location, file_name=None, force=False, indent=1):
     return location
 
 
-
 def extract_tar(location, target_dir):
     """
     Extract a tar archive at `location` in the `target_dir` directory.
     Return a list of extracted locations (either directories or files)
     """
-    locations = []
+    temp_extraction = None
     if location.endswith('.tar.zst'):
         # rare and "new" but used in msys
         subprocess.check_call(['unzstd', '-q', '-k', '-f', location])
-        location, _, _ = location.rpartition('.zst')
-        locations.append(location)
+        temp_extraction, _, _ = location.rpartition('.zst')
+        location = temp_extraction
     with open(location, 'rb') as input_tar:
         with tarfile.open(fileobj=input_tar) as tar:
             members = tar.getmembers()
             for tarinfo in members:
                 tarinfo.mode = 0o755
             tar.extractall(target_dir, members=members)
-    locations.append(target_dir)
-    return locations
-
+    if temp_extraction:
+        os.remove(temp_extraction)
 
 
 def extract_in_place(location):
     """
     Extract a tar archive at `location` in a directory created side-by-side with
     the archive and named after the archive stripped from it's extension.
     Remove this directory if it already exists.
-    
-    Return the directory where the files are extracted, and a list of all
-    extracted_locations.
+
+    Return the directory where the files are extracted.
     """
-    target_dir = location.replace('.tar.xz', '').replace('.tar.gz', '').replace('.tar.zst', '')
+    if '.tar' in location:
+        extractor = extract_tar
+        # split from tar.gz/xz/lzma/zst/bz2
+        target_dir, _, _ = location.rpartition('.tar')
+
+    elif location.endswith('.zip'):
+        extractor = extract_zip
+        target_dir = location.replace('.zip', '')
+
+    elif location.endswith('.exe'):
+        # for 7z self extracting exe
+        extractor = extract_7zip
+        target_dir = location.replace('.exe', '')
+
     if os.path.exists(target_dir):
         shutil.rmtree(target_dir)
+
     os.makedirs(target_dir, exist_ok=True)
-    extracted_locations = extract_tar(location, target_dir)
-    return target_dir, extracted_locations
+
+    extractor(location, target_dir)
+    return target_dir
+
+
+def extract_zip(location, target_dir):
+    """
+    Extract a zip archive file at location to the `target_dir` directory.
+    """
+    with zipfile.ZipFile(location) as zipf:
+        zipf.extractall(path=target_dir)
 
 
+def extract_7zip(location, target_dir):
+    """
+    Extract a 7z archive at `location` to the `target_dir` directory.
+    """
+    out = subprocess.check_output(['7z', 'x', location], cwd=target_dir)
+    if not b'Everything is Ok' in out:
+        raise Exception(out)
+
 
 def verify(fetched_location, expected_sha256=None):
     """
@@ -88,6 +116,3 @@ def verify(fetched_location, expected_sha256=None):
     with open (fetched_location, 'rb') as f:
         fsha256 = hashlib.sha256(f.read()).hexdigest()
         assert fsha256 == expected_sha256, f'Invalid SHA256 for: {fetched_location}'
-
-
-