Merge pull request #2988 from nexB/reimplement-get-installed-packages

Reimplement get installed packages

Author: pombredanne

requirements.txt

@@ -13,7 +13,7 @@ commoncode==31.0.0b4
 construct==2.10.68
 container-inspector==31.0.0
 cryptography==36.0.2
-debian-inspector==30.0.0
+debian-inspector==31.0.0b1
 dockerfile-parse==1.2.0
 dparse2==0.6.1
 extractcode==31.0.0

setup-mini.cfg

@@ -71,7 +71,7 @@ install_requires =
     colorama >= 0.3.9
     commoncode==31.0.0b4
     container-inspector >= 31.0.0
-    debian-inspector >= 30.0.0
+    debian-inspector >= 31.0.0b1
     dparse2 >= 0.6.1
     fasteners
     fingerprints >= 0.6.0
@@ -112,6 +112,8 @@ install_requires =
     xmltodict >= 0.11.0
     zipp >= 3.0.0; python_version < "3.9"
     typecode >= 30.0.0
+#    typecode[full] >= 30.0.0
+#    extractcode[full] >= 31.0.0
 
 
 [options.packages.find]
@@ -128,6 +130,8 @@ testing =
     pytest-xdist >= 2
     aboutcode-toolkit >= 7.0.2
     twine
+    black
+    isort
 
 docs =
     Sphinx >= 3.3.1

setup.cfg

@@ -71,7 +71,7 @@ install_requires =
     colorama >= 0.3.9
     commoncode==31.0.0b4
     container-inspector >= 31.0.0
-    debian-inspector >= 30.0.0
+    debian-inspector >= 31.0.0b1
     dparse2 >= 0.6.1
     fasteners
     fingerprints >= 0.6.0

src/packagedcode/alpine.py

@@ -1772,4 +1772,3 @@ def detect_licenses(locin, locout):
     # get_apkindex_licenses(loc, 'alpine-licenses.csv')
     get_apkbuild_licenses(loc, 'alpine-licenses.csv')
     detect_licenses('alpine-licenses.csv', 'alpine-licenses-detection.csv')
-

src/packagedcode/models.py

@@ -25,6 +25,7 @@
 from commoncode.datautils import Mapping
 from commoncode.datautils import String
 from commoncode.fileutils import as_posixpath
+from commoncode.resource import Resource
 from typecode import contenttype
 
 """
@@ -1201,7 +1202,7 @@ def __attrs_post_init__(self, *args, **kwargs):
             self.package_uid = build_package_uid(self.purl)
 
     def to_dict(self):
-        return  super().to_dict(with_details=False)
+        return super().to_dict(with_details=False)
 
     @classmethod
     def from_package_data(cls, package_data, datafile_path):
@@ -1340,6 +1341,24 @@ def get_packages_files(self, codebase):
                 yield resource
 
 
+@attr.attributes(slots=True)
+class PackageWithResources(Package):
+    """
+    A Package with Resources.
+    """
+
+    resources = List(
+        item_type=Resource,
+        label='List of Resources',
+        help='List of Resources for this package.',
+    )
+
+    def to_dict(self):
+        package_data = super().to_dict()
+        package_data['resources'] = [resource.to_dict() for resource in self.resources]
+        return package_data
+
+
 def get_files_for_packages(codebase):
     """
     Yield tuple of (Resource, package_uid) for all resources in codebase that are

src/packagedcode/plugin_package.py

@@ -25,6 +25,7 @@
 from packagedcode.models import Dependency
 from packagedcode.models import Package
 from packagedcode.models import PackageData
+from packagedcode.models import PackageWithResources
 
 TRACE = os.environ.get('SCANCODE_DEBUG_PACKAGE', False)
 
@@ -153,6 +154,36 @@ def process_codebase(self, codebase, strip_root=False, **kwargs):
         create_package_and_deps(codebase, strip_root=strip_root, **kwargs)
 
 
+def get_installed_packages(root_dir, processes=2, **kwargs):
+    """
+    Yield Package and their Resources as they are found in `root_dir`
+    """
+    from scancode import cli
+
+    _, codebase = cli.run_scan(
+        input=root_dir,
+        processes=processes,
+        quiet=True,
+        verbose=False,
+        max_in_memory=0,
+        return_results=False,
+        return_codebase=True,
+        system_package=True,
+    )
+
+    packages_by_uid = {}
+    for package in codebase.attributes.packages:
+        p = PackageWithResources.from_dict(package)
+        packages_by_uid[p.package_uid] = p
+
+    for resource in codebase.walk():
+        for package_uid in resource.for_packages:
+            p = packages_by_uid[package_uid]
+            p.resources.append(resource)
+
+    yield from packages_by_uid.values()
+
+
 def create_package_and_deps(codebase, strip_root=False, **kwargs):
     """
     Create and save top-level Package and Dependency from the parsed

src/scancode/cli.py

@@ -504,7 +504,9 @@ def run_scan(
     echo_func=None,
     timing=False,
     keep_temp_files=False,
+    # TODO: Review return_results as it does not return Packages and Dependencies
     return_results=True,
+    return_codebase=False,
     test_mode=False,
     test_slow_mode=False,
     test_error_mode=False,
@@ -522,6 +524,12 @@ def run_scan(
     error. See scancode() for arguments details.
     """
 
+    assert not (return_results is True and return_codebase is True), 'Only one of return_results and return_codebase can be True'
+
+    if return_codebase and max_in_memory > 0:
+        # We're keeping temp files otherwise the codebase is gone
+        keep_temp_files = True
+
     plugins_option_defaults = {clio.name: clio.default for clio in plugin_options}
     requested_options = dict(plugins_option_defaults)
     requested_options.update(kwargs)
@@ -667,7 +675,7 @@ def echo_func(*_args, **_kwargs):
                    'selected when using only scan data.')
             raise ScancodeCliUsageError(msg)
 
-        if not output_plugins and not return_results:
+        if not output_plugins and not (return_results or return_codebase):
             msg = ('ERROR: Missing output option(s): at least one output '
                    'option is required to save scan results.')
             raise ScancodeCliUsageError(msg)
@@ -1002,6 +1010,8 @@ def echo_func(*_args, **_kwargs):
             # the structure is exactly the same as the JSON output
             from formattedcode.output_json import get_results
             results = get_results(codebase, as_list=True, **requested_options)
+        elif return_codebase:
+            results = codebase
 
     finally:
         # remove temporary files
@@ -1657,3 +1667,9 @@ def get_pretty_params(ctx, generic_paths=False):
             options.append((cli_opt, value))
 
     return dict(sorted(args) + sorted(options))
+
+
+if __name__ == '__main__':
+    # We have this __main__ block so that we can run scancode as a script.
+    # This is needed so we can use the Python debugging features in VSCode.
+    scancode()