diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 36b6f2e8..062579e2 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,7 +2,14 @@ Changelog ========= -v0.0.0 +v0.6.0 +------ + +- Use latest ScanCode toolkit packagedcode including the ability to collect + extra index URLs from requirements.txt + + +v0.5.0 ------ Initial release. diff --git a/Makefile b/Makefile index ddcabcec..a11ed207 100644 --- a/Makefile +++ b/Makefile @@ -18,11 +18,11 @@ dev: isort: @echo "-> Apply isort changes to ensure proper imports ordering" - ${VENV}/bin/isort --sl -l 100 src tests + ${VENV}/bin/isort --sl -l 100 src tests setup.py --skip-glob "*/_packagedcode/*" black: @echo "-> Apply black code formatter" - ${VENV}/bin/black -l 100 src tests + ${VENV}/bin/black -l 100 src tests setup.py --exclude "_packagedcode/.*" doc8: @echo "-> Run doc8 validation" @@ -35,9 +35,9 @@ check: @${ACTIVATE} pycodestyle --max-line-length=110 \ --exclude=.eggs,etc/scripts,src/_packagedcode,venv,lib,thirdparty,docs . @echo "-> Run isort imports ordering validation" - @${ACTIVATE} isort --sl --check-only -l 100 src tests + @${ACTIVATE} isort --sl --check-only -l 100 setup.py src tests --skip-glob "*/_packagedcode/*" @echo "-> Run black validation" - @${ACTIVATE} black --check -l 100 + @${ACTIVATE} black --check -l 100 src tests setup.py --exclude "_packagedcode/.*" clean: @echo "-> Clean the Python env" diff --git a/src/_packagedcode/models.py b/src/_packagedcode/models.py index ec4faf72..ed3cf238 100644 --- a/src/_packagedcode/models.py +++ b/src/_packagedcode/models.py @@ -13,21 +13,30 @@ from fnmatch import fnmatchcase import attr +from packageurl import normalize_qualifiers +from packageurl import PackageURL + from commoncode import filetype +from commoncode.datautils import choices from commoncode.datautils import Boolean from commoncode.datautils import Date from commoncode.datautils import Integer from commoncode.datautils import List from commoncode.datautils import Mapping from commoncode.datautils import String -from commoncode.datautils import choices from commoncode.fileutils import as_posixpath -from packageurl import PackageURL -from packageurl import normalize_qualifiers +from commoncode.resource import Resource +try: + from typecode import contenttype +except ImportError: + contenttype = None -""" -Originally vendored from scancode-toolkit packagedcode.models +try: + from packagedcode import licensing +except ImportError: + licensing = None +""" This module contain data models for package and dependencies, abstracting and normalizing the small differences that exist across different package types (aka. ecosystems), manifest file formats and tools. @@ -105,7 +114,7 @@ - IdentifiablePackageData: a base class for a Package-like class with a Package URL. """ -SCANCODE_DEBUG_PACKAGE_API = os.environ.get("SCANCODE_DEBUG_PACKAGE_API", False) +SCANCODE_DEBUG_PACKAGE_API = os.environ.get('SCANCODE_DEBUG_PACKAGE_API', False) TRACE = SCANCODE_DEBUG_PACKAGE_API TRACE_UPDATE = SCANCODE_DEBUG_PACKAGE_API @@ -119,12 +128,13 @@ def logger_debug(*args): if TRACE or TRACE_UPDATE: import sys - logging.basicConfig(stream=sys.stdout) logger.setLevel(logging.DEBUG) def logger_debug(*args): - return logger.debug(" ".join(isinstance(a, str) and a or repr(a) for a in args)) + return logger.debug( + ' '.join(isinstance(a, str) and a or repr(a) for a in args) + ) class ModelMixin: @@ -176,11 +186,11 @@ def to_tuple(collection): return tuple(results) -party_person = "person" +party_person = 'person' # often loosely defined -party_project = "project" +party_project = 'project' # more formally defined -party_org = "organization" +party_org = 'organization' PARTY_TYPES = ( None, party_person, @@ -198,23 +208,31 @@ class Party(ModelMixin): type = String( repr=True, validator=choices(PARTY_TYPES), - label="party type", - help="the type of this party: One of: " + ", ".join(p for p in PARTY_TYPES if p), - ) + label='party type', + help='the type of this party: One of: ' + +', '.join(p for p in PARTY_TYPES if p)) role = String( repr=True, - label="party role", - help="A role for this party. Something such as author, " - "maintainer, contributor, owner, packager, distributor, " - "vendor, developer, owner, etc.", - ) + label='party role', + help='A role for this party. Something such as author, ' + 'maintainer, contributor, owner, packager, distributor, ' + 'vendor, developer, owner, etc.') - name = String(repr=True, label="name", help="Name of this party.") + name = String( + repr=True, + label='name', + help='Name of this party.') - email = String(repr=True, label="email", help="Email for this party.") + email = String( + repr=True, + label='email', + help='Email for this party.') - url = String(repr=True, label="url", help="URL to a primary web page for this party.") + url = String( + repr=True, + label='url', + help='URL to a primary web page for this party.') @attr.attributes(slots=True) @@ -225,34 +243,40 @@ class IdentifiablePackageData(ModelMixin): This base class is used for all package-like objects be they a manifest or an actual package instance. """ - type = String( repr=True, - label="package type", - help="A short code to identify what is the type of this " - "package. For instance gem for a Rubygem, docker for container, " - "pypi for Python Wheel or Egg, maven for a Maven Jar, " - "deb for a Debian package, etc.", - ) + label='package type', + help='A short code to identify what is the type of this ' + 'package. For instance gem for a Rubygem, docker for container, ' + 'pypi for Python Wheel or Egg, maven for a Maven Jar, ' + 'deb for a Debian package, etc.') - namespace = String(repr=True, label="package namespace", help="Namespace for this package.") + namespace = String( + repr=True, + label='package namespace', + help='Namespace for this package.') - name = String(repr=True, label="package name", help="Name of the package.") + name = String( + repr=True, + label='package name', + help='Name of the package.') - version = String(repr=True, label="package version", help="Version of the package as a string.") + version = String( + repr=True, + label='package version', + help='Version of the package as a string.') qualifiers = Mapping( default=None, value_type=str, converter=lambda v: normalize_qualifiers(v, encode=False), - label="package qualifiers", - help="Mapping of key=value pairs qualifiers for this package", - ) + label='package qualifiers', + help='Mapping of key=value pairs qualifiers for this package') subpath = String( - label="extra package subpath", - help="Subpath inside a package and relative to the root " "of this package", - ) + label='extra package subpath', + help='Subpath inside a package and relative to the root ' + 'of this package') @property def purl(self): @@ -287,10 +311,10 @@ def set_purl(self, package_url): def to_dict(self, **kwargs): mapping = super().to_dict(**kwargs) - mapping["purl"] = self.purl + mapping['purl'] = self.purl if self.qualifiers: - mapping["qualifiers"] = normalize_qualifiers( + mapping['qualifiers'] = normalize_qualifiers( qualifiers=self.qualifiers, encode=False, ) @@ -306,61 +330,49 @@ class DependentPackage(ModelMixin): purl = String( repr=True, - label="Dependent package URL", - help="A compact purl package URL. Typically when there is an " - "unresolved requirement, there is no version. " - "If the dependency is resolved, the version should be added to " - "the purl", - ) + label='Dependent package URL', + help='A compact purl package URL. Typically when there is an ' + 'unresolved requirement, there is no version. ' + 'If the dependency is resolved, the version should be added to ' + 'the purl') extracted_requirement = String( repr=True, - label="extracted version requirement", - help="String for the original version requirements and constraints. " - "Package-type specific and as found originally in a datafile.", - ) + label='extracted version requirement', + help='String for the original version requirements and constraints. ' + 'Package-type specific and as found originally in a datafile.') # ToDo: add `vers` support. See https://github.com/nexB/univers/blob/main/src/univers/version_range.py scope = String( repr=True, - label="dependency scope", - help="The scope of this dependency, such as runtime, install, etc. " - "This is package-type specific and is the original scope string.", - ) + label='dependency scope', + help='The scope of this dependency, such as runtime, install, etc. ' + 'This is package-type specific and is the original scope string.') is_runtime = Boolean( default=True, - label="is runtime flag", - help="True if this dependency is a runtime dependency.", - ) + label='is runtime flag', + help='True if this dependency is a runtime dependency.') is_optional = Boolean( default=False, - label="is optional flag", - help="True if this dependency is an optional dependency", - ) + label='is optional flag', + help='True if this dependency is an optional dependency') is_resolved = Boolean( default=False, - label="is resolved flag", - help="True if this dependency version requirement has " - "been resolved and this dependency url points to an " - "exact version.", - ) + label='is resolved flag', + help='True if this dependency version requirement has ' + 'been resolved and this dependency url points to an ' + 'exact version.') resolved_package = Mapping( - label="resolved package data", - help="A mapping of resolved package data for this dependent package, " - "either from the datafile or collected from another source. Some " - "lockfiles for Composer or Cargo contain extra dependency data.", - ) - - # dependencies = List( - # item_type="DependentPackage", - # label="dependencies", - # help="A list of DependentPackage for this package.", - # ) + label='resolved package data', + help='A mapping of resolved package data for this dependent package, ' + 'either from the datafile or collected from another source. Some ' + 'lockfiles for Composer or Cargo contain extra dependency data.' + ) @attr.attributes(slots=True) @@ -369,28 +381,29 @@ class Dependency(DependentPackage): Top-level dependency instance from parsed package data collected from data files such as a package manifest or lockfile. """ - dependency_uid = String( - label="Dependency unique id", - help="A unique identifier for this dependency instance." - "Consists of the dependency purl with a UUID qualifier.", + label='Dependency unique id', + help='A unique identifier for this dependency instance.' + 'Consists of the dependency purl with a UUID qualifier.' ) # TODO: should we also repeat the purl here: this may be redundant but this # would help avoid lookups for_package_uid = String( - label="A Package unique id", - help="The unique id of the package instance to which this dependency " - "file belongs. This is the purl with a uuid qualifier.", + label='A Package unique id', + help='The unique id of the package instance to which this dependency ' + 'file belongs. This is the purl with a uuid qualifier.' ) datafile_path = String( - label="Path to datafile.", - help="A POSIX path string to the package datafile that describes this " "dependency.", + label='Path to datafile.', + help='A POSIX path string to the package datafile that describes this ' + 'dependency.' ) datasource_id = String( - label="datasource id", help="Datasource identifier for the source of these package data." + label='datasource id', + help='Datasource identifier for the source of these package data.' ) def __attrs_post_init__(self, *args, **kwargs): @@ -415,9 +428,9 @@ def from_dependent_package( # make a copy dependent_package = dict(dependent_package) - dependent_package["datafile_path"] = datafile_path - dependent_package["datasource_id"] = datasource_id - dependent_package["for_package_uid"] = package_uid + dependent_package['datafile_path'] = datafile_path + dependent_package['datasource_id'] = datasource_id + dependent_package['for_package_uid'] = package_uid return cls.from_dict(dependent_package) @@ -445,9 +458,7 @@ def from_dependent_packages( ) else: if TRACE: - logger_debug( - f" Dependency.from_dependent_packages: dependent_package (does not have purl): {dependent_package}" - ) + logger_debug(f' Dependency.from_dependent_packages: dependent_package (does not have purl): {dependent_package}') pass @@ -456,48 +467,47 @@ class FileReference(ModelMixin): """ A reference to a file in a files listing from a manifest or data file. """ - path = String( - label="Path of this file.", - help="The file or directory POSIX path. The actual root for this path " - "is specific to a datafile format. For instance it is the rootfs " - "root for Linux system packages.", + label='Path of this file.', + help='The file or directory POSIX path. The actual root for this path ' + 'is specific to a datafile format. For instance it is the rootfs ' + 'root for Linux system packages.', repr=True, ) size = Integer( - label="file size", - help="size of the file in bytes", + label='file size', + help='size of the file in bytes', repr=False, ) sha1 = String( - label="SHA1 checksum", - help="SHA1 checksum for this file in hexadecimal", + label='SHA1 checksum', + help='SHA1 checksum for this file in hexadecimal', repr=False, ) md5 = String( - label="MD5 checksum", - help="MD5 checksum for this file in hexadecimal", + label='MD5 checksum', + help='MD5 checksum for this file in hexadecimal', repr=False, ) sha256 = String( - label="SHA256 checksum", - help="SHA256 checksum for this file in hexadecimal", + label='SHA256 checksum', + help='SHA256 checksum for this file in hexadecimal', repr=False, ) sha512 = String( - label="SHA512 checksum", - help="SHA512 checksum for this file in hexadecimal", + label='SHA512 checksum', + help='SHA512 checksum for this file in hexadecimal', repr=False, ) extra_data = Mapping( - label="extra data", - help="A mapping of arbitrary extra file reference data.", + label='extra data', + help='A mapping of arbitrary extra file reference data.', ) def update(self, other): @@ -523,131 +533,141 @@ class PackageData(IdentifiablePackageData): """ primary_language = String( - label="Primary programming language", - help="Primary programming language", - ) + label='Primary programming language', + help='Primary programming language',) description = String( - label="Description", - help="Description for this package. " - "By convention the first should be a summary when available.", - ) + label='Description', + help='Description for this package. ' + 'By convention the first should be a summary when available.') - release_date = Date(label="release date", help="Release date of the package") + release_date = Date( + label='release date', + help='Release date of the package') parties = List( item_type=Party, - label="parties", - help="A list of parties such as a person, project or organization.", - ) + label='parties', + help='A list of parties such as a person, project or organization.') - keywords = List(item_type=str, label="keywords", help="A list of keywords.") + keywords = List( + item_type=str, + label='keywords', + help='A list of keywords.') - homepage_url = String(label="homepage URL", help="URL to the homepage for this package.") + homepage_url = String( + label='homepage URL', + help='URL to the homepage for this package.') - download_url = String(label="Download URL", help="A direct download URL.") + download_url = String( + label='Download URL', + help='A direct download URL.') size = Integer( - default=None, label="download size", help="size of the package download in bytes" - ) + default=None, + label='download size', + help='size of the package download in bytes') sha1 = String( - label="SHA1 checksum", help="SHA1 checksum for this package download in hexadecimal" - ) + label='SHA1 checksum', + help='SHA1 checksum for this package download in hexadecimal') - md5 = String(label="MD5 checksum", help="MD5 checksum for this package download in hexadecimal") + md5 = String( + label='MD5 checksum', + help='MD5 checksum for this package download in hexadecimal') sha256 = String( - label="SHA256 checksum", help="SHA256 checksum for this package download in hexadecimal" - ) + label='SHA256 checksum', + help='SHA256 checksum for this package download in hexadecimal') sha512 = String( - label="SHA512 checksum", help="SHA512 checksum for this package download in hexadecimal" - ) + label='SHA512 checksum', + help='SHA512 checksum for this package download in hexadecimal') bug_tracking_url = String( - label="bug tracking URL", help="URL to the issue or bug tracker for this package" - ) + label='bug tracking URL', + help='URL to the issue or bug tracker for this package') - code_view_url = String(label="code view URL", help="a URL where the code can be browsed online") + code_view_url = String( + label='code view URL', + help='a URL where the code can be browsed online') vcs_url = String( - help="a URL to the VCS repository in the SPDX form of: " - "https://github.com/nexb/scancode-toolkit.git@405aaa4b3 " - 'See SPDX specification "Package Download Location" ' - "at https://spdx.org/spdx-specification-21-web-version#h.49x2ik5 " - ) + help='a URL to the VCS repository in the SPDX form of: ' + 'https://github.com/nexb/scancode-toolkit.git@405aaa4b3 ' + 'See SPDX specification "Package Download Location" ' + 'at https://spdx.org/spdx-specification-21-web-version#h.49x2ik5 ') copyright = String( - label="Copyright", help="Copyright statements for this package. Typically one per line." - ) + label='Copyright', + help='Copyright statements for this package. Typically one per line.') license_expression = String( - label="license expression", - help="The license expression for this package typically derived " - "from its declared license or from some other type-specific " - "routine or convention.", - ) + label='license expression', + help='The license expression for this package typically derived ' + 'from its declared license or from some other type-specific ' + 'routine or convention.') declared_license = String( - label="declared license", - help="The declared license mention, tag or text as found in a " - "package manifest. This can be a string, a list or dict of " - "strings possibly nested, as found originally in the manifest.", - ) + label='declared license', + help='The declared license mention, tag or text as found in a ' + 'package manifest. This can be a string, a list or dict of ' + 'strings possibly nested, as found originally in the manifest.') - notice_text = String(label="notice text", help="A notice text for this package.") + notice_text = String( + label='notice text', + help='A notice text for this package.') source_packages = List( item_type=str, - label="List of related source code package purls", + label='List of related source code package purls', help='A list of related source code Package URLs (aka. "purl") for ' - 'this package. For instance an SRPM is the "source package" for a ' - "binary RPM.", - ) + 'this package. For instance an SRPM is the "source package" for a ' + 'binary RPM.' + ) file_references = List( item_type=FileReference, - label="referenced files", - help="List of file paths and details for files referenced in a package " - "manifest. These may not actually exist on the filesystem. " - "The exact semantics and base of these paths is specific to a " - "package type or datafile format.", + label='referenced files', + help='List of file paths and details for files referenced in a package ' + 'manifest. These may not actually exist on the filesystem. ' + 'The exact semantics and base of these paths is specific to a ' + 'package type or datafile format.' ) extra_data = Mapping( - label="extra data", - help="A mapping of arbitrary extra package data.", + label='extra data', + help='A mapping of arbitrary extra package data.', ) dependencies = List( item_type=DependentPackage, - label="dependencies", - help="A list of DependentPackage for this package.", + label='dependencies', + help='A list of DependentPackage for this package.' ) repository_homepage_url = String( - label="package repository homepage URL.", - help="URL to the page for this package in its package repository. " - "This is typically different from the package homepage URL proper.", - ) + label='package repository homepage URL.', + help='URL to the page for this package in its package repository. ' + 'This is typically different from the package homepage URL proper.' + ) repository_download_url = String( - label="package repository download URL.", - help="download URL to download the actual archive of code of this " - "package in its package repository. " - "This may be different from the actual download URL.", - ) + label='package repository download URL.', + help='download URL to download the actual archive of code of this ' + 'package in its package repository. ' + 'This may be different from the actual download URL.' + ) api_data_url = String( - label="package repository API URL.", - help="API URL to obtain structured data for this package such as the " - "URL to a JSON or XML api its package repository.", - ) + label='package repository API URL.', + help='API URL to obtain structured data for this package such as the ' + 'URL to a JSON or XML api its package repository.' + ) datasource_id = String( - label="datasource id", - help="Datasource identifier for the source of these package data.", + label='datasource id', + help='Datasource identifier for the source of these package data.', repr=True, ) @@ -655,9 +675,9 @@ def to_dict(self, with_details=True, **kwargs): mapping = super().to_dict(with_details=with_details, **kwargs) if not with_details: # these are not used in the Package subclass - mapping.pop("file_references", None) - mapping.pop("dependencies", None) - mapping.pop("datasource_id", None) + mapping.pop('file_references', None) + mapping.pop('dependencies', None) + mapping.pop('datasource_id', None) return mapping @@ -681,21 +701,17 @@ def from_dict(cls, mapping): # these are computed attributes serialized on a package # that should not be recreated when de-serializing - computed_attributes = set( - [ - "purl", - ] - ) + computed_attributes = set(['purl', ]) fields_by_name = attr.fields_dict(cls) - extra_data = mapping.get("extra_data", {}) or {} + extra_data = mapping.get('extra_data', {}) or {} package_data = {} list_fields_by_item = { - "parties": Party, - "dependencies": DependentPackage, - "file_references": FileReference, + 'parties': Party, + 'dependencies': DependentPackage, + 'file_references': FileReference, } for name, value in mapping.items(): @@ -714,7 +730,7 @@ def from_dict(cls, mapping): else: raise Exception( f'Invalid package "scan_data" with duplicated name: {name!r}={value!r} ' - f"present both as attribute AND as extra_data: {name!r}={extra_data[name]!r}" + f'present both as attribute AND as extra_data: {name!r}={extra_data[name]!r}' ) # re-hydrate lists of typed objects @@ -739,14 +755,50 @@ def _rehydrate_list(cls, values): base_msg = 'Invalid package "scan_data "with unknown data structure.' if not isinstance(values, list) and not all(isinstance(v, dict) for v in values): raise Exception( - f"{base_msg}. Expected the value to be a list of dicts and not: " - f"{type(values)!r} for class: {cls!r}" + f'{base_msg}. Expected the value to be a list of dicts and not: ' + f'{type(values)!r} for class: {cls!r}' ) for val in values: yield cls.from_dict(val) +def compute_normalized_license(declared_license, expression_symbols=None): + """ + Return a normalized license_expression string from the ``declared_license``. + Return 'unknown' if there is a declared license but it cannot be detected + (including on errors) and return None if there is no declared license. + + Use the ``expression_symbols`` mapping of {lowered key: LicenseSymbol} + if provided. Otherwise use the standard SPDX license symbols. + """ + # Ensure declared license is always a string + if not isinstance(declared_license, str): + declared_license = repr(declared_license) + + if not declared_license: + return + + if not licensing: + if TRACE: + logger_debug( + f'Failed to compute license for {declared_license!r}: ' + 'cannot import packagedcode.licensing') + return 'unknown' + + try: + return licensing.get_normalized_expression( + query_string=declared_license, + expression_symbols=expression_symbols + ) + except Exception as e: + # we never fail just for this + # FIXME: add logging + if TRACE: + logger_debug(f'Failed to compute license for {declared_license!r}: {e!r}') + return 'unknown' + + class DatafileHandler: """ A base handler class to handle any package manifests, lockfiles and data @@ -814,7 +866,11 @@ def is_datafile(cls, location, filetypes=tuple(), _bare_filename=False): filetypes = filetypes or cls.filetypes if not filetypes: return True - return False + # we check for contenttype IFF this is available + if contenttype: + T = contenttype.get_type(location) + actual_type = T.filetype_file.lower() + return any(ft in actual_type for ft in filetypes) @classmethod def parse(cls, location): @@ -827,13 +883,286 @@ def parse(cls, location): """ raise NotImplementedError + @classmethod + def assemble(cls, package_data, resource, codebase): + """ + Given a ``package_data`` PackageData found in the ``resource`` datafile + of the ``codebase``, assemble package their files and dependencies + from one or more datafiles. + + Update ``codebase`` Resources with the package they are for. + + Yield items that can be of these types: + + - a Package to add to top-level packages with its list of Files. + - Resources that have been handled --such as this datafiles-- that should + not be further processed, + - a Dependency to add to top-level dependencies + + The approach is to find and process all the neighboring related datafiles + to this datafile at once. + + The default implementation handles this datafile only: + + - It does not include other related datafiles and manifests. + - It considers only this datafile as a package file + - It returns only this datafile resource as having been processed + + Subclasses should override to implement more complex cases where + multiple datafiles are combined and some files can be ignored. + """ + datafile_path = resource.path + # do we have enough to create a package? + if package_data.purl: + package = Package.from_package_data( + package_data=package_data, + datafile_path=datafile_path, + ) + package_uid = package.package_uid + + if not package.license_expression: + package.license_expression = cls.compute_normalized_license(package) + + cls.assign_package_to_resources( + package=package, + resource=resource, + codebase=codebase, + ) + + yield package + else: + # we have no package, so deps are not for a specific package uid + package_uid = None + + # in all cases yield possible dependencies + dependent_packages = package_data.dependencies + if dependent_packages: + yield from Dependency.from_dependent_packages( + dependent_packages=dependent_packages, + datafile_path=datafile_path, + datasource_id=package_data.datasource_id, + package_uid=package_uid, + ) + # we yield this as we do not want this further processed + yield resource + + @classmethod + def compute_normalized_license(cls, package): + """ + Return a computed license expression string or None given a ``package`` + Package object. + + Called only when using the default assemble() implementation. + Subclass can override as needed. + """ + if package.declared_license and not package.license_expression: + try: + license_expression = compute_normalized_license(package.declared_license) + except Exception: + if SCANCODE_DEBUG_PACKAGE_API: + raise + license_expression = 'unknown' + + if TRACE: + logger_debug(f' compute_normalized_license: license_expression: {license_expression}') + + return license_expression + + @classmethod + def assign_package_to_resources(cls, package, resource, codebase): + """ + Set the "for_packages" attributes to ``package`` given a + starting ``resource`` in the ``codebase``. + + This default implementation assigns the package to the whole + ``resource`` tree. Since ``resource`` is a file y default, this means + that only the datafile ``resource`` is assigned to the ``package`` by + default. + + Called only when using the default assemble() implementation. + Subclass can override as needed to assign a package to its files. + """ + # NOTE: we do not attach files to the Package level. Instead we + # update `for_packages` of a codebase resource. + package_uid = package.package_uid + if resource and package_uid: + resource.for_packages.append(package_uid) + resource.save(codebase) + for res in resource.walk(codebase): + res.for_packages.append(package_uid) + res.save(codebase) + + @classmethod + def assign_package_to_parent_tree(cls, package, resource, codebase): + """ + Set the "for_packages" attributes to ``package`` for the whole + resource tree of the parent of a ``resource`` object in the + ``codebase``. If codebase doesn't have a parent, just set the + attribute for that resource only. + + This is a convenience method that subclasses can reuse when overriding + `assign_package_to_resources()` + """ + if resource.has_parent(): + parent = resource.parent(codebase) + cls.assign_package_to_resources(package, parent, codebase) + else: + cls.assign_package_to_resources(package, resource, codebase) + + @classmethod + def assemble_from_many(cls, pkgdata_resources, codebase,): + """ + Yield Package, Resources or Dependency given a ``pkgdata_resources`` + list of tuple (PackageData, Resource) in ``codebase``. + + Create a Package from the first package_data item. Update this package + with other items. Assign to this Package the file tree from the parent + of the first resource item. + + Because of this, set the order of ``pkgdata_resources`` items carefully. + + This is a convenience method that subclasses can reuse when overriding + `assemble()` + + NOTE: ATTENTION!: this may not work well for datafile that yield + multiple PackageData for unrelated Packages + """ + package = None + package_uid = None + base_resource = None + + # process each package in sequence. The first item creates a package and + # the other only update + for package_data, resource in pkgdata_resources: + if not base_resource: + base_resource = resource + + if not package: + # create package from the first item first package_data + if package_data.purl: + package = Package.from_package_data( + package_data=package_data, + datafile_path=resource.path, + ) + package_uid = package.package_uid + if package_uid: + resource.for_packages.append(package_uid) + resource.save(codebase) + else: + # FIXME: What is the package_data is NOT for the same package as package? + # FIXME: What if the update did not do anything? (it does return True or False) + # FIXME: There we would be missing out packges AND/OR errors + package.update( + package_data=package_data, + datafile_path=resource.path, + ) + if package_uid: + resource.for_packages.append(package_uid) + resource.save(codebase) + + # in all cases yield possible dependencies + dependent_packages = package_data.dependencies + if dependent_packages: + yield from Dependency.from_dependent_packages( + dependent_packages=dependent_packages, + datafile_path=resource.path, + datasource_id=package_data.datasource_id, + package_uid=package_uid, + ) + + # we yield this as we do not want this further processed + yield resource + + # the whole parent subtree of the base_resource is for this package + if package_uid: + for res in base_resource.walk(codebase): + res.for_packages.append(package_uid) + res.save(codebase) + + if package: + if not package.license_expression: + package.license_expression = cls.compute_normalized_license(package) + yield package + + @classmethod + def assemble_from_many_datafiles(cls, datafile_name_patterns, directory, codebase): + """ + Assemble Package and Dependency from package data of the datafiles found + in multiple ``datafile_name_patterns`` name patterns (case- sensitive) + found in the ``directory`` Resource. + + Create a Package from the first package data item. Update this package + with other items. Assign to this Package the file tree from the parent + of the first resource item. + + Because of this, set the order of ``datafile_name_patterns`` items carefully. + + This is a convenience method that subclasses can reuse when overriding + `assemble()` + + NOTE: ATTENTION!: this will not work well for datafile that yields + multiple PackageData for unrelated Packages. + """ + if TRACE: + logger_debug(f'assemble_from_many_datafiles: datafile_name_patterns: {datafile_name_patterns!r}') + + if not codebase.has_single_resource: + siblings = list(directory.children(codebase)) + else: + if directory: + siblings = [directory] + else: + siblings = [] + + pkgdata_resources = [] + + # we iterate on datafile_name_patterns because their order matters + for datafile_name_pattern in datafile_name_patterns: + for sibling in siblings: + if fnmatchcase(sibling.name, datafile_name_pattern): + for package_data in sibling.package_data: + package_data = PackageData.from_dict(package_data) + pkgdata_resources.append((package_data, sibling,)) + + if pkgdata_resources: + if TRACE: + logger_debug(f' assemble_from_many_datafiles: pkgdata_resources: {pkgdata_resources!r}') + + yield from cls.assemble_from_many( + pkgdata_resources=pkgdata_resources, + codebase=codebase, + ) + + @classmethod + def create_default_package_data(cls, **kwargs): + """ + Return an empty PackageData using default values and the provided kwargs. + """ + return PackageData( + datasource_id=cls.datasource_id, + type=cls.default_package_type, + primary_language=cls.default_primary_language, + **kwargs, + ) + + +class NonAssemblableDatafileHandler(DatafileHandler): + """ + A handler that has no default implmentation for the assemble method, e.g., + it will not alone trigger the creation of a top-level Pacakge. + """ + + @classmethod + def assemble(cls, package_data, resource, codebase): + return [] + def build_package_uid(purl): """ Return a purl string with a UUID qualifier given a ``purl`` string . """ purl = PackageURL.from_string(purl) - purl.qualifiers["uuid"] = str(uuid.uuid4()) + purl.qualifiers['uuid'] = str(uuid.uuid4()) return str(purl) @@ -842,15 +1171,15 @@ def build_purl(mapping): Return a PackageURL from a ``mapping`` or None if essential type and name fields are missing. """ - ptype = mapping.get("type") - name = mapping.get("name") + ptype = mapping.get('type') + name = mapping.get('name') if not ptype or not name: return - namespace = mapping.get("namespace") - version = mapping.get("version") - qualifiers = mapping.get("qualifiers") or {} - subpath = mapping.get("subpath") + namespace = mapping.get('namespace') + version = mapping.get('version') + qualifiers = mapping.get('qualifiers') or {} + subpath = mapping.get('subpath') return PackageURL( type=ptype, name=name, @@ -869,21 +1198,21 @@ class Package(PackageData): """ package_uid = String( - label="Package unique id", - help="A unique identifier for this package instance." - "Consists of the package purl with a UUID qualifier.", + label='Package unique id', + help='A unique identifier for this package instance.' + 'Consists of the package purl with a UUID qualifier.' ) datafile_paths = List( item_type=str, - label="List of datafile paths", - help="List of datafile paths used to create this package.", + label='List of datafile paths', + help='List of datafile paths used to create this package.' ) datasource_ids = List( item_type=str, - label="datasource ids", - help="List of the datasource ids used to create this package.", + label='datasource ids', + help='List of the datasource ids used to create this package.' ) def __attrs_post_init__(self, *args, **kwargs): @@ -905,12 +1234,12 @@ def from_package_data(cls, package_data, datafile_path): elif isinstance(package_data, dict): # make a copy package_data_mapping = dict(package_data.items()) - dsid = package_data["datasource_id"] + dsid = package_data['datasource_id'] elif package_data: - raise Exception(f"Invalid type: {package_data!r}", package_data) + raise Exception(f'Invalid type: {package_data!r}', package_data) - package_data_mapping["datafile_paths"] = [datafile_path] - package_data_mapping["datasource_ids"] = [dsid] + package_data_mapping['datafile_paths'] = [datafile_path] + package_data_mapping['datasource_ids'] = [dsid] return cls.from_dict(package_data_mapping) @@ -969,7 +1298,7 @@ def update(self, package_data, datafile_path, replace=False): if not self.is_compatible(package_data, include_qualifiers=False): if TRACE_UPDATE: - logger_debug(f"update: {self.purl} not compatible with: {package_data.purl}") + logger_debug(f'update: {self.purl} not compatible with: {package_data.purl}') return False # always append these new items @@ -980,53 +1309,84 @@ def update(self, package_data, datafile_path, replace=False): new_package_data = package_data.to_dict() # update for these means combining lists of items from both sides - list_fields = set( - [ - "parties", - "dependencies", - "file_references", - ] - ) + list_fields = set([ + 'parties', + 'dependencies', + 'file_references', + ]) for name, value in existing.items(): new_value = new_package_data.get(name) if TRACE_UPDATE: - logger_debug(f"update: {name!r}={value!r} with new_value: {new_value!r}") + logger_debug(f'update: {name!r}={value!r} with new_value: {new_value!r}') if not new_value: - if TRACE_UPDATE: - logger_debug(" No new value: skipping") + if TRACE_UPDATE: logger_debug(' No new value: skipping') continue if not value: - if TRACE_UPDATE: - logger_debug(" set existing value to new") + if TRACE_UPDATE: logger_debug(' set existing value to new') setattr(self, name, new_value) continue if replace: - if TRACE_UPDATE: - logger_debug(" replace existing value to new") + if TRACE_UPDATE: logger_debug(' replace existing value to new') setattr(self, name, new_value) continue # here we do not replace... but we still merge lists/mappings - if name == "extra_data": + if name == 'extra_data': value.update(new_value) if name in list_fields: - if TRACE_UPDATE: - logger_debug(" merge lists of values") + if TRACE_UPDATE: logger_debug(' merge lists of values') merged = merge_sequences(list1=value, list2=new_value) setattr(self, name, merged) elif TRACE_UPDATE and value != new_value: - if TRACE_UPDATE: - logger_debug(" skipping update: no replace") + if TRACE_UPDATE: logger_debug(' skipping update: no replace') return True + def get_packages_files(self, codebase): + """ + Yield all the Resource of this package found in codebase. + """ + package_uid = self.package_uid + if package_uid: + for resource in codebase.walk(): + if package_uid in resource.for_packages: + yield resource + + +@attr.attributes(slots=True) +class PackageWithResources(Package): + """ + A Package with Resources. + """ + + resources = List( + item_type=Resource, + label='List of Resources', + help='List of Resources for this package.', + ) + + def to_dict(self): + package_data = super().to_dict() + package_data['resources'] = [resource.to_dict() for resource in self.resources] + return package_data + + +def get_files_for_packages(codebase): + """ + Yield tuple of (Resource, package_uid) for all resources in codebase that are + for a package. + """ + for resource in codebase.walk(): + for package_uid in resource.for_packages: + yield resource, package_uid + def merge_sequences(list1, list2, **kwargs): """ diff --git a/src/_packagedcode/pypi.py b/src/_packagedcode/pypi.py index 9cdef5b2..2eb78ce8 100644 --- a/src/_packagedcode/pypi.py +++ b/src/_packagedcode/pypi.py @@ -1,3 +1,4 @@ + # # Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. @@ -8,8 +9,6 @@ # import ast -import base64 -import io import json import logging import os @@ -20,7 +19,6 @@ from pathlib import Path import dparse2 -import importlib_metadata import pip_requirements_parser import pkginfo2 from commoncode import fileutils @@ -31,8 +29,15 @@ from _packagedcode import models from _packagedcode.utils import build_description +from _packagedcode.utils import combine_expressions +from _packagedcode.utils import yield_dependencies_from_package_data +from _packagedcode.utils import yield_dependencies_from_package_resource # FIXME: we always want to use the external library rather than the built-in for now +import importlib_metadata +import base64 +from commoncode.fileutils import as_posixpath + try: from zipfile import Path as ZipPath except ImportError: @@ -40,11 +45,10 @@ """ Detect and collect Python packages information. - -Originally vendored from scancode-toolkit packagedcode.pypi """ # TODO: add support for poetry and setup.cfg and metadata.json # TODO: add support for pex, pyz, etc. +# TODO: Add missing ABOUT file for Pyserial code TRACE = False @@ -60,16 +64,217 @@ def logger_debug(*args): logger.setLevel(logging.DEBUG) def logger_debug(*args): - return logger.debug(" ".join(isinstance(a, str) and a or repr(a) for a in args)) + return print(' '.join(isinstance(a, str) and a or repr(a) for a in args)) + + +class BasePypiHandler(models.DatafileHandler): + + @classmethod + def compute_normalized_license(cls, package): + return compute_normalized_license(package.declared_license) + + +class PythonEggPkgInfoFile(BasePypiHandler): + datasource_id = 'pypi_egg_pkginfo' + default_package_type = 'pypi' + default_primary_language = 'Python' + path_patterns = ('*/EGG-INFO/PKG-INFO',) + description = 'PyPI extracted egg PKG-INFO' + documentation_url = 'https://peps.python.org/pep-0376/' + + @classmethod + def parse(cls, location): + yield parse_metadata( + location=location, + datasource_id=cls.datasource_id, + package_type=cls.default_package_type, + ) + + @classmethod + def assign_package_to_resources(cls, package, resource, codebase): + # two levels up + root = resource.parent(codebase).parent(codebase) + if root: + return models.DatafileHandler.assign_package_to_resources(package, root, codebase) + + +class PythonEditableInstallationPkgInfoFile(BasePypiHandler): + datasource_id = 'pypi_editable_egg_pkginfo' + default_package_type = 'pypi' + default_primary_language = 'Python' + path_patterns = ('*.egg-info/PKG-INFO',) + description = 'PyPI editable local installation PKG-INFO' + documentation_url = 'https://peps.python.org/pep-0376/' + + @classmethod + def parse(cls, location): + yield parse_metadata( + location=location, + datasource_id=cls.datasource_id, + package_type=cls.default_package_type, + ) + + @classmethod + def assign_package_to_resources(cls, package, resource, codebase): + # only the parent for now... though it can be more complex + return models.DatafileHandler.assign_package_to_parent_tree(package, resource, codebase) + + +class BaseExtractedPythonLayout(BasePypiHandler): + """ + Base class for development repos, sdist tarballs and other related extracted + layourt for Python packages that can use and mix multiple datafiles. + """ + + @classmethod + def assemble(cls, package_data, resource, codebase): + # a source distribution can have many manifests + datafile_name_patterns = ( + 'Pipfile.lock', + 'Pipfile', + ) + PipRequirementsFileHandler.path_patterns + + # TODO: we want PKG-INFO first, then (setup.py, setup.cfg), then pyproject.toml for poetry + # then we have the rest of the lock files (pipfile, pipfile.lock, etc.) + + package_resource = None + if resource.name == 'PKG-INFO': + package_resource = resource + elif resource.name in datafile_name_patterns: + if resource.has_parent(): + siblings = resource.siblings(codebase) + package_resource = [r for r in siblings if r.name == 'PKG-INFO'] + if package_resource: + package_resource = package_resource[0] + + package = None + if package_resource: + pkg_data = package_resource.package_data[0] + pkg_data = models.PackageData.from_dict(pkg_data) + if pkg_data.purl: + package = models.Package.from_package_data( + package_data=pkg_data, + datafile_path=package_resource.path, + ) + package_resource.for_packages.append(package.package_uid) + package_resource.save(codebase) + yield package_resource + + yield from yield_dependencies_from_package_data( + package_data=pkg_data, + datafile_path=package_resource.path, + package_uid=package.package_uid + ) + else: + setup_resources = [] + if resource.has_parent(): + siblings = resource.siblings(codebase) + setup_resources = [ + r for r in siblings + if r.name in ('setup.py', 'setup.cfg') + and r.package_data + ] + + setup_package_data = [ + (setup_resource, models.PackageData.from_dict(setup_resource.package_data[0])) + for setup_resource in setup_resources + ] + setup_package_data = sorted(setup_package_data, key=lambda s: bool(s[1].purl), reverse=True) + for setup_resource, setup_pkg_data in setup_package_data: + if setup_pkg_data.purl: + if not package: + package = models.Package.from_package_data( + package_data=setup_pkg_data, + datafile_path=setup_resource.path, + ) + package_resource = setup_resource + else: + package.update(setup_pkg_data, setup_resource.path) + if package: + for setup_resource, setup_pkg_data in setup_package_data: + setup_resource.for_packages.append(package.package_uid) + setup_resource.save(codebase) + yield setup_resource + + yield from yield_dependencies_from_package_data( + package_data=setup_pkg_data, + datafile_path=setup_resource.path, + package_uid=package.package_uid + ) + + if package: + if not package.license_expression: + package.license_expression = compute_normalized_license(package.declared_license) + package_uid = package.package_uid + + root = package_resource.parent(codebase) + if root: + for py_res in cls.walk_pypi(resource=root, codebase=codebase): + if py_res.is_dir: + continue + if package_uid and package_uid not in py_res.for_packages: + py_res.for_packages.append(package_uid) + py_res.save(codebase) + yield py_res + elif codebase.has_single_resource: + if package_uid and package_uid not in package_resource.for_packages: + package_resource.for_packages.append(package_uid) + package_resource.save(codebase) + + yield package + + else: + package_uid = None + + if package_resource: + for sibling in package_resource.siblings(codebase): + if sibling and sibling.name in datafile_name_patterns: + yield from yield_dependencies_from_package_resource( + resource=sibling, + package_uid=package_uid + ) + + if package_uid and package_uid not in sibling.for_packages: + sibling.for_packages.append(package_uid) + sibling.save(codebase) + yield sibling + + @classmethod + def walk_pypi(cls, resource, codebase): + """ + Walk the ``codebase`` Codebase top-down, breadth-first starting from the + ``resource`` Resource. + + Skip the directory named "site-packages": this avoids + reporting nested vendored packages as being part of their parent. + Instead they will be reported on their own. + """ + for child in resource.children(codebase): + if child.name == 'site-packages': + continue + + yield child + + if child.is_dir: + for subchild in cls.walk_pypi(child, codebase): + yield subchild -class PythonSdistPkgInfoFile(models.DatafileHandler): - datasource_id = "pypi_sdist_pkginfo" - default_package_type = "pypi" - default_primary_language = "Python" - path_patterns = ("*/PKG-INFO",) - description = "PyPI extracted sdist PKG-INFO" - documentation_url = "https://peps.python.org/pep-0314/" +class PythonSdistPkgInfoFile(BaseExtractedPythonLayout): + datasource_id = 'pypi_sdist_pkginfo' + default_package_type = 'pypi' + default_primary_language = 'Python' + path_patterns = ('*/PKG-INFO',) + description = 'PyPI extracted sdist PKG-INFO' + documentation_url = 'https://peps.python.org/pep-0314/' + + @classmethod + def is_datafile(cls, location): + return ( + super().is_datafile(location) and + not PythonEggPkgInfoFile.is_datafile(location) and + not PythonEditableInstallationPkgInfoFile.is_datafile(location) + ) @classmethod def parse(cls, location): @@ -80,13 +285,13 @@ def parse(cls, location): ) -class PythonInstalledWheelMetadataFile(models.DatafileHandler): - datasource_id = "pypi_wheel_metadata" - path_patterns = ("*.dist-info/METADATA",) - default_package_type = "pypi" - default_primary_language = "Python" - description = "PyPI installed wheel METADATA" - documentation_url = "https://packaging.python.org/en/latest/specifications/core-metadata/" +class PythonInstalledWheelMetadataFile(BasePypiHandler): + datasource_id = 'pypi_wheel_metadata' + path_patterns = ('*.dist-info/METADATA',) + default_package_type = 'pypi' + default_primary_language = 'Python' + description = 'PyPI installed wheel METADATA' + documentation_url = 'https://packaging.python.org/en/latest/specifications/core-metadata/' @classmethod def parse(cls, location): @@ -96,22 +301,120 @@ def parse(cls, location): package_type=cls.default_package_type, ) + @classmethod + def assign_package_to_resources(cls, package, resource, codebase): + """ + Assign files to package for an installed wheel. This requires a bit + of navigation around as the files can be in multiple places. + """ + site_packages = resource.parent(codebase).parent(codebase).parent(codebase) + if not site_packages: + return + package_data = resource.package_data + assert len(resource.package_data) == 1, ( + f'Unsupported Pypi METADATA wheel structure: {resource.path!r} ' + f'with multiple {package_data!r}' + ) + + package_data = models.PackageData.from_dict(package_data[0]) + + package_uid = package.package_uid + + if package_uid: + # save thyself! + resource.for_packages.append(package_uid) + resource.save(codebase) + + # collect actual paths based on the file references + for file_ref in package_data.file_references: + path_ref = file_ref.path + if path_ref.startswith('..'): + # relative paths need special treatment + # most of thense are references to bin ../../../bin/wheel + cannot_resolve = False + ref_resource = None + while path_ref.startswith('..'): + _, _, path_ref.partition('../') + ref_resource = site_packages.parent(codebase) + if not ref_resource: + cannot_resolve = True + break + if cannot_resolve or not ref_resource: + # TODO:w e should log these kind of things + continue + else: + if package_uid: + ref_resource.for_packages.append(package_uid) + ref_resource.save(codebase) + else: + ref_resource = get_resource_for_path( + path=path_ref, + root=site_packages, + codebase=codebase, + ) + if ref_resource and package_uid: + ref_resource.for_packages.append(package_uid) + ref_resource.save(codebase) + + +def get_resource_for_path(path, root, codebase): + """ + Return a resource in ``codebase`` that has a ``path`` relative to the + ``root` Resource + + For example, say we start from this: + path: this/is/that therefore segments [this, is, that] + root: /usr/foo + + We would have these iterations: + iteration1 + root = /usr/foo + segments = [this, is, that] + seg this + segments = [is, that] + children = [/usr/foo/this] + root = /usr/foo/this + + iteration2 + root = /usr/foo/this + segments = [is, that] + seg is + segments = [that] + children = [/usr/foo/this/is] + root = /usr/foo/this/is + + iteration3 + root = /usr/foo/this/is + segments = [that] + seg that + segments = [] + children = [/usr/foo/this/is/that] + root = /usr/foo/this/is/that + + finally return root as /usr/foo/this/is/that + """ + segments = path.strip('/').split('/') + while segments: + seg = segments.pop(0) + children = [c for c in root.children(codebase) if c.name == seg] + if len(children) != 1: + return + else: + root = children[0] + return root + # FIXME: Implement me -class PyprojectTomlHandler(models.DatafileHandler): - datasource_id = "pypi_pyproject_toml" - path_patterns = ("*pyproject.toml",) - default_package_type = "pypi" - default_primary_language = "Python" - description = "Python pyproject.toml" - documentation_url = "https://peps.python.org/pep-0621/" +class PyprojectTomlHandler(models.NonAssemblableDatafileHandler): + datasource_id = 'pypi_pyproject_toml' + path_patterns = ('*pyproject.toml',) + default_package_type = 'pypi' + default_primary_language = 'Python' + description = 'Python pyproject.toml' + documentation_url = 'https://peps.python.org/pep-0621/' -META_DIR_SUFFIXES = ( - ".dist-info", - ".egg-info", - "EGG-INFO", -) +META_DIR_SUFFIXES = '.dist-info', '.egg-info', 'EGG-INFO', def parse_metadata(location, datasource_id, package_type): @@ -135,17 +438,19 @@ def parse_metadata(location, datasource_id, package_type): meta = dist.metadata - name = get_attribute(meta, "Name") - version = get_attribute(meta, "Version") + name = get_attribute(meta, 'Name') + version = get_attribute(meta, 'Version') - urls = get_urls(metainfo=meta, name=name, version=version) + urls, extra_data = get_urls(metainfo=meta, name=name, version=version) dependencies = get_dist_dependencies(dist) + file_references = list(get_file_references(dist)) + package_data = models.PackageData( datasource_id=datasource_id, type=package_type, - primary_language="Python", + primary_language='Python', name=name, version=version, description=get_description(meta, location), @@ -153,9 +458,14 @@ def parse_metadata(location, datasource_id, package_type): keywords=get_keywords(meta), parties=get_parties(meta), dependencies=dependencies, + file_references=file_references, + extra_data=extra_data, **urls, ) + if not package_data.license_expression and package_data.declared_license: + package_data.license_expression = models.compute_normalized_license(package_data.declared_license) + return package_data @@ -166,18 +476,44 @@ def urlsafe_b64decode(data): Copyright (c) 2012-2014 Daniel Holth and contributors. From: https://github.com/pypa/wheel/blob/66208910ab51f4008b034ef4833acfdc920f7606/src/wheel/util.py#L23 """ - pad = b"=" * (4 - (len(data) & 3)) - return base64.urlsafe_b64decode(data.encode("ASCII") + pad) + pad = b'=' * (4 - (len(data) & 3)) + return base64.urlsafe_b64decode(data.encode('ASCII') + pad) -class PypiWheelHandler(models.DatafileHandler): - datasource_id = "pypi_wheel" - path_patterns = ("*.whl",) - # filetypes = ('zip archive',) - default_package_type = "pypi" - default_primary_language = "Python" - description = "PyPI wheel" - documentation_url = "https://peps.python.org/pep-0427/" +def get_file_references(dist): + """ + Yield FileReference found in a ``dist`` importlib_metadata.Distribution. + """ + if not dist.files: + return + + for filepath in dist.files or []: + # FIXME: the path is relative to the "site-packages" directory or the + # root of a wheel but this should be a scan path + ref = models.FileReference( + path=as_posixpath(str(filepath)), + size=filepath.size, + ) + + filehash = filepath.hash + if filehash: + algo = filehash.mode + value = filehash.value + if algo in ('sha256', 'sha512'): + # convert back to hex as this is a base64 without padding otherwise + value = urlsafe_b64decode(value).hex() + setattr(ref, algo, value) + yield ref + + +class PypiWheelHandler(BasePypiHandler): + datasource_id = 'pypi_wheel' + path_patterns = ('*.whl',) + filetypes = ('zip archive',) + default_package_type = 'pypi' + default_primary_language = 'Python' + description = 'PyPI wheel' + documentation_url = 'https://peps.python.org/pep-0427/' @classmethod def parse(cls, location): @@ -186,7 +522,7 @@ def parse(cls, location): if not path.name.endswith(META_DIR_SUFFIXES): continue for metapath in path.iterdir(): - if not metapath.name.endswith("METADATA"): + if not metapath.name.endswith('METADATA'): continue yield parse_metadata( @@ -196,14 +532,14 @@ def parse(cls, location): ) -class PypiEggHandler(models.DatafileHandler): - datasource_id = "pypi_egg" - path_patterns = ("*.egg",) - # filetypes = ('zip archive',) - default_package_type = "pypi" - default_primary_language = "Python" - description = "PyPI egg" - documentation_url = "https://web.archive.org/web/20210604075235/http://peak.telecommunity.com/DevCenter/PythonEggs" +class PypiEggHandler(BasePypiHandler): + datasource_id = 'pypi_egg' + path_patterns = ('*.egg',) + filetypes = ('zip archive',) + default_package_type = 'pypi' + default_primary_language = 'Python' + description = 'PyPI egg' + documentation_url = 'https://web.archive.org/web/20210604075235/http://peak.telecommunity.com/DevCenter/PythonEggs' @classmethod def parse(cls, location): @@ -213,7 +549,7 @@ def parse(cls, location): continue for metapath in path.iterdir(): - if not metapath.name.endswith("PKG-INFO"): + if not metapath.name.endswith('PKG-INFO'): continue yield parse_metadata( @@ -223,17 +559,13 @@ def parse(cls, location): ) -class PypiSdistArchiveHandler(models.DatafileHandler): - datasource_id = "pypi_sdist" - path_patterns = ( - "*.tar.gz", - "*.tar.bz2", - "*.zip", - ) - default_package_type = "pypi" - default_primary_language = "Python" - description = "Python source distribution" - documentation_url = "https://peps.python.org/pep-0643/" +class PypiSdistArchiveHandler(BasePypiHandler): + datasource_id = 'pypi_sdist' + path_patterns = ('*.tar.gz', '*.tar.bz2', '*.zip',) + default_package_type = 'pypi' + default_primary_language = 'Python' + description = 'Python source distribution' + documentation_url = 'https://peps.python.org/pep-0643/' @classmethod def is_datafile(cls, location, filetypes=tuple()): @@ -253,7 +585,7 @@ def parse(cls, location): name = sdist.name version = sdist.version - urls = get_urls(metainfo=sdist, name=name, version=version) + urls, extra_data = get_urls(metainfo=sdist, name=name, version=version) yield models.PackageData( datasource_id=cls.datasource_id, @@ -265,17 +597,18 @@ def parse(cls, location): declared_license=get_declared_license(sdist), keywords=get_keywords(sdist), parties=get_parties(sdist), + extra_data=extra_data, **urls, ) -class PythonSetupPyHandler(models.DatafileHandler): - datasource_id = "pypi_setup_py" - path_patterns = ("*setup.py",) - default_package_type = "pypi" - default_primary_language = "Python" - description = "Python setup.py" - documentation_url = "https://docs.python.org/3/distutils/setupscript.html" +class PythonSetupPyHandler(BaseExtractedPythonLayout): + datasource_id = 'pypi_setup_py' + path_patterns = ('*setup.py',) + default_package_type = 'pypi' + default_primary_language = 'Python' + description = 'Python setup.py' + documentation_url = 'https://docs.python.org/3/distutils/setupscript.html' @classmethod def parse(cls, location): @@ -283,14 +616,19 @@ def parse(cls, location): # it may be legit to have a name-less package? # in anycase we do not want to fail because of that - name = setup_args.get("name") + name = setup_args.get('name') - version = setup_args.get("version") + version = setup_args.get('version') if not version: # search for possible dunder versions here and elsewhere version = detect_version_attribute(location) - urls = get_urls(metainfo=setup_args, name=name, version=version) + urls, extra_data = get_urls(metainfo=setup_args, name=name, version=version) + + dependencies = get_setup_py_dependencies(setup_args) + python_requires = get_setup_py_python_requires(setup_args) + extra_data.update(python_requires) + yield models.PackageData( datasource_id=cls.datasource_id, type=cls.default_package_type, @@ -298,15 +636,16 @@ def parse(cls, location): name=name, version=version, description=get_description(setup_args), - parties=get_parties(setup_args), + parties=get_setup_parties(setup_args), declared_license=get_declared_license(setup_args), - dependencies=get_setup_py_dependencies(setup_args), + dependencies=dependencies, keywords=get_keywords(setup_args), + extra_data=extra_data, **urls, ) -class BaseDependencyFileHandler(models.DatafileHandler): +class BaseDependencyFileHandler(BasePypiHandler): """ Base class for a dependency files parsed with the same library """ @@ -331,13 +670,13 @@ def parse(cls, location): ) -class SetupCfgHandler(models.DatafileHandler): - datasource_id = "pypi_setup_cfg" - path_patterns = ("*setup.cfg",) - default_package_type = "pypi" - default_primary_language = "Python" - description = "Python setup.cfg" - documentation_url = "https://peps.python.org/pep-0390/" +class SetupCfgHandler(BaseExtractedPythonLayout): + datasource_id = 'pypi_setup_cfg' + path_patterns = ('*setup.cfg',) + default_package_type = 'pypi' + default_primary_language = 'Python' + description = 'Python setup.cfg' + documentation_url = 'https://peps.python.org/pep-0390/' @classmethod def parse(cls, location): @@ -349,14 +688,14 @@ def parse(cls, location): parser.read_file(f) for section in parser.values(): - if section.name == "metadata": + if section.name == 'metadata': options = ( - "name", - "version", - "license", - "url", - "author", - "author_email", + 'name', + 'version', + 'license', + 'url', + 'author', + 'author_email', ) for name in options: content = section.get(name) @@ -365,14 +704,14 @@ def parse(cls, location): metadata[name] = content parties = [] - author = metadata.get("author") + author = metadata.get('author') if author: parties = [ models.Party( type=models.party_person, name=author, - role="author", - email=metadata.get("author_email"), + role='author', + email=metadata.get('author_email'), ) ] @@ -387,31 +726,31 @@ def parse(cls, location): yield models.PackageData( datasource_id=cls.datasource_id, type=cls.default_package_type, - name=metadata.get("name"), - version=metadata.get("version"), + name=metadata.get('name'), + version=metadata.get('version'), parties=parties, - homepage_url=metadata.get("url"), + homepage_url=metadata.get('url'), primary_language=cls.default_primary_language, dependencies=dependencies, ) class PipfileHandler(BaseDependencyFileHandler): - datasource_id = "pipfile" - path_patterns = ("*Pipfile",) - default_package_type = "pypi" - default_primary_language = "Python" - description = "Pipfile" - documentation_url = "https://github.com/pypa/pipfile" + datasource_id = 'pipfile' + path_patterns = ('*Pipfile',) + default_package_type = 'pypi' + default_primary_language = 'Python' + description = 'Pipfile' + documentation_url = 'https://github.com/pypa/pipfile' class PipfileLockHandler(BaseDependencyFileHandler): - datasource_id = "pipfile_lock" - path_patterns = ("*Pipfile.lock",) - default_package_type = "pypi" - default_primary_language = "Python" - description = "Pipfile.lock" - documentation_url = "https://github.com/pypa/pipfile" + datasource_id = 'pipfile_lock' + path_patterns = ('*Pipfile.lock',) + default_package_type = 'pypi' + default_primary_language = 'Python' + description = 'Pipfile.lock' + documentation_url = 'https://github.com/pypa/pipfile' @classmethod def parse(cls, location): @@ -421,14 +760,14 @@ def parse(cls, location): data = json.loads(content) sha256 = None - if "_meta" in data: - for name, meta in data["_meta"].items(): - if name == "hash": - sha256 = meta.get("sha256") + if '_meta' in data: + for name, meta in data['_meta'].items(): + if name == 'hash': + sha256 = meta.get('sha256') dependent_packages = parse_with_dparse2( location=location, - file_name="Pipfile.lock", + file_name='Pipfile.lock', ) yield models.PackageData( @@ -441,42 +780,43 @@ def parse(cls, location): class PipRequirementsFileHandler(BaseDependencyFileHandler): - datasource_id = "pip_requirements" + datasource_id = 'pip_requirements' path_patterns = ( - "*requirement*.txt", - "*requirement*.pip", - "*requirement*.in", - "*requires.txt", - "*requirements/*.txt", - "*requirements/*.pip", - "*requirements/*.in", - "*reqs.txt", + '*requirement*.txt', + '*requirement*.pip', + '*requirement*.in', + '*requires.txt', + '*requirements/*.txt', + '*requirements/*.pip', + '*requirements/*.in', + '*reqs.txt', ) - default_package_type = "pypi" - default_primary_language = "Python" - description = "pip requirements file" - documentation_url = "https://pip.pypa.io/en/latest/reference/requirements-file-format/" + default_package_type = 'pypi' + default_primary_language = 'Python' + description = 'pip requirements file' + documentation_url = 'https://pip.pypa.io/en/latest/reference/requirements-file-format/' @classmethod def parse(cls, location): - dependencies = get_requirements_txt_dependencies(location=location) + dependencies, extra_data = get_requirements_txt_dependencies(location=location) yield models.PackageData( datasource_id=cls.datasource_id, type=cls.default_package_type, primary_language=cls.default_primary_language, dependencies=dependencies, + extra_data=extra_data, ) - # TODO: enable nested load def get_requirements_txt_dependencies(location, include_nested=False): """ - Return a list of DependentPackage found in a requirements file at - ``location`` or an empty list. + Return a two-tuple of (list of deps, mapping of extra data) list of + DependentPackage found in a requirements file at ``location`` or tuple of + ([], {}) """ req_file = pip_requirements_parser.RequirementsFile.from_file( filename=location, @@ -485,15 +825,18 @@ def get_requirements_txt_dependencies(location, include_nested=False): if not req_file or not req_file.requirements: return [] - dependent_packages = [] + # for now we ignore errors + extra_data = {} + for opt in req_file.options: + extra_data.update(opt.options) - # for now we ignore plain options and errors + dependent_packages = [] for req in req_file.requirements: if req.name: # will be None if not pinned version = req.get_pinned_version - purl = PackageURL(type="pypi", name=req.name, version=version) + purl = PackageURL(type='pypi', name=req.name, version=version) else: # this is odd, but this can be null @@ -501,23 +844,20 @@ def get_requirements_txt_dependencies(location, include_nested=False): purl = purl and purl.to_string() or None - if req.is_editable: - requirement = req.dumps(with_name=False) - else: - requirement = req.dumps() + requirement = req.dumps() if location.endswith( ( - "dev.txt", - "test.txt", - "tests.txt", + 'dev.txt', + 'test.txt', + 'tests.txt', ) ): - scope = "development" + scope = 'development' is_runtime = False is_optional = True else: - scope = "install" + scope = 'install' is_runtime = True is_optional = False @@ -532,7 +872,7 @@ def get_requirements_txt_dependencies(location, include_nested=False): ) ) - return dependent_packages + return dependent_packages, extra_data def get_attribute(metainfo, name, multiple=False): @@ -553,8 +893,11 @@ def get_attribute(metainfo, name, multiple=False): # can use a get on dicts of emails. def attr_getter(_aname, default): - _aname = _aname.replace("-", "_") - return getattr(metainfo, _aname, default) or getattr(metainfo, _aname.lower(), default) + _aname = _aname.replace('-', '_') + return ( + getattr(metainfo, _aname, default) + or getattr(metainfo, _aname.lower(), default) + ) def item_getter(_iname, getter, default): getter = getattr(metainfo, getter, None) @@ -565,12 +908,16 @@ def item_getter(_iname, getter, default): if multiple: return ( attr_getter(name, []) - or item_getter(name, "get_all", []) - or item_getter(name, "get", []) + or item_getter(name, 'get_all', []) + or item_getter(name, 'get', []) or [] ) else: - return attr_getter(name, None) or item_getter(name, "get", None) or None + return ( + attr_getter(name, None) + or item_getter(name, 'get', None) + or None + ) def get_description(metainfo, location=None): @@ -579,17 +926,17 @@ def get_description(metainfo, location=None): """ description = None # newer metadata versions use the payload for the description - if hasattr(metainfo, "get_payload"): + if hasattr(metainfo, 'get_payload'): description = metainfo.get_payload() description = description and description.strip() or None if not description: # legacymetadata versions use the Description for the description - description = get_attribute(metainfo, "Description") + description = get_attribute(metainfo, 'Description') if not description and location: # older metadata versions can use a DESCRIPTION.rst file description = get_legacy_description(location=fileutils.parent_directory(location)) - summary = get_attribute(metainfo, "Summary") + summary = get_attribute(metainfo, 'Summary') description = clean_description(description) return build_description(summary, description) @@ -601,27 +948,30 @@ def clean_description(description): do not. We check first and cleanup if needed. """ # TODO: verify what is the impact of Description-Content-Type: if any - description = description or "" + description = description or '' description = description.strip() lines = description.splitlines(False) - space_padding = " " * 8 + space_padding = ' ' * 8 # we need cleaning if any of the first two lines starts with 8 spaces need_cleaning = any(l.startswith(space_padding) for l in lines[:2]) if not need_cleaning: return description - cleaned_lines = [line[8:] if line.startswith(space_padding) else line for line in lines] + cleaned_lines = [ + line[8:] if line.startswith(space_padding) else line + for line in lines + ] - return "\n".join(cleaned_lines) + return '\n'.join(cleaned_lines) def get_legacy_description(location): """ Return the text of a legacy DESCRIPTION.rst file. """ - location = os.path.join(location, "DESCRIPTION.rst") + location = os.path.join(location, 'DESCRIPTION.rst') if os.path.exists(location): with open(location) as i: return i.read() @@ -635,13 +985,13 @@ def get_declared_license(metainfo): declared_license = {} # TODO: We should make the declared license as it is, this should be # updated in scancode to parse a pure string - lic = get_attribute(metainfo, "License") - if lic and not lic == "UNKNOWN": - declared_license["license"] = lic + lic = get_attribute(metainfo, 'License') + if lic and not lic == 'UNKNOWN': + declared_license['license'] = lic license_classifiers, _ = get_classifiers(metainfo) if license_classifiers: - declared_license["classifiers"] = license_classifiers + declared_license['classifiers'] = license_classifiers return declared_license @@ -651,8 +1001,9 @@ def get_classifiers(metainfo): found in a ``metainfo`` object or mapping. """ - classifiers = get_attribute(metainfo, "Classifier", multiple=True) or get_attribute( - metainfo, "Classifiers", multiple=True + classifiers = ( + get_attribute(metainfo, 'Classifier', multiple=True) + or get_attribute(metainfo, 'Classifiers', multiple=True) ) if not classifiers: return [], [] @@ -660,7 +1011,7 @@ def get_classifiers(metainfo): license_classifiers = [] other_classifiers = [] for classifier in classifiers: - if classifier.startswith("License"): + if classifier.startswith('License'): license_classifiers.append(classifier) else: other_classifiers.append(classifier) @@ -672,10 +1023,10 @@ def get_keywords(metainfo): Return a list of keywords found in a ``metainfo`` object or mapping. """ keywords = [] - kws = get_attribute(metainfo, "Keywords") or [] + kws = get_attribute(metainfo, 'Keywords') or [] if kws: if isinstance(kws, str): - kws = kws.split(",") + kws = kws.split(',') elif isinstance(kws, (list, tuple)): pass else: @@ -689,39 +1040,71 @@ def get_keywords(metainfo): return keywords -def get_parties(metainfo): +def get_parties( + metainfo, + author_key='Author', + author_email_key='Author-email', + maintainer_key='Maintainer', + maintainer_email_key='Maintainer-email', + +): """ Return a list of parties found in a ``metainfo`` object or mapping. + Uses the provided keys with a default to key names used in METADATA. + setup.py and setup.cfg use lower case valid Python identifiers instead. """ parties = [] - author = get_attribute(metainfo, "Author") - author_email = get_attribute(metainfo, "Author-email") - if author or author_email: - parties.append( - models.Party( - type=models.party_person, - name=author or None, - role="author", - email=author_email or None, - ) - ) + author = get_attribute(metainfo, author_key) - maintainer = get_attribute(metainfo, "Maintainer") - maintainer_email = get_attribute(metainfo, "Maintainer-email") + author_email = get_attribute(metainfo, author_email_key) + if author or author_email: + parties.append(models.Party( + type=models.party_person, + name=author or None, + role='author', + email=author_email or None, + )) + + maintainer = get_attribute(metainfo, maintainer_key) + maintainer_email = get_attribute(metainfo, maintainer_email_key) if maintainer or maintainer_email: - parties.append( - models.Party( - type=models.party_person, - name=maintainer or None, - role="maintainer", - email=maintainer_email or None, - ) - ) + parties.append(models.Party( + type=models.party_person, + name=maintainer or None, + role='maintainer', + email=maintainer_email or None, + )) return parties +def get_setup_parties(setup_kwargs): + """ + Return a list of parties found in a ``setup_kwargs`` mapping of data found + in setup.py or setup.cfg. + """ + return get_parties( + metainfo=setup_kwargs, + author_key='author', + author_email_key='author_email', + maintainer_key='maintainer', + maintainer_email_key='maintainer_email', + ) + + +def get_setup_py_python_requires(setup_args): + """ + Return a mapping of {python_requires: value} or an empty mapping found in a + ``setup_args`` mapping of setup.py arguments. + """ + python_requires = setup_args.get('python_requires') + if python_requires: + return dict(python_requires=python_requires) + else: + return {} + + def get_setup_py_dependencies(setup_args): """ Return a list of DependentPackage found in a ``setup_args`` mapping of @@ -729,23 +1112,24 @@ def get_setup_py_dependencies(setup_args): """ dependencies = [] - python_requires = setup_args.get("python_requires") - if python_requires: - # FIXME: handle python_requires = >=3.6.* - pass - - install_requires = setup_args.get("install_requires") - dependencies.extend(get_requires_dependencies(install_requires, default_scope="install")) + install_requires = setup_args.get('install_requires') + dependencies.extend(get_requires_dependencies(install_requires, default_scope='install')) - tests_requires = setup_args.get("tests_requires") - dependencies.extend(get_requires_dependencies(tests_requires, default_scope="tests")) + tests_requires = setup_args.get('tests_requires') + dependencies.extend( + get_requires_dependencies(tests_requires, default_scope='tests') + ) - setup_requires = setup_args.get("setup_requires") - dependencies.extend(get_requires_dependencies(setup_requires, default_scope="setup")) + setup_requires = setup_args.get('setup_requires') + dependencies.extend( + get_requires_dependencies(setup_requires, default_scope='setup') + ) - extras_require = setup_args.get("extras_require", {}) + extras_require = setup_args.get('extras_require') or {} for scope, requires in extras_require.items(): - dependencies.extend(get_requires_dependencies(requires, default_scope=scope)) + dependencies.extend( + get_requires_dependencies(requires, default_scope=scope) + ) return dependencies @@ -754,7 +1138,11 @@ def is_simple_requires(requires): """ Return True if ``requires`` is a sequence of strings. """ - return requires and isinstance(requires, list) and all(isinstance(i, str) for i in requires) + return ( + requires + and isinstance(requires, list) + and all(isinstance(i, str) for i in requires) + ) def get_dist_dependencies(dist): @@ -764,11 +1152,11 @@ def get_dist_dependencies(dist): """ # we treat extras as scopes # TODO: use these for verification? - scopes = dist.metadata.get_all("Provides-Extra") or [] + scopes = dist.metadata.get_all('Provides-Extra') or [] return get_requires_dependencies(requires=dist.requires) -def get_requires_dependencies(requires, default_scope="install"): +def get_requires_dependencies(requires, default_scope='install'): """ Return a list of DependentPackage found in a ``requires`` list of requirement strings or an empty list. @@ -777,11 +1165,11 @@ def get_requires_dependencies(requires, default_scope="install"): # FIXME: when does this happen? should we log this? return [] dependent_packages = [] - for req in requires or []: + for req in (requires or []): req = Requirement(req) name = canonicalize_name(req.name) is_resolved = False - purl = PackageURL(type="pypi", name=name) + purl = PackageURL(type='pypi', name=name) # note: packaging.requirements.Requirement.specifier is a # packaging.specifiers.SpecifierSet object and a SpecifierSet._specs is # a set of either: packaging.specifiers.Specifier or @@ -798,7 +1186,7 @@ def get_requires_dependencies(requires, default_scope="install"): # equality specifier if len(specifiers) == 1: specifier = list(specifiers)[0] - if specifier.operator in ("==", "==="): + if specifier.operator in ('==', '==='): is_resolved = True purl = purl._replace(version=specifier.version) @@ -813,8 +1201,7 @@ def get_requires_dependencies(requires, default_scope="install"): is_optional=False, is_resolved=is_resolved, extracted_requirement=str(req), - ) - ) + )) return dependent_packages @@ -826,7 +1213,7 @@ def get_extra(marker): if not marker or not isinstance(marker, markers.Marker): return - marks = getattr(marker, "_markers", []) + marks = getattr(marker, '_markers', []) for mark in marks: # filter for variable(extra) == value tuples of (Variable, Op, Value) @@ -837,9 +1224,9 @@ def get_extra(marker): if ( isinstance(variable, markers.Variable) - and variable.value == "extra" + and variable.value == 'extra' and isinstance(operator, markers.Op) - and operator.value == "==" + and operator.value == '==' and isinstance(value, markers.Value) ): return value.value @@ -852,10 +1239,10 @@ def get_dparse2_supported_file_name(file_name): """ # this is kludgy but the upstream data structure and API needs this dfile_names = ( - "Pipfile.lock", - "Pipfile", - "conda.yml", - "setup.cfg", + 'Pipfile.lock', + 'Pipfile', + 'conda.yml', + 'setup.cfg', ) for dfile_name in dfile_names: @@ -881,7 +1268,7 @@ def parse_with_dparse2(location, file_name=None): for dependency in dep_file.dependencies: requirement = dependency.name is_resolved = False - purl = PackageURL(type="pypi", name=dependency.name) + purl = PackageURL(type='pypi', name=dependency.name) # note: dparse2.dependencies.Dependency.specs comes from # packaging.requirements.Requirement.specifier @@ -901,7 +1288,7 @@ def parse_with_dparse2(location, file_name=None): # are we pinned e.g. resolved? if len(specifiers) == 1: specifier = list(specifiers)[0] - if specifier.operator in ("==", "==="): + if specifier.operator in ('==', '==='): is_resolved = True purl = purl._replace(version=specifier.version) @@ -909,20 +1296,47 @@ def parse_with_dparse2(location, file_name=None): models.DependentPackage( purl=purl.to_string(), # are we always this scope? what if we have requirements-dev.txt? - scope="install", + scope='install', is_runtime=True, is_optional=False, is_resolved=is_resolved, - extracted_requirement=requirement, + extracted_requirement=requirement ) ) return dependent_packages -def get_setup_py_args(location): +def is_setup_call(statement): + """ + Return if the AST ``statement`` is a call to the setup() function. + """ + return ( + isinstance(statement, (ast.Expr, ast.Call, ast.Assign)) + and isinstance(statement.value, ast.Call) + and ( + # we look for setup and main as this is used sometimes instead of setup() + ( + isinstance(statement.value.func, ast.Name) + and statement.value.func.id in ('setup', 'main') + ) + or + # we also look for setuptools.setup when used instead of setup() + ( + isinstance(statement.value.func, ast.Attribute) + and statement.value.func.attr == 'setup' + and isinstance(statement.value.func.value, ast.Name) + and statement.value.func.value.id == 'setuptools' + ) + ) + ) + + +def get_setup_py_args_legacy(location, include_not_parsable=False): """ - Return a mapping of arguments passed to a setup.py setup() function. + Return a mapping of arguments passed to a setup.py setup() function. Also + include not parsable identifiers values such as variable name and attribute + references if ``include_not_parsable`` is True """ with open(location) as inp: setup_text = inp.read() @@ -934,58 +1348,121 @@ def get_setup_py_args(location): for statement in tree.body: # We only care about function calls or assignments to functions named # `setup` or `main` - if not ( - isinstance(statement, (ast.Expr, ast.Call, ast.Assign)) - and isinstance(statement.value, ast.Call) - and isinstance(statement.value.func, ast.Name) - # we also look for main as sometimes this is used instead of setup() - and statement.value.func.id in ("setup", "main") - ): + + # TODO: also collect top level variables assigned later as arguments values + if not is_setup_call(statement): continue # Process the arguments to the setup function - for kw in getattr(statement.value, "keywords", []): + for kw in getattr(statement.value, 'keywords', []): arg_name = kw.arg + arg_value = kw.value - if isinstance(kw.value, ast.Str): - setup_args[arg_name] = kw.value.s + # FIXME: use a recursive function to extract structured data - elif isinstance( - kw.value, - ( - ast.List, - ast.Tuple, - ast.Set, - ), - ): + if isinstance(arg_value, (ast.List, ast.Tuple, ast.Set,)): # We collect the elements of a list if the element # and tag function calls - value = [elt.s for elt in kw.value.elts if not isinstance(elt, ast.Call)] - setup_args[arg_name] = value - - # TODO: what if isinstance(kw.value, ast.Dict) - # or an expression like a call to version=get_version or version__version__ + val = [ + elt.s for elt in arg_value.elts + if not isinstance(elt, ast.Call) + ] + setup_args[arg_name] = val + + elif isinstance(arg_value, ast.Dict): + # we only collect simple name/value and name/[values] constructs + keys = [elt.value for elt in arg_value.keys] + values = [] + for val in arg_value.values: + + if isinstance(val, (ast.List, ast.Tuple, ast.Set,)): + val = [ + elt.s for elt in val.elts + if not isinstance(elt, ast.Call) + ] + values.append(val) + + elif isinstance(val, (ast.Str, ast.Constant,)): + values.append(val.s) + + else: + if include_not_parsable: + if isinstance(val, ast.Attribute): + values.append(val.attr) + + elif isinstance(val, ast.Name): + values.append(val.id) + + elif not isinstance(val, (ast.Call, ast.ListComp, ast.Subscript)): + # we used to consider only isinstance(val, ast.Str): + # instead use literal_eval and ignore failures, skipping + # only function calls this way we can get more things such + # as boolean and numbers + try: + values.append(ast.literal_eval(val.value)) + except Exception as e: + if TRACE: + logger_debug('get_setup_py_args: failed:', e) + values.append(str(val.value)) + + mapping = dict(zip(keys, values)) + setup_args[arg_name] = mapping + + elif isinstance(arg_value, (ast.Str, ast.Constant,)): + setup_args[arg_name] = arg_value.s + else: + if include_not_parsable: + if isinstance(arg_value, ast.Attribute): + setup_args[arg_name] = arg_value.attr + + elif isinstance(arg_value, ast.Name): + if arg_name: + setup_args[arg_name] = arg_value.id + + elif not isinstance(arg_value, (ast.Call, ast.ListComp, ast.Subscript,)): + # we used to consider only isinstance(kw.value, ast.Str): + # instead use literal_eval and ignore failures, skipping only + # function calls this way we can get more things such as boolean + # and numbers + try: + setup_args[arg_name] = ast.literal_eval(arg_value) + except Exception as e: + if TRACE: + logger_debug('get_setup_py_args: failed:', e) + setup_args[arg_name] = str(arg_value) + + # TODO: an expression like a call to version=get_version or version__version__ return setup_args +def get_setup_py_args(location, include_not_parsable=False): + """ + Return a mapping of arguments passed to a setup.py setup() function. Also + include not parsable identifiers values such as variable name and attribute + references if ``include_not_parsable`` is True + """ + from _packagedcode.pypi_setup_py import parse_setup_py + return parse_setup_py(location) + + def get_pypi_urls(name, version): """ Return a mapping of computed Pypi URLs for this package """ api_data_url = None if name and version: - api_data_url = f"https://pypi.org/pypi/{name}/{version}/json" + api_data_url = f'https://pypi.org/pypi/{name}/{version}/json' else: - api_data_url = name and f"https://pypi.org/pypi/{name}/json" + api_data_url = name and f'https://pypi.org/pypi/{name}/json' repository_download_url = ( name and version - and f"https://pypi.org/packages/source/{name[0]}/{name}/{name}-{version}.tar.gz" + and f'https://pypi.org/packages/source/{name[0]}/{name}/{name}-{version}.tar.gz' ) - repository_homepage_url = name and f"https://pypi.org/project/{name}" + repository_homepage_url = name and f'https://pypi.org/project/{name}' return dict( repository_homepage_url=repository_homepage_url, @@ -994,11 +1471,12 @@ def get_pypi_urls(name, version): ) -def get_urls(metainfo, name, version, extra_data=None): +def get_urls(metainfo, name, version): """ - Return a mapping for URLs of this package: - - as plain name/values for URL attributes known in PackageData - - as a nested extra_data: mapping for other URLs (possibly updating extra_data if provided). + Return a mapping of standard URLs and a mapping of extra-data URls for URLs + of this package: + - standard URLs are for URL attributes known in PackageData + - extra_data for other URLs (possibly updating extra_data if provided). """ # Misc URLs to possibly track # Project-URL: Release notes @@ -1034,7 +1512,7 @@ def get_urls(metainfo, name, version, extra_data=None): # Project-URL: Twine source # Project-URL: Say Thanks! - extra_data = extra_data or {} + extra_data = {} urls = get_pypi_urls(name, version) def add_url(_url, _utype=None, _attribute=None): @@ -1050,61 +1528,61 @@ def add_url(_url, _utype=None, _attribute=None): # get first as this is the most common one homepage_url = ( - get_attribute(metainfo, "Home-page") - or get_attribute(metainfo, "url") - or get_attribute(metainfo, "home") + get_attribute(metainfo, 'Home-page') + or get_attribute(metainfo, 'url') + or get_attribute(metainfo, 'home') ) - add_url(homepage_url, _attribute="homepage_url") + add_url(homepage_url, _attribute='homepage_url') project_urls = ( - get_attribute(metainfo, "Project-URL", multiple=True) - or get_attribute(metainfo, "project_urls") + get_attribute(metainfo, 'Project-URL', multiple=True) + or get_attribute(metainfo, 'project_urls') or [] ) - for url in project_urls: - utype, _, uvalue = url.partition(",") - uvalue = uvalue.strip() - utype = utype.strip() - utypel = utype.lower() - if utypel in ( - "tracker", - "bug reports", - "github: issues", - "bug tracker", - "issues", - "issue tracker", - ): - add_url(url, _utype=utype, _attribute="bug_tracking_url") + if isinstance(project_urls, list): + # these come from METADATA and we convert them back to a mapping + project_urls = [url.partition(', ') for url in project_urls] + project_urls = { + utype.strip(): uvalue.strip() + for utype, _, uvalue in project_urls + } + if isinstance(project_urls, dict): + for utype, url in project_urls.items(): + utypel = utype.lower() + if utypel in ( + 'tracker', + 'bug reports', + 'github: issues', + 'bug tracker', + 'issues', + 'issue tracker', + ): + add_url(url, _utype=utype, _attribute='bug_tracking_url') - elif utypel in ( - "source", - "source code", - "code", - ): - add_url(url, _utype=utype, _attribute="code_view_url") + elif utypel in ( + 'source', + 'source code', + 'code', + ): + add_url(url, _utype=utype, _attribute='code_view_url') - elif utypel in ("github", "gitlab", "github: repo", "repository"): - add_url(url, _utype=utype, _attribute="vcs_url") + elif utypel in ('github', 'gitlab', 'github: repo', 'repository'): + add_url(url, _utype=utype, _attribute='vcs_url') - elif utypel in ( - "website", - "homepage", - "home", - ): - add_url(url, _utype=utype, _attribute="homepage_url") + elif utypel in ('website', 'homepage', 'home',): + add_url(url, _utype=utype, _attribute='homepage_url') - else: - add_url(url, _utype=utype) + else: + add_url(url, _utype=utype) - # FIXME: this may not be the actual correct package download URL, so for now - # we incorrectly set this as the vcs_url - download_url = get_attribute(metainfo, "Download-URL") - add_url(download_url, _utype="Download-URL", _attribute="vcs_url") + # FIXME: this may not be the actual correct package download URL, so we keep this as an extra URL + download_url = get_attribute(metainfo, 'Download-URL') + if not download_url: + download_url = get_attribute(metainfo, 'download_url') + add_url(download_url, _utype='Download-URL') - if extra_data: - urls["extra_data"] = extra_data - return urls + return urls, extra_data def find_pattern(location, pattern): @@ -1118,7 +1596,7 @@ def find_pattern(location, pattern): SPDX-License-Identifier: BSD-3-Clause (C) 2001-2020 Chris Liechti """ - with io.open(location, encoding="utf8") as fp: + with open(location) as fp: content = fp.read() match = re.search(pattern, content) @@ -1138,8 +1616,7 @@ def find_dunder_version(location): """ pattern = re.compile(r"^__version__\s*=\s*['\"]([^'\"]*)['\"]", re.MULTILINE) match = find_pattern(location, pattern) - if TRACE: - logger_debug("find_dunder_version:", "location:", location, "match:", match) + if TRACE: logger_debug('find_dunder_version:', 'location:', location, 'match:', match) return match @@ -1150,8 +1627,7 @@ def find_plain_version(location): """ pattern = re.compile(r"^version\s*=\s*['\"]([^'\"]*)['\"]", re.MULTILINE) match = find_pattern(location, pattern) - if TRACE: - logger_debug("find_plain_version:", "location:", location, "match:", match) + if TRACE: logger_debug('find_plain_version:', 'location:', location, 'match:', match) return match @@ -1164,7 +1640,7 @@ def find_setup_py_dunder_version(location): setup( version=six.__version__, ... - would return six.__version__. + would return six.__version__ Code inspired and heavily modified from: https://github.com/pyserial/pyserial/blob/d867871e6aa333014a77498b4ac96fdd1d3bf1d8/setup.py#L34 @@ -1174,7 +1650,7 @@ def find_setup_py_dunder_version(location): pattern = re.compile(r"^\s*version\s*=\s*(.*__version__)", re.MULTILINE) match = find_pattern(location, pattern) if TRACE: - logger_debug("find_setup_py_dunder_version:", "location:", location, "match:", match) + logger_debug('find_setup_py_dunder_version:', 'location:', location, 'match:', match) return match @@ -1188,72 +1664,77 @@ def detect_version_attribute(setup_location): setup_version_arg = find_setup_py_dunder_version(setup_location) setup_py__version = find_dunder_version(setup_location) if TRACE: - logger_debug(" detect_dunder_version:", "setup_location:", setup_location) - logger_debug( - " setup_version_arg:", - repr(setup_version_arg), - ) - logger_debug( - " setup_py__version:", - repr(setup_py__version), - ) + logger_debug(' detect_version_attribute():', 'setup_location:', setup_location) + logger_debug(' find_setup_py_dunder_version(): setup_version_arg:', repr(setup_version_arg),) + logger_debug(' find_dunder_version(): setup_py__version:', repr(setup_py__version),) - if setup_version_arg == "__version__" and setup_py__version: + if setup_version_arg == '__version__' and setup_py__version: version = setup_py__version or None if TRACE: - logger_debug(" detect_dunder_version: A:", version) + logger_debug( + ' detect_dunder_version:', + "setup_version_arg == '__version__' and setup_py__version:", version) return version # here we have a more complex __version__ location # we start by adding the possible paths and file name # and we look at these in sequence - candidate_locs = [] - - if setup_version_arg and "." in setup_version_arg: - segments = setup_version_arg.split(".")[:-1] + if setup_version_arg and '.' in setup_version_arg: + segments = setup_version_arg.split('.')[:-1] else: segments = [] + if TRACE: + logger_debug(' detect_version_attribute():', 'segments:', segments) + special_names = ( - "__init__.py", - "__main__.py", - "__version__.py", - "__about__.py", - "__version.py", - "_version.py", - "version.py", - "VERSION.py", - "package_data.py", + '__init__.py', + '__main__.py', + '__version__.py', + '__about__.py', + '__version.py', + '_version.py', + 'version.py', + 'VERSION.py', + 'package_data.py', ) setup_py_dir = fileutils.parent_directory(setup_location) - src_dir = os.path.join(setup_py_dir, "src") + src_dir = os.path.join(setup_py_dir, 'src') has_src = os.path.exists(src_dir) + if TRACE: + logger_debug(' detect_version_attribute():', 'src_dir:', src_dir) + logger_debug(' detect_version_attribute():', 'has_src:', has_src) + + candidate_locs = [] if segments: for n in special_names: candidate_locs.append(segments + [n]) if has_src: for n in special_names: - candidate_locs.append(["src"] + segments + [n]) + candidate_locs.append(['src'] + segments + [n]) if len(segments) > 1: heads = segments[:-1] tail = segments[-1] - candidate_locs.append(heads + [tail + ".py"]) + candidate_locs.append(heads + [tail + '.py']) if has_src: - candidate_locs.append(["src"] + heads + [tail + ".py"]) + candidate_locs.append(['src'] + heads + [tail + '.py']) else: seg = segments[0] - candidate_locs.append([seg + ".py"]) + candidate_locs.append([seg + '.py']) if has_src: - candidate_locs.append(["src", seg + ".py"]) + candidate_locs.append(['src', seg + '.py']) candidate_locs = [ - os.path.join(setup_py_dir, *cand_loc_segs) for cand_loc_segs in candidate_locs + os.path.join(setup_py_dir, *cand_loc_segs) + for cand_loc_segs in candidate_locs ] + if TRACE: + logger_debug(' detect_version_attribute():', 'candidate_locs1:', candidate_locs) for fl in get_module_scripts( location=setup_py_dir, @@ -1263,20 +1744,28 @@ def detect_version_attribute(setup_location): candidate_locs.append(fl) if TRACE: + logger_debug(' detect_version_attribute():', 'candidate_locs2:') for loc in candidate_locs: - logger_debug(" can loc:", loc) + logger_debug(' loc:', loc) version = detect_version_in_locations( - candidate_locs=candidate_locs, detector=find_dunder_version + candidate_locs=candidate_locs, + detector=find_dunder_version ) + if TRACE: + logger_debug(' detect_version_attribute():', 'version2:', version) if version: return version - return detect_version_in_locations( + version = detect_version_in_locations( candidate_locs=candidate_locs, detector=find_plain_version, ) + if TRACE: + logger_debug(' detect_version_attribute():', 'version3:', version) + + return version def detect_version_in_locations(candidate_locs, detector=find_plain_version): @@ -1284,23 +1773,23 @@ def detect_version_in_locations(candidate_locs, detector=find_plain_version): Return the first version found in a location from the `candidate_locs` list using the `detector` callable. Return None if no version is found. """ + if TRACE: + logger_debug(' detect_version_in_locations():', 'candidate_locs:', candidate_locs) + for loc in candidate_locs: if not os.path.exists(loc): continue - if TRACE: - logger_debug("detect_version_in_locations:", "loc:", loc) + if TRACE: logger_debug(' detect_version_in_locations:', 'loc:', loc) # here the file exists try to get a dunder version version = detector(loc) if TRACE: logger_debug( - "detect_version_in_locations:", - "detector", - detector, - "version:", - version, + ' detect_version_in_locations:', + 'detector', detector, + 'version:', version, ) if version: @@ -1313,17 +1802,92 @@ def get_module_scripts(location, max_depth=1, interesting_names=()): `interesting_names` by walking the `location` directory recursively up to `max_depth` path segments extending from the root `location`. """ + if TRACE: + logger_debug( + ' get_module_scripts():', + 'location:', location, + 'max_depth:', max_depth, + 'interesting_names:', interesting_names + ) location = location.rstrip(os.path.sep) - current_depth = max_depth + if TRACE: logger_debug(' get_module_scripts:', 'location:', location) + for top, _dirs, files in os.walk(location): - if current_depth == 0: + current_depth = compute_path_depth(location, top) + if TRACE: + logger_debug(' get_module_scripts:', 'current_depth:', current_depth) + logger_debug(' get_module_scripts:', 'top:', top, '_dirs:', _dirs, 'files:', files) + if current_depth >= max_depth: break for f in files: + if TRACE: logger_debug(' get_module_scripts:', 'file:', f) + if f in interesting_names: path = os.path.join(top, f) - if TRACE: - logger_debug("get_module_scripts:", "path", path) + if TRACE: logger_debug(' get_module_scripts:', 'path:', path) yield path - current_depth -= 1 + +def compute_path_depth(base, path): + """ + Return the depth of ``path`` below ``base`` as the number of path segments + that ``path`` extends below ``base``. + For example: + >>> base = '/home/foo/bar' + >>> compute_path_depth(base, '/home/foo/bar/baz') + 1 + >>> compute_path_depth(base, base) + 0 + """ + base = base.strip(os.path.sep) + path = path.strip(os.path.sep) + + assert path.startswith(base) + subpath = path[len(base):].strip(os.path.sep) + segments = [s for s in subpath.split(os.path.sep) if s] + depth = len(segments) + if TRACE: + logger_debug( + ' compute_path_depth:', + 'base:', base, 'path:', path, 'subpath:', subpath, + 'segments:', segments, 'depth:', depth,) + return depth + + +def compute_normalized_license(declared_license): + """ + Return a normalized license expression string detected from a mapping or + list of declared license items. + """ + if not declared_license: + return + + if isinstance(declared_license, dict): + values = list(declared_license.values()) + elif isinstance(declared_license, list): + values = list(declared_license) + elif isinstance(declared_license, str): + values = [declared_license] + else: + return + + detected_licenses = [] + + for value in values: + if not value: + continue + # The value could be a string or a list + if isinstance(value, str): + detected_license = models.compute_normalized_license(value) + if detected_license: + detected_licenses.append(detected_license) + else: + # this is a list + for declared in value: + detected_license = models.compute_normalized_license(declared) + if detected_license: + detected_licenses.append(detected_license) + + if detected_licenses: + return combine_expressions(detected_licenses) diff --git a/src/_packagedcode/pypi_setup_py.py b/src/_packagedcode/pypi_setup_py.py new file mode 100644 index 00000000..e7f55c7a --- /dev/null +++ b/src/_packagedcode/pypi_setup_py.py @@ -0,0 +1,211 @@ +# +# Copyright (c) Gram and others. +# This code is copied and modified from dephell_setuptools https://github.com/pypa/setuptools +# SPDX-License-Identifier: MIT +# See https://github.com/nexB/scancode-toolkit for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + + +import ast +from pathlib import Path + +""" +Parse setup.py files. +""" + +# https://setuptools.readthedocs.io/en/latest/setuptools.html#metadata +FIELDS = { + 'author_email', + 'author', + 'classifiers', + 'dependency_links', + 'description', + 'download_url', + 'extras_require', + 'install_requires', + 'keywords', + 'license_file', + 'license', + 'long_description_content_type', + 'long_description', + 'maintainer_email', + 'maintainer', + 'metadata_version', + 'name', + 'obsoletes', + 'package_dir', + 'platforms', + 'project_urls', + 'provides', + 'python_requires', + 'requires', + 'setup_requires', + 'tests_require', + 'url', + 'version', +} + + +def is_setup_call(element): + """ + Return if the AST ``element`` is a call to the setup() function. + Note: this is derived from the code in packagedcode.pypi.py + """ + if ( + isinstance(element, ast.Call) + and ( + hasattr(element, 'func') + and isinstance(element.func, ast.Name) + and getattr(element.func, 'id', None) == 'setup' + ) or ( + hasattr(element, 'func') + and isinstance(element.func, ast.Attribute) + and getattr(element.func, 'attr', None) == 'setup' + and isinstance(element.func.value, ast.Name) + and getattr(element.func.value, 'id', None) == 'setuptools' + ) + ): + return True + + +def parse_setup_py(location): + """ + Return a mapping of setuptools.setup() call argument found in a setup.py + file at ``location`` or an empty mapping. + """ + path = Path(location) + tree = tuple(ast.parse(path.read_text(encoding='utf8')).body) + body = tuple(get_body(tree)) + + call = get_setup_call(tree) + result = get_call_kwargs(call, body) + + return clean_setup(result) + + +def get_body(elements): + """ + Yield the body from ``elements`` as a single iterable. + """ + for element in elements: + if isinstance(element, ast.FunctionDef): + yield from get_body(element.body) + continue + if isinstance(element, ast.If): + yield from get_body(element.body) + if isinstance(element, ast.Expr): + yield element.value + continue + yield element + + +def get_setup_call(elements): + """ + Return a setup() method call found in the ``elements`` or None. + """ + for element in get_body(elements): + if is_setup_call(element): + return element + elif isinstance(element, (ast.Assign,)): + if isinstance(element.value, ast.Call): + if is_setup_call(element.value): + return element.value + + +def node_to_value(node, body): + """ + Return the extracted and converted value of a node or None + """ + if node is None: + return + if hasattr(ast, 'Constant'): + if isinstance(node, ast.Constant): + return node.value + + if isinstance(node, ast.Str): + return node.s + + if isinstance(node, ast.Num): + return node.n + + if isinstance(node, (ast.List, ast.Tuple, ast.Set,)): + return [node_to_value(subnode, body) for subnode in node.elts] + + if isinstance(node, ast.Dict): + result = {} + for key, value in zip(node.keys, node.values): + result[node_to_value(key, body)] = node_to_value(value, body) + return result + + if isinstance(node, ast.Name): + variable = find_variable_in_body(body, node.id) + if variable is not None: + return node_to_value(variable, body) + + if isinstance(node, ast.Call): + if not isinstance(node.func, ast.Name): + return + if node.func.id != 'dict': + return + return get_call_kwargs(node, body) + return + + +def find_variable_in_body(body, name): + """ + Return the value of the variable ``name`` found in the ``body`` ast tree or None. + """ + for elem in body: + if not isinstance(elem, ast.Assign): + continue + for target in elem.targets: + if not isinstance(target, ast.Name): + continue + if target.id == name: + return elem.value + + +def get_call_kwargs(node: ast.Call, body): + """ + Return a mapping of setup() method call keyword arguments. + """ + result = {} + keywords = getattr(node, 'keywords', []) or [] + for keyword in keywords: + # dict unpacking + if keyword.arg is None: + value = node_to_value(keyword.value, body) + if isinstance(value, dict): + result.update(value) + continue + # keyword argument + value = node_to_value(keyword.value, body) + if value is None: + continue + result[keyword.arg] = value + return result + + +def clean_setup(data): + """ + Return a cleaned mapping from a setup ``data`` mapping. + """ + result = {k: v + for k, v in data.items() + if k in FIELDS + and (v and v is not False) + and str(v) != 'UNKNOWN' + } + + # split keywords in words + keywords = result.get('keywords') + if keywords and isinstance(keywords, str): + # some keywords are separated by coma, some by space or lines + if ',' in keywords: + keywords = [k.strip() for k in keywords.split(',')] + else: + keywords = keywords.split() + result['keywords'] = keywords + + return result diff --git a/src/_packagedcode/pypi_setup_py.py.ABOUT b/src/_packagedcode/pypi_setup_py.py.ABOUT new file mode 100644 index 00000000..0e1ef90e --- /dev/null +++ b/src/_packagedcode/pypi_setup_py.py.ABOUT @@ -0,0 +1,8 @@ +about_resource: pypi_setup_py.py +name: dephell_setuptools +author: Gram (@orsinium) +author_email: gram@orsinium.dev +homepage_url: https://github.com/dephell/dephell_setuptools +license_expression: mit +package_url: pkg:github/dephell/dephell_setuptools@v.0.2.4 +notes: code heavily modified from original \ No newline at end of file diff --git a/src/_packagedcode/pypi_setup_py.py.LICENSE b/src/_packagedcode/pypi_setup_py.py.LICENSE new file mode 100644 index 00000000..e64d3d41 --- /dev/null +++ b/src/_packagedcode/pypi_setup_py.py.LICENSE @@ -0,0 +1,20 @@ +MIT License 2019 Gram + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice (including the next +paragraph) shall be included in all copies or substantial portions of the +Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/src/_packagedcode/utils.py b/src/_packagedcode/utils.py index 7a89d455..1cddc775 100644 --- a/src/_packagedcode/utils.py +++ b/src/_packagedcode/utils.py @@ -7,26 +7,31 @@ # See https://aboutcode.org for more information about nexB OSS projects. # -""" -Originally vendored from scancode-toolkit packagedcode.utils -""" +try: + from license_expression import Licensing + from license_expression import combine_expressions as le_combine_expressions +except: + Licensing = None + le_combine_expressions = None PLAIN_URLS = ( - "https://", - "http://", + 'https://', + 'http://', ) VCS_URLS = ( - "git://", - "git+git://", - "git+https://", - "git+http://", - "hg://", - "hg+http://", - "hg+https://", - "svn://", - "svn+https://", - "svn+http://", + 'git://', + 'git+git://', + 'git+https://', + 'git+http://', + + 'hg://', + 'hg+http://', + 'hg+https://', + + 'svn://', + 'svn+https://', + 'svn+http://', ) @@ -75,35 +80,34 @@ def normalize_vcs_url(repo_url, vcs_tool=None): if repo_url.startswith(VCS_URLS + PLAIN_URLS): return repo_url - if repo_url.startswith("git@"): - tool, _, right = repo_url.partition("@") - if ":" in repo_url: - host, _, repo = right.partition(":") + if repo_url.startswith('git@'): + tool, _, right = repo_url.partition('@') + if ':' in repo_url: + host, _, repo = right.partition(':') else: # git@github.com/Filirom1/npm2aur.git - host, _, repo = right.partition("/") + host, _, repo = right.partition('/') - if any(r in host for r in ("bitbucket", "gitlab", "github")): - scheme = "https" + if any(r in host for r in ('bitbucket', 'gitlab', 'github')): + scheme = 'https' else: - scheme = "git" + scheme = 'git' - return "%(scheme)s://%(host)s/%(repo)s" % locals() + return '%(scheme)s://%(host)s/%(repo)s' % locals() # FIXME: where these URL schemes come from?? - if repo_url.startswith(("bitbucket:", "gitlab:", "github:", "gist:")): + if repo_url.startswith(('bitbucket:', 'gitlab:', 'github:', 'gist:')): hoster_urls = { - "bitbucket": "https://bitbucket.org/%(repo)s", - "github": "https://github.com/%(repo)s", - "gitlab": "https://gitlab.com/%(repo)s", - "gist": "https://gist.github.com/%(repo)s", - } - hoster, _, repo = repo_url.partition(":") + 'bitbucket': 'https://bitbucket.org/%(repo)s', + 'github': 'https://github.com/%(repo)s', + 'gitlab': 'https://gitlab.com/%(repo)s', + 'gist': 'https://gist.github.com/%(repo)s', } + hoster, _, repo = repo_url.partition(':') return hoster_urls[hoster] % locals() - if len(repo_url.split("/")) == 2: + if len(repo_url.split('/')) == 2: # implicit github, but that's only on NPM? - return f"https://github.com/{repo_url}" + return f'https://github.com/{repo_url}' return repo_url @@ -112,18 +116,36 @@ def build_description(summary, description): """ Return a description string from a summary and description """ - summary = (summary or "").strip() - description = (description or "").strip() + summary = (summary or '').strip() + description = (description or '').strip() if not description: description = summary else: if summary and summary not in description: - description = "\n".join([summary, description]) + description = '\n'.join([summary , description]) return description +_LICENSING = Licensing and Licensing() or None + + +def combine_expressions( + expressions, + relation='AND', + unique=True, + licensing=_LICENSING, +): + """ + Return a combined license expression string with relation, given a sequence of + license ``expressions`` strings or LicenseExpression objects. + """ + if not licensing: + raise Exception('combine_expressions: cannot combine combine_expressions without license_expression package.') + return expressions and str(le_combine_expressions(expressions, relation, unique, licensing)) or None + + def get_ancestor(levels_up, resource, codebase): """ Return the nth-``levels_up`` ancestor Resource of ``resource`` in @@ -163,7 +185,7 @@ def find_root_resource(path, resource, codebase): """ if not resource.path.endswith(path): return - for _seg in path.split("/"): + for _seg in path.split('/'): resource = resource.parent(codebase) if not resource: return @@ -175,7 +197,6 @@ def yield_dependencies_from_package_data(package_data, datafile_path, package_ui Yield a Dependency for each dependency from ``package_data.dependencies`` """ from _packagedcode import models - dependent_packages = package_data.dependencies if dependent_packages: yield from models.Dependency.from_dependent_packages( @@ -191,7 +212,6 @@ def yield_dependencies_from_package_resource(resource, package_uid=None): Yield a Dependency for each dependency from each package from``resource.package_data`` """ from _packagedcode import models - for pkg_data in resource.package_data: pkg_data = models.PackageData.from_dict(pkg_data) yield from yield_dependencies_from_package_data(pkg_data, resource.path, package_uid) diff --git a/tests/test_codestyle.py b/tests/test_codestyle.py index f45dc83e..77045b07 100644 --- a/tests/test_codestyle.py +++ b/tests/test_codestyle.py @@ -16,7 +16,7 @@ def test_codestyle(self): """ This test shouldn't run in proliferated repositories. """ - args = "venv/bin/black --check -l 100 setup.py tests src" + args = "make check" try: subprocess.check_output(args.split()) except subprocess.CalledProcessError as e: @@ -24,7 +24,6 @@ def test_codestyle(self): print(e.output) print("===========================================================") raise Exception( - "Black style check failed; please format the code using:\n" - " python -m black -l 100 setup.py tests src", + "Code style check failed!", e.output, ) from e