diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 477590a071f..bef2409ddfe 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -62,6 +62,18 @@ License detection: removed. This new command supports simpler reindexing using custom license texts and license rules contributed by plugins or stored in an additional directory. + +Package detection: +~~~~~~~~~~~~~~~~~~~~~ + +- Code for parsing a Maven POM, npm package.json, freebsd manifest and haxelib + JSON have been separated into two functions: one that creates a PackageData + object from the parsed Resource, and another that calls the previous function + and yields the PackageData. This was done such that we can use the package + manifest data parsing code outside of the scancode-toolkit context in other + libraries. + + v31.2.1 - 2022-10-05 ---------------------------------- diff --git a/setup.cfg b/setup.cfg index 6129fe74235..99db3f5495d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = scancode-toolkit -version = 31.2.1 +version = 31.2.2 license = Apache-2.0 AND CC-BY-4.0 AND LicenseRef-scancode-other-permissive AND LicenseRef-scancode-other-copyleft # description must be on ONE line https://github.com/pypa/setuptools/issues/1390 diff --git a/src/packagedcode/chef.py b/src/packagedcode/chef.py index 7915ea20dfe..a08423c0ccf 100644 --- a/src/packagedcode/chef.py +++ b/src/packagedcode/chef.py @@ -55,7 +55,7 @@ def chef_api_url(name, version): return name and version and f'https://supermarket.chef.io/api/v1/cookbooks/{name}/versions/{version}' -def get_urls(name, version): +def get_urls(name, version, **kwargs): """ Return a mapping of URLs given a name and version. """ diff --git a/src/packagedcode/cocoapods.py b/src/packagedcode/cocoapods.py index f3cb917023a..755b0ec58dd 100644 --- a/src/packagedcode/cocoapods.py +++ b/src/packagedcode/cocoapods.py @@ -434,7 +434,7 @@ def parse(cls, location): ) -def get_urls(name=None, version=None, homepage_url=None, vcs_url=None): +def get_urls(name=None, version=None, homepage_url=None, vcs_url=None, **kwargs): """ Return a mapping of podspec URLS. """ diff --git a/src/packagedcode/freebsd.py b/src/packagedcode/freebsd.py index 814c6e5fce9..a1ee941999a 100644 --- a/src/packagedcode/freebsd.py +++ b/src/packagedcode/freebsd.py @@ -42,20 +42,13 @@ class CompactManifestHandler(models.DatafileHandler): documentation_url = 'https://www.freebsd.org/cgi/man.cgi?pkg-create(8)#MANIFEST_FILE_DETAILS' @classmethod - def parse(cls, location): - """ - Yield one or more Package manifest objects given a file ``location`` pointing to a - package archive, manifest or similar. - """ - with io.open(location, encoding='utf-8') as loc: - freebsd_manifest = saneyaml.load(loc) - + def _parse(cls, yaml_data): package_data = models.PackageData( datasource_id=cls.datasource_id, type=cls.default_package_type, qualifiers=dict( - arch=freebsd_manifest.get('arch'), - origin=freebsd_manifest.get('origin'), + arch=yaml_data.get('arch'), + origin=yaml_data.get('origin'), ) ) @@ -69,7 +62,7 @@ def parse(cls, location): ] for source, target in plain_fields: - value = freebsd_manifest.get(source) + value = yaml_data.get(source) if value: if isinstance(value, str): value = value.strip() @@ -87,17 +80,28 @@ def parse(cls, location): for source, func in field_mappers: logger.debug('parse: %(source)r, %(func)r' % locals()) - value = freebsd_manifest.get(source) or None + value = yaml_data.get(source) or None if value: func(value, package_data) # license_mapper needs multiple fields - license_mapper(freebsd_manifest, package_data) + license_mapper(yaml_data, package_data) if package_data.declared_license: package_data.license_expression = cls.compute_normalized_license(package_data) - yield package_data + return package_data + + @classmethod + def parse(cls, location): + """ + Yield one or more Package manifest objects given a file ``location`` pointing to a + package archive, manifest or similar. + """ + with io.open(location, encoding='utf-8') as loc: + yaml_data = saneyaml.load(loc) + + yield cls._parse(yaml_data) @classmethod def compute_normalized_license(cls, package): diff --git a/src/packagedcode/haxe.py b/src/packagedcode/haxe.py index 181056ced36..f52becffa99 100644 --- a/src/packagedcode/haxe.py +++ b/src/packagedcode/haxe.py @@ -45,26 +45,7 @@ class HaxelibJsonHandler(models.DatafileHandler): documentation_url = 'https://lib.haxe.org/documentation/creating-a-haxelib-package/' @classmethod - def parse(cls, location): - """ - Yield one or more Package manifest objects given a file ``location`` pointing to a - package_data archive, manifest or similar. - - { - "name": "haxelib", - "url" : "https://lib.haxe.org/documentation/", - "license": "GPL", - "tags": ["haxelib", "core"], - "description": "The haxelib client", - "classPath": "src", - "version": "3.4.0", - "releasenote": " * Fix password input issue in Windows (#421).\n * ....", - "contributors": ["back2dos", "ncannasse", "jason", "Simn", "nadako", "andyli"] - } - """ - with io.open(location, encoding='utf-8') as loc: - json_data = json.load(loc) - + def _parse(cls, json_data): name = json_data.get('name') version = json_data.get('version') @@ -110,4 +91,27 @@ def parse(cls, location): dep = models.DependentPackage(purl=dep_purl, is_resolved=is_resolved,) package_data.dependencies.append(dep) - yield package_data + return package_data + + @classmethod + def parse(cls, location): + """ + Yield one or more Package manifest objects given a file ``location`` pointing to a + package_data archive, manifest or similar. + + { + "name": "haxelib", + "url" : "https://lib.haxe.org/documentation/", + "license": "GPL", + "tags": ["haxelib", "core"], + "description": "The haxelib client", + "classPath": "src", + "version": "3.4.0", + "releasenote": " * Fix password input issue in Windows (#421).\n * ....", + "contributors": ["back2dos", "ncannasse", "jason", "Simn", "nadako", "andyli"] + } + """ + with io.open(location, encoding='utf-8') as loc: + json_data = json.load(loc) + + yield cls._parse(json_data) diff --git a/src/packagedcode/maven.py b/src/packagedcode/maven.py index f81c9d6e607..d96fb907a3f 100644 --- a/src/packagedcode/maven.py +++ b/src/packagedcode/maven.py @@ -891,12 +891,12 @@ def has_basic_pom_attributes(pom): return basics -def get_maven_pom(location=None): +def get_maven_pom(location=None, text=None): """ Return a MavenPom object from a POM file at `location` or provided as a `text` string. """ - pom = MavenPom(location=location) + pom = MavenPom(location=location, text=text) extra_properties = {} @@ -1043,7 +1043,7 @@ def get_parties(pom): return parties -def get_urls(namespace, name, version, qualifiers, base_url='https://repo1.maven.org/maven2'): +def get_urls(namespace, name, version, qualifiers, base_url='https://repo1.maven.org/maven2', **kwargs): """ Return a mapping of URLs. """ @@ -1105,7 +1105,30 @@ def parse( Yield Packagedata objects from parsing a Maven pom file at `location` or using the provided `text` (one or the other but not both). """ - pom = get_maven_pom(location=location) + package = _parse( + datasource_id=datasource_id, + package_type=package_type, + primary_language=primary_language, + location=location, + base_url=base_url + ) + if package: + yield package + + +def _parse( + datasource_id, + package_type, + primary_language, + location=None, + text=None, + base_url='https://repo1.maven.org/maven2', +): + """ + Yield Packagedata objects from parsing a Maven pom file at `location` or + using the provided `text` (one or the other but not both). + """ + pom = get_maven_pom(location=location, text=text) if not pom: return @@ -1192,7 +1215,8 @@ def parse( if not package_data.license_expression and package_data.declared_license: package_data.license_expression = models.compute_normalized_license(package_data.declared_license) - yield package_data + return package_data + def build_vcs_and_code_view_urls(scm): """ diff --git a/src/packagedcode/models.py b/src/packagedcode/models.py index a3da606aaa2..70ff88528e1 100644 --- a/src/packagedcode/models.py +++ b/src/packagedcode/models.py @@ -307,7 +307,7 @@ def set_purl(self, package_url): for key, value in package_url.to_dict().items(): self_val = getattr(self, key) if not self_val and value: - setattr(self, attr, value) + setattr(self, key, value) def to_dict(self, **kwargs): mapping = super().to_dict(**kwargs) diff --git a/src/packagedcode/npm.py b/src/packagedcode/npm.py index 2a50d1434a3..8c68bfe4747 100644 --- a/src/packagedcode/npm.py +++ b/src/packagedcode/npm.py @@ -146,7 +146,7 @@ def walk_npm(cls, resource, codebase, depth=0): yield subchild -def get_urls(namespace, name, version): +def get_urls(namespace, name, version, **kwargs): return dict( repository_homepage_url=npm_homepage_url(namespace, name, registry='https://www.npmjs.com/package'), repository_download_url=npm_download_url(namespace, name, version, registry='https://registry.npmjs.org'), @@ -163,13 +163,10 @@ class NpmPackageJsonHandler(BaseNpmHandler): documentation_url = 'https://docs.npmjs.com/cli/v8/configuring-npm/package-json' @classmethod - def parse(cls, location): - with io.open(location, encoding='utf-8') as loc: - package_data = json.load(loc) - - name = package_data.get('name') - version = package_data.get('version') - homepage_url = package_data.get('homepage', '') + def _parse(cls, json_data): + name = json_data.get('name') + version = json_data.get('version') + homepage_url = json_data.get('homepage', '') # a package.json without name and version can be a private package @@ -188,11 +185,11 @@ def parse(cls, location): namespace=namespace or None, name=name, version=version or None, - description=package_data.get('description', '').strip() or None, + description=json_data.get('description', '').strip() or None, homepage_url=homepage_url, **urls, ) - vcs_revision = package_data.get('gitHead') or None + vcs_revision = json_data.get('gitHead') or None # mapping of top level package.json items to a function accepting as # arguments the package.json element value and returning an iterable of (key, @@ -214,7 +211,7 @@ def parse(cls, location): ] for source, func in field_mappers: - value = package_data.get(source) or None + value = json_data.get(source) or None if value: if isinstance(value, str): value = value.strip() @@ -226,14 +223,21 @@ def parse(cls, location): package.download_url = npm_download_url(package.namespace, package.name, package.version) # licenses are a tad special with many different data structures - lic = package_data.get('license') - lics = package_data.get('licenses') + lic = json_data.get('license') + lics = json_data.get('licenses') package = licenses_mapper(lic, lics, package) if not package.license_expression and package.declared_license: package.license_expression = compute_normalized_license(package.declared_license) - yield package + return package + + @classmethod + def parse(cls, location): + with io.open(location, encoding='utf-8') as loc: + json_data = json.load(loc) + + yield cls._parse(json_data) @classmethod def compute_normalized_license(cls, package): diff --git a/src/packagedcode/nuget.py b/src/packagedcode/nuget.py index b828d4ab9e8..3884d37a2bf 100644 --- a/src/packagedcode/nuget.py +++ b/src/packagedcode/nuget.py @@ -18,7 +18,7 @@ # TODO: add dependencies -def get_urls(name, version): +def get_urls(name, version, **kwargs): return dict( repository_homepage_url=f'https://www.nuget.org/packages/{name}/{version}', repository_download_url=f'https://www.nuget.org/api/v2/package/{name}/{version}', diff --git a/src/packagedcode/pypi.py b/src/packagedcode/pypi.py index e64cd490520..59b6f6a7221 100644 --- a/src/packagedcode/pypi.py +++ b/src/packagedcode/pypi.py @@ -1609,7 +1609,7 @@ def get_setup_py_args(location, include_not_parsable=False): return parse_setup_py(location) -def get_pypi_urls(name, version): +def get_pypi_urls(name, version, **kwargs): """ Return a mapping of computed Pypi URLs for this package """ diff --git a/src/packagedcode/rubygems.py b/src/packagedcode/rubygems.py index b13e4b780aa..7ad17869b8a 100644 --- a/src/packagedcode/rubygems.py +++ b/src/packagedcode/rubygems.py @@ -337,7 +337,7 @@ def compute_normalized_license(declared_license): return combine_expressions(detected_licenses) -def get_urls(name, version=None, platform=None): +def get_urls(name, version=None, platform=None, **kwargs): """ Return a mapping of standard URLs """