Skip to content

Commit 5d34442

Browse files
AyanSinhaMahapatraAyanSinhaMahapatra
committed
Update resolved package support for npm
Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>
1 parent 514e1fd commit 5d34442

9 files changed

Lines changed: 342117 additions & 5720 deletions

File tree

‎src/packagedcode/npm.py‎

Lines changed: 213 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -172,6 +172,68 @@ def walk_npm(cls, resource, codebase, depth=0):
172172
for subchild in cls.walk_npm(child, codebase, depth=depth):
173173
yield subchild
174174

175+
@classmethod
176+
def update_dependencies_by_purl(
177+
cls,
178+
dependencies,
179+
scope,
180+
dependecies_by_purl,
181+
is_runtime=False,
182+
is_optional=False,
183+
is_resolved=False,
184+
):
185+
186+
metadata_deps = ['peerDependenciesMeta', 'dependenciesMeta']
187+
if type(dependencies) == list:
188+
for subdep in dependencies:
189+
sdns, _ , sdname = subdep.rpartition('/')
190+
dep_purl = PackageURL(
191+
type=cls.default_package_type,
192+
namespace=sdns,
193+
name=sdname
194+
).to_string()
195+
dep_package = models.DependentPackage(
196+
purl=dep_purl,
197+
scope=scope,
198+
is_runtime=is_runtime,
199+
is_optional=is_optional,
200+
is_resolved=is_resolved,
201+
)
202+
dependecies_by_purl[dep_purl] = dep_package
203+
204+
elif type(dependencies) == dict:
205+
for subdep, metadata in dependencies.items():
206+
sdns, _ , sdname = subdep.rpartition('/')
207+
dep_purl = PackageURL(
208+
type=cls.default_package_type,
209+
namespace=sdns,
210+
name=sdname
211+
).to_string()
212+
213+
if scope in metadata_deps :
214+
dep_package = dependecies_by_purl.get(dep_purl)
215+
dep_package.is_optional = metadata.get("optional")
216+
continue
217+
218+
# pnpm has peer dependencies also sometimes in version?
219+
# dependencies:
220+
# '@react-spring/animated': 9.5.5_react@18.2.0
221+
# TODO: store this relation too?
222+
requirement = metadata
223+
if 'pnpm' in cls.datasource_id:
224+
if '_' in metadata:
225+
requirement, _extra = metadata.split('_')
226+
227+
dep_package = models.DependentPackage(
228+
purl=dep_purl,
229+
scope=scope,
230+
extracted_requirement=requirement,
231+
is_runtime=is_runtime,
232+
is_optional=is_optional,
233+
is_resolved=is_resolved,
234+
)
235+
dependecies_by_purl[dep_purl] = dep_package
236+
175237

176238
def get_urls(namespace, name, version, **kwargs):
177239
return dict(
@@ -308,6 +370,38 @@ def parse(cls, location, package_only=False):
308370

309371
deps_mapping = package_data.get(deps_key) or {}
310372

373+
# Top level package metadata is present here
374+
root_pkg = deps_mapping.get("")
375+
if root_pkg:
376+
pkg_name = root_pkg.get('name')
377+
pkg_ns, _ , pkg_name = pkg_name.rpartition('/')
378+
pkg_version = root_pkg.get('version')
379+
pkg_purl = PackageURL(
380+
type=cls.default_package_type,
381+
namespace=pkg_ns,
382+
name=pkg_name,
383+
version=pkg_version,
384+
).to_string()
385+
if pkg_purl != root_package_data.purl:
386+
if TRACE_NPM:
387+
logger_debug(f'BaseNpmLockHandler: parse: purl mismatch: {pkg_purl} vs {root_package_data.purl}')
388+
else:
389+
extracted_license_statement = root_pkg.get('license')
390+
if extracted_license_statement:
391+
root_package_data.extracted_license_statement = extracted_license_statement
392+
root_package_data.populate_license_fields()
393+
394+
deps_mapper(
395+
deps=root_pkg.get('devDependencies') or {},
396+
package=root_package_data,
397+
field_name='devDependencies',
398+
)
399+
deps_mapper(
400+
deps=root_pkg.get('optionalDependencies') or {},
401+
package=root_package_data,
402+
field_name='optionalDependencies',
403+
)
404+
311405
dependencies = []
312406

313407
for dep, dep_data in deps_mapping.items():
@@ -326,7 +420,7 @@ def parse(cls, location, package_only=False):
326420
if not dep:
327421
# in v2 format the first dep is the same as the top level
328422
# package and has no name
329-
pass
423+
continue
330424

331425
# only present for first top level
332426
# otherwise get name from dep
@@ -357,9 +451,6 @@ def parse(cls, location, package_only=False):
357451
is_resolved=True,
358452
)
359453

360-
# only seen in v2 for the top level package... but good to keep
361-
extracted_license_statement = dep_data.get('license')
362-
363454
# URLs and checksums
364455
misc = get_urls(ns, name, version)
365456
resolved = dep_data.get('resolved')
@@ -374,7 +465,6 @@ def parse(cls, location, package_only=False):
374465
namespace=ns,
375466
name=name,
376467
version=version,
377-
extracted_license_statement=extracted_license_statement,
378468
**misc,
379469
)
380470
resolved_package = models.PackageData.from_data(resolved_package_mapping, package_only)
@@ -385,13 +475,11 @@ def parse(cls, location, package_only=False):
385475
# v1 as name/constraint pairs
386476
subrequires = dep_data.get('requires') or {}
387477

388-
# in v1 these are further nested dependencies
478+
# in v1 these are further nested dependencies (TODO: handle these with tests)
389479
# in v2 these are name/constraint pairs like v1 requires
390480
subdependencies = dep_data.get('dependencies')
391481

392482
# v2? ignored for now
393-
dev_subdependencies = dep_data.get('devDependencies')
394-
optional_subdependencies = dep_data.get('optionalDependencies')
395483
engines = dep_data.get('engines')
396484
funding = dep_data.get('funding')
397485

@@ -401,25 +489,20 @@ def parse(cls, location, package_only=False):
401489
subdeps_data = subdependencies
402490
subdeps_data = subdeps_data or {}
403491

404-
sub_deps = []
405-
for subdep, subdep_req in subdeps_data.items():
406-
sdns, _ , sdname = subdep.rpartition('/')
407-
sdpurl = PackageURL(
408-
type=cls.default_package_type,
409-
namespace=sdns,
410-
name=sdname
411-
).to_string()
412-
sub_deps.append(
413-
models.DependentPackage(
414-
purl=sdpurl,
415-
scope=scope,
416-
extracted_requirement=subdep_req,
417-
is_runtime=is_runtime,
418-
is_optional=is_optional,
419-
is_resolved=False,
420-
)
421-
)
422-
resolved_package.dependencies = sub_deps
492+
sub_deps_by_purl = {}
493+
cls.update_dependencies_by_purl(
494+
dependencies=subdeps_data,
495+
scope=scope,
496+
dependecies_by_purl=sub_deps_by_purl,
497+
is_runtime=is_runtime,
498+
is_optional=is_optional,
499+
is_resolved=False,
500+
)
501+
502+
resolved_package.dependencies = [
503+
sub_dep.to_dict()
504+
for sub_dep in sub_deps_by_purl.values()
505+
]
423506
dependency.resolved_package = resolved_package.to_dict()
424507
dependencies.append(dependency.to_dict())
425508

@@ -535,24 +618,57 @@ def parse(cls, location, package_only=False):
535618
version=version,
536619
)
537620

538-
# TODO: add resolved_package with its own deps
621+
# TODO: what type of checksum is this?
539622
checksum = details.get('checksum')
540623
dependencies = details.get('dependencies') or {}
541624
peer_dependencies = details.get('peerDependencies') or {}
542625
dependencies_meta = details.get('dependenciesMeta') or {}
543626
# these are file references
544627
bin = details.get('bin') or []
545628

629+
deps_for_resolved_by_purl = {}
630+
cls.update_dependencies_by_purl(
631+
dependencies=dependencies,
632+
scope="dependencies",
633+
dependecies_by_purl=deps_for_resolved_by_purl,
634+
)
635+
cls.update_dependencies_by_purl(
636+
dependencies=peer_dependencies,
637+
scope="peerDependencies",
638+
dependecies_by_purl=deps_for_resolved_by_purl,
639+
)
640+
cls.update_dependencies_by_purl(
641+
dependencies=dependencies_meta,
642+
scope="dependenciesMeta",
643+
dependecies_by_purl=deps_for_resolved_by_purl,
644+
)
645+
646+
dependencies_for_resolved = [
647+
dep_package.to_dict()
648+
for dep_package in deps_for_resolved_by_purl.values()
649+
]
650+
651+
resolved_package_mapping = dict(
652+
datasource_id=cls.datasource_id,
653+
type=cls.default_package_type,
654+
primary_language=cls.default_primary_language,
655+
namespace=ns,
656+
name=name,
657+
version=version,
658+
dependencies=dependencies_for_resolved,
659+
660+
)
661+
resolved_package = models.PackageData.from_data(resolved_package_mapping)
546662
dependency = models.DependentPackage(
547-
purl=str(purl),
548-
extracted_requirement=version,
549-
is_resolved=True,
550-
# FIXME: these are NOT correct
551-
scope='dependencies',
552-
# TODO: get details from metadata
553-
is_optional=False,
554-
is_runtime=True,
555-
)
663+
purl=str(purl),
664+
extracted_requirement=version,
665+
is_resolved=True,
666+
resolved_package=resolved_package.to_dict(),
667+
# FIXME: these are NOT correct
668+
scope='dependencies',
669+
is_optional=False,
670+
is_runtime=True,
671+
)
556672
top_dependencies.append(dependency.to_dict())
557673

558674
update_dependencies_as_resolved(dependencies=top_dependencies)
@@ -790,10 +906,64 @@ def parse(cls, location, package_only=False):
790906
version=version,
791907
).to_string()
792908

793-
# TODO: add resolved_package and dependencies from the following:
794-
# 'peerDependencies', 'optionalDependencies', 'dependencies',
795-
# 'transitivePeerDependencies', 'peerDependenciesMeta'
796-
# add sha512 from 'resolution'
909+
checksum = data.get('resolution') or {}
910+
integrity = checksum.get('integrity')
911+
misc = get_algo_hexsum(integrity)
912+
913+
dependencies = data.get('dependencies') or {}
914+
optional_dependencies = data.get('optionalDependencies') or {}
915+
transitive_peer_dependencies = data.get('transitivePeerDependencies') or {}
916+
peer_dependencies = data.get('peerDependencies') or {}
917+
peer_dependencies_meta = data.get('peerDependenciesMeta') or {}
918+
919+
deps_for_resolved_by_purl = {}
920+
cls.update_dependencies_by_purl(
921+
dependencies=dependencies,
922+
scope='dependencies',
923+
dependecies_by_purl=deps_for_resolved_by_purl,
924+
is_resolved=True,
925+
)
926+
cls.update_dependencies_by_purl(
927+
dependencies=peer_dependencies,
928+
scope='peerDependencies',
929+
dependecies_by_purl=deps_for_resolved_by_purl,
930+
is_optional=True,
931+
)
932+
cls.update_dependencies_by_purl(
933+
dependencies=optional_dependencies,
934+
scope='optionalDependencies',
935+
dependecies_by_purl=deps_for_resolved_by_purl,
936+
is_resolved=True,
937+
is_optional=True,
938+
)
939+
cls.update_dependencies_by_purl(
940+
dependencies=peer_dependencies_meta,
941+
scope='peerDependenciesMeta',
942+
dependecies_by_purl=deps_for_resolved_by_purl,
943+
)
944+
cls.update_dependencies_by_purl(
945+
dependencies=transitive_peer_dependencies,
946+
scope='transitivePeerDependencies',
947+
dependecies_by_purl=deps_for_resolved_by_purl,
948+
)
949+
950+
dependencies_for_resolved = [
951+
dep_package.to_dict()
952+
for dep_package in deps_for_resolved_by_purl.values()
953+
]
954+
955+
resolved_package_mapping = dict(
956+
datasource_id=cls.datasource_id,
957+
type=cls.default_package_type,
958+
primary_language=cls.default_primary_language,
959+
namespace=namespace,
960+
name=name,
961+
version=version,
962+
dependencies=dependencies_for_resolved,
963+
**misc,
964+
)
965+
resolved_package = models.PackageData.from_data(resolved_package_mapping)
966+
797967
extra_data_fields = ["cpu", "os", "engines", "deprecated", "hasBin"]
798968

799969
is_dev = data.get("dev", False)
@@ -811,13 +981,14 @@ def parse(cls, location, package_only=False):
811981
is_optional=is_optional,
812982
is_runtime=is_runtime,
813983
is_resolved=True,
984+
resolved_package=resolved_package.to_dict(),
814985
extra_data=extra_data_deps,
815986
)
816987
dependencies_by_purl[purl] = dependency_data
817988

818989
dependencies = [
819990
dep.to_dict()
820-
for dep in list(dependencies_by_purl.values())
991+
for dep in dependencies_by_purl.values()
821992
]
822993
update_dependencies_as_resolved(dependencies=dependencies)
823994
root_package_data = dict(

0 commit comments

Comments
 (0)