From 5cf11ace687062a8300d02ce2ed361009da6c11b Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Wed, 22 May 2024 15:23:12 +0530 Subject: [PATCH 01/11] Resolve dependencies from lockfiles #1237 Reference: https://github.com/nexB/scancode.io/issues/1237 Reference: https://github.com/nexB/scancode.io/issues/1066 Signed-off-by: Ayan Sinha Mahapatra --- scanpipe/api/serializers.py | 3 + scanpipe/filters.py | 4 + .../0061_dependency_resolver_update.py | 34 + scanpipe/models.py | 51 + scanpipe/pipelines/inspect_packages.py | 20 +- scanpipe/pipelines/resolve_dependencies.py | 19 + scanpipe/pipes/__init__.py | 32 +- scanpipe/pipes/scancode.py | 120 +- .../data/alpine_3_15_4_scan_codebase.json | 28 + scanpipe/tests/data/asgiref-3.3.0.spdx.json | 42 +- .../tests/data/asgiref-3.3.0_fixtures.json | 134 +- ...asgiref-3.3.0_load_inventory_expected.json | 16 + .../data/asgiref-3.3.0_scanpipe_output.json | 108 +- .../data/asgiref-3.3.0_toolkit_scan.json | 80 +- .../asgiref-3.3.0_walk_test_fixtures.json | 134 +- .../data/basic-rootfs_root_filesystems.json | 4 + scanpipe/tests/data/centos_scan_codebase.json | 202 +++ .../data/cyclonedx/asgiref-3.3.0.cdx.json | 18 +- .../tests/data/d2d/about_files/expected.json | 4 + ...-0.6.0-py3-none-any.whl_scan_codebase.json | 24 + scanpipe/tests/data/debian_scan_codebase.json | 4 + .../dependencies/resolved_dependencies.zip | Bin 0 -> 1577 bytes scanpipe/tests/data/flume-ng-node-d2d.json | 2 + .../gcr_io_distroless_base_scan_codebase.json | 16 + .../data/is-npm-1.0.0_scan_codebase.json | 18 +- .../tests/data/is-npm-1.0.0_scan_package.json | 18 +- .../is-npm-1.0.0_scan_package_summary.json | 8 +- .../openpdf-parent-1.3.11_scan_package.json | 24 + .../multiple-is-npm-1.0.0_scan_package.json | 36 +- ...ple-is-npm-1.0.0_scan_package_summary.json | 8 +- ...esolved_dependencies_inspect_packages.json | 1127 +++++++++++++++++ .../data/scancode/is-npm-1.0.0_summary.json | 8 +- .../scancode/package_assembly_codebase.json | 22 +- scanpipe/tests/pipes/test_output.py | 2 +- scanpipe/tests/test_api.py | 4 +- scanpipe/tests/test_pipelines.py | 34 +- scanpipe/views.py | 8 + setup.cfg | 2 +- 38 files changed, 2169 insertions(+), 249 deletions(-) create mode 100644 scanpipe/migrations/0061_dependency_resolver_update.py create mode 100644 scanpipe/tests/data/dependencies/resolved_dependencies.zip create mode 100644 scanpipe/tests/data/resolved_dependencies_inspect_packages.json diff --git a/scanpipe/api/serializers.py b/scanpipe/api/serializers.py index b6fdf0de39..d42639cedf 100644 --- a/scanpipe/api/serializers.py +++ b/scanpipe/api/serializers.py @@ -380,6 +380,8 @@ class Meta: "source_packages", "extra_data", "package_uid", + "is_private", + "is_virtual", "datasource_ids", "datafile_paths", "file_references", @@ -405,6 +407,7 @@ class Meta: "is_runtime", "is_optional", "is_resolved", + "is_direct", "dependency_uid", "for_package_uid", "resolved_to_package_uid", diff --git a/scanpipe/filters.py b/scanpipe/filters.py index 4c32b13f78..f030290cd8 100644 --- a/scanpipe/filters.py +++ b/scanpipe/filters.py @@ -727,6 +727,7 @@ class DependencyFilterSet(FilterSetUtilsMixin, django_filters.FilterSet): "is_runtime", "is_optional", "is_resolved", + "is_direct", "datasource_id", "is_vulnerable", ] @@ -747,6 +748,7 @@ class DependencyFilterSet(FilterSetUtilsMixin, django_filters.FilterSet): "is_runtime", "is_optional", "is_resolved", + "is_direct", "for_package", "resolved_to_package", "datafile_resource", @@ -761,6 +763,7 @@ class DependencyFilterSet(FilterSetUtilsMixin, django_filters.FilterSet): is_runtime = StrictBooleanFilter() is_optional = StrictBooleanFilter() is_resolved = StrictBooleanFilter() + is_direct = StrictBooleanFilter() is_vulnerable = IsVulnerable(field_name="affected_by_vulnerabilities") class Meta: @@ -779,6 +782,7 @@ class Meta: "is_runtime", "is_optional", "is_resolved", + "is_direct", "datasource_id", "is_vulnerable", ] diff --git a/scanpipe/migrations/0061_dependency_resolver_update.py b/scanpipe/migrations/0061_dependency_resolver_update.py new file mode 100644 index 0000000000..0b5533d383 --- /dev/null +++ b/scanpipe/migrations/0061_dependency_resolver_update.py @@ -0,0 +1,34 @@ +# Generated by Django 5.0.6 on 2024-06-04 20:48 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("scanpipe", "0060_discovereddependency_renames"), + ] + + operations = [ + migrations.AddField( + model_name="discovereddependency", + name="is_direct", + field=models.BooleanField(default=False), + ), + migrations.AddField( + model_name="discoveredpackage", + name="is_private", + field=models.BooleanField(default=False), + ), + migrations.AddField( + model_name="discoveredpackage", + name="is_virtual", + field=models.BooleanField(default=False), + ), + migrations.AddIndex( + model_name="discovereddependency", + index=models.Index( + fields=["is_direct"], name="scanpipe_di_is_dire_6dc594_idx" + ), + ), + ] diff --git a/scanpipe/models.py b/scanpipe/models.py index 3fa0b4bcbb..9218516762 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -1003,6 +1003,19 @@ def walk_codebase_path(self): """Return files and directories path of the codebase/ directory recursively.""" return self.codebase_path.rglob("*") + def get_resource(self, path): + """ + Return the codebase resource present for a given path, + or None the resource with that path does not exist. + This path is relative to the scan location. + This is same as the Codebase.get_resource() function. + """ + # We don't want to raise an exception if there is no resource + # as this function is also called from the SCTK side + resource = self.codebaseresources.get_or_none(path=path) + if resource: + return resource + @cached_property def can_change_inputs(self): """ @@ -2971,6 +2984,8 @@ class AbstractPackage(models.Model): blank=True, help_text=_("A notice text for this package."), ) + is_private = models.BooleanField(default=False) + is_virtual = models.BooleanField(default=False) datasource_ids = models.JSONField( default=list, blank=True, @@ -3432,6 +3447,7 @@ class DiscoveredDependency( is_runtime = models.BooleanField(default=False) is_optional = models.BooleanField(default=False) is_resolved = models.BooleanField(default=False) + is_direct = models.BooleanField(default=False) objects = DiscoveredDependencyQuerySet.as_manager() @@ -3452,6 +3468,7 @@ class Meta: models.Index(fields=["is_runtime"]), models.Index(fields=["is_optional"]), models.Index(fields=["is_resolved"]), + models.Index(fields=["is_direct"]), ] constraints = [ models.UniqueConstraint( @@ -3498,6 +3515,7 @@ def create_from_data( project, dependency_data, for_package=None, + resolved_to_package=None, datafile_resource=None, datasource_id=None, strip_datafile_path_root=False, @@ -3537,6 +3555,13 @@ def create_from_data( package_uid=for_package_uid ) + if not resolved_to_package: + resolved_to_uid = dependency_data.get("resolved_to_uid") + if resolved_to_uid: + resolved_to_package = project.discoveredpackages.get( + package_uid=resolved_to_uid + ) + if not datafile_resource: datafile_path = dependency_data.get("datafile_path") if datafile_path: @@ -3562,10 +3587,36 @@ def create_from_data( return cls.objects.create( project=project, for_package=for_package, + resolved_to_package=resolved_to_package, datafile_resource=datafile_resource, **cleaned_data, ) + @classmethod + def extract_purl_data(cls, dependency_data, ignore_nulls=False): + purl_mapping = PackageURL.from_string( + purl=dependency_data.get("purl"), + ).to_dict() + purl_data = {} + + for field_name in PURL_FIELDS: + value = purl_mapping.get(field_name) + if field_name == "qualifiers": + value = normalize_qualifiers(value, encode=True) + if not ignore_nulls: + purl_data[field_name] = value or "" + else: + if value: + purl_data[field_name] = value or "" + + return purl_data + + @classmethod + def populate_dependency_uuid(cls, dependency_data): + purl = PackageURL.from_string(purl=dependency_data.get("purl")) + purl.qualifiers["uuid"] = str(uuid.uuid4()) + dependency_data["dependency_uid"] = purl.to_string() + @property def spdx_id(self): return f"SPDXRef-scancodeio-{self._meta.model_name}-{self.dependency_uid}" diff --git a/scanpipe/pipelines/inspect_packages.py b/scanpipe/pipelines/inspect_packages.py index a40528c3a5..845fb6297b 100644 --- a/scanpipe/pipelines/inspect_packages.py +++ b/scanpipe/pipelines/inspect_packages.py @@ -20,6 +20,7 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/nexB/scancode.io for support and download. +from scanpipe.pipelines import group from scanpipe.pipelines.scan_codebase import ScanCodebase from scanpipe.pipes import scancode @@ -49,7 +50,7 @@ def steps(cls): cls.flag_empty_files, cls.flag_ignored_resources, cls.scan_for_application_packages, - cls.create_packages_and_dependencies, + cls.resolve_dependencies, ) def scan_for_application_packages(self): @@ -57,15 +58,18 @@ def scan_for_application_packages(self): Scan resources for package information to add DiscoveredPackage and DiscoveredDependency objects from detected package data. """ - # `assemble` is set to False because here in this pipeline we - # only detect package_data in resources and create - # Package/Dependency instances directly instead of assembling - # the packages and assigning files to them scancode.scan_for_application_packages( project=self.project, - assemble=False, + assemble=True, package_only=True, + progress_logger=self.log, ) - def create_packages_and_dependencies(self): - scancode.process_package_data(self.project) + @group("Static Resolver") + def resolve_dependencies(self): + """ + Create packages and dependency relationships from + lockfiles or manifests containing pre-resolved + dependencies. + """ + scancode.resolve_dependencies(project=self.project) diff --git a/scanpipe/pipelines/resolve_dependencies.py b/scanpipe/pipelines/resolve_dependencies.py index d2597e8eec..fa542976b6 100644 --- a/scanpipe/pipelines/resolve_dependencies.py +++ b/scanpipe/pipelines/resolve_dependencies.py @@ -20,8 +20,10 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/nexB/scancode.io for support and download. +from scanpipe.pipelines import group from scanpipe.pipelines.scan_codebase import ScanCodebase from scanpipe.pipes import resolve +from scanpipe.pipes import scancode class ResolveDependencies(ScanCodebase): @@ -45,6 +47,8 @@ def steps(cls): cls.collect_and_create_codebase_resources, cls.flag_ignored_resources, cls.get_manifest_inputs, + cls.scan_for_application_packages, + cls.create_packages_and_dependencies, cls.get_packages_from_manifest, cls.create_resolved_packages, ) @@ -53,6 +57,20 @@ def get_manifest_inputs(self): """Locate package manifest files with a supported package resolver.""" self.manifest_resources = resolve.get_manifest_resources(self.project) + @group("Static Resolver") + def scan_for_application_packages(self): + scancode.scan_for_application_packages( + self.project, + assemble=False, + resource_qs=self.manifest_resources, + progress_logger=self.log, + ) + + @group("Static Resolver") + def create_packages_and_dependencies(self): + scancode.process_package_data(self.project, static_resolve=True) + + @group("Dynamic Resolver") def get_packages_from_manifest(self): """ Resolve package data from lockfiles/requirement files with package @@ -65,6 +83,7 @@ def get_packages_from_manifest(self): model="get_packages_from_manifest", ) + @group("Dynamic Resolver") def create_resolved_packages(self): """Create the resolved packages and their dependencies in the database.""" resolve.create_packages_and_dependencies( diff --git a/scanpipe/pipes/__init__.py b/scanpipe/pipes/__init__.py index 61e2d2a79d..64fafcc284 100644 --- a/scanpipe/pipes/__init__.py +++ b/scanpipe/pipes/__init__.py @@ -168,7 +168,12 @@ def _clean_package_data(package_data): return package_data -def update_or_create_package(project, package_data, codebase_resources=None): +def update_or_create_package( + project, + package_data, + codebase_resources=None, + is_virtual=False, +): """ Get, update or create a DiscoveredPackage then return it. Use the `project` and `package_data` mapping to lookup and creates the @@ -194,6 +199,9 @@ def update_or_create_package(project, package_data, codebase_resources=None): package = DiscoveredPackage.create_from_data(project, package_data) if package: + if is_virtual: + package.update(is_virtual=is_virtual) + if datasource_id and datasource_id not in package.datasource_ids: datasource_ids = package.datasource_ids.copy() datasource_ids.append(datasource_id) @@ -239,6 +247,7 @@ def update_or_create_dependency( project, dependency_data, for_package=None, + resolved_to_package=None, datafile_resource=None, datasource_id=None, strip_datafile_path_root=False, @@ -254,27 +263,44 @@ def update_or_create_dependency( corresponding CodebaseResource for `datafile_path`. This is used in the case where Dependency data is imported from a scancode-toolkit scan, where the root path segments are not stripped for `datafile_path`. + If the dependency is resolved and a resolved package is created, we have the + corresponsing package_uid at `resolved_to`. """ dependency = None dependency_uid = dependency_data.get("dependency_uid") + extracted_requirement = dependency_data.get("extracted_requirement") if ignore_dependency_scope(project, dependency_data): return # Do not create the DiscoveredDependency record. if not dependency_uid: - dependency_data["dependency_uid"] = uuid.uuid4() + purl_data = DiscoveredDependency.extract_purl_data(dependency_data) + dependency = DiscoveredDependency.objects.get_or_none( + project=project, + extracted_requirement=extracted_requirement, + **purl_data, + ) else: - dependency = project.discovereddependencies.get_or_none( + dependency = DiscoveredDependency.objects.get_or_none( + project=project, dependency_uid=dependency_uid, ) if dependency: dependency.update_from_data(dependency_data) + if resolved_to_package and not dependency.resolved_to_package: + dependency.update(resolved_to_package=resolved_to_package) else: + is_direct = dependency_data.get("is_direct") + if not is_direct: + pass + + DiscoveredDependency.populate_dependency_uuid(dependency_data) dependency = DiscoveredDependency.create_from_data( project, dependency_data, for_package=for_package, + resolved_to_package=resolved_to_package, datafile_resource=datafile_resource, datasource_id=datasource_id, strip_datafile_path_root=strip_datafile_path_root, diff --git a/scanpipe/pipes/scancode.py b/scanpipe/pipes/scancode.py index aea32fd6b0..29585b5da4 100644 --- a/scanpipe/pipes/scancode.py +++ b/scanpipe/pipes/scancode.py @@ -47,6 +47,8 @@ from scanpipe import pipes from scanpipe.models import CodebaseResource +from scanpipe.models import DiscoveredDependency +from scanpipe.models import DiscoveredPackage from scanpipe.pipes import flag logger = logging.getLogger("scanpipe.pipes") @@ -368,7 +370,7 @@ def scan_for_files(project, resource_qs=None, progress_logger=None): def scan_for_application_packages( - project, assemble=True, package_only=False, progress_logger=None + project, assemble=True, package_only=False, resource_qs=None, progress_logger=None ): """ Run a package scan on resources without a status for a `project`, @@ -383,7 +385,8 @@ def scan_for_application_packages( Multiprocessing is enabled by default on this pipe, the number of processes can be controlled through the SCANCODEIO_PROCESSES setting. """ - resource_qs = project.codebaseresources.no_status() + if not resource_qs: + resource_qs = project.codebaseresources.no_status() scan_func_kwargs = { "package_only": package_only, @@ -473,7 +476,7 @@ def assemble_packages(project): logger.info(f"Unknown Package assembly item type: {item!r}") -def process_package_data(project): +def process_package_data(project, static_resolve=False): """ Create instances of DiscoveredPackage and DiscoveredDependency for `project` from the parsed package data present in the CodebaseResources of `project`. @@ -482,12 +485,8 @@ def process_package_data(project): package/dependency objects are created directly from package data. """ logger.info(f"Project {project} process_package_data:") - seen_resource_paths = set() for resource in project.codebaseresources.has_package_data(): - if resource.path in seen_resource_paths: - continue - logger.info(f" Processing: {resource.path}") for package_mapping in resource.package_data: pd = packagedcode_models.PackageData.from_dict(mapping=package_mapping) @@ -516,6 +515,113 @@ def process_package_data(project): datasource_id=pd.datasource_id, ) + if static_resolve: + resolve_dependencies(project) + + +def resolve_dependencies(project): + + logger.info(f"Project {project} resolve_dependencies:") + for resource in project.codebaseresources.has_package_data(): + for package_mapping in resource.package_data: + pd = packagedcode_models.PackageData.from_dict(package_mapping) + package = None + if pd.purl: + purl_data = DiscoveredPackage.extract_purl_data(package_mapping) + package = DiscoveredPackage.objects.get_or_none( + project=project, + **purl_data, + ) + + dependencies = package_mapping.get("dependencies") or [] + update_packages_and_dependencies( + project=project, + dependencies=dependencies, + package=package, + resource=resource, + datasource_id=pd.datasource_id, + ) + + match_and_resolve_dependencies(project) + + +def update_packages_and_dependencies( + project, + dependencies, + package, + resource, + datasource_id, +): + """ + Create DiscoveredPackage and DiscoveredDependency objects from + a package_data dependencies, and also from nested resolved packages + and dependencies if present. + """ + for dep in dependencies: + resolved_package = dep.get("resolved_package") or {} + resolved_to_package = None + if resolved_package: + resolved_to_package = pipes.update_or_create_package( + project=project, + package_data=resolved_package, + codebase_resources=[resource], + is_virtual=True, + ) + + deps_from_resolved = resolved_package.get("dependencies") or [] + for dep_from_resolved in deps_from_resolved: + pipes.update_or_create_dependency( + project=project, + dependency_data=dep_from_resolved, + for_package=resolved_to_package, + datafile_resource=resource, + datasource_id=datasource_id, + ) + + pipes.update_or_create_dependency( + project=project, + dependency_data=dep, + for_package=package, + resolved_to_package=resolved_to_package, + ) + + +def match_and_resolve_dependencies(project): + """ + From a project with both direct dependency relationships (contains + only the parent package and the requirement) and indirect dependency + relationships like in lockfiles (this contains the resolved package + and the requirement), match and update dependencies to contain the + full dependency graph. + """ + for dependency in project.discovereddependencies.all(): + if dependency.resolved_to_package: + continue + + purl_data = DiscoveredDependency.extract_purl_data( + dependency_data={"purl": dependency.purl}, + ignore_nulls=True, + ) + matched_dependencies = DiscoveredDependency.objects.filter( + project=project, + extracted_requirement=dependency.extracted_requirement, + **purl_data, + ) + + other_dependencies = [ + dep for dep in matched_dependencies if dep.purl != dependency.purl + ] + if other_dependencies: + resolved_dependency = other_dependencies.pop() + dependency.update( + resolved_to_package=resolved_dependency.resolved_to_package, + ) + + # We need only the direct dependency relationships but not the from indirect + # dependency realtionships which are between the main package to resolved packages + indirect_dependencies = project.discovereddependencies.filter(is_direct=False) + indirect_dependencies.delete() + def get_packages_with_purl_from_resources(project): """ diff --git a/scanpipe/tests/data/alpine_3_15_4_scan_codebase.json b/scanpipe/tests/data/alpine_3_15_4_scan_codebase.json index aad70c1ef1..7e20f271e5 100644 --- a/scanpipe/tests/data/alpine_3_15_4_scan_codebase.json +++ b/scanpipe/tests/data/alpine_3_15_4_scan_codebase.json @@ -207,6 +207,8 @@ ] }, "package_uid": "pkg:alpine/alpine-baselayout@3.2.0-r18?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "alpine_installed_db" ], @@ -444,6 +446,8 @@ ] }, "package_uid": "pkg:alpine/alpine-keys@2.4-r1?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "alpine_installed_db" ], @@ -526,6 +530,8 @@ ], "extra_data": {}, "package_uid": "pkg:alpine/apk-tools@2.12.7-r3?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "alpine_installed_db" ], @@ -619,6 +625,8 @@ ] }, "package_uid": "pkg:alpine/busybox@1.34.1-r5?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "alpine_installed_db" ], @@ -712,6 +720,8 @@ ] }, "package_uid": "pkg:alpine/ca-certificates-bundle@20211220-r0?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "alpine_installed_db" ], @@ -829,6 +839,8 @@ ] }, "package_uid": "pkg:alpine/libcrypto1.1@1.1.1n-r0?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "alpine_installed_db" ], @@ -911,6 +923,8 @@ ], "extra_data": {}, "package_uid": "pkg:alpine/libc-utils@0.7.2-r3?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "alpine_installed_db" ], @@ -1004,6 +1018,8 @@ ] }, "package_uid": "pkg:alpine/libretls@3.3.4-r3?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "alpine_installed_db" ], @@ -1097,6 +1113,8 @@ ] }, "package_uid": "pkg:alpine/libssl1.1@1.1.1n-r0?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "alpine_installed_db" ], @@ -1190,6 +1208,8 @@ ] }, "package_uid": "pkg:alpine/musl@1.2.2-r7?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "alpine_installed_db" ], @@ -1272,6 +1292,8 @@ ], "extra_data": {}, "package_uid": "pkg:alpine/musl-utils@1.2.2-r7?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "alpine_installed_db" ], @@ -1354,6 +1376,8 @@ ], "extra_data": {}, "package_uid": "pkg:alpine/scanelf@1.3.3-r0?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "alpine_installed_db" ], @@ -1436,6 +1460,8 @@ ], "extra_data": {}, "package_uid": "pkg:alpine/ssl_client@1.34.1-r5?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "alpine_installed_db" ], @@ -1529,6 +1555,8 @@ ] }, "package_uid": "pkg:alpine/zlib@1.2.12-r0?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "alpine_installed_db" ], diff --git a/scanpipe/tests/data/asgiref-3.3.0.spdx.json b/scanpipe/tests/data/asgiref-3.3.0.spdx.json index cefb4c9cb1..0f3707137d 100644 --- a/scanpipe/tests/data/asgiref-3.3.0.spdx.json +++ b/scanpipe/tests/data/asgiref-3.3.0.spdx.json @@ -3,7 +3,7 @@ "dataLicense": "CC0-1.0", "SPDXID": "SPDXRef-DOCUMENT", "name": "scancodeio_asgiref", - "documentNamespace": "https://scancode.io/spdxdocs/8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e", + "documentNamespace": "https://scancode.io/spdxdocs/83d59103-a103-4c9a-a433-618eff973b63", "creationInfo": { "created": "2000-01-01T01:02:03Z", "creators": [ @@ -14,7 +14,7 @@ "packages": [ { "name": "asgiref", - "SPDXID": "SPDXRef-scancodeio-discoveredpackage-b6ef7c90-e3d4-4008-8b67-63f086cea2da", + "SPDXID": "SPDXRef-scancodeio-discoveredpackage-a9dcd442-4c55-49b4-9e33-8be74c9fc15b", "downloadLocation": "NOASSERTION", "licenseConcluded": "BSD-3-Clause", "copyrightText": "NOASSERTION", @@ -33,7 +33,7 @@ }, { "name": "asgiref", - "SPDXID": "SPDXRef-scancodeio-discoveredpackage-b2d24c22-0dff-4e3f-8332-413b4f4852a7", + "SPDXID": "SPDXRef-scancodeio-discoveredpackage-27357e59-c16d-40f1-8cd2-0f11b7376cdb", "downloadLocation": "NOASSERTION", "licenseConcluded": "BSD-3-Clause", "copyrightText": "NOASSERTION", @@ -52,7 +52,7 @@ }, { "name": "pytest", - "SPDXID": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest?uuid=33c92b36-8293-4fe5-8094-0eb645833284", + "SPDXID": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest?uuid=9f487b4c-4d3e-4d13-9100-89ab897c90ee", "downloadLocation": "NOASSERTION", "licenseConcluded": "NOASSERTION", "copyrightText": "NOASSERTION", @@ -68,7 +68,7 @@ }, { "name": "pytest", - "SPDXID": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest?uuid=21776640-36c9-4111-a6c4-ea5a550c85e0", + "SPDXID": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest?uuid=909f72ab-1d6b-41f6-b41a-2b3be388870a", "downloadLocation": "NOASSERTION", "licenseConcluded": "NOASSERTION", "copyrightText": "NOASSERTION", @@ -84,7 +84,7 @@ }, { "name": "pytest-asyncio", - "SPDXID": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest-asyncio?uuid=bf9520e1-0b6a-4c00-90c4-b5e7afb6de3e", + "SPDXID": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest-asyncio?uuid=7c26cfc3-080d-4d3c-bcfe-fd6a495c5d3a", "downloadLocation": "NOASSERTION", "licenseConcluded": "NOASSERTION", "copyrightText": "NOASSERTION", @@ -100,7 +100,7 @@ }, { "name": "pytest-asyncio", - "SPDXID": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest-asyncio?uuid=1963fa7d-3e64-4975-b4e7-80492466fefb", + "SPDXID": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest-asyncio?uuid=e474c205-ceef-444a-a0d7-aee32c8716df", "downloadLocation": "NOASSERTION", "licenseConcluded": "NOASSERTION", "copyrightText": "NOASSERTION", @@ -116,33 +116,33 @@ } ], "documentDescribes": [ - "SPDXRef-scancodeio-discoveredpackage-b6ef7c90-e3d4-4008-8b67-63f086cea2da", - "SPDXRef-scancodeio-discoveredpackage-b2d24c22-0dff-4e3f-8332-413b4f4852a7", - "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest?uuid=33c92b36-8293-4fe5-8094-0eb645833284", - "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest?uuid=21776640-36c9-4111-a6c4-ea5a550c85e0", - "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest-asyncio?uuid=bf9520e1-0b6a-4c00-90c4-b5e7afb6de3e", - "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest-asyncio?uuid=1963fa7d-3e64-4975-b4e7-80492466fefb" + "SPDXRef-scancodeio-discoveredpackage-a9dcd442-4c55-49b4-9e33-8be74c9fc15b", + "SPDXRef-scancodeio-discoveredpackage-27357e59-c16d-40f1-8cd2-0f11b7376cdb", + "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest?uuid=9f487b4c-4d3e-4d13-9100-89ab897c90ee", + "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest?uuid=909f72ab-1d6b-41f6-b41a-2b3be388870a", + "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest-asyncio?uuid=7c26cfc3-080d-4d3c-bcfe-fd6a495c5d3a", + "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest-asyncio?uuid=e474c205-ceef-444a-a0d7-aee32c8716df" ], "files": [], "relationships": [ { - "spdxElementId": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest?uuid=33c92b36-8293-4fe5-8094-0eb645833284", - "relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-b6ef7c90-e3d4-4008-8b67-63f086cea2da", + "spdxElementId": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest?uuid=9f487b4c-4d3e-4d13-9100-89ab897c90ee", + "relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-a9dcd442-4c55-49b4-9e33-8be74c9fc15b", "relationshipType": "DEPENDENCY_OF" }, { - "spdxElementId": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest?uuid=21776640-36c9-4111-a6c4-ea5a550c85e0", - "relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-b2d24c22-0dff-4e3f-8332-413b4f4852a7", + "spdxElementId": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest?uuid=909f72ab-1d6b-41f6-b41a-2b3be388870a", + "relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-27357e59-c16d-40f1-8cd2-0f11b7376cdb", "relationshipType": "DEPENDENCY_OF" }, { - "spdxElementId": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest-asyncio?uuid=bf9520e1-0b6a-4c00-90c4-b5e7afb6de3e", - "relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-b6ef7c90-e3d4-4008-8b67-63f086cea2da", + "spdxElementId": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest-asyncio?uuid=7c26cfc3-080d-4d3c-bcfe-fd6a495c5d3a", + "relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-a9dcd442-4c55-49b4-9e33-8be74c9fc15b", "relationshipType": "DEPENDENCY_OF" }, { - "spdxElementId": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest-asyncio?uuid=1963fa7d-3e64-4975-b4e7-80492466fefb", - "relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-b2d24c22-0dff-4e3f-8332-413b4f4852a7", + "spdxElementId": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest-asyncio?uuid=e474c205-ceef-444a-a0d7-aee32c8716df", + "relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-27357e59-c16d-40f1-8cd2-0f11b7376cdb", "relationshipType": "DEPENDENCY_OF" } ], diff --git a/scanpipe/tests/data/asgiref-3.3.0_fixtures.json b/scanpipe/tests/data/asgiref-3.3.0_fixtures.json index bdf26ca322..0cce145f35 100644 --- a/scanpipe/tests/data/asgiref-3.3.0_fixtures.json +++ b/scanpipe/tests/data/asgiref-3.3.0_fixtures.json @@ -1,13 +1,13 @@ [ { "model": "scanpipe.project", - "pk": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e", + "pk": "83d59103-a103-4c9a-a433-618eff973b63", "fields": { "extra_data": {}, - "created_date": "2024-03-01T11:33:36.591Z", + "created_date": "2024-06-13T14:23:21.548Z", "name": "asgiref", - "slug": "asgiref-8d3058f3", - "work_directory": "/tmp/tmp3x6obwss/projects/asgiref-8d3058f3", + "slug": "asgiref-83d59103", + "work_directory": "/tmp/tmp5ihgt8lg/projects/asgiref-83d59103", "is_archived": false, "notes": "", "settings": {} @@ -15,17 +15,17 @@ }, { "model": "scanpipe.run", - "pk": "dfe51f4c-9867-462b-977c-1897531bb35f", + "pk": "ceb011c2-6d52-4c3a-bbff-c53e45f0313e", "fields": { "task_id": null, "task_start_date": null, "task_end_date": null, "task_exitcode": null, "task_output": "", - "log": "2024-03-01 11:33:36.59 Pipeline [scan_codebase] starting\n2024-03-01 11:33:36.59 Step [download_missing_inputs] starting\n2024-03-01 11:33:36.59 Step [download_missing_inputs] completed in 0 seconds\n2024-03-01 11:33:36.59 Step [copy_inputs_to_codebase_directory] starting\n2024-03-01 11:33:36.59 Step [copy_inputs_to_codebase_directory] completed in 0 seconds\n2024-03-01 11:33:36.59 Step [extract_archives] starting\n2024-03-01 11:33:36.66 Step [extract_archives] completed in 0 seconds\n2024-03-01 11:33:36.66 Step [collect_and_create_codebase_resources] starting\n2024-03-01 11:33:36.85 Step [collect_and_create_codebase_resources] completed in 0 seconds\n2024-03-01 11:33:36.85 Step [flag_empty_files] starting\n2024-03-01 11:33:36.85 Step [flag_empty_files] completed in 0 seconds\n2024-03-01 11:33:36.85 Step [flag_ignored_resources] starting\n2024-03-01 11:33:36.85 Step [flag_ignored_resources] completed in 0 seconds\n2024-03-01 11:33:36.85 Step [scan_for_application_packages] starting\n2024-03-01 11:33:36.89 Progress: 11% (2/18)\n2024-03-01 11:33:36.92 Progress: 22% (4/18)\n2024-03-01 11:33:36.93 Progress: 33% (6/18)\n2024-03-01 11:33:36.93 Progress: 44% (8/18)\n2024-03-01 11:33:36.93 Progress: 55% (10/18)\n2024-03-01 11:33:36.93 Progress: 66% (12/18)\n2024-03-01 11:33:36.93 Progress: 77% (14/18)\n2024-03-01 11:33:36.93 Progress: 88% (16/18)\n2024-03-01 11:33:39.89 Progress: 100% (18/18)\n2024-03-01 11:33:40.08 Step [scan_for_application_packages] completed in 3 seconds\n2024-03-01 11:33:40.08 Step [scan_for_files] starting\n2024-03-01 11:33:50.84 Progress: 12% (2/16) ETA: 79 seconds (1.3 minutes)\n2024-03-01 11:33:51.20 Progress: 25% (4/16) ETA: 33 seconds\n2024-03-01 11:33:51.91 Progress: 37% (6/16) ETA: 20 seconds\n2024-03-01 11:33:52.72 Progress: 50% (8/16) ETA: 13 seconds\n2024-03-01 11:33:52.97 Progress: 62% (10/16) ETA: 8 seconds\n2024-03-01 11:33:53.15 Progress: 75% (12/16) ETA: 4 seconds\n2024-03-01 11:33:53.16 Progress: 87% (14/16) ETA: 2 seconds\n2024-03-01 11:33:53.56 Progress: 100% (16/16)\n2024-03-01 11:33:53.85 Step [scan_for_files] completed in 14 seconds\n2024-03-01 11:33:53.85 Pipeline completed in 17 seconds\n", - "project": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e", + "log": "2024-06-13 14:23:21.55 Pipeline [scan_codebase] starting\n2024-06-13 14:23:21.55 Step [download_missing_inputs] starting\n2024-06-13 14:23:21.55 Step [download_missing_inputs] completed in 0 seconds\n2024-06-13 14:23:21.55 Step [copy_inputs_to_codebase_directory] starting\n2024-06-13 14:23:21.55 Step [copy_inputs_to_codebase_directory] completed in 0 seconds\n2024-06-13 14:23:21.55 Step [extract_archives] starting\n2024-06-13 14:23:21.60 Step [extract_archives] completed in 0 seconds\n2024-06-13 14:23:21.60 Step [collect_and_create_codebase_resources] starting\n2024-06-13 14:23:21.67 Step [collect_and_create_codebase_resources] completed in 0 seconds\n2024-06-13 14:23:21.67 Step [flag_empty_files] starting\n2024-06-13 14:23:21.67 Step [flag_empty_files] completed in 0 seconds\n2024-06-13 14:23:21.67 Step [flag_ignored_resources] starting\n2024-06-13 14:23:21.67 Step [flag_ignored_resources] completed in 0 seconds\n2024-06-13 14:23:21.68 Step [scan_for_application_packages] starting\n2024-06-13 14:23:21.69 Progress: 11% (2/18)\n2024-06-13 14:23:21.70 Progress: 22% (4/18)\n2024-06-13 14:23:21.71 Progress: 33% (6/18)\n2024-06-13 14:23:21.71 Progress: 44% (8/18)\n2024-06-13 14:23:21.71 Progress: 55% (10/18)\n2024-06-13 14:23:21.71 Progress: 66% (12/18)\n2024-06-13 14:23:21.71 Progress: 77% (14/18)\n2024-06-13 14:23:21.71 Progress: 88% (16/18)\n2024-06-13 14:23:24.08 Progress: 100% (18/18)\n2024-06-13 14:23:26.20 Step [scan_for_application_packages] completed in 5 seconds\n2024-06-13 14:23:26.20 Step [scan_for_files] starting\n2024-06-13 14:23:26.32 Progress: 12% (2/16) ETA: 1 seconds\n2024-06-13 14:23:26.33 Progress: 25% (4/16)\n2024-06-13 14:23:26.35 Progress: 37% (6/16)\n2024-06-13 14:23:26.56 Progress: 50% (8/16)\n2024-06-13 14:23:26.60 Progress: 62% (10/16)\n2024-06-13 14:23:26.63 Progress: 75% (12/16)\n2024-06-13 14:23:26.79 Progress: 87% (14/16)\n2024-06-13 14:23:26.88 Progress: 100% (16/16)\n2024-06-13 14:23:26.90 Step [scan_for_files] completed in 1 seconds\n2024-06-13 14:23:26.90 Pipeline completed in 5 seconds\n", + "project": "83d59103-a103-4c9a-a433-618eff973b63", "pipeline_name": "scan_codebase", - "created_date": "2024-03-01T11:33:36.594Z", + "created_date": "2024-06-13T14:23:21.551Z", "scancodeio_version": "", "description": "Scan a codebase for application packages, licenses, and copyrights.", "current_step": "", @@ -41,7 +41,7 @@ "sha256": "a5098bc870b80e7b872bff60bb363c7f2c2c89078759f6c47b53ff8c525a152e", "sha512": "", "extra_data": {}, - "project": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e", + "project": "83d59103-a103-4c9a-a433-618eff973b63", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -113,6 +113,8 @@ "Documentation": "https://asgi.readthedocs.io/", "Further Documentation": "https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions" }, + "is_private": false, + "is_virtual": false, "qualifiers": {}, "description": "ASGI specs, helper code, and adapters\nasgiref\n=======\n\n.. image:: https://api.travis-ci.org/django/asgiref.svg\n :target: https://travis-ci.org/django/asgiref\n\n.. image:: https://img.shields.io/pypi/v/asgiref.svg\n :target: https://pypi.python.org/pypi/asgiref\n\nASGI is a standard for Python asynchronous web apps and servers to communicate\nwith each other, and positioned as an asynchronous successor to WSGI. You can\nread more at https://asgi.readthedocs.io/en/latest/\n\nThis package includes ASGI base libraries, such as:\n\n* Sync-to-async and async-to-sync function wrappers, ``asgiref.sync``\n* Server base classes, ``asgiref.server``\n* A WSGI-to-ASGI adapter, in ``asgiref.wsgi``\n\n\nFunction wrappers\n-----------------\n\nThese allow you to wrap or decorate async or sync functions to call them from\nthe other style (so you can call async functions from a synchronous thread,\nor vice-versa).\n\nIn particular:\n\n* AsyncToSync lets a synchronous subthread stop and wait while the async\n function is called on the main thread's event loop, and then control is\n returned to the thread when the async function is finished.\n\n* SyncToAsync lets async code call a synchronous function, which is run in\n a threadpool and control returned to the async coroutine when the synchronous\n function completes.\n\nThe idea is to make it easier to call synchronous APIs from async code and\nasynchronous APIs from synchronous code so it's easier to transition code from\none style to the other. In the case of Channels, we wrap the (synchronous)\nDjango view system with SyncToAsync to allow it to run inside the (asynchronous)\nASGI server.\n\nNote that exactly what threads things run in is very specific, and aimed to\nkeep maximum compatibility with old synchronous code. See\n\"Synchronous code & Threads\" below for a full explanation. By default,\n``sync_to_async`` will run all synchronous code in the program in the same\nthread for safety reasons; you can disable this for more performance with\n``@sync_to_async(thread_sensitive=False)``, but make sure that your code does\nnot rely on anything bound to threads (like database connections) when you do.\n\n\nThreadlocal replacement\n-----------------------\n\nThis is a drop-in replacement for ``threading.local`` that works with both\nthreads and asyncio Tasks. Even better, it will proxy values through from a\ntask-local context to a thread-local context when you use ``sync_to_async``\nto run things in a threadpool, and vice-versa for ``async_to_sync``.\n\nIf you instead want true thread- and task-safety, you can set\n``thread_critical`` on the Local object to ensure this instead.\n\n\nServer base classes\n-------------------\n\nIncludes a ``StatelessServer`` class which provides all the hard work of\nwriting a stateless server (as in, does not handle direct incoming sockets\nbut instead consumes external streams or sockets to work out what is happening).\n\nAn example of such a server would be a chatbot server that connects out to\na central chat server and provides a \"connection scope\" per user chatting to\nit. There's only one actual connection, but the server has to separate things\ninto several scopes for easier writing of the code.\n\nYou can see an example of this being used in `frequensgi `_.\n\n\nWSGI-to-ASGI adapter\n--------------------\n\nAllows you to wrap a WSGI application so it appears as a valid ASGI application.\n\nSimply wrap it around your WSGI application like so::\n\n asgi_application = WsgiToAsgi(wsgi_application)\n\nThe WSGI application will be run in a synchronous threadpool, and the wrapped\nASGI application will be one that accepts ``http`` class messages.\n\nPlease note that not all extended features of WSGI may be supported (such as\nfile handles for incoming POST bodies).\n\n\nDependencies\n------------\n\n``asgiref`` requires Python 3.5 or higher.\n\n\nContributing\n------------\n\nPlease refer to the\n`main Channels contributing docs `_.\n\n\nTesting\n'''''''\n\nTo run tests, make sure you have installed the ``tests`` extra with the package::\n\n cd asgiref/\n pip install -e .[tests]\n pytest\n\n\nBuilding the documentation\n''''''''''''''''''''''''''\n\nThe documentation uses `Sphinx `_::\n\n cd asgiref/docs/\n pip install sphinx\n\nTo build the docs, you can use the default tools::\n\n sphinx-build -b html . _build/html # or `make html`, if you've got make set up\n cd _build/html\n python -m http.server\n\n...or you can use ``sphinx-autobuild`` to run a server and rebuild/reload\nyour documentation changes automatically::\n\n pip install sphinx-autobuild\n sphinx-autobuild . _build/html\n\n\nImplementation Details\n----------------------\n\nSynchronous code & threads\n''''''''''''''''''''''''''\n\nThe ``asgiref.sync`` module provides two wrappers that let you go between\nasynchronous and synchronous code at will, while taking care of the rough edges\nfor you.\n\nUnfortunately, the rough edges are numerous, and the code has to work especially\nhard to keep things in the same thread as much as possible. Notably, the\nrestrictions we are working with are:\n\n* All synchronous code called through ``SyncToAsync`` and marked with\n ``thread_sensitive`` should run in the same thread as each other (and if the\n outer layer of the program is synchronous, the main thread)\n\n* If a thread already has a running async loop, ``AsyncToSync`` can't run things\n on that loop if it's blocked on synchronous code that is above you in the\n call stack.\n\nThe first compromise you get to might be that ``thread_sensitive`` code should\njust run in the same thread and not spawn in a sub-thread, fulfilling the first\nrestriction, but that immediately runs you into the second restriction.\n\nThe only real solution is to essentially have a variant of ThreadPoolExecutor\nthat executes any ``thread_sensitive`` code on the outermost synchronous\nthread - either the main thread, or a single spawned subthread.\n\nThis means you now have two basic states:\n\n* If the outermost layer of your program is synchronous, then all async code\n run through ``AsyncToSync`` will run in a per-call event loop in arbitary\n sub-threads, while all ``thread_sensitive`` code will run in the main thread.\n\n* If the outermost layer of your program is asynchronous, then all async code\n runs on the main thread's event loop, and all ``thread_sensitive`` synchronous\n code will run in a single shared sub-thread.\n\nCruicially, this means that in both cases there is a thread which is a shared\nresource that all ``thread_sensitive`` code must run on, and there is a chance\nthat this thread is currently blocked on its own ``AsyncToSync`` call. Thus,\n``AsyncToSync`` needs to act as an executor for thread code while it's blocking.\n\nThe ``CurrentThreadExecutor`` class provides this functionality; rather than\nsimply waiting on a Future, you can call its ``run_until_future`` method and\nit will run submitted code until that Future is done. This means that code\ninside the call can then run code on your thread.\n\n\nMaintenance and Security\n------------------------\n\nTo report security issues, please contact security@djangoproject.com. For GPG\nsignatures and more security process information, see\nhttps://docs.djangoproject.com/en/dev/internals/security/.\n\nTo report bugs or request new features, please open a new GitHub issue.\n\nThis repository is part of the Channels project. For the shepherd and maintenance team, please see the\n`main Channels readme `_.", "notice_text": null, @@ -121,6 +123,7 @@ { "purl": "pkg:pypi/pytest", "scope": "tests", + "is_direct": true, "extra_data": {}, "is_runtime": true, "is_optional": true, @@ -131,6 +134,7 @@ { "purl": "pkg:pypi/pytest-asyncio", "scope": "tests", + "is_direct": true, "extra_data": {}, "is_runtime": true, "is_optional": true, @@ -143,7 +147,7 @@ "homepage_url": "https://github.com/django/asgiref/", "release_date": null, "code_view_url": null, - "datasource_ids": "pypi_wheel", + "datasource_id": "pypi_wheel", "file_references": [ { "md5": null, @@ -342,7 +346,7 @@ "sha256": "", "sha512": "", "extra_data": {}, - "project": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e", + "project": "83d59103-a103-4c9a-a433-618eff973b63", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -382,7 +386,7 @@ "sha256": "", "sha512": "", "extra_data": {}, - "project": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e", + "project": "83d59103-a103-4c9a-a433-618eff973b63", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -422,7 +426,7 @@ "sha256": "", "sha512": "", "extra_data": {}, - "project": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e", + "project": "83d59103-a103-4c9a-a433-618eff973b63", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -462,7 +466,7 @@ "sha256": "6e89108c2cf0c0446174188f76f60465ae1c1f14f83427807df40d52a27cb2c8", "sha512": "", "extra_data": {}, - "project": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e", + "project": "83d59103-a103-4c9a-a433-618eff973b63", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -502,7 +506,7 @@ "sha256": "b846415d1b514e9c1dff14a22deb906d794bc546ca6129f950a18cd091e2a669", "sha512": "", "extra_data": {}, - "project": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e", + "project": "83d59103-a103-4c9a-a433-618eff973b63", "detected_license_expression": "bsd-new", "detected_license_expression_spdx": "BSD-3-Clause", "license_detections": [ @@ -548,7 +552,7 @@ "authors": [], "emails": [], "urls": [], - "compliance_alert": "", + "compliance_alert": "missing", "path": "asgiref-3.3.0-py3-none-any.whl-extract/asgiref-3.3.0.dist-info/LICENSE", "rootfs_path": "", "status": "scanned", @@ -577,7 +581,7 @@ "sha256": "70f98f4eb9f6068b192b5464fcdf69e29a8ff09962bfce84bbb052baeee44f33", "sha512": "", "extra_data": {}, - "project": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e", + "project": "83d59103-a103-4c9a-a433-618eff973b63", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -649,6 +653,8 @@ "Documentation": "https://asgi.readthedocs.io/", "Further Documentation": "https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions" }, + "is_private": false, + "is_virtual": false, "qualifiers": {}, "description": "ASGI specs, helper code, and adapters\nasgiref\n=======\n\n.. image:: https://api.travis-ci.org/django/asgiref.svg\n :target: https://travis-ci.org/django/asgiref\n\n.. image:: https://img.shields.io/pypi/v/asgiref.svg\n :target: https://pypi.python.org/pypi/asgiref\n\nASGI is a standard for Python asynchronous web apps and servers to communicate\nwith each other, and positioned as an asynchronous successor to WSGI. You can\nread more at https://asgi.readthedocs.io/en/latest/\n\nThis package includes ASGI base libraries, such as:\n\n* Sync-to-async and async-to-sync function wrappers, ``asgiref.sync``\n* Server base classes, ``asgiref.server``\n* A WSGI-to-ASGI adapter, in ``asgiref.wsgi``\n\n\nFunction wrappers\n-----------------\n\nThese allow you to wrap or decorate async or sync functions to call them from\nthe other style (so you can call async functions from a synchronous thread,\nor vice-versa).\n\nIn particular:\n\n* AsyncToSync lets a synchronous subthread stop and wait while the async\n function is called on the main thread's event loop, and then control is\n returned to the thread when the async function is finished.\n\n* SyncToAsync lets async code call a synchronous function, which is run in\n a threadpool and control returned to the async coroutine when the synchronous\n function completes.\n\nThe idea is to make it easier to call synchronous APIs from async code and\nasynchronous APIs from synchronous code so it's easier to transition code from\none style to the other. In the case of Channels, we wrap the (synchronous)\nDjango view system with SyncToAsync to allow it to run inside the (asynchronous)\nASGI server.\n\nNote that exactly what threads things run in is very specific, and aimed to\nkeep maximum compatibility with old synchronous code. See\n\"Synchronous code & Threads\" below for a full explanation. By default,\n``sync_to_async`` will run all synchronous code in the program in the same\nthread for safety reasons; you can disable this for more performance with\n``@sync_to_async(thread_sensitive=False)``, but make sure that your code does\nnot rely on anything bound to threads (like database connections) when you do.\n\n\nThreadlocal replacement\n-----------------------\n\nThis is a drop-in replacement for ``threading.local`` that works with both\nthreads and asyncio Tasks. Even better, it will proxy values through from a\ntask-local context to a thread-local context when you use ``sync_to_async``\nto run things in a threadpool, and vice-versa for ``async_to_sync``.\n\nIf you instead want true thread- and task-safety, you can set\n``thread_critical`` on the Local object to ensure this instead.\n\n\nServer base classes\n-------------------\n\nIncludes a ``StatelessServer`` class which provides all the hard work of\nwriting a stateless server (as in, does not handle direct incoming sockets\nbut instead consumes external streams or sockets to work out what is happening).\n\nAn example of such a server would be a chatbot server that connects out to\na central chat server and provides a \"connection scope\" per user chatting to\nit. There's only one actual connection, but the server has to separate things\ninto several scopes for easier writing of the code.\n\nYou can see an example of this being used in `frequensgi `_.\n\n\nWSGI-to-ASGI adapter\n--------------------\n\nAllows you to wrap a WSGI application so it appears as a valid ASGI application.\n\nSimply wrap it around your WSGI application like so::\n\n asgi_application = WsgiToAsgi(wsgi_application)\n\nThe WSGI application will be run in a synchronous threadpool, and the wrapped\nASGI application will be one that accepts ``http`` class messages.\n\nPlease note that not all extended features of WSGI may be supported (such as\nfile handles for incoming POST bodies).\n\n\nDependencies\n------------\n\n``asgiref`` requires Python 3.5 or higher.\n\n\nContributing\n------------\n\nPlease refer to the\n`main Channels contributing docs `_.\n\n\nTesting\n'''''''\n\nTo run tests, make sure you have installed the ``tests`` extra with the package::\n\n cd asgiref/\n pip install -e .[tests]\n pytest\n\n\nBuilding the documentation\n''''''''''''''''''''''''''\n\nThe documentation uses `Sphinx `_::\n\n cd asgiref/docs/\n pip install sphinx\n\nTo build the docs, you can use the default tools::\n\n sphinx-build -b html . _build/html # or `make html`, if you've got make set up\n cd _build/html\n python -m http.server\n\n...or you can use ``sphinx-autobuild`` to run a server and rebuild/reload\nyour documentation changes automatically::\n\n pip install sphinx-autobuild\n sphinx-autobuild . _build/html\n\n\nImplementation Details\n----------------------\n\nSynchronous code & threads\n''''''''''''''''''''''''''\n\nThe ``asgiref.sync`` module provides two wrappers that let you go between\nasynchronous and synchronous code at will, while taking care of the rough edges\nfor you.\n\nUnfortunately, the rough edges are numerous, and the code has to work especially\nhard to keep things in the same thread as much as possible. Notably, the\nrestrictions we are working with are:\n\n* All synchronous code called through ``SyncToAsync`` and marked with\n ``thread_sensitive`` should run in the same thread as each other (and if the\n outer layer of the program is synchronous, the main thread)\n\n* If a thread already has a running async loop, ``AsyncToSync`` can't run things\n on that loop if it's blocked on synchronous code that is above you in the\n call stack.\n\nThe first compromise you get to might be that ``thread_sensitive`` code should\njust run in the same thread and not spawn in a sub-thread, fulfilling the first\nrestriction, but that immediately runs you into the second restriction.\n\nThe only real solution is to essentially have a variant of ThreadPoolExecutor\nthat executes any ``thread_sensitive`` code on the outermost synchronous\nthread - either the main thread, or a single spawned subthread.\n\nThis means you now have two basic states:\n\n* If the outermost layer of your program is synchronous, then all async code\n run through ``AsyncToSync`` will run in a per-call event loop in arbitary\n sub-threads, while all ``thread_sensitive`` code will run in the main thread.\n\n* If the outermost layer of your program is asynchronous, then all async code\n runs on the main thread's event loop, and all ``thread_sensitive`` synchronous\n code will run in a single shared sub-thread.\n\nCruicially, this means that in both cases there is a thread which is a shared\nresource that all ``thread_sensitive`` code must run on, and there is a chance\nthat this thread is currently blocked on its own ``AsyncToSync`` call. Thus,\n``AsyncToSync`` needs to act as an executor for thread code while it's blocking.\n\nThe ``CurrentThreadExecutor`` class provides this functionality; rather than\nsimply waiting on a Future, you can call its ``run_until_future`` method and\nit will run submitted code until that Future is done. This means that code\ninside the call can then run code on your thread.\n\n\nMaintenance and Security\n------------------------\n\nTo report security issues, please contact security@djangoproject.com. For GPG\nsignatures and more security process information, see\nhttps://docs.djangoproject.com/en/dev/internals/security/.\n\nTo report bugs or request new features, please open a new GitHub issue.\n\nThis repository is part of the Channels project. For the shepherd and maintenance team, please see the\n`main Channels readme `_.", "notice_text": null, @@ -657,6 +663,7 @@ { "purl": "pkg:pypi/pytest", "scope": "tests", + "is_direct": true, "extra_data": {}, "is_runtime": true, "is_optional": true, @@ -667,6 +674,7 @@ { "purl": "pkg:pypi/pytest-asyncio", "scope": "tests", + "is_direct": true, "extra_data": {}, "is_runtime": true, "is_optional": true, @@ -679,7 +687,7 @@ "homepage_url": "https://github.com/django/asgiref/", "release_date": null, "code_view_url": null, - "datasource_ids": "pypi_wheel_metadata", + "datasource_id": "pypi_wheel_metadata", "file_references": [ { "md5": null, @@ -878,7 +886,7 @@ "sha256": "11546323af45e6a5639bf620a9c4d73e74c0bf705f494af4595007b923f75e8a", "sha512": "", "extra_data": {}, - "project": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e", + "project": "83d59103-a103-4c9a-a433-618eff973b63", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -918,7 +926,7 @@ "sha256": "2c1983592aa38f0bfb0afacc73ddc5b46ce10e8e89ceaa9fed1e5fc6361b608d", "sha512": "", "extra_data": {}, - "project": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e", + "project": "83d59103-a103-4c9a-a433-618eff973b63", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -958,7 +966,7 @@ "sha256": "30f49b9094bff904a42caeec32515715fe625a56dc48bd7c0e3d9988c0ad4bd7", "sha512": "", "extra_data": {}, - "project": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e", + "project": "83d59103-a103-4c9a-a433-618eff973b63", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -998,7 +1006,7 @@ "sha256": "fa4651a3b79201a4dc44a4096cd49ec8f427e912ea0ee05c666357b413a8afe7", "sha512": "", "extra_data": {}, - "project": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e", + "project": "83d59103-a103-4c9a-a433-618eff973b63", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -1038,7 +1046,7 @@ "sha256": "ee0fcf4a8e6fa9df8a4643bb48e82892d496afce44b6c8b8aea2721755545e1c", "sha512": "", "extra_data": {}, - "project": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e", + "project": "83d59103-a103-4c9a-a433-618eff973b63", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -1078,7 +1086,7 @@ "sha256": "3151f66c476208c3154cb6c4fb557a2a253bab82f0ab33fb3c8b9f7976be9e33", "sha512": "", "extra_data": {}, - "project": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e", + "project": "83d59103-a103-4c9a-a433-618eff973b63", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -1118,7 +1126,7 @@ "sha256": "ddd445b778c097fc75c2bf69ad964cbadd3bd6999d1dd2306d39d401855e8e3e", "sha512": "", "extra_data": {}, - "project": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e", + "project": "83d59103-a103-4c9a-a433-618eff973b63", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -1158,7 +1166,7 @@ "sha256": "ddbc8d455eceb68fc583c67e7c4ad0277c867fb39095c51ec5b37f70342e8334", "sha512": "", "extra_data": {}, - "project": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e", + "project": "83d59103-a103-4c9a-a433-618eff973b63", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -1198,7 +1206,7 @@ "sha256": "126c3e3a8a75a517d2739612304607804cf5f34da63fa25d03a6f11f7edb6f2f", "sha512": "", "extra_data": {}, - "project": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e", + "project": "83d59103-a103-4c9a-a433-618eff973b63", "detected_license_expression": "apache-2.0", "detected_license_expression_spdx": "Apache-2.0", "license_detections": [ @@ -1248,7 +1256,7 @@ "start_line": 83 } ], - "compliance_alert": "", + "compliance_alert": "missing", "path": "asgiref-3.3.0-py3-none-any.whl-extract/asgiref/timeout.py", "rootfs_path": "", "status": "scanned", @@ -1277,7 +1285,7 @@ "sha256": "f8bd1ea3fb8afddabb10f8efd66796d41446cad51168ef4d3c44b19c973d0ad0", "sha512": "", "extra_data": {}, - "project": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e", + "project": "83d59103-a103-4c9a-a433-618eff973b63", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -1317,7 +1325,7 @@ "sha256": "885267fee0fea687875a02ceb929ca095312d47aaa57e20e4ce382f397caaf4d", "sha512": "", "extra_data": {}, - "project": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e", + "project": "83d59103-a103-4c9a-a433-618eff973b63", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -1367,8 +1375,8 @@ "Documentation": "https://asgi.readthedocs.io/", "Further Documentation": "https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions" }, - "project": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e", - "compliance_alert": "", + "project": "83d59103-a103-4c9a-a433-618eff973b63", + "compliance_alert": "missing", "affected_by_vulnerabilities": [], "filename": "", "primary_language": "Python", @@ -1438,7 +1446,14 @@ "other_license_detections": [], "extracted_license_statement": "license: BSD\nclassifiers:\n - 'License :: OSI Approved :: BSD License'\n", "notice_text": "", - "datasource_ids": "", + "is_private": false, + "is_virtual": false, + "datasource_ids": [ + "pypi_wheel" + ], + "datafile_paths": [ + "asgiref-3.3.0-py3-none-any.whl" + ], "file_references": [], "parties": [ { @@ -1449,10 +1464,10 @@ "email": "foundation@djangoproject.com" } ], - "uuid": "b6ef7c90-e3d4-4008-8b67-63f086cea2da", + "uuid": "a9dcd442-4c55-49b4-9e33-8be74c9fc15b", "missing_resources": [], "modified_resources": [], - "package_uid": "pkg:pypi/asgiref@3.3.0?uuid=90cd6d93-1c46-477a-bd4a-47aa6b3ce773", + "package_uid": "pkg:pypi/asgiref@3.3.0?uuid=eca680a3-115b-494b-8581-81c93384e22d", "keywords": [ "Development Status :: 5 - Production/Stable", "Environment :: Web Environment", @@ -1493,8 +1508,8 @@ "Documentation": "https://asgi.readthedocs.io/", "Further Documentation": "https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions" }, - "project": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e", - "compliance_alert": "", + "project": "83d59103-a103-4c9a-a433-618eff973b63", + "compliance_alert": "missing", "affected_by_vulnerabilities": [], "filename": "", "primary_language": "Python", @@ -1564,7 +1579,14 @@ "other_license_detections": [], "extracted_license_statement": "license: BSD\nclassifiers:\n - 'License :: OSI Approved :: BSD License'\n", "notice_text": "", - "datasource_ids": "", + "is_private": false, + "is_virtual": false, + "datasource_ids": [ + "pypi_wheel_metadata" + ], + "datafile_paths": [ + "asgiref-3.3.0-py3-none-any.whl-extract/asgiref-3.3.0.dist-info/METADATA" + ], "file_references": [], "parties": [ { @@ -1575,10 +1597,10 @@ "email": "foundation@djangoproject.com" } ], - "uuid": "b2d24c22-0dff-4e3f-8332-413b4f4852a7", + "uuid": "27357e59-c16d-40f1-8cd2-0f11b7376cdb", "missing_resources": [], "modified_resources": [], - "package_uid": "pkg:pypi/asgiref@3.3.0?uuid=fb05a7ee-1078-4d60-bf71-2ba031414f1c", + "package_uid": "pkg:pypi/asgiref@3.3.0?uuid=510ea8b0-d7a7-4531-96ea-f41cb4384976", "keywords": [ "Development Status :: 5 - Production/Stable", "Environment :: Web Environment", @@ -1624,16 +1646,18 @@ "qualifiers": "", "subpath": "", "affected_by_vulnerabilities": [], - "project": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e", - "dependency_uid": "pkg:pypi/pytest?uuid=33c92b36-8293-4fe5-8094-0eb645833284", + "project": "83d59103-a103-4c9a-a433-618eff973b63", + "dependency_uid": "pkg:pypi/pytest?uuid=9f487b4c-4d3e-4d13-9100-89ab897c90ee", "for_package": 1, + "resolved_to_package": null, "datafile_resource": 1, "extracted_requirement": "pytest; extra == \"tests\"", "scope": "tests", "datasource_id": "pypi_wheel", "is_runtime": true, "is_optional": true, - "is_resolved": false + "is_resolved": false, + "is_direct": true } }, { @@ -1647,16 +1671,18 @@ "qualifiers": "", "subpath": "", "affected_by_vulnerabilities": [], - "project": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e", - "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=bf9520e1-0b6a-4c00-90c4-b5e7afb6de3e", + "project": "83d59103-a103-4c9a-a433-618eff973b63", + "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=7c26cfc3-080d-4d3c-bcfe-fd6a495c5d3a", "for_package": 1, + "resolved_to_package": null, "datafile_resource": 1, "extracted_requirement": "pytest-asyncio; extra == \"tests\"", "scope": "tests", "datasource_id": "pypi_wheel", "is_runtime": true, "is_optional": true, - "is_resolved": false + "is_resolved": false, + "is_direct": true } }, { @@ -1670,16 +1696,18 @@ "qualifiers": "", "subpath": "", "affected_by_vulnerabilities": [], - "project": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e", - "dependency_uid": "pkg:pypi/pytest?uuid=21776640-36c9-4111-a6c4-ea5a550c85e0", + "project": "83d59103-a103-4c9a-a433-618eff973b63", + "dependency_uid": "pkg:pypi/pytest?uuid=909f72ab-1d6b-41f6-b41a-2b3be388870a", "for_package": 2, + "resolved_to_package": null, "datafile_resource": 7, "extracted_requirement": "pytest; extra == \"tests\"", "scope": "tests", "datasource_id": "pypi_wheel_metadata", "is_runtime": true, "is_optional": true, - "is_resolved": false + "is_resolved": false, + "is_direct": true } }, { @@ -1693,16 +1721,18 @@ "qualifiers": "", "subpath": "", "affected_by_vulnerabilities": [], - "project": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e", - "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=1963fa7d-3e64-4975-b4e7-80492466fefb", + "project": "83d59103-a103-4c9a-a433-618eff973b63", + "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=e474c205-ceef-444a-a0d7-aee32c8716df", "for_package": 2, + "resolved_to_package": null, "datafile_resource": 7, "extracted_requirement": "pytest-asyncio; extra == \"tests\"", "scope": "tests", "datasource_id": "pypi_wheel_metadata", "is_runtime": true, "is_optional": true, - "is_resolved": false + "is_resolved": false, + "is_direct": true } } ] diff --git a/scanpipe/tests/data/asgiref-3.3.0_load_inventory_expected.json b/scanpipe/tests/data/asgiref-3.3.0_load_inventory_expected.json index 037d59811b..1f01c8b4fd 100644 --- a/scanpipe/tests/data/asgiref-3.3.0_load_inventory_expected.json +++ b/scanpipe/tests/data/asgiref-3.3.0_load_inventory_expected.json @@ -136,6 +136,8 @@ "Further Documentation": "https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions" }, "package_uid": "pkg:pypi/asgiref@3.3.0?uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "pypi_wheel" ], @@ -255,6 +257,8 @@ "Further Documentation": "https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions" }, "package_uid": "pkg:pypi/asgiref@3.3.0?uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "pypi_wheel_metadata" ], @@ -275,6 +279,7 @@ "is_runtime": true, "is_optional": true, "is_resolved": false, + "is_direct": true, "dependency_uid": "pkg:pypi/pytest?uuid=fixed-uid-done-for-testing-5642512d1758", "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=fixed-uid-done-for-testing-5642512d1758", "resolved_to_package_uid": null, @@ -290,6 +295,7 @@ "is_runtime": true, "is_optional": true, "is_resolved": false, + "is_direct": true, "dependency_uid": "pkg:pypi/pytest?uuid=fixed-uid-done-for-testing-5642512d1758", "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=fixed-uid-done-for-testing-5642512d1758", "resolved_to_package_uid": null, @@ -305,6 +311,7 @@ "is_runtime": true, "is_optional": true, "is_resolved": false, + "is_direct": true, "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=fixed-uid-done-for-testing-5642512d1758", "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=fixed-uid-done-for-testing-5642512d1758", "resolved_to_package_uid": null, @@ -320,6 +327,7 @@ "is_runtime": true, "is_optional": true, "is_resolved": false, + "is_direct": true, "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=fixed-uid-done-for-testing-5642512d1758", "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=fixed-uid-done-for-testing-5642512d1758", "resolved_to_package_uid": null, @@ -398,6 +406,8 @@ "Documentation": "https://asgi.readthedocs.io/", "Further Documentation": "https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions" }, + "is_private": false, + "is_virtual": false, "qualifiers": {}, "notice_text": null, "api_data_url": "https://pypi.org/pypi/asgiref/3.3.0/json", @@ -405,6 +415,7 @@ { "purl": "pkg:pypi/pytest", "scope": "tests", + "is_direct": true, "extra_data": {}, "is_runtime": true, "is_optional": true, @@ -415,6 +426,7 @@ { "purl": "pkg:pypi/pytest-asyncio", "scope": "tests", + "is_direct": true, "extra_data": {}, "is_runtime": true, "is_optional": true, @@ -885,6 +897,8 @@ "Documentation": "https://asgi.readthedocs.io/", "Further Documentation": "https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions" }, + "is_private": false, + "is_virtual": false, "qualifiers": {}, "notice_text": null, "api_data_url": "https://pypi.org/pypi/asgiref/3.3.0/json", @@ -892,6 +906,7 @@ { "purl": "pkg:pypi/pytest", "scope": "tests", + "is_direct": true, "extra_data": {}, "is_runtime": true, "is_optional": true, @@ -902,6 +917,7 @@ { "purl": "pkg:pypi/pytest-asyncio", "scope": "tests", + "is_direct": true, "extra_data": {}, "is_runtime": true, "is_optional": true, diff --git a/scanpipe/tests/data/asgiref-3.3.0_scanpipe_output.json b/scanpipe/tests/data/asgiref-3.3.0_scanpipe_output.json index 475a47ad66..1393778a64 100644 --- a/scanpipe/tests/data/asgiref-3.3.0_scanpipe_output.json +++ b/scanpipe/tests/data/asgiref-3.3.0_scanpipe_output.json @@ -2,18 +2,18 @@ "headers": [ { "tool_name": "scanpipe", - "tool_version": "v33.1.0-14-gd4b299c", + "tool_version": "v34.5.0-8-g21a9308", "other_tools": [ - "pkg:pypi/scancode-toolkit@32.0.8" + "pkg:pypi/scancode-toolkit@32.1.0" ], "notice": "Generated with ScanCode.io and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied.\nNo content created from ScanCode.io should be considered or used as legal advice.\nConsult an Attorney for any legal advice.\nScanCode.io is a free software code scanning tool from nexB Inc. and others\nlicensed under the Apache License version 2.0.\nScanCode is a trademark of nexB Inc.\nVisit https://github.com/nexB/scancode.io for support and download.\n", - "uuid": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e", - "created_date": "2024-03-01T11:33:36.591Z", + "uuid": "83d59103-a103-4c9a-a433-618eff973b63", + "created_date": "2024-06-13T14:23:21.548Z", "notes": "", "settings": {}, "input_sources": [ { - "uuid": "df2fef03-0f8c-41b6-a645-e8a4de4e0f65", + "uuid": "075464b8-34ad-44ef-b5d3-99a50d79c5d4", "filename": "asgiref-3.3.0-py3-none-any.whl", "download_url": "", "is_uploaded": true, @@ -28,15 +28,15 @@ "pipeline_name": "scan_codebase", "status": "not_started", "description": "Scan a codebase for application packages, licenses, and copyrights.", - "uuid": "dfe51f4c-9867-462b-977c-1897531bb35f", - "created_date": "2024-03-01T11:33:36.594169Z", + "uuid": "ceb011c2-6d52-4c3a-bbff-c53e45f0313e", + "created_date": "2024-06-13T14:23:21.551739Z", "scancodeio_version": "", "task_id": null, "task_start_date": null, "task_end_date": null, "task_exitcode": null, "task_output": "", - "log": "2024-03-01 11:33:36.59 Pipeline [scan_codebase] starting\n2024-03-01 11:33:36.59 Step [download_missing_inputs] starting\n2024-03-01 11:33:36.59 Step [download_missing_inputs] completed in 0 seconds\n2024-03-01 11:33:36.59 Step [copy_inputs_to_codebase_directory] starting\n2024-03-01 11:33:36.59 Step [copy_inputs_to_codebase_directory] completed in 0 seconds\n2024-03-01 11:33:36.59 Step [extract_archives] starting\n2024-03-01 11:33:36.66 Step [extract_archives] completed in 0 seconds\n2024-03-01 11:33:36.66 Step [collect_and_create_codebase_resources] starting\n2024-03-01 11:33:36.85 Step [collect_and_create_codebase_resources] completed in 0 seconds\n2024-03-01 11:33:36.85 Step [flag_empty_files] starting\n2024-03-01 11:33:36.85 Step [flag_empty_files] completed in 0 seconds\n2024-03-01 11:33:36.85 Step [flag_ignored_resources] starting\n2024-03-01 11:33:36.85 Step [flag_ignored_resources] completed in 0 seconds\n2024-03-01 11:33:36.85 Step [scan_for_application_packages] starting\n2024-03-01 11:33:36.89 Progress: 11% (2/18)\n2024-03-01 11:33:36.92 Progress: 22% (4/18)\n2024-03-01 11:33:36.93 Progress: 33% (6/18)\n2024-03-01 11:33:36.93 Progress: 44% (8/18)\n2024-03-01 11:33:36.93 Progress: 55% (10/18)\n2024-03-01 11:33:36.93 Progress: 66% (12/18)\n2024-03-01 11:33:36.93 Progress: 77% (14/18)\n2024-03-01 11:33:36.93 Progress: 88% (16/18)\n2024-03-01 11:33:39.89 Progress: 100% (18/18)\n2024-03-01 11:33:40.08 Step [scan_for_application_packages] completed in 3 seconds\n2024-03-01 11:33:40.08 Step [scan_for_files] starting\n2024-03-01 11:33:50.84 Progress: 12% (2/16) ETA: 79 seconds (1.3 minutes)\n2024-03-01 11:33:51.20 Progress: 25% (4/16) ETA: 33 seconds\n2024-03-01 11:33:51.91 Progress: 37% (6/16) ETA: 20 seconds\n2024-03-01 11:33:52.72 Progress: 50% (8/16) ETA: 13 seconds\n2024-03-01 11:33:52.97 Progress: 62% (10/16) ETA: 8 seconds\n2024-03-01 11:33:53.15 Progress: 75% (12/16) ETA: 4 seconds\n2024-03-01 11:33:53.16 Progress: 87% (14/16) ETA: 2 seconds\n2024-03-01 11:33:53.56 Progress: 100% (16/16)\n2024-03-01 11:33:53.85 Step [scan_for_files] completed in 14 seconds\n2024-03-01 11:33:53.85 Pipeline completed in 17 seconds\n", + "log": "2024-06-13 14:23:21.55 Pipeline [scan_codebase] starting\n2024-06-13 14:23:21.55 Step [download_missing_inputs] starting\n2024-06-13 14:23:21.55 Step [download_missing_inputs] completed in 0 seconds\n2024-06-13 14:23:21.55 Step [copy_inputs_to_codebase_directory] starting\n2024-06-13 14:23:21.55 Step [copy_inputs_to_codebase_directory] completed in 0 seconds\n2024-06-13 14:23:21.55 Step [extract_archives] starting\n2024-06-13 14:23:21.60 Step [extract_archives] completed in 0 seconds\n2024-06-13 14:23:21.60 Step [collect_and_create_codebase_resources] starting\n2024-06-13 14:23:21.67 Step [collect_and_create_codebase_resources] completed in 0 seconds\n2024-06-13 14:23:21.67 Step [flag_empty_files] starting\n2024-06-13 14:23:21.67 Step [flag_empty_files] completed in 0 seconds\n2024-06-13 14:23:21.67 Step [flag_ignored_resources] starting\n2024-06-13 14:23:21.67 Step [flag_ignored_resources] completed in 0 seconds\n2024-06-13 14:23:21.68 Step [scan_for_application_packages] starting\n2024-06-13 14:23:21.69 Progress: 11% (2/18)\n2024-06-13 14:23:21.70 Progress: 22% (4/18)\n2024-06-13 14:23:21.71 Progress: 33% (6/18)\n2024-06-13 14:23:21.71 Progress: 44% (8/18)\n2024-06-13 14:23:21.71 Progress: 55% (10/18)\n2024-06-13 14:23:21.71 Progress: 66% (12/18)\n2024-06-13 14:23:21.71 Progress: 77% (14/18)\n2024-06-13 14:23:21.71 Progress: 88% (16/18)\n2024-06-13 14:23:24.08 Progress: 100% (18/18)\n2024-06-13 14:23:26.20 Step [scan_for_application_packages] completed in 5 seconds\n2024-06-13 14:23:26.20 Step [scan_for_files] starting\n2024-06-13 14:23:26.32 Progress: 12% (2/16) ETA: 1 seconds\n2024-06-13 14:23:26.33 Progress: 25% (4/16)\n2024-06-13 14:23:26.35 Progress: 37% (6/16)\n2024-06-13 14:23:26.56 Progress: 50% (8/16)\n2024-06-13 14:23:26.60 Progress: 62% (10/16)\n2024-06-13 14:23:26.63 Progress: 75% (12/16)\n2024-06-13 14:23:26.79 Progress: 87% (14/16)\n2024-06-13 14:23:26.88 Progress: 100% (16/16)\n2024-06-13 14:23:26.90 Step [scan_for_files] completed in 1 seconds\n2024-06-13 14:23:26.90 Pipeline completed in 5 seconds\n", "execution_time": null } ], @@ -146,7 +146,7 @@ "other_license_expression_spdx": "", "other_license_detections": [], "extracted_license_statement": "license: BSD\nclassifiers:\n - 'License :: OSI Approved :: BSD License'\n", - "compliance_alert": "", + "compliance_alert": "missing", "notice_text": "", "source_packages": [], "extra_data": { @@ -154,8 +154,15 @@ "Documentation": "https://asgi.readthedocs.io/", "Further Documentation": "https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions" }, - "package_uid": "pkg:pypi/asgiref@3.3.0?uuid=90cd6d93-1c46-477a-bd4a-47aa6b3ce773", - "datasource_id": "", + "package_uid": "pkg:pypi/asgiref@3.3.0?uuid=eca680a3-115b-494b-8581-81c93384e22d", + "is_private": false, + "is_virtual": false, + "datasource_ids": [ + "pypi_wheel" + ], + "datafile_paths": [ + "asgiref-3.3.0-py3-none-any.whl" + ], "file_references": [], "missing_resources": [], "modified_resources": [], @@ -263,7 +270,7 @@ "other_license_expression_spdx": "", "other_license_detections": [], "extracted_license_statement": "license: BSD\nclassifiers:\n - 'License :: OSI Approved :: BSD License'\n", - "compliance_alert": "", + "compliance_alert": "missing", "notice_text": "", "source_packages": [], "extra_data": { @@ -271,8 +278,15 @@ "Documentation": "https://asgi.readthedocs.io/", "Further Documentation": "https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions" }, - "package_uid": "pkg:pypi/asgiref@3.3.0?uuid=fb05a7ee-1078-4d60-bf71-2ba031414f1c", - "datasource_id": "", + "package_uid": "pkg:pypi/asgiref@3.3.0?uuid=510ea8b0-d7a7-4531-96ea-f41cb4384976", + "is_private": false, + "is_virtual": false, + "datasource_ids": [ + "pypi_wheel_metadata" + ], + "datafile_paths": [ + "asgiref-3.3.0-py3-none-any.whl-extract/asgiref-3.3.0.dist-info/METADATA" + ], "file_references": [], "missing_resources": [], "modified_resources": [], @@ -287,8 +301,10 @@ "is_runtime": true, "is_optional": true, "is_resolved": false, - "dependency_uid": "pkg:pypi/pytest?uuid=33c92b36-8293-4fe5-8094-0eb645833284", - "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=90cd6d93-1c46-477a-bd4a-47aa6b3ce773", + "is_direct": true, + "dependency_uid": "pkg:pypi/pytest?uuid=9f487b4c-4d3e-4d13-9100-89ab897c90ee", + "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=eca680a3-115b-494b-8581-81c93384e22d", + "resolved_to_package_uid": null, "datafile_path": "asgiref-3.3.0-py3-none-any.whl", "datasource_id": "pypi_wheel", "package_type": "pypi", @@ -301,8 +317,10 @@ "is_runtime": true, "is_optional": true, "is_resolved": false, - "dependency_uid": "pkg:pypi/pytest?uuid=21776640-36c9-4111-a6c4-ea5a550c85e0", - "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=fb05a7ee-1078-4d60-bf71-2ba031414f1c", + "is_direct": true, + "dependency_uid": "pkg:pypi/pytest?uuid=909f72ab-1d6b-41f6-b41a-2b3be388870a", + "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=510ea8b0-d7a7-4531-96ea-f41cb4384976", + "resolved_to_package_uid": null, "datafile_path": "asgiref-3.3.0-py3-none-any.whl-extract/asgiref-3.3.0.dist-info/METADATA", "datasource_id": "pypi_wheel_metadata", "package_type": "pypi", @@ -315,8 +333,10 @@ "is_runtime": true, "is_optional": true, "is_resolved": false, - "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=bf9520e1-0b6a-4c00-90c4-b5e7afb6de3e", - "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=90cd6d93-1c46-477a-bd4a-47aa6b3ce773", + "is_direct": true, + "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=7c26cfc3-080d-4d3c-bcfe-fd6a495c5d3a", + "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=eca680a3-115b-494b-8581-81c93384e22d", + "resolved_to_package_uid": null, "datafile_path": "asgiref-3.3.0-py3-none-any.whl", "datasource_id": "pypi_wheel", "package_type": "pypi", @@ -329,8 +349,10 @@ "is_runtime": true, "is_optional": true, "is_resolved": false, - "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=1963fa7d-3e64-4975-b4e7-80492466fefb", - "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=fb05a7ee-1078-4d60-bf71-2ba031414f1c", + "is_direct": true, + "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=e474c205-ceef-444a-a0d7-aee32c8716df", + "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=510ea8b0-d7a7-4531-96ea-f41cb4384976", + "resolved_to_package_uid": null, "datafile_path": "asgiref-3.3.0-py3-none-any.whl-extract/asgiref-3.3.0.dist-info/METADATA", "datasource_id": "pypi_wheel_metadata", "package_type": "pypi", @@ -411,6 +433,8 @@ "Documentation": "https://asgi.readthedocs.io/", "Further Documentation": "https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions" }, + "is_private": false, + "is_virtual": false, "qualifiers": {}, "description": "ASGI specs, helper code, and adapters\nasgiref\n=======\n\n.. image:: https://api.travis-ci.org/django/asgiref.svg\n :target: https://travis-ci.org/django/asgiref\n\n.. image:: https://img.shields.io/pypi/v/asgiref.svg\n :target: https://pypi.python.org/pypi/asgiref\n\nASGI is a standard for Python asynchronous web apps and servers to communicate\nwith each other, and positioned as an asynchronous successor to WSGI. You can\nread more at https://asgi.readthedocs.io/en/latest/\n\nThis package includes ASGI base libraries, such as:\n\n* Sync-to-async and async-to-sync function wrappers, ``asgiref.sync``\n* Server base classes, ``asgiref.server``\n* A WSGI-to-ASGI adapter, in ``asgiref.wsgi``\n\n\nFunction wrappers\n-----------------\n\nThese allow you to wrap or decorate async or sync functions to call them from\nthe other style (so you can call async functions from a synchronous thread,\nor vice-versa).\n\nIn particular:\n\n* AsyncToSync lets a synchronous subthread stop and wait while the async\n function is called on the main thread's event loop, and then control is\n returned to the thread when the async function is finished.\n\n* SyncToAsync lets async code call a synchronous function, which is run in\n a threadpool and control returned to the async coroutine when the synchronous\n function completes.\n\nThe idea is to make it easier to call synchronous APIs from async code and\nasynchronous APIs from synchronous code so it's easier to transition code from\none style to the other. In the case of Channels, we wrap the (synchronous)\nDjango view system with SyncToAsync to allow it to run inside the (asynchronous)\nASGI server.\n\nNote that exactly what threads things run in is very specific, and aimed to\nkeep maximum compatibility with old synchronous code. See\n\"Synchronous code & Threads\" below for a full explanation. By default,\n``sync_to_async`` will run all synchronous code in the program in the same\nthread for safety reasons; you can disable this for more performance with\n``@sync_to_async(thread_sensitive=False)``, but make sure that your code does\nnot rely on anything bound to threads (like database connections) when you do.\n\n\nThreadlocal replacement\n-----------------------\n\nThis is a drop-in replacement for ``threading.local`` that works with both\nthreads and asyncio Tasks. Even better, it will proxy values through from a\ntask-local context to a thread-local context when you use ``sync_to_async``\nto run things in a threadpool, and vice-versa for ``async_to_sync``.\n\nIf you instead want true thread- and task-safety, you can set\n``thread_critical`` on the Local object to ensure this instead.\n\n\nServer base classes\n-------------------\n\nIncludes a ``StatelessServer`` class which provides all the hard work of\nwriting a stateless server (as in, does not handle direct incoming sockets\nbut instead consumes external streams or sockets to work out what is happening).\n\nAn example of such a server would be a chatbot server that connects out to\na central chat server and provides a \"connection scope\" per user chatting to\nit. There's only one actual connection, but the server has to separate things\ninto several scopes for easier writing of the code.\n\nYou can see an example of this being used in `frequensgi `_.\n\n\nWSGI-to-ASGI adapter\n--------------------\n\nAllows you to wrap a WSGI application so it appears as a valid ASGI application.\n\nSimply wrap it around your WSGI application like so::\n\n asgi_application = WsgiToAsgi(wsgi_application)\n\nThe WSGI application will be run in a synchronous threadpool, and the wrapped\nASGI application will be one that accepts ``http`` class messages.\n\nPlease note that not all extended features of WSGI may be supported (such as\nfile handles for incoming POST bodies).\n\n\nDependencies\n------------\n\n``asgiref`` requires Python 3.5 or higher.\n\n\nContributing\n------------\n\nPlease refer to the\n`main Channels contributing docs `_.\n\n\nTesting\n'''''''\n\nTo run tests, make sure you have installed the ``tests`` extra with the package::\n\n cd asgiref/\n pip install -e .[tests]\n pytest\n\n\nBuilding the documentation\n''''''''''''''''''''''''''\n\nThe documentation uses `Sphinx `_::\n\n cd asgiref/docs/\n pip install sphinx\n\nTo build the docs, you can use the default tools::\n\n sphinx-build -b html . _build/html # or `make html`, if you've got make set up\n cd _build/html\n python -m http.server\n\n...or you can use ``sphinx-autobuild`` to run a server and rebuild/reload\nyour documentation changes automatically::\n\n pip install sphinx-autobuild\n sphinx-autobuild . _build/html\n\n\nImplementation Details\n----------------------\n\nSynchronous code & threads\n''''''''''''''''''''''''''\n\nThe ``asgiref.sync`` module provides two wrappers that let you go between\nasynchronous and synchronous code at will, while taking care of the rough edges\nfor you.\n\nUnfortunately, the rough edges are numerous, and the code has to work especially\nhard to keep things in the same thread as much as possible. Notably, the\nrestrictions we are working with are:\n\n* All synchronous code called through ``SyncToAsync`` and marked with\n ``thread_sensitive`` should run in the same thread as each other (and if the\n outer layer of the program is synchronous, the main thread)\n\n* If a thread already has a running async loop, ``AsyncToSync`` can't run things\n on that loop if it's blocked on synchronous code that is above you in the\n call stack.\n\nThe first compromise you get to might be that ``thread_sensitive`` code should\njust run in the same thread and not spawn in a sub-thread, fulfilling the first\nrestriction, but that immediately runs you into the second restriction.\n\nThe only real solution is to essentially have a variant of ThreadPoolExecutor\nthat executes any ``thread_sensitive`` code on the outermost synchronous\nthread - either the main thread, or a single spawned subthread.\n\nThis means you now have two basic states:\n\n* If the outermost layer of your program is synchronous, then all async code\n run through ``AsyncToSync`` will run in a per-call event loop in arbitary\n sub-threads, while all ``thread_sensitive`` code will run in the main thread.\n\n* If the outermost layer of your program is asynchronous, then all async code\n runs on the main thread's event loop, and all ``thread_sensitive`` synchronous\n code will run in a single shared sub-thread.\n\nCruicially, this means that in both cases there is a thread which is a shared\nresource that all ``thread_sensitive`` code must run on, and there is a chance\nthat this thread is currently blocked on its own ``AsyncToSync`` call. Thus,\n``AsyncToSync`` needs to act as an executor for thread code while it's blocking.\n\nThe ``CurrentThreadExecutor`` class provides this functionality; rather than\nsimply waiting on a Future, you can call its ``run_until_future`` method and\nit will run submitted code until that Future is done. This means that code\ninside the call can then run code on your thread.\n\n\nMaintenance and Security\n------------------------\n\nTo report security issues, please contact security@djangoproject.com. For GPG\nsignatures and more security process information, see\nhttps://docs.djangoproject.com/en/dev/internals/security/.\n\nTo report bugs or request new features, please open a new GitHub issue.\n\nThis repository is part of the Channels project. For the shepherd and maintenance team, please see the\n`main Channels readme `_.", "notice_text": null, @@ -419,6 +443,7 @@ { "purl": "pkg:pypi/pytest", "scope": "tests", + "is_direct": true, "extra_data": {}, "is_runtime": true, "is_optional": true, @@ -429,6 +454,7 @@ { "purl": "pkg:pypi/pytest-asyncio", "scope": "tests", + "is_direct": true, "extra_data": {}, "is_runtime": true, "is_optional": true, @@ -630,7 +656,7 @@ } ], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=90cd6d93-1c46-477a-bd4a-47aa6b3ce773" + "pkg:pypi/asgiref@3.3.0?uuid=eca680a3-115b-494b-8581-81c93384e22d" ], "emails": [], "urls": [], @@ -789,7 +815,7 @@ ], "license_clues": [], "percentage_of_license_text": 95.11, - "compliance_alert": "", + "compliance_alert": "missing", "copyrights": [ { "end_line": 1, @@ -807,7 +833,7 @@ "authors": [], "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=fb05a7ee-1078-4d60-bf71-2ba031414f1c" + "pkg:pypi/asgiref@3.3.0?uuid=510ea8b0-d7a7-4531-96ea-f41cb4384976" ], "emails": [], "urls": [], @@ -886,6 +912,8 @@ "Documentation": "https://asgi.readthedocs.io/", "Further Documentation": "https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions" }, + "is_private": false, + "is_virtual": false, "qualifiers": {}, "description": "ASGI specs, helper code, and adapters\nasgiref\n=======\n\n.. image:: https://api.travis-ci.org/django/asgiref.svg\n :target: https://travis-ci.org/django/asgiref\n\n.. image:: https://img.shields.io/pypi/v/asgiref.svg\n :target: https://pypi.python.org/pypi/asgiref\n\nASGI is a standard for Python asynchronous web apps and servers to communicate\nwith each other, and positioned as an asynchronous successor to WSGI. You can\nread more at https://asgi.readthedocs.io/en/latest/\n\nThis package includes ASGI base libraries, such as:\n\n* Sync-to-async and async-to-sync function wrappers, ``asgiref.sync``\n* Server base classes, ``asgiref.server``\n* A WSGI-to-ASGI adapter, in ``asgiref.wsgi``\n\n\nFunction wrappers\n-----------------\n\nThese allow you to wrap or decorate async or sync functions to call them from\nthe other style (so you can call async functions from a synchronous thread,\nor vice-versa).\n\nIn particular:\n\n* AsyncToSync lets a synchronous subthread stop and wait while the async\n function is called on the main thread's event loop, and then control is\n returned to the thread when the async function is finished.\n\n* SyncToAsync lets async code call a synchronous function, which is run in\n a threadpool and control returned to the async coroutine when the synchronous\n function completes.\n\nThe idea is to make it easier to call synchronous APIs from async code and\nasynchronous APIs from synchronous code so it's easier to transition code from\none style to the other. In the case of Channels, we wrap the (synchronous)\nDjango view system with SyncToAsync to allow it to run inside the (asynchronous)\nASGI server.\n\nNote that exactly what threads things run in is very specific, and aimed to\nkeep maximum compatibility with old synchronous code. See\n\"Synchronous code & Threads\" below for a full explanation. By default,\n``sync_to_async`` will run all synchronous code in the program in the same\nthread for safety reasons; you can disable this for more performance with\n``@sync_to_async(thread_sensitive=False)``, but make sure that your code does\nnot rely on anything bound to threads (like database connections) when you do.\n\n\nThreadlocal replacement\n-----------------------\n\nThis is a drop-in replacement for ``threading.local`` that works with both\nthreads and asyncio Tasks. Even better, it will proxy values through from a\ntask-local context to a thread-local context when you use ``sync_to_async``\nto run things in a threadpool, and vice-versa for ``async_to_sync``.\n\nIf you instead want true thread- and task-safety, you can set\n``thread_critical`` on the Local object to ensure this instead.\n\n\nServer base classes\n-------------------\n\nIncludes a ``StatelessServer`` class which provides all the hard work of\nwriting a stateless server (as in, does not handle direct incoming sockets\nbut instead consumes external streams or sockets to work out what is happening).\n\nAn example of such a server would be a chatbot server that connects out to\na central chat server and provides a \"connection scope\" per user chatting to\nit. There's only one actual connection, but the server has to separate things\ninto several scopes for easier writing of the code.\n\nYou can see an example of this being used in `frequensgi `_.\n\n\nWSGI-to-ASGI adapter\n--------------------\n\nAllows you to wrap a WSGI application so it appears as a valid ASGI application.\n\nSimply wrap it around your WSGI application like so::\n\n asgi_application = WsgiToAsgi(wsgi_application)\n\nThe WSGI application will be run in a synchronous threadpool, and the wrapped\nASGI application will be one that accepts ``http`` class messages.\n\nPlease note that not all extended features of WSGI may be supported (such as\nfile handles for incoming POST bodies).\n\n\nDependencies\n------------\n\n``asgiref`` requires Python 3.5 or higher.\n\n\nContributing\n------------\n\nPlease refer to the\n`main Channels contributing docs `_.\n\n\nTesting\n'''''''\n\nTo run tests, make sure you have installed the ``tests`` extra with the package::\n\n cd asgiref/\n pip install -e .[tests]\n pytest\n\n\nBuilding the documentation\n''''''''''''''''''''''''''\n\nThe documentation uses `Sphinx `_::\n\n cd asgiref/docs/\n pip install sphinx\n\nTo build the docs, you can use the default tools::\n\n sphinx-build -b html . _build/html # or `make html`, if you've got make set up\n cd _build/html\n python -m http.server\n\n...or you can use ``sphinx-autobuild`` to run a server and rebuild/reload\nyour documentation changes automatically::\n\n pip install sphinx-autobuild\n sphinx-autobuild . _build/html\n\n\nImplementation Details\n----------------------\n\nSynchronous code & threads\n''''''''''''''''''''''''''\n\nThe ``asgiref.sync`` module provides two wrappers that let you go between\nasynchronous and synchronous code at will, while taking care of the rough edges\nfor you.\n\nUnfortunately, the rough edges are numerous, and the code has to work especially\nhard to keep things in the same thread as much as possible. Notably, the\nrestrictions we are working with are:\n\n* All synchronous code called through ``SyncToAsync`` and marked with\n ``thread_sensitive`` should run in the same thread as each other (and if the\n outer layer of the program is synchronous, the main thread)\n\n* If a thread already has a running async loop, ``AsyncToSync`` can't run things\n on that loop if it's blocked on synchronous code that is above you in the\n call stack.\n\nThe first compromise you get to might be that ``thread_sensitive`` code should\njust run in the same thread and not spawn in a sub-thread, fulfilling the first\nrestriction, but that immediately runs you into the second restriction.\n\nThe only real solution is to essentially have a variant of ThreadPoolExecutor\nthat executes any ``thread_sensitive`` code on the outermost synchronous\nthread - either the main thread, or a single spawned subthread.\n\nThis means you now have two basic states:\n\n* If the outermost layer of your program is synchronous, then all async code\n run through ``AsyncToSync`` will run in a per-call event loop in arbitary\n sub-threads, while all ``thread_sensitive`` code will run in the main thread.\n\n* If the outermost layer of your program is asynchronous, then all async code\n runs on the main thread's event loop, and all ``thread_sensitive`` synchronous\n code will run in a single shared sub-thread.\n\nCruicially, this means that in both cases there is a thread which is a shared\nresource that all ``thread_sensitive`` code must run on, and there is a chance\nthat this thread is currently blocked on its own ``AsyncToSync`` call. Thus,\n``AsyncToSync`` needs to act as an executor for thread code while it's blocking.\n\nThe ``CurrentThreadExecutor`` class provides this functionality; rather than\nsimply waiting on a Future, you can call its ``run_until_future`` method and\nit will run submitted code until that Future is done. This means that code\ninside the call can then run code on your thread.\n\n\nMaintenance and Security\n------------------------\n\nTo report security issues, please contact security@djangoproject.com. For GPG\nsignatures and more security process information, see\nhttps://docs.djangoproject.com/en/dev/internals/security/.\n\nTo report bugs or request new features, please open a new GitHub issue.\n\nThis repository is part of the Channels project. For the shepherd and maintenance team, please see the\n`main Channels readme `_.", "notice_text": null, @@ -894,6 +922,7 @@ { "purl": "pkg:pypi/pytest", "scope": "tests", + "is_direct": true, "extra_data": {}, "is_runtime": true, "is_optional": true, @@ -904,6 +933,7 @@ { "purl": "pkg:pypi/pytest-asyncio", "scope": "tests", + "is_direct": true, "extra_data": {}, "is_runtime": true, "is_optional": true, @@ -1105,7 +1135,7 @@ } ], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=fb05a7ee-1078-4d60-bf71-2ba031414f1c" + "pkg:pypi/asgiref@3.3.0?uuid=510ea8b0-d7a7-4531-96ea-f41cb4384976" ], "emails": [], "urls": [], @@ -1142,7 +1172,7 @@ "authors": [], "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=fb05a7ee-1078-4d60-bf71-2ba031414f1c" + "pkg:pypi/asgiref@3.3.0?uuid=510ea8b0-d7a7-4531-96ea-f41cb4384976" ], "emails": [], "urls": [], @@ -1179,7 +1209,7 @@ "authors": [], "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=fb05a7ee-1078-4d60-bf71-2ba031414f1c" + "pkg:pypi/asgiref@3.3.0?uuid=510ea8b0-d7a7-4531-96ea-f41cb4384976" ], "emails": [], "urls": [], @@ -1216,7 +1246,7 @@ "authors": [], "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=fb05a7ee-1078-4d60-bf71-2ba031414f1c" + "pkg:pypi/asgiref@3.3.0?uuid=510ea8b0-d7a7-4531-96ea-f41cb4384976" ], "emails": [], "urls": [], @@ -1253,7 +1283,7 @@ "authors": [], "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=fb05a7ee-1078-4d60-bf71-2ba031414f1c" + "pkg:pypi/asgiref@3.3.0?uuid=510ea8b0-d7a7-4531-96ea-f41cb4384976" ], "emails": [], "urls": [], @@ -1290,7 +1320,7 @@ "authors": [], "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=fb05a7ee-1078-4d60-bf71-2ba031414f1c" + "pkg:pypi/asgiref@3.3.0?uuid=510ea8b0-d7a7-4531-96ea-f41cb4384976" ], "emails": [], "urls": [], @@ -1327,7 +1357,7 @@ "authors": [], "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=fb05a7ee-1078-4d60-bf71-2ba031414f1c" + "pkg:pypi/asgiref@3.3.0?uuid=510ea8b0-d7a7-4531-96ea-f41cb4384976" ], "emails": [], "urls": [], @@ -1364,7 +1394,7 @@ "authors": [], "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=fb05a7ee-1078-4d60-bf71-2ba031414f1c" + "pkg:pypi/asgiref@3.3.0?uuid=510ea8b0-d7a7-4531-96ea-f41cb4384976" ], "emails": [], "urls": [], @@ -1401,7 +1431,7 @@ "authors": [], "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=fb05a7ee-1078-4d60-bf71-2ba031414f1c" + "pkg:pypi/asgiref@3.3.0?uuid=510ea8b0-d7a7-4531-96ea-f41cb4384976" ], "emails": [], "urls": [], @@ -1438,7 +1468,7 @@ "authors": [], "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=fb05a7ee-1078-4d60-bf71-2ba031414f1c" + "pkg:pypi/asgiref@3.3.0?uuid=510ea8b0-d7a7-4531-96ea-f41cb4384976" ], "emails": [], "urls": [], @@ -1475,7 +1505,7 @@ "authors": [], "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=fb05a7ee-1078-4d60-bf71-2ba031414f1c" + "pkg:pypi/asgiref@3.3.0?uuid=510ea8b0-d7a7-4531-96ea-f41cb4384976" ], "emails": [], "urls": [], @@ -1529,13 +1559,13 @@ ], "license_clues": [], "percentage_of_license_text": 1.22, - "compliance_alert": "", + "compliance_alert": "missing", "copyrights": [], "holders": [], "authors": [], "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=fb05a7ee-1078-4d60-bf71-2ba031414f1c" + "pkg:pypi/asgiref@3.3.0?uuid=510ea8b0-d7a7-4531-96ea-f41cb4384976" ], "emails": [], "urls": [ @@ -1588,7 +1618,7 @@ "authors": [], "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=fb05a7ee-1078-4d60-bf71-2ba031414f1c" + "pkg:pypi/asgiref@3.3.0?uuid=510ea8b0-d7a7-4531-96ea-f41cb4384976" ], "emails": [], "urls": [], diff --git a/scanpipe/tests/data/asgiref-3.3.0_toolkit_scan.json b/scanpipe/tests/data/asgiref-3.3.0_toolkit_scan.json index 9f4220caca..9d65b79a11 100644 --- a/scanpipe/tests/data/asgiref-3.3.0_toolkit_scan.json +++ b/scanpipe/tests/data/asgiref-3.3.0_toolkit_scan.json @@ -2,7 +2,7 @@ "headers": [ { "tool_name": "scancode-toolkit", - "tool_version": "32.0.8", + "tool_version": "32.1.0", "options": { "--copyright": true, "--info": true, @@ -10,10 +10,10 @@ "--package": true }, "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", - "start_timestamp": "2024-03-01T113353.857112", - "end_timestamp": "2024-03-01T113357.331569", + "start_timestamp": "2024-06-13T142326.906987", + "end_timestamp": "2024-06-13T142327.694160", "output_format_version": "3.1.0", - "duration": 3.4744701385498047, + "duration": 0.7872016429901123, "message": null, "errors": [], "warnings": [], @@ -21,8 +21,8 @@ "system_environment": { "operating_system": "linux", "cpu_architecture": "64", - "platform": "Linux-5.15.0-94-generic-x86_64-with-glibc2.35", - "platform_version": "#104-Ubuntu SMP Tue Jan 9 15:25:40 UTC 2024", + "platform": "Linux-5.15.0-112-generic-x86_64-with-glibc2.35", + "platform_version": "#122-Ubuntu SMP Thu May 23 07:48:21 UTC 2024", "python_version": "3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0]" }, "spdx_license_list_version": "3.23", @@ -130,6 +130,8 @@ "extracted_license_statement": "license: BSD\nclassifiers:\n - 'License :: OSI Approved :: BSD License'\n", "notice_text": null, "source_packages": [], + "is_private": false, + "is_virtual": false, "extra_data": { "Documentation": "https://asgi.readthedocs.io/", "Further Documentation": "https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions", @@ -138,7 +140,7 @@ "repository_homepage_url": "https://pypi.org/project/asgiref", "repository_download_url": "https://pypi.org/packages/source/a/asgiref/asgiref-3.3.0.tar.gz", "api_data_url": "https://pypi.org/pypi/asgiref/3.3.0/json", - "package_uid": "pkg:pypi/asgiref@3.3.0?uuid=2a37d192-e2ae-4392-8b01-fe06ee3e74b4", + "package_uid": "pkg:pypi/asgiref@3.3.0?uuid=6497a3d3-f29b-408c-8eb6-5c9a262cae90", "datafile_paths": [ "codebase/asgiref-3.3.0-py3-none-any.whl" ], @@ -246,6 +248,8 @@ "extracted_license_statement": "license: BSD\nclassifiers:\n - 'License :: OSI Approved :: BSD License'\n", "notice_text": null, "source_packages": [], + "is_private": false, + "is_virtual": false, "extra_data": { "Documentation": "https://asgi.readthedocs.io/", "Further Documentation": "https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions", @@ -254,7 +258,7 @@ "repository_homepage_url": "https://pypi.org/project/asgiref", "repository_download_url": "https://pypi.org/packages/source/a/asgiref/asgiref-3.3.0.tar.gz", "api_data_url": "https://pypi.org/pypi/asgiref/3.3.0/json", - "package_uid": "pkg:pypi/asgiref@3.3.0?uuid=38e17ba8-d7a8-4e7b-afa3-95aa693da1bd", + "package_uid": "pkg:pypi/asgiref@3.3.0?uuid=6bcf52b9-d880-4574-b1fd-f5dabd882b75", "datafile_paths": [ "codebase/asgiref-3.3.0-py3-none-any.whl-extract/asgiref-3.3.0.dist-info/METADATA" ], @@ -272,10 +276,11 @@ "is_runtime": true, "is_optional": true, "is_resolved": false, + "is_direct": true, "resolved_package": {}, "extra_data": {}, - "dependency_uid": "pkg:pypi/pytest?uuid=d3e15d40-b017-421c-a972-9736873402b0", - "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=2a37d192-e2ae-4392-8b01-fe06ee3e74b4", + "dependency_uid": "pkg:pypi/pytest?uuid=a151ecd1-4069-48e0-8612-2d78064b6fdc", + "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=6497a3d3-f29b-408c-8eb6-5c9a262cae90", "datafile_path": "codebase/asgiref-3.3.0-py3-none-any.whl", "datasource_id": "pypi_wheel" }, @@ -286,10 +291,11 @@ "is_runtime": true, "is_optional": true, "is_resolved": false, + "is_direct": true, "resolved_package": {}, "extra_data": {}, - "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=c88ff8c0-c55b-43a8-978f-e87a06b5a292", - "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=2a37d192-e2ae-4392-8b01-fe06ee3e74b4", + "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=f36d232d-fa28-4ff4-8095-44bc504c8ede", + "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=6497a3d3-f29b-408c-8eb6-5c9a262cae90", "datafile_path": "codebase/asgiref-3.3.0-py3-none-any.whl", "datasource_id": "pypi_wheel" }, @@ -300,10 +306,11 @@ "is_runtime": true, "is_optional": true, "is_resolved": false, + "is_direct": true, "resolved_package": {}, "extra_data": {}, - "dependency_uid": "pkg:pypi/pytest?uuid=6e128dd2-5668-4340-afbe-ecb969d40c80", - "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=38e17ba8-d7a8-4e7b-afa3-95aa693da1bd", + "dependency_uid": "pkg:pypi/pytest?uuid=bf7aeeb1-44f2-4958-91ec-8b887c95f9a1", + "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=6bcf52b9-d880-4574-b1fd-f5dabd882b75", "datafile_path": "codebase/asgiref-3.3.0-py3-none-any.whl-extract/asgiref-3.3.0.dist-info/METADATA", "datasource_id": "pypi_wheel_metadata" }, @@ -314,10 +321,11 @@ "is_runtime": true, "is_optional": true, "is_resolved": false, + "is_direct": true, "resolved_package": {}, "extra_data": {}, - "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=b54e289c-79a4-433f-8bb5-ebdb2d63023a", - "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=38e17ba8-d7a8-4e7b-afa3-95aa693da1bd", + "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=84b2883c-b748-4224-b0e2-1b3c8906396a", + "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=6bcf52b9-d880-4574-b1fd-f5dabd882b75", "datafile_path": "codebase/asgiref-3.3.0-py3-none-any.whl-extract/asgiref-3.3.0.dist-info/METADATA", "datasource_id": "pypi_wheel_metadata" } @@ -455,7 +463,7 @@ "base_name": "asgiref-3.3.0-py3-none-any", "extension": ".whl", "size": 19948, - "date": "2024-03-01", + "date": "2024-06-13", "sha1": "c03f67211a311b13d1294ac8af7cb139ee34c4f9", "md5": "5bce1df6dedc53a41a9a6b40d7b1699e", "sha256": "a5098bc870b80e7b872bff60bb363c7f2c2c89078759f6c47b53ff8c525a152e", @@ -696,6 +704,8 @@ "extra_data": {} } ], + "is_private": false, + "is_virtual": false, "extra_data": { "Documentation": "https://asgi.readthedocs.io/", "Further Documentation": "https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions", @@ -709,6 +719,7 @@ "is_runtime": true, "is_optional": true, "is_resolved": false, + "is_direct": true, "resolved_package": {}, "extra_data": {} }, @@ -719,6 +730,7 @@ "is_runtime": true, "is_optional": true, "is_resolved": false, + "is_direct": true, "resolved_package": {}, "extra_data": {} } @@ -731,7 +743,7 @@ } ], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=2a37d192-e2ae-4392-8b01-fe06ee3e74b4" + "pkg:pypi/asgiref@3.3.0?uuid=6497a3d3-f29b-408c-8eb6-5c9a262cae90" ], "detected_license_expression": null, "detected_license_expression_spdx": null, @@ -838,7 +850,7 @@ "is_script": false, "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=38e17ba8-d7a8-4e7b-afa3-95aa693da1bd" + "pkg:pypi/asgiref@3.3.0?uuid=6bcf52b9-d880-4574-b1fd-f5dabd882b75" ], "detected_license_expression": null, "detected_license_expression_spdx": null, @@ -875,7 +887,7 @@ "is_script": true, "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=38e17ba8-d7a8-4e7b-afa3-95aa693da1bd" + "pkg:pypi/asgiref@3.3.0?uuid=6bcf52b9-d880-4574-b1fd-f5dabd882b75" ], "detected_license_expression": null, "detected_license_expression_spdx": null, @@ -912,7 +924,7 @@ "is_script": true, "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=38e17ba8-d7a8-4e7b-afa3-95aa693da1bd" + "pkg:pypi/asgiref@3.3.0?uuid=6bcf52b9-d880-4574-b1fd-f5dabd882b75" ], "detected_license_expression": null, "detected_license_expression_spdx": null, @@ -949,7 +961,7 @@ "is_script": true, "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=38e17ba8-d7a8-4e7b-afa3-95aa693da1bd" + "pkg:pypi/asgiref@3.3.0?uuid=6bcf52b9-d880-4574-b1fd-f5dabd882b75" ], "detected_license_expression": null, "detected_license_expression_spdx": null, @@ -986,7 +998,7 @@ "is_script": true, "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=38e17ba8-d7a8-4e7b-afa3-95aa693da1bd" + "pkg:pypi/asgiref@3.3.0?uuid=6bcf52b9-d880-4574-b1fd-f5dabd882b75" ], "detected_license_expression": null, "detected_license_expression_spdx": null, @@ -1023,7 +1035,7 @@ "is_script": true, "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=38e17ba8-d7a8-4e7b-afa3-95aa693da1bd" + "pkg:pypi/asgiref@3.3.0?uuid=6bcf52b9-d880-4574-b1fd-f5dabd882b75" ], "detected_license_expression": null, "detected_license_expression_spdx": null, @@ -1060,7 +1072,7 @@ "is_script": true, "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=38e17ba8-d7a8-4e7b-afa3-95aa693da1bd" + "pkg:pypi/asgiref@3.3.0?uuid=6bcf52b9-d880-4574-b1fd-f5dabd882b75" ], "detected_license_expression": null, "detected_license_expression_spdx": null, @@ -1097,7 +1109,7 @@ "is_script": true, "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=38e17ba8-d7a8-4e7b-afa3-95aa693da1bd" + "pkg:pypi/asgiref@3.3.0?uuid=6bcf52b9-d880-4574-b1fd-f5dabd882b75" ], "detected_license_expression": "apache-2.0", "detected_license_expression_spdx": "Apache-2.0", @@ -1156,7 +1168,7 @@ "is_script": true, "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=38e17ba8-d7a8-4e7b-afa3-95aa693da1bd" + "pkg:pypi/asgiref@3.3.0?uuid=6bcf52b9-d880-4574-b1fd-f5dabd882b75" ], "detected_license_expression": null, "detected_license_expression_spdx": null, @@ -1228,7 +1240,7 @@ "is_script": false, "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=38e17ba8-d7a8-4e7b-afa3-95aa693da1bd" + "pkg:pypi/asgiref@3.3.0?uuid=6bcf52b9-d880-4574-b1fd-f5dabd882b75" ], "detected_license_expression": "bsd-new", "detected_license_expression_spdx": "BSD-3-Clause", @@ -1525,6 +1537,8 @@ "extra_data": {} } ], + "is_private": false, + "is_virtual": false, "extra_data": { "Documentation": "https://asgi.readthedocs.io/", "Further Documentation": "https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions", @@ -1538,6 +1552,7 @@ "is_runtime": true, "is_optional": true, "is_resolved": false, + "is_direct": true, "resolved_package": {}, "extra_data": {} }, @@ -1548,6 +1563,7 @@ "is_runtime": true, "is_optional": true, "is_resolved": false, + "is_direct": true, "resolved_package": {}, "extra_data": {} } @@ -1560,7 +1576,7 @@ } ], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=38e17ba8-d7a8-4e7b-afa3-95aa693da1bd" + "pkg:pypi/asgiref@3.3.0?uuid=6bcf52b9-d880-4574-b1fd-f5dabd882b75" ], "detected_license_expression": "bsd-new", "detected_license_expression_spdx": "BSD-3-Clause", @@ -1646,7 +1662,7 @@ "is_script": false, "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=38e17ba8-d7a8-4e7b-afa3-95aa693da1bd" + "pkg:pypi/asgiref@3.3.0?uuid=6bcf52b9-d880-4574-b1fd-f5dabd882b75" ], "detected_license_expression": null, "detected_license_expression_spdx": null, @@ -1683,7 +1699,7 @@ "is_script": false, "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=38e17ba8-d7a8-4e7b-afa3-95aa693da1bd" + "pkg:pypi/asgiref@3.3.0?uuid=6bcf52b9-d880-4574-b1fd-f5dabd882b75" ], "detected_license_expression": null, "detected_license_expression_spdx": null, @@ -1720,7 +1736,7 @@ "is_script": false, "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=38e17ba8-d7a8-4e7b-afa3-95aa693da1bd" + "pkg:pypi/asgiref@3.3.0?uuid=6bcf52b9-d880-4574-b1fd-f5dabd882b75" ], "detected_license_expression": null, "detected_license_expression_spdx": null, diff --git a/scanpipe/tests/data/asgiref-3.3.0_walk_test_fixtures.json b/scanpipe/tests/data/asgiref-3.3.0_walk_test_fixtures.json index b4a732e505..dfcd1d102f 100644 --- a/scanpipe/tests/data/asgiref-3.3.0_walk_test_fixtures.json +++ b/scanpipe/tests/data/asgiref-3.3.0_walk_test_fixtures.json @@ -1,13 +1,13 @@ [ { "model": "scanpipe.project", - "pk": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e", + "pk": "83d59103-a103-4c9a-a433-618eff973b63", "fields": { "extra_data": {}, - "created_date": "2024-03-01T11:33:36.591Z", + "created_date": "2024-06-13T14:23:21.548Z", "name": "asgiref", - "slug": "asgiref-8d3058f3", - "work_directory": "/tmp/tmp3x6obwss/projects/asgiref-8d3058f3", + "slug": "asgiref-83d59103", + "work_directory": "/tmp/tmp5ihgt8lg/projects/asgiref-83d59103", "is_archived": false, "notes": "", "settings": {} @@ -15,17 +15,17 @@ }, { "model": "scanpipe.run", - "pk": "dfe51f4c-9867-462b-977c-1897531bb35f", + "pk": "ceb011c2-6d52-4c3a-bbff-c53e45f0313e", "fields": { "task_id": null, "task_start_date": null, "task_end_date": null, "task_exitcode": null, "task_output": "", - "log": "2024-03-01 11:33:36.59 Pipeline [scan_codebase] starting\n2024-03-01 11:33:36.59 Step [download_missing_inputs] starting\n2024-03-01 11:33:36.59 Step [download_missing_inputs] completed in 0 seconds\n2024-03-01 11:33:36.59 Step [copy_inputs_to_codebase_directory] starting\n2024-03-01 11:33:36.59 Step [copy_inputs_to_codebase_directory] completed in 0 seconds\n2024-03-01 11:33:36.59 Step [extract_archives] starting\n2024-03-01 11:33:36.66 Step [extract_archives] completed in 0 seconds\n2024-03-01 11:33:36.66 Step [collect_and_create_codebase_resources] starting\n2024-03-01 11:33:36.85 Step [collect_and_create_codebase_resources] completed in 0 seconds\n2024-03-01 11:33:36.85 Step [flag_empty_files] starting\n2024-03-01 11:33:36.85 Step [flag_empty_files] completed in 0 seconds\n2024-03-01 11:33:36.85 Step [flag_ignored_resources] starting\n2024-03-01 11:33:36.85 Step [flag_ignored_resources] completed in 0 seconds\n2024-03-01 11:33:36.85 Step [scan_for_application_packages] starting\n2024-03-01 11:33:36.89 Progress: 11% (2/18)\n2024-03-01 11:33:36.92 Progress: 22% (4/18)\n2024-03-01 11:33:36.93 Progress: 33% (6/18)\n2024-03-01 11:33:36.93 Progress: 44% (8/18)\n2024-03-01 11:33:36.93 Progress: 55% (10/18)\n2024-03-01 11:33:36.93 Progress: 66% (12/18)\n2024-03-01 11:33:36.93 Progress: 77% (14/18)\n2024-03-01 11:33:36.93 Progress: 88% (16/18)\n2024-03-01 11:33:39.89 Progress: 100% (18/18)\n2024-03-01 11:33:40.08 Step [scan_for_application_packages] completed in 3 seconds\n2024-03-01 11:33:40.08 Step [scan_for_files] starting\n2024-03-01 11:33:50.84 Progress: 12% (2/16) ETA: 79 seconds (1.3 minutes)\n2024-03-01 11:33:51.20 Progress: 25% (4/16) ETA: 33 seconds\n2024-03-01 11:33:51.91 Progress: 37% (6/16) ETA: 20 seconds\n2024-03-01 11:33:52.72 Progress: 50% (8/16) ETA: 13 seconds\n2024-03-01 11:33:52.97 Progress: 62% (10/16) ETA: 8 seconds\n2024-03-01 11:33:53.15 Progress: 75% (12/16) ETA: 4 seconds\n2024-03-01 11:33:53.16 Progress: 87% (14/16) ETA: 2 seconds\n2024-03-01 11:33:53.56 Progress: 100% (16/16)\n2024-03-01 11:33:53.85 Step [scan_for_files] completed in 14 seconds\n2024-03-01 11:33:53.85 Pipeline completed in 17 seconds\n", - "project": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e", + "log": "2024-06-13 14:23:21.55 Pipeline [scan_codebase] starting\n2024-06-13 14:23:21.55 Step [download_missing_inputs] starting\n2024-06-13 14:23:21.55 Step [download_missing_inputs] completed in 0 seconds\n2024-06-13 14:23:21.55 Step [copy_inputs_to_codebase_directory] starting\n2024-06-13 14:23:21.55 Step [copy_inputs_to_codebase_directory] completed in 0 seconds\n2024-06-13 14:23:21.55 Step [extract_archives] starting\n2024-06-13 14:23:21.60 Step [extract_archives] completed in 0 seconds\n2024-06-13 14:23:21.60 Step [collect_and_create_codebase_resources] starting\n2024-06-13 14:23:21.67 Step [collect_and_create_codebase_resources] completed in 0 seconds\n2024-06-13 14:23:21.67 Step [flag_empty_files] starting\n2024-06-13 14:23:21.67 Step [flag_empty_files] completed in 0 seconds\n2024-06-13 14:23:21.67 Step [flag_ignored_resources] starting\n2024-06-13 14:23:21.67 Step [flag_ignored_resources] completed in 0 seconds\n2024-06-13 14:23:21.68 Step [scan_for_application_packages] starting\n2024-06-13 14:23:21.69 Progress: 11% (2/18)\n2024-06-13 14:23:21.70 Progress: 22% (4/18)\n2024-06-13 14:23:21.71 Progress: 33% (6/18)\n2024-06-13 14:23:21.71 Progress: 44% (8/18)\n2024-06-13 14:23:21.71 Progress: 55% (10/18)\n2024-06-13 14:23:21.71 Progress: 66% (12/18)\n2024-06-13 14:23:21.71 Progress: 77% (14/18)\n2024-06-13 14:23:21.71 Progress: 88% (16/18)\n2024-06-13 14:23:24.08 Progress: 100% (18/18)\n2024-06-13 14:23:26.20 Step [scan_for_application_packages] completed in 5 seconds\n2024-06-13 14:23:26.20 Step [scan_for_files] starting\n2024-06-13 14:23:26.32 Progress: 12% (2/16) ETA: 1 seconds\n2024-06-13 14:23:26.33 Progress: 25% (4/16)\n2024-06-13 14:23:26.35 Progress: 37% (6/16)\n2024-06-13 14:23:26.56 Progress: 50% (8/16)\n2024-06-13 14:23:26.60 Progress: 62% (10/16)\n2024-06-13 14:23:26.63 Progress: 75% (12/16)\n2024-06-13 14:23:26.79 Progress: 87% (14/16)\n2024-06-13 14:23:26.88 Progress: 100% (16/16)\n2024-06-13 14:23:26.90 Step [scan_for_files] completed in 1 seconds\n2024-06-13 14:23:26.90 Pipeline completed in 5 seconds\n", + "project": "83d59103-a103-4c9a-a433-618eff973b63", "pipeline_name": "scan_codebase", - "created_date": "2024-03-01T11:33:36.594Z", + "created_date": "2024-06-13T14:23:21.551Z", "scancodeio_version": "", "description": "Scan a codebase for application packages, licenses, and copyrights.", "current_step": "", @@ -41,7 +41,7 @@ "sha256": "a5098bc870b80e7b872bff60bb363c7f2c2c89078759f6c47b53ff8c525a152e", "sha512": "", "extra_data": {}, - "project": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e", + "project": "83d59103-a103-4c9a-a433-618eff973b63", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -113,6 +113,8 @@ "Documentation": "https://asgi.readthedocs.io/", "Further Documentation": "https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions" }, + "is_private": false, + "is_virtual": false, "qualifiers": {}, "description": "ASGI specs, helper code, and adapters\nasgiref\n=======\n\n.. image:: https://api.travis-ci.org/django/asgiref.svg\n :target: https://travis-ci.org/django/asgiref\n\n.. image:: https://img.shields.io/pypi/v/asgiref.svg\n :target: https://pypi.python.org/pypi/asgiref\n\nASGI is a standard for Python asynchronous web apps and servers to communicate\nwith each other, and positioned as an asynchronous successor to WSGI. You can\nread more at https://asgi.readthedocs.io/en/latest/\n\nThis package includes ASGI base libraries, such as:\n\n* Sync-to-async and async-to-sync function wrappers, ``asgiref.sync``\n* Server base classes, ``asgiref.server``\n* A WSGI-to-ASGI adapter, in ``asgiref.wsgi``\n\n\nFunction wrappers\n-----------------\n\nThese allow you to wrap or decorate async or sync functions to call them from\nthe other style (so you can call async functions from a synchronous thread,\nor vice-versa).\n\nIn particular:\n\n* AsyncToSync lets a synchronous subthread stop and wait while the async\n function is called on the main thread's event loop, and then control is\n returned to the thread when the async function is finished.\n\n* SyncToAsync lets async code call a synchronous function, which is run in\n a threadpool and control returned to the async coroutine when the synchronous\n function completes.\n\nThe idea is to make it easier to call synchronous APIs from async code and\nasynchronous APIs from synchronous code so it's easier to transition code from\none style to the other. In the case of Channels, we wrap the (synchronous)\nDjango view system with SyncToAsync to allow it to run inside the (asynchronous)\nASGI server.\n\nNote that exactly what threads things run in is very specific, and aimed to\nkeep maximum compatibility with old synchronous code. See\n\"Synchronous code & Threads\" below for a full explanation. By default,\n``sync_to_async`` will run all synchronous code in the program in the same\nthread for safety reasons; you can disable this for more performance with\n``@sync_to_async(thread_sensitive=False)``, but make sure that your code does\nnot rely on anything bound to threads (like database connections) when you do.\n\n\nThreadlocal replacement\n-----------------------\n\nThis is a drop-in replacement for ``threading.local`` that works with both\nthreads and asyncio Tasks. Even better, it will proxy values through from a\ntask-local context to a thread-local context when you use ``sync_to_async``\nto run things in a threadpool, and vice-versa for ``async_to_sync``.\n\nIf you instead want true thread- and task-safety, you can set\n``thread_critical`` on the Local object to ensure this instead.\n\n\nServer base classes\n-------------------\n\nIncludes a ``StatelessServer`` class which provides all the hard work of\nwriting a stateless server (as in, does not handle direct incoming sockets\nbut instead consumes external streams or sockets to work out what is happening).\n\nAn example of such a server would be a chatbot server that connects out to\na central chat server and provides a \"connection scope\" per user chatting to\nit. There's only one actual connection, but the server has to separate things\ninto several scopes for easier writing of the code.\n\nYou can see an example of this being used in `frequensgi `_.\n\n\nWSGI-to-ASGI adapter\n--------------------\n\nAllows you to wrap a WSGI application so it appears as a valid ASGI application.\n\nSimply wrap it around your WSGI application like so::\n\n asgi_application = WsgiToAsgi(wsgi_application)\n\nThe WSGI application will be run in a synchronous threadpool, and the wrapped\nASGI application will be one that accepts ``http`` class messages.\n\nPlease note that not all extended features of WSGI may be supported (such as\nfile handles for incoming POST bodies).\n\n\nDependencies\n------------\n\n``asgiref`` requires Python 3.5 or higher.\n\n\nContributing\n------------\n\nPlease refer to the\n`main Channels contributing docs `_.\n\n\nTesting\n'''''''\n\nTo run tests, make sure you have installed the ``tests`` extra with the package::\n\n cd asgiref/\n pip install -e .[tests]\n pytest\n\n\nBuilding the documentation\n''''''''''''''''''''''''''\n\nThe documentation uses `Sphinx `_::\n\n cd asgiref/docs/\n pip install sphinx\n\nTo build the docs, you can use the default tools::\n\n sphinx-build -b html . _build/html # or `make html`, if you've got make set up\n cd _build/html\n python -m http.server\n\n...or you can use ``sphinx-autobuild`` to run a server and rebuild/reload\nyour documentation changes automatically::\n\n pip install sphinx-autobuild\n sphinx-autobuild . _build/html\n\n\nImplementation Details\n----------------------\n\nSynchronous code & threads\n''''''''''''''''''''''''''\n\nThe ``asgiref.sync`` module provides two wrappers that let you go between\nasynchronous and synchronous code at will, while taking care of the rough edges\nfor you.\n\nUnfortunately, the rough edges are numerous, and the code has to work especially\nhard to keep things in the same thread as much as possible. Notably, the\nrestrictions we are working with are:\n\n* All synchronous code called through ``SyncToAsync`` and marked with\n ``thread_sensitive`` should run in the same thread as each other (and if the\n outer layer of the program is synchronous, the main thread)\n\n* If a thread already has a running async loop, ``AsyncToSync`` can't run things\n on that loop if it's blocked on synchronous code that is above you in the\n call stack.\n\nThe first compromise you get to might be that ``thread_sensitive`` code should\njust run in the same thread and not spawn in a sub-thread, fulfilling the first\nrestriction, but that immediately runs you into the second restriction.\n\nThe only real solution is to essentially have a variant of ThreadPoolExecutor\nthat executes any ``thread_sensitive`` code on the outermost synchronous\nthread - either the main thread, or a single spawned subthread.\n\nThis means you now have two basic states:\n\n* If the outermost layer of your program is synchronous, then all async code\n run through ``AsyncToSync`` will run in a per-call event loop in arbitary\n sub-threads, while all ``thread_sensitive`` code will run in the main thread.\n\n* If the outermost layer of your program is asynchronous, then all async code\n runs on the main thread's event loop, and all ``thread_sensitive`` synchronous\n code will run in a single shared sub-thread.\n\nCruicially, this means that in both cases there is a thread which is a shared\nresource that all ``thread_sensitive`` code must run on, and there is a chance\nthat this thread is currently blocked on its own ``AsyncToSync`` call. Thus,\n``AsyncToSync`` needs to act as an executor for thread code while it's blocking.\n\nThe ``CurrentThreadExecutor`` class provides this functionality; rather than\nsimply waiting on a Future, you can call its ``run_until_future`` method and\nit will run submitted code until that Future is done. This means that code\ninside the call can then run code on your thread.\n\n\nMaintenance and Security\n------------------------\n\nTo report security issues, please contact security@djangoproject.com. For GPG\nsignatures and more security process information, see\nhttps://docs.djangoproject.com/en/dev/internals/security/.\n\nTo report bugs or request new features, please open a new GitHub issue.\n\nThis repository is part of the Channels project. For the shepherd and maintenance team, please see the\n`main Channels readme `_.", "notice_text": null, @@ -121,6 +123,7 @@ { "purl": "pkg:pypi/pytest", "scope": "tests", + "is_direct": true, "extra_data": {}, "is_runtime": true, "is_optional": true, @@ -131,6 +134,7 @@ { "purl": "pkg:pypi/pytest-asyncio", "scope": "tests", + "is_direct": true, "extra_data": {}, "is_runtime": true, "is_optional": true, @@ -143,7 +147,7 @@ "homepage_url": "https://github.com/django/asgiref/", "release_date": null, "code_view_url": null, - "datasource_ids": "pypi_wheel", + "datasource_id": "pypi_wheel", "file_references": [ { "md5": null, @@ -342,7 +346,7 @@ "sha256": "", "sha512": "", "extra_data": {}, - "project": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e", + "project": "83d59103-a103-4c9a-a433-618eff973b63", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -382,7 +386,7 @@ "sha256": "", "sha512": "", "extra_data": {}, - "project": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e", + "project": "83d59103-a103-4c9a-a433-618eff973b63", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -422,7 +426,7 @@ "sha256": "", "sha512": "", "extra_data": {}, - "project": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e", + "project": "83d59103-a103-4c9a-a433-618eff973b63", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -462,7 +466,7 @@ "sha256": "6e89108c2cf0c0446174188f76f60465ae1c1f14f83427807df40d52a27cb2c8", "sha512": "", "extra_data": {}, - "project": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e", + "project": "83d59103-a103-4c9a-a433-618eff973b63", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -502,7 +506,7 @@ "sha256": "b846415d1b514e9c1dff14a22deb906d794bc546ca6129f950a18cd091e2a669", "sha512": "", "extra_data": {}, - "project": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e", + "project": "83d59103-a103-4c9a-a433-618eff973b63", "detected_license_expression": "bsd-new", "detected_license_expression_spdx": "BSD-3-Clause", "license_detections": [ @@ -548,7 +552,7 @@ "authors": [], "emails": [], "urls": [], - "compliance_alert": "", + "compliance_alert": "missing", "path": "asgiref-3.3.0.whl-extract/asgiref-3.3.0.dist-info/LICENSE", "rootfs_path": "", "status": "scanned", @@ -577,7 +581,7 @@ "sha256": "70f98f4eb9f6068b192b5464fcdf69e29a8ff09962bfce84bbb052baeee44f33", "sha512": "", "extra_data": {}, - "project": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e", + "project": "83d59103-a103-4c9a-a433-618eff973b63", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -649,6 +653,8 @@ "Documentation": "https://asgi.readthedocs.io/", "Further Documentation": "https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions" }, + "is_private": false, + "is_virtual": false, "qualifiers": {}, "description": "ASGI specs, helper code, and adapters\nasgiref\n=======\n\n.. image:: https://api.travis-ci.org/django/asgiref.svg\n :target: https://travis-ci.org/django/asgiref\n\n.. image:: https://img.shields.io/pypi/v/asgiref.svg\n :target: https://pypi.python.org/pypi/asgiref\n\nASGI is a standard for Python asynchronous web apps and servers to communicate\nwith each other, and positioned as an asynchronous successor to WSGI. You can\nread more at https://asgi.readthedocs.io/en/latest/\n\nThis package includes ASGI base libraries, such as:\n\n* Sync-to-async and async-to-sync function wrappers, ``asgiref.sync``\n* Server base classes, ``asgiref.server``\n* A WSGI-to-ASGI adapter, in ``asgiref.wsgi``\n\n\nFunction wrappers\n-----------------\n\nThese allow you to wrap or decorate async or sync functions to call them from\nthe other style (so you can call async functions from a synchronous thread,\nor vice-versa).\n\nIn particular:\n\n* AsyncToSync lets a synchronous subthread stop and wait while the async\n function is called on the main thread's event loop, and then control is\n returned to the thread when the async function is finished.\n\n* SyncToAsync lets async code call a synchronous function, which is run in\n a threadpool and control returned to the async coroutine when the synchronous\n function completes.\n\nThe idea is to make it easier to call synchronous APIs from async code and\nasynchronous APIs from synchronous code so it's easier to transition code from\none style to the other. In the case of Channels, we wrap the (synchronous)\nDjango view system with SyncToAsync to allow it to run inside the (asynchronous)\nASGI server.\n\nNote that exactly what threads things run in is very specific, and aimed to\nkeep maximum compatibility with old synchronous code. See\n\"Synchronous code & Threads\" below for a full explanation. By default,\n``sync_to_async`` will run all synchronous code in the program in the same\nthread for safety reasons; you can disable this for more performance with\n``@sync_to_async(thread_sensitive=False)``, but make sure that your code does\nnot rely on anything bound to threads (like database connections) when you do.\n\n\nThreadlocal replacement\n-----------------------\n\nThis is a drop-in replacement for ``threading.local`` that works with both\nthreads and asyncio Tasks. Even better, it will proxy values through from a\ntask-local context to a thread-local context when you use ``sync_to_async``\nto run things in a threadpool, and vice-versa for ``async_to_sync``.\n\nIf you instead want true thread- and task-safety, you can set\n``thread_critical`` on the Local object to ensure this instead.\n\n\nServer base classes\n-------------------\n\nIncludes a ``StatelessServer`` class which provides all the hard work of\nwriting a stateless server (as in, does not handle direct incoming sockets\nbut instead consumes external streams or sockets to work out what is happening).\n\nAn example of such a server would be a chatbot server that connects out to\na central chat server and provides a \"connection scope\" per user chatting to\nit. There's only one actual connection, but the server has to separate things\ninto several scopes for easier writing of the code.\n\nYou can see an example of this being used in `frequensgi `_.\n\n\nWSGI-to-ASGI adapter\n--------------------\n\nAllows you to wrap a WSGI application so it appears as a valid ASGI application.\n\nSimply wrap it around your WSGI application like so::\n\n asgi_application = WsgiToAsgi(wsgi_application)\n\nThe WSGI application will be run in a synchronous threadpool, and the wrapped\nASGI application will be one that accepts ``http`` class messages.\n\nPlease note that not all extended features of WSGI may be supported (such as\nfile handles for incoming POST bodies).\n\n\nDependencies\n------------\n\n``asgiref`` requires Python 3.5 or higher.\n\n\nContributing\n------------\n\nPlease refer to the\n`main Channels contributing docs `_.\n\n\nTesting\n'''''''\n\nTo run tests, make sure you have installed the ``tests`` extra with the package::\n\n cd asgiref/\n pip install -e .[tests]\n pytest\n\n\nBuilding the documentation\n''''''''''''''''''''''''''\n\nThe documentation uses `Sphinx `_::\n\n cd asgiref/docs/\n pip install sphinx\n\nTo build the docs, you can use the default tools::\n\n sphinx-build -b html . _build/html # or `make html`, if you've got make set up\n cd _build/html\n python -m http.server\n\n...or you can use ``sphinx-autobuild`` to run a server and rebuild/reload\nyour documentation changes automatically::\n\n pip install sphinx-autobuild\n sphinx-autobuild . _build/html\n\n\nImplementation Details\n----------------------\n\nSynchronous code & threads\n''''''''''''''''''''''''''\n\nThe ``asgiref.sync`` module provides two wrappers that let you go between\nasynchronous and synchronous code at will, while taking care of the rough edges\nfor you.\n\nUnfortunately, the rough edges are numerous, and the code has to work especially\nhard to keep things in the same thread as much as possible. Notably, the\nrestrictions we are working with are:\n\n* All synchronous code called through ``SyncToAsync`` and marked with\n ``thread_sensitive`` should run in the same thread as each other (and if the\n outer layer of the program is synchronous, the main thread)\n\n* If a thread already has a running async loop, ``AsyncToSync`` can't run things\n on that loop if it's blocked on synchronous code that is above you in the\n call stack.\n\nThe first compromise you get to might be that ``thread_sensitive`` code should\njust run in the same thread and not spawn in a sub-thread, fulfilling the first\nrestriction, but that immediately runs you into the second restriction.\n\nThe only real solution is to essentially have a variant of ThreadPoolExecutor\nthat executes any ``thread_sensitive`` code on the outermost synchronous\nthread - either the main thread, or a single spawned subthread.\n\nThis means you now have two basic states:\n\n* If the outermost layer of your program is synchronous, then all async code\n run through ``AsyncToSync`` will run in a per-call event loop in arbitary\n sub-threads, while all ``thread_sensitive`` code will run in the main thread.\n\n* If the outermost layer of your program is asynchronous, then all async code\n runs on the main thread's event loop, and all ``thread_sensitive`` synchronous\n code will run in a single shared sub-thread.\n\nCruicially, this means that in both cases there is a thread which is a shared\nresource that all ``thread_sensitive`` code must run on, and there is a chance\nthat this thread is currently blocked on its own ``AsyncToSync`` call. Thus,\n``AsyncToSync`` needs to act as an executor for thread code while it's blocking.\n\nThe ``CurrentThreadExecutor`` class provides this functionality; rather than\nsimply waiting on a Future, you can call its ``run_until_future`` method and\nit will run submitted code until that Future is done. This means that code\ninside the call can then run code on your thread.\n\n\nMaintenance and Security\n------------------------\n\nTo report security issues, please contact security@djangoproject.com. For GPG\nsignatures and more security process information, see\nhttps://docs.djangoproject.com/en/dev/internals/security/.\n\nTo report bugs or request new features, please open a new GitHub issue.\n\nThis repository is part of the Channels project. For the shepherd and maintenance team, please see the\n`main Channels readme `_.", "notice_text": null, @@ -657,6 +663,7 @@ { "purl": "pkg:pypi/pytest", "scope": "tests", + "is_direct": true, "extra_data": {}, "is_runtime": true, "is_optional": true, @@ -667,6 +674,7 @@ { "purl": "pkg:pypi/pytest-asyncio", "scope": "tests", + "is_direct": true, "extra_data": {}, "is_runtime": true, "is_optional": true, @@ -679,7 +687,7 @@ "homepage_url": "https://github.com/django/asgiref/", "release_date": null, "code_view_url": null, - "datasource_ids": "pypi_wheel_metadata", + "datasource_id": "pypi_wheel_metadata", "file_references": [ { "md5": null, @@ -878,7 +886,7 @@ "sha256": "11546323af45e6a5639bf620a9c4d73e74c0bf705f494af4595007b923f75e8a", "sha512": "", "extra_data": {}, - "project": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e", + "project": "83d59103-a103-4c9a-a433-618eff973b63", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -918,7 +926,7 @@ "sha256": "2c1983592aa38f0bfb0afacc73ddc5b46ce10e8e89ceaa9fed1e5fc6361b608d", "sha512": "", "extra_data": {}, - "project": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e", + "project": "83d59103-a103-4c9a-a433-618eff973b63", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -958,7 +966,7 @@ "sha256": "30f49b9094bff904a42caeec32515715fe625a56dc48bd7c0e3d9988c0ad4bd7", "sha512": "", "extra_data": {}, - "project": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e", + "project": "83d59103-a103-4c9a-a433-618eff973b63", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -998,7 +1006,7 @@ "sha256": "fa4651a3b79201a4dc44a4096cd49ec8f427e912ea0ee05c666357b413a8afe7", "sha512": "", "extra_data": {}, - "project": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e", + "project": "83d59103-a103-4c9a-a433-618eff973b63", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -1038,7 +1046,7 @@ "sha256": "ee0fcf4a8e6fa9df8a4643bb48e82892d496afce44b6c8b8aea2721755545e1c", "sha512": "", "extra_data": {}, - "project": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e", + "project": "83d59103-a103-4c9a-a433-618eff973b63", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -1078,7 +1086,7 @@ "sha256": "3151f66c476208c3154cb6c4fb557a2a253bab82f0ab33fb3c8b9f7976be9e33", "sha512": "", "extra_data": {}, - "project": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e", + "project": "83d59103-a103-4c9a-a433-618eff973b63", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -1118,7 +1126,7 @@ "sha256": "ddd445b778c097fc75c2bf69ad964cbadd3bd6999d1dd2306d39d401855e8e3e", "sha512": "", "extra_data": {}, - "project": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e", + "project": "83d59103-a103-4c9a-a433-618eff973b63", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -1158,7 +1166,7 @@ "sha256": "ddbc8d455eceb68fc583c67e7c4ad0277c867fb39095c51ec5b37f70342e8334", "sha512": "", "extra_data": {}, - "project": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e", + "project": "83d59103-a103-4c9a-a433-618eff973b63", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -1198,7 +1206,7 @@ "sha256": "126c3e3a8a75a517d2739612304607804cf5f34da63fa25d03a6f11f7edb6f2f", "sha512": "", "extra_data": {}, - "project": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e", + "project": "83d59103-a103-4c9a-a433-618eff973b63", "detected_license_expression": "apache-2.0", "detected_license_expression_spdx": "Apache-2.0", "license_detections": [ @@ -1248,7 +1256,7 @@ "start_line": 83 } ], - "compliance_alert": "", + "compliance_alert": "missing", "path": "asgiref-3.3.0.whl-extract/asgiref/timeout.py", "rootfs_path": "", "status": "scanned", @@ -1277,7 +1285,7 @@ "sha256": "f8bd1ea3fb8afddabb10f8efd66796d41446cad51168ef4d3c44b19c973d0ad0", "sha512": "", "extra_data": {}, - "project": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e", + "project": "83d59103-a103-4c9a-a433-618eff973b63", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -1317,7 +1325,7 @@ "sha256": "885267fee0fea687875a02ceb929ca095312d47aaa57e20e4ce382f397caaf4d", "sha512": "", "extra_data": {}, - "project": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e", + "project": "83d59103-a103-4c9a-a433-618eff973b63", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -1367,8 +1375,8 @@ "Documentation": "https://asgi.readthedocs.io/", "Further Documentation": "https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions" }, - "project": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e", - "compliance_alert": "", + "project": "83d59103-a103-4c9a-a433-618eff973b63", + "compliance_alert": "missing", "affected_by_vulnerabilities": [], "filename": "", "primary_language": "Python", @@ -1438,7 +1446,14 @@ "other_license_detections": [], "extracted_license_statement": "license: BSD\nclassifiers:\n - 'License :: OSI Approved :: BSD License'\n", "notice_text": "", - "datasource_ids": "", + "is_private": false, + "is_virtual": false, + "datasource_ids": [ + "pypi_wheel" + ], + "datafile_paths": [ + "asgiref-3.3.0-py3-none-any.whl" + ], "file_references": [], "parties": [ { @@ -1449,10 +1464,10 @@ "email": "foundation@djangoproject.com" } ], - "uuid": "b6ef7c90-e3d4-4008-8b67-63f086cea2da", + "uuid": "a9dcd442-4c55-49b4-9e33-8be74c9fc15b", "missing_resources": [], "modified_resources": [], - "package_uid": "pkg:pypi/asgiref@3.3.0?uuid=90cd6d93-1c46-477a-bd4a-47aa6b3ce773", + "package_uid": "pkg:pypi/asgiref@3.3.0?uuid=eca680a3-115b-494b-8581-81c93384e22d", "keywords": [ "Development Status :: 5 - Production/Stable", "Environment :: Web Environment", @@ -1493,8 +1508,8 @@ "Documentation": "https://asgi.readthedocs.io/", "Further Documentation": "https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions" }, - "project": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e", - "compliance_alert": "", + "project": "83d59103-a103-4c9a-a433-618eff973b63", + "compliance_alert": "missing", "affected_by_vulnerabilities": [], "filename": "", "primary_language": "Python", @@ -1564,7 +1579,14 @@ "other_license_detections": [], "extracted_license_statement": "license: BSD\nclassifiers:\n - 'License :: OSI Approved :: BSD License'\n", "notice_text": "", - "datasource_ids": "", + "is_private": false, + "is_virtual": false, + "datasource_ids": [ + "pypi_wheel_metadata" + ], + "datafile_paths": [ + "asgiref-3.3.0-py3-none-any.whl-extract/asgiref-3.3.0.dist-info/METADATA" + ], "file_references": [], "parties": [ { @@ -1575,10 +1597,10 @@ "email": "foundation@djangoproject.com" } ], - "uuid": "b2d24c22-0dff-4e3f-8332-413b4f4852a7", + "uuid": "27357e59-c16d-40f1-8cd2-0f11b7376cdb", "missing_resources": [], "modified_resources": [], - "package_uid": "pkg:pypi/asgiref@3.3.0?uuid=fb05a7ee-1078-4d60-bf71-2ba031414f1c", + "package_uid": "pkg:pypi/asgiref@3.3.0?uuid=510ea8b0-d7a7-4531-96ea-f41cb4384976", "keywords": [ "Development Status :: 5 - Production/Stable", "Environment :: Web Environment", @@ -1624,16 +1646,18 @@ "qualifiers": "", "subpath": "", "affected_by_vulnerabilities": [], - "project": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e", - "dependency_uid": "pkg:pypi/pytest?uuid=33c92b36-8293-4fe5-8094-0eb645833284", + "project": "83d59103-a103-4c9a-a433-618eff973b63", + "dependency_uid": "pkg:pypi/pytest?uuid=9f487b4c-4d3e-4d13-9100-89ab897c90ee", "for_package": 1, + "resolved_to_package": null, "datafile_resource": 1, "extracted_requirement": "pytest; extra == \"tests\"", "scope": "tests", "datasource_id": "pypi_wheel", "is_runtime": true, "is_optional": true, - "is_resolved": false + "is_resolved": false, + "is_direct": true } }, { @@ -1647,16 +1671,18 @@ "qualifiers": "", "subpath": "", "affected_by_vulnerabilities": [], - "project": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e", - "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=bf9520e1-0b6a-4c00-90c4-b5e7afb6de3e", + "project": "83d59103-a103-4c9a-a433-618eff973b63", + "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=7c26cfc3-080d-4d3c-bcfe-fd6a495c5d3a", "for_package": 1, + "resolved_to_package": null, "datafile_resource": 1, "extracted_requirement": "pytest-asyncio; extra == \"tests\"", "scope": "tests", "datasource_id": "pypi_wheel", "is_runtime": true, "is_optional": true, - "is_resolved": false + "is_resolved": false, + "is_direct": true } }, { @@ -1670,16 +1696,18 @@ "qualifiers": "", "subpath": "", "affected_by_vulnerabilities": [], - "project": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e", - "dependency_uid": "pkg:pypi/pytest?uuid=21776640-36c9-4111-a6c4-ea5a550c85e0", + "project": "83d59103-a103-4c9a-a433-618eff973b63", + "dependency_uid": "pkg:pypi/pytest?uuid=909f72ab-1d6b-41f6-b41a-2b3be388870a", "for_package": 2, + "resolved_to_package": null, "datafile_resource": 7, "extracted_requirement": "pytest; extra == \"tests\"", "scope": "tests", "datasource_id": "pypi_wheel_metadata", "is_runtime": true, "is_optional": true, - "is_resolved": false + "is_resolved": false, + "is_direct": true } }, { @@ -1693,16 +1721,18 @@ "qualifiers": "", "subpath": "", "affected_by_vulnerabilities": [], - "project": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e", - "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=1963fa7d-3e64-4975-b4e7-80492466fefb", + "project": "83d59103-a103-4c9a-a433-618eff973b63", + "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=e474c205-ceef-444a-a0d7-aee32c8716df", "for_package": 2, + "resolved_to_package": null, "datafile_resource": 7, "extracted_requirement": "pytest-asyncio; extra == \"tests\"", "scope": "tests", "datasource_id": "pypi_wheel_metadata", "is_runtime": true, "is_optional": true, - "is_resolved": false + "is_resolved": false, + "is_direct": true } } ] \ No newline at end of file diff --git a/scanpipe/tests/data/basic-rootfs_root_filesystems.json b/scanpipe/tests/data/basic-rootfs_root_filesystems.json index 53122fa5d5..244fd9f0dc 100644 --- a/scanpipe/tests/data/basic-rootfs_root_filesystems.json +++ b/scanpipe/tests/data/basic-rootfs_root_filesystems.json @@ -199,6 +199,8 @@ "multi_arch": "same" }, "package_uid": "pkg:deb/ubuntu/libncurses5@6.1-1ubuntu1.18.04?arch=amd64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "debian_installed_status_db", "debian_copyright_in_package" @@ -317,6 +319,8 @@ "multi_arch": "same" }, "package_uid": "pkg:deb/ubuntu/libndp0@1.4-2ubuntu0.16.04.1?arch=amd64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "debian_installed_status_db", "debian_copyright_in_package" diff --git a/scanpipe/tests/data/centos_scan_codebase.json b/scanpipe/tests/data/centos_scan_codebase.json index f6717dc9ba..cba4477db1 100644 --- a/scanpipe/tests/data/centos_scan_codebase.json +++ b/scanpipe/tests/data/centos_scan_codebase.json @@ -293,6 +293,8 @@ ] }, "package_uid": "pkg:rpm/audit-libs@3.0?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -365,6 +367,8 @@ "source_packages": [], "extra_data": {}, "package_uid": "pkg:rpm/basesystem@11?arch=noarch&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -1480,6 +1484,8 @@ ] }, "package_uid": "pkg:rpm/bash@4.4.19?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -1699,6 +1705,8 @@ ] }, "package_uid": "pkg:rpm/brotli@1.0.6?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -1830,6 +1838,8 @@ ] }, "package_uid": "pkg:rpm/bzip2-libs@1.0.6?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -2265,6 +2275,8 @@ ] }, "package_uid": "pkg:rpm/ca-certificates@2018.2.24?arch=noarch&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -3204,6 +3216,8 @@ ] }, "package_uid": "pkg:rpm/chkconfig@1.11?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -4183,6 +4197,8 @@ ] }, "package_uid": "pkg:rpm/coreutils-single@8.30?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -4874,6 +4890,8 @@ ] }, "package_uid": "pkg:rpm/crypto-policies@20181217?arch=noarch&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -5085,6 +5103,8 @@ ] }, "package_uid": "pkg:rpm/curl@7.61.1?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -5456,6 +5476,8 @@ ] }, "package_uid": "pkg:rpm/cyrus-sasl-lib@2.1.27?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -5666,6 +5688,8 @@ ] }, "package_uid": "pkg:rpm/elfutils-libelf@0.174?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -5853,6 +5877,8 @@ ] }, "package_uid": "pkg:rpm/expat@2.2.5?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -143960,6 +143986,8 @@ ] }, "package_uid": "pkg:rpm/filesystem@3.8?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -145043,6 +145071,8 @@ ] }, "package_uid": "pkg:rpm/gawk@4.2.1?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -146390,6 +146420,8 @@ ] }, "package_uid": "pkg:rpm/glib2@2.56.4?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -152621,6 +152653,8 @@ ] }, "package_uid": "pkg:rpm/glibc@2.28?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -153188,6 +153222,8 @@ ] }, "package_uid": "pkg:rpm/glibc-common@2.28?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -153320,6 +153356,8 @@ "source_packages": [], "extra_data": {}, "package_uid": "pkg:rpm/glibc-minimal-langpack@2.28?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -153530,6 +153568,8 @@ ] }, "package_uid": "pkg:rpm/gmp@6.1.2?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -155021,6 +155061,8 @@ ] }, "package_uid": "pkg:rpm/gnupg2@2.2.9?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -155367,6 +155409,8 @@ ] }, "package_uid": "pkg:rpm/gnutls@3.6.5?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -155657,6 +155701,8 @@ ] }, "package_uid": "pkg:rpm/gobject-introspection@1.56.1?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -155860,6 +155906,8 @@ ] }, "package_uid": "pkg:rpm/gpgme@1.10.0?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -155932,6 +155980,8 @@ "source_packages": [], "extra_data": {}, "package_uid": "pkg:rpm/gpg-pubkey@d4082792?uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -156004,6 +156054,8 @@ "source_packages": [], "extra_data": {}, "package_uid": "pkg:rpm/gpg-pubkey@fd431d51?uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -156599,6 +156651,8 @@ ] }, "package_uid": "pkg:rpm/grep@3.1?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -156810,6 +156864,8 @@ ] }, "package_uid": "pkg:rpm/info@6.5?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -156957,6 +157013,8 @@ ] }, "package_uid": "pkg:rpm/json-c@0.13.1?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -157552,6 +157610,8 @@ ] }, "package_uid": "pkg:rpm/json-glib@1.4.4?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -157706,6 +157766,8 @@ ] }, "package_uid": "pkg:rpm/keyutils-libs@1.5.10?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -158277,6 +158339,8 @@ ] }, "package_uid": "pkg:rpm/krb5-libs@1.16.1?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -158360,6 +158424,8 @@ ] }, "package_uid": "pkg:rpm/langpacks-en@1.0?arch=noarch&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -158475,6 +158541,8 @@ ] }, "package_uid": "pkg:rpm/libacl@2.2.53?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -158654,6 +158722,8 @@ ] }, "package_uid": "pkg:rpm/libarchive@3.3.2?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -158864,6 +158934,8 @@ ] }, "package_uid": "pkg:rpm/libassuan@2.5.1?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -158987,6 +159059,8 @@ ] }, "package_uid": "pkg:rpm/libattr@2.4.48?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -159118,6 +159192,8 @@ ] }, "package_uid": "pkg:rpm/libblkid@2.32.1?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -159417,6 +159493,8 @@ ] }, "package_uid": "pkg:rpm/libcap@2.25?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -159548,6 +159626,8 @@ ] }, "package_uid": "pkg:rpm/libcap-ng@0.7.9?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -159679,6 +159759,8 @@ ] }, "package_uid": "pkg:rpm/libcom_err@1.44.3?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -159810,6 +159892,8 @@ ] }, "package_uid": "pkg:rpm/libcurl@7.61.1?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -159965,6 +160049,8 @@ ] }, "package_uid": "pkg:rpm/libdb@5.3.28?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -160528,6 +160614,8 @@ ] }, "package_uid": "pkg:rpm/libdb-utils@5.3.28?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -161075,6 +161163,8 @@ ] }, "package_uid": "pkg:rpm/libdnf@0.22.5?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -161222,6 +161312,8 @@ ] }, "package_uid": "pkg:rpm/libffi@3.1?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -161430,6 +161522,8 @@ ] }, "package_uid": "pkg:rpm/libgcc@8.2.1?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -161617,6 +161711,8 @@ ] }, "package_uid": "pkg:rpm/libgcrypt@1.8.3?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -161996,6 +162092,8 @@ ] }, "package_uid": "pkg:rpm/libgpg-error@1.31?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -162389,6 +162487,8 @@ ] }, "package_uid": "pkg:rpm/libidn2@2.0.5?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -162630,6 +162730,8 @@ ] }, "package_uid": "pkg:rpm/libksba@1.3.5?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -162777,6 +162879,8 @@ ] }, "package_uid": "pkg:rpm/libmetalink@0.1.3?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -162964,6 +163068,8 @@ ] }, "package_uid": "pkg:rpm/libmodulemd1@1.8.0?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -163095,6 +163201,8 @@ ] }, "package_uid": "pkg:rpm/libmount@2.32.1?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -163226,6 +163334,8 @@ ] }, "package_uid": "pkg:rpm/libnghttp2@1.33.0?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -163997,6 +164107,8 @@ ] }, "package_uid": "pkg:rpm/libpeas@1.22.0?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -164128,6 +164240,8 @@ ] }, "package_uid": "pkg:rpm/libpsl@0.20.2?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -164267,6 +164381,8 @@ ] }, "package_uid": "pkg:rpm/librepo@1.9.2?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -164406,6 +164522,8 @@ ] }, "package_uid": "pkg:rpm/librhsm@0.0.3?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -164569,6 +164687,8 @@ ] }, "package_uid": "pkg:rpm/libselinux@2.8?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -164692,6 +164812,8 @@ ] }, "package_uid": "pkg:rpm/libsepol@2.8?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -164855,6 +164977,8 @@ ] }, "package_uid": "pkg:rpm/libsigsegv@2.11?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -164994,6 +165118,8 @@ ] }, "package_uid": "pkg:rpm/libsmartcols@2.32.1?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -165157,6 +165283,8 @@ ] }, "package_uid": "pkg:rpm/libsolv@0.6.35?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -165344,6 +165472,8 @@ ] }, "package_uid": "pkg:rpm/libssh@0.8.5?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -165712,6 +165842,8 @@ ] }, "package_uid": "pkg:rpm/libstdc%2B%2B@8.2.1?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -165906,6 +166038,8 @@ ] }, "package_uid": "pkg:rpm/libtasn1@4.13?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -166092,6 +166226,8 @@ ] }, "package_uid": "pkg:rpm/libunistring@0.9.9?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -166255,6 +166391,8 @@ ] }, "package_uid": "pkg:rpm/libusbx@1.0.22?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -166386,6 +166524,8 @@ ] }, "package_uid": "pkg:rpm/libuuid@2.32.1?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -166557,6 +166697,8 @@ ] }, "package_uid": "pkg:rpm/libverto@0.3.0?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -166775,6 +166917,8 @@ ] }, "package_uid": "pkg:rpm/libxcrypt@4.1.1?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -167018,6 +167162,8 @@ ] }, "package_uid": "pkg:rpm/libxml2@2.9.7?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -167165,6 +167311,8 @@ ] }, "package_uid": "pkg:rpm/libyaml@0.1.7?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -167272,6 +167420,8 @@ ] }, "package_uid": "pkg:rpm/lua-libs@5.3.4?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -167403,6 +167553,8 @@ ] }, "package_uid": "pkg:rpm/lz4-libs@1.8.1.2?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -167542,6 +167694,8 @@ ] }, "package_uid": "pkg:rpm/microdnf@3.0.1?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -167752,6 +167906,8 @@ ] }, "package_uid": "pkg:rpm/mpfr@3.1.6?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -169155,6 +169311,8 @@ ] }, "package_uid": "pkg:rpm/ncurses-base@6.1?arch=noarch&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -169582,6 +169740,8 @@ ] }, "package_uid": "pkg:rpm/ncurses-libs@6.1?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -169848,6 +170008,8 @@ ] }, "package_uid": "pkg:rpm/nettle@3.4.1?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -169979,6 +170141,8 @@ ] }, "package_uid": "pkg:rpm/npth@1.5?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -170294,6 +170458,8 @@ ] }, "package_uid": "pkg:rpm/openldap@2.4.46?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -170617,6 +170783,8 @@ ] }, "package_uid": "pkg:rpm/openssl-libs@1.1.1?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -170908,6 +171076,8 @@ ] }, "package_uid": "pkg:rpm/p11-kit@0.23.14?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -171071,6 +171241,8 @@ ] }, "package_uid": "pkg:rpm/p11-kit-trust@0.23.14?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -171266,6 +171438,8 @@ ] }, "package_uid": "pkg:rpm/pcre@8.42?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -171469,6 +171643,8 @@ ] }, "package_uid": "pkg:rpm/pcre2@10.32?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -171880,6 +172056,8 @@ ] }, "package_uid": "pkg:rpm/popt@1.16?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -171987,6 +172165,8 @@ ] }, "package_uid": "pkg:rpm/publicsuffix-list-dafsa@20180723?arch=noarch&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -172174,6 +172354,8 @@ ] }, "package_uid": "pkg:rpm/readline@7.0?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -172441,6 +172623,8 @@ ] }, "package_uid": "pkg:rpm/redhat-release@8.0?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -174500,6 +174684,8 @@ ] }, "package_uid": "pkg:rpm/rpm@4.14.2?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -174662,6 +174848,8 @@ ] }, "package_uid": "pkg:rpm/rpm-libs@4.14.2?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -175169,6 +175357,8 @@ ] }, "package_uid": "pkg:rpm/sed@4.5?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -175540,6 +175730,8 @@ ] }, "package_uid": "pkg:rpm/setup@2.12.2?arch=noarch&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -175671,6 +175863,8 @@ ] }, "package_uid": "pkg:rpm/sqlite-libs@3.26.0?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -175914,6 +176108,8 @@ ] }, "package_uid": "pkg:rpm/systemd-libs@239?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -190829,6 +191025,8 @@ ] }, "package_uid": "pkg:rpm/tzdata@2019a?arch=noarch&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -190952,6 +191150,8 @@ ] }, "package_uid": "pkg:rpm/xz-libs@5.2.4?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], @@ -191107,6 +191307,8 @@ ] }, "package_uid": "pkg:rpm/zlib@1.2.11?arch=x86_64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "rpm_installed_database_bdb" ], diff --git a/scanpipe/tests/data/cyclonedx/asgiref-3.3.0.cdx.json b/scanpipe/tests/data/cyclonedx/asgiref-3.3.0.cdx.json index 355ad72367..1146f3e070 100644 --- a/scanpipe/tests/data/cyclonedx/asgiref-3.3.0.cdx.json +++ b/scanpipe/tests/data/cyclonedx/asgiref-3.3.0.cdx.json @@ -6,7 +6,7 @@ "version": 1, "metadata": { "component": { - "bom-ref": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e", + "bom-ref": "83d59103-a103-4c9a-a433-618eff973b63", "name": "asgiref", "type": "library" }, @@ -26,7 +26,7 @@ }, "components": [ { - "bom-ref": "pkg:pypi/asgiref@3.3.0?uuid=90cd6d93-1c46-477a-bd4a-47aa6b3ce773", + "bom-ref": "pkg:pypi/asgiref@3.3.0?uuid=eca680a3-115b-494b-8581-81c93384e22d", "copyright": "", "description": "ASGI specs, helper code, and adapters\nasgiref\n=======\n\n.. image:: https://api.travis-ci.org/django/asgiref.svg\n :target: https://travis-ci.org/django/asgiref\n\n.. image:: https://img.shields.io/pypi/v/asgiref.svg\n :target: https://pypi.python.org/pypi/asgiref\n\nASGI is a standard for Python asynchronous web apps and servers to communicate\nwith each other, and positioned as an asynchronous successor to WSGI. You can\nread more at https://asgi.readthedocs.io/en/latest/\n\nThis package includes ASGI base libraries, such as:\n\n* Sync-to-async and async-to-sync function wrappers, ``asgiref.sync``\n* Server base classes, ``asgiref.server``\n* A WSGI-to-ASGI adapter, in ``asgiref.wsgi``\n\n\nFunction wrappers\n-----------------\n\nThese allow you to wrap or decorate async or sync functions to call them from\nthe other style (so you can call async functions from a synchronous thread,\nor vice-versa).\n\nIn particular:\n\n* AsyncToSync lets a synchronous subthread stop and wait while the async\n function is called on the main thread's event loop, and then control is\n returned to the thread when the async function is finished.\n\n* SyncToAsync lets async code call a synchronous function, which is run in\n a threadpool and control returned to the async coroutine when the synchronous\n function completes.\n\nThe idea is to make it easier to call synchronous APIs from async code and\nasynchronous APIs from synchronous code so it's easier to transition code from\none style to the other. In the case of Channels, we wrap the (synchronous)\nDjango view system with SyncToAsync to allow it to run inside the (asynchronous)\nASGI server.\n\nNote that exactly what threads things run in is very specific, and aimed to\nkeep maximum compatibility with old synchronous code. See\n\"Synchronous code & Threads\" below for a full explanation. By default,\n``sync_to_async`` will run all synchronous code in the program in the same\nthread for safety reasons; you can disable this for more performance with\n``@sync_to_async(thread_sensitive=False)``, but make sure that your code does\nnot rely on anything bound to threads (like database connections) when you do.\n\n\nThreadlocal replacement\n-----------------------\n\nThis is a drop-in replacement for ``threading.local`` that works with both\nthreads and asyncio Tasks. Even better, it will proxy values through from a\ntask-local context to a thread-local context when you use ``sync_to_async``\nto run things in a threadpool, and vice-versa for ``async_to_sync``.\n\nIf you instead want true thread- and task-safety, you can set\n``thread_critical`` on the Local object to ensure this instead.\n\n\nServer base classes\n-------------------\n\nIncludes a ``StatelessServer`` class which provides all the hard work of\nwriting a stateless server (as in, does not handle direct incoming sockets\nbut instead consumes external streams or sockets to work out what is happening).\n\nAn example of such a server would be a chatbot server that connects out to\na central chat server and provides a \"connection scope\" per user chatting to\nit. There's only one actual connection, but the server has to separate things\ninto several scopes for easier writing of the code.\n\nYou can see an example of this being used in `frequensgi `_.\n\n\nWSGI-to-ASGI adapter\n--------------------\n\nAllows you to wrap a WSGI application so it appears as a valid ASGI application.\n\nSimply wrap it around your WSGI application like so::\n\n asgi_application = WsgiToAsgi(wsgi_application)\n\nThe WSGI application will be run in a synchronous threadpool, and the wrapped\nASGI application will be one that accepts ``http`` class messages.\n\nPlease note that not all extended features of WSGI may be supported (such as\nfile handles for incoming POST bodies).\n\n\nDependencies\n------------\n\n``asgiref`` requires Python 3.5 or higher.\n\n\nContributing\n------------\n\nPlease refer to the\n`main Channels contributing docs `_.\n\n\nTesting\n'''''''\n\nTo run tests, make sure you have installed the ``tests`` extra with the package::\n\n cd asgiref/\n pip install -e .[tests]\n pytest\n\n\nBuilding the documentation\n''''''''''''''''''''''''''\n\nThe documentation uses `Sphinx `_::\n\n cd asgiref/docs/\n pip install sphinx\n\nTo build the docs, you can use the default tools::\n\n sphinx-build -b html . _build/html # or `make html`, if you've got make set up\n cd _build/html\n python -m http.server\n\n...or you can use ``sphinx-autobuild`` to run a server and rebuild/reload\nyour documentation changes automatically::\n\n pip install sphinx-autobuild\n sphinx-autobuild . _build/html\n\n\nImplementation Details\n----------------------\n\nSynchronous code & threads\n''''''''''''''''''''''''''\n\nThe ``asgiref.sync`` module provides two wrappers that let you go between\nasynchronous and synchronous code at will, while taking care of the rough edges\nfor you.\n\nUnfortunately, the rough edges are numerous, and the code has to work especially\nhard to keep things in the same thread as much as possible. Notably, the\nrestrictions we are working with are:\n\n* All synchronous code called through ``SyncToAsync`` and marked with\n ``thread_sensitive`` should run in the same thread as each other (and if the\n outer layer of the program is synchronous, the main thread)\n\n* If a thread already has a running async loop, ``AsyncToSync`` can't run things\n on that loop if it's blocked on synchronous code that is above you in the\n call stack.\n\nThe first compromise you get to might be that ``thread_sensitive`` code should\njust run in the same thread and not spawn in a sub-thread, fulfilling the first\nrestriction, but that immediately runs you into the second restriction.\n\nThe only real solution is to essentially have a variant of ThreadPoolExecutor\nthat executes any ``thread_sensitive`` code on the outermost synchronous\nthread - either the main thread, or a single spawned subthread.\n\nThis means you now have two basic states:\n\n* If the outermost layer of your program is synchronous, then all async code\n run through ``AsyncToSync`` will run in a per-call event loop in arbitary\n sub-threads, while all ``thread_sensitive`` code will run in the main thread.\n\n* If the outermost layer of your program is asynchronous, then all async code\n runs on the main thread's event loop, and all ``thread_sensitive`` synchronous\n code will run in a single shared sub-thread.\n\nCruicially, this means that in both cases there is a thread which is a shared\nresource that all ``thread_sensitive`` code must run on, and there is a chance\nthat this thread is currently blocked on its own ``AsyncToSync`` call. Thus,\n``AsyncToSync`` needs to act as an executor for thread code while it's blocking.\n\nThe ``CurrentThreadExecutor`` class provides this functionality; rather than\nsimply waiting on a Future, you can call its ``run_until_future`` method and\nit will run submitted code until that Future is done. This means that code\ninside the call can then run code on your thread.\n\n\nMaintenance and Security\n------------------------\n\nTo report security issues, please contact security@djangoproject.com. For GPG\nsignatures and more security process information, see\nhttps://docs.djangoproject.com/en/dev/internals/security/.\n\nTo report bugs or request new features, please open a new GitHub issue.\n\nThis repository is part of the Channels project. For the shepherd and maintenance team, please see the\n`main Channels readme `_.", "externalReferences": [ @@ -64,7 +64,7 @@ "version": "3.3.0" }, { - "bom-ref": "pkg:pypi/asgiref@3.3.0?uuid=fb05a7ee-1078-4d60-bf71-2ba031414f1c", + "bom-ref": "pkg:pypi/asgiref@3.3.0?uuid=510ea8b0-d7a7-4531-96ea-f41cb4384976", "copyright": "", "description": "ASGI specs, helper code, and adapters\nasgiref\n=======\n\n.. image:: https://api.travis-ci.org/django/asgiref.svg\n :target: https://travis-ci.org/django/asgiref\n\n.. image:: https://img.shields.io/pypi/v/asgiref.svg\n :target: https://pypi.python.org/pypi/asgiref\n\nASGI is a standard for Python asynchronous web apps and servers to communicate\nwith each other, and positioned as an asynchronous successor to WSGI. You can\nread more at https://asgi.readthedocs.io/en/latest/\n\nThis package includes ASGI base libraries, such as:\n\n* Sync-to-async and async-to-sync function wrappers, ``asgiref.sync``\n* Server base classes, ``asgiref.server``\n* A WSGI-to-ASGI adapter, in ``asgiref.wsgi``\n\n\nFunction wrappers\n-----------------\n\nThese allow you to wrap or decorate async or sync functions to call them from\nthe other style (so you can call async functions from a synchronous thread,\nor vice-versa).\n\nIn particular:\n\n* AsyncToSync lets a synchronous subthread stop and wait while the async\n function is called on the main thread's event loop, and then control is\n returned to the thread when the async function is finished.\n\n* SyncToAsync lets async code call a synchronous function, which is run in\n a threadpool and control returned to the async coroutine when the synchronous\n function completes.\n\nThe idea is to make it easier to call synchronous APIs from async code and\nasynchronous APIs from synchronous code so it's easier to transition code from\none style to the other. In the case of Channels, we wrap the (synchronous)\nDjango view system with SyncToAsync to allow it to run inside the (asynchronous)\nASGI server.\n\nNote that exactly what threads things run in is very specific, and aimed to\nkeep maximum compatibility with old synchronous code. See\n\"Synchronous code & Threads\" below for a full explanation. By default,\n``sync_to_async`` will run all synchronous code in the program in the same\nthread for safety reasons; you can disable this for more performance with\n``@sync_to_async(thread_sensitive=False)``, but make sure that your code does\nnot rely on anything bound to threads (like database connections) when you do.\n\n\nThreadlocal replacement\n-----------------------\n\nThis is a drop-in replacement for ``threading.local`` that works with both\nthreads and asyncio Tasks. Even better, it will proxy values through from a\ntask-local context to a thread-local context when you use ``sync_to_async``\nto run things in a threadpool, and vice-versa for ``async_to_sync``.\n\nIf you instead want true thread- and task-safety, you can set\n``thread_critical`` on the Local object to ensure this instead.\n\n\nServer base classes\n-------------------\n\nIncludes a ``StatelessServer`` class which provides all the hard work of\nwriting a stateless server (as in, does not handle direct incoming sockets\nbut instead consumes external streams or sockets to work out what is happening).\n\nAn example of such a server would be a chatbot server that connects out to\na central chat server and provides a \"connection scope\" per user chatting to\nit. There's only one actual connection, but the server has to separate things\ninto several scopes for easier writing of the code.\n\nYou can see an example of this being used in `frequensgi `_.\n\n\nWSGI-to-ASGI adapter\n--------------------\n\nAllows you to wrap a WSGI application so it appears as a valid ASGI application.\n\nSimply wrap it around your WSGI application like so::\n\n asgi_application = WsgiToAsgi(wsgi_application)\n\nThe WSGI application will be run in a synchronous threadpool, and the wrapped\nASGI application will be one that accepts ``http`` class messages.\n\nPlease note that not all extended features of WSGI may be supported (such as\nfile handles for incoming POST bodies).\n\n\nDependencies\n------------\n\n``asgiref`` requires Python 3.5 or higher.\n\n\nContributing\n------------\n\nPlease refer to the\n`main Channels contributing docs `_.\n\n\nTesting\n'''''''\n\nTo run tests, make sure you have installed the ``tests`` extra with the package::\n\n cd asgiref/\n pip install -e .[tests]\n pytest\n\n\nBuilding the documentation\n''''''''''''''''''''''''''\n\nThe documentation uses `Sphinx `_::\n\n cd asgiref/docs/\n pip install sphinx\n\nTo build the docs, you can use the default tools::\n\n sphinx-build -b html . _build/html # or `make html`, if you've got make set up\n cd _build/html\n python -m http.server\n\n...or you can use ``sphinx-autobuild`` to run a server and rebuild/reload\nyour documentation changes automatically::\n\n pip install sphinx-autobuild\n sphinx-autobuild . _build/html\n\n\nImplementation Details\n----------------------\n\nSynchronous code & threads\n''''''''''''''''''''''''''\n\nThe ``asgiref.sync`` module provides two wrappers that let you go between\nasynchronous and synchronous code at will, while taking care of the rough edges\nfor you.\n\nUnfortunately, the rough edges are numerous, and the code has to work especially\nhard to keep things in the same thread as much as possible. Notably, the\nrestrictions we are working with are:\n\n* All synchronous code called through ``SyncToAsync`` and marked with\n ``thread_sensitive`` should run in the same thread as each other (and if the\n outer layer of the program is synchronous, the main thread)\n\n* If a thread already has a running async loop, ``AsyncToSync`` can't run things\n on that loop if it's blocked on synchronous code that is above you in the\n call stack.\n\nThe first compromise you get to might be that ``thread_sensitive`` code should\njust run in the same thread and not spawn in a sub-thread, fulfilling the first\nrestriction, but that immediately runs you into the second restriction.\n\nThe only real solution is to essentially have a variant of ThreadPoolExecutor\nthat executes any ``thread_sensitive`` code on the outermost synchronous\nthread - either the main thread, or a single spawned subthread.\n\nThis means you now have two basic states:\n\n* If the outermost layer of your program is synchronous, then all async code\n run through ``AsyncToSync`` will run in a per-call event loop in arbitary\n sub-threads, while all ``thread_sensitive`` code will run in the main thread.\n\n* If the outermost layer of your program is asynchronous, then all async code\n runs on the main thread's event loop, and all ``thread_sensitive`` synchronous\n code will run in a single shared sub-thread.\n\nCruicially, this means that in both cases there is a thread which is a shared\nresource that all ``thread_sensitive`` code must run on, and there is a chance\nthat this thread is currently blocked on its own ``AsyncToSync`` call. Thus,\n``AsyncToSync`` needs to act as an executor for thread code while it's blocking.\n\nThe ``CurrentThreadExecutor`` class provides this functionality; rather than\nsimply waiting on a Future, you can call its ``run_until_future`` method and\nit will run submitted code until that Future is done. This means that code\ninside the call can then run code on your thread.\n\n\nMaintenance and Security\n------------------------\n\nTo report security issues, please contact security@djangoproject.com. For GPG\nsignatures and more security process information, see\nhttps://docs.djangoproject.com/en/dev/internals/security/.\n\nTo report bugs or request new features, please open a new GitHub issue.\n\nThis repository is part of the Channels project. For the shepherd and maintenance team, please see the\n`main Channels readme `_.", "evidence": { @@ -112,23 +112,23 @@ "dependencies": [ { "dependsOn": [ - "pkg:pypi/asgiref@3.3.0?uuid=90cd6d93-1c46-477a-bd4a-47aa6b3ce773", - "pkg:pypi/asgiref@3.3.0?uuid=fb05a7ee-1078-4d60-bf71-2ba031414f1c" + "pkg:pypi/asgiref@3.3.0?uuid=510ea8b0-d7a7-4531-96ea-f41cb4384976", + "pkg:pypi/asgiref@3.3.0?uuid=eca680a3-115b-494b-8581-81c93384e22d" ], - "ref": "8d3058f3-ec1f-487d-8c5f-b2d3b26cda3e" + "ref": "83d59103-a103-4c9a-a433-618eff973b63" }, { - "ref": "pkg:pypi/asgiref@3.3.0?uuid=90cd6d93-1c46-477a-bd4a-47aa6b3ce773" + "ref": "pkg:pypi/asgiref@3.3.0?uuid=510ea8b0-d7a7-4531-96ea-f41cb4384976" }, { - "ref": "pkg:pypi/asgiref@3.3.0?uuid=fb05a7ee-1078-4d60-bf71-2ba031414f1c" + "ref": "pkg:pypi/asgiref@3.3.0?uuid=eca680a3-115b-494b-8581-81c93384e22d" } ], "vulnerabilities": [ { "affects": [ { - "ref": "urn:cdx:pkg:pypi/asgiref@3.3.0?uuid=fb05a7ee-1078-4d60-bf71-2ba031414f1c" + "ref": "urn:cdx:pkg:pypi/asgiref@3.3.0?uuid=510ea8b0-d7a7-4531-96ea-f41cb4384976" } ], "bom-ref": "BomRef", diff --git a/scanpipe/tests/data/d2d/about_files/expected.json b/scanpipe/tests/data/d2d/about_files/expected.json index 4628bce4be..e1b242fde8 100644 --- a/scanpipe/tests/data/d2d/about_files/expected.json +++ b/scanpipe/tests/data/d2d/about_files/expected.json @@ -72,6 +72,8 @@ "source_packages": [], "extra_data": {}, "package_uid": "", + "is_private": false, + "is_virtual": false, "datasource_ids": [], "datafile_paths": [], "file_references": [], @@ -123,6 +125,8 @@ ] }, "package_uid": "", + "is_private": false, + "is_virtual": false, "datasource_ids": [], "datafile_paths": [], "file_references": [], diff --git a/scanpipe/tests/data/daglib-0.6.0-py3-none-any.whl_scan_codebase.json b/scanpipe/tests/data/daglib-0.6.0-py3-none-any.whl_scan_codebase.json index 69c7f52f87..2b560db9ba 100644 --- a/scanpipe/tests/data/daglib-0.6.0-py3-none-any.whl_scan_codebase.json +++ b/scanpipe/tests/data/daglib-0.6.0-py3-none-any.whl_scan_codebase.json @@ -127,6 +127,8 @@ "Documentation": "https://mharrisb1.github.io/daglib/" }, "package_uid": "pkg:pypi/daglib@0.6.0?uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "pypi_wheel" ], @@ -235,6 +237,8 @@ "Documentation": "https://mharrisb1.github.io/daglib/" }, "package_uid": "pkg:pypi/daglib@0.6.0?uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "pypi_wheel_metadata" ], @@ -255,6 +259,7 @@ "is_runtime": true, "is_optional": false, "is_resolved": false, + "is_direct": true, "dependency_uid": "pkg:pypi/dask?uuid=fixed-uid-done-for-testing-5642512d1758", "for_package_uid": "pkg:pypi/daglib@0.6.0?uuid=fixed-uid-done-for-testing-5642512d1758", "resolved_to_package_uid": null, @@ -270,6 +275,7 @@ "is_runtime": true, "is_optional": false, "is_resolved": false, + "is_direct": true, "dependency_uid": "pkg:pypi/dask?uuid=fixed-uid-done-for-testing-5642512d1758", "for_package_uid": "pkg:pypi/daglib@0.6.0?uuid=fixed-uid-done-for-testing-5642512d1758", "resolved_to_package_uid": null, @@ -285,6 +291,7 @@ "is_runtime": true, "is_optional": true, "is_resolved": false, + "is_direct": true, "dependency_uid": "pkg:pypi/graphviz?uuid=fixed-uid-done-for-testing-5642512d1758", "for_package_uid": "pkg:pypi/daglib@0.6.0?uuid=fixed-uid-done-for-testing-5642512d1758", "resolved_to_package_uid": null, @@ -300,6 +307,7 @@ "is_runtime": true, "is_optional": true, "is_resolved": false, + "is_direct": true, "dependency_uid": "pkg:pypi/graphviz?uuid=fixed-uid-done-for-testing-5642512d1758", "for_package_uid": "pkg:pypi/daglib@0.6.0?uuid=fixed-uid-done-for-testing-5642512d1758", "resolved_to_package_uid": null, @@ -315,6 +323,7 @@ "is_runtime": true, "is_optional": true, "is_resolved": false, + "is_direct": true, "dependency_uid": "pkg:pypi/ipycytoscape?uuid=fixed-uid-done-for-testing-5642512d1758", "for_package_uid": "pkg:pypi/daglib@0.6.0?uuid=fixed-uid-done-for-testing-5642512d1758", "resolved_to_package_uid": null, @@ -330,6 +339,7 @@ "is_runtime": true, "is_optional": true, "is_resolved": false, + "is_direct": true, "dependency_uid": "pkg:pypi/ipycytoscape?uuid=fixed-uid-done-for-testing-5642512d1758", "for_package_uid": "pkg:pypi/daglib@0.6.0?uuid=fixed-uid-done-for-testing-5642512d1758", "resolved_to_package_uid": null, @@ -345,6 +355,7 @@ "is_runtime": true, "is_optional": false, "is_resolved": false, + "is_direct": true, "dependency_uid": "pkg:pypi/networkx?uuid=fixed-uid-done-for-testing-5642512d1758", "for_package_uid": "pkg:pypi/daglib@0.6.0?uuid=fixed-uid-done-for-testing-5642512d1758", "resolved_to_package_uid": null, @@ -360,6 +371,7 @@ "is_runtime": true, "is_optional": false, "is_resolved": false, + "is_direct": true, "dependency_uid": "pkg:pypi/networkx?uuid=fixed-uid-done-for-testing-5642512d1758", "for_package_uid": "pkg:pypi/daglib@0.6.0?uuid=fixed-uid-done-for-testing-5642512d1758", "resolved_to_package_uid": null, @@ -427,6 +439,8 @@ "extra_data": { "Documentation": "https://mharrisb1.github.io/daglib/" }, + "is_private": false, + "is_virtual": false, "qualifiers": {}, "notice_text": null, "api_data_url": "https://pypi.org/pypi/daglib/0.6.0/json", @@ -434,6 +448,7 @@ { "purl": "pkg:pypi/dask", "scope": "install", + "is_direct": true, "extra_data": {}, "is_runtime": true, "is_optional": false, @@ -444,6 +459,7 @@ { "purl": "pkg:pypi/graphviz", "scope": "graphviz", + "is_direct": true, "extra_data": {}, "is_runtime": true, "is_optional": true, @@ -454,6 +470,7 @@ { "purl": "pkg:pypi/ipycytoscape", "scope": "ipycytoscape", + "is_direct": true, "extra_data": {}, "is_runtime": true, "is_optional": true, @@ -464,6 +481,7 @@ { "purl": "pkg:pypi/networkx", "scope": "install", + "is_direct": true, "extra_data": {}, "is_runtime": true, "is_optional": false, @@ -819,6 +837,8 @@ "extra_data": { "Documentation": "https://mharrisb1.github.io/daglib/" }, + "is_private": false, + "is_virtual": false, "qualifiers": {}, "notice_text": null, "api_data_url": "https://pypi.org/pypi/daglib/0.6.0/json", @@ -826,6 +846,7 @@ { "purl": "pkg:pypi/dask", "scope": "install", + "is_direct": true, "extra_data": {}, "is_runtime": true, "is_optional": false, @@ -836,6 +857,7 @@ { "purl": "pkg:pypi/graphviz", "scope": "graphviz", + "is_direct": true, "extra_data": {}, "is_runtime": true, "is_optional": true, @@ -846,6 +868,7 @@ { "purl": "pkg:pypi/ipycytoscape", "scope": "ipycytoscape", + "is_direct": true, "extra_data": {}, "is_runtime": true, "is_optional": true, @@ -856,6 +879,7 @@ { "purl": "pkg:pypi/networkx", "scope": "install", + "is_direct": true, "extra_data": {}, "is_runtime": true, "is_optional": false, diff --git a/scanpipe/tests/data/debian_scan_codebase.json b/scanpipe/tests/data/debian_scan_codebase.json index de5a7820df..7586d79380 100644 --- a/scanpipe/tests/data/debian_scan_codebase.json +++ b/scanpipe/tests/data/debian_scan_codebase.json @@ -276,6 +276,8 @@ ] }, "package_uid": "pkg:deb/ubuntu/libncurses5@6.1-1ubuntu1.18.04?arch=amd64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "debian_installed_status_db", "debian_copyright_in_package", @@ -414,6 +416,8 @@ ] }, "package_uid": "pkg:deb/ubuntu/libndp0@1.4-2ubuntu0.16.04.1?arch=amd64&uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "debian_installed_status_db", "debian_copyright_in_package", diff --git a/scanpipe/tests/data/dependencies/resolved_dependencies.zip b/scanpipe/tests/data/dependencies/resolved_dependencies.zip new file mode 100644 index 0000000000000000000000000000000000000000..9b6ab79671888fa1c54db19b36aad8aefc667299 GIT binary patch literal 1577 zcmWIWW@Zs#-~d9e*uxPFQ1FY9fq{oXfuSHVIXf{uRWGYJKQA-Jjd2e){7T>ReNs!PP-&o;7I=5 z;S9uxxg;?2k93Gj>Yf)YU0SXpvJO$Tzk(O#KALHF@?}f+ zT+vMl_0?;|55E4kjH%wo-5~wmiET2cO>I{z@A=v`O{K)G$@mk?>dT+=*2kqTyj;9? zoB1Ezt8p#G6P6w-^A{4!Um4I8wN^|ddls9Lt*p_k=j-1`Oj7-_SdByV?5B$PTwlFb z7Ps!-aDSSK(5u&rES}iUI@`1E(Vi3ME?xaGVU5bB`=;|{XFuA{vhma8uB%T~c_yxM zZIOsLY?a2yE5cBk=CDM&ib0WsckTeoq}^BE`rm%S>#b2cY%dv{TXel# zW2I0<6Pv1U&_2eum%AVTk~w;6F;l96t06~#^QsK4Ai?Cug$Ek~EV(qV_As;huzIqqTIxBtfY0A4VzFQ)! zjOJBLUb4`l>%Ng$p1S7YyD9E>l8!oFl?#b`A23g#rs&sw9=S4^L&;+KW}<8i8u9Up zEY6l(Auf^+Pp__O2x$`tY2XR6 zv%qrF?&g0--}zqIXMJ~91gpu*@}%AJ+ia%n?h9TOIK}Vn&Z*CqZ+y7>?a#zd@|8dL zDE50#?X6mx_dRgJ`kiem3p_ul74dv@7b)stP~_utRcH}#Rtb*1#?$*MLX}6#r`zPI{yc{{bsE{*b0dp>co+KQ z1j*n2S8-&<{l^D)=>Dp@|M}&AZO@dFp;gSeT&bCVgT7B+pke+h+H!5z#~m-%O4NUUWWBXwMj=c8 z&Q(X(ibgQHEm$p)5IS)pqlA9ry7s^mKQgx-Fz9eA)VQp_?nm|B+vl&J&Rl(e@0q}y zDeBvQr&ngN$iH|UbKm&L;waV4alhF?ncU@tB1a#vU<7650B=Sn5oSagi(CMJ%2*iK u(g\n2009 Philipp Kern \n2011 Michael Shuler \nVarious Debian Contributors\nMozilla Contributors", "namespace": null, "extra_data": {}, + "is_private": false, + "is_virtual": false, "qualifiers": {}, "notice_text": null, "api_data_url": null, diff --git a/scanpipe/tests/data/is-npm-1.0.0_scan_codebase.json b/scanpipe/tests/data/is-npm-1.0.0_scan_codebase.json index 985603dfb9..46eecc0921 100644 --- a/scanpipe/tests/data/is-npm-1.0.0_scan_codebase.json +++ b/scanpipe/tests/data/is-npm-1.0.0_scan_codebase.json @@ -104,8 +104,14 @@ "extracted_license_statement": "- MIT\n", "notice_text": "", "source_packages": [], - "extra_data": {}, + "extra_data": { + "engines": { + "node": ">=0.10.0" + } + }, "package_uid": "pkg:npm/is-npm@1.0.0?uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "npm_package_json" ], @@ -126,6 +132,7 @@ "is_runtime": false, "is_optional": true, "is_resolved": false, + "is_direct": true, "dependency_uid": "pkg:npm/ava?uuid=fixed-uid-done-for-testing-5642512d1758", "for_package_uid": "pkg:npm/is-npm@1.0.0?uuid=fixed-uid-done-for-testing-5642512d1758", "resolved_to_package_uid": null, @@ -319,7 +326,13 @@ ], "copyright": null, "namespace": null, - "extra_data": {}, + "extra_data": { + "engines": { + "node": ">=0.10.0" + } + }, + "is_private": false, + "is_virtual": false, "qualifiers": {}, "notice_text": null, "api_data_url": "https://registry.npmjs.org/is-npm/1.0.0", @@ -327,6 +340,7 @@ { "purl": "pkg:npm/ava", "scope": "devDependencies", + "is_direct": true, "extra_data": {}, "is_runtime": false, "is_optional": true, diff --git a/scanpipe/tests/data/is-npm-1.0.0_scan_package.json b/scanpipe/tests/data/is-npm-1.0.0_scan_package.json index 9348bc77ef..c30eee492e 100644 --- a/scanpipe/tests/data/is-npm-1.0.0_scan_package.json +++ b/scanpipe/tests/data/is-npm-1.0.0_scan_package.json @@ -121,7 +121,13 @@ "extracted_license_statement": "- MIT\n", "notice_text": null, "source_packages": [], - "extra_data": {}, + "is_private": false, + "is_virtual": false, + "extra_data": { + "engines": { + "node": ">=0.10.0" + } + }, "repository_homepage_url": "https://www.npmjs.com/package/is-npm", "repository_download_url": "https://registry.npmjs.org/is-npm/-/is-npm-1.0.0.tgz", "api_data_url": "https://registry.npmjs.org/is-npm/1.0.0", @@ -143,6 +149,7 @@ "is_runtime": false, "is_optional": true, "is_resolved": false, + "is_direct": true, "resolved_package": {}, "extra_data": {}, "dependency_uid": "pkg:npm/ava?uuid=fixed-uid-done-for-testing-5642512d1758", @@ -396,7 +403,13 @@ "notice_text": null, "source_packages": [], "file_references": [], - "extra_data": {}, + "is_private": false, + "is_virtual": false, + "extra_data": { + "engines": { + "node": ">=0.10.0" + } + }, "dependencies": [ { "purl": "pkg:npm/ava", @@ -405,6 +418,7 @@ "is_runtime": false, "is_optional": true, "is_resolved": false, + "is_direct": true, "resolved_package": {}, "extra_data": {} } diff --git a/scanpipe/tests/data/is-npm-1.0.0_scan_package_summary.json b/scanpipe/tests/data/is-npm-1.0.0_scan_package_summary.json index cc66567e3c..bcd46b009b 100644 --- a/scanpipe/tests/data/is-npm-1.0.0_scan_package_summary.json +++ b/scanpipe/tests/data/is-npm-1.0.0_scan_package_summary.json @@ -219,8 +219,14 @@ "extracted_license_statement": "- MIT\n", "notice_text": "", "source_packages": [], - "extra_data": {}, + "extra_data": { + "engines": { + "node": ">=0.10.0" + } + }, "package_uid": "pkg:npm/is-npm@1.0.0?uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "npm_package_json" ], diff --git a/scanpipe/tests/data/manifests/openpdf-parent-1.3.11_scan_package.json b/scanpipe/tests/data/manifests/openpdf-parent-1.3.11_scan_package.json index 224e7c5452..005b0a94bc 100644 --- a/scanpipe/tests/data/manifests/openpdf-parent-1.3.11_scan_package.json +++ b/scanpipe/tests/data/manifests/openpdf-parent-1.3.11_scan_package.json @@ -116,6 +116,8 @@ "source_packages": [ "pkg:maven/com.github.librepdf/openpdf-parent@1.3.11?classifier=sources" ], + "is_private": false, + "is_virtual": false, "extra_data": {}, "repository_homepage_url": "https://repo1.maven.org/maven2/com/github/librepdf/openpdf-parent/1.3.11/", "repository_download_url": "https://repo1.maven.org/maven2/com/github/librepdf/openpdf-parent/1.3.11/openpdf-parent-1.3.11.jar", @@ -138,6 +140,7 @@ "is_runtime": false, "is_optional": true, "is_resolved": true, + "is_direct": true, "resolved_package": {}, "extra_data": {}, "dependency_uid": "pkg:maven/org.assertj/assertj-core@3.12.1?uuid=fixed-uid-done-for-testing-5642512d1758", @@ -152,6 +155,7 @@ "is_runtime": false, "is_optional": true, "is_resolved": true, + "is_direct": true, "resolved_package": {}, "extra_data": {}, "dependency_uid": "pkg:maven/org.junit.jupiter/junit-jupiter-api@5.5.1?uuid=fixed-uid-done-for-testing-5642512d1758", @@ -166,6 +170,7 @@ "is_runtime": false, "is_optional": true, "is_resolved": true, + "is_direct": true, "resolved_package": {}, "extra_data": {}, "dependency_uid": "pkg:maven/org.junit.jupiter/junit-jupiter-params@5.5.1?uuid=fixed-uid-done-for-testing-5642512d1758", @@ -180,6 +185,7 @@ "is_runtime": false, "is_optional": true, "is_resolved": true, + "is_direct": true, "resolved_package": {}, "extra_data": {}, "dependency_uid": "pkg:maven/org.junit.jupiter/junit-jupiter-engine@5.5.1?uuid=fixed-uid-done-for-testing-5642512d1758", @@ -194,6 +200,7 @@ "is_runtime": false, "is_optional": true, "is_resolved": true, + "is_direct": true, "resolved_package": {}, "extra_data": {}, "dependency_uid": "pkg:maven/org.mockito/mockito-core@2.25.1?uuid=fixed-uid-done-for-testing-5642512d1758", @@ -208,6 +215,7 @@ "is_runtime": false, "is_optional": true, "is_resolved": true, + "is_direct": true, "resolved_package": {}, "extra_data": {}, "dependency_uid": "pkg:maven/org.hamcrest/hamcrest-library@2.1?uuid=fixed-uid-done-for-testing-5642512d1758", @@ -222,6 +230,7 @@ "is_runtime": false, "is_optional": true, "is_resolved": true, + "is_direct": true, "resolved_package": {}, "extra_data": {}, "dependency_uid": "pkg:maven/org.hamcrest/hamcrest-core@2.1?uuid=fixed-uid-done-for-testing-5642512d1758", @@ -236,6 +245,7 @@ "is_runtime": false, "is_optional": false, "is_resolved": true, + "is_direct": true, "resolved_package": {}, "extra_data": {}, "dependency_uid": "pkg:maven/org.bouncycastle/bcprov-jdk15on@1.63?uuid=fixed-uid-done-for-testing-5642512d1758", @@ -250,6 +260,7 @@ "is_runtime": false, "is_optional": false, "is_resolved": true, + "is_direct": true, "resolved_package": {}, "extra_data": {}, "dependency_uid": "pkg:maven/org.bouncycastle/bcpkix-jdk15on@1.63?uuid=fixed-uid-done-for-testing-5642512d1758", @@ -264,6 +275,7 @@ "is_runtime": false, "is_optional": false, "is_resolved": true, + "is_direct": true, "resolved_package": {}, "extra_data": {}, "dependency_uid": "pkg:maven/com.github.spotbugs/spotbugs@3.1.12?uuid=fixed-uid-done-for-testing-5642512d1758", @@ -440,6 +452,8 @@ "pkg:maven/com.github.librepdf/openpdf-parent@1.3.11?classifier=sources" ], "file_references": [], + "is_private": false, + "is_virtual": false, "extra_data": {}, "dependencies": [ { @@ -449,6 +463,7 @@ "is_runtime": false, "is_optional": true, "is_resolved": true, + "is_direct": true, "resolved_package": {}, "extra_data": {} }, @@ -459,6 +474,7 @@ "is_runtime": false, "is_optional": true, "is_resolved": true, + "is_direct": true, "resolved_package": {}, "extra_data": {} }, @@ -469,6 +485,7 @@ "is_runtime": false, "is_optional": true, "is_resolved": true, + "is_direct": true, "resolved_package": {}, "extra_data": {} }, @@ -479,6 +496,7 @@ "is_runtime": false, "is_optional": true, "is_resolved": true, + "is_direct": true, "resolved_package": {}, "extra_data": {} }, @@ -489,6 +507,7 @@ "is_runtime": false, "is_optional": true, "is_resolved": true, + "is_direct": true, "resolved_package": {}, "extra_data": {} }, @@ -499,6 +518,7 @@ "is_runtime": false, "is_optional": true, "is_resolved": true, + "is_direct": true, "resolved_package": {}, "extra_data": {} }, @@ -509,6 +529,7 @@ "is_runtime": false, "is_optional": true, "is_resolved": true, + "is_direct": true, "resolved_package": {}, "extra_data": {} }, @@ -519,6 +540,7 @@ "is_runtime": false, "is_optional": false, "is_resolved": true, + "is_direct": true, "resolved_package": {}, "extra_data": {} }, @@ -529,6 +551,7 @@ "is_runtime": false, "is_optional": false, "is_resolved": true, + "is_direct": true, "resolved_package": {}, "extra_data": {} }, @@ -539,6 +562,7 @@ "is_runtime": false, "is_optional": false, "is_resolved": true, + "is_direct": true, "resolved_package": {}, "extra_data": {} } diff --git a/scanpipe/tests/data/multiple-is-npm-1.0.0_scan_package.json b/scanpipe/tests/data/multiple-is-npm-1.0.0_scan_package.json index bfcf90c9fc..28a8efb328 100644 --- a/scanpipe/tests/data/multiple-is-npm-1.0.0_scan_package.json +++ b/scanpipe/tests/data/multiple-is-npm-1.0.0_scan_package.json @@ -121,7 +121,13 @@ "extracted_license_statement": "- MIT\n", "notice_text": null, "source_packages": [], - "extra_data": {}, + "is_private": false, + "is_virtual": false, + "extra_data": { + "engines": { + "node": ">=0.10.0" + } + }, "repository_homepage_url": "https://www.npmjs.com/package/is-npm", "repository_download_url": "https://registry.npmjs.org/is-npm/-/is-npm-1.0.0.tgz", "api_data_url": "https://registry.npmjs.org/is-npm/1.0.0", @@ -203,7 +209,13 @@ "extracted_license_statement": "- MIT\n", "notice_text": null, "source_packages": [], - "extra_data": {}, + "is_private": false, + "is_virtual": false, + "extra_data": { + "engines": { + "node": ">=0.10.0" + } + }, "repository_homepage_url": "https://www.npmjs.com/package/is-npm", "repository_download_url": "https://registry.npmjs.org/is-npm/-/is-npm-1.0.0.tgz", "api_data_url": "https://registry.npmjs.org/is-npm/1.0.0", @@ -225,6 +237,7 @@ "is_runtime": false, "is_optional": true, "is_resolved": false, + "is_direct": true, "resolved_package": {}, "extra_data": {}, "dependency_uid": "pkg:npm/ava?uuid=fixed-uid-done-for-testing-5642512d1758", @@ -239,6 +252,7 @@ "is_runtime": false, "is_optional": true, "is_resolved": false, + "is_direct": true, "resolved_package": {}, "extra_data": {}, "dependency_uid": "pkg:npm/ava?uuid=fixed-uid-done-for-testing-5642512d1758", @@ -602,7 +616,13 @@ "notice_text": null, "source_packages": [], "file_references": [], - "extra_data": {}, + "is_private": false, + "is_virtual": false, + "extra_data": { + "engines": { + "node": ">=0.10.0" + } + }, "dependencies": [ { "purl": "pkg:npm/ava", @@ -611,6 +631,7 @@ "is_runtime": false, "is_optional": true, "is_resolved": false, + "is_direct": true, "resolved_package": {}, "extra_data": {} } @@ -865,7 +886,13 @@ "notice_text": null, "source_packages": [], "file_references": [], - "extra_data": {}, + "is_private": false, + "is_virtual": false, + "extra_data": { + "engines": { + "node": ">=0.10.0" + } + }, "dependencies": [ { "purl": "pkg:npm/ava", @@ -874,6 +901,7 @@ "is_runtime": false, "is_optional": true, "is_resolved": false, + "is_direct": true, "resolved_package": {}, "extra_data": {} } diff --git a/scanpipe/tests/data/multiple-is-npm-1.0.0_scan_package_summary.json b/scanpipe/tests/data/multiple-is-npm-1.0.0_scan_package_summary.json index d350cc70e5..38f16f31a7 100644 --- a/scanpipe/tests/data/multiple-is-npm-1.0.0_scan_package_summary.json +++ b/scanpipe/tests/data/multiple-is-npm-1.0.0_scan_package_summary.json @@ -243,8 +243,14 @@ "extracted_license_statement": "- MIT\n", "notice_text": "", "source_packages": [], - "extra_data": {}, + "extra_data": { + "engines": { + "node": ">=0.10.0" + } + }, "package_uid": "pkg:npm/is-npm@1.0.0?uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "npm_package_json" ], diff --git a/scanpipe/tests/data/resolved_dependencies_inspect_packages.json b/scanpipe/tests/data/resolved_dependencies_inspect_packages.json new file mode 100644 index 0000000000..c06064c60d --- /dev/null +++ b/scanpipe/tests/data/resolved_dependencies_inspect_packages.json @@ -0,0 +1,1127 @@ +{ + "headers": [ + { + "tool_name": "scanpipe", + "notice": "Generated with ScanCode.io and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied.\nNo content created from ScanCode.io should be considered or used as legal advice.\nConsult an Attorney for any legal advice.\nScanCode.io is a free software code scanning tool from nexB Inc. and others\nlicensed under the Apache License version 2.0.\nScanCode is a trademark of nexB Inc.\nVisit https://github.com/nexB/scancode.io for support and download.\n", + "input_sources": [ + { + "filename": "resolved_dependencies.zip", + "is_uploaded": false, + "is_file": true, + "exists": true + } + ], + "runs": [ + { + "pipeline_name": "inspect_packages", + "status": "not_started", + "scancodeio_version": "", + "task_id": null, + "task_start_date": null, + "task_end_date": null, + "task_exitcode": null, + "task_output": "", + "execution_time": null + } + ], + "extra_data": {} + } + ], + "packages": [ + { + "purl": "pkg:npm/athena-express@6.0.4", + "type": "npm", + "namespace": "", + "name": "athena-express", + "version": "6.0.4", + "qualifiers": "", + "subpath": "", + "tag": "", + "primary_language": "JavaScript", + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "", + "download_url": "https://registry.yarnpkg.com/athena-express/-/athena-express-6.0.4.tgz", + "bug_tracking_url": "", + "code_view_url": "", + "vcs_url": "", + "repository_homepage_url": "https://www.npmjs.com/package/athena-express", + "repository_download_url": "https://registry.npmjs.org/athena-express/-/athena-express-6.0.4.tgz", + "api_data_url": "https://registry.npmjs.org/athena-express/6.0.4", + "md5": "", + "sha1": "bb457dcc967686faea2138f934828a331b98c8f6", + "sha256": "", + "sha512": "02d0272ee0a30050812b2780926a607d4ccbd0d0c0bda64d82992e914ac1f68e7e53ffb0cdcac7bdc2a946e7c2f905c9ed6ffc512f5b3c28fc275b28d2c8ff7c", + "copyright": "", + "holder": "", + "declared_license_expression": "", + "declared_license_expression_spdx": "", + "license_detections": [], + "other_license_expression": "", + "other_license_expression_spdx": "", + "other_license_detections": [], + "extracted_license_statement": "", + "notice_text": "", + "source_packages": [], + "extra_data": {}, + "package_uid": "", + "is_private": false, + "is_virtual": true, + "datasource_ids": [ + "yarn_lock_v1" + ], + "datafile_paths": [], + "file_references": [], + "missing_resources": [], + "modified_resources": [], + "affected_by_vulnerabilities": [] + }, + { + "purl": "pkg:npm/bluebird@3.7.2", + "type": "npm", + "namespace": "", + "name": "bluebird", + "version": "3.7.2", + "qualifiers": "", + "subpath": "", + "tag": "", + "primary_language": "JavaScript", + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "", + "download_url": "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz", + "bug_tracking_url": "", + "code_view_url": "", + "vcs_url": "", + "repository_homepage_url": "https://www.npmjs.com/package/bluebird", + "repository_download_url": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", + "api_data_url": "https://registry.npmjs.org/bluebird/3.7.2", + "md5": "", + "sha1": "9f229c15be272454ffa973ace0dbee79a1b0c36f", + "sha256": "", + "sha512": "5e9363e860d0cdd7d6fabd969e7ef189201ded33378f39311970464ed58ab925efd71515f9acf1026f2375664dd3a413424fb63765c1f6344392f6e6426711b6", + "copyright": "", + "holder": "", + "declared_license_expression": "", + "declared_license_expression_spdx": "", + "license_detections": [], + "other_license_expression": "", + "other_license_expression_spdx": "", + "other_license_detections": [], + "extracted_license_statement": "", + "notice_text": "", + "source_packages": [], + "extra_data": {}, + "package_uid": "", + "is_private": false, + "is_virtual": true, + "datasource_ids": [ + "yarn_lock_v1" + ], + "datafile_paths": [], + "file_references": [], + "missing_resources": [], + "modified_resources": [], + "affected_by_vulnerabilities": [] + }, + { + "purl": "pkg:npm/create-athena-partition@1.0.0", + "type": "npm", + "namespace": "", + "name": "create-athena-partition", + "version": "1.0.0", + "qualifiers": "", + "subpath": "", + "tag": "", + "primary_language": "JavaScript", + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "", + "download_url": "https://registry.npmjs.org/create-athena-partition/-/create-athena-partition-1.0.0.tgz", + "bug_tracking_url": "", + "code_view_url": "", + "vcs_url": "", + "repository_homepage_url": "", + "repository_download_url": "", + "api_data_url": "", + "md5": "", + "sha1": "", + "sha256": "", + "sha512": "", + "copyright": "", + "holder": "", + "declared_license_expression": "unknown", + "declared_license_expression_spdx": "LicenseRef-scancode-unknown", + "license_detections": [ + { + "matches": [ + { + "score": 100.0, + "matcher": "5-undetected", + "end_line": 1, + "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/package-manifest-unknown-cb9ea49fe36cb2e1ba6d87c68e1195a492f762cf", + "from_file": "resolved_dependencies.zip-extract/package.json", + "start_line": 1, + "matched_text": "license - UNLICENSED", + "match_coverage": 100.0, + "matched_length": 2, + "rule_relevance": 100, + "rule_identifier": "package-manifest-unknown-cb9ea49fe36cb2e1ba6d87c68e1195a492f762cf", + "license_expression": "unknown", + "spdx_license_expression": "LicenseRef-scancode-unknown" + } + ], + "identifier": "unknown-0669ac45-20f6-defd-ec9f-2b6aafc9f944", + "license_expression": "unknown", + "license_expression_spdx": "LicenseRef-scancode-unknown" + } + ], + "other_license_expression": "", + "other_license_expression_spdx": "", + "other_license_detections": [], + "extracted_license_statement": "- UNLICENSED\n", + "notice_text": "", + "source_packages": [], + "extra_data": {}, + "package_uid": "pkg:npm/create-athena-partition@1.0.0?uuid=fixed-uid-done-for-testing-5642512d1758", + "is_private": true, + "is_virtual": false, + "datasource_ids": [ + "npm_package_json" + ], + "datafile_paths": [ + "resolved_dependencies.zip-extract/package.json" + ], + "file_references": [], + "missing_resources": [], + "modified_resources": [], + "affected_by_vulnerabilities": [] + }, + { + "purl": "pkg:npm/csvtojson@2.0.10", + "type": "npm", + "namespace": "", + "name": "csvtojson", + "version": "2.0.10", + "qualifiers": "", + "subpath": "", + "tag": "", + "primary_language": "JavaScript", + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "", + "download_url": "https://registry.yarnpkg.com/csvtojson/-/csvtojson-2.0.10.tgz", + "bug_tracking_url": "", + "code_view_url": "", + "vcs_url": "", + "repository_homepage_url": "https://www.npmjs.com/package/csvtojson", + "repository_download_url": "https://registry.npmjs.org/csvtojson/-/csvtojson-2.0.10.tgz", + "api_data_url": "https://registry.npmjs.org/csvtojson/2.0.10", + "md5": "", + "sha1": "11e7242cc630da54efce7958a45f443210357574", + "sha256": "", + "sha512": "954585c462b286b68a096f10821cfa6747f697f3ea0755b700ed07289cc6210e49452951eb9d5e9090e21896c14f8b1134dbf975d9d2195127b313fd5d1095a5", + "copyright": "", + "holder": "", + "declared_license_expression": "", + "declared_license_expression_spdx": "", + "license_detections": [], + "other_license_expression": "", + "other_license_expression_spdx": "", + "other_license_detections": [], + "extracted_license_statement": "", + "notice_text": "", + "source_packages": [], + "extra_data": {}, + "package_uid": "", + "is_private": false, + "is_virtual": true, + "datasource_ids": [ + "yarn_lock_v1" + ], + "datafile_paths": [], + "file_references": [], + "missing_resources": [], + "modified_resources": [], + "affected_by_vulnerabilities": [] + }, + { + "purl": "pkg:npm/is-utf8@0.2.1", + "type": "npm", + "namespace": "", + "name": "is-utf8", + "version": "0.2.1", + "qualifiers": "", + "subpath": "", + "tag": "", + "primary_language": "JavaScript", + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "", + "download_url": "https://registry.yarnpkg.com/is-utf8/-/is-utf8-0.2.1.tgz", + "bug_tracking_url": "", + "code_view_url": "", + "vcs_url": "", + "repository_homepage_url": "https://www.npmjs.com/package/is-utf8", + "repository_download_url": "https://registry.npmjs.org/is-utf8/-/is-utf8-0.2.1.tgz", + "api_data_url": "https://registry.npmjs.org/is-utf8/0.2.1", + "md5": "", + "sha1": "4b0da1442104d1b336340e80797e865cf39f7d72", + "sha256": "", + "sha512": "", + "copyright": "", + "holder": "", + "declared_license_expression": "", + "declared_license_expression_spdx": "", + "license_detections": [], + "other_license_expression": "", + "other_license_expression_spdx": "", + "other_license_detections": [], + "extracted_license_statement": "", + "notice_text": "", + "source_packages": [], + "extra_data": {}, + "package_uid": "", + "is_private": false, + "is_virtual": true, + "datasource_ids": [ + "yarn_lock_v1" + ], + "datafile_paths": [], + "file_references": [], + "missing_resources": [], + "modified_resources": [], + "affected_by_vulnerabilities": [] + }, + { + "purl": "pkg:npm/lodash@4.17.21", + "type": "npm", + "namespace": "", + "name": "lodash", + "version": "4.17.21", + "qualifiers": "", + "subpath": "", + "tag": "", + "primary_language": "JavaScript", + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "", + "download_url": "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz", + "bug_tracking_url": "", + "code_view_url": "", + "vcs_url": "", + "repository_homepage_url": "https://www.npmjs.com/package/lodash", + "repository_download_url": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "api_data_url": "https://registry.npmjs.org/lodash/4.17.21", + "md5": "", + "sha1": "679591c564c3bffaae8454cf0b3df370c3d6911c", + "sha256": "", + "sha512": "bf690311ee7b95e713ba568322e3533f2dd1cb880b189e99d4edef13592b81764daec43e2c54c61d5c558dc5cfb35ecb85b65519e74026ff17675b6f8f916f4a", + "copyright": "", + "holder": "", + "declared_license_expression": "", + "declared_license_expression_spdx": "", + "license_detections": [], + "other_license_expression": "", + "other_license_expression_spdx": "", + "other_license_detections": [], + "extracted_license_statement": "", + "notice_text": "", + "source_packages": [], + "extra_data": {}, + "package_uid": "", + "is_private": false, + "is_virtual": true, + "datasource_ids": [ + "yarn_lock_v1" + ], + "datafile_paths": [], + "file_references": [], + "missing_resources": [], + "modified_resources": [], + "affected_by_vulnerabilities": [] + }, + { + "purl": "pkg:npm/strip-bom@2.0.0", + "type": "npm", + "namespace": "", + "name": "strip-bom", + "version": "2.0.0", + "qualifiers": "", + "subpath": "", + "tag": "", + "primary_language": "JavaScript", + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "", + "download_url": "https://registry.yarnpkg.com/strip-bom/-/strip-bom-2.0.0.tgz", + "bug_tracking_url": "", + "code_view_url": "", + "vcs_url": "", + "repository_homepage_url": "https://www.npmjs.com/package/strip-bom", + "repository_download_url": "https://registry.npmjs.org/strip-bom/-/strip-bom-2.0.0.tgz", + "api_data_url": "https://registry.npmjs.org/strip-bom/2.0.0", + "md5": "", + "sha1": "6219a85616520491f35788bdbf1447a99c7e6b0e", + "sha256": "", + "sha512": "", + "copyright": "", + "holder": "", + "declared_license_expression": "", + "declared_license_expression_spdx": "", + "license_detections": [], + "other_license_expression": "", + "other_license_expression_spdx": "", + "other_license_detections": [], + "extracted_license_statement": "", + "notice_text": "", + "source_packages": [], + "extra_data": {}, + "package_uid": "", + "is_private": false, + "is_virtual": true, + "datasource_ids": [ + "yarn_lock_v1" + ], + "datafile_paths": [], + "file_references": [], + "missing_resources": [], + "modified_resources": [], + "affected_by_vulnerabilities": [] + } + ], + "dependencies": [ + { + "purl": "pkg:npm/athena-express", + "extracted_requirement": "^6.0.4", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false, + "is_direct": true, + "dependency_uid": "pkg:npm/athena-express?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:npm/create-athena-partition@1.0.0?uuid=fixed-uid-done-for-testing-5642512d1758", + "resolved_to_package_uid": "", + "datafile_path": "resolved_dependencies.zip-extract/package.json", + "datasource_id": "npm_package_json", + "package_type": "npm", + "affected_by_vulnerabilities": [] + }, + { + "purl": "pkg:npm/bluebird", + "extracted_requirement": "^3.5.1", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "is_direct": true, + "dependency_uid": "pkg:npm/bluebird?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "", + "resolved_to_package_uid": "", + "datafile_path": "resolved_dependencies.zip-extract/yarn.lock", + "datasource_id": "yarn_lock_v1", + "package_type": "npm", + "affected_by_vulnerabilities": [] + }, + { + "purl": "pkg:npm/csvtojson", + "extracted_requirement": "^2.0.10", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "is_direct": true, + "dependency_uid": "pkg:npm/csvtojson?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "", + "resolved_to_package_uid": "", + "datafile_path": "resolved_dependencies.zip-extract/yarn.lock", + "datasource_id": "yarn_lock_v1", + "package_type": "npm", + "affected_by_vulnerabilities": [] + }, + { + "purl": "pkg:npm/is-utf8", + "extracted_requirement": "^0.2.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "is_direct": true, + "dependency_uid": "pkg:npm/is-utf8?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "", + "resolved_to_package_uid": "", + "datafile_path": "resolved_dependencies.zip-extract/yarn.lock", + "datasource_id": "yarn_lock_v1", + "package_type": "npm", + "affected_by_vulnerabilities": [] + }, + { + "purl": "pkg:npm/lodash", + "extracted_requirement": "^4.17.3", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "is_direct": true, + "dependency_uid": "pkg:npm/lodash?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "", + "resolved_to_package_uid": "", + "datafile_path": "resolved_dependencies.zip-extract/yarn.lock", + "datasource_id": "yarn_lock_v1", + "package_type": "npm", + "affected_by_vulnerabilities": [] + }, + { + "purl": "pkg:npm/strip-bom", + "extracted_requirement": "^2.0.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "is_direct": true, + "dependency_uid": "pkg:npm/strip-bom?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "", + "resolved_to_package_uid": "", + "datafile_path": "resolved_dependencies.zip-extract/yarn.lock", + "datasource_id": "yarn_lock_v1", + "package_type": "npm", + "affected_by_vulnerabilities": [] + } + ], + "files": [ + { + "path": "resolved_dependencies.zip", + "type": "file", + "name": "resolved_dependencies.zip", + "status": "", + "tag": "", + "extension": ".zip", + "md5": "412ba8411bd121539d151f0bd39e2ed2", + "sha1": "215dfb066677df22a2cd61f2550deaa7d1583472", + "sha256": "0d4157ba9da49fbf101cd21b37ac93175568d2d5e30884b235002fb2f5f31c6a", + "sha512": "", + "programming_language": "", + "is_binary": true, + "is_text": false, + "is_archive": true, + "is_media": false, + "is_key_file": false, + "detected_license_expression": "", + "detected_license_expression_spdx": "", + "license_detections": [], + "license_clues": [], + "percentage_of_license_text": null, + "copyrights": [], + "holders": [], + "authors": [], + "package_data": [], + "for_packages": [], + "emails": [], + "urls": [], + "extra_data": {} + }, + { + "path": "resolved_dependencies.zip-extract", + "type": "directory", + "name": "resolved_dependencies.zip-extract", + "status": "", + "tag": "", + "extension": ".zip-extract", + "md5": "", + "sha1": "", + "sha256": "", + "sha512": "", + "programming_language": "", + "is_binary": false, + "is_text": false, + "is_archive": false, + "is_media": false, + "is_key_file": false, + "detected_license_expression": "", + "detected_license_expression_spdx": "", + "license_detections": [], + "license_clues": [], + "percentage_of_license_text": null, + "copyrights": [], + "holders": [], + "authors": [], + "package_data": [], + "for_packages": [], + "emails": [], + "urls": [], + "extra_data": {} + }, + { + "path": "resolved_dependencies.zip-extract/package.json", + "type": "file", + "name": "package.json", + "status": "application-package", + "tag": "", + "extension": ".json", + "md5": "7001c0ea1ebdc9e52bf8225ad9e997c0", + "sha1": "f86079c55d42a7204a9e5e8aa6617174d43d4fc5", + "sha256": "15861d5a2202f1334290ac3a4fbd80991e41497e58134dde4d94d10d67752dbe", + "sha512": "", + "programming_language": "", + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_key_file": false, + "detected_license_expression": "", + "detected_license_expression_spdx": "", + "license_detections": [], + "license_clues": [], + "percentage_of_license_text": null, + "copyrights": [], + "holders": [], + "authors": [], + "package_data": [ + { + "md5": null, + "name": "create-athena-partition", + "purl": "pkg:npm/create-athena-partition@1.0.0", + "sha1": null, + "type": "npm", + "holder": null, + "sha256": null, + "sha512": null, + "parties": [], + "subpath": null, + "vcs_url": null, + "version": "1.0.0", + "keywords": [], + "copyright": null, + "namespace": null, + "extra_data": {}, + "is_private": true, + "is_virtual": false, + "qualifiers": {}, + "notice_text": null, + "api_data_url": null, + "dependencies": [ + { + "purl": "pkg:npm/athena-express", + "scope": "dependencies", + "is_direct": true, + "extra_data": {}, + "is_runtime": true, + "is_optional": false, + "is_resolved": false, + "resolved_package": {}, + "extracted_requirement": "^6.0.4" + } + ], + "download_url": "https://registry.npmjs.org/create-athena-partition/-/create-athena-partition-1.0.0.tgz", + "homepage_url": null, + "release_date": null, + "code_view_url": null, + "datasource_id": "npm_package_json", + "file_references": [], + "source_packages": [], + "bug_tracking_url": null, + "primary_language": "JavaScript", + "license_detections": [ + { + "matches": [ + { + "score": 100.0, + "matcher": "5-undetected", + "end_line": 1, + "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/package-manifest-unknown-cb9ea49fe36cb2e1ba6d87c68e1195a492f762cf", + "from_file": null, + "start_line": 1, + "matched_text": "license - UNLICENSED", + "match_coverage": 100.0, + "matched_length": 2, + "rule_relevance": 100, + "rule_identifier": "package-manifest-unknown-cb9ea49fe36cb2e1ba6d87c68e1195a492f762cf", + "license_expression": "unknown", + "spdx_license_expression": "LicenseRef-scancode-unknown" + } + ], + "identifier": "unknown-0669ac45-20f6-defd-ec9f-2b6aafc9f944", + "license_expression": "unknown", + "license_expression_spdx": "LicenseRef-scancode-unknown" + } + ], + "repository_download_url": null, + "repository_homepage_url": null, + "other_license_detections": [], + "other_license_expression": null, + "declared_license_expression": "unknown", + "extracted_license_statement": "- UNLICENSED\n", + "other_license_expression_spdx": null, + "declared_license_expression_spdx": "LicenseRef-scancode-unknown" + } + ], + "for_packages": [ + "pkg:npm/create-athena-partition@1.0.0?uuid=fixed-uid-done-for-testing-5642512d1758" + ], + "emails": [], + "urls": [], + "extra_data": {} + }, + { + "path": "resolved_dependencies.zip-extract/yarn.lock", + "type": "file", + "name": "yarn.lock", + "status": "application-package", + "tag": "", + "extension": ".lock", + "md5": "0481ffd2443fcad568a09fae75b9d231", + "sha1": "396a29e290d34e6409e1801510d6b674ee4a2d98", + "sha256": "af63cd9d3ce4b209f073cc750a9b19a92a2cfdf3ad44cd9b4b207258d702ceba", + "sha512": "", + "programming_language": "", + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_key_file": false, + "detected_license_expression": "", + "detected_license_expression_spdx": "", + "license_detections": [], + "license_clues": [], + "percentage_of_license_text": null, + "copyrights": [], + "holders": [], + "authors": [], + "package_data": [ + { + "md5": null, + "name": null, + "purl": null, + "sha1": null, + "type": "npm", + "holder": null, + "sha256": null, + "sha512": null, + "parties": [], + "subpath": null, + "vcs_url": null, + "version": null, + "keywords": [], + "copyright": null, + "namespace": null, + "extra_data": {}, + "is_private": false, + "is_virtual": false, + "qualifiers": {}, + "notice_text": null, + "api_data_url": null, + "dependencies": [ + { + "purl": "pkg:npm/athena-express@6.0.4", + "scope": "dependencies", + "is_direct": false, + "extra_data": {}, + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": { + "md5": null, + "name": "athena-express", + "purl": "pkg:npm/athena-express@6.0.4", + "sha1": "bb457dcc967686faea2138f934828a331b98c8f6", + "type": "npm", + "holder": null, + "sha256": null, + "sha512": "02d0272ee0a30050812b2780926a607d4ccbd0d0c0bda64d82992e914ac1f68e7e53ffb0cdcac7bdc2a946e7c2f905c9ed6ffc512f5b3c28fc275b28d2c8ff7c", + "parties": [], + "subpath": null, + "vcs_url": null, + "version": "6.0.4", + "keywords": [], + "copyright": null, + "namespace": "", + "extra_data": {}, + "is_private": false, + "is_virtual": true, + "qualifiers": {}, + "notice_text": null, + "api_data_url": "https://registry.npmjs.org/athena-express/6.0.4", + "dependencies": [ + { + "purl": "pkg:npm/csvtojson", + "scope": "dependencies", + "is_direct": true, + "extra_data": {}, + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {}, + "extracted_requirement": "^2.0.10" + } + ], + "download_url": "https://registry.yarnpkg.com/athena-express/-/athena-express-6.0.4.tgz", + "homepage_url": null, + "release_date": null, + "code_view_url": null, + "datasource_id": "yarn_lock_v1", + "file_references": [], + "source_packages": [], + "bug_tracking_url": null, + "primary_language": "JavaScript", + "license_detections": [], + "repository_download_url": "https://registry.npmjs.org/athena-express/-/athena-express-6.0.4.tgz", + "repository_homepage_url": "https://www.npmjs.com/package/athena-express", + "other_license_detections": [], + "other_license_expression": null, + "declared_license_expression": null, + "extracted_license_statement": null, + "other_license_expression_spdx": null, + "declared_license_expression_spdx": null + }, + "extracted_requirement": "^6.0.4" + }, + { + "purl": "pkg:npm/bluebird@3.7.2", + "scope": "dependencies", + "is_direct": false, + "extra_data": {}, + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": { + "md5": null, + "name": "bluebird", + "purl": "pkg:npm/bluebird@3.7.2", + "sha1": "9f229c15be272454ffa973ace0dbee79a1b0c36f", + "type": "npm", + "holder": null, + "sha256": null, + "sha512": "5e9363e860d0cdd7d6fabd969e7ef189201ded33378f39311970464ed58ab925efd71515f9acf1026f2375664dd3a413424fb63765c1f6344392f6e6426711b6", + "parties": [], + "subpath": null, + "vcs_url": null, + "version": "3.7.2", + "keywords": [], + "copyright": null, + "namespace": "", + "extra_data": {}, + "is_private": false, + "is_virtual": true, + "qualifiers": {}, + "notice_text": null, + "api_data_url": "https://registry.npmjs.org/bluebird/3.7.2", + "dependencies": [], + "download_url": "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz", + "homepage_url": null, + "release_date": null, + "code_view_url": null, + "datasource_id": "yarn_lock_v1", + "file_references": [], + "source_packages": [], + "bug_tracking_url": null, + "primary_language": "JavaScript", + "license_detections": [], + "repository_download_url": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", + "repository_homepage_url": "https://www.npmjs.com/package/bluebird", + "other_license_detections": [], + "other_license_expression": null, + "declared_license_expression": null, + "extracted_license_statement": null, + "other_license_expression_spdx": null, + "declared_license_expression_spdx": null + }, + "extracted_requirement": "^3.5.1" + }, + { + "purl": "pkg:npm/csvtojson@2.0.10", + "scope": "dependencies", + "is_direct": false, + "extra_data": {}, + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": { + "md5": null, + "name": "csvtojson", + "purl": "pkg:npm/csvtojson@2.0.10", + "sha1": "11e7242cc630da54efce7958a45f443210357574", + "type": "npm", + "holder": null, + "sha256": null, + "sha512": "954585c462b286b68a096f10821cfa6747f697f3ea0755b700ed07289cc6210e49452951eb9d5e9090e21896c14f8b1134dbf975d9d2195127b313fd5d1095a5", + "parties": [], + "subpath": null, + "vcs_url": null, + "version": "2.0.10", + "keywords": [], + "copyright": null, + "namespace": "", + "extra_data": {}, + "is_private": false, + "is_virtual": true, + "qualifiers": {}, + "notice_text": null, + "api_data_url": "https://registry.npmjs.org/csvtojson/2.0.10", + "dependencies": [ + { + "purl": "pkg:npm/bluebird", + "scope": "dependencies", + "is_direct": true, + "extra_data": {}, + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {}, + "extracted_requirement": "^3.5.1" + }, + { + "purl": "pkg:npm/lodash", + "scope": "dependencies", + "is_direct": true, + "extra_data": {}, + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {}, + "extracted_requirement": "^4.17.3" + }, + { + "purl": "pkg:npm/strip-bom", + "scope": "dependencies", + "is_direct": true, + "extra_data": {}, + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {}, + "extracted_requirement": "^2.0.0" + } + ], + "download_url": "https://registry.yarnpkg.com/csvtojson/-/csvtojson-2.0.10.tgz", + "homepage_url": null, + "release_date": null, + "code_view_url": null, + "datasource_id": "yarn_lock_v1", + "file_references": [], + "source_packages": [], + "bug_tracking_url": null, + "primary_language": "JavaScript", + "license_detections": [], + "repository_download_url": "https://registry.npmjs.org/csvtojson/-/csvtojson-2.0.10.tgz", + "repository_homepage_url": "https://www.npmjs.com/package/csvtojson", + "other_license_detections": [], + "other_license_expression": null, + "declared_license_expression": null, + "extracted_license_statement": null, + "other_license_expression_spdx": null, + "declared_license_expression_spdx": null + }, + "extracted_requirement": "^2.0.10" + }, + { + "purl": "pkg:npm/is-utf8@0.2.1", + "scope": "dependencies", + "is_direct": false, + "extra_data": {}, + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": { + "md5": null, + "name": "is-utf8", + "purl": "pkg:npm/is-utf8@0.2.1", + "sha1": "4b0da1442104d1b336340e80797e865cf39f7d72", + "type": "npm", + "holder": null, + "sha256": null, + "sha512": null, + "parties": [], + "subpath": null, + "vcs_url": null, + "version": "0.2.1", + "keywords": [], + "copyright": null, + "namespace": "", + "extra_data": {}, + "is_private": false, + "is_virtual": true, + "qualifiers": {}, + "notice_text": null, + "api_data_url": "https://registry.npmjs.org/is-utf8/0.2.1", + "dependencies": [], + "download_url": "https://registry.yarnpkg.com/is-utf8/-/is-utf8-0.2.1.tgz", + "homepage_url": null, + "release_date": null, + "code_view_url": null, + "datasource_id": "yarn_lock_v1", + "file_references": [], + "source_packages": [], + "bug_tracking_url": null, + "primary_language": "JavaScript", + "license_detections": [], + "repository_download_url": "https://registry.npmjs.org/is-utf8/-/is-utf8-0.2.1.tgz", + "repository_homepage_url": "https://www.npmjs.com/package/is-utf8", + "other_license_detections": [], + "other_license_expression": null, + "declared_license_expression": null, + "extracted_license_statement": null, + "other_license_expression_spdx": null, + "declared_license_expression_spdx": null + }, + "extracted_requirement": "^0.2.0" + }, + { + "purl": "pkg:npm/lodash@4.17.21", + "scope": "dependencies", + "is_direct": false, + "extra_data": {}, + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": { + "md5": null, + "name": "lodash", + "purl": "pkg:npm/lodash@4.17.21", + "sha1": "679591c564c3bffaae8454cf0b3df370c3d6911c", + "type": "npm", + "holder": null, + "sha256": null, + "sha512": "bf690311ee7b95e713ba568322e3533f2dd1cb880b189e99d4edef13592b81764daec43e2c54c61d5c558dc5cfb35ecb85b65519e74026ff17675b6f8f916f4a", + "parties": [], + "subpath": null, + "vcs_url": null, + "version": "4.17.21", + "keywords": [], + "copyright": null, + "namespace": "", + "extra_data": {}, + "is_private": false, + "is_virtual": true, + "qualifiers": {}, + "notice_text": null, + "api_data_url": "https://registry.npmjs.org/lodash/4.17.21", + "dependencies": [], + "download_url": "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz", + "homepage_url": null, + "release_date": null, + "code_view_url": null, + "datasource_id": "yarn_lock_v1", + "file_references": [], + "source_packages": [], + "bug_tracking_url": null, + "primary_language": "JavaScript", + "license_detections": [], + "repository_download_url": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "repository_homepage_url": "https://www.npmjs.com/package/lodash", + "other_license_detections": [], + "other_license_expression": null, + "declared_license_expression": null, + "extracted_license_statement": null, + "other_license_expression_spdx": null, + "declared_license_expression_spdx": null + }, + "extracted_requirement": "^4.17.3" + }, + { + "purl": "pkg:npm/strip-bom@2.0.0", + "scope": "dependencies", + "is_direct": false, + "extra_data": {}, + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": { + "md5": null, + "name": "strip-bom", + "purl": "pkg:npm/strip-bom@2.0.0", + "sha1": "6219a85616520491f35788bdbf1447a99c7e6b0e", + "type": "npm", + "holder": null, + "sha256": null, + "sha512": null, + "parties": [], + "subpath": null, + "vcs_url": null, + "version": "2.0.0", + "keywords": [], + "copyright": null, + "namespace": "", + "extra_data": {}, + "is_private": false, + "is_virtual": true, + "qualifiers": {}, + "notice_text": null, + "api_data_url": "https://registry.npmjs.org/strip-bom/2.0.0", + "dependencies": [ + { + "purl": "pkg:npm/is-utf8", + "scope": "dependencies", + "is_direct": true, + "extra_data": {}, + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "resolved_package": {}, + "extracted_requirement": "^0.2.0" + } + ], + "download_url": "https://registry.yarnpkg.com/strip-bom/-/strip-bom-2.0.0.tgz", + "homepage_url": null, + "release_date": null, + "code_view_url": null, + "datasource_id": "yarn_lock_v1", + "file_references": [], + "source_packages": [], + "bug_tracking_url": null, + "primary_language": "JavaScript", + "license_detections": [], + "repository_download_url": "https://registry.npmjs.org/strip-bom/-/strip-bom-2.0.0.tgz", + "repository_homepage_url": "https://www.npmjs.com/package/strip-bom", + "other_license_detections": [], + "other_license_expression": null, + "declared_license_expression": null, + "extracted_license_statement": null, + "other_license_expression_spdx": null, + "declared_license_expression_spdx": null + }, + "extracted_requirement": "^2.0.0" + } + ], + "download_url": null, + "homepage_url": null, + "release_date": null, + "code_view_url": null, + "datasource_id": "yarn_lock_v1", + "file_references": [], + "source_packages": [], + "bug_tracking_url": null, + "primary_language": "JavaScript", + "license_detections": [], + "repository_download_url": null, + "repository_homepage_url": null, + "other_license_detections": [], + "other_license_expression": null, + "declared_license_expression": null, + "extracted_license_statement": null, + "other_license_expression_spdx": null, + "declared_license_expression_spdx": null + } + ], + "for_packages": [ + "pkg:npm/bluebird@3.7.2?uuid=fixed-uid-done-for-testing-5642512d1758", + "pkg:npm/lodash@4.17.21?uuid=fixed-uid-done-for-testing-5642512d1758", + "pkg:npm/strip-bom@2.0.0?uuid=fixed-uid-done-for-testing-5642512d1758", + "pkg:npm/is-utf8@0.2.1?uuid=fixed-uid-done-for-testing-5642512d1758", + "pkg:npm/create-athena-partition@1.0.0?uuid=fixed-uid-done-for-testing-5642512d1758", + "pkg:npm/athena-express@6.0.4?uuid=fixed-uid-done-for-testing-5642512d1758", + "pkg:npm/csvtojson@2.0.10?uuid=fixed-uid-done-for-testing-5642512d1758" + ], + "emails": [], + "urls": [], + "extra_data": {} + } + ], + "relations": [] +} \ No newline at end of file diff --git a/scanpipe/tests/data/scancode/is-npm-1.0.0_summary.json b/scanpipe/tests/data/scancode/is-npm-1.0.0_summary.json index 8f00569f72..ec5eb63ce6 100644 --- a/scanpipe/tests/data/scancode/is-npm-1.0.0_summary.json +++ b/scanpipe/tests/data/scancode/is-npm-1.0.0_summary.json @@ -226,8 +226,14 @@ "compliance_alert": "", "notice_text": "", "source_packages": [], - "extra_data": {}, + "extra_data": { + "engines": { + "node": ">=0.10.0" + } + }, "package_uid": "pkg:npm/is-npm@1.0.0?uuid=ba110d49-b6f2-4c86-8d89-a6fd34838ca8", + "is_private": false, + "is_virtual": false, "datasource_ids": [ "npm_package_json" ], diff --git a/scanpipe/tests/data/scancode/package_assembly_codebase.json b/scanpipe/tests/data/scancode/package_assembly_codebase.json index 85239ed90d..eae0d76de9 100644 --- a/scanpipe/tests/data/scancode/package_assembly_codebase.json +++ b/scanpipe/tests/data/scancode/package_assembly_codebase.json @@ -2,16 +2,16 @@ "headers": [ { "tool_name": "scancode-toolkit", - "tool_version": "32.0.8", + "tool_version": "32.1.0", "options": { "--info": true, "--package": true }, "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", - "start_timestamp": "2024-03-01T113357.465177", - "end_timestamp": "2024-03-01T113357.680774", + "start_timestamp": "2024-06-13T142327.801919", + "end_timestamp": "2024-06-13T142327.894178", "output_format_version": "3.1.0", - "duration": 0.2156081199645996, + "duration": 0.09227418899536133, "message": null, "errors": [], "warnings": [], @@ -19,8 +19,8 @@ "system_environment": { "operating_system": "linux", "cpu_architecture": "64", - "platform": "Linux-5.15.0-94-generic-x86_64-with-glibc2.35", - "platform_version": "#104-Ubuntu SMP Tue Jan 9 15:25:40 UTC 2024", + "platform": "Linux-5.15.0-112-generic-x86_64-with-glibc2.35", + "platform_version": "#122-Ubuntu SMP Thu May 23 07:48:21 UTC 2024", "python_version": "3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0]" }, "spdx_license_list_version": "3.23", @@ -85,11 +85,13 @@ "extracted_license_statement": "- MIT\n", "notice_text": null, "source_packages": [], + "is_private": false, + "is_virtual": false, "extra_data": {}, "repository_homepage_url": "https://www.npmjs.com/package/test", "repository_download_url": "https://registry.npmjs.org/test/-/test-0.1.0.tgz", "api_data_url": "https://registry.npmjs.org/test/0.1.0", - "package_uid": "pkg:npm/test@0.1.0?uuid=393cff2b-c004-445c-b055-6e8b0af8a904", + "package_uid": "pkg:npm/test@0.1.0?uuid=a5225ee0-cc90-49cf-80bc-370004b8ced5", "datafile_paths": [ "package_assembly_codebase.tar.gz-extract/test/get_package_resources/package.json" ], @@ -260,6 +262,8 @@ "notice_text": null, "source_packages": [], "file_references": [], + "is_private": false, + "is_virtual": false, "extra_data": {}, "dependencies": [], "repository_homepage_url": "https://www.npmjs.com/package/test", @@ -270,7 +274,7 @@ } ], "for_packages": [ - "pkg:npm/test@0.1.0?uuid=393cff2b-c004-445c-b055-6e8b0af8a904" + "pkg:npm/test@0.1.0?uuid=a5225ee0-cc90-49cf-80bc-370004b8ced5" ], "files_count": 0, "dirs_count": 0, @@ -299,7 +303,7 @@ "is_script": false, "package_data": [], "for_packages": [ - "pkg:npm/test@0.1.0?uuid=393cff2b-c004-445c-b055-6e8b0af8a904" + "pkg:npm/test@0.1.0?uuid=a5225ee0-cc90-49cf-80bc-370004b8ced5" ], "files_count": 0, "dirs_count": 0, diff --git a/scanpipe/tests/pipes/test_output.py b/scanpipe/tests/pipes/test_output.py index 471d56c1b1..26fc067d61 100644 --- a/scanpipe/tests/pipes/test_output.py +++ b/scanpipe/tests/pipes/test_output.py @@ -247,7 +247,7 @@ def test_scanpipe_pipes_outputs_to_cyclonedx(self, regen=FIXTURES_REGEN): project = Project.objects.get(name="asgiref") package = project.discoveredpackages.get( - uuid="b2d24c22-0dff-4e3f-8332-413b4f4852a7" + uuid="27357e59-c16d-40f1-8cd2-0f11b7376cdb" ) package.other_license_expression_spdx = "Apache-2.0 AND LicenseRef-test" diff --git a/scanpipe/tests/test_api.py b/scanpipe/tests/test_api.py index 5dc039151c..a94fb2d84a 100644 --- a/scanpipe/tests/test_api.py +++ b/scanpipe/tests/test_api.py @@ -997,8 +997,8 @@ def test_scanpipe_api_serializer_get_model_serializer(self): get_model_serializer(None) def test_scanpipe_api_serializer_get_serializer_fields(self): - self.assertEqual(46, len(get_serializer_fields(DiscoveredPackage))) - self.assertEqual(13, len(get_serializer_fields(DiscoveredDependency))) + self.assertEqual(48, len(get_serializer_fields(DiscoveredPackage))) + self.assertEqual(14, len(get_serializer_fields(DiscoveredDependency))) self.assertEqual(33, len(get_serializer_fields(CodebaseResource))) self.assertEqual(5, len(get_serializer_fields(CodebaseRelation))) self.assertEqual(7, len(get_serializer_fields(ProjectMessage))) diff --git a/scanpipe/tests/test_pipelines.py b/scanpipe/tests/test_pipelines.py index b481b45701..2c55dd1d85 100644 --- a/scanpipe/tests/test_pipelines.py +++ b/scanpipe/tests/test_pipelines.py @@ -639,11 +639,14 @@ def test_scanpipe_inspect_packages_creates_packages_npm(self): package = project1.discoveredpackages.get() dependency = project1.discovereddependencies.get() - self.assertEqual(1, package.codebase_resources.count()) + self.assertEqual(3, package.codebase_resources.count()) self.assertEqual("pkg:npm/is-npm@1.0.0", dependency.for_package.purl) self.assertEqual(package.datasource_ids, [dependency.datasource_id]) self.assertEqual( - package.codebase_resources.get().path, dependency.datafile_resource.path + package.codebase_resources.get( + path="is-npm-1.0.0.tgz-extract/package/package.json" + ).path, + dependency.datafile_resource.path, ) def test_scanpipe_inspect_packages_creates_packages_pypi(self): @@ -664,6 +667,33 @@ def test_scanpipe_inspect_packages_creates_packages_pypi(self): self.assertEqual(0, project1.discoveredpackages.count()) self.assertEqual(26, project1.discovereddependencies.count()) + def test_scanpipe_inspect_packages_with_resolved_dependencies(self): + pipeline_name = "inspect_packages" + project1 = Project.objects.create(name="Analysis") + + input_location = ( + self.data_location / "dependencies" / "resolved_dependencies.zip" + ) + project1.copy_input_from(input_location) + + run = project1.add_pipeline( + pipeline_name=pipeline_name, + selected_groups=["Static Resolver"], + ) + pipeline = run.make_pipeline_instance() + + exitcode, out = pipeline.execute() + self.assertEqual(0, exitcode, msg=out) + self.assertEqual(4, project1.codebaseresources.count()) + self.assertEqual(7, project1.discoveredpackages.count()) + self.assertEqual(6, project1.discovereddependencies.count()) + + result_file = output.to_json(project1) + expected_file = ( + self.data_location / "resolved_dependencies_inspect_packages.json" + ) + self.assertPipelineResultEqual(expected_file, result_file, regen=True) + def test_scanpipe_scan_codebase_can_process_wheel(self): pipeline_name = "scan_codebase" project1 = Project.objects.create(name="Analysis") diff --git a/scanpipe/views.py b/scanpipe/views.py index 880de02ed2..25afc709ab 100644 --- a/scanpipe/views.py +++ b/scanpipe/views.py @@ -1543,6 +1543,10 @@ class DiscoveredDependencyListView( "field_name": "is_resolved", "filter_fieldname": "is_resolved", }, + { + "field_name": "is_direct", + "filter_fieldname": "is_direct", + }, "for_package", "resolved_to_package", "datafile_resource", @@ -1730,6 +1734,7 @@ class CodebaseResourceDetailsView( "extra_data": { "fields": [ {"field_name": "extra_data", "render_func": render_as_yaml}, + {"field_name": "package_data", "render_func": render_as_yaml}, ], "verbose_name": "Extra", "icon_class": "fa-solid fa-plus-square", @@ -1883,6 +1888,8 @@ class DiscoveredPackageDetailsView( "missing_resources", "modified_resources", "package_uid", + "is_private", + "is_virtual", "datasource_ids", "datafile_paths", ], @@ -2048,6 +2055,7 @@ class DiscoveredDependencyDetailsView( "is_runtime", "is_optional", "is_resolved", + "is_direct", ], "icon_class": "fa-solid fa-info-circle", }, diff --git a/setup.cfg b/setup.cfg index 11402c6a54..cb2a56636c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -72,7 +72,7 @@ install_requires = # Docker container-inspector==33.0.0 # ScanCode-toolkit - scancode-toolkit[packages]==32.1.0 + scancode-toolkit[packages] @ git+https://github.com/nexB/scancode-toolkit.git@b24b29f5e04b72a723fcc0e55e5d4cb60fc879e4 extractcode[full]==31.0.0 commoncode==31.2.1 # FetchCode From 04fc0e1f2f7be62ade8f03b945d456f294d372fd Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Fri, 14 Jun 2024 16:50:03 +0530 Subject: [PATCH 02/11] Address feedback and add improvements Signed-off-by: Ayan Sinha Mahapatra --- scanpipe/filters.py | 4 ++ .../0061_dependency_resolver_update.py | 12 +++++ scanpipe/models.py | 44 ++++++++++--------- scanpipe/pipes/__init__.py | 7 --- ...esolved_dependencies_inspect_packages.json | 44 +++++++++---------- scanpipe/tests/test_pipelines.py | 10 ++++- 6 files changed, 69 insertions(+), 52 deletions(-) diff --git a/scanpipe/filters.py b/scanpipe/filters.py index f030290cd8..4b97acf5ca 100644 --- a/scanpipe/filters.py +++ b/scanpipe/filters.py @@ -684,6 +684,8 @@ class PackageFilterSet(FilterSetUtilsMixin, django_filters.FilterSet): declared_license_expression = django_filters.filters.CharFilter( widget=HasValueDropdownWidget ) + is_private = StrictBooleanFilter() + is_virtual = StrictBooleanFilter() class Meta: model = DiscoveredPackage @@ -717,6 +719,8 @@ class Meta: "is_vulnerable", "compliance_alert", "tag", + "is_private", + "is_virtual", ] diff --git a/scanpipe/migrations/0061_dependency_resolver_update.py b/scanpipe/migrations/0061_dependency_resolver_update.py index 0b5533d383..abe4288373 100644 --- a/scanpipe/migrations/0061_dependency_resolver_update.py +++ b/scanpipe/migrations/0061_dependency_resolver_update.py @@ -31,4 +31,16 @@ class Migration(migrations.Migration): fields=["is_direct"], name="scanpipe_di_is_dire_6dc594_idx" ), ), + migrations.AddIndex( + model_name="discoveredpackage", + index=models.Index( + fields=["is_private"], name="scanpipe_di_is_priv_9ffd1a_idx" + ), + ), + migrations.AddIndex( + model_name="discoveredpackage", + index=models.Index( + fields=["is_virtual"], name="scanpipe_di_is_virt_c5c176_idx" + ), + ), ] diff --git a/scanpipe/models.py b/scanpipe/models.py index 28d92c7ea1..89e5f54709 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -3099,6 +3099,8 @@ class Meta: models.Index(fields=["sha512"]), models.Index(fields=["compliance_alert"]), models.Index(fields=["tag"]), + models.Index(fields=["is_private"]), + models.Index(fields=["is_virtual"]), ] constraints = [ models.UniqueConstraint( @@ -3126,15 +3128,7 @@ def purl(self): @classmethod def extract_purl_data(cls, package_data): - purl_data = {} - - for field_name in PURL_FIELDS: - value = package_data.get(field_name) - if field_name == "qualifiers": - value = normalize_qualifiers(value, encode=True) - purl_data[field_name] = value or "" - - return purl_data + return normalize_package_url_data(package_data) @classmethod def create_from_data(cls, project, package_data): @@ -3619,19 +3613,8 @@ def extract_purl_data(cls, dependency_data, ignore_nulls=False): purl_mapping = PackageURL.from_string( purl=dependency_data.get("purl"), ).to_dict() - purl_data = {} - - for field_name in PURL_FIELDS: - value = purl_mapping.get(field_name) - if field_name == "qualifiers": - value = normalize_qualifiers(value, encode=True) - if not ignore_nulls: - purl_data[field_name] = value or "" - else: - if value: - purl_data[field_name] = value or "" - return purl_data + return normalize_package_url_data(purl_mapping, ignore_nulls) @classmethod def populate_dependency_uuid(cls, dependency_data): @@ -3666,6 +3649,25 @@ def as_spdx(self): ) +def normalize_package_url_data(purl_mapping, ignore_nulls=False): + """ + Normalize a mapping of purl data so database queries with + purl data can be executed. + """ + normalized_purl_mapping = {} + for field_name in PURL_FIELDS: + value = purl_mapping.get(field_name) + if field_name == "qualifiers": + value = normalize_qualifiers(value, encode=True) + if not ignore_nulls: + normalized_purl_mapping[field_name] = value or "" + else: + if value: + normalized_purl_mapping[field_name] = value or "" + + return normalized_purl_mapping + + class WebhookSubscription(UUIDPKModel, ProjectRelatedModel): target_url = models.URLField(_("Target URL"), max_length=1024) created_date = models.DateTimeField(auto_now_add=True, editable=False) diff --git a/scanpipe/pipes/__init__.py b/scanpipe/pipes/__init__.py index 64fafcc284..d5dc538103 100644 --- a/scanpipe/pipes/__init__.py +++ b/scanpipe/pipes/__init__.py @@ -199,9 +199,6 @@ def update_or_create_package( package = DiscoveredPackage.create_from_data(project, package_data) if package: - if is_virtual: - package.update(is_virtual=is_virtual) - if datasource_id and datasource_id not in package.datasource_ids: datasource_ids = package.datasource_ids.copy() datasource_ids.append(datasource_id) @@ -291,10 +288,6 @@ def update_or_create_dependency( if resolved_to_package and not dependency.resolved_to_package: dependency.update(resolved_to_package=resolved_to_package) else: - is_direct = dependency_data.get("is_direct") - if not is_direct: - pass - DiscoveredDependency.populate_dependency_uuid(dependency_data) dependency = DiscoveredDependency.create_from_data( project, diff --git a/scanpipe/tests/data/resolved_dependencies_inspect_packages.json b/scanpipe/tests/data/resolved_dependencies_inspect_packages.json index c06064c60d..4aedd79f7c 100644 --- a/scanpipe/tests/data/resolved_dependencies_inspect_packages.json +++ b/scanpipe/tests/data/resolved_dependencies_inspect_packages.json @@ -65,7 +65,7 @@ "notice_text": "", "source_packages": [], "extra_data": {}, - "package_uid": "", + "package_uid": "pkg:npm/athena-express@6.0.4?uuid=fixed-uid-done-for-testing-5642512d1758", "is_private": false, "is_virtual": true, "datasource_ids": [ @@ -114,7 +114,7 @@ "notice_text": "", "source_packages": [], "extra_data": {}, - "package_uid": "", + "package_uid": "pkg:npm/bluebird@3.7.2?uuid=fixed-uid-done-for-testing-5642512d1758", "is_private": false, "is_virtual": true, "datasource_ids": [ @@ -237,7 +237,7 @@ "notice_text": "", "source_packages": [], "extra_data": {}, - "package_uid": "", + "package_uid": "pkg:npm/csvtojson@2.0.10?uuid=fixed-uid-done-for-testing-5642512d1758", "is_private": false, "is_virtual": true, "datasource_ids": [ @@ -286,7 +286,7 @@ "notice_text": "", "source_packages": [], "extra_data": {}, - "package_uid": "", + "package_uid": "pkg:npm/is-utf8@0.2.1?uuid=fixed-uid-done-for-testing-5642512d1758", "is_private": false, "is_virtual": true, "datasource_ids": [ @@ -335,7 +335,7 @@ "notice_text": "", "source_packages": [], "extra_data": {}, - "package_uid": "", + "package_uid": "pkg:npm/lodash@4.17.21?uuid=fixed-uid-done-for-testing-5642512d1758", "is_private": false, "is_virtual": true, "datasource_ids": [ @@ -384,7 +384,7 @@ "notice_text": "", "source_packages": [], "extra_data": {}, - "package_uid": "", + "package_uid": "pkg:npm/strip-bom@2.0.0?uuid=fixed-uid-done-for-testing-5642512d1758", "is_private": false, "is_virtual": true, "datasource_ids": [ @@ -408,7 +408,7 @@ "is_direct": true, "dependency_uid": "pkg:npm/athena-express?uuid=fixed-uid-done-for-testing-5642512d1758", "for_package_uid": "pkg:npm/create-athena-partition@1.0.0?uuid=fixed-uid-done-for-testing-5642512d1758", - "resolved_to_package_uid": "", + "resolved_to_package_uid": "pkg:npm/athena-express@6.0.4?uuid=fixed-uid-done-for-testing-5642512d1758", "datafile_path": "resolved_dependencies.zip-extract/package.json", "datasource_id": "npm_package_json", "package_type": "npm", @@ -423,8 +423,8 @@ "is_resolved": true, "is_direct": true, "dependency_uid": "pkg:npm/bluebird?uuid=fixed-uid-done-for-testing-5642512d1758", - "for_package_uid": "", - "resolved_to_package_uid": "", + "for_package_uid": "pkg:npm/csvtojson@2.0.10?uuid=fixed-uid-done-for-testing-5642512d1758", + "resolved_to_package_uid": "pkg:npm/bluebird@3.7.2?uuid=fixed-uid-done-for-testing-5642512d1758", "datafile_path": "resolved_dependencies.zip-extract/yarn.lock", "datasource_id": "yarn_lock_v1", "package_type": "npm", @@ -439,8 +439,8 @@ "is_resolved": true, "is_direct": true, "dependency_uid": "pkg:npm/csvtojson?uuid=fixed-uid-done-for-testing-5642512d1758", - "for_package_uid": "", - "resolved_to_package_uid": "", + "for_package_uid": "pkg:npm/athena-express@6.0.4?uuid=fixed-uid-done-for-testing-5642512d1758", + "resolved_to_package_uid": "pkg:npm/csvtojson@2.0.10?uuid=fixed-uid-done-for-testing-5642512d1758", "datafile_path": "resolved_dependencies.zip-extract/yarn.lock", "datasource_id": "yarn_lock_v1", "package_type": "npm", @@ -455,8 +455,8 @@ "is_resolved": true, "is_direct": true, "dependency_uid": "pkg:npm/is-utf8?uuid=fixed-uid-done-for-testing-5642512d1758", - "for_package_uid": "", - "resolved_to_package_uid": "", + "for_package_uid": "pkg:npm/strip-bom@2.0.0?uuid=fixed-uid-done-for-testing-5642512d1758", + "resolved_to_package_uid": "pkg:npm/is-utf8@0.2.1?uuid=fixed-uid-done-for-testing-5642512d1758", "datafile_path": "resolved_dependencies.zip-extract/yarn.lock", "datasource_id": "yarn_lock_v1", "package_type": "npm", @@ -471,8 +471,8 @@ "is_resolved": true, "is_direct": true, "dependency_uid": "pkg:npm/lodash?uuid=fixed-uid-done-for-testing-5642512d1758", - "for_package_uid": "", - "resolved_to_package_uid": "", + "for_package_uid": "pkg:npm/csvtojson@2.0.10?uuid=fixed-uid-done-for-testing-5642512d1758", + "resolved_to_package_uid": "pkg:npm/lodash@4.17.21?uuid=fixed-uid-done-for-testing-5642512d1758", "datafile_path": "resolved_dependencies.zip-extract/yarn.lock", "datasource_id": "yarn_lock_v1", "package_type": "npm", @@ -487,8 +487,8 @@ "is_resolved": true, "is_direct": true, "dependency_uid": "pkg:npm/strip-bom?uuid=fixed-uid-done-for-testing-5642512d1758", - "for_package_uid": "", - "resolved_to_package_uid": "", + "for_package_uid": "pkg:npm/csvtojson@2.0.10?uuid=fixed-uid-done-for-testing-5642512d1758", + "resolved_to_package_uid": "pkg:npm/strip-bom@2.0.0?uuid=fixed-uid-done-for-testing-5642512d1758", "datafile_path": "resolved_dependencies.zip-extract/yarn.lock", "datasource_id": "yarn_lock_v1", "package_type": "npm", @@ -1110,13 +1110,13 @@ } ], "for_packages": [ - "pkg:npm/bluebird@3.7.2?uuid=fixed-uid-done-for-testing-5642512d1758", - "pkg:npm/lodash@4.17.21?uuid=fixed-uid-done-for-testing-5642512d1758", - "pkg:npm/strip-bom@2.0.0?uuid=fixed-uid-done-for-testing-5642512d1758", - "pkg:npm/is-utf8@0.2.1?uuid=fixed-uid-done-for-testing-5642512d1758", "pkg:npm/create-athena-partition@1.0.0?uuid=fixed-uid-done-for-testing-5642512d1758", + "pkg:npm/bluebird@3.7.2?uuid=fixed-uid-done-for-testing-5642512d1758", + "pkg:npm/csvtojson@2.0.10?uuid=fixed-uid-done-for-testing-5642512d1758", "pkg:npm/athena-express@6.0.4?uuid=fixed-uid-done-for-testing-5642512d1758", - "pkg:npm/csvtojson@2.0.10?uuid=fixed-uid-done-for-testing-5642512d1758" + "pkg:npm/strip-bom@2.0.0?uuid=fixed-uid-done-for-testing-5642512d1758", + "pkg:npm/lodash@4.17.21?uuid=fixed-uid-done-for-testing-5642512d1758", + "pkg:npm/is-utf8@0.2.1?uuid=fixed-uid-done-for-testing-5642512d1758" ], "emails": [], "urls": [], diff --git a/scanpipe/tests/test_pipelines.py b/scanpipe/tests/test_pipelines.py index 05b94cb593..8bd1a4b047 100644 --- a/scanpipe/tests/test_pipelines.py +++ b/scanpipe/tests/test_pipelines.py @@ -470,7 +470,13 @@ def _normalize_package_uids(self, data): if isinstance(value, (list, dict)): value = self._normalize_package_uids(value) if ( - key in ("package_uid", "dependency_uid", "for_package_uid") + key + in ( + "package_uid", + "dependency_uid", + "for_package_uid", + "resolved_to_package_uid", + ) and value ): value = purl_with_fake_uuid(value) @@ -692,7 +698,7 @@ def test_scanpipe_inspect_packages_with_resolved_dependencies(self): expected_file = ( self.data_location / "resolved_dependencies_inspect_packages.json" ) - self.assertPipelineResultEqual(expected_file, result_file, regen=True) + self.assertPipelineResultEqual(expected_file, result_file) def test_scanpipe_scan_codebase_can_process_wheel(self): pipeline_name = "scan_codebase" From f3385a8aed5850f4e11dc5dfe4a4fa311cf328f4 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Mon, 17 Jun 2024 19:38:32 +0530 Subject: [PATCH 03/11] Improve dependency resolving from lockfiles #1237 Resolves dependency for cases where multiple requirements are resolved by one package and all the version requirements are joined for that package. Reference: https://github.com/nexB/scancode.io/issues/1237 Signed-off-by: Ayan Sinha Mahapatra --- scanpipe/pipes/scancode.py | 17 +++++++++++++++++ setup.cfg | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/scanpipe/pipes/scancode.py b/scanpipe/pipes/scancode.py index 29585b5da4..fcd9368951 100644 --- a/scanpipe/pipes/scancode.py +++ b/scanpipe/pipes/scancode.py @@ -602,6 +602,7 @@ def match_and_resolve_dependencies(project): dependency_data={"purl": dependency.purl}, ignore_nulls=True, ) + matched_dependencies = DiscoveredDependency.objects.filter( project=project, extracted_requirement=dependency.extracted_requirement, @@ -611,6 +612,22 @@ def match_and_resolve_dependencies(project): other_dependencies = [ dep for dep in matched_dependencies if dep.purl != dependency.purl ] + if not other_dependencies: + # We also have cases where multiple dependency requirements have one + # resolved package and the extracted requirements field is combined + matched_dependencies = DiscoveredDependency.objects.filter( + project=project, + **purl_data, + ) + other_dependencies = [ + dep + for dep in matched_dependencies + if ( + dep.purl != dependency.purl + and dependency.extracted_requirement in dep.extracted_requirement + ) + ] + if other_dependencies: resolved_dependency = other_dependencies.pop() dependency.update( diff --git a/setup.cfg b/setup.cfg index 76d76fe7d2..b334b295cc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -72,7 +72,7 @@ install_requires = # Docker container-inspector==33.0.0 # ScanCode-toolkit - scancode-toolkit[packages] @ git+https://github.com/nexB/scancode-toolkit.git@b24b29f5e04b72a723fcc0e55e5d4cb60fc879e4 + scancode-toolkit[packages] @ git+https://github.com/nexB/scancode-toolkit.git@ce8e0d1ddc9e8968958313e82481d46498d51f25 extractcode[full]==31.0.0 commoncode==31.2.1 # FetchCode From 65e652d0a59985f90810e7d2a6b8c3c36d9f5b35 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Wed, 19 Jun 2024 16:46:14 +0530 Subject: [PATCH 04/11] Update scancode-toolkit and fix tests Signed-off-by: Ayan Sinha Mahapatra --- .../data/resolved_dependencies_inspect_packages.json | 8 ++++---- scanpipe/tests/test_pipelines.py | 10 ++++++---- scanpipe/views.py | 4 ---- setup.cfg | 2 +- 4 files changed, 11 insertions(+), 13 deletions(-) diff --git a/scanpipe/tests/data/resolved_dependencies_inspect_packages.json b/scanpipe/tests/data/resolved_dependencies_inspect_packages.json index 4aedd79f7c..20f0b184af 100644 --- a/scanpipe/tests/data/resolved_dependencies_inspect_packages.json +++ b/scanpipe/tests/data/resolved_dependencies_inspect_packages.json @@ -1110,13 +1110,13 @@ } ], "for_packages": [ - "pkg:npm/create-athena-partition@1.0.0?uuid=fixed-uid-done-for-testing-5642512d1758", + "pkg:npm/athena-express@6.0.4?uuid=fixed-uid-done-for-testing-5642512d1758", "pkg:npm/bluebird@3.7.2?uuid=fixed-uid-done-for-testing-5642512d1758", + "pkg:npm/create-athena-partition@1.0.0?uuid=fixed-uid-done-for-testing-5642512d1758", "pkg:npm/csvtojson@2.0.10?uuid=fixed-uid-done-for-testing-5642512d1758", - "pkg:npm/athena-express@6.0.4?uuid=fixed-uid-done-for-testing-5642512d1758", - "pkg:npm/strip-bom@2.0.0?uuid=fixed-uid-done-for-testing-5642512d1758", + "pkg:npm/is-utf8@0.2.1?uuid=fixed-uid-done-for-testing-5642512d1758", "pkg:npm/lodash@4.17.21?uuid=fixed-uid-done-for-testing-5642512d1758", - "pkg:npm/is-utf8@0.2.1?uuid=fixed-uid-done-for-testing-5642512d1758" + "pkg:npm/strip-bom@2.0.0?uuid=fixed-uid-done-for-testing-5642512d1758" ], "emails": [], "urls": [], diff --git a/scanpipe/tests/test_pipelines.py b/scanpipe/tests/test_pipelines.py index 8bd1a4b047..ed5c03ff84 100644 --- a/scanpipe/tests/test_pipelines.py +++ b/scanpipe/tests/test_pipelines.py @@ -481,10 +481,12 @@ def _normalize_package_uids(self, data): ): value = purl_with_fake_uuid(value) if key == "for_packages" and value: - value = [ - self.purl_fields_with_fake_uuid(package_uid, key) - for package_uid in value - ] + value = sorted( + [ + self.purl_fields_with_fake_uuid(package_uid, key) + for package_uid in value + ] + ) if is_local_files and key in ("name", "namespace", "purl") and value: value = self.purl_fields_with_fake_uuid(value, key) normalized_data[key] = value diff --git a/scanpipe/views.py b/scanpipe/views.py index 25afc709ab..203c778bde 100644 --- a/scanpipe/views.py +++ b/scanpipe/views.py @@ -1543,10 +1543,6 @@ class DiscoveredDependencyListView( "field_name": "is_resolved", "filter_fieldname": "is_resolved", }, - { - "field_name": "is_direct", - "filter_fieldname": "is_direct", - }, "for_package", "resolved_to_package", "datafile_resource", diff --git a/setup.cfg b/setup.cfg index b334b295cc..67b7344e59 100644 --- a/setup.cfg +++ b/setup.cfg @@ -72,7 +72,7 @@ install_requires = # Docker container-inspector==33.0.0 # ScanCode-toolkit - scancode-toolkit[packages] @ git+https://github.com/nexB/scancode-toolkit.git@ce8e0d1ddc9e8968958313e82481d46498d51f25 + scancode-toolkit[packages] @ git+https://github.com/nexB/scancode-toolkit.git@ae22b45d23122df13ff5d6907a9bb6e117c63026 extractcode[full]==31.0.0 commoncode==31.2.1 # FetchCode From 92a3bf3148d81e670fef4bce541b7b9372002794 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Thu, 20 Jun 2024 19:54:05 +0530 Subject: [PATCH 05/11] Bump scancode-toolkit to v32.2.0 Reference: https://github.com/nexB/scancode-toolkit/releases/tag/v32.2.0 Signed-off-by: Ayan Sinha Mahapatra --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 67b7344e59..eb025df2e6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -72,7 +72,7 @@ install_requires = # Docker container-inspector==33.0.0 # ScanCode-toolkit - scancode-toolkit[packages] @ git+https://github.com/nexB/scancode-toolkit.git@ae22b45d23122df13ff5d6907a9bb6e117c63026 + scancode-toolkit[packages]==32.2.0 extractcode[full]==31.0.0 commoncode==31.2.1 # FetchCode From 7c9e0e8537bf1f9e42f42ec5bed836522f3c3e4e Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Thu, 20 Jun 2024 20:17:56 +0530 Subject: [PATCH 06/11] Regenerate test fixtures and expectations Signed-off-by: Ayan Sinha Mahapatra --- scanpipe/tests/data/asgiref-3.3.0.spdx.json | 42 +++++----- .../tests/data/asgiref-3.3.0_fixtures.json | 80 +++++++++---------- .../data/asgiref-3.3.0_scanpipe_output.json | 66 +++++++-------- .../data/asgiref-3.3.0_toolkit_scan.json | 64 +++++++-------- .../asgiref-3.3.0_walk_test_fixtures.json | 80 +++++++++---------- .../data/cyclonedx/asgiref-3.3.0.cdx.json | 18 ++--- .../tests/data/is-npm-1.0.0_scan_package.json | 4 +- .../openpdf-parent-1.3.11_scan_package.json | 4 +- .../multiple-is-npm-1.0.0_scan_package.json | 4 +- .../scancode/package_assembly_codebase.json | 18 ++--- scanpipe/tests/pipes/test_output.py | 2 +- 11 files changed, 191 insertions(+), 191 deletions(-) diff --git a/scanpipe/tests/data/asgiref-3.3.0.spdx.json b/scanpipe/tests/data/asgiref-3.3.0.spdx.json index 0f3707137d..423dcff55d 100644 --- a/scanpipe/tests/data/asgiref-3.3.0.spdx.json +++ b/scanpipe/tests/data/asgiref-3.3.0.spdx.json @@ -3,7 +3,7 @@ "dataLicense": "CC0-1.0", "SPDXID": "SPDXRef-DOCUMENT", "name": "scancodeio_asgiref", - "documentNamespace": "https://scancode.io/spdxdocs/83d59103-a103-4c9a-a433-618eff973b63", + "documentNamespace": "https://scancode.io/spdxdocs/0d37341f-f1fc-45c9-bc02-db1eabaccae7", "creationInfo": { "created": "2000-01-01T01:02:03Z", "creators": [ @@ -14,7 +14,7 @@ "packages": [ { "name": "asgiref", - "SPDXID": "SPDXRef-scancodeio-discoveredpackage-a9dcd442-4c55-49b4-9e33-8be74c9fc15b", + "SPDXID": "SPDXRef-scancodeio-discoveredpackage-c9fcc7fb-3613-47d9-92ba-3b3470e5798c", "downloadLocation": "NOASSERTION", "licenseConcluded": "BSD-3-Clause", "copyrightText": "NOASSERTION", @@ -33,7 +33,7 @@ }, { "name": "asgiref", - "SPDXID": "SPDXRef-scancodeio-discoveredpackage-27357e59-c16d-40f1-8cd2-0f11b7376cdb", + "SPDXID": "SPDXRef-scancodeio-discoveredpackage-bafafbff-e654-4a8f-aa74-61b19ed5ce86", "downloadLocation": "NOASSERTION", "licenseConcluded": "BSD-3-Clause", "copyrightText": "NOASSERTION", @@ -52,7 +52,7 @@ }, { "name": "pytest", - "SPDXID": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest?uuid=9f487b4c-4d3e-4d13-9100-89ab897c90ee", + "SPDXID": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest?uuid=52eb9840-989c-407d-8e4b-309535ef9df5", "downloadLocation": "NOASSERTION", "licenseConcluded": "NOASSERTION", "copyrightText": "NOASSERTION", @@ -68,7 +68,7 @@ }, { "name": "pytest", - "SPDXID": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest?uuid=909f72ab-1d6b-41f6-b41a-2b3be388870a", + "SPDXID": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest?uuid=7b581a48-b4b1-40e6-a7e5-a14de28ca546", "downloadLocation": "NOASSERTION", "licenseConcluded": "NOASSERTION", "copyrightText": "NOASSERTION", @@ -84,7 +84,7 @@ }, { "name": "pytest-asyncio", - "SPDXID": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest-asyncio?uuid=7c26cfc3-080d-4d3c-bcfe-fd6a495c5d3a", + "SPDXID": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest-asyncio?uuid=9e2bf13a-0bdf-4894-a0d3-e44c28893055", "downloadLocation": "NOASSERTION", "licenseConcluded": "NOASSERTION", "copyrightText": "NOASSERTION", @@ -100,7 +100,7 @@ }, { "name": "pytest-asyncio", - "SPDXID": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest-asyncio?uuid=e474c205-ceef-444a-a0d7-aee32c8716df", + "SPDXID": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest-asyncio?uuid=02fcb56f-f0d2-438c-989e-54cb8009ebe6", "downloadLocation": "NOASSERTION", "licenseConcluded": "NOASSERTION", "copyrightText": "NOASSERTION", @@ -116,33 +116,33 @@ } ], "documentDescribes": [ - "SPDXRef-scancodeio-discoveredpackage-a9dcd442-4c55-49b4-9e33-8be74c9fc15b", - "SPDXRef-scancodeio-discoveredpackage-27357e59-c16d-40f1-8cd2-0f11b7376cdb", - "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest?uuid=9f487b4c-4d3e-4d13-9100-89ab897c90ee", - "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest?uuid=909f72ab-1d6b-41f6-b41a-2b3be388870a", - "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest-asyncio?uuid=7c26cfc3-080d-4d3c-bcfe-fd6a495c5d3a", - "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest-asyncio?uuid=e474c205-ceef-444a-a0d7-aee32c8716df" + "SPDXRef-scancodeio-discoveredpackage-c9fcc7fb-3613-47d9-92ba-3b3470e5798c", + "SPDXRef-scancodeio-discoveredpackage-bafafbff-e654-4a8f-aa74-61b19ed5ce86", + "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest?uuid=52eb9840-989c-407d-8e4b-309535ef9df5", + "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest?uuid=7b581a48-b4b1-40e6-a7e5-a14de28ca546", + "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest-asyncio?uuid=9e2bf13a-0bdf-4894-a0d3-e44c28893055", + "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest-asyncio?uuid=02fcb56f-f0d2-438c-989e-54cb8009ebe6" ], "files": [], "relationships": [ { - "spdxElementId": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest?uuid=9f487b4c-4d3e-4d13-9100-89ab897c90ee", - "relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-a9dcd442-4c55-49b4-9e33-8be74c9fc15b", + "spdxElementId": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest?uuid=52eb9840-989c-407d-8e4b-309535ef9df5", + "relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-c9fcc7fb-3613-47d9-92ba-3b3470e5798c", "relationshipType": "DEPENDENCY_OF" }, { - "spdxElementId": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest?uuid=909f72ab-1d6b-41f6-b41a-2b3be388870a", - "relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-27357e59-c16d-40f1-8cd2-0f11b7376cdb", + "spdxElementId": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest?uuid=7b581a48-b4b1-40e6-a7e5-a14de28ca546", + "relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-bafafbff-e654-4a8f-aa74-61b19ed5ce86", "relationshipType": "DEPENDENCY_OF" }, { - "spdxElementId": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest-asyncio?uuid=7c26cfc3-080d-4d3c-bcfe-fd6a495c5d3a", - "relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-a9dcd442-4c55-49b4-9e33-8be74c9fc15b", + "spdxElementId": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest-asyncio?uuid=9e2bf13a-0bdf-4894-a0d3-e44c28893055", + "relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-c9fcc7fb-3613-47d9-92ba-3b3470e5798c", "relationshipType": "DEPENDENCY_OF" }, { - "spdxElementId": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest-asyncio?uuid=e474c205-ceef-444a-a0d7-aee32c8716df", - "relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-27357e59-c16d-40f1-8cd2-0f11b7376cdb", + "spdxElementId": "SPDXRef-scancodeio-discovereddependency-pkg:pypi/pytest-asyncio?uuid=02fcb56f-f0d2-438c-989e-54cb8009ebe6", + "relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-bafafbff-e654-4a8f-aa74-61b19ed5ce86", "relationshipType": "DEPENDENCY_OF" } ], diff --git a/scanpipe/tests/data/asgiref-3.3.0_fixtures.json b/scanpipe/tests/data/asgiref-3.3.0_fixtures.json index 0cce145f35..7e34023fb4 100644 --- a/scanpipe/tests/data/asgiref-3.3.0_fixtures.json +++ b/scanpipe/tests/data/asgiref-3.3.0_fixtures.json @@ -1,13 +1,13 @@ [ { "model": "scanpipe.project", - "pk": "83d59103-a103-4c9a-a433-618eff973b63", + "pk": "0d37341f-f1fc-45c9-bc02-db1eabaccae7", "fields": { "extra_data": {}, - "created_date": "2024-06-13T14:23:21.548Z", + "created_date": "2024-06-20T14:35:43.180Z", "name": "asgiref", - "slug": "asgiref-83d59103", - "work_directory": "/tmp/tmp5ihgt8lg/projects/asgiref-83d59103", + "slug": "asgiref-0d37341f", + "work_directory": "/tmp/tmpi98ore1a/projects/asgiref-0d37341f", "is_archived": false, "notes": "", "settings": {} @@ -15,17 +15,17 @@ }, { "model": "scanpipe.run", - "pk": "ceb011c2-6d52-4c3a-bbff-c53e45f0313e", + "pk": "c2be3b12-1cf4-4f1d-9802-411f73c17e58", "fields": { "task_id": null, "task_start_date": null, "task_end_date": null, "task_exitcode": null, "task_output": "", - "log": "2024-06-13 14:23:21.55 Pipeline [scan_codebase] starting\n2024-06-13 14:23:21.55 Step [download_missing_inputs] starting\n2024-06-13 14:23:21.55 Step [download_missing_inputs] completed in 0 seconds\n2024-06-13 14:23:21.55 Step [copy_inputs_to_codebase_directory] starting\n2024-06-13 14:23:21.55 Step [copy_inputs_to_codebase_directory] completed in 0 seconds\n2024-06-13 14:23:21.55 Step [extract_archives] starting\n2024-06-13 14:23:21.60 Step [extract_archives] completed in 0 seconds\n2024-06-13 14:23:21.60 Step [collect_and_create_codebase_resources] starting\n2024-06-13 14:23:21.67 Step [collect_and_create_codebase_resources] completed in 0 seconds\n2024-06-13 14:23:21.67 Step [flag_empty_files] starting\n2024-06-13 14:23:21.67 Step [flag_empty_files] completed in 0 seconds\n2024-06-13 14:23:21.67 Step [flag_ignored_resources] starting\n2024-06-13 14:23:21.67 Step [flag_ignored_resources] completed in 0 seconds\n2024-06-13 14:23:21.68 Step [scan_for_application_packages] starting\n2024-06-13 14:23:21.69 Progress: 11% (2/18)\n2024-06-13 14:23:21.70 Progress: 22% (4/18)\n2024-06-13 14:23:21.71 Progress: 33% (6/18)\n2024-06-13 14:23:21.71 Progress: 44% (8/18)\n2024-06-13 14:23:21.71 Progress: 55% (10/18)\n2024-06-13 14:23:21.71 Progress: 66% (12/18)\n2024-06-13 14:23:21.71 Progress: 77% (14/18)\n2024-06-13 14:23:21.71 Progress: 88% (16/18)\n2024-06-13 14:23:24.08 Progress: 100% (18/18)\n2024-06-13 14:23:26.20 Step [scan_for_application_packages] completed in 5 seconds\n2024-06-13 14:23:26.20 Step [scan_for_files] starting\n2024-06-13 14:23:26.32 Progress: 12% (2/16) ETA: 1 seconds\n2024-06-13 14:23:26.33 Progress: 25% (4/16)\n2024-06-13 14:23:26.35 Progress: 37% (6/16)\n2024-06-13 14:23:26.56 Progress: 50% (8/16)\n2024-06-13 14:23:26.60 Progress: 62% (10/16)\n2024-06-13 14:23:26.63 Progress: 75% (12/16)\n2024-06-13 14:23:26.79 Progress: 87% (14/16)\n2024-06-13 14:23:26.88 Progress: 100% (16/16)\n2024-06-13 14:23:26.90 Step [scan_for_files] completed in 1 seconds\n2024-06-13 14:23:26.90 Pipeline completed in 5 seconds\n", - "project": "83d59103-a103-4c9a-a433-618eff973b63", + "log": "2024-06-20 14:35:43.18 Pipeline [scan_codebase] starting\n2024-06-20 14:35:43.18 Step [download_missing_inputs] starting\n2024-06-20 14:35:43.18 Step [download_missing_inputs] completed in 0 seconds\n2024-06-20 14:35:43.18 Step [copy_inputs_to_codebase_directory] starting\n2024-06-20 14:35:43.18 Step [copy_inputs_to_codebase_directory] completed in 0 seconds\n2024-06-20 14:35:43.18 Step [extract_archives] starting\n2024-06-20 14:35:43.24 Step [extract_archives] completed in 0 seconds\n2024-06-20 14:35:43.24 Step [collect_and_create_codebase_resources] starting\n2024-06-20 14:35:43.32 Step [collect_and_create_codebase_resources] completed in 0 seconds\n2024-06-20 14:35:43.32 Step [flag_empty_files] starting\n2024-06-20 14:35:43.32 Step [flag_empty_files] completed in 0 seconds\n2024-06-20 14:35:43.32 Step [flag_ignored_resources] starting\n2024-06-20 14:35:43.32 Step [flag_ignored_resources] completed in 0 seconds\n2024-06-20 14:35:43.32 Step [scan_for_application_packages] starting\n2024-06-20 14:35:43.35 Progress: 11% (2/18)\n2024-06-20 14:35:43.37 Progress: 22% (4/18)\n2024-06-20 14:35:43.37 Progress: 33% (6/18)\n2024-06-20 14:35:43.37 Progress: 44% (8/18)\n2024-06-20 14:35:43.37 Progress: 55% (10/18)\n2024-06-20 14:35:43.37 Progress: 66% (12/18)\n2024-06-20 14:35:43.37 Progress: 77% (14/18)\n2024-06-20 14:35:43.37 Progress: 88% (16/18)\n2024-06-20 14:35:46.48 Progress: 100% (18/18)\n2024-06-20 14:35:49.01 Step [scan_for_application_packages] completed in 6 seconds\n2024-06-20 14:35:49.01 Step [scan_for_files] starting\n2024-06-20 14:35:49.18 Progress: 12% (2/16) ETA: 1 seconds\n2024-06-20 14:35:49.19 Progress: 25% (4/16) ETA: 1 seconds\n2024-06-20 14:35:49.19 Progress: 37% (6/16)\n2024-06-20 14:35:49.49 Progress: 50% (8/16)\n2024-06-20 14:35:49.59 Progress: 62% (10/16)\n2024-06-20 14:35:49.64 Progress: 75% (12/16)\n2024-06-20 14:35:49.85 Progress: 87% (14/16)\n2024-06-20 14:35:50.04 Progress: 100% (16/16)\n2024-06-20 14:35:50.07 Step [scan_for_files] completed in 1 seconds\n2024-06-20 14:35:50.08 Pipeline completed in 7 seconds\n", + "project": "0d37341f-f1fc-45c9-bc02-db1eabaccae7", "pipeline_name": "scan_codebase", - "created_date": "2024-06-13T14:23:21.551Z", + "created_date": "2024-06-20T14:35:43.184Z", "scancodeio_version": "", "description": "Scan a codebase for application packages, licenses, and copyrights.", "current_step": "", @@ -41,7 +41,7 @@ "sha256": "a5098bc870b80e7b872bff60bb363c7f2c2c89078759f6c47b53ff8c525a152e", "sha512": "", "extra_data": {}, - "project": "83d59103-a103-4c9a-a433-618eff973b63", + "project": "0d37341f-f1fc-45c9-bc02-db1eabaccae7", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -346,7 +346,7 @@ "sha256": "", "sha512": "", "extra_data": {}, - "project": "83d59103-a103-4c9a-a433-618eff973b63", + "project": "0d37341f-f1fc-45c9-bc02-db1eabaccae7", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -386,7 +386,7 @@ "sha256": "", "sha512": "", "extra_data": {}, - "project": "83d59103-a103-4c9a-a433-618eff973b63", + "project": "0d37341f-f1fc-45c9-bc02-db1eabaccae7", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -426,7 +426,7 @@ "sha256": "", "sha512": "", "extra_data": {}, - "project": "83d59103-a103-4c9a-a433-618eff973b63", + "project": "0d37341f-f1fc-45c9-bc02-db1eabaccae7", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -466,7 +466,7 @@ "sha256": "6e89108c2cf0c0446174188f76f60465ae1c1f14f83427807df40d52a27cb2c8", "sha512": "", "extra_data": {}, - "project": "83d59103-a103-4c9a-a433-618eff973b63", + "project": "0d37341f-f1fc-45c9-bc02-db1eabaccae7", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -506,7 +506,7 @@ "sha256": "b846415d1b514e9c1dff14a22deb906d794bc546ca6129f950a18cd091e2a669", "sha512": "", "extra_data": {}, - "project": "83d59103-a103-4c9a-a433-618eff973b63", + "project": "0d37341f-f1fc-45c9-bc02-db1eabaccae7", "detected_license_expression": "bsd-new", "detected_license_expression_spdx": "BSD-3-Clause", "license_detections": [ @@ -581,7 +581,7 @@ "sha256": "70f98f4eb9f6068b192b5464fcdf69e29a8ff09962bfce84bbb052baeee44f33", "sha512": "", "extra_data": {}, - "project": "83d59103-a103-4c9a-a433-618eff973b63", + "project": "0d37341f-f1fc-45c9-bc02-db1eabaccae7", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -886,7 +886,7 @@ "sha256": "11546323af45e6a5639bf620a9c4d73e74c0bf705f494af4595007b923f75e8a", "sha512": "", "extra_data": {}, - "project": "83d59103-a103-4c9a-a433-618eff973b63", + "project": "0d37341f-f1fc-45c9-bc02-db1eabaccae7", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -926,7 +926,7 @@ "sha256": "2c1983592aa38f0bfb0afacc73ddc5b46ce10e8e89ceaa9fed1e5fc6361b608d", "sha512": "", "extra_data": {}, - "project": "83d59103-a103-4c9a-a433-618eff973b63", + "project": "0d37341f-f1fc-45c9-bc02-db1eabaccae7", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -966,7 +966,7 @@ "sha256": "30f49b9094bff904a42caeec32515715fe625a56dc48bd7c0e3d9988c0ad4bd7", "sha512": "", "extra_data": {}, - "project": "83d59103-a103-4c9a-a433-618eff973b63", + "project": "0d37341f-f1fc-45c9-bc02-db1eabaccae7", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -1006,7 +1006,7 @@ "sha256": "fa4651a3b79201a4dc44a4096cd49ec8f427e912ea0ee05c666357b413a8afe7", "sha512": "", "extra_data": {}, - "project": "83d59103-a103-4c9a-a433-618eff973b63", + "project": "0d37341f-f1fc-45c9-bc02-db1eabaccae7", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -1046,7 +1046,7 @@ "sha256": "ee0fcf4a8e6fa9df8a4643bb48e82892d496afce44b6c8b8aea2721755545e1c", "sha512": "", "extra_data": {}, - "project": "83d59103-a103-4c9a-a433-618eff973b63", + "project": "0d37341f-f1fc-45c9-bc02-db1eabaccae7", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -1086,7 +1086,7 @@ "sha256": "3151f66c476208c3154cb6c4fb557a2a253bab82f0ab33fb3c8b9f7976be9e33", "sha512": "", "extra_data": {}, - "project": "83d59103-a103-4c9a-a433-618eff973b63", + "project": "0d37341f-f1fc-45c9-bc02-db1eabaccae7", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -1126,7 +1126,7 @@ "sha256": "ddd445b778c097fc75c2bf69ad964cbadd3bd6999d1dd2306d39d401855e8e3e", "sha512": "", "extra_data": {}, - "project": "83d59103-a103-4c9a-a433-618eff973b63", + "project": "0d37341f-f1fc-45c9-bc02-db1eabaccae7", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -1166,7 +1166,7 @@ "sha256": "ddbc8d455eceb68fc583c67e7c4ad0277c867fb39095c51ec5b37f70342e8334", "sha512": "", "extra_data": {}, - "project": "83d59103-a103-4c9a-a433-618eff973b63", + "project": "0d37341f-f1fc-45c9-bc02-db1eabaccae7", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -1206,7 +1206,7 @@ "sha256": "126c3e3a8a75a517d2739612304607804cf5f34da63fa25d03a6f11f7edb6f2f", "sha512": "", "extra_data": {}, - "project": "83d59103-a103-4c9a-a433-618eff973b63", + "project": "0d37341f-f1fc-45c9-bc02-db1eabaccae7", "detected_license_expression": "apache-2.0", "detected_license_expression_spdx": "Apache-2.0", "license_detections": [ @@ -1285,7 +1285,7 @@ "sha256": "f8bd1ea3fb8afddabb10f8efd66796d41446cad51168ef4d3c44b19c973d0ad0", "sha512": "", "extra_data": {}, - "project": "83d59103-a103-4c9a-a433-618eff973b63", + "project": "0d37341f-f1fc-45c9-bc02-db1eabaccae7", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -1325,7 +1325,7 @@ "sha256": "885267fee0fea687875a02ceb929ca095312d47aaa57e20e4ce382f397caaf4d", "sha512": "", "extra_data": {}, - "project": "83d59103-a103-4c9a-a433-618eff973b63", + "project": "0d37341f-f1fc-45c9-bc02-db1eabaccae7", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -1375,7 +1375,7 @@ "Documentation": "https://asgi.readthedocs.io/", "Further Documentation": "https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions" }, - "project": "83d59103-a103-4c9a-a433-618eff973b63", + "project": "0d37341f-f1fc-45c9-bc02-db1eabaccae7", "compliance_alert": "missing", "affected_by_vulnerabilities": [], "filename": "", @@ -1464,10 +1464,10 @@ "email": "foundation@djangoproject.com" } ], - "uuid": "a9dcd442-4c55-49b4-9e33-8be74c9fc15b", + "uuid": "c9fcc7fb-3613-47d9-92ba-3b3470e5798c", "missing_resources": [], "modified_resources": [], - "package_uid": "pkg:pypi/asgiref@3.3.0?uuid=eca680a3-115b-494b-8581-81c93384e22d", + "package_uid": "pkg:pypi/asgiref@3.3.0?uuid=4705892e-924f-483a-a951-05109c930b3d", "keywords": [ "Development Status :: 5 - Production/Stable", "Environment :: Web Environment", @@ -1508,7 +1508,7 @@ "Documentation": "https://asgi.readthedocs.io/", "Further Documentation": "https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions" }, - "project": "83d59103-a103-4c9a-a433-618eff973b63", + "project": "0d37341f-f1fc-45c9-bc02-db1eabaccae7", "compliance_alert": "missing", "affected_by_vulnerabilities": [], "filename": "", @@ -1597,10 +1597,10 @@ "email": "foundation@djangoproject.com" } ], - "uuid": "27357e59-c16d-40f1-8cd2-0f11b7376cdb", + "uuid": "bafafbff-e654-4a8f-aa74-61b19ed5ce86", "missing_resources": [], "modified_resources": [], - "package_uid": "pkg:pypi/asgiref@3.3.0?uuid=510ea8b0-d7a7-4531-96ea-f41cb4384976", + "package_uid": "pkg:pypi/asgiref@3.3.0?uuid=52c2a744-d7a9-4afc-b38a-5d86a19d9c37", "keywords": [ "Development Status :: 5 - Production/Stable", "Environment :: Web Environment", @@ -1646,8 +1646,8 @@ "qualifiers": "", "subpath": "", "affected_by_vulnerabilities": [], - "project": "83d59103-a103-4c9a-a433-618eff973b63", - "dependency_uid": "pkg:pypi/pytest?uuid=9f487b4c-4d3e-4d13-9100-89ab897c90ee", + "project": "0d37341f-f1fc-45c9-bc02-db1eabaccae7", + "dependency_uid": "pkg:pypi/pytest?uuid=52eb9840-989c-407d-8e4b-309535ef9df5", "for_package": 1, "resolved_to_package": null, "datafile_resource": 1, @@ -1671,8 +1671,8 @@ "qualifiers": "", "subpath": "", "affected_by_vulnerabilities": [], - "project": "83d59103-a103-4c9a-a433-618eff973b63", - "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=7c26cfc3-080d-4d3c-bcfe-fd6a495c5d3a", + "project": "0d37341f-f1fc-45c9-bc02-db1eabaccae7", + "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=9e2bf13a-0bdf-4894-a0d3-e44c28893055", "for_package": 1, "resolved_to_package": null, "datafile_resource": 1, @@ -1696,8 +1696,8 @@ "qualifiers": "", "subpath": "", "affected_by_vulnerabilities": [], - "project": "83d59103-a103-4c9a-a433-618eff973b63", - "dependency_uid": "pkg:pypi/pytest?uuid=909f72ab-1d6b-41f6-b41a-2b3be388870a", + "project": "0d37341f-f1fc-45c9-bc02-db1eabaccae7", + "dependency_uid": "pkg:pypi/pytest?uuid=7b581a48-b4b1-40e6-a7e5-a14de28ca546", "for_package": 2, "resolved_to_package": null, "datafile_resource": 7, @@ -1721,8 +1721,8 @@ "qualifiers": "", "subpath": "", "affected_by_vulnerabilities": [], - "project": "83d59103-a103-4c9a-a433-618eff973b63", - "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=e474c205-ceef-444a-a0d7-aee32c8716df", + "project": "0d37341f-f1fc-45c9-bc02-db1eabaccae7", + "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=02fcb56f-f0d2-438c-989e-54cb8009ebe6", "for_package": 2, "resolved_to_package": null, "datafile_resource": 7, diff --git a/scanpipe/tests/data/asgiref-3.3.0_scanpipe_output.json b/scanpipe/tests/data/asgiref-3.3.0_scanpipe_output.json index 1393778a64..97d2cd3153 100644 --- a/scanpipe/tests/data/asgiref-3.3.0_scanpipe_output.json +++ b/scanpipe/tests/data/asgiref-3.3.0_scanpipe_output.json @@ -2,18 +2,18 @@ "headers": [ { "tool_name": "scanpipe", - "tool_version": "v34.5.0-8-g21a9308", + "tool_version": "v34.6.2-9-g4aef6db", "other_tools": [ - "pkg:pypi/scancode-toolkit@32.1.0" + "pkg:pypi/scancode-toolkit@32.2.0" ], "notice": "Generated with ScanCode.io and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied.\nNo content created from ScanCode.io should be considered or used as legal advice.\nConsult an Attorney for any legal advice.\nScanCode.io is a free software code scanning tool from nexB Inc. and others\nlicensed under the Apache License version 2.0.\nScanCode is a trademark of nexB Inc.\nVisit https://github.com/nexB/scancode.io for support and download.\n", - "uuid": "83d59103-a103-4c9a-a433-618eff973b63", - "created_date": "2024-06-13T14:23:21.548Z", + "uuid": "0d37341f-f1fc-45c9-bc02-db1eabaccae7", + "created_date": "2024-06-20T14:35:43.180Z", "notes": "", "settings": {}, "input_sources": [ { - "uuid": "075464b8-34ad-44ef-b5d3-99a50d79c5d4", + "uuid": "1d424247-8840-43f1-bfe0-277d73c72106", "filename": "asgiref-3.3.0-py3-none-any.whl", "download_url": "", "is_uploaded": true, @@ -28,15 +28,15 @@ "pipeline_name": "scan_codebase", "status": "not_started", "description": "Scan a codebase for application packages, licenses, and copyrights.", - "uuid": "ceb011c2-6d52-4c3a-bbff-c53e45f0313e", - "created_date": "2024-06-13T14:23:21.551739Z", + "uuid": "c2be3b12-1cf4-4f1d-9802-411f73c17e58", + "created_date": "2024-06-20T14:35:43.184006Z", "scancodeio_version": "", "task_id": null, "task_start_date": null, "task_end_date": null, "task_exitcode": null, "task_output": "", - "log": "2024-06-13 14:23:21.55 Pipeline [scan_codebase] starting\n2024-06-13 14:23:21.55 Step [download_missing_inputs] starting\n2024-06-13 14:23:21.55 Step [download_missing_inputs] completed in 0 seconds\n2024-06-13 14:23:21.55 Step [copy_inputs_to_codebase_directory] starting\n2024-06-13 14:23:21.55 Step [copy_inputs_to_codebase_directory] completed in 0 seconds\n2024-06-13 14:23:21.55 Step [extract_archives] starting\n2024-06-13 14:23:21.60 Step [extract_archives] completed in 0 seconds\n2024-06-13 14:23:21.60 Step [collect_and_create_codebase_resources] starting\n2024-06-13 14:23:21.67 Step [collect_and_create_codebase_resources] completed in 0 seconds\n2024-06-13 14:23:21.67 Step [flag_empty_files] starting\n2024-06-13 14:23:21.67 Step [flag_empty_files] completed in 0 seconds\n2024-06-13 14:23:21.67 Step [flag_ignored_resources] starting\n2024-06-13 14:23:21.67 Step [flag_ignored_resources] completed in 0 seconds\n2024-06-13 14:23:21.68 Step [scan_for_application_packages] starting\n2024-06-13 14:23:21.69 Progress: 11% (2/18)\n2024-06-13 14:23:21.70 Progress: 22% (4/18)\n2024-06-13 14:23:21.71 Progress: 33% (6/18)\n2024-06-13 14:23:21.71 Progress: 44% (8/18)\n2024-06-13 14:23:21.71 Progress: 55% (10/18)\n2024-06-13 14:23:21.71 Progress: 66% (12/18)\n2024-06-13 14:23:21.71 Progress: 77% (14/18)\n2024-06-13 14:23:21.71 Progress: 88% (16/18)\n2024-06-13 14:23:24.08 Progress: 100% (18/18)\n2024-06-13 14:23:26.20 Step [scan_for_application_packages] completed in 5 seconds\n2024-06-13 14:23:26.20 Step [scan_for_files] starting\n2024-06-13 14:23:26.32 Progress: 12% (2/16) ETA: 1 seconds\n2024-06-13 14:23:26.33 Progress: 25% (4/16)\n2024-06-13 14:23:26.35 Progress: 37% (6/16)\n2024-06-13 14:23:26.56 Progress: 50% (8/16)\n2024-06-13 14:23:26.60 Progress: 62% (10/16)\n2024-06-13 14:23:26.63 Progress: 75% (12/16)\n2024-06-13 14:23:26.79 Progress: 87% (14/16)\n2024-06-13 14:23:26.88 Progress: 100% (16/16)\n2024-06-13 14:23:26.90 Step [scan_for_files] completed in 1 seconds\n2024-06-13 14:23:26.90 Pipeline completed in 5 seconds\n", + "log": "2024-06-20 14:35:43.18 Pipeline [scan_codebase] starting\n2024-06-20 14:35:43.18 Step [download_missing_inputs] starting\n2024-06-20 14:35:43.18 Step [download_missing_inputs] completed in 0 seconds\n2024-06-20 14:35:43.18 Step [copy_inputs_to_codebase_directory] starting\n2024-06-20 14:35:43.18 Step [copy_inputs_to_codebase_directory] completed in 0 seconds\n2024-06-20 14:35:43.18 Step [extract_archives] starting\n2024-06-20 14:35:43.24 Step [extract_archives] completed in 0 seconds\n2024-06-20 14:35:43.24 Step [collect_and_create_codebase_resources] starting\n2024-06-20 14:35:43.32 Step [collect_and_create_codebase_resources] completed in 0 seconds\n2024-06-20 14:35:43.32 Step [flag_empty_files] starting\n2024-06-20 14:35:43.32 Step [flag_empty_files] completed in 0 seconds\n2024-06-20 14:35:43.32 Step [flag_ignored_resources] starting\n2024-06-20 14:35:43.32 Step [flag_ignored_resources] completed in 0 seconds\n2024-06-20 14:35:43.32 Step [scan_for_application_packages] starting\n2024-06-20 14:35:43.35 Progress: 11% (2/18)\n2024-06-20 14:35:43.37 Progress: 22% (4/18)\n2024-06-20 14:35:43.37 Progress: 33% (6/18)\n2024-06-20 14:35:43.37 Progress: 44% (8/18)\n2024-06-20 14:35:43.37 Progress: 55% (10/18)\n2024-06-20 14:35:43.37 Progress: 66% (12/18)\n2024-06-20 14:35:43.37 Progress: 77% (14/18)\n2024-06-20 14:35:43.37 Progress: 88% (16/18)\n2024-06-20 14:35:46.48 Progress: 100% (18/18)\n2024-06-20 14:35:49.01 Step [scan_for_application_packages] completed in 6 seconds\n2024-06-20 14:35:49.01 Step [scan_for_files] starting\n2024-06-20 14:35:49.18 Progress: 12% (2/16) ETA: 1 seconds\n2024-06-20 14:35:49.19 Progress: 25% (4/16) ETA: 1 seconds\n2024-06-20 14:35:49.19 Progress: 37% (6/16)\n2024-06-20 14:35:49.49 Progress: 50% (8/16)\n2024-06-20 14:35:49.59 Progress: 62% (10/16)\n2024-06-20 14:35:49.64 Progress: 75% (12/16)\n2024-06-20 14:35:49.85 Progress: 87% (14/16)\n2024-06-20 14:35:50.04 Progress: 100% (16/16)\n2024-06-20 14:35:50.07 Step [scan_for_files] completed in 1 seconds\n2024-06-20 14:35:50.08 Pipeline completed in 7 seconds\n", "execution_time": null } ], @@ -154,7 +154,7 @@ "Documentation": "https://asgi.readthedocs.io/", "Further Documentation": "https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions" }, - "package_uid": "pkg:pypi/asgiref@3.3.0?uuid=eca680a3-115b-494b-8581-81c93384e22d", + "package_uid": "pkg:pypi/asgiref@3.3.0?uuid=4705892e-924f-483a-a951-05109c930b3d", "is_private": false, "is_virtual": false, "datasource_ids": [ @@ -278,7 +278,7 @@ "Documentation": "https://asgi.readthedocs.io/", "Further Documentation": "https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions" }, - "package_uid": "pkg:pypi/asgiref@3.3.0?uuid=510ea8b0-d7a7-4531-96ea-f41cb4384976", + "package_uid": "pkg:pypi/asgiref@3.3.0?uuid=52c2a744-d7a9-4afc-b38a-5d86a19d9c37", "is_private": false, "is_virtual": false, "datasource_ids": [ @@ -302,8 +302,8 @@ "is_optional": true, "is_resolved": false, "is_direct": true, - "dependency_uid": "pkg:pypi/pytest?uuid=9f487b4c-4d3e-4d13-9100-89ab897c90ee", - "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=eca680a3-115b-494b-8581-81c93384e22d", + "dependency_uid": "pkg:pypi/pytest?uuid=52eb9840-989c-407d-8e4b-309535ef9df5", + "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=4705892e-924f-483a-a951-05109c930b3d", "resolved_to_package_uid": null, "datafile_path": "asgiref-3.3.0-py3-none-any.whl", "datasource_id": "pypi_wheel", @@ -318,8 +318,8 @@ "is_optional": true, "is_resolved": false, "is_direct": true, - "dependency_uid": "pkg:pypi/pytest?uuid=909f72ab-1d6b-41f6-b41a-2b3be388870a", - "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=510ea8b0-d7a7-4531-96ea-f41cb4384976", + "dependency_uid": "pkg:pypi/pytest?uuid=7b581a48-b4b1-40e6-a7e5-a14de28ca546", + "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=52c2a744-d7a9-4afc-b38a-5d86a19d9c37", "resolved_to_package_uid": null, "datafile_path": "asgiref-3.3.0-py3-none-any.whl-extract/asgiref-3.3.0.dist-info/METADATA", "datasource_id": "pypi_wheel_metadata", @@ -334,8 +334,8 @@ "is_optional": true, "is_resolved": false, "is_direct": true, - "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=7c26cfc3-080d-4d3c-bcfe-fd6a495c5d3a", - "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=eca680a3-115b-494b-8581-81c93384e22d", + "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=9e2bf13a-0bdf-4894-a0d3-e44c28893055", + "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=4705892e-924f-483a-a951-05109c930b3d", "resolved_to_package_uid": null, "datafile_path": "asgiref-3.3.0-py3-none-any.whl", "datasource_id": "pypi_wheel", @@ -350,8 +350,8 @@ "is_optional": true, "is_resolved": false, "is_direct": true, - "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=e474c205-ceef-444a-a0d7-aee32c8716df", - "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=510ea8b0-d7a7-4531-96ea-f41cb4384976", + "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=02fcb56f-f0d2-438c-989e-54cb8009ebe6", + "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=52c2a744-d7a9-4afc-b38a-5d86a19d9c37", "resolved_to_package_uid": null, "datafile_path": "asgiref-3.3.0-py3-none-any.whl-extract/asgiref-3.3.0.dist-info/METADATA", "datasource_id": "pypi_wheel_metadata", @@ -656,7 +656,7 @@ } ], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=eca680a3-115b-494b-8581-81c93384e22d" + "pkg:pypi/asgiref@3.3.0?uuid=4705892e-924f-483a-a951-05109c930b3d" ], "emails": [], "urls": [], @@ -833,7 +833,7 @@ "authors": [], "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=510ea8b0-d7a7-4531-96ea-f41cb4384976" + "pkg:pypi/asgiref@3.3.0?uuid=52c2a744-d7a9-4afc-b38a-5d86a19d9c37" ], "emails": [], "urls": [], @@ -1135,7 +1135,7 @@ } ], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=510ea8b0-d7a7-4531-96ea-f41cb4384976" + "pkg:pypi/asgiref@3.3.0?uuid=52c2a744-d7a9-4afc-b38a-5d86a19d9c37" ], "emails": [], "urls": [], @@ -1172,7 +1172,7 @@ "authors": [], "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=510ea8b0-d7a7-4531-96ea-f41cb4384976" + "pkg:pypi/asgiref@3.3.0?uuid=52c2a744-d7a9-4afc-b38a-5d86a19d9c37" ], "emails": [], "urls": [], @@ -1209,7 +1209,7 @@ "authors": [], "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=510ea8b0-d7a7-4531-96ea-f41cb4384976" + "pkg:pypi/asgiref@3.3.0?uuid=52c2a744-d7a9-4afc-b38a-5d86a19d9c37" ], "emails": [], "urls": [], @@ -1246,7 +1246,7 @@ "authors": [], "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=510ea8b0-d7a7-4531-96ea-f41cb4384976" + "pkg:pypi/asgiref@3.3.0?uuid=52c2a744-d7a9-4afc-b38a-5d86a19d9c37" ], "emails": [], "urls": [], @@ -1283,7 +1283,7 @@ "authors": [], "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=510ea8b0-d7a7-4531-96ea-f41cb4384976" + "pkg:pypi/asgiref@3.3.0?uuid=52c2a744-d7a9-4afc-b38a-5d86a19d9c37" ], "emails": [], "urls": [], @@ -1320,7 +1320,7 @@ "authors": [], "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=510ea8b0-d7a7-4531-96ea-f41cb4384976" + "pkg:pypi/asgiref@3.3.0?uuid=52c2a744-d7a9-4afc-b38a-5d86a19d9c37" ], "emails": [], "urls": [], @@ -1357,7 +1357,7 @@ "authors": [], "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=510ea8b0-d7a7-4531-96ea-f41cb4384976" + "pkg:pypi/asgiref@3.3.0?uuid=52c2a744-d7a9-4afc-b38a-5d86a19d9c37" ], "emails": [], "urls": [], @@ -1394,7 +1394,7 @@ "authors": [], "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=510ea8b0-d7a7-4531-96ea-f41cb4384976" + "pkg:pypi/asgiref@3.3.0?uuid=52c2a744-d7a9-4afc-b38a-5d86a19d9c37" ], "emails": [], "urls": [], @@ -1431,7 +1431,7 @@ "authors": [], "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=510ea8b0-d7a7-4531-96ea-f41cb4384976" + "pkg:pypi/asgiref@3.3.0?uuid=52c2a744-d7a9-4afc-b38a-5d86a19d9c37" ], "emails": [], "urls": [], @@ -1468,7 +1468,7 @@ "authors": [], "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=510ea8b0-d7a7-4531-96ea-f41cb4384976" + "pkg:pypi/asgiref@3.3.0?uuid=52c2a744-d7a9-4afc-b38a-5d86a19d9c37" ], "emails": [], "urls": [], @@ -1505,7 +1505,7 @@ "authors": [], "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=510ea8b0-d7a7-4531-96ea-f41cb4384976" + "pkg:pypi/asgiref@3.3.0?uuid=52c2a744-d7a9-4afc-b38a-5d86a19d9c37" ], "emails": [], "urls": [], @@ -1565,7 +1565,7 @@ "authors": [], "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=510ea8b0-d7a7-4531-96ea-f41cb4384976" + "pkg:pypi/asgiref@3.3.0?uuid=52c2a744-d7a9-4afc-b38a-5d86a19d9c37" ], "emails": [], "urls": [ @@ -1618,7 +1618,7 @@ "authors": [], "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=510ea8b0-d7a7-4531-96ea-f41cb4384976" + "pkg:pypi/asgiref@3.3.0?uuid=52c2a744-d7a9-4afc-b38a-5d86a19d9c37" ], "emails": [], "urls": [], diff --git a/scanpipe/tests/data/asgiref-3.3.0_toolkit_scan.json b/scanpipe/tests/data/asgiref-3.3.0_toolkit_scan.json index 9d65b79a11..edc9bb99e5 100644 --- a/scanpipe/tests/data/asgiref-3.3.0_toolkit_scan.json +++ b/scanpipe/tests/data/asgiref-3.3.0_toolkit_scan.json @@ -2,7 +2,7 @@ "headers": [ { "tool_name": "scancode-toolkit", - "tool_version": "32.1.0", + "tool_version": "32.2.0", "options": { "--copyright": true, "--info": true, @@ -10,10 +10,10 @@ "--package": true }, "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", - "start_timestamp": "2024-06-13T142326.906987", - "end_timestamp": "2024-06-13T142327.694160", - "output_format_version": "3.1.0", - "duration": 0.7872016429901123, + "start_timestamp": "2024-06-20T143550.082400", + "end_timestamp": "2024-06-20T143551.343015", + "output_format_version": "3.2.0", + "duration": 1.2606468200683594, "message": null, "errors": [], "warnings": [], @@ -25,7 +25,7 @@ "platform_version": "#122-Ubuntu SMP Thu May 23 07:48:21 UTC 2024", "python_version": "3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0]" }, - "spdx_license_list_version": "3.23", + "spdx_license_list_version": "3.24", "files_count": 15 } } @@ -140,7 +140,7 @@ "repository_homepage_url": "https://pypi.org/project/asgiref", "repository_download_url": "https://pypi.org/packages/source/a/asgiref/asgiref-3.3.0.tar.gz", "api_data_url": "https://pypi.org/pypi/asgiref/3.3.0/json", - "package_uid": "pkg:pypi/asgiref@3.3.0?uuid=6497a3d3-f29b-408c-8eb6-5c9a262cae90", + "package_uid": "pkg:pypi/asgiref@3.3.0?uuid=45853fcf-d50a-41e7-8575-615fbf5f2fcc", "datafile_paths": [ "codebase/asgiref-3.3.0-py3-none-any.whl" ], @@ -258,7 +258,7 @@ "repository_homepage_url": "https://pypi.org/project/asgiref", "repository_download_url": "https://pypi.org/packages/source/a/asgiref/asgiref-3.3.0.tar.gz", "api_data_url": "https://pypi.org/pypi/asgiref/3.3.0/json", - "package_uid": "pkg:pypi/asgiref@3.3.0?uuid=6bcf52b9-d880-4574-b1fd-f5dabd882b75", + "package_uid": "pkg:pypi/asgiref@3.3.0?uuid=ec9d584a-0961-42b7-ae50-897a4a0ed991", "datafile_paths": [ "codebase/asgiref-3.3.0-py3-none-any.whl-extract/asgiref-3.3.0.dist-info/METADATA" ], @@ -279,8 +279,8 @@ "is_direct": true, "resolved_package": {}, "extra_data": {}, - "dependency_uid": "pkg:pypi/pytest?uuid=a151ecd1-4069-48e0-8612-2d78064b6fdc", - "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=6497a3d3-f29b-408c-8eb6-5c9a262cae90", + "dependency_uid": "pkg:pypi/pytest?uuid=eb1a4f8a-d143-42a8-92b4-2812e7799b42", + "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=45853fcf-d50a-41e7-8575-615fbf5f2fcc", "datafile_path": "codebase/asgiref-3.3.0-py3-none-any.whl", "datasource_id": "pypi_wheel" }, @@ -294,8 +294,8 @@ "is_direct": true, "resolved_package": {}, "extra_data": {}, - "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=f36d232d-fa28-4ff4-8095-44bc504c8ede", - "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=6497a3d3-f29b-408c-8eb6-5c9a262cae90", + "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=ad12e610-74a0-444f-8534-463c80a8c91d", + "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=45853fcf-d50a-41e7-8575-615fbf5f2fcc", "datafile_path": "codebase/asgiref-3.3.0-py3-none-any.whl", "datasource_id": "pypi_wheel" }, @@ -309,8 +309,8 @@ "is_direct": true, "resolved_package": {}, "extra_data": {}, - "dependency_uid": "pkg:pypi/pytest?uuid=bf7aeeb1-44f2-4958-91ec-8b887c95f9a1", - "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=6bcf52b9-d880-4574-b1fd-f5dabd882b75", + "dependency_uid": "pkg:pypi/pytest?uuid=fed8b279-02b7-49e7-8861-0ec442a83520", + "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=ec9d584a-0961-42b7-ae50-897a4a0ed991", "datafile_path": "codebase/asgiref-3.3.0-py3-none-any.whl-extract/asgiref-3.3.0.dist-info/METADATA", "datasource_id": "pypi_wheel_metadata" }, @@ -324,8 +324,8 @@ "is_direct": true, "resolved_package": {}, "extra_data": {}, - "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=84b2883c-b748-4224-b0e2-1b3c8906396a", - "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=6bcf52b9-d880-4574-b1fd-f5dabd882b75", + "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=9709beb0-796d-4d75-97cf-54ef5bc2eca5", + "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=ec9d584a-0961-42b7-ae50-897a4a0ed991", "datafile_path": "codebase/asgiref-3.3.0-py3-none-any.whl-extract/asgiref-3.3.0.dist-info/METADATA", "datasource_id": "pypi_wheel_metadata" } @@ -463,7 +463,7 @@ "base_name": "asgiref-3.3.0-py3-none-any", "extension": ".whl", "size": 19948, - "date": "2024-06-13", + "date": "2024-06-20", "sha1": "c03f67211a311b13d1294ac8af7cb139ee34c4f9", "md5": "5bce1df6dedc53a41a9a6b40d7b1699e", "sha256": "a5098bc870b80e7b872bff60bb363c7f2c2c89078759f6c47b53ff8c525a152e", @@ -743,7 +743,7 @@ } ], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=6497a3d3-f29b-408c-8eb6-5c9a262cae90" + "pkg:pypi/asgiref@3.3.0?uuid=45853fcf-d50a-41e7-8575-615fbf5f2fcc" ], "detected_license_expression": null, "detected_license_expression_spdx": null, @@ -850,7 +850,7 @@ "is_script": false, "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=6bcf52b9-d880-4574-b1fd-f5dabd882b75" + "pkg:pypi/asgiref@3.3.0?uuid=ec9d584a-0961-42b7-ae50-897a4a0ed991" ], "detected_license_expression": null, "detected_license_expression_spdx": null, @@ -887,7 +887,7 @@ "is_script": true, "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=6bcf52b9-d880-4574-b1fd-f5dabd882b75" + "pkg:pypi/asgiref@3.3.0?uuid=ec9d584a-0961-42b7-ae50-897a4a0ed991" ], "detected_license_expression": null, "detected_license_expression_spdx": null, @@ -924,7 +924,7 @@ "is_script": true, "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=6bcf52b9-d880-4574-b1fd-f5dabd882b75" + "pkg:pypi/asgiref@3.3.0?uuid=ec9d584a-0961-42b7-ae50-897a4a0ed991" ], "detected_license_expression": null, "detected_license_expression_spdx": null, @@ -961,7 +961,7 @@ "is_script": true, "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=6bcf52b9-d880-4574-b1fd-f5dabd882b75" + "pkg:pypi/asgiref@3.3.0?uuid=ec9d584a-0961-42b7-ae50-897a4a0ed991" ], "detected_license_expression": null, "detected_license_expression_spdx": null, @@ -998,7 +998,7 @@ "is_script": true, "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=6bcf52b9-d880-4574-b1fd-f5dabd882b75" + "pkg:pypi/asgiref@3.3.0?uuid=ec9d584a-0961-42b7-ae50-897a4a0ed991" ], "detected_license_expression": null, "detected_license_expression_spdx": null, @@ -1035,7 +1035,7 @@ "is_script": true, "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=6bcf52b9-d880-4574-b1fd-f5dabd882b75" + "pkg:pypi/asgiref@3.3.0?uuid=ec9d584a-0961-42b7-ae50-897a4a0ed991" ], "detected_license_expression": null, "detected_license_expression_spdx": null, @@ -1072,7 +1072,7 @@ "is_script": true, "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=6bcf52b9-d880-4574-b1fd-f5dabd882b75" + "pkg:pypi/asgiref@3.3.0?uuid=ec9d584a-0961-42b7-ae50-897a4a0ed991" ], "detected_license_expression": null, "detected_license_expression_spdx": null, @@ -1109,7 +1109,7 @@ "is_script": true, "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=6bcf52b9-d880-4574-b1fd-f5dabd882b75" + "pkg:pypi/asgiref@3.3.0?uuid=ec9d584a-0961-42b7-ae50-897a4a0ed991" ], "detected_license_expression": "apache-2.0", "detected_license_expression_spdx": "Apache-2.0", @@ -1168,7 +1168,7 @@ "is_script": true, "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=6bcf52b9-d880-4574-b1fd-f5dabd882b75" + "pkg:pypi/asgiref@3.3.0?uuid=ec9d584a-0961-42b7-ae50-897a4a0ed991" ], "detected_license_expression": null, "detected_license_expression_spdx": null, @@ -1240,7 +1240,7 @@ "is_script": false, "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=6bcf52b9-d880-4574-b1fd-f5dabd882b75" + "pkg:pypi/asgiref@3.3.0?uuid=ec9d584a-0961-42b7-ae50-897a4a0ed991" ], "detected_license_expression": "bsd-new", "detected_license_expression_spdx": "BSD-3-Clause", @@ -1576,7 +1576,7 @@ } ], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=6bcf52b9-d880-4574-b1fd-f5dabd882b75" + "pkg:pypi/asgiref@3.3.0?uuid=ec9d584a-0961-42b7-ae50-897a4a0ed991" ], "detected_license_expression": "bsd-new", "detected_license_expression_spdx": "BSD-3-Clause", @@ -1662,7 +1662,7 @@ "is_script": false, "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=6bcf52b9-d880-4574-b1fd-f5dabd882b75" + "pkg:pypi/asgiref@3.3.0?uuid=ec9d584a-0961-42b7-ae50-897a4a0ed991" ], "detected_license_expression": null, "detected_license_expression_spdx": null, @@ -1699,7 +1699,7 @@ "is_script": false, "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=6bcf52b9-d880-4574-b1fd-f5dabd882b75" + "pkg:pypi/asgiref@3.3.0?uuid=ec9d584a-0961-42b7-ae50-897a4a0ed991" ], "detected_license_expression": null, "detected_license_expression_spdx": null, @@ -1736,7 +1736,7 @@ "is_script": false, "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=6bcf52b9-d880-4574-b1fd-f5dabd882b75" + "pkg:pypi/asgiref@3.3.0?uuid=ec9d584a-0961-42b7-ae50-897a4a0ed991" ], "detected_license_expression": null, "detected_license_expression_spdx": null, diff --git a/scanpipe/tests/data/asgiref-3.3.0_walk_test_fixtures.json b/scanpipe/tests/data/asgiref-3.3.0_walk_test_fixtures.json index dfcd1d102f..31f26dc1c2 100644 --- a/scanpipe/tests/data/asgiref-3.3.0_walk_test_fixtures.json +++ b/scanpipe/tests/data/asgiref-3.3.0_walk_test_fixtures.json @@ -1,13 +1,13 @@ [ { "model": "scanpipe.project", - "pk": "83d59103-a103-4c9a-a433-618eff973b63", + "pk": "0d37341f-f1fc-45c9-bc02-db1eabaccae7", "fields": { "extra_data": {}, - "created_date": "2024-06-13T14:23:21.548Z", + "created_date": "2024-06-20T14:35:43.180Z", "name": "asgiref", - "slug": "asgiref-83d59103", - "work_directory": "/tmp/tmp5ihgt8lg/projects/asgiref-83d59103", + "slug": "asgiref-0d37341f", + "work_directory": "/tmp/tmpi98ore1a/projects/asgiref-0d37341f", "is_archived": false, "notes": "", "settings": {} @@ -15,17 +15,17 @@ }, { "model": "scanpipe.run", - "pk": "ceb011c2-6d52-4c3a-bbff-c53e45f0313e", + "pk": "c2be3b12-1cf4-4f1d-9802-411f73c17e58", "fields": { "task_id": null, "task_start_date": null, "task_end_date": null, "task_exitcode": null, "task_output": "", - "log": "2024-06-13 14:23:21.55 Pipeline [scan_codebase] starting\n2024-06-13 14:23:21.55 Step [download_missing_inputs] starting\n2024-06-13 14:23:21.55 Step [download_missing_inputs] completed in 0 seconds\n2024-06-13 14:23:21.55 Step [copy_inputs_to_codebase_directory] starting\n2024-06-13 14:23:21.55 Step [copy_inputs_to_codebase_directory] completed in 0 seconds\n2024-06-13 14:23:21.55 Step [extract_archives] starting\n2024-06-13 14:23:21.60 Step [extract_archives] completed in 0 seconds\n2024-06-13 14:23:21.60 Step [collect_and_create_codebase_resources] starting\n2024-06-13 14:23:21.67 Step [collect_and_create_codebase_resources] completed in 0 seconds\n2024-06-13 14:23:21.67 Step [flag_empty_files] starting\n2024-06-13 14:23:21.67 Step [flag_empty_files] completed in 0 seconds\n2024-06-13 14:23:21.67 Step [flag_ignored_resources] starting\n2024-06-13 14:23:21.67 Step [flag_ignored_resources] completed in 0 seconds\n2024-06-13 14:23:21.68 Step [scan_for_application_packages] starting\n2024-06-13 14:23:21.69 Progress: 11% (2/18)\n2024-06-13 14:23:21.70 Progress: 22% (4/18)\n2024-06-13 14:23:21.71 Progress: 33% (6/18)\n2024-06-13 14:23:21.71 Progress: 44% (8/18)\n2024-06-13 14:23:21.71 Progress: 55% (10/18)\n2024-06-13 14:23:21.71 Progress: 66% (12/18)\n2024-06-13 14:23:21.71 Progress: 77% (14/18)\n2024-06-13 14:23:21.71 Progress: 88% (16/18)\n2024-06-13 14:23:24.08 Progress: 100% (18/18)\n2024-06-13 14:23:26.20 Step [scan_for_application_packages] completed in 5 seconds\n2024-06-13 14:23:26.20 Step [scan_for_files] starting\n2024-06-13 14:23:26.32 Progress: 12% (2/16) ETA: 1 seconds\n2024-06-13 14:23:26.33 Progress: 25% (4/16)\n2024-06-13 14:23:26.35 Progress: 37% (6/16)\n2024-06-13 14:23:26.56 Progress: 50% (8/16)\n2024-06-13 14:23:26.60 Progress: 62% (10/16)\n2024-06-13 14:23:26.63 Progress: 75% (12/16)\n2024-06-13 14:23:26.79 Progress: 87% (14/16)\n2024-06-13 14:23:26.88 Progress: 100% (16/16)\n2024-06-13 14:23:26.90 Step [scan_for_files] completed in 1 seconds\n2024-06-13 14:23:26.90 Pipeline completed in 5 seconds\n", - "project": "83d59103-a103-4c9a-a433-618eff973b63", + "log": "2024-06-20 14:35:43.18 Pipeline [scan_codebase] starting\n2024-06-20 14:35:43.18 Step [download_missing_inputs] starting\n2024-06-20 14:35:43.18 Step [download_missing_inputs] completed in 0 seconds\n2024-06-20 14:35:43.18 Step [copy_inputs_to_codebase_directory] starting\n2024-06-20 14:35:43.18 Step [copy_inputs_to_codebase_directory] completed in 0 seconds\n2024-06-20 14:35:43.18 Step [extract_archives] starting\n2024-06-20 14:35:43.24 Step [extract_archives] completed in 0 seconds\n2024-06-20 14:35:43.24 Step [collect_and_create_codebase_resources] starting\n2024-06-20 14:35:43.32 Step [collect_and_create_codebase_resources] completed in 0 seconds\n2024-06-20 14:35:43.32 Step [flag_empty_files] starting\n2024-06-20 14:35:43.32 Step [flag_empty_files] completed in 0 seconds\n2024-06-20 14:35:43.32 Step [flag_ignored_resources] starting\n2024-06-20 14:35:43.32 Step [flag_ignored_resources] completed in 0 seconds\n2024-06-20 14:35:43.32 Step [scan_for_application_packages] starting\n2024-06-20 14:35:43.35 Progress: 11% (2/18)\n2024-06-20 14:35:43.37 Progress: 22% (4/18)\n2024-06-20 14:35:43.37 Progress: 33% (6/18)\n2024-06-20 14:35:43.37 Progress: 44% (8/18)\n2024-06-20 14:35:43.37 Progress: 55% (10/18)\n2024-06-20 14:35:43.37 Progress: 66% (12/18)\n2024-06-20 14:35:43.37 Progress: 77% (14/18)\n2024-06-20 14:35:43.37 Progress: 88% (16/18)\n2024-06-20 14:35:46.48 Progress: 100% (18/18)\n2024-06-20 14:35:49.01 Step [scan_for_application_packages] completed in 6 seconds\n2024-06-20 14:35:49.01 Step [scan_for_files] starting\n2024-06-20 14:35:49.18 Progress: 12% (2/16) ETA: 1 seconds\n2024-06-20 14:35:49.19 Progress: 25% (4/16) ETA: 1 seconds\n2024-06-20 14:35:49.19 Progress: 37% (6/16)\n2024-06-20 14:35:49.49 Progress: 50% (8/16)\n2024-06-20 14:35:49.59 Progress: 62% (10/16)\n2024-06-20 14:35:49.64 Progress: 75% (12/16)\n2024-06-20 14:35:49.85 Progress: 87% (14/16)\n2024-06-20 14:35:50.04 Progress: 100% (16/16)\n2024-06-20 14:35:50.07 Step [scan_for_files] completed in 1 seconds\n2024-06-20 14:35:50.08 Pipeline completed in 7 seconds\n", + "project": "0d37341f-f1fc-45c9-bc02-db1eabaccae7", "pipeline_name": "scan_codebase", - "created_date": "2024-06-13T14:23:21.551Z", + "created_date": "2024-06-20T14:35:43.184Z", "scancodeio_version": "", "description": "Scan a codebase for application packages, licenses, and copyrights.", "current_step": "", @@ -41,7 +41,7 @@ "sha256": "a5098bc870b80e7b872bff60bb363c7f2c2c89078759f6c47b53ff8c525a152e", "sha512": "", "extra_data": {}, - "project": "83d59103-a103-4c9a-a433-618eff973b63", + "project": "0d37341f-f1fc-45c9-bc02-db1eabaccae7", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -346,7 +346,7 @@ "sha256": "", "sha512": "", "extra_data": {}, - "project": "83d59103-a103-4c9a-a433-618eff973b63", + "project": "0d37341f-f1fc-45c9-bc02-db1eabaccae7", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -386,7 +386,7 @@ "sha256": "", "sha512": "", "extra_data": {}, - "project": "83d59103-a103-4c9a-a433-618eff973b63", + "project": "0d37341f-f1fc-45c9-bc02-db1eabaccae7", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -426,7 +426,7 @@ "sha256": "", "sha512": "", "extra_data": {}, - "project": "83d59103-a103-4c9a-a433-618eff973b63", + "project": "0d37341f-f1fc-45c9-bc02-db1eabaccae7", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -466,7 +466,7 @@ "sha256": "6e89108c2cf0c0446174188f76f60465ae1c1f14f83427807df40d52a27cb2c8", "sha512": "", "extra_data": {}, - "project": "83d59103-a103-4c9a-a433-618eff973b63", + "project": "0d37341f-f1fc-45c9-bc02-db1eabaccae7", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -506,7 +506,7 @@ "sha256": "b846415d1b514e9c1dff14a22deb906d794bc546ca6129f950a18cd091e2a669", "sha512": "", "extra_data": {}, - "project": "83d59103-a103-4c9a-a433-618eff973b63", + "project": "0d37341f-f1fc-45c9-bc02-db1eabaccae7", "detected_license_expression": "bsd-new", "detected_license_expression_spdx": "BSD-3-Clause", "license_detections": [ @@ -581,7 +581,7 @@ "sha256": "70f98f4eb9f6068b192b5464fcdf69e29a8ff09962bfce84bbb052baeee44f33", "sha512": "", "extra_data": {}, - "project": "83d59103-a103-4c9a-a433-618eff973b63", + "project": "0d37341f-f1fc-45c9-bc02-db1eabaccae7", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -886,7 +886,7 @@ "sha256": "11546323af45e6a5639bf620a9c4d73e74c0bf705f494af4595007b923f75e8a", "sha512": "", "extra_data": {}, - "project": "83d59103-a103-4c9a-a433-618eff973b63", + "project": "0d37341f-f1fc-45c9-bc02-db1eabaccae7", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -926,7 +926,7 @@ "sha256": "2c1983592aa38f0bfb0afacc73ddc5b46ce10e8e89ceaa9fed1e5fc6361b608d", "sha512": "", "extra_data": {}, - "project": "83d59103-a103-4c9a-a433-618eff973b63", + "project": "0d37341f-f1fc-45c9-bc02-db1eabaccae7", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -966,7 +966,7 @@ "sha256": "30f49b9094bff904a42caeec32515715fe625a56dc48bd7c0e3d9988c0ad4bd7", "sha512": "", "extra_data": {}, - "project": "83d59103-a103-4c9a-a433-618eff973b63", + "project": "0d37341f-f1fc-45c9-bc02-db1eabaccae7", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -1006,7 +1006,7 @@ "sha256": "fa4651a3b79201a4dc44a4096cd49ec8f427e912ea0ee05c666357b413a8afe7", "sha512": "", "extra_data": {}, - "project": "83d59103-a103-4c9a-a433-618eff973b63", + "project": "0d37341f-f1fc-45c9-bc02-db1eabaccae7", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -1046,7 +1046,7 @@ "sha256": "ee0fcf4a8e6fa9df8a4643bb48e82892d496afce44b6c8b8aea2721755545e1c", "sha512": "", "extra_data": {}, - "project": "83d59103-a103-4c9a-a433-618eff973b63", + "project": "0d37341f-f1fc-45c9-bc02-db1eabaccae7", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -1086,7 +1086,7 @@ "sha256": "3151f66c476208c3154cb6c4fb557a2a253bab82f0ab33fb3c8b9f7976be9e33", "sha512": "", "extra_data": {}, - "project": "83d59103-a103-4c9a-a433-618eff973b63", + "project": "0d37341f-f1fc-45c9-bc02-db1eabaccae7", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -1126,7 +1126,7 @@ "sha256": "ddd445b778c097fc75c2bf69ad964cbadd3bd6999d1dd2306d39d401855e8e3e", "sha512": "", "extra_data": {}, - "project": "83d59103-a103-4c9a-a433-618eff973b63", + "project": "0d37341f-f1fc-45c9-bc02-db1eabaccae7", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -1166,7 +1166,7 @@ "sha256": "ddbc8d455eceb68fc583c67e7c4ad0277c867fb39095c51ec5b37f70342e8334", "sha512": "", "extra_data": {}, - "project": "83d59103-a103-4c9a-a433-618eff973b63", + "project": "0d37341f-f1fc-45c9-bc02-db1eabaccae7", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -1206,7 +1206,7 @@ "sha256": "126c3e3a8a75a517d2739612304607804cf5f34da63fa25d03a6f11f7edb6f2f", "sha512": "", "extra_data": {}, - "project": "83d59103-a103-4c9a-a433-618eff973b63", + "project": "0d37341f-f1fc-45c9-bc02-db1eabaccae7", "detected_license_expression": "apache-2.0", "detected_license_expression_spdx": "Apache-2.0", "license_detections": [ @@ -1285,7 +1285,7 @@ "sha256": "f8bd1ea3fb8afddabb10f8efd66796d41446cad51168ef4d3c44b19c973d0ad0", "sha512": "", "extra_data": {}, - "project": "83d59103-a103-4c9a-a433-618eff973b63", + "project": "0d37341f-f1fc-45c9-bc02-db1eabaccae7", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -1325,7 +1325,7 @@ "sha256": "885267fee0fea687875a02ceb929ca095312d47aaa57e20e4ce382f397caaf4d", "sha512": "", "extra_data": {}, - "project": "83d59103-a103-4c9a-a433-618eff973b63", + "project": "0d37341f-f1fc-45c9-bc02-db1eabaccae7", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -1375,7 +1375,7 @@ "Documentation": "https://asgi.readthedocs.io/", "Further Documentation": "https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions" }, - "project": "83d59103-a103-4c9a-a433-618eff973b63", + "project": "0d37341f-f1fc-45c9-bc02-db1eabaccae7", "compliance_alert": "missing", "affected_by_vulnerabilities": [], "filename": "", @@ -1464,10 +1464,10 @@ "email": "foundation@djangoproject.com" } ], - "uuid": "a9dcd442-4c55-49b4-9e33-8be74c9fc15b", + "uuid": "c9fcc7fb-3613-47d9-92ba-3b3470e5798c", "missing_resources": [], "modified_resources": [], - "package_uid": "pkg:pypi/asgiref@3.3.0?uuid=eca680a3-115b-494b-8581-81c93384e22d", + "package_uid": "pkg:pypi/asgiref@3.3.0?uuid=4705892e-924f-483a-a951-05109c930b3d", "keywords": [ "Development Status :: 5 - Production/Stable", "Environment :: Web Environment", @@ -1508,7 +1508,7 @@ "Documentation": "https://asgi.readthedocs.io/", "Further Documentation": "https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions" }, - "project": "83d59103-a103-4c9a-a433-618eff973b63", + "project": "0d37341f-f1fc-45c9-bc02-db1eabaccae7", "compliance_alert": "missing", "affected_by_vulnerabilities": [], "filename": "", @@ -1597,10 +1597,10 @@ "email": "foundation@djangoproject.com" } ], - "uuid": "27357e59-c16d-40f1-8cd2-0f11b7376cdb", + "uuid": "bafafbff-e654-4a8f-aa74-61b19ed5ce86", "missing_resources": [], "modified_resources": [], - "package_uid": "pkg:pypi/asgiref@3.3.0?uuid=510ea8b0-d7a7-4531-96ea-f41cb4384976", + "package_uid": "pkg:pypi/asgiref@3.3.0?uuid=52c2a744-d7a9-4afc-b38a-5d86a19d9c37", "keywords": [ "Development Status :: 5 - Production/Stable", "Environment :: Web Environment", @@ -1646,8 +1646,8 @@ "qualifiers": "", "subpath": "", "affected_by_vulnerabilities": [], - "project": "83d59103-a103-4c9a-a433-618eff973b63", - "dependency_uid": "pkg:pypi/pytest?uuid=9f487b4c-4d3e-4d13-9100-89ab897c90ee", + "project": "0d37341f-f1fc-45c9-bc02-db1eabaccae7", + "dependency_uid": "pkg:pypi/pytest?uuid=52eb9840-989c-407d-8e4b-309535ef9df5", "for_package": 1, "resolved_to_package": null, "datafile_resource": 1, @@ -1671,8 +1671,8 @@ "qualifiers": "", "subpath": "", "affected_by_vulnerabilities": [], - "project": "83d59103-a103-4c9a-a433-618eff973b63", - "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=7c26cfc3-080d-4d3c-bcfe-fd6a495c5d3a", + "project": "0d37341f-f1fc-45c9-bc02-db1eabaccae7", + "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=9e2bf13a-0bdf-4894-a0d3-e44c28893055", "for_package": 1, "resolved_to_package": null, "datafile_resource": 1, @@ -1696,8 +1696,8 @@ "qualifiers": "", "subpath": "", "affected_by_vulnerabilities": [], - "project": "83d59103-a103-4c9a-a433-618eff973b63", - "dependency_uid": "pkg:pypi/pytest?uuid=909f72ab-1d6b-41f6-b41a-2b3be388870a", + "project": "0d37341f-f1fc-45c9-bc02-db1eabaccae7", + "dependency_uid": "pkg:pypi/pytest?uuid=7b581a48-b4b1-40e6-a7e5-a14de28ca546", "for_package": 2, "resolved_to_package": null, "datafile_resource": 7, @@ -1721,8 +1721,8 @@ "qualifiers": "", "subpath": "", "affected_by_vulnerabilities": [], - "project": "83d59103-a103-4c9a-a433-618eff973b63", - "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=e474c205-ceef-444a-a0d7-aee32c8716df", + "project": "0d37341f-f1fc-45c9-bc02-db1eabaccae7", + "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=02fcb56f-f0d2-438c-989e-54cb8009ebe6", "for_package": 2, "resolved_to_package": null, "datafile_resource": 7, diff --git a/scanpipe/tests/data/cyclonedx/asgiref-3.3.0.cdx.json b/scanpipe/tests/data/cyclonedx/asgiref-3.3.0.cdx.json index 1146f3e070..c29d481cd3 100644 --- a/scanpipe/tests/data/cyclonedx/asgiref-3.3.0.cdx.json +++ b/scanpipe/tests/data/cyclonedx/asgiref-3.3.0.cdx.json @@ -6,7 +6,7 @@ "version": 1, "metadata": { "component": { - "bom-ref": "83d59103-a103-4c9a-a433-618eff973b63", + "bom-ref": "0d37341f-f1fc-45c9-bc02-db1eabaccae7", "name": "asgiref", "type": "library" }, @@ -26,7 +26,7 @@ }, "components": [ { - "bom-ref": "pkg:pypi/asgiref@3.3.0?uuid=eca680a3-115b-494b-8581-81c93384e22d", + "bom-ref": "pkg:pypi/asgiref@3.3.0?uuid=4705892e-924f-483a-a951-05109c930b3d", "copyright": "", "description": "ASGI specs, helper code, and adapters\nasgiref\n=======\n\n.. image:: https://api.travis-ci.org/django/asgiref.svg\n :target: https://travis-ci.org/django/asgiref\n\n.. image:: https://img.shields.io/pypi/v/asgiref.svg\n :target: https://pypi.python.org/pypi/asgiref\n\nASGI is a standard for Python asynchronous web apps and servers to communicate\nwith each other, and positioned as an asynchronous successor to WSGI. You can\nread more at https://asgi.readthedocs.io/en/latest/\n\nThis package includes ASGI base libraries, such as:\n\n* Sync-to-async and async-to-sync function wrappers, ``asgiref.sync``\n* Server base classes, ``asgiref.server``\n* A WSGI-to-ASGI adapter, in ``asgiref.wsgi``\n\n\nFunction wrappers\n-----------------\n\nThese allow you to wrap or decorate async or sync functions to call them from\nthe other style (so you can call async functions from a synchronous thread,\nor vice-versa).\n\nIn particular:\n\n* AsyncToSync lets a synchronous subthread stop and wait while the async\n function is called on the main thread's event loop, and then control is\n returned to the thread when the async function is finished.\n\n* SyncToAsync lets async code call a synchronous function, which is run in\n a threadpool and control returned to the async coroutine when the synchronous\n function completes.\n\nThe idea is to make it easier to call synchronous APIs from async code and\nasynchronous APIs from synchronous code so it's easier to transition code from\none style to the other. In the case of Channels, we wrap the (synchronous)\nDjango view system with SyncToAsync to allow it to run inside the (asynchronous)\nASGI server.\n\nNote that exactly what threads things run in is very specific, and aimed to\nkeep maximum compatibility with old synchronous code. See\n\"Synchronous code & Threads\" below for a full explanation. By default,\n``sync_to_async`` will run all synchronous code in the program in the same\nthread for safety reasons; you can disable this for more performance with\n``@sync_to_async(thread_sensitive=False)``, but make sure that your code does\nnot rely on anything bound to threads (like database connections) when you do.\n\n\nThreadlocal replacement\n-----------------------\n\nThis is a drop-in replacement for ``threading.local`` that works with both\nthreads and asyncio Tasks. Even better, it will proxy values through from a\ntask-local context to a thread-local context when you use ``sync_to_async``\nto run things in a threadpool, and vice-versa for ``async_to_sync``.\n\nIf you instead want true thread- and task-safety, you can set\n``thread_critical`` on the Local object to ensure this instead.\n\n\nServer base classes\n-------------------\n\nIncludes a ``StatelessServer`` class which provides all the hard work of\nwriting a stateless server (as in, does not handle direct incoming sockets\nbut instead consumes external streams or sockets to work out what is happening).\n\nAn example of such a server would be a chatbot server that connects out to\na central chat server and provides a \"connection scope\" per user chatting to\nit. There's only one actual connection, but the server has to separate things\ninto several scopes for easier writing of the code.\n\nYou can see an example of this being used in `frequensgi `_.\n\n\nWSGI-to-ASGI adapter\n--------------------\n\nAllows you to wrap a WSGI application so it appears as a valid ASGI application.\n\nSimply wrap it around your WSGI application like so::\n\n asgi_application = WsgiToAsgi(wsgi_application)\n\nThe WSGI application will be run in a synchronous threadpool, and the wrapped\nASGI application will be one that accepts ``http`` class messages.\n\nPlease note that not all extended features of WSGI may be supported (such as\nfile handles for incoming POST bodies).\n\n\nDependencies\n------------\n\n``asgiref`` requires Python 3.5 or higher.\n\n\nContributing\n------------\n\nPlease refer to the\n`main Channels contributing docs `_.\n\n\nTesting\n'''''''\n\nTo run tests, make sure you have installed the ``tests`` extra with the package::\n\n cd asgiref/\n pip install -e .[tests]\n pytest\n\n\nBuilding the documentation\n''''''''''''''''''''''''''\n\nThe documentation uses `Sphinx `_::\n\n cd asgiref/docs/\n pip install sphinx\n\nTo build the docs, you can use the default tools::\n\n sphinx-build -b html . _build/html # or `make html`, if you've got make set up\n cd _build/html\n python -m http.server\n\n...or you can use ``sphinx-autobuild`` to run a server and rebuild/reload\nyour documentation changes automatically::\n\n pip install sphinx-autobuild\n sphinx-autobuild . _build/html\n\n\nImplementation Details\n----------------------\n\nSynchronous code & threads\n''''''''''''''''''''''''''\n\nThe ``asgiref.sync`` module provides two wrappers that let you go between\nasynchronous and synchronous code at will, while taking care of the rough edges\nfor you.\n\nUnfortunately, the rough edges are numerous, and the code has to work especially\nhard to keep things in the same thread as much as possible. Notably, the\nrestrictions we are working with are:\n\n* All synchronous code called through ``SyncToAsync`` and marked with\n ``thread_sensitive`` should run in the same thread as each other (and if the\n outer layer of the program is synchronous, the main thread)\n\n* If a thread already has a running async loop, ``AsyncToSync`` can't run things\n on that loop if it's blocked on synchronous code that is above you in the\n call stack.\n\nThe first compromise you get to might be that ``thread_sensitive`` code should\njust run in the same thread and not spawn in a sub-thread, fulfilling the first\nrestriction, but that immediately runs you into the second restriction.\n\nThe only real solution is to essentially have a variant of ThreadPoolExecutor\nthat executes any ``thread_sensitive`` code on the outermost synchronous\nthread - either the main thread, or a single spawned subthread.\n\nThis means you now have two basic states:\n\n* If the outermost layer of your program is synchronous, then all async code\n run through ``AsyncToSync`` will run in a per-call event loop in arbitary\n sub-threads, while all ``thread_sensitive`` code will run in the main thread.\n\n* If the outermost layer of your program is asynchronous, then all async code\n runs on the main thread's event loop, and all ``thread_sensitive`` synchronous\n code will run in a single shared sub-thread.\n\nCruicially, this means that in both cases there is a thread which is a shared\nresource that all ``thread_sensitive`` code must run on, and there is a chance\nthat this thread is currently blocked on its own ``AsyncToSync`` call. Thus,\n``AsyncToSync`` needs to act as an executor for thread code while it's blocking.\n\nThe ``CurrentThreadExecutor`` class provides this functionality; rather than\nsimply waiting on a Future, you can call its ``run_until_future`` method and\nit will run submitted code until that Future is done. This means that code\ninside the call can then run code on your thread.\n\n\nMaintenance and Security\n------------------------\n\nTo report security issues, please contact security@djangoproject.com. For GPG\nsignatures and more security process information, see\nhttps://docs.djangoproject.com/en/dev/internals/security/.\n\nTo report bugs or request new features, please open a new GitHub issue.\n\nThis repository is part of the Channels project. For the shepherd and maintenance team, please see the\n`main Channels readme `_.", "externalReferences": [ @@ -64,7 +64,7 @@ "version": "3.3.0" }, { - "bom-ref": "pkg:pypi/asgiref@3.3.0?uuid=510ea8b0-d7a7-4531-96ea-f41cb4384976", + "bom-ref": "pkg:pypi/asgiref@3.3.0?uuid=52c2a744-d7a9-4afc-b38a-5d86a19d9c37", "copyright": "", "description": "ASGI specs, helper code, and adapters\nasgiref\n=======\n\n.. image:: https://api.travis-ci.org/django/asgiref.svg\n :target: https://travis-ci.org/django/asgiref\n\n.. image:: https://img.shields.io/pypi/v/asgiref.svg\n :target: https://pypi.python.org/pypi/asgiref\n\nASGI is a standard for Python asynchronous web apps and servers to communicate\nwith each other, and positioned as an asynchronous successor to WSGI. You can\nread more at https://asgi.readthedocs.io/en/latest/\n\nThis package includes ASGI base libraries, such as:\n\n* Sync-to-async and async-to-sync function wrappers, ``asgiref.sync``\n* Server base classes, ``asgiref.server``\n* A WSGI-to-ASGI adapter, in ``asgiref.wsgi``\n\n\nFunction wrappers\n-----------------\n\nThese allow you to wrap or decorate async or sync functions to call them from\nthe other style (so you can call async functions from a synchronous thread,\nor vice-versa).\n\nIn particular:\n\n* AsyncToSync lets a synchronous subthread stop and wait while the async\n function is called on the main thread's event loop, and then control is\n returned to the thread when the async function is finished.\n\n* SyncToAsync lets async code call a synchronous function, which is run in\n a threadpool and control returned to the async coroutine when the synchronous\n function completes.\n\nThe idea is to make it easier to call synchronous APIs from async code and\nasynchronous APIs from synchronous code so it's easier to transition code from\none style to the other. In the case of Channels, we wrap the (synchronous)\nDjango view system with SyncToAsync to allow it to run inside the (asynchronous)\nASGI server.\n\nNote that exactly what threads things run in is very specific, and aimed to\nkeep maximum compatibility with old synchronous code. See\n\"Synchronous code & Threads\" below for a full explanation. By default,\n``sync_to_async`` will run all synchronous code in the program in the same\nthread for safety reasons; you can disable this for more performance with\n``@sync_to_async(thread_sensitive=False)``, but make sure that your code does\nnot rely on anything bound to threads (like database connections) when you do.\n\n\nThreadlocal replacement\n-----------------------\n\nThis is a drop-in replacement for ``threading.local`` that works with both\nthreads and asyncio Tasks. Even better, it will proxy values through from a\ntask-local context to a thread-local context when you use ``sync_to_async``\nto run things in a threadpool, and vice-versa for ``async_to_sync``.\n\nIf you instead want true thread- and task-safety, you can set\n``thread_critical`` on the Local object to ensure this instead.\n\n\nServer base classes\n-------------------\n\nIncludes a ``StatelessServer`` class which provides all the hard work of\nwriting a stateless server (as in, does not handle direct incoming sockets\nbut instead consumes external streams or sockets to work out what is happening).\n\nAn example of such a server would be a chatbot server that connects out to\na central chat server and provides a \"connection scope\" per user chatting to\nit. There's only one actual connection, but the server has to separate things\ninto several scopes for easier writing of the code.\n\nYou can see an example of this being used in `frequensgi `_.\n\n\nWSGI-to-ASGI adapter\n--------------------\n\nAllows you to wrap a WSGI application so it appears as a valid ASGI application.\n\nSimply wrap it around your WSGI application like so::\n\n asgi_application = WsgiToAsgi(wsgi_application)\n\nThe WSGI application will be run in a synchronous threadpool, and the wrapped\nASGI application will be one that accepts ``http`` class messages.\n\nPlease note that not all extended features of WSGI may be supported (such as\nfile handles for incoming POST bodies).\n\n\nDependencies\n------------\n\n``asgiref`` requires Python 3.5 or higher.\n\n\nContributing\n------------\n\nPlease refer to the\n`main Channels contributing docs `_.\n\n\nTesting\n'''''''\n\nTo run tests, make sure you have installed the ``tests`` extra with the package::\n\n cd asgiref/\n pip install -e .[tests]\n pytest\n\n\nBuilding the documentation\n''''''''''''''''''''''''''\n\nThe documentation uses `Sphinx `_::\n\n cd asgiref/docs/\n pip install sphinx\n\nTo build the docs, you can use the default tools::\n\n sphinx-build -b html . _build/html # or `make html`, if you've got make set up\n cd _build/html\n python -m http.server\n\n...or you can use ``sphinx-autobuild`` to run a server and rebuild/reload\nyour documentation changes automatically::\n\n pip install sphinx-autobuild\n sphinx-autobuild . _build/html\n\n\nImplementation Details\n----------------------\n\nSynchronous code & threads\n''''''''''''''''''''''''''\n\nThe ``asgiref.sync`` module provides two wrappers that let you go between\nasynchronous and synchronous code at will, while taking care of the rough edges\nfor you.\n\nUnfortunately, the rough edges are numerous, and the code has to work especially\nhard to keep things in the same thread as much as possible. Notably, the\nrestrictions we are working with are:\n\n* All synchronous code called through ``SyncToAsync`` and marked with\n ``thread_sensitive`` should run in the same thread as each other (and if the\n outer layer of the program is synchronous, the main thread)\n\n* If a thread already has a running async loop, ``AsyncToSync`` can't run things\n on that loop if it's blocked on synchronous code that is above you in the\n call stack.\n\nThe first compromise you get to might be that ``thread_sensitive`` code should\njust run in the same thread and not spawn in a sub-thread, fulfilling the first\nrestriction, but that immediately runs you into the second restriction.\n\nThe only real solution is to essentially have a variant of ThreadPoolExecutor\nthat executes any ``thread_sensitive`` code on the outermost synchronous\nthread - either the main thread, or a single spawned subthread.\n\nThis means you now have two basic states:\n\n* If the outermost layer of your program is synchronous, then all async code\n run through ``AsyncToSync`` will run in a per-call event loop in arbitary\n sub-threads, while all ``thread_sensitive`` code will run in the main thread.\n\n* If the outermost layer of your program is asynchronous, then all async code\n runs on the main thread's event loop, and all ``thread_sensitive`` synchronous\n code will run in a single shared sub-thread.\n\nCruicially, this means that in both cases there is a thread which is a shared\nresource that all ``thread_sensitive`` code must run on, and there is a chance\nthat this thread is currently blocked on its own ``AsyncToSync`` call. Thus,\n``AsyncToSync`` needs to act as an executor for thread code while it's blocking.\n\nThe ``CurrentThreadExecutor`` class provides this functionality; rather than\nsimply waiting on a Future, you can call its ``run_until_future`` method and\nit will run submitted code until that Future is done. This means that code\ninside the call can then run code on your thread.\n\n\nMaintenance and Security\n------------------------\n\nTo report security issues, please contact security@djangoproject.com. For GPG\nsignatures and more security process information, see\nhttps://docs.djangoproject.com/en/dev/internals/security/.\n\nTo report bugs or request new features, please open a new GitHub issue.\n\nThis repository is part of the Channels project. For the shepherd and maintenance team, please see the\n`main Channels readme `_.", "evidence": { @@ -112,23 +112,23 @@ "dependencies": [ { "dependsOn": [ - "pkg:pypi/asgiref@3.3.0?uuid=510ea8b0-d7a7-4531-96ea-f41cb4384976", - "pkg:pypi/asgiref@3.3.0?uuid=eca680a3-115b-494b-8581-81c93384e22d" + "pkg:pypi/asgiref@3.3.0?uuid=4705892e-924f-483a-a951-05109c930b3d", + "pkg:pypi/asgiref@3.3.0?uuid=52c2a744-d7a9-4afc-b38a-5d86a19d9c37" ], - "ref": "83d59103-a103-4c9a-a433-618eff973b63" + "ref": "0d37341f-f1fc-45c9-bc02-db1eabaccae7" }, { - "ref": "pkg:pypi/asgiref@3.3.0?uuid=510ea8b0-d7a7-4531-96ea-f41cb4384976" + "ref": "pkg:pypi/asgiref@3.3.0?uuid=4705892e-924f-483a-a951-05109c930b3d" }, { - "ref": "pkg:pypi/asgiref@3.3.0?uuid=eca680a3-115b-494b-8581-81c93384e22d" + "ref": "pkg:pypi/asgiref@3.3.0?uuid=52c2a744-d7a9-4afc-b38a-5d86a19d9c37" } ], "vulnerabilities": [ { "affects": [ { - "ref": "urn:cdx:pkg:pypi/asgiref@3.3.0?uuid=510ea8b0-d7a7-4531-96ea-f41cb4384976" + "ref": "urn:cdx:pkg:pypi/asgiref@3.3.0?uuid=52c2a744-d7a9-4afc-b38a-5d86a19d9c37" } ], "bom-ref": "BomRef", diff --git a/scanpipe/tests/data/is-npm-1.0.0_scan_package.json b/scanpipe/tests/data/is-npm-1.0.0_scan_package.json index c30eee492e..db1010775e 100644 --- a/scanpipe/tests/data/is-npm-1.0.0_scan_package.json +++ b/scanpipe/tests/data/is-npm-1.0.0_scan_package.json @@ -14,12 +14,12 @@ "--summary": true }, "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", - "output_format_version": "3.1.0", + "output_format_version": "3.2.0", "message": null, "errors": [], "warnings": [], "extra_data": { - "spdx_license_list_version": "3.23", + "spdx_license_list_version": "3.24", "files_count": 3 } } diff --git a/scanpipe/tests/data/manifests/openpdf-parent-1.3.11_scan_package.json b/scanpipe/tests/data/manifests/openpdf-parent-1.3.11_scan_package.json index 005b0a94bc..5582c1598f 100644 --- a/scanpipe/tests/data/manifests/openpdf-parent-1.3.11_scan_package.json +++ b/scanpipe/tests/data/manifests/openpdf-parent-1.3.11_scan_package.json @@ -14,12 +14,12 @@ "--summary": true }, "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", - "output_format_version": "3.1.0", + "output_format_version": "3.2.0", "message": null, "errors": [], "warnings": [], "extra_data": { - "spdx_license_list_version": "3.23", + "spdx_license_list_version": "3.24", "files_count": 1 } } diff --git a/scanpipe/tests/data/multiple-is-npm-1.0.0_scan_package.json b/scanpipe/tests/data/multiple-is-npm-1.0.0_scan_package.json index 28a8efb328..fcba010313 100644 --- a/scanpipe/tests/data/multiple-is-npm-1.0.0_scan_package.json +++ b/scanpipe/tests/data/multiple-is-npm-1.0.0_scan_package.json @@ -14,12 +14,12 @@ "--summary": true }, "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", - "output_format_version": "3.1.0", + "output_format_version": "3.2.0", "message": null, "errors": [], "warnings": [], "extra_data": { - "spdx_license_list_version": "3.23", + "spdx_license_list_version": "3.24", "files_count": 6 } } diff --git a/scanpipe/tests/data/scancode/package_assembly_codebase.json b/scanpipe/tests/data/scancode/package_assembly_codebase.json index eae0d76de9..857823da2c 100644 --- a/scanpipe/tests/data/scancode/package_assembly_codebase.json +++ b/scanpipe/tests/data/scancode/package_assembly_codebase.json @@ -2,16 +2,16 @@ "headers": [ { "tool_name": "scancode-toolkit", - "tool_version": "32.1.0", + "tool_version": "32.2.0", "options": { "--info": true, "--package": true }, "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", - "start_timestamp": "2024-06-13T142327.801919", - "end_timestamp": "2024-06-13T142327.894178", - "output_format_version": "3.1.0", - "duration": 0.09227418899536133, + "start_timestamp": "2024-06-20T143551.606618", + "end_timestamp": "2024-06-20T143551.806679", + "output_format_version": "3.2.0", + "duration": 0.2000892162322998, "message": null, "errors": [], "warnings": [], @@ -23,7 +23,7 @@ "platform_version": "#122-Ubuntu SMP Thu May 23 07:48:21 UTC 2024", "python_version": "3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0]" }, - "spdx_license_list_version": "3.23", + "spdx_license_list_version": "3.24", "files_count": 3 } } @@ -91,7 +91,7 @@ "repository_homepage_url": "https://www.npmjs.com/package/test", "repository_download_url": "https://registry.npmjs.org/test/-/test-0.1.0.tgz", "api_data_url": "https://registry.npmjs.org/test/0.1.0", - "package_uid": "pkg:npm/test@0.1.0?uuid=a5225ee0-cc90-49cf-80bc-370004b8ced5", + "package_uid": "pkg:npm/test@0.1.0?uuid=52a13e12-1ca3-4d62-8468-0ef2122c1fb8", "datafile_paths": [ "package_assembly_codebase.tar.gz-extract/test/get_package_resources/package.json" ], @@ -274,7 +274,7 @@ } ], "for_packages": [ - "pkg:npm/test@0.1.0?uuid=a5225ee0-cc90-49cf-80bc-370004b8ced5" + "pkg:npm/test@0.1.0?uuid=52a13e12-1ca3-4d62-8468-0ef2122c1fb8" ], "files_count": 0, "dirs_count": 0, @@ -303,7 +303,7 @@ "is_script": false, "package_data": [], "for_packages": [ - "pkg:npm/test@0.1.0?uuid=a5225ee0-cc90-49cf-80bc-370004b8ced5" + "pkg:npm/test@0.1.0?uuid=52a13e12-1ca3-4d62-8468-0ef2122c1fb8" ], "files_count": 0, "dirs_count": 0, diff --git a/scanpipe/tests/pipes/test_output.py b/scanpipe/tests/pipes/test_output.py index 26fc067d61..9baa8081bc 100644 --- a/scanpipe/tests/pipes/test_output.py +++ b/scanpipe/tests/pipes/test_output.py @@ -247,7 +247,7 @@ def test_scanpipe_pipes_outputs_to_cyclonedx(self, regen=FIXTURES_REGEN): project = Project.objects.get(name="asgiref") package = project.discoveredpackages.get( - uuid="27357e59-c16d-40f1-8cd2-0f11b7376cdb" + uuid="bafafbff-e654-4a8f-aa74-61b19ed5ce86" ) package.other_license_expression_spdx = "Apache-2.0 AND LicenseRef-test" From 72cef364817de9036084be3d829df4f27a9b6c82 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Thu, 27 Jun 2024 19:50:59 +0530 Subject: [PATCH 07/11] Improve dependency resolver for lockfiles Handle various lockfile cases where: * Same package/dependencies are present in different lockfiles * Independent lockfiles without a manifest and root package * Ecosystems which have only a single version of package in their environment * Dependency graphs where a resolved package can have many parent packages. Signed-off-by: Ayan Sinha Mahapatra --- scanpipe/models.py | 26 +++++++++++-- scanpipe/pipes/__init__.py | 37 ++++++++++++++++--- scanpipe/pipes/scancode.py | 17 ++++++++- .../templates/scanpipe/dependency_list.html | 3 ++ scanpipe/views.py | 4 ++ 5 files changed, 76 insertions(+), 11 deletions(-) diff --git a/scanpipe/models.py b/scanpipe/models.py index a2e7f3da78..550b11aa58 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -3477,10 +3477,28 @@ class DiscoveredDependency( "The identifier for the datafile handler used to obtain this dependency." ), ) - is_runtime = models.BooleanField(default=False) - is_optional = models.BooleanField(default=False) - is_resolved = models.BooleanField(default=False) - is_direct = models.BooleanField(default=False) + is_runtime = models.BooleanField( + default=False, + help_text=_("True if this dependency is a runtime dependency."), + ) + is_optional = models.BooleanField( + default=False, + help_text=_("True if this dependency is an optional dependency"), + ) + is_resolved = models.BooleanField( + default=False, + help_text=_( + "True if this dependency version requirement has been pinned " + "and this dependency points to an exact version." + ), + ) + is_direct = models.BooleanField( + default=False, + help_text=_( + "True if this is a direct, first-level dependency relationship " + "for a package." + ), + ) objects = DiscoveredDependencyQuerySet.as_manager() diff --git a/scanpipe/pipes/__init__.py b/scanpipe/pipes/__init__.py index d5dc538103..c5b0a015b3 100644 --- a/scanpipe/pipes/__init__.py +++ b/scanpipe/pipes/__init__.py @@ -270,9 +270,10 @@ def update_or_create_dependency( if ignore_dependency_scope(project, dependency_data): return # Do not create the DiscoveredDependency record. + dependencies = [] if not dependency_uid: purl_data = DiscoveredDependency.extract_purl_data(dependency_data) - dependency = DiscoveredDependency.objects.get_or_none( + dependencies = DiscoveredDependency.objects.filter( project=project, extracted_requirement=extracted_requirement, **purl_data, @@ -282,12 +283,36 @@ def update_or_create_dependency( project=project, dependency_uid=dependency_uid, ) + if dependency: + dependencies.append(dependency) + + for dependency in dependencies: + # This dependency relationship is for a new package + if ( + for_package + and dependency.for_package + and dependency.for_package != for_package + ): + DiscoveredDependency.populate_dependency_uuid(dependency_data) + dependency = DiscoveredDependency.create_from_data( + project, + dependency_data, + for_package=for_package, + resolved_to_package=resolved_to_package, + datafile_resource=datafile_resource, + datasource_id=datasource_id, + strip_datafile_path_root=strip_datafile_path_root, + ) + break - if dependency: - dependency.update_from_data(dependency_data) - if resolved_to_package and not dependency.resolved_to_package: - dependency.update(resolved_to_package=resolved_to_package) - else: + elif dependency: + dependency.update_from_data(dependency_data) + if resolved_to_package and not dependency.resolved_to_package: + dependency.update(resolved_to_package=resolved_to_package) + if for_package and not dependency.for_package: + dependency.update(for_package=for_package) + + if not dependencies: DiscoveredDependency.populate_dependency_uuid(dependency_data) dependency = DiscoveredDependency.create_from_data( project, diff --git a/scanpipe/pipes/scancode.py b/scanpipe/pipes/scancode.py index fcd9368951..879a49e51b 100644 --- a/scanpipe/pipes/scancode.py +++ b/scanpipe/pipes/scancode.py @@ -528,11 +528,15 @@ def resolve_dependencies(project): package = None if pd.purl: purl_data = DiscoveredPackage.extract_purl_data(package_mapping) - package = DiscoveredPackage.objects.get_or_none( + packages = DiscoveredPackage.objects.filter( project=project, **purl_data, ) + for package in packages: + if resource.location in package.datafile_paths: + break + dependencies = package_mapping.get("dependencies") or [] update_packages_and_dependencies( project=project, @@ -583,6 +587,8 @@ def update_packages_and_dependencies( dependency_data=dep, for_package=package, resolved_to_package=resolved_to_package, + datafile_resource=resource, + datasource_id=datasource_id, ) @@ -628,6 +634,15 @@ def match_and_resolve_dependencies(project): ) ] + # This should be done only in the case of lockfiles where only one version + # of a package is present for an environment + if not other_dependencies: + other_dependencies = [ + dep + for dep in matched_dependencies + if (dep.purl == dependency.purl and dep.resolved_to_package) + ] + if other_dependencies: resolved_dependency = other_dependencies.pop() dependency.update( diff --git a/scanpipe/templates/scanpipe/dependency_list.html b/scanpipe/templates/scanpipe/dependency_list.html index e26ea9c24e..51ba0821f4 100644 --- a/scanpipe/templates/scanpipe/dependency_list.html +++ b/scanpipe/templates/scanpipe/dependency_list.html @@ -48,6 +48,9 @@ {{ dependency.is_resolved }} + + {{ dependency.is_direct }} + {% if dependency.for_package %} {# CAUTION: Avoid relying on get_absolute_url to prevent unnecessary query triggers #} diff --git a/scanpipe/views.py b/scanpipe/views.py index 334c4f7437..2ed65b679c 100644 --- a/scanpipe/views.py +++ b/scanpipe/views.py @@ -1546,6 +1546,10 @@ class DiscoveredDependencyListView( "field_name": "is_resolved", "filter_fieldname": "is_resolved", }, + { + "field_name": "is_direct", + "filter_fieldname": "is_direct", + }, "for_package", "resolved_to_package", "datafile_resource", From 44711eafc66c9fae1b06f5c7fc2946abe97c8059 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Fri, 28 Jun 2024 16:44:00 +0530 Subject: [PATCH 08/11] Address feedback and refactor code Signed-off-by: Ayan Sinha Mahapatra --- .../0062_dependency_resolver_update.py | 39 ++++++++- scanpipe/models.py | 19 +++- scanpipe/pipes/__init__.py | 6 +- scanpipe/pipes/scancode.py | 86 +++++++++++++------ scanpipe/tests/test_models.py | 13 +++ scanpipe/tests/test_pipelines.py | 17 ++-- 6 files changed, 137 insertions(+), 43 deletions(-) diff --git a/scanpipe/migrations/0062_dependency_resolver_update.py b/scanpipe/migrations/0062_dependency_resolver_update.py index 8bafb6db79..733b83c12c 100644 --- a/scanpipe/migrations/0062_dependency_resolver_update.py +++ b/scanpipe/migrations/0062_dependency_resolver_update.py @@ -13,17 +13,50 @@ class Migration(migrations.Migration): migrations.AddField( model_name="discovereddependency", name="is_direct", - field=models.BooleanField(default=False), + field=models.BooleanField( + default=False, + help_text="True if this is a direct, first-level dependency relationship for a package.", + ), ), migrations.AddField( model_name="discoveredpackage", name="is_private", - field=models.BooleanField(default=False), + field=models.BooleanField( + default=False, + help_text="True if this is a private package, either not meant to be published on a repository, and/or a local package without a name and version used primarily to track dependencies and other information.", + ), ), migrations.AddField( model_name="discoveredpackage", name="is_virtual", - field=models.BooleanField(default=False), + field=models.BooleanField( + default=False, + help_text="True if this package is created only from a manifest or lockfile, and not from its actual packaged code. The files of this package are not present in the codebase.", + ), + ), + migrations.AlterField( + model_name="discovereddependency", + name="is_optional", + field=models.BooleanField( + default=False, + help_text="True if this dependency is an optional dependency", + ), + ), + migrations.AlterField( + model_name="discovereddependency", + name="is_resolved", + field=models.BooleanField( + default=False, + help_text="True if this dependency version requirement has been pinned and this dependency points to an exact version.", + ), + ), + migrations.AlterField( + model_name="discovereddependency", + name="is_runtime", + field=models.BooleanField( + default=False, + help_text="True if this dependency is a runtime dependency.", + ), ), migrations.AddIndex( model_name="discovereddependency", diff --git a/scanpipe/models.py b/scanpipe/models.py index 6e641fc026..7f67f5e23c 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -3074,8 +3074,23 @@ class AbstractPackage(models.Model): blank=True, help_text=_("A notice text for this package."), ) - is_private = models.BooleanField(default=False) - is_virtual = models.BooleanField(default=False) + is_private = models.BooleanField( + default=False, + help_text=_( + "True if this is a private package, either not meant to be " + "published on a repository, and/or a local package without a " + "name and version used primarily to track dependencies and " + "other information." + ), + ) + is_virtual = models.BooleanField( + default=False, + help_text=_( + "True if this package is created only from a manifest or lockfile, " + "and not from its actual packaged code. The files of this package " + "are not present in the codebase." + ), + ) datasource_ids = models.JSONField( default=list, blank=True, diff --git a/scanpipe/pipes/__init__.py b/scanpipe/pipes/__init__.py index c5b0a015b3..16c232dadd 100644 --- a/scanpipe/pipes/__init__.py +++ b/scanpipe/pipes/__init__.py @@ -287,12 +287,12 @@ def update_or_create_dependency( dependencies.append(dependency) for dependency in dependencies: - # This dependency relationship is for a new package - if ( + is_for_new_package = ( for_package and dependency.for_package and dependency.for_package != for_package - ): + ) + if is_for_new_package: DiscoveredDependency.populate_dependency_uuid(dependency_data) dependency = DiscoveredDependency.create_from_data( project, diff --git a/scanpipe/pipes/scancode.py b/scanpipe/pipes/scancode.py index 7a079d4c76..e72a9c4713 100644 --- a/scanpipe/pipes/scancode.py +++ b/scanpipe/pipes/scancode.py @@ -489,38 +489,73 @@ def process_package_data(project, static_resolve=False): for resource in project.codebaseresources.has_package_data(): logger.info(f" Processing: {resource.path}") for package_mapping in resource.package_data: - pd = packagedcode_models.PackageData.from_dict(mapping=package_mapping) - if not pd.can_assemble: - continue + create_packages_and_dependencies_from_mapping( + project=project, + resource=resource, + package_mapping=package_mapping, + find_package=False, + ) - logger.info(f" Package data: {pd.purl}") + if static_resolve: + resolve_dependencies(project) - package_data = pd.to_dict() - dependencies = package_data.pop("dependencies") - package = None - if pd.purl: - package = pipes.update_or_create_package( - project=project, - package_data=package_data, - codebase_resources=[resource], - ) +def create_packages_and_dependencies_from_mapping( + project, + resource, + package_mapping, + find_package=False, +): + """ + Create or update packages and dependencies from a `package_mapping`, + for a respective `resource` and `project`. - for dep in dependencies: - pipes.update_or_create_dependency( - project=project, - dependency_data=dep, - for_package=package, - datafile_resource=resource, - datasource_id=pd.datasource_id, - ) + If `find_package` is True, find the package with the respective purl data, + instead of trying to create it. + """ + pd = packagedcode_models.PackageData.from_dict(mapping=package_mapping) + if not pd.can_assemble: + return - if static_resolve: - resolve_dependencies(project) + logger.info(f" Package data: {pd.purl}") + package_data = pd.to_dict() + dependencies = package_data.pop("dependencies") -def resolve_dependencies(project): + package = None + if pd.purl: + if find_package: + purl_data = DiscoveredPackage.extract_purl_data(package_mapping) + packages = DiscoveredPackage.objects.filter( + project=project, + **purl_data, + ) + for package in packages: + if resource.location in package.datafile_paths: + break + else: + package = pipes.update_or_create_package( + project=project, + package_data=package_data, + codebase_resources=[resource], + ) + + update_packages_and_dependencies( + project=project, + dependencies=dependencies, + package=package, + resource=resource, + datasource_id=pd.datasource_id, + process_resolved=False, + ) + + +def resolve_dependencies(project): + """ + Match and merge resolved dependencies to create a dependency graph of + direct dependency relations between resolved packages. + """ logger.info(f"Project {project} resolve_dependencies:") for resource in project.codebaseresources.has_package_data(): for package_mapping in resource.package_data: @@ -555,6 +590,7 @@ def update_packages_and_dependencies( package, resource, datasource_id, + process_resolved=True, ): """ Create DiscoveredPackage and DiscoveredDependency objects from @@ -564,7 +600,7 @@ def update_packages_and_dependencies( for dep in dependencies: resolved_package = dep.get("resolved_package") or {} resolved_to_package = None - if resolved_package: + if process_resolved and resolved_package: resolved_to_package = pipes.update_or_create_package( project=project, package_data=resolved_package, diff --git a/scanpipe/tests/test_models.py b/scanpipe/tests/test_models.py index d7c89bd440..dc1f2c9354 100644 --- a/scanpipe/tests/test_models.py +++ b/scanpipe/tests/test_models.py @@ -49,6 +49,7 @@ from django.utils import timezone from packagedcode.models import PackageData +from packageurl import PackageURL from requests.exceptions import RequestException from rq.job import JobStatus @@ -65,6 +66,7 @@ from scanpipe.models import UUIDTaggedItem from scanpipe.models import convert_glob_to_django_regex from scanpipe.models import get_project_work_directory +from scanpipe.models import normalize_package_url_data from scanpipe.pipes.fetch import Download from scanpipe.pipes.input import copy_input from scanpipe.tests import dependency_data1 @@ -730,6 +732,17 @@ def test_scanpipe_project_get_ignored_dependency_scopes_index(self): expected = {"npm": ["devDependencies"], "pypi": ["tests", "build"]} self.assertEqual(expected, self.project1.get_ignored_dependency_scopes_index()) + def test_scanpipe_normalize_package_url_data(self): + purl = PackageURL.from_string("pkg:npm/athena-express@6.0.4") + purl_data = normalize_package_url_data(purl_mapping=purl.to_dict()) + self.assertEqual(purl_data.get("namespace"), "") + + purl_data = normalize_package_url_data( + purl_mapping=purl.to_dict(), + ignore_nulls=True, + ) + self.assertEqual(purl_data.get("namespace"), None) + def test_scanpipe_project_get_ignored_vulnerabilities_set(self): self.project1.settings = { "ignored_vulnerabilities": [ diff --git a/scanpipe/tests/test_pipelines.py b/scanpipe/tests/test_pipelines.py index 850b106aa9..010a0ec763 100644 --- a/scanpipe/tests/test_pipelines.py +++ b/scanpipe/tests/test_pipelines.py @@ -557,6 +557,12 @@ def _normalize_package_uids(self, data): Return the `data`, where any `package_uid` value has been normalized with `purl_with_fake_uuid()` """ + fields_with_package_uids = [ + "package_uid", + "dependency_uid", + "for_package_uid", + "resolved_to_package_uid", + ] if isinstance(data, list): return [self._normalize_package_uids(entry) for entry in data] @@ -568,16 +574,7 @@ def _normalize_package_uids(self, data): for key, value in data.items(): if isinstance(value, (list, dict)): value = self._normalize_package_uids(value) - if ( - key - in ( - "package_uid", - "dependency_uid", - "for_package_uid", - "resolved_to_package_uid", - ) - and value - ): + if key in fields_with_package_uids and value: value = purl_with_fake_uuid(value) if key == "for_packages" and value: value = sorted( From 023423c3349771e90dcc43aff2b7e63bdbc2d80b Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Fri, 28 Jun 2024 20:41:06 +0530 Subject: [PATCH 09/11] FIx bugs for resolving python packages Signed-off-by: Ayan Sinha Mahapatra --- scanpipe/models.py | 5 ++++ scanpipe/pipes/__init__.py | 49 +++++++++++++++++++++++--------------- scanpipe/pipes/scancode.py | 8 ++++++- 3 files changed, 42 insertions(+), 20 deletions(-) diff --git a/scanpipe/models.py b/scanpipe/models.py index 7f67f5e23c..14c84b9ecd 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -74,6 +74,7 @@ from licensedcode.cache import get_licensing from matchcode_toolkit.fingerprinting import IGNORED_DIRECTORY_FINGERPRINTS from packagedcode.models import build_package_uid +from packagedcode.utils import get_base_purl from packageurl import PackageURL from packageurl import normalize_qualifiers from packageurl.contrib.django.models import PackageURLMixin @@ -3618,6 +3619,10 @@ def get_absolute_url(self): def purl(self): return self.package_url + @property + def base_purl(self): + return get_base_purl(self.package_url) + @property def package_type(self): return self.type diff --git a/scanpipe/pipes/__init__.py b/scanpipe/pipes/__init__.py index 16c232dadd..f891d78e19 100644 --- a/scanpipe/pipes/__init__.py +++ b/scanpipe/pipes/__init__.py @@ -263,28 +263,10 @@ def update_or_create_dependency( If the dependency is resolved and a resolved package is created, we have the corresponsing package_uid at `resolved_to`. """ - dependency = None - dependency_uid = dependency_data.get("dependency_uid") - extracted_requirement = dependency_data.get("extracted_requirement") - if ignore_dependency_scope(project, dependency_data): return # Do not create the DiscoveredDependency record. - dependencies = [] - if not dependency_uid: - purl_data = DiscoveredDependency.extract_purl_data(dependency_data) - dependencies = DiscoveredDependency.objects.filter( - project=project, - extracted_requirement=extracted_requirement, - **purl_data, - ) - else: - dependency = DiscoveredDependency.objects.get_or_none( - project=project, - dependency_uid=dependency_uid, - ) - if dependency: - dependencies.append(dependency) + dependencies = get_dependencies(project, dependency_data) for dependency in dependencies: is_for_new_package = ( @@ -327,6 +309,35 @@ def update_or_create_dependency( return dependency +def get_dependencies(project, dependency_data): + """ + Given a `dependency_data` mapping, get a list of DiscoveredDependency objects + for that `project` with similar dependency data. + """ + dependency = None + dependency_uid = dependency_data.get("dependency_uid") + extracted_requirement = dependency_data.get("extracted_requirement") + if not extracted_requirement: + extracted_requirement = "" + dependencies = [] + if not dependency_uid: + purl_data = DiscoveredDependency.extract_purl_data(dependency_data) + dependencies = DiscoveredDependency.objects.filter( + project=project, + extracted_requirement=extracted_requirement, + **purl_data, + ) + else: + dependency = DiscoveredDependency.objects.get_or_none( + project=project, + dependency_uid=dependency_uid, + ) + if dependency: + dependencies.append(dependency) + + return dependencies + + def get_or_create_relation(project, relation_data): """ Get or create a CodebaseRelation then return it. diff --git a/scanpipe/pipes/scancode.py b/scanpipe/pipes/scancode.py index e72a9c4713..083523d7f9 100644 --- a/scanpipe/pipes/scancode.py +++ b/scanpipe/pipes/scancode.py @@ -644,6 +644,9 @@ def match_and_resolve_dependencies(project): dependency_data={"purl": dependency.purl}, ignore_nulls=True, ) + extracted_requirement = dependency.extracted_requirement + if not extracted_requirement: + extracted_requirement = "" matched_dependencies = DiscoveredDependency.objects.filter( project=project, @@ -676,7 +679,10 @@ def match_and_resolve_dependencies(project): other_dependencies = [ dep for dep in matched_dependencies - if (dep.purl == dependency.purl and dep.resolved_to_package) + if ( + dep.base_purl == dependency.base_purl + and dep.resolved_to_package + ) ] if other_dependencies: From a9b2ed345b62612618c7b0a7b2380af3d5154bc0 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Fri, 28 Jun 2024 21:49:41 +0530 Subject: [PATCH 10/11] Add unit tests and refactor code Signed-off-by: Ayan Sinha Mahapatra --- scanpipe/pipes/scancode.py | 31 +++---- scanpipe/tests/pipes/test_scancode.py | 126 ++++++++++++++++++++++++++ 2 files changed, 138 insertions(+), 19 deletions(-) diff --git a/scanpipe/pipes/scancode.py b/scanpipe/pipes/scancode.py index 083523d7f9..02c1a1fa38 100644 --- a/scanpipe/pipes/scancode.py +++ b/scanpipe/pipes/scancode.py @@ -494,6 +494,7 @@ def process_package_data(project, static_resolve=False): resource=resource, package_mapping=package_mapping, find_package=False, + process_resolved=False, ) if static_resolve: @@ -505,6 +506,7 @@ def create_packages_and_dependencies_from_mapping( resource, package_mapping, find_package=False, + process_resolved=False, ): """ Create or update packages and dependencies from a `package_mapping`, @@ -512,6 +514,8 @@ def create_packages_and_dependencies_from_mapping( If `find_package` is True, find the package with the respective purl data, instead of trying to create it. + If `process_resolved` is True, also create packages and dependency relations + from the resolved packages of dependencies of this `package_mapping`. """ pd = packagedcode_models.PackageData.from_dict(mapping=package_mapping) if not pd.can_assemble: @@ -547,7 +551,7 @@ def create_packages_and_dependencies_from_mapping( package=package, resource=resource, datasource_id=pd.datasource_id, - process_resolved=False, + process_resolved=process_resolved, ) @@ -559,26 +563,12 @@ def resolve_dependencies(project): logger.info(f"Project {project} resolve_dependencies:") for resource in project.codebaseresources.has_package_data(): for package_mapping in resource.package_data: - pd = packagedcode_models.PackageData.from_dict(package_mapping) - package = None - if pd.purl: - purl_data = DiscoveredPackage.extract_purl_data(package_mapping) - packages = DiscoveredPackage.objects.filter( - project=project, - **purl_data, - ) - - for package in packages: - if resource.location in package.datafile_paths: - break - - dependencies = package_mapping.get("dependencies") or [] - update_packages_and_dependencies( + create_packages_and_dependencies_from_mapping( project=project, - dependencies=dependencies, - package=package, resource=resource, - datasource_id=pd.datasource_id, + package_mapping=package_mapping, + find_package=True, + process_resolved=True, ) match_and_resolve_dependencies(project) @@ -596,6 +586,9 @@ def update_packages_and_dependencies( Create DiscoveredPackage and DiscoveredDependency objects from a package_data dependencies, and also from nested resolved packages and dependencies if present. + + If `process_resolved` is True, also create packages and dependency relations + from the resolved packages of `dependencies`. """ for dep in dependencies: resolved_package = dep.get("resolved_package") or {} diff --git a/scanpipe/tests/pipes/test_scancode.py b/scanpipe/tests/pipes/test_scancode.py index 2a24b59d1c..cf16de928a 100644 --- a/scanpipe/tests/pipes/test_scancode.py +++ b/scanpipe/tests/pipes/test_scancode.py @@ -569,3 +569,129 @@ def test_scanpipe_scancode_process_package_data(self): self.assertEqual(1, project1.discoveredpackages.count()) self.assertEqual(6, project1.discovereddependencies.count()) + + def test_scanpipe_scancode_create_packages_and_dependencies_from_mapping(self): + pipeline_name = "inspect_packages" + project1 = Project.objects.create(name="Analysis") + + input_location = self.data / "dependencies" / "resolved_dependencies.zip" + project1.copy_input_from(input_location) + + run = project1.add_pipeline( + pipeline_name=pipeline_name, + selected_groups=[], + ) + pipeline = run.make_pipeline_instance() + exitcode, out = pipeline.execute() + self.assertEqual(0, exitcode, msg=out) + + self.assertEqual(1, project1.discoveredpackages.count()) + self.assertEqual(7, project1.discovereddependencies.count()) + + yarn_resource = project1.codebaseresources.get( + path="resolved_dependencies.zip-extract/yarn.lock" + ) + lockfile_package_data = yarn_resource.package_data[0] + scancode.create_packages_and_dependencies_from_mapping( + project=project1, + resource=yarn_resource, + package_mapping=lockfile_package_data, + find_package=False, + process_resolved=True, + ) + + self.assertEqual(7, project1.discoveredpackages.count()) + self.assertEqual(12, project1.discovereddependencies.count()) + + def test_scanpipe_scancode_resolve_dependencies(self): + project1 = Project.objects.create(name="Analysis") + pkg_1 = DiscoveredPackage.objects.create( + project=project1, + type="npm", + name="bluebird", + version="3.7.2", + ) + DiscoveredDependency.objects.create( + project=project1, + type="npm", + name="bluebird", + extracted_requirement="^3.5.1", + is_direct=False, + resolved_to_package=pkg_1, + ) + dep_2 = DiscoveredDependency.objects.create( + project=project1, + type="npm", + name="bluebird", + extracted_requirement="^3.5.1", + is_direct=True, + ) + scancode.match_and_resolve_dependencies(project1) + + self.assertEqual(1, project1.discoveredpackages.count()) + self.assertEqual(1, project1.discovereddependencies.count()) + resolved_dep = project1.discovereddependencies.get(name="bluebird") + self.assertEqual(resolved_dep, dep_2) + self.assertEqual(resolved_dep.resolved_to_package, pkg_1) + + def test_scanpipe_scancode_resolve_dependencies_complex_requirements(self): + project1 = Project.objects.create(name="Analysis") + pkg_1 = DiscoveredPackage.objects.create( + project=project1, + type="npm", + name="bluebird", + version="3.7.2", + ) + DiscoveredDependency.objects.create( + project=project1, + type="npm", + name="bluebird", + extracted_requirement="^3.5.1", + is_direct=False, + resolved_to_package=pkg_1, + ) + dep_2 = DiscoveredDependency.objects.create( + project=project1, + type="npm", + name="bluebird", + extracted_requirement="^3.5.1 || ^3.5.0", + is_direct=True, + ) + scancode.match_and_resolve_dependencies(project1) + + self.assertEqual(1, project1.discoveredpackages.count()) + self.assertEqual(1, project1.discovereddependencies.count()) + resolved_dep = project1.discovereddependencies.get(name="bluebird") + self.assertEqual(resolved_dep, dep_2) + self.assertEqual(resolved_dep.resolved_to_package, pkg_1) + + def test_scanpipe_scancode_resolve_dependencies_no_requirements(self): + project1 = Project.objects.create(name="Analysis") + pkg_1 = DiscoveredPackage.objects.create( + project=project1, + type="npm", + name="bluebird", + version="3.7.2", + ) + DiscoveredDependency.objects.create( + project=project1, + type="npm", + name="bluebird", + extracted_requirement="^3.5.1", + is_direct=False, + resolved_to_package=pkg_1, + ) + dep_2 = DiscoveredDependency.objects.create( + project=project1, + type="npm", + name="bluebird", + extracted_requirement="", + is_direct=True, + ) + scancode.match_and_resolve_dependencies(project1) + + self.assertEqual(1, project1.discoveredpackages.count()) + self.assertEqual(1, project1.discovereddependencies.count()) + resolved_dep = project1.discovereddependencies.get(name="bluebird") + self.assertEqual(resolved_dep, dep_2) + self.assertEqual(resolved_dep.resolved_to_package, pkg_1) From f843a405edadf2d666fe2390efd71142fc731291 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Mon, 1 Jul 2024 14:30:15 +0530 Subject: [PATCH 11/11] Address comments and add CHANGELOG entries Signed-off-by: Ayan Sinha Mahapatra --- CHANGELOG.rst | 18 ++++++++++++++++++ scanpipe/pipelines/resolve_dependencies.py | 15 +++++++++++++-- scanpipe/pipes/__init__.py | 5 ++--- scanpipe/pipes/scancode.py | 21 ++++++++++++--------- 4 files changed, 45 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a52d53f539..2979b69981 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -15,6 +15,24 @@ v34.6.4 (unreleased) related work directories created more than a specified number of days ago. https://github.com/nexB/scancode.io/issues/1289 +- Update the ``inspect_packages`` pipeline to have an optional ``Static Resolver`` + group to create resolved packages and dependency relationships from lockfiles + and manifests having pre-resolved dependencies. Also update this pipeline to + perform package assembly from multiple manifests and files to create + discovered packages. Also update the ``resolve_dependencies`` pipeline to have + the same ``Static Resolver`` group and mode the dynamic resolution part to a new + optional ``Dynamic Resolver`` group. + See https://github.com/nexB/scancode.io/pull/1244 + +- Add a new attribute ``is_direct`` to the DiscoveredDependency model and two new + attributes ``is_private`` and ``is_virtual`` to the DiscoveredPackage model. + Also update the UIs to show these attributes and show the ``package_data`` field + contents for CodebaseResources in the ``extra_data`` tab. + See https://github.com/nexB/scancode.io/pull/1244 + +- Update scancode-toolkit to version ``32.2.0``. For the complete list of updates + and improvements see https://github.com/nexB/scancode-toolkit/releases/tag/v32.2.0 + v34.6.3 (2024-06-21) -------------------- diff --git a/scanpipe/pipelines/resolve_dependencies.py b/scanpipe/pipelines/resolve_dependencies.py index fa542976b6..03ddd702b6 100644 --- a/scanpipe/pipelines/resolve_dependencies.py +++ b/scanpipe/pipelines/resolve_dependencies.py @@ -59,15 +59,23 @@ def get_manifest_inputs(self): @group("Static Resolver") def scan_for_application_packages(self): + """ + Scan and assemble application packages from package manifests + and lockfiles. + """ scancode.scan_for_application_packages( self.project, - assemble=False, + assemble=True, resource_qs=self.manifest_resources, progress_logger=self.log, ) @group("Static Resolver") def create_packages_and_dependencies(self): + """ + Create the statically resolved packages and their dependencies + in the database. + """ scancode.process_package_data(self.project, static_resolve=True) @group("Dynamic Resolver") @@ -85,7 +93,10 @@ def get_packages_from_manifest(self): @group("Dynamic Resolver") def create_resolved_packages(self): - """Create the resolved packages and their dependencies in the database.""" + """ + Create the dynamically resolved packages and their dependencies + in the database. + """ resolve.create_packages_and_dependencies( project=self.project, packages=self.resolved_packages, diff --git a/scanpipe/pipes/__init__.py b/scanpipe/pipes/__init__.py index f891d78e19..1f52270dfd 100644 --- a/scanpipe/pipes/__init__.py +++ b/scanpipe/pipes/__init__.py @@ -316,9 +316,8 @@ def get_dependencies(project, dependency_data): """ dependency = None dependency_uid = dependency_data.get("dependency_uid") - extracted_requirement = dependency_data.get("extracted_requirement") - if not extracted_requirement: - extracted_requirement = "" + extracted_requirement = dependency_data.get("extracted_requirement") or "" + dependencies = [] if not dependency_uid: purl_data = DiscoveredDependency.extract_purl_data(dependency_data) diff --git a/scanpipe/pipes/scancode.py b/scanpipe/pipes/scancode.py index 02c1a1fa38..9cb708284e 100644 --- a/scanpipe/pipes/scancode.py +++ b/scanpipe/pipes/scancode.py @@ -648,7 +648,9 @@ def match_and_resolve_dependencies(project): ) other_dependencies = [ - dep for dep in matched_dependencies if dep.purl != dependency.purl + matched_dependency + for matched_dependency in matched_dependencies + if matched_dependency.purl != dependency.purl ] if not other_dependencies: # We also have cases where multiple dependency requirements have one @@ -658,11 +660,12 @@ def match_and_resolve_dependencies(project): **purl_data, ) other_dependencies = [ - dep - for dep in matched_dependencies + matched_dependency + for matched_dependency in matched_dependencies if ( - dep.purl != dependency.purl - and dependency.extracted_requirement in dep.extracted_requirement + matched_dependency.purl != dependency.purl + and dependency.extracted_requirement + in matched_dependency.extracted_requirement ) ] @@ -670,11 +673,11 @@ def match_and_resolve_dependencies(project): # of a package is present for an environment if not other_dependencies: other_dependencies = [ - dep - for dep in matched_dependencies + matched_dependency + for matched_dependency in matched_dependencies if ( - dep.base_purl == dependency.base_purl - and dep.resolved_to_package + matched_dependency.base_purl == dependency.base_purl + and matched_dependency.resolved_to_package ) ]