Refactor the "Import manifest" feature as "Load SBOMs" #61

The process is now using the new specialized "load_sbom" pipeline.

Signed-off-by: tdruez <tdruez@nexb.com>

Author: tdruez

dejacode_toolkit/scancodeio.py

@@ -76,10 +76,10 @@ def submit_scan(self, uri, user_uuid, dataspace_uuid):
         logger.debug(f'{self.label}: submit scan uri="{uri}" webhook_url="{webhook_url}"')
         return self.request_post(url=self.project_api_url, json=data)
 
-    def submit_manifest_inspection(self, project_name, file_location, user_uuid, execute_now=False):
+    def submit_load_sbom(self, project_name, file_location, user_uuid, execute_now=False):
         data = {
             "name": project_name,
-            "pipeline": "inspect_packages",
+            "pipeline": "load_sbom",
             "execute_now": execute_now,
         }
         files = {
@@ -92,7 +92,7 @@ def submit_manifest_inspection(self, project_name, file_location, user_uuid, exe
         data["webhook_url"] = webhook_url
 
         logger.debug(
-            f"{self.label}: submit manifest inspection "
+            f"{self.label}: submit load sbom "
             f'project_name="{project_name}" webhook_url="{webhook_url}"'
         )
         return self.request_post(url=self.project_api_url, data=data, files=files)

dje/tasks.py

@@ -115,15 +115,12 @@ def scancodeio_submit_scan(uris, user_uuid, dataspace_uuid):
 
 
 @job
-def scancodeio_submit_manifest_inspection(scancodeproject_uuid, user_uuid):
-    """
-    Submit the provided `uris` to ScanCode.io as an asynchronous task.
-    Only publicly available URLs are sent to ScanCode.io.
-    """
+def scancodeio_submit_load_sbom(scancodeproject_uuid, user_uuid):
+    """Submit the provided SBOM file to ScanCode.io as an asynchronous task."""
     from dje.models import DejacodeUser
 
     logger.info(
-        f"Entering scancodeio_submit_manifest_inspection task with "
+        f"Entering scancodeio_submit_load_sbom task with "
         f"scancodeproject_uuid={scancodeproject_uuid} user_uuid={user_uuid}"
     )
 
@@ -140,24 +137,24 @@ def scancodeio_submit_manifest_inspection(scancodeproject_uuid, user_uuid):
     # Create a Project instance on ScanCode.io without immediate execution of the
     # pipeline. This allows to get instant feedback from ScanCode.io about the Project
     # creation status and its related data, even in SYNC mode.
-    response = scancodeio.submit_manifest_inspection(
+    response = scancodeio.submit_load_sbom(
         project_name=scancodeproject_uuid,
         file_location=scancode_project.input_file.path,
         user_uuid=user_uuid,
         execute_now=False,
     )
 
     if not response:
-        logger.info("Error submitting the manifest to ScanCode.io server")
+        logger.info("Error submitting the SBOM file to ScanCode.io server")
         scancode_project.status = ScanCodeProject.Status.FAILURE
-        msg = "- Error: Manifest could not be submitted to ScanCode.io"
+        msg = "- Error: SBOM could not be submitted to ScanCode.io"
         scancode_project.append_to_log(msg, save=True)
         return
 
     logger.info("Update the ScanCodeProject instance")
     scancode_project.status = ScanCodeProject.Status.SUBMITTED
     scancode_project.project_uuid = response.get("uuid")
-    msg = "- Manifest submitted to ScanCode.io for inspection"
+    msg = "- SBOM file submitted to ScanCode.io for inspection"
     scancode_project.append_to_log(msg, save=True)
 
     # Delay the execution of the pipeline after the ScancodeProject instance was
@@ -192,8 +189,8 @@ def pull_project_data_from_scancodeio(scancodeproject_uuid):
         status=ScanCodeProject.Status.IMPORT_STARTED
     )
 
-    if scancode_project.type == scancode_project.ProjectType.IMPORT_FROM_MANIFEST:
-        notification_verb = "Import packages from manifest"
+    if scancode_project.type == scancode_project.ProjectType.LOAD_SBOMS:
+        notification_verb = "Load Packages from SBOMs"
     else:
         notification_verb = "Import packages from ScanCode.io"
 

docs/tutorial-1.rst

@@ -19,23 +19,23 @@ you can refer to :ref:`data_model_product` for details about each fields.
 
 .. note:: You are ready to assign Inventory objects to your Product!
 
-Import a Software Bill of Materials (SBOM) to your Product
-==========================================================
+Load a Software Bill of Materials (SBOM) to your Product
+========================================================
 
-You have the flexibility to employ either your CycloneDX or SPDX
+You have the flexibility to employ either your CycloneDX, SPDX, or AboutFile
 Software Bill of Materials (SBOMs).
 
 Alternatively, you can conveniently download one of the provided examples from
 the following
 `GitHub repository <https://github.com/nexB/dejacode/tree/main/docs/sboms/>`_.
 
 On the Product details page, from the :guilabel:`Scan` dropdown, select
-:guilabel:`Import Packages from manifest`:
+:guilabel:`Load Packages from SBOMs`:
 
-* Click the :guilabel:`Choose file/Browse` button on the **Manifest file** field.
+* Click the :guilabel:`Choose File` button on the **SBOM file or zip archive** field.
 * Select your SBOM (.cdx.json or .spdx.json) and click the :guilabel:`Open` button.
 * Check the :guilabel:`Update existing packages with discovered packages data` option.
-* Click the :guilabel:`Import Packages` button.
+* Click the :guilabel:`Load Packages` button.
 
 DejaCode presents the :guilabel:`Imports` tab. Refresh your screen from the browser
 to see the status of your import.

product_portfolio/forms.py

@@ -561,9 +561,9 @@ def helper(self):
         return helper
 
 
-class ImportManifestForm(forms.Form):
+class LoadSBOMsForm(forms.Form):
     input_file = SmartFileField(
-        label=_("Manifest file"),
+        label=_("SBOM file or zip archive"),
         required=True,
     )
     update_existing_packages = forms.BooleanField(
@@ -594,7 +594,7 @@ def helper(self):
         helper.form_method = "post"
         helper.form_id = "import-manifest-form"
         helper.attrs = {"autocomplete": "off"}
-        helper.add_input(Submit("submit", "Import Packages"))
+        helper.add_input(Submit("submit", "Load Packages", css_class="btn-success"))
         return helper
 
 

product_portfolio/migrations/0004_alter_scancodeproject_type.py

@@ -0,0 +1,26 @@
+# Generated by Django 4.2.8 on 2024-02-28 07:28
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("product_portfolio", "0003_alter_scancodeproject_status"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="scancodeproject",
+            name="type",
+            field=models.CharField(
+                choices=[
+                    ("IMPORT_FROM_MANIFEST", "Import from Manifest"),
+                    ("LOAD_SBOMS", "Load SBOMs"),
+                    ("PULL_FROM_SCANCODEIO", "Pull from ScanCode.io"),
+                ],
+                db_index=True,
+                help_text="The type of import, for the ProjectType choices.",
+                max_length=50,
+            ),
+        ),
+    ]

product_portfolio/models.py

@@ -251,8 +251,8 @@ def get_license_summary_url(self):
     def get_check_package_version_url(self):
         return self.get_url("check_package_version")
 
-    def get_import_manifest_url(self):
-        return self.get_url("import_manifest")
+    def get_load_sboms_url(self):
+        return self.get_url("load_sboms")
 
     def get_pull_project_data_url(self):
         return self.get_url("pull_project_data")
@@ -1119,7 +1119,9 @@ class ScanCodeProject(HistoryFieldsMixin, DataspacedModel):
     """Wrap a ScanCode.io Project."""
 
     class ProjectType(models.TextChoices):
+        # This type was replaced by LOAD_SBOMS but is kept for backward compatibility
         IMPORT_FROM_MANIFEST = "IMPORT_FROM_MANIFEST", _("Import from Manifest")
+        LOAD_SBOMS = "LOAD_SBOMS", _("Load SBOMs")
         PULL_FROM_SCANCODEIO = "PULL_FROM_SCANCODEIO", _("Pull from ScanCode.io")
 
     class Status(models.TextChoices):

…duct_portfolio/import_manifest_form.html …s/product_portfolio/load_sboms_form.htmlproduct_portfolio/templates/product_portfolio/import_manifest_form.html renamed to product_portfolio/templates/product_portfolio/load_sboms_form.html product_portfolio/templates/product_portfolio/import_manifest_form.html renamed to product_portfolio/templates/product_portfolio/load_sboms_form.html

@@ -2,7 +2,7 @@
 {% load i18n static crispy_forms_tags %}
 {% load inject_preserved_filters from dje_tags %}
 
-{% block page_title %}{% trans "Import Packages from manifest" %}{% endblock %}
+{% block page_title %}{% trans "Load Packages from SBOMs" %}{% endblock %}
 
 {% block content %}
   <div class="header">
@@ -14,38 +14,50 @@
             / {{ object.get_absolute_link }}
           </div>
           <h1 class="header-title">
-            {% trans "Import Packages from manifest" %}
+            {% trans "Load Packages from SBOMs" %}
           </h1>
         </div>
       </div>
     </div>
   </div>
 
-  <p class="lead">
-    Import and create Packages from a package manifest, lockfile, and SBOM (SPDX document).
-  </p>
-
   {% include 'includes/messages_alert.html' %}
 
-  <div class="alert alert-primary" role="alert">
-    When you upload your <strong>Manifest file</strong>, DejaCode will:
-    <ul class="mb-0 mt-2">
-      <li><strong>Submit the manifest to ScanCode.io</strong> for scan inspection.</li>
-      <li><strong>Retrieve discovered packages</strong> identified by ScanCode.io and import those packages in DejaCode.</li>
-      <li><strong>Assign the packages</strong> to this product.</li>
-    </ul>
-  </div>
-
-  <div class="alert alert-warning">
+  <div class="alert alert-success">
     <div>
-      The following manifest types are supported:
-      <ul class="mb-0 mt-2">
-        <li>Python <strong>requirements.txt</strong></li>
+      DejaCode supports the following SBOM formats:
+      <ul class="mt-2">
         <li>CycloneDX BOM as JSON <strong>bom.json</strong> and <strong>.cdx.json</strong></li>
         <li>SPDX document as JSON <strong>.spdx.json</strong></li>
         <li>AboutCode <strong>.ABOUT</strong> files</li>
       </ul>
     </div>
+    <strong>Multiple SBOMs:</strong>
+    You can provide multiple SBOMs by packaging them into a <strong>zip archive</strong>.
+    DejaCode will handle and process them accordingly.
+  </div>
+
+  <div class="alert alert-primary" role="alert">
+    When you upload your <strong>Software Bill of Materials (SBOM) file to DejaCode</strong>,
+    the following process will occur:
+    <ul class="mb-0 mt-2">
+      <li>
+        <strong>Submission to ScanCode.io</strong>
+        Your SBOM file will be submitted to ScanCode.io for thorough scan inspection.
+      </li>
+      <li>
+        <strong>Package Discovery</strong>
+        ScanCode.io will identify and discover packages within your SBOM.
+      </li>
+      <li>
+        <strong>Package Importation</strong>
+        DejaCode will retrieve the discovered packages from ScanCode.io and import them into its system.
+      </li>
+      <li>
+        <strong>Package Assignment</strong>
+        The imported packages will be assigned to the corresponding product within DejaCode.
+      </li>
+    </ul>
   </div>
 
   <div class="row">
@@ -60,7 +72,7 @@ <h1 class="header-title">
   <script>
     $(document).ready(function () {
       $('form#import-manifest-form').on('submit', function () {
-        NEXB.displayOverlay("Importing Packages from manifest...");
+        NEXB.displayOverlay("Load Packages from SBOM...");
       })
     });
   </script>

product_portfolio/templates/product_portfolio/product_details.html

@@ -38,7 +38,7 @@
           {% if has_change_permission %}
             <a class="dropdown-item" href="{{ object.get_import_from_scan_url }}"><i class="fas fa-file-upload"></i> {% trans 'Import data from Scan' %}</a>
             {% if request.user.dataspace.enable_package_scanning %}
-              <a class="dropdown-item" href="{{ object.get_import_manifest_url }}"><i class="fas fa-file-upload"></i> {% trans 'Import Packages from manifest' %}</a>
+              <a class="dropdown-item" href="{{ object.get_load_sboms_url }}"><i class="fas fa-file-upload"></i> {% trans 'Load Packages from SBOMs' %}</a>
             {% endif %}
             {% if pull_project_data_form %}
               <a class="dropdown-item" style="margin-left: -3px;" href="#" data-bs-toggle="modal" data-bs-target="#pull-project-data-modal"><i class="fas fa-cloud-download-alt"></i> {% trans 'Pull ScanCode.io Project data' %}</a>

product_portfolio/templates/product_portfolio/scancodeio_project_status.html

@@ -1,3 +1,7 @@
+<ul class="list-unstyled">
+  <li><strong>Package count:</strong> {{ scan_data.package_count }}</li>
+  <li><strong>Resource count:</strong> {{ scan_data.resource_count }}</li>
+</ul>
 {% for run in scan_data.runs %}
   <ul class="list-unstyled mb-1">
     <li><strong>Pipeline name:</strong> {{ run.pipeline_name }}</li>

product_portfolio/tests/test_models.py

@@ -731,7 +731,7 @@ def test_product_portfolio_scancode_project_model_can_start_import(self):
         scancode_project = ScanCodeProject.objects.create(
             product=self.product1,
             dataspace=self.product1.dataspace,
-            type=ScanCodeProject.ProjectType.IMPORT_FROM_MANIFEST,
+            type=ScanCodeProject.ProjectType.LOAD_SBOMS,
         )
         self.assertTrue(scancode_project.can_start_import)