Update sctk to V32.1

matchcode/tests/__init__.py

@@ -0,0 +1,13 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# purldb is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/purldb for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import os 
+
+
+FIXTURES_REGEN = os.environ.get("MATCHCODE_TEST_FIXTURES_REGEN", False)

matchcode/tests/test_api.py

@@ -16,6 +16,7 @@
 from matchcode.indexing import index_package_directories
 from matchcode.utils import load_resources_from_scan
 from matchcode.utils import MatchcodeTestCase
+from matchcode.tests import FIXTURES_REGEN
 
 
 class ApproximateDirectoryStructureIndexAPITestCase(MatchcodeTestCase):

matchcode/tests/test_index_packages.py

@@ -28,6 +28,7 @@
 from matchcode.utils import MatchcodeTestCase
 from packagedb.models import Package
 from packagedb.models import Resource
+from matchcode.tests import FIXTURES_REGEN
 
 
 class IndexPackagesTestCase(MatchcodeTestCase):

matchcode/tests/test_match.py

@@ -25,6 +25,7 @@
 from matchcode.utils import index_packages_sha1
 from matchcode.utils import load_resources_from_scan
 from matchcode.utils import MatchcodeTestCase
+from matchcode.tests import FIXTURES_REGEN
 
 
 def run_do_match_from_scan(scan_file_location, match_type):
@@ -108,25 +109,25 @@ def test_do_match_package_archive_match(self):
         input_file = self.get_test_loc('models/match-test.json')
         vc = run_do_match_from_scan(input_file, EXACT_PACKAGE_ARCHIVE_MATCH)
         expected = self.get_test_loc('models/match-test-exact-package-results.json')
-        self.check_codebase(vc, expected, regen=False)
+        self.check_codebase(vc, expected, regen=FIXTURES_REGEN)
 
     def test_do_match_approximate_directory_structure_match(self):
         input_file = self.get_test_loc('models/match-test.json')
         vc = run_do_match_from_scan(input_file, APPROXIMATE_DIRECTORY_STRUCTURE_MATCH)
         expected = self.get_test_loc('models/match-test-approximate-directory-structure-results.json')
-        self.check_codebase(vc, expected, regen=False)
+        self.check_codebase(vc, expected, regen=FIXTURES_REGEN)
 
     def test_do_match_approximate_directory_content_match(self):
         input_file = self.get_test_loc('models/match-test.json')
         vc = run_do_match_from_scan(input_file, APPROXIMATE_DIRECTORY_CONTENT_MATCH)
         expected = self.get_test_loc('models/match-test-approximate-directory-content-results.json')
-        self.check_codebase(vc, expected, regen=False)
+        self.check_codebase(vc, expected, regen=FIXTURES_REGEN)
 
     def test_do_match_package_file_match(self):
         input_file = self.get_test_loc('models/match-test.json')
         vc = run_do_match_from_scan(input_file, EXACT_FILE_MATCH)
         expected = self.get_test_loc('models/match-test-exact-file-results.json')
-        self.check_codebase(vc, expected, regen=False)
+        self.check_codebase(vc, expected, regen=FIXTURES_REGEN)
 
 
 class MatchNestedPackagesTestCase(MatchcodeTestCase):
@@ -166,7 +167,7 @@ def test_do_match_approximate_directory_structure_match(self):
         input_file = self.get_test_loc('match/nested/nested.json')
         vc = run_do_match_from_scan(input_file, APPROXIMATE_DIRECTORY_STRUCTURE_MATCH)
         expected = self.get_test_loc('match/nested/nested-expected.json')
-        self.check_codebase(vc, expected, regen=False)
+        self.check_codebase(vc, expected, regen=FIXTURES_REGEN)
 
 
 class MatchUtilityFunctionsTestCase(MatchcodeTestCase):
@@ -275,52 +276,52 @@ def test_match_ApproximateDirectoryStructureIndex_abbrev_1_0_3(self):
         input_file = self.get_test_loc('match/directory-matching/abbrev-1.0.3-i.json')
         vc = run_do_match_from_scan(input_file, APPROXIMATE_DIRECTORY_STRUCTURE_MATCH)
         expected = self.get_test_loc('match/directory-matching/abbrev-1.0.3-i-expected.json')
-        self.check_codebase(vc, expected, regen=False)
+        self.check_codebase(vc, expected, regen=FIXTURES_REGEN)
 
     def test_match_ApproximateDirectoryStructureIndex_abbrev_1_0_4(self):
         input_file = self.get_test_loc('match/directory-matching/abbrev-1.0.4-i.json')
         vc = run_do_match_from_scan(input_file, APPROXIMATE_DIRECTORY_STRUCTURE_MATCH)
         expected = self.get_test_loc('match/directory-matching/abbrev-1.0.4-i-expected.json')
-        self.check_codebase(vc, expected, regen=False)
+        self.check_codebase(vc, expected, regen=FIXTURES_REGEN)
 
     def test_match_ApproximateDirectoryStructureIndex_abbrev_1_0_5(self):
         input_file = self.get_test_loc('match/directory-matching/abbrev-1.0.5-i.json')
         vc = run_do_match_from_scan(input_file, APPROXIMATE_DIRECTORY_STRUCTURE_MATCH)
         expected = self.get_test_loc('match/directory-matching/abbrev-1.0.5-i-expected.json')
-        self.check_codebase(vc, expected, regen=False)
+        self.check_codebase(vc, expected, regen=FIXTURES_REGEN)
 
     def test_match_ApproximateDirectoryStructureIndex_abbrev_1_0_6(self):
         input_file = self.get_test_loc('match/directory-matching/abbrev-1.0.6-i.json')
         vc = run_do_match_from_scan(input_file, APPROXIMATE_DIRECTORY_STRUCTURE_MATCH)
         expected = self.get_test_loc('match/directory-matching/abbrev-1.0.6-i-expected.json')
-        self.check_codebase(vc, expected, regen=False)
+        self.check_codebase(vc, expected, regen=FIXTURES_REGEN)
 
     def test_match_ApproximateDirectoryStructureIndex_abbrev_1_0_7(self):
         input_file = self.get_test_loc('match/directory-matching/abbrev-1.0.7-i.json')
         vc = run_do_match_from_scan(input_file, APPROXIMATE_DIRECTORY_STRUCTURE_MATCH)
         expected = self.get_test_loc('match/directory-matching/abbrev-1.0.7-i-expected.json')
-        self.check_codebase(vc, expected, regen=False)
+        self.check_codebase(vc, expected, regen=FIXTURES_REGEN)
 
     def test_match_ApproximateDirectoryStructureIndex_abbrev_1_0_9(self):
         input_file = self.get_test_loc('match/directory-matching/abbrev-1.0.9-i.json')
         vc = run_do_match_from_scan(input_file, APPROXIMATE_DIRECTORY_STRUCTURE_MATCH)
         expected = self.get_test_loc('match/directory-matching/abbrev-1.0.9-i-expected.json')
-        self.check_codebase(vc, expected, regen=False)
+        self.check_codebase(vc, expected, regen=FIXTURES_REGEN)
 
     def test_match_ApproximateDirectoryStructureIndex_abbrev_1_1_0(self):
         input_file = self.get_test_loc('match/directory-matching/abbrev-1.1.0-i.json')
         vc = run_do_match_from_scan(input_file, APPROXIMATE_DIRECTORY_STRUCTURE_MATCH)
         expected = self.get_test_loc('match/directory-matching/abbrev-1.1.0-i-expected.json')
-        self.check_codebase(vc, expected, regen=False)
+        self.check_codebase(vc, expected, regen=FIXTURES_REGEN)
 
     def test_match_ApproximateDirectoryStructureIndex_abbrev_1_1_1(self):
         input_file = self.get_test_loc('match/directory-matching/abbrev-1.1.1-i.json')
         vc = run_do_match_from_scan(input_file, APPROXIMATE_DIRECTORY_STRUCTURE_MATCH)
         expected = self.get_test_loc('match/directory-matching/abbrev-1.1.1-i-expected.json')
-        self.check_codebase(vc, expected, regen=False)
+        self.check_codebase(vc, expected, regen=FIXTURES_REGEN)
 
     def test_match_ApproximateDirectoryStructureIndex_get_stdin_3_0_2(self):
         input_file = self.get_test_loc('match/directory-matching/get-stdin-3.0.2-i.json')
         vc = run_do_match_from_scan(input_file, APPROXIMATE_DIRECTORY_STRUCTURE_MATCH)
         expected = self.get_test_loc('match/directory-matching/get-stdin-3.0.2-i-expected.json')
-        self.check_codebase(vc, expected, regen=False)
+        self.check_codebase(vc, expected, regen=FIXTURES_REGEN)

matchcode/tests/test_models.py

@@ -25,6 +25,7 @@
 from matchcode.utils import index_package_files_sha1
 from matchcode.utils import load_resources_from_scan
 from matchcode.utils import MatchcodeTestCase
+from matchcode.tests import FIXTURES_REGEN
 
 
 EXACT_PACKAGE_ARCHIVE_MATCH = 0
@@ -127,7 +128,7 @@ def test_ExactFileIndex_match(self):
             resource.save(codebase)
 
         expected = self.get_test_loc('models/exact-file-matching-standalone-test-results.json')
-        self.check_codebase(codebase, expected, regen=False)
+        self.check_codebase(codebase, expected, regen=FIXTURES_REGEN)
 
 
 class ApproximateDirectoryMatchingIndexModelTestCase(MatchcodeTestCase):
@@ -184,7 +185,7 @@ def test_ApproximateDirectoryStructureIndex_match_subdir(self):
                 resource.save(codebase)
 
         expected = self.get_test_loc('models/directory-matching/async-0.2.9-i-expected-structure.json')
-        self.check_codebase(codebase, expected, regen=False)
+        self.check_codebase(codebase, expected, regen=FIXTURES_REGEN)
 
     def test_ApproximateDirectoryContentIndex_match_subdir(self):
         scan_location = self.get_test_loc('models/directory-matching/async-0.2.9-i.json')
@@ -207,7 +208,7 @@ def test_ApproximateDirectoryContentIndex_match_subdir(self):
                 resource.save(codebase)
 
         expected = self.get_test_loc('models/directory-matching/async-0.2.9-i-expected-content.json')
-        self.check_codebase(codebase, expected, regen=False)
+        self.check_codebase(codebase, expected, regen=FIXTURES_REGEN)
 
 
 class MatchcodeModelUtilsTestCase(MatchcodeTestCase):

matchcode/utils.py

@@ -21,6 +21,8 @@
 
 from matchcode_toolkit.fingerprinting import hexstring_to_binarray
 
+from matchcode.tests import FIXTURES_REGEN
+
 
 ############## TEST UTILITIES ##############
 """
@@ -48,7 +50,7 @@ def get_test_loc(cls, path):
 
 class CodebaseTester(object):
     def check_codebase(self, codebase, expected_codebase_json_loc,
-                       regen=False, remove_file_date=True):
+                       regen=FIXTURES_REGEN, remove_file_date=True):
         """
         Check the Resources of the `codebase` Codebase objects are the same
         as the data in the `expected_codebase_json_loc` JSON file location,

minecode/mappers/apache.py

@@ -55,10 +55,10 @@ def get_packages(self, uri, resource_uri):
         package version.
         """
         metadata = json.loads(resource_uri.data, object_pairs_hook=OrderedDict)
-        return build_packages_from_projects(metadata)
+        return build_packages_from_projects(metadata, uri=uri)
 
 
-def build_packages_from_projects(metadata):
+def build_packages_from_projects(metadata, uri=None):
     """
     Yield Package built from Apache a `metadata` mapping
     which is a dictionary keyed by project name and values are project_metadata.
@@ -70,6 +70,7 @@ def build_packages_from_projects(metadata):
         descriptions = [d for d in (short_desc, long_desc) if d and d.strip()]
         description = '\n'.join(descriptions)
         common_data = dict(
+            datasource_id="apache_json",
             type='apache',
             name=project_name,
             description=description,
@@ -96,7 +97,7 @@ def build_packages_from_projects(metadata):
             parties = common_data.get('parties')
             if not parties:
                 common_data['parties'] = []
-            common_data['parties'].append(party)
+            common_data['parties'].append(party.to_dict())
 
         # license is just a URL in the json file, for example:
         # http://usefulinc.com/doap/licenses/asl20
@@ -127,10 +128,16 @@ def build_packages_from_projects(metadata):
                     rdata['release_date'] = parse_date(release.get('created'))
                 else:
                     logger.warn('Unexpected date format for release date: {}'.format(release.get('created')))
-                package = scan_models.Package(**rdata)
+                package = scan_models.Package.from_package_data(
+                    package_data=rdata,
+                    datafile_path=uri,
+                )
                 yield package
         else:
-            package = scan_models.Package(**common_data)
+            package = scan_models.Package.from_package_data(
+                    package_data=common_data,
+                    datafile_path=uri,
+                )
             yield package
 
 

minecode/mappers/bower.py

@@ -55,15 +55,15 @@ def build_packages_from_jsonfile(metadata, uri=None, purl=None):
     if devdependencies:
         for key, value in devdependencies.items():
             dev_dependencies.append(
-                DependentPackage(purl=key, extracted_requirement=value, scope='devdependency')
+                DependentPackage(purl=key, extracted_requirement=value, scope='devdependency').to_dict()
             )
 
     dependencies = content.get('dependencies')
     dependencies_build = []
     if dependencies:
         for key, value in dependencies.items():
             dependencies_build.append(
-                DependentPackage(purl=key, extracted_requirement=value, scope='runtime')
+                DependentPackage(purl=key, extracted_requirement=value, scope='runtime').to_dict()
             )
 
     if name:
@@ -80,6 +80,7 @@ def build_packages_from_jsonfile(metadata, uri=None, purl=None):
             vcs_url=vcs_repo,
             keywords=keywords_content,
             homepage_url=content.get('homepage'),
+            datasource_id='bower_json',
         )
 
         if extracted_license_statement:
@@ -90,7 +91,7 @@ def build_packages_from_jsonfile(metadata, uri=None, purl=None):
             parties = common_data.get('parties')
             if not parties:
                 common_data['parties'] = []
-            common_data['parties'].append(scan_models.Party(name=author_content, role='author',))
+            common_data['parties'].append(scan_models.Party(name=author_content, role='author',).to_dict())
         else:
             parties = common_data.get('parties')
             if not parties:
@@ -99,7 +100,7 @@ def build_packages_from_jsonfile(metadata, uri=None, purl=None):
             for author in author_content:
                 author_split = author.split(':')
                 if len(author_split) > 1:
-                    common_data['parties'].append(scan_models.Party(name=author_split[1].strip(), role='author',))
+                    common_data['parties'].append(scan_models.Party(name=author_split[1].strip(), role='author',).to_dict())
 
         dependencies = []
         if dependencies_build:
@@ -108,7 +109,10 @@ def build_packages_from_jsonfile(metadata, uri=None, purl=None):
             dependencies.extend(dev_dependencies)
         if len(dependencies) > 0:
             common_data['dependencies'] = dependencies
-        package = scan_models.Package(**common_data)
+        package = scan_models.Package.from_package_data(
+            package_data=common_data,
+            datafile_path=uri,
+        )
         package.set_purl(purl)
         yield package
 

minecode/mappers/cpan.py

@@ -50,6 +50,7 @@ def build_packages_from_release_json(metadata, uri=None):
         extracted_license_statement = [l for l in release.get('license', []) if l and l.strip()]
 
         common_data = dict(
+            datasource_id="cpan_release_json",
             type='cpan',
             name=name,
             description=release.get('abstract'),
@@ -58,7 +59,7 @@ def build_packages_from_release_json(metadata, uri=None):
             extracted_license_statement=extracted_license_statement,
             # the date format passing is like:
             # "2014-04-20T21:30:13"
-            release_date=parse_date(release.get('date'))
+            release_date=parse_date(release.get('date')),
         )
 
         # Get the homepage_url, declared_license and vcs_repository/vcs_tool under resources section.
@@ -101,9 +102,12 @@ def build_packages_from_release_json(metadata, uri=None):
                 type=scan_models.party_person,
                 name=release.get('author'), role='author')
             common_data['parties'] = common_data.get('parties', [])
-            common_data['parties'].append(party)
+            common_data['parties'].append(party.to_dict())
 
-        package = scan_models.Package(**common_data)
+        package = scan_models.Package.from_package_data(
+            package_data=common_data,
+            datafile_path=uri,
+        )
         package_url = PackageURL(type='cpan', name=release.get('name'), version=release.get('version'))
         package.set_purl(package_url.to_string())
         yield package
@@ -173,6 +177,7 @@ def build_packages_from_metafile(metadata, uri=None, purl=None):
             # https://spdx.org/spdx-specification-21-web-version#h.49x2ik5
             vcs_repo = vcs_tool + '+' + vcs_repo
         common_data = dict(
+            datasource_id="cpan_metadata_json",
             type='cpan',
             name=name,
             description=content.get('abstract', name),
@@ -201,9 +206,9 @@ def build_packages_from_metafile(metadata, uri=None, purl=None):
                 email=author_email
             )
 
-            parties.append(party)
+            parties.append(party.to_dict())
 
-        package = scan_models.PackageData(**common_data)
+        package = scan_models.PackageData.from_data(package_data=common_data)
         package.set_purl(purl)
         yield package
 
@@ -260,6 +265,7 @@ def build_packages_from_readmefile(metadata, uri=None, purl=None):
             vcs_repo = vcs_tool + '+' + vcs_repo
         copyr = content.get('COPYRIGHT and LICENSE')
         common_data = dict(
+            datasource_id="cpan_readme",
             type='cpan',
             name=name,
             description=content.get('ABSTRACT', name),
@@ -284,7 +290,7 @@ def build_packages_from_readmefile(metadata, uri=None, purl=None):
             keywords_content = [content.get('KEYWORDS')]
         common_data['keywords'] = keywords_content
 
-        package = scan_models.PackageData(**common_data)
+        package = scan_models.PackageData.from_data(package_data=common_data)
         package.set_purl(purl)
         yield package