Support SCTK v32.1

Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>

Author: AyanSinhaMahapatra

matchcode/utils.py

@@ -21,6 +21,8 @@
 
 from matchcode_toolkit.fingerprinting import hexstring_to_binarray
 
+from matchcode.tests import FIXTURES_REGEN
+
 
 ############## TEST UTILITIES ##############
 """
@@ -48,7 +50,7 @@ def get_test_loc(cls, path):
 
 class CodebaseTester(object):
     def check_codebase(self, codebase, expected_codebase_json_loc,
-                       regen=False, remove_file_date=True):
+                       regen=FIXTURES_REGEN, remove_file_date=True):
         """
         Check the Resources of the `codebase` Codebase objects are the same
         as the data in the `expected_codebase_json_loc` JSON file location,

minecode/mappers/apache.py

@@ -55,10 +55,10 @@ def get_packages(self, uri, resource_uri):
         package version.
         """
         metadata = json.loads(resource_uri.data, object_pairs_hook=OrderedDict)
-        return build_packages_from_projects(metadata)
+        return build_packages_from_projects(metadata, uri=uri)
 
 
-def build_packages_from_projects(metadata):
+def build_packages_from_projects(metadata, uri=None):
     """
     Yield Package built from Apache a `metadata` mapping
     which is a dictionary keyed by project name and values are project_metadata.
@@ -70,6 +70,7 @@ def build_packages_from_projects(metadata):
         descriptions = [d for d in (short_desc, long_desc) if d and d.strip()]
         description = '\n'.join(descriptions)
         common_data = dict(
+            datasource_id="apache_json",
             type='apache',
             name=project_name,
             description=description,
@@ -96,7 +97,7 @@ def build_packages_from_projects(metadata):
             parties = common_data.get('parties')
             if not parties:
                 common_data['parties'] = []
-            common_data['parties'].append(party)
+            common_data['parties'].append(party.to_dict())
 
         # license is just a URL in the json file, for example:
         # http://usefulinc.com/doap/licenses/asl20
@@ -127,10 +128,16 @@ def build_packages_from_projects(metadata):
                     rdata['release_date'] = parse_date(release.get('created'))
                 else:
                     logger.warn('Unexpected date format for release date: {}'.format(release.get('created')))
-                package = scan_models.Package(**rdata)
+                package = scan_models.Package.from_package_data(
+                    package_data=rdata,
+                    datafile_path=uri,
+                )
                 yield package
         else:
-            package = scan_models.Package(**common_data)
+            package = scan_models.Package.from_package_data(
+                    package_data=common_data,
+                    datafile_path=uri,
+                )
             yield package
 
 

minecode/mappers/bower.py

@@ -55,15 +55,15 @@ def build_packages_from_jsonfile(metadata, uri=None, purl=None):
     if devdependencies:
         for key, value in devdependencies.items():
             dev_dependencies.append(
-                DependentPackage(purl=key, extracted_requirement=value, scope='devdependency')
+                DependentPackage(purl=key, extracted_requirement=value, scope='devdependency').to_dict()
             )
 
     dependencies = content.get('dependencies')
     dependencies_build = []
     if dependencies:
         for key, value in dependencies.items():
             dependencies_build.append(
-                DependentPackage(purl=key, extracted_requirement=value, scope='runtime')
+                DependentPackage(purl=key, extracted_requirement=value, scope='runtime').to_dict()
             )
 
     if name:
@@ -80,6 +80,7 @@ def build_packages_from_jsonfile(metadata, uri=None, purl=None):
             vcs_url=vcs_repo,
             keywords=keywords_content,
             homepage_url=content.get('homepage'),
+            datasource_id='bower_json',
         )
 
         if extracted_license_statement:
@@ -90,7 +91,7 @@ def build_packages_from_jsonfile(metadata, uri=None, purl=None):
             parties = common_data.get('parties')
             if not parties:
                 common_data['parties'] = []
-            common_data['parties'].append(scan_models.Party(name=author_content, role='author',))
+            common_data['parties'].append(scan_models.Party(name=author_content, role='author',).to_dict())
         else:
             parties = common_data.get('parties')
             if not parties:
@@ -99,7 +100,7 @@ def build_packages_from_jsonfile(metadata, uri=None, purl=None):
             for author in author_content:
                 author_split = author.split(':')
                 if len(author_split) > 1:
-                    common_data['parties'].append(scan_models.Party(name=author_split[1].strip(), role='author',))
+                    common_data['parties'].append(scan_models.Party(name=author_split[1].strip(), role='author',).to_dict())
 
         dependencies = []
         if dependencies_build:
@@ -108,7 +109,10 @@ def build_packages_from_jsonfile(metadata, uri=None, purl=None):
             dependencies.extend(dev_dependencies)
         if len(dependencies) > 0:
             common_data['dependencies'] = dependencies
-        package = scan_models.Package(**common_data)
+        package = scan_models.Package.from_package_data(
+            package_data=common_data,
+            datafile_path=uri,
+        )
         package.set_purl(purl)
         yield package
 

minecode/mappers/cpan.py

@@ -50,6 +50,7 @@ def build_packages_from_release_json(metadata, uri=None):
         extracted_license_statement = [l for l in release.get('license', []) if l and l.strip()]
 
         common_data = dict(
+            datasource_id="cpan_release_json",
             type='cpan',
             name=name,
             description=release.get('abstract'),
@@ -58,7 +59,7 @@ def build_packages_from_release_json(metadata, uri=None):
             extracted_license_statement=extracted_license_statement,
             # the date format passing is like:
             # "2014-04-20T21:30:13"
-            release_date=parse_date(release.get('date'))
+            release_date=parse_date(release.get('date')),
         )
 
         # Get the homepage_url, declared_license and vcs_repository/vcs_tool under resources section.
@@ -101,9 +102,12 @@ def build_packages_from_release_json(metadata, uri=None):
                 type=scan_models.party_person,
                 name=release.get('author'), role='author')
             common_data['parties'] = common_data.get('parties', [])
-            common_data['parties'].append(party)
+            common_data['parties'].append(party.to_dict())
 
-        package = scan_models.Package(**common_data)
+        package = scan_models.Package.from_package_data(
+            package_data=common_data,
+            datafile_path=uri,
+        )
         package_url = PackageURL(type='cpan', name=release.get('name'), version=release.get('version'))
         package.set_purl(package_url.to_string())
         yield package
@@ -173,6 +177,7 @@ def build_packages_from_metafile(metadata, uri=None, purl=None):
             # https://spdx.org/spdx-specification-21-web-version#h.49x2ik5
             vcs_repo = vcs_tool + '+' + vcs_repo
         common_data = dict(
+            datasource_id="cpan_metadata_json",
             type='cpan',
             name=name,
             description=content.get('abstract', name),
@@ -201,9 +206,9 @@ def build_packages_from_metafile(metadata, uri=None, purl=None):
                 email=author_email
             )
 
-            parties.append(party)
+            parties.append(party.to_dict())
 
-        package = scan_models.PackageData(**common_data)
+        package = scan_models.PackageData.from_data(package_data=common_data)
         package.set_purl(purl)
         yield package
 
@@ -260,6 +265,7 @@ def build_packages_from_readmefile(metadata, uri=None, purl=None):
             vcs_repo = vcs_tool + '+' + vcs_repo
         copyr = content.get('COPYRIGHT and LICENSE')
         common_data = dict(
+            datasource_id="cpan_readme",
             type='cpan',
             name=name,
             description=content.get('ABSTRACT', name),
@@ -284,7 +290,7 @@ def build_packages_from_readmefile(metadata, uri=None, purl=None):
             keywords_content = [content.get('KEYWORDS')]
         common_data['keywords'] = keywords_content
 
-        package = scan_models.PackageData(**common_data)
+        package = scan_models.PackageData.from_data(package_data=common_data)
         package.set_purl(purl)
         yield package
 

minecode/mappers/cran.py

@@ -47,7 +47,7 @@ def get_dependencies(depends):
     if not dependencies:
         return dep_pkgs
     for name in dependencies:
-        dep_pkgs.append(scan_models.DependentPackage(purl=name))
+        dep_pkgs.append(scan_models.DependentPackage(purl=name).to_dict())
     return dep_pkgs
 
 
@@ -60,7 +60,7 @@ def comma_separated(text):
     return [t.strip() for t in text.split(',') if t and t.strip()]
 
 
-def build_packages_from_html(metadata, uri, purl=None):
+def build_packages_from_html(metadata, uri=None, purl=None):
     """
     Yield Package built from Cpan a `metadata` content
     metadata: json metadata content
@@ -69,6 +69,7 @@ def build_packages_from_html(metadata, uri, purl=None):
     """
     # Parse the name from the url, for example: https://cloud.r-project.org/web/packages/ANN2/index.html
     common_data = dict(
+        datasource_id="cran_metadata",
         type='cran',
         name=uri.rpartition('/')[0].rpartition('/')[-1]
     )
@@ -115,15 +116,15 @@ def build_packages_from_html(metadata, uri, purl=None):
                     if not parties:
                         common_data['parties'] = []
                     party = scan_models.Party(type=scan_models.party_person, name=value, role='author')
-                    common_data['parties'].append(party)
+                    common_data['parties'].append(party.to_dict())
                 elif key == 'Maintainer:':
                     maintainer_split = value.split('<')
                     if len(maintainer_split) > 1:
                         parties = common_data.get('parties')
                         if not parties:
                             common_data['parties'] = []
                         party = scan_models.Party(type=scan_models.party_person, name=maintainer_split[0].rstrip(), role='maintainer', email=maintainer_split[1].replace('>', '').replace(' at ', '@'))
-                        common_data['parties'].append(party)
+                        common_data['parties'].append(party.to_dict())
                 elif 'source' in key or 'binaries' in key:
                     if type(value) == list:
                         for url in value:
@@ -140,12 +141,18 @@ def build_packages_from_html(metadata, uri, purl=None):
 
     if download_urls:  # for else statement will have else running always if there is no break statement
         for download_url in download_urls:
-            package = scan_models.Package(**common_data)
+            package = scan_models.Package.from_package_data(
+                package_data=common_data,
+                datafile_path=uri,
+            )
             package.download_url = download_url
             package.set_purl(purl)
             yield package
     else:
         # Yield a package without download_url
-        package = scan_models.Package(**common_data)
+        package = scan_models.Package.from_package_data(
+            package_data=common_data,
+            datafile_path=uri,
+        )
         package.set_purl(purl)
         yield package

minecode/mappers/debian.py

@@ -377,6 +377,7 @@ def build_packages_from_dist_archive(metadata, uri):
     # version by analysing the uri
     version = uri[folder_length:uri.index('/', folder_length)]
     common_data = dict(
+        datasource_id="debian_archive_file",
         name=name,
         version=version,
     )
@@ -415,12 +416,18 @@ def get_resourceuri_by_uri(uri):
 
     if download_urls:
         for download_url in download_urls:
-            package = scan_models.Package(**common_data)
+            package = scan_models.Package.from_package_data(
+                package_data=common_data,
+                datafile_path=uri,
+            )
             package['download_url'] = download_url
             yield package
     else:
         # yield package without a download_url value
-        package = scan_models.DebianPackage(**common_data)
+        package = scan_models.Package.from_package_data(
+                package_data=common_data,
+                datafile_path=uri,
+            )
         # FIXME: this is NOT RIGHT: purl is not defined
-        package.set_purl(purl)
+        package.set_purl(package.purl)
         yield package

minecode/mappers/eclipse.py

@@ -29,10 +29,10 @@ def get_packages(self, uri, resource_uri):
         """
         # FIXME: JSON deserialization should be handled eventually by the framework
         metadata = json.loads(resource_uri.data)
-        return build_packages_with_json(metadata, resource_uri.package_url)
+        return build_packages_with_json(metadata, resource_uri.package_url, uri)
 
 
-def build_packages_with_json(metadata, purl=None):
+def build_packages_with_json(metadata, purl=None, uri=None):
     """
     Yield Package built from Eclipse a `metadata` mapping
     The package can contain multiple projects, and each project can contain
@@ -44,6 +44,7 @@ def build_packages_with_json(metadata, purl=None):
     projects = metadata['projects']
     for project, project_metadata in projects.items():
         common_data = dict(
+            datasource_id="eclipse_metadata",
             type='eclipse',
             name=project,
         )
@@ -70,7 +71,10 @@ def build_packages_with_json(metadata, purl=None):
             durl = download_url.get('url')
             if durl:
                 common_data['download_url'] = durl
-                package = scan_models.Package(**common_data)
+                package = scan_models.Package.from_package_data(
+                    package_data=common_data,
+                    datafile_path=uri,
+                )
                 package.set_purl(purl)
                 yield package
 
@@ -84,16 +88,19 @@ def get_packages(self, uri, resource_uri):
         Yield as many Package as there are download URLs.
         """
         # FIXME: JSON deserialization should be handled eventually by the framework
-        return build_packages(resource_uri.data, resource_uri.package_url)
+        return build_packages(resource_uri.data, resource_uri.package_url, uri)
 
 
-def build_packages(html_text, purl=None):
+def build_packages(html_text, purl=None, uri=None):
     """
     Yield Package objects built from `html_text`and the `purl` package URL
     string.
     """
     page = BeautifulSoup(html_text, 'lxml')
-    common_data = dict(type='eclipse')
+    common_data = dict(
+        datasource_id="eclipse_html",
+        type='eclipse',
+    )
 
     extracted_license_statement = []
     for meta in page.find_all(name='meta'):
@@ -126,7 +133,10 @@ def build_packages(html_text, purl=None):
         if str(a.contents[0]).strip() == 'Downloads':
             download_data = dict(download_url=a['href'],)
             download_data.update(common_data)
-            package = scan_models.Package(**download_data)
+            package = scan_models.Package.from_package_data(
+                package_data=download_data,
+                datafile_path=uri,
+            )
             package.set_purl(purl)
             yield package
 
@@ -151,8 +161,14 @@ def build_packages(html_text, purl=None):
 
                 version = a.contents[0]
                 href = a['href']
-                download_data = dict(version=version, download_url=href)
+                download_data = dict(
+                    version=version,
+                    download_url=href,
+                )
                 download_data.update(common_data)
-                package = scan_models.Package(**download_data)
+                package = scan_models.Package.from_package_data(
+                    package_data=download_data,
+                    datafile_path=uri,
+                )
                 package.set_purl(purl)
                 yield package