Merge pull request #2776 from fangxlmr/fix-yaml-dup-keys

Fix invalid license yaml files by resolving duplicated keys

Author: pombredanne

src/licensedcode/data/licenses/agpl-1.0-plus.yml

@@ -15,7 +15,6 @@ other_spdx_license_keys:
 text_urls:
     - http://www.affero.org/oagpl.html
 faq_url: http://www.affero.org/oagf.html
-minimum_coverage: 99
 ignorable_copyrights:
   - Copyright (c) 2002 Affero Inc.
   - copyright (c) 1989, 1991 Free Software Foundation, Inc.

src/licensedcode/data/licenses/agpl-3.0-plus.yml

@@ -20,8 +20,6 @@ other_urls:
     - http://www.opensource.org/licenses/AGPL-3.0
     - https://opensource.org/licenses/AGPL-3.0
     - https://www.gnu.org/licenses/agpl.txt
-ignorable_urls:
-    - http://www.gnu.org/licenses/
 minimum_coverage: 99
 ignorable_copyrights:
   - Copyright (c) 2007 Free Software Foundation, Inc. <https://fsf.org/>
@@ -31,4 +29,3 @@ ignorable_urls:
   - http://www.gnu.org/licenses/
   - https://fsf.org/
   - https://www.gnu.org/licenses/
-

src/licensedcode/data/licenses/ecos.yml

@@ -5,12 +5,13 @@ name: GPL 2.0 or later with eCos Exception
 category: Copyleft Limited
 owner: eCos
 homepage_url: http://ecos.sourceware.org/ecos-license/
-notes: Per SPDX.org, this is really GPL v2 or later + an exception
 is_exception: yes
 spdx_license_key: eCos-2.0
 text_urls:
     - http://www.fsf.org/licensing/licenses/ecos-license.html
     - http://www.gnu.org/licenses/ecos-license.html
 faq_url: http://ecos.sourceware.org/license-overview.html
 minimum_coverage: 100
-notes: replaced by ecos-exception-2.0
+notes: |
+    Per SPDX.org, this is really GPL v2 or later + an exception.
+    Replaced by ecos-exception-2.0.

src/licensedcode/data/licenses/ekioh.yml

@@ -4,9 +4,9 @@ short_name: Ekioh License
 name: Ekioh License
 category: Permissive
 owner: Ekioh
-notes: see also https://kryogenix.org/code/browser/licence.html
 notes: |
     Obsolete License -- do not use.  Except for the dangling ", subject to the
     following conditions:" text, this license is essentially identical to the
     MIT-0-Clause (mit-0) license, which has been accepted in the open source
     community relatively recently.
+    See also https://kryogenix.org/code/browser/licence.html.

src/licensedcode/data/licenses/gpl-2.0-plus-openssl.yml

@@ -7,4 +7,3 @@ owner: magnumripper
 homepage_url: https://github.com/magnumripper/JohnTheRipper/blob/1.8.0/doc/LICENSE
 notes: replaced by openssl-exception-gpl-2.0-plus
 is_exception: yes
-notes: replaced by openssl-exception-gpl-2.0-plus

src/licensedcode/data/licenses/gpl-3.0-plus.yml

@@ -16,8 +16,6 @@ other_urls:
     - http://www.opensource.org/licenses/GPL-3.0
     - https://opensource.org/licenses/GPL-3.0
     - https://www.gnu.org/licenses/gpl-3.0-standalone.html
-ignorable_urls:
-    - http://www.gnu.org/licenses/
 minimum_coverage: 99
 ignorable_copyrights:
   - Copyright (c) 2007 Free Software Foundation, Inc. <https://fsf.org/>
@@ -28,4 +26,3 @@ ignorable_urls:
   - https://fsf.org/
   - https://www.gnu.org/licenses/
   - https://www.gnu.org/licenses/why-not-lgpl.html
-

src/licensedcode/data/licenses/lgpl-2.1-spell-checker.yml

@@ -20,4 +20,3 @@ standard_notice: |-
     this file, you may extend this exception to your version of the
     file, but you are not obligated to do so. If you do not wish to
     do so, delete this exception statement from your version.
-notes: replaced by spell-checker-exception-lgpl-2.1-plus

src/licensedcode/data/licenses/openssl.yml

@@ -12,7 +12,6 @@ spdx_license_key: LicenseRef-scancode-openssl
 faq_url: http://www.openssl.org/support/faq.html
 other_urls:
     - http://www.openssl.org/source/license.html
-minimum_coverage: 30
 ignorable_authors:
     - the OpenSSL Project
 ignorable_urls:

src/licensedcode/data/rules/bsd-new_or_gpl-2.0_30.yml

@@ -1,9 +1,9 @@
 license_expression: bsd-new OR gpl-2.0
 is_license_notice: yes
 relevance: 99
-notes: this is also rarely seen with the bsd-simplified license, hence the lowered relevance 
 referenced_filenames:
     - LICENSE
     - COPYING
-notes: this is used sometimes with a bsd-new and sometimes with a 
- bsd-simplified and is both long enough and vaguely defined
+notes: |
+    This is used sometimes with a bsd-new and is both long enough and vaguely defined.
+    This is also rarely seen with the bsd-simplified license, hence the lowered relevance.

src/licensedcode/models.py

@@ -258,7 +258,7 @@ def load(self):
         """
         try:
             with io.open(self.data_file, encoding='utf-8') as f:
-                data = saneyaml.load(f.read())
+                data = saneyaml.load(f.read(), allow_duplicate_keys=False)
 
             for k, v in data.items():
                 if k == 'minimum_coverage':
@@ -1228,7 +1228,7 @@ def load(self):
         """
         try:
             with io.open(self.data_file, encoding='utf-8') as f:
-                data = saneyaml.load(f.read())
+                data = saneyaml.load(f.read(), allow_duplicate_keys=False)
         except Exception as e:
             print('#############################')
             print('INVALID LICENSE RULE FILE:', f'file://{self.data_file}')