Merge branch 'develop' into rename-first-column-in-csv

Author: JRavi2

.travis.yml

@@ -84,15 +84,24 @@ matrix:
       script: 
         - source bin/activate
         - cd docs
-        # This script tests the Sphinx Documentation build
-        # Also checks for broken links in the documentation
-        - ./scripts/sphinx_build_link_check.sh
-        # This script checks for documentation style errors
+        # Check that the Sphinx Documentation build minimally
+        - sphinx-build -E source build
+        # Check for documentation style errors
         - ./scripts/doc8_style_check.sh
       language: python
       python: "3.6"
       dist: bionic
 
+    - os: linux
+      sudo: required
+      script: 
+        # This checks .ABOUT file links using `about check`
+        - source bin/activate
+        - ./bin/about check thirdparty/
+      language: python
+      python: "3.6"
+      dist: bionic
+
     # Refer https://github.com/dkhamsing/awesome_bot#command-line for more info.
     - os: linux
       sudo: required
@@ -107,7 +116,7 @@ matrix:
         # This checks for broken links inside README.md files of plugins/plugins-builtin folders.
         - awesome_bot */**/*.md --allow-redirect --allow-dupe --skip-save-results
         # This checks for broken links inside Thirdparty .ABOUT files
-        - awesome_bot */*.ABOUT --allow-redirect --allow-dupe --skip-save-results
+        - awesome_bot thirdparty/*.ABOUT --allow-redirect --allow-dupe --skip-save-results
 
 addons:
   homebrew:

AUTHORS.rst

@@ -16,3 +16,6 @@ The following organizations or individuals have contributed to ScanCode:
 - Tushar Mittal @techytushar
 - Martin Petkov @MartinPetkov
 - Mrinal Paliwal @mriiinal
+- Mankaran Singh @MankaranSingh
+- Ravi Jain @JRavi2
+- Tushar Goel @TG1999

CHANGELOG.rst

@@ -1,6 +1,12 @@
 Changelog
 =========
 
+Outputs:
+ - The "headers" attribute in JSON outputs now contains a 'duration' field. #1942
+
+OS support:
+ - Add 10.15 macOS tests to azure-pipelines
+
 v3.0.2 (2019-02-15)
 -------------------
 

README.rst

@@ -162,7 +162,7 @@ License
 =======
 
 * Apache-2.0 with an acknowledgement required to accompany the scan output.
-* Public domain CC-0 for reference datasets.
+* Public domain CC0-1.0 for reference datasets.
 * Multiple licenses (GPL2/3, LGPL, MIT, BSD, etc.) for third-party components.
 
 See the NOTICE file and the .ABOUT files that document the origin and license of

azure-pipelines.yml

@@ -83,21 +83,21 @@ jobs:
 
     - template: etc/ci/azure-mac.yml
       parameters:
-          job_name: macos1013_py27
-          image_name: macos-10.13
+          job_name: macos1014_py27
+          image_name: macos-10.14
           python_versions: ['2.7']
           test_suites:
               all: bin/py.test -n 2 -vvs --reruns=3 --ignore=tests/scancode
               scancode: bin/py.test -vvs --reruns=3 tests/scancode
 
     - template: etc/ci/azure-mac.yml
       parameters:
-          job_name: macos1014_py27
-          image_name: macos-10.14
+          job_name: macos1015_py27
+          image_name: macos-10.15
           python_versions: ['2.7']
           test_suites:
               all: bin/py.test -n 2 -vvs --reruns=3 --ignore=tests/scancode
-              scancode: bin/py.test -vvs --reruns=3 tests/scancode
+              scancode: bin/py.test -vvs --reruns=3 tests/scancode          
 
 
 ################################################################################
@@ -236,19 +236,19 @@ jobs:
 
     - template: etc/ci/azure-mac.yml
       parameters:
-          job_name: macos_1013_py36
-          image_name: macos-10.13
+          job_name: macos1014_py36
+          image_name: macos-10.14
           python_versions: ['3.6']
           test_suites:
               all: bin/py.test -n 2 -vvs --reruns=3
 
     - template: etc/ci/azure-mac.yml
       parameters:
-          job_name: macos1014_py36
-          image_name: macos-10.14
+          job_name: macos1015_py36
+          image_name: macos-10.15
           python_versions: ['3.6']
           test_suites:
-              all: bin/py.test -n 2 -vvs --reruns=3
+              all: bin/py.test -n 2 -vvs --reruns=3          
 
     - template: etc/ci/azure-win.yml
       parameters:

docs/source/getting-started/home.rst

@@ -20,7 +20,7 @@ ScanCode tries to address this issue by offering:
 - Your choice of JSON or other output formats (SPDX, HTML, CSV) for integration with other tools
 - ScanCode workbench for Visualization
 - Well-tested, easy to hack, and well-documented code
-- Release of the code and reference data under attribution licenses (Apache 2.0 and CC-BY-1.0)
+- Release of the code and reference data under attribution licenses (Apache 2.0 and CC0-1.0)
 - Plugin System for easily adding new Functionality to Scans.
 - Python 3 Unicode Capabilities for better supporting users from 100+ languages.
 - Extensive Documentation Support.

docs/source/getting-started/whats-new.rst

@@ -49,7 +49,7 @@ deprecated hereafter. The new documentation has improved support in terms of:
 #. Consistent with the Latest Version
 #. Command Line Interface Reference
 #. Getting Started Support for Newcomers
-#. Restructured to fit into the recommended `4 Category Doc Format <https://www.divio.com/blog/documentation/>`_
+#. Restructured to fit into the recommended `4 Category Doc Format <https://docs.divio.com/en/latest/>`_
 #. Updated Tutorials/How To's
 #. Updated Plugin Support
 

docs/source/license.rst

@@ -3,4 +3,4 @@ License
 
 ScanCode Toolkit includes documents that are dedicated to the Public Domain using the
 Creative Commons CC0 1.0 Universal (CC0 1.0) Public Domain Dedication:
-http://creativecommons.org/publicdomain/zero/1.0/
+https://creativecommons.org/publicdomain/zero/1.0/

docs/source/misc/faq.rst

@@ -37,7 +37,7 @@ crafted regex):
   a grammar; it has a few thousand tests.
 - licenses and copyrights are detected in texts and binaries
 
-Licensecheck (available here for reference: /https://metacpan.org/release/App-Licensecheck )
+Licensecheck (available here for reference: /https://metacpan.org/pod/App::Licensecheck )
 is a Perl script using hand-crafted regex patterns to find typical copyright statements and
 about 50 common licenses. There are about 50 license detection tests.
 

etc/scripts/buildrules.py

@@ -184,19 +184,23 @@ def cli(licenses_file):
 
     print()
     for rule in rules_data:
+        is_negative = rule.data.get('is_negative')
+        is_false_positive = rule.data.get('is_false_positive')
         existing = rule_exists(rule.text)
-        if existing:
-            print('Skipping existing rule:', existing, 'with text:\n', rule.text[:50].strip(), '...')
+        if existing and not is_negative:
+            print('Skipping existing non-negative rule:', existing, 'with text:\n', rule.text[:50].strip(), '...')
             continue
-
-        if rule.data.get('is_negative'):
+        
+        if is_negative:
             base_name = 'not-a-license'
         else:
             license_expression = rule.data.get('license_expression')
             if not license_expression:
                 raise Exception('Missing license_expression for text:', rule)
             licensing.parse(license_expression, validate=True, simple=True)
             base_name = license_expression
+            if is_false_positive:
+                base_name = 'false-positive_' + base_name
 
         base_loc = find_rule_base_loc(base_name)
 
@@ -218,8 +222,8 @@ def cli(licenses_file):
         else:
             rules_tokens.add(rule_tokens)
             rulerec.dump()
-            models.update_ignorables(rulerec, verbose=True)
-            print('Rule added:', rulerec.identifier)
+            models.update_ignorables(rulerec, verbose=False)
+            print('Rule added:', 'file://' + rulerec.data_file, '\n', 'file://' + rulerec.text_file,)
 
 
 if __name__ == '__main__':