Merge pull request #1835 from nexB/1833-install-pypi-py27

Make PyPI installation work on Python 2 #1833

Author: pombredanne

.travis.yml

@@ -1,11 +1,13 @@
+
+################################################################################
+# We use Travis to run minimal smoke tests suites on Pythons 2.7 and 3.6
+# on macOS and Linux
+# We also run the documentation build and the ABOUT file checks
+################################################################################
+
 matrix:
   include:
-    - os: osx
-      env: 
-        - PYTHON="3.6.8"
-        - TEST_SUITE="bin/py.test -vvs --reruns 3 --cov=src --cov-report=term --cov-report=xml -n2 --ignore=tests/scancode"
-      language: generic
-      python:
+    # Run minimal test suite
     - os: osx
       env: 
         - PYTHON="3.6.8"
@@ -16,74 +18,45 @@ matrix:
     - os: osx
       env: 
         - PYTHON="2.7.15"
-        - TEST_SUITE="bin/py.test -vvs --reruns 3 --cov=src --cov-report=term --cov-report=xml -n2 --ignore=tests/scancode"
-      language: generic
-      python:
-    - os: osx
-      env: 
-        - PYTHON="2.7.15"
-        - TEST_SUITE="bin/py.test -vvs --reruns 3 --cov=src --cov-report=term --cov-report=xml tests/scancode"
+        - TEST_SUITE="bin/py.test -vvs --reruns 3 tests/scancode"
       language: generic
       python:
 
     - os: linux
       sudo: required
-      env: TEST_SUITE="bin/py.test -vvs --reruns 3 --cov=src --cov-report=term --cov-report=xml -n2 --ignore=tests/scancode"
-      language: python
-      python: "2.7"
-      dist: xenial
-    - os: linux
-      sudo: required
-      env: TEST_SUITE="bin/py.test -vvs --reruns 3 --cov=src --cov-report=term --cov-report=xml tests/scancode"
+      env: TEST_SUITE="bin/py.test -vvs --reruns 3 tests/scancode"
       language: python
       python: "2.7"
       dist: xenial
 
     - os: linux
       sudo: required
-      env: TEST_SUITE="bin/py.test -vvs --reruns 3 --cov=src --cov-report=term --cov-report=xml -n2 --ignore=tests/scancode"
-      language: python
-      python: "2.7"
-      dist: bionic
-    - os: linux
-      sudo: required
-      env: TEST_SUITE="bin/py.test -vvs --reruns 3 --cov=src --cov-report=term --cov-report=xml tests/scancode"
+      env: TEST_SUITE="bin/py.test -vvs --reruns 3 tests/scancode"
       language: python
       python: "2.7"
       dist: bionic
 
-    - os: linux
-      sudo: required
-      env: TEST_SUITE="bin/py.test -vvs --reruns 3 --cov=src --cov-report=term --cov-report=xml -n2 --ignore=tests/scancode"
-      language: python
-      python: "3.6"
-      dist: xenial
     - os: linux
       sudo: required
       env: TEST_SUITE="bin/py.test -vvs --reruns 3 tests/scancode"
       language: python
       python: "3.6"
       dist: xenial
 
-    - os: linux
-      sudo: required
-      env: TEST_SUITE="bin/py.test -vvs --reruns 3 --cov=src --cov-report=term --cov-report=xml -n2 --ignore=tests/scancode"
-      language: python
-      python: "3.6"
-      dist: bionic
     - os: linux
       sudo: required
       env: TEST_SUITE="bin/py.test -vvs --reruns 3 tests/scancode"
       language: python
       python: "3.6"
       dist: bionic
 
-    # Refer https://scancode-toolkit.readthedocs.io/en/latest/contribute/contrib_doc.html#continuous-integration
+    # Check and lint the documentation files
     - os: linux
       sudo: required
       script: 
         - source bin/activate
         - cd docs
+        - pip install -r requirements-doc.txt
         # Check that the Sphinx Documentation build minimally
         - sphinx-build -E source build
         # Check for documentation style errors
@@ -92,32 +65,19 @@ matrix:
       python: "3.6"
       dist: bionic
 
+    # Check and lint .ABOUT files
     - os: linux
       sudo: required
       script: 
-        # This checks .ABOUT file links using `about check`
         - source bin/activate
         - ./bin/about check thirdparty/
+        - ./bin/about check src/
+        - ./bin/about check etc/
+        - ./bin/about check scancode-toolkit.ABOUT
       language: python
       python: "3.6"
       dist: bionic
 
-    # Refer https://github.com/dkhamsing/awesome_bot#command-line for more info.
-    - os: linux
-      sudo: required
-      before_install:
-        - gem install awesome_bot
-      script:
-        # These check for broken links inside .md files used for GitHub templates
-        - awesome_bot .github/*.md --allow-redirect --allow-dupe --skip-save-results
-        - awesome_bot .github/ISSUE_TEMPLATE/*.md --allow-redirect --allow-dupe --skip-save-results
-        # This checks for broken links inside .rst files at Project Root, like README.rst
-        - awesome_bot *.rst --allow-redirect --allow-dupe --skip-save-results
-        # This checks for broken links inside README.md files of plugins/plugins-builtin folders.
-        - awesome_bot */**/*.md --allow-redirect --allow-dupe --skip-save-results
-        # This checks for broken links inside Thirdparty .ABOUT files
-        - awesome_bot thirdparty/*.ABOUT --allow-redirect --allow-dupe --skip-save-results
-
 addons:
   homebrew:
     packages:
@@ -127,7 +87,7 @@ addons:
         - xz
         - zlib
         - pyenv
-    update: true
+    update: false
 
 before_install:
   - chmod +x ./docs/scripts/sphinx_build_link_check.sh
@@ -138,15 +98,10 @@ install:
 
 script:
   # If debugging, use a subset of tests to wait less:
-  # - ./bin/py.test -n 2 -vvs --cov=src tests/scancode
+  # - ./bin/py.test -n 2 -vvs tests/scancode
   # - echo $TEST_SUITE
   - $TEST_SUITE
 
-# Pay attention to both calls below. They will succeed even if there
-# is no coverage information or connection to reporting website fails.
-after_success:
-  - ./bin/codecov
-
 notifications:
   irc:
     channels:

MANIFEST.in

@@ -8,8 +8,6 @@ graft docs
 graft tests
 graft plugins-builtin
 
-prune src/scancode_toolkit.egg-info
-
 include .bumpversion.cfg
 include .coveragerc
 include .gitattributes
@@ -28,10 +26,8 @@ include conftest.py
 include CONTRIBUTING.rst
 include extractcode
 include extractcode.bat
-include ISSUE_TEMPLATE.md
 include MANIFEST.in
 include NOTICE
-include pyvenv.cfg
 include README.rst
 include scancode
 include scancode-toolkit.ABOUT

appveyor.yml

@@ -1,14 +1,23 @@
+
+################################################################################
+# We use  Appveyor to run minimal smoke tests suites on Pythons 2.7 and 3.6
+# on Windows 32 and 64 bits
+################################################################################
 environment:
     matrix:
-        - TEST_SUITE: "py.test -vvs --reruns 3 -n2 --ignore=tests/scancode"
-        - TEST_SUITE: "py.test -vvs --reruns 3     tests/scancode"
+        - PYTHON: "C:\\Python27"
+        - PYTHON: "C:\\Python27-x64"
+        - PYTHON: "C:\\Python36"
+        - PYTHON: "C:\\Python36-x64"
+#        - PYTHON: "C:\\Python37"
+#        - PYTHON: "C:\\Python37-x64"
+#        - PYTHON: "C:\\Python38"
+#        - PYTHON: "C:\\Python38-x64"
 
-install:
-    - configure
 
 build: off
 
 
 test_script:
-    - python -c "import sys;print sys.getdefaultencoding()"
-    - cmd: "%TEST_SUITE%"
+    - python -c "import sys;print(sys.getdefaultencoding())"
+    - cmd: "configure --python %PYTHON%\\python.exe && Scripts\\pytest -vvs --reruns 3 tests\\scancode"

azure-pipelines.yml

@@ -1,4 +1,9 @@
 
+################################################################################
+# We use Azure to run the full tests suites on Pythons 2.7 and 3.6
+# on Windows (32 and 64), macOS and Linux (64 various distro)
+################################################################################
+
 jobs:
 
 ################################################################################
@@ -58,18 +63,6 @@ jobs:
                   --cov=src --cov-report=term --cov-report=xml
                 bin/codecov --token "$CODECOV_TOKEN"
 
-              plugins-scancode-fingerprint: |
-                bin/pip install -e plugins/scancode-fingerprint
-                bin/py.test -vvs plugins/scancode-fingerprint/tests/
-                bin/codecov --token "$CODECOV_TOKEN"
-
-              plugins-scancode-compiledcode: |
-                bin/pip install -e plugins/scancode-dwarfdump-manylinux1_x86_64
-                bin/pip install -e plugins/scancode-ctags-manylinux1_x86_64
-                bin/pip install -e plugins/scancode-readelf-manylinux1_x86_64
-                bin/pip install -e plugins/scancode-compiledcode
-                bin/py.test -vvs plugins/scancode-compiledcode/tests/
-
     - template: etc/ci/azure-win.yml
       parameters:
           job_name: Win2016_32_py27
@@ -81,14 +74,16 @@ jobs:
               scancode: Scripts\py.test -vvs --reruns=3 tests\scancode
               lic_cluecode: Scripts\py.test -vvs  --reruns=3 tests\cluecode  tests\licensedcode
 
-    - template: etc/ci/azure-mac.yml
+    - template: etc/ci/azure-win.yml
       parameters:
-          job_name: macos1014_py27
-          image_name: macos-10.14
+          job_name: Win2016_64_py27
+          image_name: vs2017-win2016
           python_versions: ['2.7']
+          python_architecture: x64
           test_suites:
-              all: bin/py.test -n 2 -vvs --reruns=3 --ignore=tests/scancode
-              scancode: bin/py.test -vvs --reruns=3 tests/scancode
+              misc: Scripts\py.test -vvs --reruns=3 --ignore=tests\scancode --ignore=tests\cluecode  --ignore=tests\licensedcode
+              scancode: Scripts\py.test -vvs --reruns=3 tests\scancode
+              lic_cluecode: Scripts\py.test -vvs  --reruns=3 tests\cluecode  tests\licensedcode
 
     - template: etc/ci/azure-mac.yml
       parameters:
@@ -97,23 +92,13 @@ jobs:
           python_versions: ['2.7']
           test_suites:
               all: bin/py.test -n 2 -vvs --reruns=3 --ignore=tests/scancode
-              scancode: bin/py.test -vvs --reruns=3 tests/scancode          
+              scancode: bin/py.test -vvs --reruns=3 tests/scancode
 
 
 ################################################################################
 # These jobs are using containers and their own Python 2.7
 ################################################################################
 
-#    - template: etc/ci/azure-container-deb.yml
-#      parameters:
-#          job_name: ubuntu16_py27
-#          container: ubuntu:xenial
-#          python_path: python2.7
-#          python_version: '2.7'
-#          install_python: sudo apt-get install -y python python-dev
-#          test_suite_label: all
-#          test_suite: bin/py.test -n 2 -vvs --reruns=3
-
     - template: etc/ci/azure-container-deb.yml
       parameters:
           job_name: ubuntu18_py27
@@ -179,8 +164,9 @@ jobs:
 #          test_suite_label: all
 #          test_suite: bin/py.test -n 2 -vvs --reruns=3
 
+
 ########################################################################
-# RELEASE on 2.7
+# RELEASE on 2.7. Also check that we can pip install
 ########################################################################
 
     - job: Build_release_archive_py2
@@ -198,6 +184,9 @@ jobs:
           - script: ./etc/release/release.sh
             displayName: 'Build installable releases'
 
+          - script: ./etc/release/pip-install2.sh
+            displayName: 'Test pip installation'
+
 
 ########################################################################
 # RELEASE on 3.6. Also check that we can pip install
@@ -236,26 +225,29 @@ jobs:
 
     - template: etc/ci/azure-mac.yml
       parameters:
-          job_name: macos1014_py36
-          image_name: macos-10.14
+          job_name: macos1015_py36
+          image_name: macos-10.15
           python_versions: ['3.6']
           test_suites:
               all: bin/py.test -n 2 -vvs --reruns=3
 
-    - template: etc/ci/azure-mac.yml
+    - template: etc/ci/azure-win.yml
       parameters:
-          job_name: macos1015_py36
-          image_name: macos-10.15
+          job_name: Win2016_32_py36
+          image_name: vs2017-win2016
           python_versions: ['3.6']
+          python_architecture: x86
           test_suites:
-              all: bin/py.test -n 2 -vvs --reruns=3          
+              misc: Scripts\py.test -vvs --reruns=3 --ignore=tests\scancode --ignore=tests\cluecode  --ignore=tests\licensedcode
+              scancode: Scripts\py.test -vvs --reruns=3 tests\scancode
+              lic_cluecode: Scripts\py.test -vvs  --reruns=3 tests\cluecode  tests\licensedcode
 
     - template: etc/ci/azure-win.yml
       parameters:
-          job_name: Win2016_32_py36
+          job_name: Win2016_64_py36
           image_name: vs2017-win2016
           python_versions: ['3.6']
-          python_architecture: x86
+          python_architecture: x64
           test_suites:
               misc: Scripts\py.test -vvs --reruns=3 --ignore=tests\scancode --ignore=tests\cluecode  --ignore=tests\licensedcode
               scancode: Scripts\py.test -vvs --reruns=3 tests\scancode
@@ -324,7 +316,7 @@ jobs:
 #          install_python: echo "Python is pre-installed"
 #          test_suite_label: all
 #          test_suite: bin/py.test -n 2 -vvs --reruns=3
-#
+
 
 ################################################################################
 # These jobs are using containers and their own Python 3.7
@@ -384,20 +376,3 @@ jobs:
 #            sudo make altinstall
 #          test_suite_label: basic tests
 #          test_suite: bin/py.test -n 2 -vvs --reruns=3
-
-
-################################################################################
-# Misc: we do not support yet Windows 64
-################################################################################
-
-# 
-#    - template: etc/ci/azure-win.yml
-#      parameters:
-#          job_name: Win2016_64
-#          image_name: vs2017-win2016
-#          python_versions: ['2.7', '3.6']
-#          python_architecture: x64
-#          test_suites:
-#              all: Scripts\py.test -n 2 -vvs --ignore=tests\scancode
-#              scancode: Scripts\py.test -vvs tests\scancode
-

configure

@@ -20,6 +20,12 @@ export TPP_DIR="thirdparty"
 
 # default configurations for dev
 CONF_DEFAULT="etc/conf/dev"
+
+if [[ "$CONF_DEFAULT" == "etc/conf/dev" ]]; then
+    export TPP_DIR_DEV="thirdparty/dev"
+fi
+
+
 #################################
 
 CFG_CMD_LINE_ARGS="$@"