Version 21.5.12 preparation

.gitattributes

@@ -1,2 +1,3 @@
 # Ignore all Git auto CR/LF line endings conversions
 * -text
+pyproject.toml export-subst

.travis.yml

@@ -16,7 +16,7 @@ python:
   - "3.9"
 
 # Scripts to run at install stage
-install: ./configure
+install: ./configure --dev
 
 # Scripts to run at script stage
 script: tmp/bin/pytest -vvs -n 2

CHANGELOG.rst

@@ -5,44 +5,55 @@ vNext
 -----
 
 
+Version 21.5.12
+---------------
+
+- Add new function to find a command or shared object file in the PATH (e.g. in
+  environment variables). See commoncode.command.find_in_path()
+- Add new simplified the commoncode.command.execute() function. 
+- Add support for Python 3.10
+- Update tests to cope with Python 3.6 bug https://bugs.python.org/issue26919
+- Adopt latest skeleton with configure scripts updates
+
+Breaking API changes:
+
+- commoncode.command.load_shared_library() now ignores the lib_dir argument
+- commoncode.command.execute2() is deprecated and ignores the lib_dir argument
+  it is replaced by commoncode.command.execute()
+- In commoncode.testcase get_test_loc() "exists" argument has been renamed to
+  "must_exist". It has also been added to FileDrivenTesting.get_test_loc()
+  method.
+
+
 Version 21.4.28
 ---------------
 
-*2020-04-28*
 - Add new function to get a Resource path stripped from its root path segment
 
 
 Version 21.1.21
 ---------------
 
-*2020-01-21*
 - Improve error reporting when oding missing DLLs
 - Clean config and improve basic documentation
 
 
 Version 21.1.14
 ---------------
 
-*2020-11-12*
 - Update dependencies
-
-*2020-11-12*
 - Add Azure Pipelines CI support
-
-*2021-01-14*
 - Drop Python 2 support
 - Update license
 
 
 Version 20.10.08
 ----------------
 
-*2020-10-08*
 - Add support for both python 2 + 3
-
-*2020-10-08*
 - Add CI support for python 2 + 3
 
+
 Version 20.10
 -------------
 
@@ -52,12 +63,11 @@ Version 20.10
 Version 20.09.30
 ----------------
 
-*2020-09-25*
 - Update to PEP 517/518 development practices
 - Add some minimal documentation
 
+
 Version 20.09
 -------------
 
-*2020-09-24*
 - Initial release.

NOTICE

@@ -1,19 +1,7 @@
 #
-# Copyright (c) nexB Inc. and others.
+# Copyright (c) nexB Inc. and others. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
-#
-# Visit https://aboutcode.org and https://github.com/nexB/ for support and download.
-# ScanCode is a trademark of nexB Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/commoncode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
 #

README.rst

@@ -13,12 +13,13 @@ Visit https://aboutcode.org and https://github.com/nexB/ for support and downloa
 
 To install this package use::
 
-    pip install commoncode]
+    pip install commoncode
 
 
-To set up the development environment::
 
-    source configure
+Alternatively, to set up a development environment::
+
+    source configure --dev
 
 To run unit tests::
 

azure-pipelines.yml

@@ -7,39 +7,39 @@
 
 jobs:
 
-    - template: etc/ci/azure-linux.yml
+    - template: etc/ci/azure-posix.yml
       parameters:
           job_name: ubuntu16_cpython
           image_name: ubuntu-16.04
           python_versions: ['3.6', '3.7', '3.8', '3.9']
           test_suites:
               all: tmp/bin/pytest -vvs
 
-    - template: etc/ci/azure-linux.yml
+    - template: etc/ci/azure-posix.yml
       parameters:
           job_name: ubuntu18_cpython
           image_name: ubuntu-18.04
           python_versions: ['3.6', '3.7', '3.8', '3.9']
           test_suites:
               all: tmp/bin/pytest -n 2 -vvs
 
-    - template: etc/ci/azure-linux.yml
+    - template: etc/ci/azure-posix.yml
       parameters:
           job_name: ubuntu20_cpython
           image_name: ubuntu-20.04
           python_versions: ['3.6', '3.7', '3.8', '3.9']
           test_suites:
               all: tmp/bin/pytest -n 2 -vvs
 
-    - template: etc/ci/azure-mac.yml
+    - template: etc/ci/azure-posix.yml
       parameters:
           job_name: macos1014_cpython
           image_name: macos-10.14
           python_versions: ['3.6', '3.7', '3.8', '3.9']
           test_suites:
               all: tmp/bin/pytest -n 2 -vvs
 
-    - template: etc/ci/azure-mac.yml
+    - template: etc/ci/azure-posix.yml
       parameters:
           job_name: macos1015_cpython
           image_name: macos-10.15

configure

@@ -1,44 +1,169 @@
 #!/usr/bin/env bash
 #
-# Copyright (c) nexB Inc. and others.
+# Copyright (c) nexB Inc. and others. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/ for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
 #
 
 set -e
 #set -x
 
-# source this script for a basic setup and configuration for local development
+################################
+# A configuration script to set things up: 
+# create a virtualenv and install or update thirdparty packages.
+# Source this script for initial configuration
+# Use configure --help for details
+#
+# This script will search for a virtualenv.pyz app in etc/thirdparty/virtualenv.pyz
+# Otherwise it will download the latest from the VIRTUALENV_PYZ_URL default
+################################
+CLI_ARGS=$1
 
-CONFIGURE_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+################################
+# Defaults. Change these variables to customize this script
+################################
 
+# Requirement arguments passed to pip and used by default or with --dev.
+REQUIREMENTS="--editable . --constraint requirements.txt"
+DEV_REQUIREMENTS="--editable .[testing] --constraint requirements.txt"
 
-if [[ "$1" == "--clean" ]]; then
-    rm -rf "$CONFIGURE_ROOT_DIR/tmp"
-    exit
+# where we create a virtualenv
+VIRTUALENV_DIR=tmp
+
+# Cleanable files and directories with the --clean option
+CLEANABLE="
+    build
+	tmp"
+
+# extra  arguments passed to pip
+PIP_EXTRA_ARGS=" "
+
+# the URL to download virtualenv.pyz if needed
+VIRTUALENV_PYZ_URL=https://bootstrap.pypa.io/virtualenv.pyz
+################################
+
+
+################################
+# Current directory where this script lives
+CFG_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+CFG_BIN_DIR=$CFG_ROOT_DIR/$VIRTUALENV_DIR/bin
+
+
+################################
+# Set the quiet flag to empty if not defined 
+if [[ "$CFG_QUIET" == "" ]]; then
+    CFG_QUIET=" "
 fi
 
 
-if [[ "$PYTHON_EXE" == "" ]]; then
-    PYTHON_EXE=python3
+################################
+# find a proper Python to run
+# Use environment variables or a file if available.
+# Otherwise the latest Python by default.
+if [[ "$PYTHON_EXECUTABLE" == "" ]]; then
+    # check for a file named PYTHON_EXECUTABLE 
+    if [ -f "$CFG_ROOT_DIR/PYTHON_EXECUTABLE" ]; then
+        PYTHON_EXECUTABLE=$(cat "$CFG_ROOT_DIR/PYTHON_EXECUTABLE")
+    else
+        PYTHON_EXECUTABLE=python3
+    fi
 fi
 
 
-function setup {
-    # create a virtualenv on Python
-    mkdir -p $CONFIGURE_ROOT_DIR/tmp
-    wget -O $CONFIGURE_ROOT_DIR/tmp/virtualenv.pyz https://bootstrap.pypa.io/virtualenv.pyz
-    $PYTHON_EXE $CONFIGURE_ROOT_DIR/tmp/virtualenv.pyz $CONFIGURE_ROOT_DIR/tmp
-    source $CONFIGURE_ROOT_DIR/tmp/bin/activate
-    $CONFIGURE_ROOT_DIR/tmp/bin/pip install --upgrade pip virtualenv setuptools wheel
+################################
+cli_help() {
+    echo An initial configuration script
+    echo "  usage: ./configure [options]"
+    echo
+    echo The default is to configure for regular use. Use --dev for development.
+    echo
+    echo The options are:
+    echo " --clean: clean built and installed files and exit."
+    echo " --dev:   configure the environment for development."
+    echo " --help:  display this help message and exit."
+    echo
+    echo By default, the python interpreter version found in the path is used.
+    echo Alternatively, the PYTHON_EXECUTABLE environment variable can be set to
+    echo configure another Python executable interpreter to use. If this is not
+    echo set, a file named PYTHON_EXECUTABLE containing a single line with the
+    echo path of the Python executable to use will be checked last.
+    set +e
+    exit
 }
 
 
-setup
+clean() {
+    # Remove cleanable file and directories and files from the root dir.
+    echo "* Cleaning ..."
+    for cln in $CLEANABLE;
+        do rm -rf "${CFG_ROOT_DIR:?}/${cln:?}";
+    done
+    set +e
+    exit
+}
 
-$CONFIGURE_ROOT_DIR/tmp/bin/pip install -e .[testing]
 
-if [ -f "$CONFIGURE_ROOT_DIR/tmp/bin/activate" ]; then
-    source "$CONFIGURE_ROOT_DIR/tmp/bin/activate"
-fi
+create_virtualenv() {
+    # create a virtualenv for Python
+    # Note: we do not use the bundled Python 3 "venv" because its behavior and
+    # presence is not consistent across Linux distro and sometimes pip is not
+    # included either by default. The virtualenv.pyz app cures all these issues.
+
+    VENV_DIR="$1"
+    if [ ! -f "$CFG_BIN_DIR/python" ]; then
+
+        mkdir -p "$CFG_ROOT_DIR/$VENV_DIR"
+
+        if [ -f "$CFG_ROOT_DIR/etc/thirdparty/virtualenv.pyz" ]; then
+            VIRTUALENV_PYZ="$CFG_ROOT_DIR/etc/thirdparty/virtualenv.pyz"
+        else
+            VIRTUALENV_PYZ="$CFG_ROOT_DIR/$VENV_DIR/virtualenv.pyz"
+            wget -O "$VIRTUALENV_PYZ" "$VIRTUALENV_PYZ_URL"
+        fi
+
+        $PYTHON_EXECUTABLE "$VIRTUALENV_PYZ" \
+            --wheel embed --pip embed --setuptools embed \
+            --seeder pip \
+            --never-download \
+            --no-periodic-update \
+            --no-vcs-ignore \
+            $CFG_QUIET \
+            "$CFG_ROOT_DIR/$VENV_DIR"
+    fi
+}
+
+
+install_packages() {
+    # install requirements in virtualenv
+    # note: --no-build-isolation means that pip/wheel/setuptools will not
+    # be reinstalled a second time and reused from the virtualenv and this
+    # speeds up the installation.
+    # We always have the PEP517 build dependencies installed already.
+
+    "$CFG_BIN_DIR/pip" install \
+        --upgrade \
+        --no-build-isolation \
+        $CFG_QUIET \
+        $PIP_EXTRA_ARGS \
+        $1
+}
+
+
+################################
+# Main command line entry point
+CFG_DEV_MODE=0
+CFG_REQUIREMENTS=$REQUIREMENTS
+
+case "$CLI_ARGS" in
+    --help)    cli_help;;
+    --clean)   clean;;
+    --dev)     CFG_REQUIREMENTS="$DEV_REQUIREMENTS" && CFG_DEV_MODE=1;;
+esac
+
+create_virtualenv "$VIRTUALENV_DIR"
+install_packages "$CFG_REQUIREMENTS"
+. "$CFG_BIN_DIR/activate"
 
 set +e