Disable trace of configure.py

Signed-off-by: Abhishek Kumar <abhishek.kasyap09@gmail.com>

Author: Abhishek-Dev09

etc/configure.py

@@ -131,7 +131,7 @@
     bin_dir_name = 'bin'
 
 # set to True to trace command line executaion
-TRACE = True
+TRACE = False
 
 
 def call(cmd, root_dir):

setup.py

@@ -119,33 +119,39 @@ def read(*names, **kwargs):
         'License :: OSI Approved :: Apache Software License',
         'License :: CC0 1.0 Universal (CC0 1.0) Public Domain Dedication',
         'Programming Language :: Python',
+        'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3.6',
         'Topic :: Utilities',
     ],
     keywords=[
         'open source', 'scan', 'license', 'package', 'dependency',
         'copyright', 'filetype', 'author', 'extract', 'licensing',
     ],
-    python_requires='!=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, <4',
+    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, <4',
     install_requires=[
         # Hack to support pip 8 (for those poor sods forced to use ubuntu 16.04's system pip)
         # See https://github.com/nexB/scancode-toolkit/issues/1463
+        'more_itertools <  6.0.0; python_version == "2.7"',
         # end hack
 
         # cluecode
         # Some nltk version ranges are buggy
         'nltk >= 3.2, < 4.0',
+        'py2_ipaddress >= 2.0, <3.5; python_version<"3"',
         'urlpy',
         'publicsuffix2',
         'fingerprints >= 0.6.0, < 1.0.0',
 
         # extractcode
         'patch >= 1.15, < 1.20 ',
         # to work around bug http://bugs.python.org/issue19839
+        # on multistream bzip2 files: this can removed in Python 3.
+        'bz2file >= 0.98; python_version<"3"',
         'extractcode_libarchive',
         'extractcode_7z',
 
         # commoncode
+        'backports.os == 0.1.1; python_version<"3"',
         'future >= 0.16.0',
         'text_unidecode >= 1.0, < 2.0',
         'saneyaml',
@@ -186,7 +192,8 @@ def read(*names, **kwargs):
         'dparse >= 0.4.1',
 
         # used to fix mojibake in Windows PE
-        'ftfy>=  5.0.0',
+        'ftfy <  5.0.0; python_version == "2.7"',
+        'ftfy>=  5.0.0; python_version > "3"',
 
         # scancode
         'click >= 6.0.0, < 7.0.0',

src/commoncode/codec.py

@@ -30,33 +30,75 @@
 from base64 import standard_b64decode as stddecode
 from base64 import urlsafe_b64encode as b64encode
 
+from commoncode.system import py2
+
+
 """
 Numbers to bytes or strings and URLs coder/decoders.
 """
 
-# noop
-c2i = lambda c: c
-i2c = lambda i: bytes([i])
 
+if py2:
+    c2i = ord
+    i2c = chr
+else:
+    # noop
+    c2i = lambda c: c
+    i2c = lambda i: bytes([i])
 
-def num_to_bin(num):
-    """
-    Convert a `num` integer or long to a binary string byte-ordered such that
-    the least significant bytes are at the beginning of the string (aka. big
-    endian).
-    """
-    # Zero is not encoded but returned as an empty value
-    if num == 0:
-        return b'\x00'
 
-    return num.to_bytes((num.bit_length() + 7) // 8, 'big')
+if py2:
+    def num_to_bin(num):
+        """
+        Convert a `num` integer or long to a binary string byte-ordered such
+        that the least significant bytes are at the beginning of the string
+        (aka. big endian).
+        """
+        # Zero is not encoded but returned as an empty value
+        if num == 0:
+            return b'\x00'
 
+        binstr = []
+        while num > 0:
+            # add the least significant byte value
+            binstr.append(i2c(num & 0xFF))
+            # shift the next byte to least significant and repeat
+            num = num >> 8
 
-def bin_to_num(binstr):
-    """
-    Convert a big endian byte-ordered binary string to an integer or long.
-    """
-    return int.from_bytes(binstr, byteorder='big', signed=False)
+        # reverse the list now such that the most significant
+        # byte is at the start of this string to speed decoding
+        return b''.join(reversed(binstr))
+
+
+    def bin_to_num(binstr):
+        """
+        Convert a big endian byte-ordered binary string to an integer or long.
+        """
+        num = 0
+        for charac in binstr:
+            # the most significant byte is a the start of the string so we multiply
+            # that value by 256 (e.g. <<8) and add the value of the current byte,
+            # then move to next byte in the string and repeat
+            num = (num << 8) + c2i(charac)
+        return num
+else:
+    def num_to_bin(num):
+        """
+        Convert a `num` integer or long to a binary string byte-ordered such that
+        the least significant bytes are at the beginning of the string (aka. big
+        endian).
+        """
+        # Zero is not encoded but returned as an empty value
+        if num == 0:
+            return b'\x00'
+
+        return num.to_bytes((num.bit_length() + 7) // 8, 'big')
+
+    def bin_to_num(binstr):
+        """
+        Convert a big endian byte-ordered binary string to an integer or long.
+        """
+        return int.from_bytes(binstr, byteorder='big', signed=False)
 
 
 def urlsafe_b64encode(s):

src/commoncode/command.py

@@ -46,6 +46,8 @@
 from commoncode.system import on_linux
 from commoncode.system import on_posix
 from commoncode.system import on_windows
+from commoncode.system import py2
+from commoncode.system import py3
 from commoncode import text
 
 
@@ -79,9 +81,14 @@
 curr_dir = path.dirname(path.dirname(path.abspath(__file__)))
 
 
-PATH_ENV_VAR = 'PATH'
-LD_LIBRARY_PATH = 'LD_LIBRARY_PATH'
-DYLD_LIBRARY_PATH = 'DYLD_LIBRARY_PATH'
+if py2:
+    PATH_ENV_VAR = b'PATH'
+    LD_LIBRARY_PATH = b'LD_LIBRARY_PATH'
+    DYLD_LIBRARY_PATH = b'DYLD_LIBRARY_PATH'
+else:
+    PATH_ENV_VAR = 'PATH'
+    LD_LIBRARY_PATH = 'LD_LIBRARY_PATH'
+    DYLD_LIBRARY_PATH = 'DYLD_LIBRARY_PATH'
 
 
 def execute2(cmd_loc, args, lib_dir=None, cwd=None, env=None, to_files=False, log=TRACE):
@@ -107,8 +114,12 @@ def execute2(cmd_loc, args, lib_dir=None, cwd=None, env=None, to_files=False, lo
     # temp files for stderr and stdout
     tmp_dir = get_temp_dir(prefix='cmd-')
 
-    stdout = 'stdout'
-    stderr = 'stderr'
+    if on_linux and py2:
+        stdout = b'stdout'
+        stderr = b'stderr'
+    else:
+        stdout = 'stdout'
+        stderr = 'stderr'
 
     sop = path.join(tmp_dir, stdout)
     sep = path.join(tmp_dir, stderr)
@@ -127,7 +138,10 @@ def execute2(cmd_loc, args, lib_dir=None, cwd=None, env=None, to_files=False, lo
     proc = None
     rc = 100
 
-    okwargs = dict(mode='w', encoding='utf-8')
+    if py2:
+        okwargs = dict(mode='wb')
+    if py3:
+        okwargs = dict(mode='w', encoding='utf-8')
 
     try:
         with io.open(sop, **okwargs) as stdout, io.open(sep, **okwargs) as stderr:
@@ -181,7 +195,13 @@ def get_env(base_vars=None, lib_dir=None):
         env_vars.update(
             {DYLD_LIBRARY_PATH: update_path_var(dyld_lib_path, new_path)})
 
-    env_vars = {text.as_unicode(k): text.as_unicode(v) for k, v in env_vars.items()}
+    if py2:
+        # ensure that we use bytes on py2 and unicode on py3
+        def to_bytes(s):
+            return s if isinstance(s, bytes) else s.encode('utf-8')
+        env_vars = {to_bytes(k): to_bytes(v) for k, v in env_vars.items()}
+    else:
+        env_vars = {text.as_unicode(k): text.as_unicode(v) for k, v in env_vars.items()}
 
     return env_vars
 
@@ -228,8 +248,15 @@ def load_shared_library(dll_path, lib_dir):
     if lib_dir and not path.exists(lib_dir):
         raise ImportError('Shared library "lib_dir" does not exists: %(lib_dir)r' % locals())
 
-    if not isinstance(dll_path, compat.unicode):
-        dll_path = fsdecode(dll_path)
+    if on_linux and py2:
+        # bytes only there ...
+        if not isinstance(dll_path, bytes):
+            dll_path = fsencode(dll_path)
+
+    else:
+        # ... unicode everywhere else
+        if not isinstance(dll_path, compat.unicode):
+            dll_path = fsdecode(dll_path)
 
     try:
         with pushd(lib_dir):
@@ -277,10 +304,16 @@ def update_path_var(existing_path_var, new_path, pathsep=PATH_ENV_SEP):
 
     # ensure we use unicode or bytes depending on OSes
     # TODO: deal also with Python versions
-    # ... and unicode otherwise
-    existing_path_var = fsdecode(existing_path_var)
-    new_path = fsdecode(new_path)
-    pathsep = fsdecode(pathsep)
+    if on_linux and py2:
+        # bytes ...
+        existing_path_var = fsencode(existing_path_var)
+        new_path = fsencode(new_path)
+        pathsep = fsencode(pathsep)
+    else:
+        # ... and unicode otherwise
+        existing_path_var = fsdecode(existing_path_var)
+        new_path = fsdecode(new_path)
+        pathsep = fsdecode(pathsep)
 
     path_elements = existing_path_var.split(pathsep)
 
@@ -296,9 +329,14 @@ def update_path_var(existing_path_var, new_path, pathsep=PATH_ENV_SEP):
         # new path is already in PATH, change nothing
         updated_path_var = existing_path_var
 
-    # ... else use unicode
-    if not isinstance(updated_path_var, compat.unicode):
-        updated_path_var = fsdecode(updated_path_var)
+    if py2:
+        # always use bytes for env vars...
+        if isinstance(updated_path_var, compat.unicode):
+            updated_path_var = fsencode(updated_path_var)
+    else:
+        # ... else use unicode
+        if not isinstance(updated_path_var, compat.unicode):
+            updated_path_var = fsdecode(updated_path_var)
 
     # at this stage new_path_env is unicode on all OSes on Py3
     # and on Py2 it is bytes on Linux and unicode elsewhere

src/commoncode/fileset.py

@@ -32,6 +32,7 @@
 from commoncode import fileutils
 from commoncode import paths
 from commoncode.system import on_linux
+from commoncode.system import py2
 
 
 TRACE = False
@@ -44,8 +45,8 @@
     logger.setLevel(logging.DEBUG)
 
 
-POSIX_PATH_SEP = u'/'
-EMPTY_STRING = u''
+POSIX_PATH_SEP = b'/' if on_linux and py2 else u'/'
+EMPTY_STRING = b'' if on_linux and py2 else u''
 
 
 """
@@ -186,7 +187,7 @@ def load(location):
     fn = os.path.abspath(os.path.normpath(os.path.expanduser(location)))
     msg = ('File %(location)s does not exist or not a file.') % locals()
     assert (os.path.exists(fn) and os.path.isfile(fn)), msg
-    mode = 'r'
+    mode = 'rb' if on_linux and py2 else 'r'
     with open(fn, mode) as f:
         return [l.strip() for l in f if l and l.strip()]