Automate the creation of requirement.txt and fetch deps

etc/scripts/freeze_and_update_reqs.py

@@ -0,0 +1,133 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright nexB Inc. and others. All rights reserved.
+# http://nexb.com and https://github.com/nexB/scancode-toolkit/
+# The ScanCode software is licensed under the Apache License version 2.0.
+# Data generated with ScanCode require an acknowledgment.
+# ScanCode is a trademark of nexB Inc.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# When you publish or redistribute any data created with ScanCode or any ScanCode
+# derivative work, you must accompany this data with the following acknowledgment:
+#
+#  Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+#  OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+#  ScanCode should be considered or used as legal advice. Consult an Attorney
+#  for any legal advice.
+#  ScanCode is a free software code scanning tool from nexB Inc. and others.
+#  Visit https://github.com/nexB/scancode-toolkit/ for support and download.
+
+from __future__ import absolute_import
+from __future__ import print_function
+
+import argparse
+import fnmatch
+from commoncode.fileutils import resource_iter
+import os
+import subprocess
+import shutil
+import sys
+
+python_version = str(sys.version_info[0]) + str(sys.version_info[1])
+py_abi = "{0}cp{1}{0}".format("*", python_version)
+
+
+def generate_req_text(input_dir, output_file=False, package_name=False):
+    """
+    Generate a requirement.txt file of all dependencies present in thirdparty.
+    """
+    thirdparty = list(resource_iter(input_dir, with_dirs=False))
+    # FIXME this code is for py 3.6 and later we will update for all version
+    dependencies = [
+        files
+        for files in thirdparty
+        if fnmatch.fnmatchcase(files, "*py3*")
+        or fnmatch.fnmatchcase(files, py_abi)
+        or (
+            fnmatch.fnmatchcase(files, "*tar.gz*")
+            and not fnmatch.fnmatchcase(files, "*py2-ipaddress-3.4.1.tar.gz*")
+        )
+    ]
+    if not (os.path.isdir("required_deps")):
+        os.mkdir("required_deps")
+    for deps in dependencies:
+        shutil.copy(deps, "required_deps")
+    subprocess.run(
+        [
+            "pip-compile",
+            "--generate-hashes",
+            "--find-links",
+            "required_deps",
+            "--upgrade",
+            "--output-file",
+            output_file,
+            "--verbose",
+            "--upgrade-package",
+            "package_name",
+            "--pip-args",
+            "--no-index",
+        ]
+    )
+    shutil.rmtree("required_deps")
+
+
+def main_with_args(args: str) -> None:
+    parser = argparse.ArgumentParser(
+        description="""Creates a archive for specific OS and specific python.
+EXAMPLE:
+freeze_and_update_reqs.py \\
+  --deps_directory DEPS_DIRECTORY \\
+  --output OUTPUT \\
+  --upgrade_package PACKAGE_NAME \\
+""",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+
+    parser.add_argument(
+        "--deps_directory",
+        help="Required: Thirdparty Dependencies directory to be archived. ",
+        type=str,
+        required=True,
+    )
+
+    parser.add_argument(
+        "--output",
+        help="Output file name. Required if more than one input file is given. Will be derived from input file otherwise. ",
+        type=str,
+        default="requirements.txt",
+    )
+
+    parser.add_argument(
+        "--upgrade",
+        help="Upgrade all dependencies to new version. ",
+        action="store_true",
+    )
+
+    parser.add_argument(
+        "--upgrade_package",
+        help="Specify particular packages to upgrade. ",
+        type=str,
+        default=None,
+    )
+
+    args = parser.parse_args()
+
+    tpdir = args.deps_directory
+    output_file = args.output
+    package_name = args.upgrade_package
+    upgrade = args.upgrade or None
+    generate_req_text(tpdir, output_file, package_name)
+
+
+def main() -> None:
+    main_with_args(sys.argv[1:])
+
+
+if __name__ == "__main__":
+    main()

etc/scripts/scancode.py

@@ -0,0 +1,128 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright nexB Inc. and others. All rights reserved.
+# http://nexb.com and https://github.com/nexB/scancode-toolkit/
+# The ScanCode software is licensed under the Apache License version 2.0.
+# Data generated with ScanCode require an acknowledgment.
+# ScanCode is a trademark of nexB Inc.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# When you publish or redistribute any data created with ScanCode or any ScanCode
+# derivative work, you must accompany this data with the following acknowledgment:
+#
+#  Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+#  OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+#  ScanCode should be considered or used as legal advice. Consult an Attorney
+#  for any legal advice.
+#  ScanCode is a free software code scanning tool from nexB Inc. and others.
+#  Visit https://github.com/nexB/scancode-toolkit/ for support and download.
+
+from __future__ import absolute_import
+from __future__ import print_function
+
+import argparse
+import fnmatch
+from commoncode.fileutils import resource_iter
+import sys
+
+
+def search_package(package_name, target, version=False):
+    """
+    Search specific package in given directory with all corresponding files.
+    """
+
+    if version:
+        package_name = "*{}-{}*".format(package_name, version)
+    else:
+        package_name = "*{}*".format(package_name)
+    thirdparty = list(resource_iter(target, with_dirs=False))
+    dependency = [
+        files for files in thirdparty if fnmatch.fnmatchcase(files, package_name)
+    ]
+    if dependency:
+        whl = [
+            files for files in dependency if files.endswith(".whl")
+        ]  ## There are multiple version of same package So list of wheel will be considered.
+        sdist = [files for files in dependency if files.endswith(".tar.gz")]
+        about = [files for files in dependency if files.endswith(".ABOUT")]
+        notice = [files for files in dependency if files.endswith(".NOTICE")]
+        license = [files for files in dependency if files.endswith(".LICENSE")]
+        print(*whl, sep="\n")
+        print("\n")
+        if sdist:
+            print(*sdist, sep="\n")
+        else:
+            print("Corresponding sdist does not exits in target\n")
+        print("\n")
+        if about:
+            print(*about, sep="\n")
+        else:
+            print("Corresponding .ABOUT does not exits in target\n")
+        print("\n")
+        if license:
+            print(*licence, sep="\n")
+        else:
+            print("Corresponding .LICENSE does not exits in target\n")
+        print("\n")
+        if notice:
+            print(*notice, sep="\n")
+        else:
+            print("Corresponding .NOTICE does not exits in target\n")
+        print("\n")
+    else:
+        print("Specified package does not exist\n")
+
+
+def main_with_args(args: str) -> None:
+    parser = argparse.ArgumentParser(
+        description="""Fetch a specific package with version in given target like thirdparty by default.
+EXAMPLE:
+scancode.py \\
+  --fetch PACKAGE_NAME \\
+  --target TARGET_DIR \\
+  --version VERSION_OF_PACKAGE \\
+""",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+
+    parser.add_argument(
+        "--fetch",
+        help="Required: Specific Dependencies  to be fetched. ",
+        type=str,
+        required=True,
+    )
+
+    parser.add_argument(
+        "--target",
+        help=" a target directory where the built wheels and tarballs would be fetched. ",
+        type=str,
+        default="thirdparty",
+    )
+
+    parser.add_argument(
+        "--version",
+        help="Specify version of dependencies to be fetched. ",
+        type=str,
+        default=None,
+    )
+
+    args = parser.parse_args()
+
+    package_name = args.fetch
+    target = args.target
+    version = args.version
+    search_package(package_name, target, version)
+
+
+def main() -> None:
+    main_with_args(sys.argv[1:])
+
+
+if __name__ == "__main__":
+    main()

thirdparty/setuptools-41.2.0-py2.py3-none-any.whl.ABOUT

@@ -0,0 +1,15 @@
+about_resource: setuptools-41.2.0-py2.py3-none-any.whl
+name: setuptools
+version: 41.2.0
+download_url: https://files.pythonhosted.org/packages/b2/86/095d2f7829badc207c893dd4ac767e871f6cd547145df797ea26baea4e2e/setuptools-41.2.0-py2.py3-none-any.whl#sha256=4380abcf2a4ffd1a5ba22d687c6d690dce83b2b51c70e9c6d09f7e8c7e8040dc
+description: Easily download, build, install, upgrade, and uninstall Python packages
+homepage_url: https://pypi.python.org/pypi/setuptools
+license_expression: python AND mit
+copyright: Copyright (c) Python Packaging Authority
+attribute: yes
+track_changes: yes
+owner: Python Packaging Authority
+owner_url: https://www.pypa.io/en/latest/
+contact: distutils-sig@python.org
+checksum_md5: c339d06c65fe171f686b0ad76751f3bf
+checksum_sha1: fd41702d63ed12ebb789c6dc52f1944e0bec0f33