Automate requirements.txt creation

etc/scripts/freeze_and_update_reqs.py

@@ -0,0 +1,135 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2019 nexB Inc. and others. All rights reserved.
+# http://nexb.com and https://github.com/nexB/scancode-toolkit/
+# The ScanCode software is licensed under the Apache License version 2.0.
+# Data generated with ScanCode require an acknowledgment.
+# ScanCode is a trademark of nexB Inc.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# When you publish or redistribute any data created with ScanCode or any ScanCode
+# derivative work, you must accompany this data with the following acknowledgment:
+#
+#  Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+#  OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+#  ScanCode should be considered or used as legal advice. Consult an Attorney
+#  for any legal advice.
+#  ScanCode is a free software code scanning tool from nexB Inc. and others.
+#  Visit https://github.com/nexB/scancode-toolkit/ for support and download.
+
+
+from __future__ import absolute_import
+from __future__ import print_function
+
+
+import fnmatch
+import hashlib
+import io
+import os
+import re
+import subprocess
+
+
+def is_package_exist_with_hashes(package_name,hash_of_file):
+    """ Check if any line in the file contains package with same hashes """
+    # Open the file in read only mode
+    line_with_hash = "{} \\\n    --hash=sha256:{}".format(package_name,hash_of_file)
+    with open('requirements.txt') as file:
+        # Read all lines in the file one by one
+        for line in file:
+            # For each line, check if line contains the package with same hashes
+            if line_with_hash in line:
+                return True
+    return False
+
+def is_package_exist_without_hashes(package_name):
+    """ Check if any line in the file contains package without same hash """
+    # Open the file in read only mode
+    line_with_hash = "{} \\\n".format(package_name) 
+    with open('requirements.txt') as file:
+        # Read all lines in the file one by one
+        for line in file:
+            # For each line, check if line contains the string
+            if line_with_hash in line:
+                return True
+    return False
+
+
+def add_package(package_name,hash_of_file):
+    with open('requirements.txt', 'a') as file:
+    # Append package with hashes at the end of file
+        line = "{} \\\n    --hash=sha256:{}\n".format(package_name,hash_of_file)
+        file.write(line)
+
+def append_requirement_file(package_name,hash_of_file):
+    if is_package_exist_with_hashes(package_name,hash_of_file):
+        return
+    else:
+        inputfile = open('requirements.txt').readlines()
+        write_file = open('requirements.txt','w')
+        for line in inputfile:
+            write_file.write(line)
+            current_line = "{} \\\n".format(package_name)
+            if current_line in line:
+                new_line = "    --hash=sha256:{} \\".format(hash_of_file)        
+                write_file.write(new_line + "\n") 
+        write_file.close()
+
+
+def read_chunks(file, size=io.DEFAULT_BUFFER_SIZE):
+    """Yield pieces of data from a file-like object until EOF."""
+    while True:
+        chunk = file.read(size)
+        if not chunk:
+            break
+        yield chunk
+
+def hash_of_file(path):
+    # type: (str) -> str
+    """Return the hash digest of a file."""
+    with open(path, 'rb') as archive:
+        hash = hashlib.new('sha256')
+        for chunk in read_chunks(archive):
+            hash.update(chunk)
+    return hash.hexdigest()
+
+def main():
+    os.chdir(os.pardir) #go to etc
+    os.chdir(os.pardir)  # go to scancode-toolkit
+    cwd = os.getcwd()
+    print("Current working directory:", cwd) 
+    fp= open('requirements.txt','w') #Create empty file because it must exits before opening the file in read mode.
+    for subdir, dirs, files in os.walk(cwd+"/thirdparty"):
+        for filename in files:
+            filepath = subdir + os.sep + filename
+            if filepath.endswith(".whl") and (fnmatch.fnmatchcase(filename, "*py3*") or fnmatch.fnmatchcase(filename, "*cp36*")):
+                name = filename.split('-')[0]
+                version = filename.split('-')[1]
+                package_name = name + "==" + version
+                hs= hash_of_file(filepath)
+                if is_package_exist_without_hashes(package_name):
+                    append_requirement_file(package_name,hs)
+                else: add_package(package_name,hs)
+
+            elif filepath.endswith(".gz") and not filename.startswith("py2-"):
+                name = filename.split('-')[0]
+                version = re.findall("\d+\.\d+\.+\d|\d+\.\d+\d*|\d+\d*",filename)
+                if len(version)>=2:
+                    version[0]= version[0]+version[1]
+                if re.search(r'\d', name):
+                    version[0]= version[1]
+                hs= hash_of_file(filepath)
+                package_name = name + "==" + version[0]
+                if is_package_exist_without_hashes(package_name):
+                    append_requirement_file(package_name,hs)       
+                else: add_package(package_name,hs)
+    fp.close()
+
+if __name__ == '__main__':
+    main()

github_release.py

@@ -0,0 +1,56 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2019 nexB Inc. and others. All rights reserved.
+# http://nexb.com and https://github.com/nexB/scancode-toolkit/
+# The ScanCode software is licensed under the Apache License version 2.0.
+# Data generated with ScanCode require an acknowledgment.
+# ScanCode is a trademark of nexB Inc.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# When you publish or redistribute any data created with ScanCode or any ScanCode
+# derivative work, you must accompany this data with the following acknowledgment:
+#
+#  Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+#  OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+#  ScanCode should be considered or used as legal advice. Consult an Attorney
+#  for any legal advice.
+#  ScanCode is a free software code scanning tool from nexB Inc. and others.
+#  Visit https://github.com/nexB/scancode-toolkit/ for support and download.
+
+
+from __future__ import absolute_import
+from __future__ import print_function
+
+import fnmatch
+import os
+import subprocess
+
+
+def main():
+    cwd = os.getcwd()
+    print("Current working directory:", cwd) 
+    subprocess.run(["pip3","install","github-release-retry"])
+    token = input ("Enter GITHUB_TOKEN :")
+
+    os.environ ["GITHUB_TOKEN"] = token
+
+    tag =  input ("Enter tag_name :")
+    repo = input ("Enter name of repository name :")
+    body = input ("Enter body string :")
+    user = input ("Enter Github user name :")
+    limit = input ("Enter retry limit :")
+    for subdir, dirs, files in os.walk(cwd+"/thirdparty"):
+        for filename in files:
+            if fnmatch.fnmatchcase(filename, "*py3*") or fnmatch.fnmatchcase(filename, "*cp36*") or (fnmatch.fnmatchcase(filename, "*tar.gz*") and not fnmatch.fnmatchcase(filename, "*py2*")):
+                filepath = subdir + os.sep + filename
+                subprocess.run(["python3","-m","github_release_retry.github_release_retry","--user",user,"--repo",repo,"--tag_name",tag,"--body_string",body,"--retry_limit",limit,filepath])
+
+if __name__ == '__main__':
+    main()
+