From 02c6debae11f9775e735a1840ef81425c174151d Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 24 Jun 2022 20:12:46 +0200 Subject: [PATCH 1/7] Add test protocol to validate resolution #6 Signed-off-by: Philippe Ombredanne --- docs/source/index.rst | 1 + docs/source/test-protocol.rst | 137 ++++++++++++++++++++++++++++++++++ 2 files changed, 138 insertions(+) create mode 100644 docs/source/test-protocol.rst diff --git a/docs/source/index.rst b/docs/source/index.rst index 8a3c599d..3cc78c9a 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -6,6 +6,7 @@ Welcome to nexb-skeleton's documentation! :caption: Contents: dependencies-design + test-protocol Indices and tables ================== diff --git a/docs/source/test-protocol.rst b/docs/source/test-protocol.rst new file mode 100644 index 00000000..f7fc2dce --- /dev/null +++ b/docs/source/test-protocol.rst @@ -0,0 +1,137 @@ +============================================================== + Validation tests for Python package dependency resolution +============================================================== + +In order to validate that the python-inspector dependency resolution is correct +here is a procedure. + + +Requirements +*************** + +You need to: + +- have access to some Python codebase to use as a test bed. +- have a working installation of python-inspector +- have a working installation of scancode-toolkit v31.x (inlucing RCs) or higher + + +Test protocol +*************** + +Step 1: Collect development code details +-------------------------------------------- + +In this step, you will collect the list of requirements files that exist +in the codebase.You can use scancode:: + + scancode --package --info --json-pp --processes 4 + +You then need to review the scanc results to assemble a list of of pip requirement files used in development. + +Or a simpler approach is to use the ``find`` command:: + + find -name "requirement*.txt" > + +The output is a list of pip requirement files used in the development codebase. + + + +Step 2: Build your code +---------------------------- + +In this step, you need to run the build of your codebase and obtain the set +of deployed binaries. Building requires both dependency resolution and installation +of the resolved packages. The set of installed packages is used to establish the +ground truth and the expected results. + +As an output, keep the Python version that is used and the directory(ies) where +the deployed code is built named further down as . + + +Step 3: Collect built code details +--------------------------------------- + +In this step, you will extract the deployed code and run a scan to +collect existing packages as resolved during the original build. + +For this we will use extractcode and scancode with these commands:: + + extractcode --shallow + scancode --package --info --json-pp --processes 4 + +You can adjust the number of processes up or down based on available CPU cores. + +The output is the file that contains all the detected +packages that were installed during the build. + + +Step 4: Resolve dependencies using development requirement files +-------------------------------------------------------------------- + +In this step, you will resolve the dependencies using python-inspector dad +command for each of the requirements files identified in Step 1 using the +Python version identified in Step 2. Run this command for each requirements +file, using each time a different output file name. We assume here Python +version 3.8 (note the absence of dot when passed as a command line option:: + + dad --python-version 38 --requirement --json + +The output is a list of JSON files with resolved packages for each of the +input requirements files. + + +Step 5: Collect expected resolved packages +---------------------------------------------- + +In this step, you need to collect the list of built packages "purl" found in +the top level "packages" attribute of the JSON scan output from Step 3. + +The output is a list of expected purl with a version. + + +Step 6: Collect actual resolved packages +---------------------------------------------- + +In this step, you need to collect the list of purl "package" attribute resolved +found in the top level "resolved_dependencies" attribute of the python-inspector +resolution output from Step 4. + +The output is a list of actual purl with a version. + + +Step 7: Compare expected with actual resolved packages +--------------------------------------------------------- + +In this step, you compare the output from Step 5 and Step 6 and validate the +correctness of python-inpector resolution. + +The output is a list of differences: + +- list 1: expected purls not present in the resolved purls. +- list 2: resolved purls not present in the expected purls. + + +(and if needed you can also collate the list of similar purls for reference). + + +If the build, build scan and resolution worked as expected, the list 1 and list 2 +should be empty. The differences need to be investigated. + +The possible causes could be: + +- extra actual resolved packages when some of the requirement files from Step 3 + also contains files for requirements that are not deployed (e.g. test, tools.) + +- missing actual resolved packages when the reference codebase from Step 1 uses + other build manifests beyond requirements.txt files. If this is the case, you + should carefully identify which other build manifests are used by reviewing + the development scan from Step 3. and report each manifest format as enhancement + request in the python-inspector issue tracker. + +- finally it can be a bug in python-inspector proper. Please report an issue + in the python-inspector issue tracker. Attach the list or JSON output(s) + from Step 1, Step 3 and Step 4. And the results of the review of Step 5, 6 + and 7. Alternatively you can share these files with python-inspector + maintainers if these are private. + From 8777200ca9099abe9272ec9bd4b1522dc94d52aa Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Fri, 24 Jun 2022 23:51:02 +0530 Subject: [PATCH 2/7] Fix failing tests Signed-off-by: Tushar Goel --- docs/source/test-protocol.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/source/test-protocol.rst b/docs/source/test-protocol.rst index f7fc2dce..9c741e35 100644 --- a/docs/source/test-protocol.rst +++ b/docs/source/test-protocol.rst @@ -27,7 +27,8 @@ in the codebase.You can use scancode:: scancode --package --info --json-pp --processes 4 -You then need to review the scanc results to assemble a list of of pip requirement files used in development. +You then need to review the scanc results to assemble a list +of pip requirement files used in development. Or a simpler approach is to use the ``find`` command:: @@ -43,7 +44,7 @@ Step 2: Build your code In this step, you need to run the build of your codebase and obtain the set of deployed binaries. Building requires both dependency resolution and installation of the resolved packages. The set of installed packages is used to establish the -ground truth and the expected results. +ground truth and the expected results. As an output, keep the Python version that is used and the directory(ies) where the deployed code is built named further down as . @@ -63,7 +64,7 @@ For this we will use extractcode and scancode with these commands:: You can adjust the number of processes up or down based on available CPU cores. The output is the file that contains all the detected -packages that were installed during the build. +packages that were installed during the build. Step 4: Resolve dependencies using development requirement files @@ -129,7 +130,7 @@ The possible causes could be: the development scan from Step 3. and report each manifest format as enhancement request in the python-inspector issue tracker. -- finally it can be a bug in python-inspector proper. Please report an issue +- finally it can be a bug in python-inspector proper. Please report an issue in the python-inspector issue tracker. Attach the list or JSON output(s) from Step 1, Step 3 and Step 4. And the results of the review of Step 5, 6 and 7. Alternatively you can share these files with python-inspector From cbd957ee4f8c189400eb9bbec14e66e7d2bee407 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 25 Jun 2022 00:15:20 +0200 Subject: [PATCH 3/7] Use better doc page titles Signed-off-by: Philippe Ombredanne --- docs/source/dependencies-design.rst | 6 +++--- docs/source/index.rst | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/source/dependencies-design.rst b/docs/source/dependencies-design.rst index 952f3817..113fa1c1 100644 --- a/docs/source/dependencies-design.rst +++ b/docs/source/dependencies-design.rst @@ -1,6 +1,6 @@ -========================================= - Resolving Python package dependencies -========================================= +==================================================== + Python package dependencies resolver design +==================================================== This is a design to create a new command line tool to resolve Python diff --git a/docs/source/index.rst b/docs/source/index.rst index 3cc78c9a..c5ac78f0 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,5 +1,5 @@ -Welcome to nexb-skeleton's documentation! -========================================= +Welcome to python-inspector's documentation! +================================================ .. toctree:: :maxdepth: 2 From 07c3172a7effdf54d3096f4f2c436b6a3380971e Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Thu, 30 Jun 2022 22:56:17 +0530 Subject: [PATCH 4/7] Add netrc file option in doc Signed-off-by: Tushar Goel --- docs/source/test-protocol.rst | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/source/test-protocol.rst b/docs/source/test-protocol.rst index 48b883cf..b7c07f6b 100644 --- a/docs/source/test-protocol.rst +++ b/docs/source/test-protocol.rst @@ -80,7 +80,8 @@ file, using each time a different output file name. We assume here Python version 3.8 (note the absence of dot when passed as a command line option:: dad --python-version 38 --requirement \ - --json + --json \ + --netrc The output is a list of JSON files with resolved packages for each of the input requirements files. @@ -92,17 +93,17 @@ Step 5: Collect expected resolved packages In this step, you need to collect the list of built packages "purl" found in the top level "packages" attribute of the JSON scan output from Step 3. -The output is a list of expected purl with a version. +The output is a list of expected purls with a version. Step 6: Collect actual resolved packages ---------------------------------------------- -In this step, you need to collect the list of purl "package" attribute resolved +In this step, you need to collect the list of purls "package" attribute resolved found in the top level "resolved_dependencies" attribute of the python-inspector resolution output from Step 4. -The output is a list of actual purl with a version. +The output is a list of actual purls with a version. Step 7: Compare expected with actual resolved packages @@ -153,7 +154,7 @@ And the reference Python version is 3.8:: mkdir -p ~/tmp/pyinsp-example/ cd ~/tmp/pyinsp-example/ - git https://github.com/tjcsl/ion + git clone https://github.com/tjcsl/ion Another example could be https://github.com/digitalocean/sample-django @@ -256,7 +257,7 @@ Step 5: Collect expected resolved packages TODO: explain how to do this in details -The output is a list of expected purl with a version. +The output is a list of expected purls with a version. Step 6: Collect actual resolved packages @@ -264,7 +265,7 @@ Step 6: Collect actual resolved packages TODO: explain how to do this in details -The output is a list of actual purl with a version. +The output is a list of actual purls with a version. Step 7: Compare expected with actual resolved packages From 056489a126a5742e55f6ce1aabfdf4a812743552 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Sun, 3 Jul 2022 03:02:38 +0530 Subject: [PATCH 5/7] Complete end to end example Signed-off-by: Tushar Goel --- docs/source/test-protocol.rst | 40 +++++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/docs/source/test-protocol.rst b/docs/source/test-protocol.rst index b7c07f6b..6a2be9b1 100644 --- a/docs/source/test-protocol.rst +++ b/docs/source/test-protocol.rst @@ -255,7 +255,21 @@ The output files are:: Step 5: Collect expected resolved packages ---------------------------------------------- -TODO: explain how to do this in details +Run this python script to generate text file with expected purls + +:: + + import json + with open("~/tmp/pyinsp-example/codebase/output/deployed-scan-output.json") as f: + scancode_data = json.load(f) + scancode_purls = [] + for package in scancode_data["packages"]: + if package["purl"] not in scancode_purls: + scancode_purls.append(package["purl"]) + scancode_purls = sorted(scancode_purls) + with open("~/tmp/pyinsp-example/codebase/output/scan.txt", "w") as f: + f.writelines("\n".join(scancode_purls)) + The output is a list of expected purls with a version. @@ -263,7 +277,25 @@ The output is a list of expected purls with a version. Step 6: Collect actual resolved packages ---------------------------------------------- -TODO: explain how to do this in details +Run this python script to generate text file with actual purls + +:: + + import json + py_insp_purls = [] + for json_file in [ + "~/tmp/pyinsp-example/output/resolved-rtd-requirements.txt.json", + "~/tmp/pyinsp-example/output/resolved-requirements.txt.json", + ]: + with open(json_file) as f: + py_insp_data = json.load(f) + for package in py_insp_data["packages"]: + if package["purl"] not in py_insp_purls: + py_insp_purls.append(package["purl"]) + py_insp_purls = sorted(py_insp_purls) + with open("~/tmp/pyinsp-example/codebase/output/py-insp.txt", "w") as f: + f.writelines("\n".join(py_insp_purls)) + The output is a list of actual purls with a version. @@ -271,6 +303,6 @@ The output is a list of actual purls with a version. Step 7: Compare expected with actual resolved packages --------------------------------------------------------- -TODO: explain how to do this in details - +We run a sdiff command:: + sdiff ~/tmp/pyinsp-example/codebase/output/py-insp.txt ~/tmp/pyinsp-example/codebase/output/scan.txt From fe3afa3963fbe45a72c8af4091ba4725be960feb Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Wed, 6 Jul 2022 21:00:25 +0530 Subject: [PATCH 6/7] Bump dependencies and fix documentation Signed-off-by: Tushar Goel --- docs/source/test-protocol.rst | 6 +++--- tests/test_resolution.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/source/test-protocol.rst b/docs/source/test-protocol.rst index 6a2be9b1..b7f64e6c 100644 --- a/docs/source/test-protocol.rst +++ b/docs/source/test-protocol.rst @@ -289,9 +289,9 @@ Run this python script to generate text file with actual purls ]: with open(json_file) as f: py_insp_data = json.load(f) - for package in py_insp_data["packages"]: - if package["purl"] not in py_insp_purls: - py_insp_purls.append(package["purl"]) + for package in py_insp_data["resolved_dependencies"]: + if package["package"] not in py_insp_purls: + py_insp_purls.append(package["package"]) py_insp_purls = sorted(py_insp_purls) with open("~/tmp/pyinsp-example/codebase/output/py-insp.txt", "w") as f: f.writelines("\n".join(py_insp_purls)) diff --git a/tests/test_resolution.py b/tests/test_resolution.py index c8e8b94e..1832947f 100644 --- a/tests/test_resolution.py +++ b/tests/test_resolution.py @@ -82,11 +82,11 @@ def test_get_resolved_dependencies_with_flask_and_python_36(): assert as_list == [ "pkg:pypi/click@8.1.3", "pkg:pypi/flask@2.1.2", - "pkg:pypi/importlib-metadata@4.11.4", + "pkg:pypi/importlib-metadata@4.12.0", "pkg:pypi/itsdangerous@2.1.2", "pkg:pypi/jinja2@3.1.2", "pkg:pypi/markupsafe@2.0.1", - "pkg:pypi/typing-extensions@4.2.0", + "pkg:pypi/typing-extensions@4.3.0", "pkg:pypi/werkzeug@2.1.2", "pkg:pypi/zipp@3.8.0", ] @@ -107,7 +107,7 @@ def test_get_resolved_dependencies_with_tilde_requirement_using_json_api(): assert as_list == [ "pkg:pypi/click@8.1.3", "pkg:pypi/flask@2.1.2", - "pkg:pypi/importlib-metadata@4.11.4", + "pkg:pypi/importlib-metadata@4.12.0", "pkg:pypi/itsdangerous@2.1.2", "pkg:pypi/jinja2@3.1.2", "pkg:pypi/markupsafe@2.1.1", From f6a84b295645e248ec964c42c4252a85449142b2 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Thu, 7 Jul 2022 21:53:40 +0530 Subject: [PATCH 7/7] Replace writelines with write Signed-off-by: Tushar Goel --- docs/source/test-protocol.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/test-protocol.rst b/docs/source/test-protocol.rst index b7f64e6c..d5e918a8 100644 --- a/docs/source/test-protocol.rst +++ b/docs/source/test-protocol.rst @@ -268,7 +268,7 @@ Run this python script to generate text file with expected purls scancode_purls.append(package["purl"]) scancode_purls = sorted(scancode_purls) with open("~/tmp/pyinsp-example/codebase/output/scan.txt", "w") as f: - f.writelines("\n".join(scancode_purls)) + f.write("\n".join(scancode_purls)) The output is a list of expected purls with a version. @@ -294,7 +294,7 @@ Run this python script to generate text file with actual purls py_insp_purls.append(package["package"]) py_insp_purls = sorted(py_insp_purls) with open("~/tmp/pyinsp-example/codebase/output/py-insp.txt", "w") as f: - f.writelines("\n".join(py_insp_purls)) + f.write("\n".join(py_insp_purls)) The output is a list of actual purls with a version.