refactor(pypi): add whl_deps_library repo rule (#3960)

Summary:
- Add a new repo rule to just read metadata.json
- Add integration tests for the repository rules in `whl_library.bzl`
  file.
- Make some of the arguments optional in the BUILD.bazel code
  generation.

No changelog, because the rule is not yet exposed to the user in any
way.

Split out of #3856
Work towards #2948
Fixes #3071

---------

Co-authored-by: Richard Levasseur <richardlev@gmail.com>

Author: aignas

.bazelrc.deleted_packages

@@ -44,6 +44,7 @@ common --deleted_packages=tests/integration/toolchain_target_settings
 common --deleted_packages=tests/integration/unified_pypi
 common --deleted_packages=tests/integration/uv_lock
 common --deleted_packages=tests/integration/validate_test_main
+common --deleted_packages=tests/integration/whl_library
 common --deleted_packages=tests/modules/another_module
 common --deleted_packages=tests/modules/other
 common --deleted_packages=tests/modules/other/nspkg_delta

python/private/pypi/generate_whl_library_build_bazel.bzl

@@ -25,6 +25,7 @@ _RENDER = {
     "extras": render.list,
     "group_deps": render.list,
     "include": str,
+    "repo": lambda maybe_label: repr(str(maybe_label)),
     "requires_dist": render.list,
     "srcs_exclude": render.list,
     "tags": render.list,
@@ -38,19 +39,22 @@ _TEMPLATE = """\
 
 package(default_visibility = ["//visibility:public"])
 
+{fn}(
+{kwargs}
+)
+"""
+
+_PURL = """\
 package_metadata(
     name = "package_metadata",
     purl = {purl},
     visibility = ["//:__subpackages__"],
 )
-
-{fn}(
-{kwargs}
-)
 """
 
 def generate_whl_library_build_bazel(
         *,
+        metadata_version,
         annotation = None,
         config_load,
         purl = None,
@@ -59,6 +63,7 @@ def generate_whl_library_build_bazel(
     """Generate a BUILD file for an unzipped Wheel
 
     Args:
+        metadata_version: The version to use for tag generation.
         annotation: The annotation for the build file.
         config_load: {type}`str` The location from where to load the config.
         purl: The purl.
@@ -74,14 +79,26 @@ def generate_whl_library_build_bazel(
         """load("@package_metadata//rules:package_metadata.bzl", "package_metadata")""",
     ]
 
-    fn = "whl_library_targets_from_requires"
+    if kwargs.get("repo"):
+        fn = "whl_library_deps_targets"
+    else:
+        fn = "whl_library_targets"
+
+    tags = [
+        "pypi_name={}".format(kwargs.get("metadata_name")),
+        "pypi_version={}".format(metadata_version),
+    ]
+    kwargs["tags"] = tags
+
     if not requires_dist:
         # no deps, we can leave the extra loads out
         pass
-    else:
+    elif config_load:
         loads.append("""load("{}", "{}")""".format(config_load, "packages"))
         kwargs["include"] = "packages"
         kwargs["requires_dist"] = requires_dist
+    else:
+        kwargs["requires_dist"] = requires_dist
 
     loads.extend([
         """load("@rules_python//python/private/pypi:whl_library_targets.bzl", "{}")""".format(fn),
@@ -106,9 +123,8 @@ def generate_whl_library_build_bazel(
                     "{} = {},".format(k, _RENDER.get(k, repr)(v))
                     for k, v in sorted(kwargs.items())
                 ])),
-                purl = repr(purl),
             ),
-        ] + additional_content,
+        ] + ([_PURL.format(purl = repr(purl))] if purl else []) + additional_content,
     )
 
     # NOTE: Ensure that we terminate with a new line

python/private/pypi/pep508_deps.bzl

@@ -44,6 +44,12 @@ def deps(
         * deps_select: {type}`dict[str, list[str]]` dependencies to include on particular
               subset of target platforms.
     """
+    if not requires_dist:
+        return struct(
+            deps = [],
+            deps_select = {},
+        )
+
     reqs = sorted(
         [requirement(r) for r in requires_dist],
         key = lambda x: "{}:{}:".format(x.name, sorted(x.extras), x.marker),

python/private/pypi/whl_library.bzl

@@ -332,6 +332,12 @@ def _whl_extract(rctx, *, whl_path, logger, sdist_filename = None):
         read_fn = rctx.read,
         logger = logger,
     )
+    rctx.file("metadata.json", json.encode_indent({
+        "name": metadata.name,
+        "provides_extra": metadata.provides_extra,
+        "requires_dist": metadata.requires_dist,
+        "version": metadata.version,
+    }))
     namespace_package_files = pypi_repo_utils.find_namespace_package_files(rctx, install_dir_path)
 
     entry_points = _get_entry_points(rctx, install_dir_path, metadata)
@@ -429,6 +435,8 @@ def _whl_archive_impl(rctx):
             whl_path = rctx.path(filename)
         else:
             fail("Only wheels are supported")
+    else:
+        fail("Either 'whl_file' or 'urls' and 'filename' needs to be specified")
 
     return _whl_extract(rctx, whl_path = whl_path, logger = logger)
 
@@ -571,9 +579,6 @@ For example if your whl depends on `numpy` and your Python package repo is named
     "index_url": attr.string(
         doc = "The index_url that the package will be downloaded from.",
     ),
-    "repo": attr.string(
-        doc = "Pointer to parent repo name. Used to make these rules rerun if the parent repo changes.",
-    ),
     "repo_prefix": attr.string(
         doc = """
 Prefix for the generated packages will be of the form `@<prefix><sanitized-package-name>//...`
@@ -676,7 +681,6 @@ whl_archive = repository_rule(
             "group_deps",
             "group_name",
             "index_url",
-            "repo",
             "repo_prefix",
             "requirement",
             "sha256",
@@ -702,7 +706,74 @@ Does not depend on any python.
     ],
 )
 
-def whl_library(name, **kwargs):
+def _whl_deps_library_impl(rctx):
+    logger = repo_utils.logger(rctx)
+
+    if rctx.attr.metadata_file and rctx.attr.metadata:
+        logger.fail("Only one of 'metadata_file' and 'metadata' can be specified")
+        return
+    if not (rctx.attr.metadata_file or rctx.attr.metadata):
+        logger.fail("At least one of 'metadata_file' and 'metadata' must be specified")
+        return
+
+    if rctx.attr.metadata_file:
+        metadata_contents = rctx.read(rctx.attr.metadata_file)
+    else:
+        metadata_contents = rctx.attr.metadata
+
+    metadata = struct(**json.decode(metadata_contents))
+
+    build_file_contents = generate_whl_library_build_bazel(
+        dep_template = rctx.attr.dep_template or "@{}{{name}}//:{{target}}".format(
+            rctx.attr.repo_prefix,
+        ),
+        config_load = rctx.attr.config_load,
+        metadata_name = metadata.name,
+        metadata_version = metadata.version,
+        requires_dist = metadata.requires_dist,
+        group_deps = rctx.attr.group_deps,
+        group_name = rctx.attr.group_name,
+        repo = rctx.attr.repo or (
+            str(rctx.attr.metadata_file) if rctx.attr.metadata_file else None
+        ),
+        extras = requirement(rctx.attr.requirement).extras,
+    )
+    rctx.file("BUILD.bazel", build_file_contents)
+
+whl_deps_library = repository_rule(
+    attrs = {
+        k: _pip_archive_attrs[k]
+        for k in [
+            "config_load",
+            "dep_template",
+            "group_deps",
+            "group_name",
+            "requirement",
+        ]
+    } | {
+        "metadata": attr.string(
+            doc = """
+The subset of the METADATA contents that is needed for generation of the dependencies.
+* name: {type}`str`
+* version: {type}`str`
+* provides_extra: {type}`list[str]`
+* requires_dist: {type}`list[str]`
+""",
+        ),
+        "metadata_file": attr.label(doc = "An alternative way to pass {attr}`metadata` but as a file."),
+        "repo": attr.label(doc = "A label at the root of the repo to get stuff from."),
+    },
+    doc = """
+A repo rule that reuses the sources from a different place and then creates the necessary targets
+so that this can be used in the repo.
+
+Does not depend on any python.
+""",
+    implementation = _whl_deps_library_impl,
+    environ = [REPO_DEBUG_ENV_VAR],
+)
+
+def whl_library(name, repo = None, **kwargs):
     """Create a whl_library.
 
     This proxies to one of the underlying implementations:
@@ -711,15 +782,18 @@ def whl_library(name, **kwargs):
 
     Args:
         name: {type}`str` The name of the repo.
+        repo: Unused, will be dropped in the next major release.
         **kwargs: The args passed to the underlying implementation.
 
     Returns:
         the repo metadata.
     """
+    _ = repo  # buildifier: disable=unused-variable
+
     whl_file = kwargs.get("whl_file")
     urls = kwargs.get("urls", [])
     filename = kwargs.get("filename")
     if whl_file or (urls and filename and filename.endswith(".whl")):
-        return whl_archive(name = name, **kwargs)
-
-    return pip_archive(name = name, **kwargs)
+        whl_archive(name = name, **kwargs)
+    else:
+        pip_archive(name = name, **kwargs)