Add model prediction and rule integration

.github/workflows/tests.yml

@@ -0,0 +1,60 @@
+name: Tests
+
+on: [push, pull_request]
+
+permissions: {}
+
+jobs:
+  base:
+    runs-on: ubuntu-24.04
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.13"]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
+        with:
+          persist-credentials: false
+      - name: Set up Python
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install development dependencies
+        run: ./configure --dev
+      - name: Run code checks
+        run: |
+          venv/bin/ruff check
+          make doc8
+      - name: Run tests
+        run: venv/bin/pytest -q
+      - name: Build distributions
+        if: matrix.python-version == '3.13'
+        run: |
+          venv/bin/python -m build
+          venv/bin/twine check dist/*
+
+  ml:
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
+        with:
+          persist-credentials: false
+      - name: Set up Python
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405
+        with:
+          python-version: "3.12"
+      - name: Install ML dependencies
+        run: |
+          ./configure --dev
+          venv/bin/pip install -e ".[training]"
+      - name: Run ML and integration tests
+        env:
+          USE_TF: "0"
+        run: |
+          venv/bin/pytest -q \
+            tests/test_training.py \
+            tests/test_model.py \
+            tests/test_export.py \
+            tests/test_inference.py \
+            tests/test_model_rules.py

CHANGELOG.rst

@@ -10,3 +10,4 @@ v0.0.0
 - Add required phrase dataset extraction.
 - Add composite rule required phrase updates.
 - Add required phrase model training and ONNX export.
+- Add model prediction and rule integration.

README.rst

@@ -54,8 +54,12 @@ reload and validation succeed. See ``docs/source/training.rst`` for details.
 Run read-only prediction
 ========================
 
-Load a validated final model and return candidate required phrases without
-changing a ScanCode rule or file:
+Install the inference dependencies, then load a validated final model and
+return candidate required phrases without changing a ScanCode rule or file:
+
+.. code-block:: console
+
+    python -m pip install ".[inference]"
 
 .. code-block:: python
 
@@ -66,6 +70,22 @@ changing a ScanCode rule or file:
 
 Predictions require human review before they are added to license rules.
 
+Add predicted phrases to rules
+==============================
+
+Review predictions before modifying rules. Then run the integration command on
+a final model directory or Hugging Face repository:
+
+.. code-block:: console
+
+    add-model-required-phrases --model model-output/final-model --dry-run --verbose
+
+For a Hugging Face repository, also provide its full commit hash with
+``--model-revision``. The command rejects phrase text found more than once in a
+rule because ScanCode's mutation helper would mark every occurrence. It validates
+the complete rule update and writes each changed rule once. Rebuild the ScanCode
+license index after applying changes without ``--dry-run``.
+
 Development
 ===========
 

azure-pipelines.yml

@@ -28,7 +28,8 @@ jobs:
                     tests/test_training.py \
                     tests/test_model.py \
                     tests/test_export.py \
-                    tests/test_inference.py
+                    tests/test_inference.py \
+                    tests/test_model_rules.py
             displayName: Run training unit tests
 
     - template: etc/ci/azure-posix.yml

docs/source/index.rst

@@ -11,6 +11,7 @@ license rules.
    dataset
    composite_rules
    training
+   model_rules
    contribute/contrib_doc
 
 Indices and tables

docs/source/model_rules.rst

@@ -0,0 +1,25 @@
+Add model-predicted required phrases
+====================================
+
+Use the command after reviewing predictions from a validated final model:
+
+.. code-block:: console
+
+    add-model-required-phrases \
+        --model model-output/final-model \
+        --dry-run \
+        --verbose
+
+Install the ``inference`` extra before using this command. ``--model`` accepts
+a local final-model directory or a Hugging Face repository. Remote models also
+require their full commit hash through ``--model-revision``. The model must pass
+the publication checks before inference starts.
+
+The command skips rules that cannot receive generated required phrases and
+rules that already contain required-phrase markers. It rejects phrase text
+found more than once because ScanCode would mark every occurrence. The complete
+rule update is checked in memory and each changed rule is written once.
+
+Use ``--license-expression`` to process one expression and ``--limit`` for a
+small review run. Remove ``--dry-run`` only after reviewing the predictions.
+Rebuild the ScanCode license index after writing rules.

setup.cfg

@@ -55,12 +55,20 @@ where = src
 [options.entry_points]
 console_scripts =
     add-composite-required-phrases = scancode_required_phrases.composite_rules:add_composite_required_phrases
+    add-model-required-phrases = scancode_required_phrases.model_rules:add_model_required_phrases
     build-required-phrases-dataset = scancode_required_phrases.dataset:main
     export-required-phrase-model = scancode_required_phrases.export:main
     train-required-phrase-model = scancode_required_phrases.training:main
 
 
 [options.extras_require]
+inference =
+    huggingface-hub == 0.36.2
+    pytorch-crf == 0.7.2
+    safetensors >= 0.4
+    sentencepiece >= 0.2
+    torch >= 2.0
+    transformers == 4.57.3
 training =
     accelerate >= 0.33
     huggingface-hub == 0.36.2
@@ -77,6 +85,7 @@ onnx =
     onnx >= 1.16
     onnxruntime >= 1.18
 dev =
+    build
     pytest >= 7.0.1
     pytest-xdist >= 2
     aboutcode-toolkit >= 7.0.2

src/scancode_required_phrases/inference.py

@@ -10,6 +10,7 @@
 
 from licensedcode.tokenize import required_phrase_splitter
 
+from scancode_required_phrases.training import encode_complete_words
 from scancode_required_phrases.training import extract_spans
 from scancode_required_phrases.training import first_subword_positions
 from scancode_required_phrases.training import ID2LABEL
@@ -82,20 +83,22 @@ def predict(self, text):
         if not words:
             return PredictionResult(words=(), phrases=(), truncated=False)
 
-        encoding = self.tokenizer(
-            words,
-            is_split_into_words=True,
-            truncation=True,
+        encoding, truncated = encode_complete_words(
+            tokens=words,
+            tokenizer=self.tokenizer,
             max_length=self.max_length,
-            return_tensors="pt",
         )
         positions = first_subword_positions(encoding.word_ids())
         if not positions:
             return PredictionResult(words=tuple(words), phrases=(), truncated=False)
 
         device = next(self.model.parameters()).device
-        input_ids = encoding["input_ids"].to(device)
-        attention_mask = encoding["attention_mask"].to(device)
+        input_ids = torch.tensor([encoding["input_ids"]], dtype=torch.long, device=device)
+        attention_mask = torch.tensor(
+            [encoding["attention_mask"]],
+            dtype=torch.long,
+            device=device,
+        )
 
         with torch.inference_mode():
             emissions = self.model.emissions(input_ids, attention_mask)
@@ -110,11 +113,8 @@ def predict(self, text):
             free = self.model.crf(word_emissions, tags, mask=mask, reduction="none")
 
             labels = [ID2LABEL[int(label)] for label in decoded]
-            truncated = len(labels) < len(words)
             predictions = []
             for start, end in extract_spans(labels):
-                if truncated and end == len(labels) - 1:
-                    continue
                 predictions.append(
                     PhrasePrediction(
                         text=" ".join(words[start : end + 1]),