Harden model prediction and rule updates

Signed-off-by: Kaushik <kaushikrjpm10@gmail.com>

Author: Kaushik-Kumar-CEG

.github/workflows/tests.yml

@@ -0,0 +1,60 @@
+name: Tests
+
+on: [push, pull_request]
+
+permissions: {}
+
+jobs:
+  base:
+    runs-on: ubuntu-24.04
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.13"]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
+        with:
+          persist-credentials: false
+      - name: Set up Python
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install development dependencies
+        run: ./configure --dev
+      - name: Run code checks
+        run: |
+          venv/bin/ruff check
+          make doc8
+      - name: Run tests
+        run: venv/bin/pytest -q
+      - name: Build distributions
+        if: matrix.python-version == '3.13'
+        run: |
+          venv/bin/python -m build
+          venv/bin/twine check dist/*
+
+  ml:
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
+        with:
+          persist-credentials: false
+      - name: Set up Python
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405
+        with:
+          python-version: "3.12"
+      - name: Install ML dependencies
+        run: |
+          ./configure --dev
+          venv/bin/pip install -e ".[training]"
+      - name: Run ML and integration tests
+        env:
+          USE_TF: "0"
+        run: |
+          venv/bin/pytest -q \
+            tests/test_training.py \
+            tests/test_model.py \
+            tests/test_export.py \
+            tests/test_inference.py \
+            tests/test_model_rules.py

README.rst

@@ -54,8 +54,12 @@ reload and validation succeed. See ``docs/source/training.rst`` for details.
 Run read-only prediction
 ========================
 
-Load a validated final model and return candidate required phrases without
-changing a ScanCode rule or file:
+Install the inference dependencies, then load a validated final model and
+return candidate required phrases without changing a ScanCode rule or file:
+
+.. code-block:: console
+
+    python -m pip install ".[inference]"
 
 .. code-block:: python
 
@@ -76,9 +80,11 @@ a final model directory or Hugging Face repository:
 
     add-model-required-phrases --model model-output/final-model --dry-run --verbose
 
-The command validates each candidate with ScanCode's required-phrase helpers and
-writes each changed rule once. Rebuild the ScanCode license index after applying
-changes without ``--dry-run``.
+For a Hugging Face repository, also provide its full commit hash with
+``--model-revision``. The command rejects phrase text found more than once in a
+rule because ScanCode's mutation helper would mark every occurrence. It validates
+the complete rule update and writes each changed rule once. Rebuild the ScanCode
+license index after applying changes without ``--dry-run``.
 
 Development
 ===========

azure-pipelines.yml

@@ -28,7 +28,8 @@ jobs:
                     tests/test_training.py \
                     tests/test_model.py \
                     tests/test_export.py \
-                    tests/test_inference.py
+                    tests/test_inference.py \
+                    tests/test_model_rules.py
             displayName: Run training unit tests
 
     - template: etc/ci/azure-posix.yml

docs/source/model_rules.rst

@@ -10,13 +10,15 @@ Use the command after reviewing predictions from a validated final model:
         --dry-run \
         --verbose
 
-``--model`` accepts a local final-model directory or a Hugging Face repository.
-The model must pass the hardened publication checks before inference starts.
+Install the ``inference`` extra before using this command. ``--model`` accepts
+a local final-model directory or a Hugging Face repository. Remote models also
+require their full commit hash through ``--model-revision``. The model must pass
+the publication checks before inference starts.
 
 The command skips rules that cannot receive generated required phrases and
-rules that already contain required-phrase markers. Each prediction must pass
-ScanCode's candidate and locatability checks. Accepted phrases are applied in
-memory and each changed rule is written once.
+rules that already contain required-phrase markers. It rejects phrase text
+found more than once because ScanCode would mark every occurrence. The complete
+rule update is checked in memory and each changed rule is written once.
 
 Use ``--license-expression`` to process one expression and ``--limit`` for a
 small review run. Remove ``--dry-run`` only after reviewing the predictions.

setup.cfg

@@ -62,6 +62,13 @@ console_scripts =
 
 
 [options.extras_require]
+inference =
+    huggingface-hub == 0.36.2
+    pytorch-crf == 0.7.2
+    safetensors >= 0.4
+    sentencepiece >= 0.2
+    torch >= 2.0
+    transformers == 4.57.3
 training =
     accelerate >= 0.33
     huggingface-hub == 0.36.2
@@ -78,6 +85,7 @@ onnx =
     onnx >= 1.16
     onnxruntime >= 1.18
 dev =
+    build
     pytest >= 7.0.1
     pytest-xdist >= 2
     aboutcode-toolkit >= 7.0.2

src/scancode_required_phrases/inference.py

@@ -10,6 +10,7 @@
 
 from licensedcode.tokenize import required_phrase_splitter
 
+from scancode_required_phrases.training import encode_complete_words
 from scancode_required_phrases.training import extract_spans
 from scancode_required_phrases.training import first_subword_positions
 from scancode_required_phrases.training import ID2LABEL
@@ -41,41 +42,6 @@ def words_from_text(text):
     return required_phrase_splitter(unicodedata.normalize("NFKC", text))
 
 
-def _word_counts(word_ids):
-    counts = {}
-    for word_id in word_ids:
-        if word_id is not None:
-            counts[word_id] = counts.get(word_id, 0) + 1
-    return counts
-
-
-def encode_words(tokenizer, words, max_length):
-    """Encode the longest complete-word prefix and report truncation."""
-    call = dict(
-        is_split_into_words=True,
-        add_special_tokens=True,
-        return_tensors="pt",
-    )
-    full = tokenizer(words, truncation=False, **call)
-    encoding = tokenizer(words, truncation=True, max_length=max_length, **call)
-
-    full_counts = _word_counts(full.word_ids())
-    retained_counts = _word_counts(encoding.word_ids())
-    covered_words = max(retained_counts, default=-1) + 1
-    complete_words = covered_words
-
-    if covered_words and retained_counts[covered_words - 1] != full_counts[covered_words - 1]:
-        complete_words -= 1
-        encoding = tokenizer(words[:complete_words], truncation=False, **call)
-
-    if not complete_words:
-        raise ValueError("Tokenizer retained no complete words")
-    if encoding["input_ids"].shape[1] > max_length:
-        raise ValueError("Complete-word encoding exceeds the model maximum length")
-
-    return encoding, complete_words < len(words)
-
-
 def span_confidence(crf, word_emissions, tags, mask, free, span):
     """Return the CRF probability mass agreeing with one decoded span."""
     start, end = span
@@ -117,18 +83,22 @@ def predict(self, text):
         if not words:
             return PredictionResult(words=(), phrases=(), truncated=False)
 
-        encoding, truncated = encode_words(
+        encoding, truncated = encode_complete_words(
+            tokens=words,
             tokenizer=self.tokenizer,
-            words=words,
             max_length=self.max_length,
         )
         positions = first_subword_positions(encoding.word_ids())
         if not positions:
             return PredictionResult(words=tuple(words), phrases=(), truncated=False)
 
         device = next(self.model.parameters()).device
-        input_ids = encoding["input_ids"].to(device)
-        attention_mask = encoding["attention_mask"].to(device)
+        input_ids = torch.tensor([encoding["input_ids"]], dtype=torch.long, device=device)
+        attention_mask = torch.tensor(
+            [encoding["attention_mask"]],
+            dtype=torch.long,
+            device=device,
+        )
 
         with torch.inference_mode():
             emissions = self.model.emissions(input_ids, attention_mask)