scancode-toolkit/tests/textcode/test_pdf.py at 43db560cf8d686d8300fec06b4fb892674c25c6a · aboutcode-org/scancode-toolkit · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
#
# Copyright (c) nexB Inc. and others. All rights reserved.
# ScanCode is a trademark of nexB Inc.
# SPDX-License-Identifier: Apache-2.0
# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
# See https://github.com/nexB/scancode-toolkit for support or download.
# See https://aboutcode.org for more information about nexB OSS projects.
#

import os

import pytest

from commoncode.testcase import FileBasedTesting
from textcode import pdf
from textcode.analysis import numbered_text_lines


class TestPdf(FileBasedTesting):
    test_data_dir = os.path.join(os.path.dirname(__file__), 'data')

    def test_get_text_lines(self):
        test_file = self.get_test_loc('pdf/pdf.pdf')
        result = pdf.get_text_lines(test_file)
        expected = b'''pdf

"""
Extracts text from a pdf file.
"""
import contextlib
from StringIO import StringIO

from pdfminer.pdfinterp import PDFResourceManager, PDFPageInterpreter
from pdfminer.pdfpage import PDFPage
from pdfminer.converter import TextConverter

def get_text(location):
    rs_mgr = PDFResourceManager()
    extracted_text = StringIO()
    with contextlib.closing(TextConverter(rs_mgr, extracted_text)) as extractor:
        with open(location, \'rb\') as pdf_file:
            interpreter = PDFPageInterpreter(rs_mgr, extractor)
            pages = PDFPage.get_pages(pdf_file, check_extractable=True)
            for page in pages:
                interpreter.process_page(page)
    return extracted_text

Page 1

\x0c'''.splitlines(True)

        assert result == expected

    def test_get_text_lines_extracts_all_pages_up_to_max_pages(self):
        # regression test: text extraction must not stop after the first page
        test_file = self.get_test_loc('pdf/multi_page.pdf')
        result = pdf.get_text_lines(test_file)
        text = b''.join(result)
        assert b'This is page 1 of a multi-page test document.' in text
        assert b'This notice is on page 2.' in text
        assert b'The last page 5 of this test document.' in text
        # the default max_pages=5 must still be honored
        assert b'Page 6' not in text

    def test_get_text_lines_returns_lines_when_max_pages_is_one(self):
        # regression test: reaching max_pages must not return None
        test_file = self.get_test_loc('pdf/multi_page.pdf')
        result = pdf.get_text_lines(test_file, max_pages=1)
        assert result
        text = b''.join(result)
        assert b'This is page 1 of a multi-page test document.' in text
        assert b'page 2' not in text

    def test_get_text_lines_extracts_all_pages_when_max_pages_is_zero(self):
        test_file = self.get_test_loc('pdf/multi_page.pdf')
        result = pdf.get_text_lines(test_file, max_pages=0)
        text = b''.join(result)
        assert b'Page 6 is beyond the default max_pages limit.' in text
        assert b'Page 7 is also beyond the default max_pages limit.' in text

    def test_pdfminer_can_parse_faulty_broadcom_doc(self):
        # test for https://github.com/euske/pdfminer/issues/118
        test_file = self.get_test_loc('pdf/pdfminer_bug_118/faulty.pdf')
        from pdfminer.pdfparser import PDFParser
        from pdfminer.pdfdocument import PDFDocument
        with open(test_file, 'rb') as inputfile:
            parser = PDFParser(inputfile)
            PDFDocument(parser)

    def test_get_text_lines_can_parse_faulty_broadcom_doc(self):
        test_file = self.get_test_loc('pdf/pdfminer_bug_118/faulty.pdf')
        result = list(pdf.get_text_lines(test_file))
        expected = [
            b'Programmer\xe2\x80\x99s Guide\n',
            b'BCM5756M\n', b'\n',
            b'Host Programmer Interface Specification for the \n',
            b'NetXtreme\xc2\xae and NetLink\xe2\x84\xa2 Family of Highly \n',
            b'Integrated Media Access Controllers\n',
            b'\n',
            b'5300 California Avenue  \xe2\x80\xa2  Irvine, CA 92617  (cid:129)  Phone: 949-926-5000  (cid:129)  Fax: 949-926-5203\n',
            b'\n',
            b'5756M-PG101-R\n',
            b'\n',
            b'10/15/07\n',
            b'\n',
            b'\x0c']
        assert result == expected

    def test_pdfminer_can_parse_apache_fop_test_pdf(self):
        test_file = self.get_test_loc('pdf/fop_test_pdf_1.5_test.pdf')
        result = pdf.get_text_lines(test_file)
        for expected in apache_fop_expected:
            assert expected in result

    @pytest.mark.xfail(reason='Latest pdfminer.six from 2022 has a regression')
    def test_numbered_text_lines_does_not_fail_on_autocad_test_pdf(self):
        test_file = self.get_test_loc('pdf/AutoCad_Diagram.pdf')
        result = list(numbered_text_lines(test_file))
        assert result == []


apache_fop_expected = [
    b'This is the page header\n',
    b'About Apache FOP\n',
    b'It  is  a  print  formatter  driv-\n',
    b'en  by  XSL  formatting  ob-\n',
    b'jects (XSL-FO) and an out-\n',
    b'ter1. FOP has a nice logo:\n',
    b'Header 1.1 Header 1.2\n',
    b'See the FOP website for more information\n'
]