|
| 1 | +# -*- coding: utf-8 -*- |
| 2 | +# |
| 3 | +# Copyright (c) nexB Inc. and others. All rights reserved. |
| 4 | +# ScanCode is a trademark of nexB Inc. |
| 5 | +# SPDX-License-Identifier: Apache-2.0 |
| 6 | +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. |
| 7 | +# See https://github.com/nexB/scancode-toolkit for support or download. |
| 8 | +# See https://aboutcode.org for more information about nexB OSS projects. |
| 9 | +# |
| 10 | + |
| 11 | +import pytest |
| 12 | + |
| 13 | +from cluecode.finder_data import JUNK_URL_PREFIXES |
| 14 | +from cluecode.finder_data import JUNK_URLS |
| 15 | +from cluecode.finder_data import JUNK_DOMAIN_SUFFIXES |
| 16 | +from cluecode.finder_data import classify_url |
| 17 | + |
| 18 | + |
| 19 | +@pytest.mark.parametrize('url_prefix', sorted(JUNK_URL_PREFIXES)) |
| 20 | +def test_classify_url__junk_urls_prefixes(url_prefix): |
| 21 | + assert not classify_url(url_prefix) |
| 22 | + assert classify_url(f"foobar/{url_prefix}") |
| 23 | + |
| 24 | + |
| 25 | +@pytest.mark.parametrize('url', sorted(JUNK_URLS)) |
| 26 | +def test_classify_url__junk_urls(url): |
| 27 | + assert not classify_url(url) |
| 28 | + assert classify_url(f"{url}/foobar") |
| 29 | + |
| 30 | + |
| 31 | +@pytest.mark.parametrize('suffix', sorted(JUNK_DOMAIN_SUFFIXES)) |
| 32 | +def test_classify_url__junk_domain_suffix(suffix): |
| 33 | + assert not classify_url(f"http://foo/bar{suffix}") |
| 34 | + assert classify_url(f"{suffix}/some/bar") |
| 35 | + |
| 36 | + |
| 37 | +MORE_JUNK_URLS = ''' |
| 38 | + http://www.adobe.com/2006/mxml |
| 39 | + http://www.w3.org/1999/XSL/Transform |
| 40 | + http://docs.oasis-open.org/ns/xri/xrd-1.0 |
| 41 | + http://www.w3.org/2001/XMLSchema-instance |
| 42 | + http://www.w3.org/2001/XMLSchema}string |
| 43 | + http://www.w3.org/2001/XMLSchema |
| 44 | + http://java.sun.com/xml/ns/persistence/persistence_1_0.xsd |
| 45 | + http://bing.com |
| 46 | + http://google.com |
| 47 | + http://msn.com |
| 48 | + http://maven.apache.org/maven-v4_0_0.xsd |
| 49 | + http://maven.apache.org/POM/4.0.0 |
| 50 | + http://www.w3.org/MarkUp/DTD/xhtml-rdfa-1.dtd |
| 51 | + http://www.w3.org/1999/02/22-rdf-syntax-ns |
| 52 | + http://www.w3.org/1999/xhtml |
| 53 | + http://www.w3.org/1999/XMLSchema |
| 54 | + http://www.w3.org/1999/XMLSchema-instance |
| 55 | + http://www.w3.org/2000/svg |
| 56 | + http://www.w3.org/2001/XMLSchema |
| 57 | + http://www.w3.org/2000/10/XMLSchema |
| 58 | + http://www.w3.org/2000/10/XMLSchema-instance |
| 59 | + http://www.w3.org/2001/XMLSchema |
| 60 | + http://www.w3.org/2001/XMLSchema-instance |
| 61 | + http://www.w3.org/2002/12/soap-encoding |
| 62 | + http://www.w3.org/2002/12/soap-envelope |
| 63 | + http://www.w3.org/2005/Atom |
| 64 | + http://www.w3.org/2006/01/wsdl |
| 65 | + http://www.w3.org/2006/01/wsdl/http |
| 66 | + http://www.w3.org/2006/01/wsdl/soap |
| 67 | + http://www.w3.org/2006/vcard/ns |
| 68 | + http://www.w3.org/International/O-URL-and-ident.html |
| 69 | + http://www.w3.org/MarkUp |
| 70 | + http://www.w3.org/WAI/GL |
| 71 | + http://xml.apache.org/axis/session |
| 72 | + http://xml.apache.org/xml-soap |
| 73 | + http://docs.oasis-open.org/ns/xri/xrd-1.0 |
| 74 | + http://cobertura.sourceforge.net/xml/coverage-01.dtd |
| 75 | + http://findbugs.googlecode.com/svn/trunk/findbugs/etc/docbook/docbookx.dtd |
| 76 | + http://hibernate.sourceforge.net/hibernate-configuration-2.0.dtd |
| 77 | + http://hibernate.sourceforge.net/hibernate-generic.dtd |
| 78 | + http://hibernate.sourceforge.net/hibernate-mapping-2.0.dtd |
| 79 | + http://www.opensymphony.com/xwork/xwork-1.0.dtd |
| 80 | + http://]hostname |
| 81 | + http://+ |
| 82 | + http://www |
| 83 | + http://www.w3.org/1999/xhtml |
| 84 | + http://www.w3.org/1999/XSL/Transform |
| 85 | + http://www.w3.org/2001/XMLSchema |
| 86 | + http://www.w3.org/2001/XMLSchema-instance |
| 87 | + http://www.w3.org/hypertext/WWW/Protocols/HTTP/HTRESP.html |
| 88 | + http://www.w3.org/hypertext/WWW/Protocols/HTTP/Object_Headers.html |
| 89 | + http://www.w3.org/P3P |
| 90 | + http://www.w3.org/pub/WWW |
| 91 | + http://www.w3.org/TR/html4/strict.dtd |
| 92 | + http://www.w3.org/TR/REC-html40/loose.dtd |
| 93 | + http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd |
| 94 | + http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd |
| 95 | + http://www.w3.org/TR/xslt |
| 96 | + https: |
| 97 | + https://+ |
| 98 | + http://www.example.com |
| 99 | + http://www.example.com/dir/file |
| 100 | + http://www.example.com:dir/file |
| 101 | + http://www.your.org.here |
| 102 | + http://hostname |
| 103 | + https://www.trustedcomputinggroup.org/XML/SCHEMA/TNCCS_1.0.xsd |
| 104 | + http://glade.gnome.org/glade-2.0.dtd |
| 105 | + http://pagesperso-orange.fr/sebastien.godard/sysstat.dtd |
| 106 | + http://www.freedesktop.org/standards/dbus/1.0/busconfig.dtd |
| 107 | + http://www.freedesktop.org/standards/dbus/1.0/introspect.dtd |
| 108 | + http://gcc.gnu.org/bugs.html |
| 109 | + http://nsis.sf.net/NSIS_Error |
| 110 | + http://www.springframework.org/dtd/ |
| 111 | + http://www.slickedit.com/dtd/ |
| 112 | + http://www.oexchange.org/spec/0.8/ |
| 113 | + http://www.puppycrawl.com/dtds/ |
| 114 | + http://adobe.com/AS3/2006/builtin |
| 115 | + http://careers.msn.com |
| 116 | + http://foo.bar.baz |
| 117 | + http://foo.bar.com |
| 118 | + http://foobar.com |
| 119 | + http://java.sun.com/xml/ns/ |
| 120 | + http://java.sun.com/j2se/1.4/docs/ |
| 121 | + http://java.sun.com/j2se/1.5.0/docs/ |
| 122 | + http://developer.apple.com/certificationauthority/ |
| 123 | + http://www.apple.com/appleca/ |
| 124 | + https://www.apple.com/certificateauthority/ |
| 125 | + http://schemas.microsoft.com/ |
| 126 | + http://dublincore.org/schemas/ |
| 127 | + http://www.w3.org/TR/ |
| 128 | + http://www.apple.com/DTDs |
| 129 | + http://apache.org/xml/features/ |
| 130 | + http://apache.org/xml/properties/ |
| 131 | + http://crl.verisign.com/ |
| 132 | + http://crl.globalsign.net/ |
| 133 | + http://crl.microsoft.com/ |
| 134 | + http://crl.thawte.com/ |
| 135 | + http://CSC3-2004-crl.verisign.com |
| 136 | + http://csc3-2009-2-crl.verisign.com |
| 137 | + http://dellincca.dell.com/crl |
| 138 | + http://ts-crl.ws.symantec.com |
| 139 | + http://java.sun.com/dtd/ |
| 140 | + http://java.sun.com/j2ee/dtds/ |
| 141 | + http://jakarta.apache.org/commons/dtds/ |
| 142 | + http://jakarta.apache.org/struts/dtds/ |
| 143 | + http://www.jboss.org/j2ee/dtd/ |
| 144 | + http://glassfish.org/dtds/ |
| 145 | + http://docbook.org/xml/simple/ |
| 146 | + http://www.oasis-open.org/docbook/xml/ |
| 147 | + http://www.w3.org/XML/1998/namespace |
| 148 | + https://www.w3.org/XML/1998/namespace |
| 149 | + http://www.w3.org/2000/xmlns/ |
| 150 | + https://www.w3.org/2000/xmlns/ |
| 151 | + http://ts-aia.ws.symantec.com/ |
| 152 | + https://ts-aia.ws.symantec.com/ |
| 153 | + https://www.verisign.com/rpa |
| 154 | + http://csc3-2010-crl.verisign.com/ |
| 155 | + https://www.verisign.com/rpa |
| 156 | + http://csc3-2010-aia.verisign.com/ |
| 157 | + https://www.verisign.com/cps |
| 158 | + http://logo.verisign.com/ |
| 159 | + http://ocsp2.globalsign.com/ |
| 160 | + http://crl.globalsign.com/ |
| 161 | + http://secure.globalsign.com/cacert/ |
| 162 | + https://www.globalsign.com/repository/ |
| 163 | + http://www.microsoft.com/pki/certs/ |
| 164 | + http://www.microsoft.com/pkiops/crl |
| 165 | + http://www.microsoft.com/PKI/ |
| 166 | +'''.split() |
| 167 | + |
| 168 | + |
| 169 | +@pytest.mark.parametrize('url', sorted(MORE_JUNK_URLS)) |
| 170 | +def test_classify_url__more_junk_urls(url): |
| 171 | + assert not classify_url(url) |
0 commit comments