#267 Remove memory leak in license detection match merging

 * Do not carry line_by_pos mapping in matches. Instead only inject
   start and end line in matches at the end using the query line_by_pos
   mapping kept only globally and not on a per-match basis.
   This was the source of a major memory leak when license matches were
   being updated and combined during a merge() run.
 * Use a list-based mapping for line_by_pos instead of a dic for smaller
   memory footprint.
 * Use slots for QueryRun attributes for smaller memory footprint
 * Replace "solid" attribute for Rules with a minimum_score that a match
   to a rule must equal or exceed. solid is now minimum_score: 100. Use
   this in other rules with various minimu score as needed. 
 * Remove remaining references to "gaps".
 * Fix incorrect Rule thresholds computation for minimum lengths.
 * Do not use license matches cache for now (the sqlite-backed
   diskcache-based implementation is the source of major slowdown).
 * Various minor cleanup and updates on rules, licenses and their
   corresponding tests.
 * New batch of frequent tokens for license detection.
 * Add new tests contributed by @yahalom5776
 * Add new license match filter for matches to a whole rule made of a
   single token that is surrounded by unknown or single letter tokens
   such as in "a b c d e  GPL 1 2 3  4" to discard some false positive
   (in this case for a GPL). This required to add tracking of query
   tokens made on a single character.

Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com>

Author: pombredanne

src/licensedcode/__init__.py

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2015 nexB Inc. and others. All rights reserved.
+# Copyright (c) 2016 nexB Inc. and others. All rights reserved.
 # http://nexb.com and https://github.com/nexB/scancode-toolkit/
 # The ScanCode software is licensed under the Apache License version 2.0.
 # Data generated with ScanCode require an acknowledgment.
@@ -52,9 +52,6 @@
 MIN_MATCH_LENGTH = 4
 MIN_MATCH_HIGH_LENGTH = 3
 
-# maximum number of non-matching tokens that can be skipped
-MAX_GAP_SKIP = 15
-
 # maximum distance between two matches to merge
 MAX_DIST = 120
 

src/licensedcode/data/licenses/day-spec.LICENSE

@@ -1,3 +1,10 @@
+[Day Specification License]
+
+Day Management AG ("Licensor") is willing to license this specification
+to you ONLY UPON THE CONDITION THAT YOU ACCEPT ALL OF THE TERMS CONTAINED
+IN THIS LICENSE AGREEMENT ("Agreement"). Please read the terms and
+conditions of this Agreement carefully.
+
 Content Repository for JavaTM Technology API Specification ("Specification")
 Version: 1.0
 Status: FCS

src/licensedcode/data/licenses/tatu-ylonen.LICENSE

@@ -1,11 +1,5 @@
 Tatu Ylonen License 
 
-* Author: Tatu Ylonen <ylo@cs.hut.fi>
- * Copyright (c) 1995 Tatu Ylonen <ylo@cs.hut.fi>, Espoo, Finland
- *                    All rights reserved
- * Versions of malloc and friends that check their results, and never return
- * failure (they call fatal if they encounter an error).
- *
  * As far as I am concerned, the code I have written for this software
  * can be used freely for any purpose.  Any derived versions of this
  * software must be clearly marked as such, and if the derived work is

src/licensedcode/data/licenses/zlib.SPDX

@@ -1,3 +1,3 @@
 licenses:
     - 3pp-proprietary-08
-solid: yes
+minimum_score: 100

src/licensedcode/data/rules/3pp-proprietary-08_3.yml

@@ -1,3 +1,3 @@
 licenses:
     - adobe-scl
-solid: yes
+minimum_score: 100

src/licensedcode/data/rules/adobe-scl_2.yml

@@ -1,5 +1,5 @@
 licenses:
     - afl-2.1
     - bsd-new
-solid: yes
+minimum_score: 100
 license_choice: yes

src/licensedcode/data/rules/afl-2.1_or_bsd-new_4.yml

@@ -1,6 +1,6 @@
 licenses:
     - apache-1.1
-solid: yes
+minimum_score: 100
 notes: >
     Notice found in http://apache.org/licenses/LICENSE-1.1 
     but it is really specific to an Apache Software Foundation component.

src/licensedcode/data/rules/apache-1.1_31.yml

@@ -1,4 +1,4 @@
 licenses:
     - apache-1.1
-solid: yes
+minimum_score: 100
 notes: Old, original Apache 1.1 URL

src/licensedcode/data/rules/apache-1.1_36.yml

@@ -1,3 +1,3 @@
 licenses:
     - apache-1.1
-solid: yes
+minimum_score: 100