diff --git a/libmozdata/clouseau.py b/libmozdata/clouseau.py index 9fff6f00..cc982b46 100644 --- a/libmozdata/clouseau.py +++ b/libmozdata/clouseau.py @@ -2,6 +2,9 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. + +from libmozdata.utils import batched + from . import config from .connection import Connection, Query @@ -54,14 +57,20 @@ def get_by_signatures(cls, signatures, product=None, channel=None): dict: the reports by signatures """ data = {} - cls( - params={ - "signatures": signatures, - "product": product, - "channel": channel, - }, - handler=cls._default_handler, - handlerdata=data, - ).wait() + requests = [ + cls( + params={ + "signatures": signatures_batch, + "product": product, + "channel": channel, + }, + handler=cls._default_handler, + handlerdata=data, + ) + for signatures_batch in batched(signatures, 20) + ] + + for request in requests: + request.wait() return data diff --git a/libmozdata/utils.py b/libmozdata/utils.py index e7f77ec3..171c7fd8 100644 --- a/libmozdata/utils.py +++ b/libmozdata/utils.py @@ -8,6 +8,7 @@ import os.path import random from datetime import date, datetime, timedelta +from itertools import count import dateutil.parser import pytz @@ -381,3 +382,39 @@ def get_language(path): for lang, names in langs.items(): if name in names or extension in names: return lang + + +# This is roughly equivalent to the itertools.islice which was added in +# Python 3.10. +# Copied from https://docs.python.org/3/library/itertools.html#itertools.islice +def islice(iterable, *args): + # islice('ABCDEFG', 2) → A B + # islice('ABCDEFG', 2, 4) → C D + # islice('ABCDEFG', 2, None) → C D E F G + # islice('ABCDEFG', 0, None, 2) → A C E G + + s = slice(*args) + start = 0 if s.start is None else s.start + stop = s.stop + step = 1 if s.step is None else s.step + if start < 0 or (stop is not None and stop < 0) or step <= 0: + raise ValueError + + indices = count() if stop is None else range(max(start, stop)) + next_i = start + for i, element in zip(indices, iterable): + if i == next_i: + yield element + next_i += step + + +# This is roughly equivalent to the itertools.batched which was added in +# Python 3.12. +# Copied from https://docs.python.org/3/library/itertools.html#itertools.batched +def batched(iterable, n): + # batched('ABCDEFG', 3) → ABC DEF G + if n < 1: + raise ValueError("n must be at least one") + iterator = iter(iterable) + while batch := tuple(islice(iterator, n)): + yield batch