From f18eda8b888168916fbf41acb83d79f8aa6ef4e3 Mon Sep 17 00:00:00 2001 From: Suhaib Mujahid Date: Wed, 17 Jul 2024 09:34:23 -0400 Subject: [PATCH 1/4] Fetch report from Clouseau in batches --- libmozdata/clouseau.py | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/libmozdata/clouseau.py b/libmozdata/clouseau.py index 9fff6f00..4ad9bc77 100644 --- a/libmozdata/clouseau.py +++ b/libmozdata/clouseau.py @@ -2,6 +2,8 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. +from itertools import batched + from . import config from .connection import Connection, Query @@ -54,14 +56,20 @@ def get_by_signatures(cls, signatures, product=None, channel=None): dict: the reports by signatures """ data = {} - cls( - params={ - "signatures": signatures, - "product": product, - "channel": channel, - }, - handler=cls._default_handler, - handlerdata=data, - ).wait() + requests = [ + cls( + params={ + "signatures": signatures_batch, + "product": product, + "channel": channel, + }, + handler=cls._default_handler, + handlerdata=data, + ) + for signatures_batch in batched(signatures, 20) + ] + + for request in requests: + request.wait() return data From 27a9a8d5b1b5343244695174809d8f366c47d69f Mon Sep 17 00:00:00 2001 From: Suhaib Mujahid Date: Wed, 17 Jul 2024 10:12:04 -0400 Subject: [PATCH 2/4] Copy the implementation of `itertools.batched()` --- libmozdata/clouseau.py | 3 ++- libmozdata/utils.py | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/libmozdata/clouseau.py b/libmozdata/clouseau.py index 4ad9bc77..cc982b46 100644 --- a/libmozdata/clouseau.py +++ b/libmozdata/clouseau.py @@ -2,7 +2,8 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. -from itertools import batched + +from libmozdata.utils import batched from . import config from .connection import Connection, Query diff --git a/libmozdata/utils.py b/libmozdata/utils.py index e7f77ec3..4575395e 100644 --- a/libmozdata/utils.py +++ b/libmozdata/utils.py @@ -8,6 +8,7 @@ import os.path import random from datetime import date, datetime, timedelta +from itertools import count import dateutil.parser import pytz @@ -381,3 +382,39 @@ def get_language(path): for lang, names in langs.items(): if name in names or extension in names: return lang + + +# This is a roughly equivalent to the itertools.islice which was added in +# Python 3.10. +# Copied from https://docs.python.org/3/library/itertools.html#itertools.islice +def islice(iterable, *args): + # islice('ABCDEFG', 2) → A B + # islice('ABCDEFG', 2, 4) → C D + # islice('ABCDEFG', 2, None) → C D E F G + # islice('ABCDEFG', 0, None, 2) → A C E G + + s = slice(*args) + start = 0 if s.start is None else s.start + stop = s.stop + step = 1 if s.step is None else s.step + if start < 0 or (stop is not None and stop < 0) or step <= 0: + raise ValueError + + indices = count() if stop is None else range(max(start, stop)) + next_i = start + for i, element in zip(indices, iterable): + if i == next_i: + yield element + next_i += step + + +# This is a roughly equivalent to the itertools.batched which was added in +# Python 3.12. +# Copied from https://docs.python.org/3/library/itertools.html#itertools.batched +def batched(iterable, n): + # batched('ABCDEFG', 3) → ABC DEF G + if n < 1: + raise ValueError("n must be at least one") + iterator = iter(iterable) + while batch := tuple(islice(iterator, n)): + yield batch From 9b30ccc34647af08060054746eef48a5182fb658 Mon Sep 17 00:00:00 2001 From: Suhaib Mujahid Date: Wed, 17 Jul 2024 10:15:56 -0400 Subject: [PATCH 3/4] Update libmozdata/utils.py Co-authored-by: Marco Castelluccio --- libmozdata/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libmozdata/utils.py b/libmozdata/utils.py index 4575395e..113fb7ff 100644 --- a/libmozdata/utils.py +++ b/libmozdata/utils.py @@ -384,7 +384,7 @@ def get_language(path): return lang -# This is a roughly equivalent to the itertools.islice which was added in +# This is roughly equivalent to the itertools.islice which was added in # Python 3.10. # Copied from https://docs.python.org/3/library/itertools.html#itertools.islice def islice(iterable, *args): From 34508e7cc7b0dea4472d0fe10bdab5c81ba0b50a Mon Sep 17 00:00:00 2001 From: Suhaib Mujahid Date: Wed, 17 Jul 2024 10:22:42 -0400 Subject: [PATCH 4/4] Update libmozdata/utils.py Co-authored-by: Marco Castelluccio --- libmozdata/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libmozdata/utils.py b/libmozdata/utils.py index 113fb7ff..171c7fd8 100644 --- a/libmozdata/utils.py +++ b/libmozdata/utils.py @@ -408,7 +408,7 @@ def islice(iterable, *args): next_i += step -# This is a roughly equivalent to the itertools.batched which was added in +# This is roughly equivalent to the itertools.batched which was added in # Python 3.12. # Copied from https://docs.python.org/3/library/itertools.html#itertools.batched def batched(iterable, n):