From aeb25284e104169893a9121bbb88e12e53c0f3bc Mon Sep 17 00:00:00 2001 From: Boris Mbarga Date: Fri, 17 Jan 2025 08:08:07 +0000 Subject: [PATCH 1/2] Split the is_importing stat in two We want to use two new different sockstats, `import_skip_push_limit` and `import_soft_throttling` which are respectively used to allow a higher push limit, and implement soft throttling policy for imports. We still forward the `is_importing` stat for to skip push limit for backward compatibility. We plan to remove it on a subsequent PR. --- internal/governor/conn.go | 4 ++++ internal/governor/governor.go | 6 ++++++ internal/spokes/spokes.go | 10 +++++++++- 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/internal/governor/conn.go b/internal/governor/conn.go index 864cff6..fd00f13 100644 --- a/internal/governor/conn.go +++ b/internal/governor/conn.go @@ -211,6 +211,10 @@ func readSockstat(environ []string) updateData { res.GroupLeader = sockstat.GetBool(parts[1]) case "is_importing": res.IsImporting = sockstat.BoolValue(parts[1]) + case "import_skip_push_limit": + res.ImportSkipPushLimit = sockstat.BoolValue(parts[1]) + case "import_soft_throttling": + res.ImportSoftThrottling = sockstat.BoolValue(parts[1]) } } diff --git a/internal/governor/governor.go b/internal/governor/governor.go index 5354b87..7c3db1d 100644 --- a/internal/governor/governor.go +++ b/internal/governor/governor.go @@ -76,6 +76,12 @@ type updateData struct { CommandID string `json:"command_id,omitempty"` // IsImporting is true if the command is an import. IsImporting bool `json:"is_importing,omitempty"` + // ImportSkipPushLimit is true if the command is an import and + // we want to skip the push limit for a command. + ImportSkipPushLimit bool `json:"import_skip_push_limit,omitempty"` + // ImportSoftThrottling is true if the command is an import and + // we want to apply it some soft throttling policies. + ImportSoftThrottling bool `json:"import_soft_throttling,omitempty"` } func update(w io.Writer, ud updateData) error { diff --git a/internal/spokes/spokes.go b/internal/spokes/spokes.go index e4254f2..e651256 100644 --- a/internal/spokes/spokes.go +++ b/internal/spokes/spokes.go @@ -955,7 +955,11 @@ func (r *spokesReceivePack) isFsckConfigEnabled() bool { } func (r *spokesReceivePack) getMaxInputSize() (int, error) { - if isImporting() { + // We want to skip the default push limit when the `import_skip_push_limit` + // stat is set only. + // We keep using the `is_import` here for backward compatibility only, + // which should be removed on a subsequent PR. + if isImporting() || skipPushLimit() { return 80 * 1024 * 1024 * 1024, nil /* 80 GB */ } @@ -1234,6 +1238,10 @@ func isImporting() bool { return sockstat.GetBool("is_importing") } +func skipPushLimit() bool { + return sockstat.GetBool("import_skip_push_limit") +} + func allowBadDate() bool { return isImporting() && sockstat.GetBool("allow_baddate_in_import") } From ea702de45654ce43fa1279db1ae4e08bf3bf578b Mon Sep 17 00:00:00 2001 From: Boris Mbarga Date: Fri, 17 Jan 2025 08:28:32 +0000 Subject: [PATCH 2/2] Dont use is_importing for push limit Now that we have the `import_skip_push_limit` stat available, we can use it to skip push limiti. --- internal/spokes/spokes.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/internal/spokes/spokes.go b/internal/spokes/spokes.go index e651256..b74cee9 100644 --- a/internal/spokes/spokes.go +++ b/internal/spokes/spokes.go @@ -955,11 +955,7 @@ func (r *spokesReceivePack) isFsckConfigEnabled() bool { } func (r *spokesReceivePack) getMaxInputSize() (int, error) { - // We want to skip the default push limit when the `import_skip_push_limit` - // stat is set only. - // We keep using the `is_import` here for backward compatibility only, - // which should be removed on a subsequent PR. - if isImporting() || skipPushLimit() { + if skipPushLimit() { return 80 * 1024 * 1024 * 1024, nil /* 80 GB */ }