Skip to content

add Gemini 3.8 Flash and GPT-6 Astra #3

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
simonpcouch wants to merge 2 commits into
base: main
Choose a base branch
from
Open

add Gemini 3.8 Flash and GPT-6 Astra #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1fb67bb
set up Gemini 3.8 Flash and GPT-6 Astra runs
simonpcouch Sep 11, 2026
7e87119
add Gemini 3.8 Flash and GPT-6 Astra results
simonpcouch Sep 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 41 additions & 8 deletions data-raw/bluff_results.R
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,31 @@ task_costs <- purrr::map(task_files, function(f) {
cost_df <- tsk$get_cost()
task_name <- gsub("tsk_|\\.rda", "", basename(f))
solver <- cost_df |> dplyr::filter(source == "solver")
solver_chats <- tsk$get_samples()$solver_chat
cached_input_tokens <- purrr::map_dbl(solver_chats, function(chat) {
sum(chat$get_tokens()$cached_input)
})
cache_write_tokens <- purrr::map_dbl(solver_chats, function(chat) {
purrr::map_dbl(chat$get_turns(), function(turn) {
if (!inherits(turn, "ellmer::AssistantTurn")) {
return(0)
}
purrr::pluck(
turn@json,
"usage",
"input_tokens_details",
"cache_write_tokens",
.default = 0
)
}) |>
sum()
})
tibble(
task_name = task_name,
solver_input_tokens = solver$input,
solver_output_tokens = solver$output,
solver_cached_input_tokens = sum(cached_input_tokens),
solver_cache_write_tokens = sum(cache_write_tokens),
cost = as.numeric(gsub("\\$", "", solver$price))
)
}) |>
Expand All @@ -30,21 +51,31 @@ task_costs <- purrr::map(task_files, function(f) {
# - Claude Sonnet 5: standard $3 input / $15 output per MTok. (An
# introductory $2 / $10 rate runs through 2026-08-31, but we use the
# durable list price to stay comparable with the other models.)
# - GPT-6 Astra: $10 input / $1 cached input / $12.50 cache write / $50
# output per MTok
# (https://developers.openai.com/api/docs/pricing)
# - Gemini 3.8 Flash: promotional $0.75 input / $0.075 cached input / $3.75
# output per MTok through 2026-12-31
# (https://ai.google.dev/gemini-api/docs/pricing)
manual_prices <- tribble(
~task_name , ~input_per_mtok , ~output_per_mtok ,
"claude_fable_5" , 10 , 50 ,
"gemma4_26b_a4b" , 0.30 , 1.50 ,
"sonnet_5_medium" , 3 , 15 ,
"sonnet_5_nonthinking" , 3 , 15 ,
~task_name , ~input_per_mtok, ~cached_input_per_mtok,
~cache_write_per_mtok, ~output_per_mtok,
"claude_fable_5" , 10 , 0 , 0 , 50,
"gemma4_26b_a4b" , 0.30, 0 , 0 , 1.50,
"sonnet_5_medium" , 3 , 0 , 0 , 15,
"sonnet_5_nonthinking" , 3 , 0 , 0 , 15,
"gpt_6_astra_medium" , 10 , 1 , 12.5, 50,
"gemini_3_8_flash_medium" , 0.75, 0.075, 0 , 3.75,
)

task_costs <- task_costs |>
left_join(manual_prices, by = "task_name") |>
mutate(
cost = if_else(
is.na(cost),
(solver_input_tokens *
input_per_mtok +
!is.na(input_per_mtok),
((solver_input_tokens - solver_cache_write_tokens) * input_per_mtok +
solver_cached_input_tokens * cached_input_per_mtok +
solver_cache_write_tokens * cache_write_per_mtok +
solver_output_tokens * output_per_mtok) /
1e6,
cost
Expand Down Expand Up @@ -109,13 +140,15 @@ bluff_results <-
model == "gpt_5_nano_medium" ~ "GPT-5 nano (medium)",
model == "gpt_5_4_nano_medium" ~ "GPT-5.4 nano (medium)",
model == "gpt_5_4_nano_nonthinking" ~ "GPT-5.4 nano (non-thinking)",
model == "gpt_6_astra_medium" ~ "GPT-6 Astra (medium)",
model == "gemini_2_5_pro_medium" ~ "Gemini 2.5 Pro (medium)",
model == "gemini_2_5_flash_medium" ~ "Gemini 2.5 Flash (medium)",
model ==
"gemini_2_5_flash_nonthinking" ~ "Gemini 2.5 Flash (non-thinking)",
model == "gemini_3_flash_medium" ~ "Gemini 3 Flash (medium)",
model == "gemini_3_1_pro_medium" ~ "Gemini 3.1 Pro (medium)",
model == "gemini_3_5_flash_medium" ~ "Gemini 3.5 Flash (medium)",
model == "gemini_3_8_flash_medium" ~ "Gemini 3.8 Flash (medium)",
model == "gemma4_26b_a4b" ~ "Gemma4 26B A4B"
),
thinking = stringr::str_detect(model, "\\(medium\\)")
Expand Down
2 changes: 2 additions & 0 deletions data-raw/model_metadata.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,12 @@ gpt_5_4_nano_medium,OpenAI,2026-03-17,https://developers.openai.com/api/docs/cha
gpt_5_4_nano_nonthinking,OpenAI,2026-03-17,https://developers.openai.com/api/docs/changelog
gpt_5_5_medium,OpenAI,2026-04-23,https://developers.openai.com/api/docs/changelog
gpt_5_5_nonthinking,OpenAI,2026-04-23,https://developers.openai.com/api/docs/changelog
gpt_6_astra_medium,OpenAI,2026-09-03,https://developers.openai.com/api/docs/changelog
gemini_2_5_pro_medium,Google,2025-03-25,https://ai.google.dev/gemini-api/docs/changelog
gemini_2_5_flash_medium,Google,2025-06-17,https://ai.google.dev/gemini-api/docs/changelog
gemini_2_5_flash_nonthinking,Google,2025-06-17,https://ai.google.dev/gemini-api/docs/changelog
gemini_3_flash_medium,Google,2025-12-17,https://ai.google.dev/gemini-api/docs/changelog
gemini_3_1_pro_medium,Google,2025-11-18,https://ai.google.dev/gemini-api/docs/changelog
gemini_3_5_flash_medium,Google,2026-05-19,https://ai.google.dev/gemini-api/docs/changelog
gemini_3_8_flash_medium,Google,2026-09-02,https://ai.google.dev/gemini-api/docs/changelog
gemma4_26b_a4b,Google,2026-03-31,https://ai.google.dev/gemma/docs/releases
Binary file modified data/bluff_results.rda
View file
Open in desktop
Binary file not shown.
Loading