Examples

Budgeting a prompt

use token_estimate::{estimate, remaining, Profile};

let system = "You are a careful assistant.";
let context = std::fs::read_to_string("notes.md").unwrap_or_default();

let budget = 128_000;
let used = estimate(system, Profile::Prose) + estimate(&context, Profile::Mixed);
println!("{} tokens left", remaining(&context, budget, Profile::Mixed));
assert!(used <= budget);

Choosing a margin

use token_estimate::{estimate, estimate_with_margin, Profile};

let text = "some prose to measure";
let base = estimate(text, Profile::Prose);

// A 20% margin when overflow is expensive.
assert!(estimate_with_margin(text, Profile::Prose, 20) > base);
assert_eq!(estimate_with_margin(text, Profile::Prose, 0), base);

Whitespace does not inflate the estimate

use token_estimate::{estimate, Profile};

let tight  = "alpha beta gamma";
let padded = "alpha        beta        gamma";
assert_eq!(estimate(tight, Profile::Prose), estimate(padded, Profile::Prose));

Non-Latin text is a floor, not a figure

use token_estimate::{estimate, Profile};

let ascii = "aaaaaaaaaaaaaaaaaaaa";
let cjk   = "説明説明説明説明説明説明説明説明説明説明";
// Same character count, higher estimate — but still only a floor.
assert!(estimate(cjk, Profile::Prose) > estimate(ascii, Profile::Prose));