-- SPDX-FileCopyrightText: © 2026 Vladimir Zorin <vladimir@deviant.guru>
-- SPDX-License-Identifier: OWL-1.0 or later

local testimony = require("testimony")
local core = require("mneme.core")

local testify = testimony.new("== mneme: ft analyzer ==")

testify:that("splits words on whitespace and punctuation", function()
	local terms = core.ft_analyze("hello world, foo-bar")
	testimony.assert_not_nil(terms)
	testimony.assert_equal(4, #terms)
	testimony.assert_equal("hello", terms[1])
	testimony.assert_equal("world", terms[2])
	testimony.assert_equal("foo", terms[3])
	testimony.assert_equal("bar", terms[4])
end)

testify:that("lowercases ASCII characters", function()
	local terms = core.ft_analyze("Hello WORLD FoO")
	testimony.assert_not_nil(terms)
	testimony.assert_equal(3, #terms)
	testimony.assert_equal("hello", terms[1])
	testimony.assert_equal("world", terms[2])
	testimony.assert_equal("foo", terms[3])
end)

testify:that("skips single-character tokens", function()
	local terms = core.ft_analyze("a b cd ef g hi")
	testimony.assert_not_nil(terms)
	testimony.assert_equal(3, #terms)
	testimony.assert_equal("cd", terms[1])
	testimony.assert_equal("ef", terms[2])
	testimony.assert_equal("hi", terms[3])
end)

testify:that("handles empty input", function()
	local terms = core.ft_analyze("")
	testimony.assert_not_nil(terms)
	testimony.assert_equal(0, #terms)
end)

testify:that("handles whitespace-only input", function()
	local terms = core.ft_analyze("   \t\n  ")
	testimony.assert_not_nil(terms)
	testimony.assert_equal(0, #terms)
end)

testify:that("preserves digits in words", function()
	local terms = core.ft_analyze("http2 rfc8446 h2o")
	testimony.assert_not_nil(terms)
	testimony.assert_equal(3, #terms)
	testimony.assert_equal("http2", terms[1])
	testimony.assert_equal("rfc8446", terms[2])
	testimony.assert_equal("h2o", terms[3])
end)

testify:that("preserves UTF-8 multibyte sequences", function()
	local terms = core.ft_analyze("hello wörld café")
	testimony.assert_not_nil(terms)
	testimony.assert_equal(3, #terms)
	testimony.assert_equal("hello", terms[1])
	-- UTF-8 bytes are preserved (non-ASCII not lowercased)
	testimony.assert_equal("wörld", terms[2])
	testimony.assert_equal("café", terms[3])
end)

testify:that("handles punctuation-heavy text", function()
	local terms = core.ft_analyze("foo...bar!! baz? (qux)")
	testimony.assert_not_nil(terms)
	testimony.assert_equal(4, #terms)
	testimony.assert_equal("foo", terms[1])
	testimony.assert_equal("bar", terms[2])
	testimony.assert_equal("baz", terms[3])
	testimony.assert_equal("qux", terms[4])
end)

testify:that("handles numbers-only tokens", function()
	local terms = core.ft_analyze("42 7 100 3")
	testimony.assert_not_nil(terms)
	testimony.assert_equal(2, #terms)
	testimony.assert_equal("42", terms[1])
	testimony.assert_equal("100", terms[2])
end)

testify:conclude()