{"id":3502300,"url":"https://alion.io/job/true-talent-solutions-partners-research-engineer-evals","title":"Research Engineer, Evals","company":{"id":4415660,"name":"True Talent Solutions Partners","domain":"truetalentsolutionspartners.com","url":"https://alion.io/company/true-talent-solutions-partners","size_band":null,"is_staffing_agency":true,"employer_type":"agency","is_intermediary":false,"listed_via":null,"ats_vendor":"Ashby","truth_index":null},"role":"AI/ML","role_family":"AI/ML","seniority":"middle","employment_type":"full_time","work_mode":"on_site","remote_scope":null,"remote_scope_basis":null,"remote_working_hours":null,"hiring_geo_confidence":"structured","locations":["San Francisco, United States"],"countries":["US"],"hiring_countries":[],"hiring_countries_total":0,"salary":{"min":250000,"max":290000,"currency":"USD","period":"year","gross":null,"usd_annual":290000},"salary_estimate":null,"experience_years_min":4,"visa_sponsorship":true,"relocation_package":false,"has_equity":false,"technologies":[{"name":"LLM","optional":false},{"name":"Machine Learning","optional":false},{"name":"NLP","optional":false},{"name":"Python","optional":false}],"status":"closed","first_seen_at":"2026-10-30T02:22:07Z","employer_posted_date":"2026-10-30","last_verified_at":"2026-11-03T00:58:51Z","board_verified":false,"closed_at":"2026-11-03T00:58:51Z","days_open":3,"trust":{"level":"not_scored","repost_count":null,"flags":[],"days_open":3},"description":"About the Role\nBuild evaluation systems for a web data extraction product, defining how output quality is measured across diverse websites, formats, and edge cases. You will combine research and engineering to turn evaluation results into actionable model and product improvements.\nWhat You'll Do\nDesign rigorous metrics for evaluating structured extraction quality across varied web content.\n\nBuild scalable evaluation pipelines, testing harnesses, and supporting data systems.\n\nGenerate and curate ground-truth datasets for trustworthy measurement.\n\nConnect evaluation findings to model and product decisions, closing the quality feedback loop.\n\nMake evaluation results clear and useful to the broader team.\n\nWhat We're Looking For\nTypically 4 to 10 or more years of relevant experience, including at least 2 years working on evaluations.\n\nStrong Python skills and experience with machine learning, LLMs, data pipelines, web scraping, NLP, evaluation frameworks, and statistical analysis.\n\nExperience building evaluation systems for LLM-based structured extraction from messy, real-world web pages or documents, including field-level accuracy measurement at scale.\n\nExperience designing evaluation methods and metrics, building ground-truth datasets, and applying statistical rigor, including sampling, estimator bias and variance, error bounds, and judge-to-human agreement.\n\nAbility to combine research methodology with engineering implementation and use evaluation results to guide real model or product decisions.\n\nCompensation & Benefits\nBase salary: $250,000 to $290,000 USD annually. Visa sponsorship is available.\nLocation\nHybrid, based in San Francisco, California. The role requires working on-site five days a week in San Francisco or Toronto, with relocation before starting if needed.","description_format":"text","description_chars":1799,"description_truncated":false,"requirements":{"experience_years_min":4,"management_years_min":null,"team_size_min":null,"manages_managers":false,"education":null,"security_clearance":false,"languages":[]},"benefits":[],"hiring_locations":[],"hiring_excludes":[],"relocation_offered":true,"industries":[],"lifecycle":[{"event":"open","at":"2026-10-30T03:52:28Z"},{"event":"close","at":"2026-11-03T00:58:51Z"}],"visa":[],"liveness":null,"pay":{"stated_usd_annual":290000,"is_top_pay":true},"html_url":"https://alion.io/job/true-talent-solutions-partners-research-engineer-evals","json_url":"https://alion.io/job/true-talent-solutions-partners-research-engineer-evals.json","meta":{"generated_at":"2026-11-03T07:00:43Z","cache_seconds":300,"methodology":"https://alion.io/methodology","terms":"https://alion.io/terms","contact":"https://alion.io/contact","api":"https://alion.io/developers","about":"Alion is a live layer of people, companies and AI agents: who they are, whether they are real and active right now, what they do and how to work with them, readable by people and by agents and paid per call.","catalog":"https://alion.io/catalog.json","usage":{"tier":"crawler_verified","counted_by":"address","units_charged":1,"used_today":9845,"day_limit":null,"remaining_today":null,"minute_limit":300,"resets_at":"2026-11-04T00:00:00Z"}},"offers":[{"id":"company.slices","title":"One company in depth, by slice","status":"live","price":{"credits":0.02,"usd":0.002,"plus_per_slice":{"credits":0.05,"usd":0.005}},"unit":"per company, plus each slice with data","note":"the employer in depth","call":{"mcp_tool":"get_company","arguments":{"id":4415660},"rest":"https://alion.io/mcp/rest/get_company?id=4415660"},"human":"https://alion.io/catalog?offer=company.slices&for=job%2Ftrue-talent-solutions-partners-research-engineer-evals"},{"id":"market.stats","title":"A market slice: pay, demand and time to fill","status":"live","price":{"credits":1,"usd":0.1},"unit":"per slice","note":"pay, demand and time to fill for this role and place","call":{"mcp_tool":"market_stats"},"human":"https://alion.io/catalog?offer=market.stats&for=job%2Ftrue-talent-solutions-partners-research-engineer-evals"},{"id":"job.search","title":"Open jobs by role, technology, place, pay and visa","status":"live","price":{"credits":0.02,"usd":0.002},"unit":"per posting in a list","note":"similar open postings","call":{"mcp_tool":"search_jobs"},"human":"https://alion.io/catalog?offer=job.search&for=job%2Ftrue-talent-solutions-partners-research-engineer-evals"},{"id":"company.verify","title":"Is this company real and active right now","status":"pilot","price":null,"unit":"per company","request":{"url":"https://alion.io/catalog/request","method":"POST","body":"{\"offer\": \"company.verify\", \"for\": \"job/true-talent-solutions-partners-research-engineer-evals\", \"note\": \"what you need it for\"}"},"human":"https://alion.io/catalog?offer=company.verify&for=job%2Ftrue-talent-solutions-partners-research-engineer-evals"}],"agent":{"mcp":"https://alion.io/mcp/jobs","mcp_open":"https://alion.io/mcp/open","next":[{"tool":"get_job","arguments":{"id":3502300}},{"tool":"search_jobs","arguments":{"company":"truetalentsolutionspartners.com","detail":"compact"}},{"tool":"match_people","arguments":{"job_id":3502300},"note":"people open to offers who fit this vacancy"}],"note":"The same data through MCP tools: https://alion.io/mcp/open needs no sign-in (compact searches, postings, people); https://alion.io/mcp with an Alion account has everything. Docs: https://alion.io/developers"}}