{"id":3576861,"url":"https://alion.io/job/raydar-ai-evaluation-engineer","title":"AI Evaluation Engineer","company":{"id":4478753,"name":"Raydar","domain":"raydar.com","url":"https://alion.io/company/raydar-com","size_band":"1001-5000","is_staffing_agency":false,"employer_type":"direct","is_intermediary":false,"listed_via":null,"ats_vendor":"Workable","truth_index":null},"role":"AI/ML","role_family":"AI/ML","seniority":null,"employment_type":"full_time","work_mode":"remote","remote_scope":"stated_countries","remote_scope_basis":"posting_text","remote_working_hours":null,"hiring_geo_confidence":"explicit","locations":[],"countries":[],"hiring_countries":["US","CA"],"hiring_countries_total":2,"salary":{"min":200000,"max":300000,"currency":"USD","period":"year","gross":null,"usd_annual":300000},"salary_estimate":null,"experience_years_min":null,"visa_sponsorship":false,"relocation_package":false,"has_equity":false,"technologies":[],"status":"live","first_seen_at":"2026-10-15T00:00:00Z","employer_posted_date":"2026-10-15","last_verified_at":"2026-11-02T05:11:27Z","board_verified":true,"closed_at":null,"days_open":18,"trust":{"level":"ok","repost_count":null,"flags":[],"days_open":18},"description":"About the company\nOur client is a software company.\nThe role\nRaydar is recruiting for this role on behalf of our client. Lead the creation of a measurement approach for an AI system that carries out technical design work. Draw on both hands-on engineering judgment and evaluation know-how, and collaborate with domain experts and other teams to convert findings into clear improvement priorities.\nWhat you'll do\n- Establish a new assessment framework that defines quality for design work where several answers can be valid.\n- Collaborate with domain experts, external users and internal teams to see how seasoned engineers operate and where the system struggles.\n- Reproduce realistic design tasks and typical failure cases in repeatable test setups that developers can run again.\n- Create and tune automated graders alongside human review processes, and confirm that the graders agree with expert opinion.\n- Convert assessment results into a ranked list of improvements to the system's reasoning.\nRequirements\nWhat we're looking for\n- About five or more years of professional experience in hardware engineering, AI, or a blend of both, with depth in one and practical familiarity with the other.\n- Substantial hands-on background assessing AI systems, including authoring test tasks, building automated graders and benchmarking them against human review.\n- Solid grasp of how autonomous, agent-style AI systems behave.\n- Grounding in a hardware engineering field, enough to tell sound design from weak design.\n- Comfort working directly with external users, specialists, partners and technical colleagues across functions.\n- Self-assurance to guide work among many stakeholders in a role with high ownership.\n- Based in the US or Canada, and able to travel to an office for one week every 6 weeks.\nBonus points\n- Prior time at a robotics or physical AI company.\n- Experience building things from zero in an early-stage startup.\nBenefits\nCompensation and benefits\n- Base salary: USD 200,000 to 300,000 per year\n- Equity\nLocation and work model\n- Remote (US or Canada)\n- Full-time","description_format":"text","description_chars":2079,"description_truncated":false,"requirements":{"experience_years_min":null,"management_years_min":null,"team_size_min":null,"manages_managers":false,"education":null,"security_clearance":false,"languages":[]},"benefits":["Equity"],"hiring_locations":[{"name":"United States","iso":"US","kind":"country"},{"name":"Canada","iso":"CA","kind":"country"}],"hiring_excludes":[],"relocation_offered":false,"industries":[],"lifecycle":[{"event":"open","at":"2026-10-31T22:14:15Z"}],"visa":[],"liveness":{"score":44,"band":"fade","label":"Fading","p_open":1,"p_active":0.794,"p_room":0.55,"age_days":18,"expected_fill_days":17,"reasons":["conf:1","velocity","win:tail","comp:brand"],"computed_at":"2026-11-02T06:13:15Z"},"pay":{"stated_usd_annual":300000,"is_top_pay":true},"html_url":"https://alion.io/job/raydar-ai-evaluation-engineer","json_url":"https://alion.io/job/raydar-ai-evaluation-engineer.json","meta":{"generated_at":"2026-11-02T06:22:41Z","cache_seconds":300,"methodology":"https://alion.io/methodology","terms":"https://alion.io/terms","contact":"https://alion.io/contact","api":"https://alion.io/developers","about":"Alion is a live layer of people, companies and AI agents: who they are, whether they are real and active right now, what they do and how to work with them, readable by people and by agents and paid per call.","catalog":"https://alion.io/catalog.json","usage":{"tier":"crawler_verified","counted_by":"address","units_charged":1,"used_today":9186,"day_limit":null,"remaining_today":null,"minute_limit":300,"resets_at":"2026-11-03T00:00:00Z"}},"offers":[{"id":"company.slices","title":"One company in depth, by slice","status":"live","price":{"credits":0.02,"usd":0.002,"plus_per_slice":{"credits":0.05,"usd":0.005}},"unit":"per company, plus each slice with data","note":"the employer in depth","call":{"mcp_tool":"get_company","arguments":{"id":4478753},"rest":"https://alion.io/mcp/rest/get_company?id=4478753"},"human":"https://alion.io/catalog?offer=company.slices&for=job%2Fraydar-ai-evaluation-engineer"},{"id":"market.stats","title":"A market slice: pay, demand and time to fill","status":"live","price":{"credits":1,"usd":0.1},"unit":"per slice","note":"pay, demand and time to fill for this role and place","call":{"mcp_tool":"market_stats"},"human":"https://alion.io/catalog?offer=market.stats&for=job%2Fraydar-ai-evaluation-engineer"},{"id":"job.search","title":"Open jobs by role, technology, place, pay and visa","status":"live","price":{"credits":0.02,"usd":0.002},"unit":"per posting in a list","note":"similar open postings","call":{"mcp_tool":"search_jobs"},"human":"https://alion.io/catalog?offer=job.search&for=job%2Fraydar-ai-evaluation-engineer"},{"id":"company.verify","title":"Is this company real and active right now","status":"pilot","price":null,"unit":"per company","request":{"url":"https://alion.io/catalog/request","method":"POST","body":"{\"offer\": \"company.verify\", \"for\": \"job/raydar-ai-evaluation-engineer\", \"note\": \"what you need it for\"}"},"human":"https://alion.io/catalog?offer=company.verify&for=job%2Fraydar-ai-evaluation-engineer"}],"agent":{"mcp":"https://alion.io/mcp/jobs","mcp_open":"https://alion.io/mcp/open","next":[{"tool":"get_job","arguments":{"id":3576861}},{"tool":"search_jobs","arguments":{"company":"raydar.com","detail":"compact"}},{"tool":"match_people","arguments":{"job_id":3576861},"note":"people open to offers who fit this vacancy"}],"note":"The same data through MCP tools: https://alion.io/mcp/open needs no sign-in (compact searches, postings, people); https://alion.io/mcp with an Alion account has everything. Docs: https://alion.io/developers"}}