{"id":2345982,"url":"https://alion.io/job/yo-ai-labs-ai-agent-evaluator-remote","title":"AI Agent Evaluator - Remote","company":{"id":3824455,"name":"YO AI Labs","domain":"yoailabs.com","url":"https://alion.io/company/yoailabs","size_band":"5000+","is_staffing_agency":false,"employer_type":"direct","is_intermediary":false,"listed_via":null,"ats_vendor":"Manatal","truth_index":{"grade":"A","score":100,"open_postings":19,"ghost_share":0,"stale_share":0,"repost_share":0,"time_to_fill_p50_days":11,"computed_at":"2026-10-22T06:10:10Z"}},"role":"Operations","role_family":"Operations","seniority":"junior","employment_type":null,"work_mode":"on_site","remote_scope":null,"remote_scope_basis":null,"remote_working_hours":null,"hiring_geo_confidence":"explicit","locations":["New York, United States"],"countries":["US"],"hiring_countries":[],"hiring_countries_total":0,"salary":null,"salary_estimate":{"min_usd":72000,"max_usd":149000,"period":"year","method":"role_seniority_country_remote_cell","sample_n":2427},"experience_years_min":2,"visa_sponsorship":false,"relocation_package":false,"has_equity":false,"technologies":[{"name":"AI Agents","optional":false}],"status":"live","first_seen_at":"2026-10-12T12:37:49Z","employer_posted_date":"2026-10-12","last_verified_at":"2026-10-22T20:15:44Z","board_verified":true,"closed_at":null,"days_open":10,"trust":{"level":"ok","repost_count":null,"flags":[],"days_open":10},"description":"Role Title: AI Agent Evaluator\nRole Type: Contractor\nLocation: United States\nWe are seeking detail-oriented professionals to evaluate AI agents performing everyday tasks. You'll test two AI agents side by side, assess their performance, and provide feedback to improve their reliability and user experience.\nKey Responsibilities\nTest AI agents on everyday tasks, including shopping, travel, bookings, bill payments, email follow-ups, and forms.\nApprove or stop agents at key steps.\nScreen-record each testing session.\nScore task completion, quality, control issues, user satisfaction, and time.\nProvide clear written feedback on agent performance and usability.\nRequired Qualifications\nBased in the United States with a bachelor's degree.\nRegularly use AI tools in daily life.\n2-3 years of experience, preferably in engineering, business management, or office operations.\nStrong attention to detail and ability to follow instructions.\nExcellent written communication and documentation skills.\nAn active personal AI account (Free, Go, or Plus; not Pro) and a smartphone capable of screen recording.\nPreferred: Relevant domain expertise. No coding or AI development experience is required.\nSelection Process\nSubmit an application and complete screening questions.\nAttend a 15-30-minute interview.\nComplete profile review and onboarding.\nProject Details\nTask duration: Approximately 1.5 hours per run, including evidence and scoring.\nKickoff: Monday, October 12.\n Onboarding: Before the project begins, subject to scheduling.","description_format":"text","description_chars":1522,"description_truncated":false,"requirements":{"experience_years_min":2,"management_years_min":null,"team_size_min":null,"manages_managers":false,"education":{"level":"bachelor","optional":false},"security_clearance":false,"languages":[]},"benefits":[],"hiring_locations":[{"name":"United States","iso":"US","kind":"country"}],"hiring_excludes":[],"relocation_offered":false,"industries":[],"lifecycle":[{"event":"open","at":"2026-10-12T12:37:49Z"}],"visa":[],"liveness":{"score":51,"band":"ok","label":"Likely open","p_open":1,"p_active":0.676,"p_room":0.75,"age_days":9,"expected_fill_days":11,"reasons":["conf:1","velocity","win:late","comp:junior,brand"],"computed_at":"2026-10-22T06:10:10Z"},"pay":null,"html_url":"https://alion.io/job/yo-ai-labs-ai-agent-evaluator-remote","json_url":"https://alion.io/job/yo-ai-labs-ai-agent-evaluator-remote.json","meta":{"generated_at":"2026-10-22T22:20:24Z","cache_seconds":300,"methodology":"https://alion.io/methodology","terms":"https://alion.io/terms","contact":"https://alion.io/contact","api":"https://alion.io/developers","about":"Alion is a live layer of people, companies and AI agents: who they are, whether they are real and active right now, what they do and how to work with them, readable by people and by agents and paid per call.","catalog":"https://alion.io/catalog.json","usage":{"tier":"crawler_verified","counted_by":"address","units_charged":1,"used_today":7455,"day_limit":null,"remaining_today":null,"minute_limit":300,"resets_at":"2026-10-23T00:00:00Z"}},"offers":[{"id":"company.slices","title":"One company in depth, by slice","status":"live","price":{"credits":0.02,"usd":0.002,"plus_per_slice":{"credits":0.05,"usd":0.005}},"unit":"per company, plus each slice with data","note":"the employer in depth","call":{"mcp_tool":"get_company","arguments":{"id":3824455},"rest":"https://alion.io/mcp/rest/get_company?id=3824455"},"human":"https://alion.io/catalog?offer=company.slices&for=job%2Fyo-ai-labs-ai-agent-evaluator-remote"},{"id":"market.stats","title":"A market slice: pay, demand and time to fill","status":"live","price":{"credits":1,"usd":0.1},"unit":"per slice","note":"pay, demand and time to fill for this role and place","call":{"mcp_tool":"market_stats"},"human":"https://alion.io/catalog?offer=market.stats&for=job%2Fyo-ai-labs-ai-agent-evaluator-remote"},{"id":"job.search","title":"Open jobs by role, technology, place, pay and visa","status":"live","price":{"credits":0.02,"usd":0.002},"unit":"per posting in a list","note":"similar open postings","call":{"mcp_tool":"search_jobs"},"human":"https://alion.io/catalog?offer=job.search&for=job%2Fyo-ai-labs-ai-agent-evaluator-remote"},{"id":"company.verify","title":"Is this company real and active right now","status":"pilot","price":null,"unit":"per company","request":{"url":"https://alion.io/catalog/request","method":"POST","body":"{\"offer\": \"company.verify\", \"for\": \"job/yo-ai-labs-ai-agent-evaluator-remote\", \"note\": \"what you need it for\"}"},"human":"https://alion.io/catalog?offer=company.verify&for=job%2Fyo-ai-labs-ai-agent-evaluator-remote"}]}