{"id":2063029,"url":"https://alion.io/job/bt-group-qa-ai-evaluator","title":"QA - AI Evaluator","company":{"id":51586,"name":"BT Group","domain":"bt.com","url":"https://alion.io/company/bt-group","size_band":"5000+","is_staffing_agency":false,"employer_type":"direct","is_intermediary":false,"listed_via":null,"ats_vendor":"SuccessFactors","truth_index":{"grade":"A","score":97,"open_postings":3,"ghost_share":0,"stale_share":0.333,"repost_share":0,"time_to_fill_p50_days":11,"computed_at":"2026-10-13T05:45:00Z"}},"role":"Operations","role_family":"Operations","seniority":"middle","employment_type":null,"work_mode":"on_site","remote_scope":null,"remote_scope_basis":null,"remote_working_hours":null,"hiring_geo_confidence":"structured","locations":["Bengaluru, India"],"countries":["IN"],"hiring_countries":[],"hiring_countries_total":0,"salary":null,"salary_estimate":{"min_usd":13000,"max_usd":30000,"period":"year","method":"global_role_cell_scaled_by_country","sample_n":2095},"experience_years_min":3,"visa_sponsorship":false,"relocation_package":false,"has_equity":false,"technologies":[{"name":"AI Agents","optional":false},{"name":"Hallucination","optional":false},{"name":"LLM","optional":false},{"name":"Multi-Agent Systems","optional":false},{"name":"Prompt Engineering","optional":false},{"name":"RAG","optional":false}],"status":"live","first_seen_at":"2026-10-08T02:42:57Z","employer_posted_date":null,"last_verified_at":"2026-10-08T02:42:57Z","board_verified":false,"closed_at":null,"days_open":5,"trust":{"level":"not_scored","repost_count":null,"flags":[],"days_open":5},"description":"The QA AI Evaluator will be responsible for ensuring the quality, reliability, governance, and business effectiveness of Agentic AI solutions. The role focuses on evaluating AI agents, prompts, LLM outputs, and end-to-end workflows to ensure accurate, consistent, and trusted outcomes.\nResponsibilities:\nDefine AI evaluation frameworks, standards, KPIs, and success criteria.\nValidate agent accuracy, reasoning, reliability, workflow quality, and business outcomes.\nAssess prompts, LLM responses, hallucination risks, and AI-generated content quality.\nBenchmark models, tools, prompts, and agent configurations to improve performance.\nEnsure Responsible AI, security, privacy, governance, and compliance requirements are met.\nProvide clear insights, risks, recommendations, and scorecards to leadership and stakeholders.\nDefine and govern AI quality standards, evaluation frameworks, KPIs, and success criteria.\nValidate Agentic AI solutions for accuracy, reasoning, reliability, and business effectiveness.\nAssess prompts, LLM outputs, and end-to-end agent workflows to ensure quality outcomes.\nBenchmark models, prompts, tools, and agent configurations to optimize performance.\nMeasure and improve AI quality metrics, including accuracy, relevance, completeness, and hallucination rates.\nIdentify risks, quality gaps, and opportunities for continuous improvement.\nEnsure Responsible AI, security, privacy, governance, and compliance requirements are met.\nSupport AI adoption, user acceptance, and organizational change initiatives.\nCollaborate with product, engineering, architecture, QA, and business stakeholders.\nPresent insights, risks, performance metrics, and strategic recommendations to leadership.\nRequirements:\nStrong experience in AI testing, quality engineering, AI/LLM evaluation, or agentic AI assurance.\nDeep understanding of LLMs, prompt engineering, RAG, AI agents, and multi-agent architectures.\nExpertise in defining evaluation frameworks, quality metrics, KPIs, scorecards, and benchmarking approaches.\nExperience validating AI accuracy, reasoning, reliability, hallucinations, and business outcomes.\nKnowledge of Responsible AI, Security, Privacy, Governance, Risk, and Compliance practices.\nStrong analytical and problem-solving skills with experience in root cause analysis and continuous improvement.\nFamiliarity with AI evaluation tools, test automation, and data-driven quality assessment techniques.\nAbility to collaborate effectively with product, engineering, architecture, QA, and business stakeholders.\nExcellent communication and stakeholder management skills, including presenting insights to leadership.\nExperience driving AI adoption, change management, and business transformation initiatives.\nEssential:\n3-4 years of experience with strong experience in AI testing, quality engineering, AI/LLM evaluation, or agentic AI assurance.\nGood understanding of LLMs, prompt engineering, RAG, AI agents, and multi-agent workflows.\nAbility to define evaluation frameworks, quality metrics, KPIs, scorecards, and success criteria.\nHands-on experience validating AI accuracy, reasoning, reliability, hallucination risk, relevance, completeness, and business outcomes.\nStrong analytical, problem-solving, stakeholder management, and communication skills.\nDesirable:\nMinimum 3-4 years of experience with AI evaluation tools, automation frameworks, and data-driven quality assessment techniques.\nExposure to benchmarking models, prompts, tools, and agent configurations to improve performance.\nExperience supporting AI adoption, user acceptance, change management, or wider business transformation initiatives.","description_format":"text","description_chars":3634,"description_truncated":false,"requirements":{"experience_years_min":3,"management_years_min":null,"team_size_min":null,"manages_managers":false,"education":null,"security_clearance":false,"languages":[]},"benefits":[],"hiring_locations":[],"hiring_excludes":[],"relocation_offered":false,"industries":["Artificial Intelligence","Telecommunications","Broadband"],"lifecycle":[{"event":"open","at":"2026-10-08T02:42:57Z"}],"visa":[{"country":"GB","licensed_sponsor":true,"evidence":"Licensed UK visa sponsor (Senior or Specialist Worker, Skilled Worker)","filings_12m":null,"filings_prev_12m":null,"green_card_filings_12m":null,"median_offered_wage_usd":null,"route":"Global Business Mobility: Senior or Specialist Worker; Skilled Worker","cap_exempt":false,"checked_at":"2026-10-13T06:20:30+00:00","sources":["UK Home Office: register of licensed sponsors (workers)"],"filings_for_role_12m":0}],"liveness":{"score":80,"band":"hot","label":"Hiring now","p_open":1,"p_active":0.843,"p_room":0.945,"age_days":5,"expected_fill_days":11,"reasons":["seen:5","velocity","win:mid"],"computed_at":"2026-10-13T05:45:00Z"},"pay":null,"html_url":"https://alion.io/job/bt-group-qa-ai-evaluator","json_url":"https://alion.io/job/bt-group-qa-ai-evaluator.json","meta":{"generated_at":"2026-10-14T01:25:20Z","cache_seconds":300,"methodology":"https://alion.io/methodology","terms":"https://alion.io/terms","contact":"https://alion.io/contact","api":"https://alion.io/developers","about":"Alion is a live layer of people, companies and AI agents: who they are, whether they are real and active right now, what they do and how to work with them, readable by people and by agents and paid per call.","catalog":"https://alion.io/catalog.json","usage":{"tier":"crawler_verified","counted_by":"address","units_charged":1,"used_today":2097,"day_limit":null,"remaining_today":null,"minute_limit":300,"resets_at":"2026-10-15T00:00:00Z"}},"offers":[{"id":"company.slices","title":"One company in depth, by slice","status":"live","price":{"credits":0.02,"usd":0.002,"plus_per_slice":{"credits":0.05,"usd":0.005}},"unit":"per company, plus each slice with data","note":"the employer in depth","call":{"mcp_tool":"get_company","arguments":{"id":51586},"rest":"https://alion.io/mcp/rest/get_company?id=51586"},"human":"https://alion.io/catalog?offer=company.slices&for=job%2Fbt-group-qa-ai-evaluator"},{"id":"market.stats","title":"A market slice: pay, demand and time to fill","status":"live","price":{"credits":1,"usd":0.1},"unit":"per slice","note":"pay, demand and time to fill for this role and place","call":{"mcp_tool":"market_stats"},"human":"https://alion.io/catalog?offer=market.stats&for=job%2Fbt-group-qa-ai-evaluator"},{"id":"job.search","title":"Open jobs by role, technology, place, pay and visa","status":"live","price":{"credits":0.02,"usd":0.002},"unit":"per posting in a list","note":"similar open postings","call":{"mcp_tool":"search_jobs"},"human":"https://alion.io/catalog?offer=job.search&for=job%2Fbt-group-qa-ai-evaluator"},{"id":"company.verify","title":"Is this company real and active right now","status":"pilot","price":null,"unit":"per company","request":{"url":"https://alion.io/catalog/request","method":"POST","body":"{\"offer\": \"company.verify\", \"for\": \"job/bt-group-qa-ai-evaluator\", \"note\": \"what you need it for\"}"},"human":"https://alion.io/catalog?offer=company.verify&for=job%2Fbt-group-qa-ai-evaluator"}]}