{"id":4309796,"url":"https://alion.io/job/newvision-software-consultancy-ai-qa-engineer","title":"AI QA Engineer","company":{"id":4489094,"name":"NewVision Software & Consultancy","domain":"newvision-software.com","url":"https://alion.io/company/newvision-software-and-consultancy","size_band":"501-1000","is_staffing_agency":false,"employer_type":"services","is_intermediary":false,"listed_via":null,"ats_vendor":"Career site","truth_index":null},"role":"QA","role_family":"QA","seniority":"middle","employment_type":null,"work_mode":"on_site","remote_scope":null,"remote_scope_basis":null,"remote_working_hours":null,"hiring_geo_confidence":"structured","locations":["Pune, India"],"countries":["IN"],"hiring_countries":[],"hiring_countries_total":0,"salary":null,"salary_estimate":null,"experience_years_min":3,"visa_sponsorship":false,"relocation_package":false,"has_equity":false,"technologies":[{"name":"AI Agents","optional":false},{"name":"Arize Phoenix","optional":false},{"name":"Azure","optional":false},{"name":"CI/CD","optional":false},{"name":"DeepEval","optional":false},{"name":"GitHub Actions","optional":false},{"name":"Hallucination","optional":false},{"name":"Java","optional":false},{"name":"LLM","optional":false},{"name":"OpenAI","optional":false},{"name":"Playwright","optional":false},{"name":"Pytest","optional":false},{"name":"Python","optional":false},{"name":"RAG","optional":false},{"name":"Selenium","optional":false}],"status":"live","first_seen_at":"2026-08-13T00:00:00Z","employer_posted_date":"2026-11-14","last_verified_at":"2026-11-14T11:21:27Z","board_verified":true,"closed_at":null,"days_open":93,"trust":{"level":"ok","repost_count":null,"flags":[],"days_open":93},"description":"The AI Test Engineer is responsible for validating the quality, reliability, safety, performance, and governance of AI systems, including LLMs, RAG pipelines, and AI agents. Unlike traditional QA role, this position focuses on LLM evaluation, behavioral testing, hallucination, bias detection, AI test automation, and alignment with Responsible AI governance standards.\nKey Responsibilities:\n· Define AI test strategy for LLMs, AI agents, and orchestration workflows leveraging Tools & framework like DeepEval, Arize\n· Design and maintain golden evaluation datasets, prompt test cases, and benchmarking frameworks for LLMs, RAG systems\n· Perform regression testing across prompt updates, agent logic changes, multi-step agent workflows and model upgrades\n· Measure and analyse accuracy, consistency, latency, throughput, and cost efficiency of AI services\n· Integrate AI test suites into CI/CD pipelines with automated quality gates\n· Conduct red-team testing to identify safety, compliance, security, and prompt-injection vulnerabilities\n· Provide actionable insights through evaluation reports, metrics, and release readiness assessments\nQualifications:\n· 3-6 years of experience in AI Test Automation\n· Proven experience in testing AI/ML/LLM-based systems\n· Strong understanding of:\nPrompt behavior, model evaluation, dataset curation and risk management\n· Hands-on experience with:\nPython, Pytest,Pyrit, Java\nTest automation tools (Selenium, Playwright)\n· AI evaluation tools and frameworks\n· Azure AI Services / Azure OpenAI, CI/CD pipelines and GitHub Actions\n· Test automation integration into DevOps workflows","description_format":"text","description_chars":1617,"description_truncated":false,"requirements":{"experience_years_min":3,"management_years_min":null,"team_size_min":null,"manages_managers":false,"education":null,"security_clearance":false,"languages":[]},"benefits":[],"hiring_locations":[],"hiring_excludes":[],"relocation_offered":false,"industries":["IT Consulting & Digital Transformation","Custom Software Development"],"lifecycle":[{"event":"open","at":"2026-11-14T11:21:27Z"}],"visa":[],"liveness":{"score":11,"band":"cold","label":"Long shot","p_open":1,"p_active":0.406,"p_room":0.28,"age_days":93,"expected_fill_days":30,"reasons":["conf:1","win:tail","crowd:brand"],"computed_at":"2026-11-14T13:02:51Z"},"pay":null,"html_url":"https://alion.io/job/newvision-software-consultancy-ai-qa-engineer","json_url":"https://alion.io/job/newvision-software-consultancy-ai-qa-engineer.json","meta":{"generated_at":"2026-11-14T13:02:51Z","cache_seconds":300,"methodology":"https://alion.io/methodology","terms":"https://alion.io/terms","contact":"https://alion.io/contact","api":"https://alion.io/developers","about":"Alion is a live layer of people, companies and AI agents: who they are, whether they are real and active right now, what they do and how to work with them, readable by people and by agents and paid per call.","catalog":"https://alion.io/catalog.json","usage":{"tier":"crawler_verified","counted_by":"address","units_charged":1,"used_today":13064,"day_limit":null,"remaining_today":null,"minute_limit":300,"resets_at":"2026-11-15T00:00:00Z"}},"offers":[{"id":"company.slices","title":"One company in depth, by slice","status":"live","price":{"credits":0.1,"usd":0.01,"plus_per_slice":{"credits":0.05,"usd":0.005}},"unit":"per company, plus each slice with data","note":"the employer in depth","call":{"mcp_tool":"get_company","arguments":{"id":4489094},"rest":"https://alion.io/mcp/rest/get_company?id=4489094"},"human":"https://alion.io/catalog?offer=company.slices&for=job%2Fnewvision-software-consultancy-ai-qa-engineer"},{"id":"market.stats","title":"A market slice: pay, demand and time to fill","status":"free","price":{"credits":0,"usd":0},"unit":"per slice","note":"pay, demand and time to fill for this role and place","call":{"mcp_tool":"market_stats"},"human":"https://alion.io/catalog?offer=market.stats&for=job%2Fnewvision-software-consultancy-ai-qa-engineer"},{"id":"job.search","title":"Open jobs by role, technology, place, pay and visa","status":"live","price":{"credits":0.1,"usd":0.01},"unit":"per posting in a list","note":"similar open postings","call":{"mcp_tool":"search_jobs"},"human":"https://alion.io/catalog?offer=job.search&for=job%2Fnewvision-software-consultancy-ai-qa-engineer"},{"id":"company.verify","title":"Is this company real and active right now","status":"pilot","price":null,"unit":"per company","request":{"url":"https://alion.io/catalog/request","method":"POST","body":"{\"offer\": \"company.verify\", \"for\": \"job/newvision-software-consultancy-ai-qa-engineer\", \"note\": \"what you need it for\"}"},"human":"https://alion.io/catalog?offer=company.verify&for=job%2Fnewvision-software-consultancy-ai-qa-engineer"}],"agent":{"mcp":"https://alion.io/mcp/jobs","mcp_open":"https://alion.io/mcp/open","next":[{"tool":"get_job","arguments":{"id":4309796}},{"tool":"search_jobs","arguments":{"company":"newvision-software.com","detail":"compact"}},{"tool":"match_people","arguments":{"job_id":4309796},"note":"people open to offers who fit this vacancy"}],"note":"The same data through MCP tools: https://alion.io/mcp/open needs no sign-in (compact searches, postings, people); https://alion.io/mcp with an Alion account has everything. Docs: https://alion.io/developers"}}