{"id":2771312,"url":"https://alion.io/job/value-lane-consulting-ai-quality-evaluation-engineer-llm-architecture","title":"AI Quality & Evaluation Engineer - LLM Architecture","company":{"id":3800676,"name":"Value Lane Consulting","domain":"value-lane.com","url":"https://alion.io/company/value-lane-consulting","size_band":null,"is_staffing_agency":false,"employer_type":"direct","is_intermediary":false,"listed_via":null,"ats_vendor":null,"truth_index":null},"role":"AI/ML","role_family":"AI/ML","seniority":"middle","employment_type":null,"work_mode":"remote","remote_scope":"stated_countries","remote_scope_basis":"inferred_company_offices","remote_working_hours":null,"hiring_geo_confidence":"inferred","locations":["Bengaluru, India"],"countries":["IN"],"hiring_countries":["IN"],"hiring_countries_total":1,"salary":null,"salary_estimate":{"min_usd":19000,"max_usd":40000,"period":"year","method":"role_seniority_country_remote_cell","sample_n":8},"experience_years_min":3,"visa_sponsorship":false,"relocation_package":false,"has_equity":false,"technologies":[{"name":"AI Agents","optional":false},{"name":"CI/CD","optional":false},{"name":"LLM","optional":false},{"name":"Python","optional":false},{"name":"RAG","optional":false}],"status":"live","first_seen_at":"2026-08-19T11:37:15Z","employer_posted_date":null,"last_verified_at":"2026-08-19T11:37:15Z","board_verified":false,"closed_at":null,"days_open":61,"trust":{"level":"not_scored","repost_count":null,"flags":[],"days_open":61},"description":"Role Overview :\n\nAs an AI Quality & Evaluation Engineer, you will be at the forefront of ensuring the reliability, safety, and performance of our next-generation generative AI systems. You will bridge the gap between raw model outputs and production-grade intelligence by designing rigorous evaluation frameworks that measure accuracy, bias, and alignment. Working closely with data scientists, product managers, and software engineers, you will translate complex model behaviors into actionable quality metrics. Your work directly influences the end-user experience, ensuring that our AI solutions remain trustworthy, scalable, and highly effective in solving real-world business challenges.\n\nKey Responsibilities :\n\n- Design and implement automated evaluation pipelines to benchmark LLM performance against human-centric quality standards.\n\n- Develop custom test suites and synthetic datasets to stress-test models, ensuring robustness across diverse edge cases and user scenarios.\n\n- Collaborate with cross-functional teams to define success metrics for AI features, driving continuous improvement in model precision and recall.\n\n- Analyze model failure modes and provide data-driven insights to research teams to accelerate the iterative refinement process.\n\n- Build scalable automation frameworks that integrate seamlessly into our CI/CD pipelines, reducing the time-to-market for high-quality AI deployments.\n\nRequired Skillset :\n\n- Demonstrated proficiency in Python for building evaluation scripts, data processing, and automation testing frameworks.\n\n- Deep understanding of LLM architectures and the nuances of evaluating generative models, including RAG systems and fine-tuned agents.\n\n- Proven ability to translate ambiguous quality requirements into structured, measurable testing protocols.\n\n- Strong analytical mindset with the capacity to communicate complex technical findings to non-technical stakeholders clearly and concisely.\n\n- Experience working in a fast-paced, collaborative environment, with the flexibility to thrive in our Bangalore-based hybrid office setup.\n\n- A degree in Computer Science, Data Science, or a related quantitative field, supported by 3 - 6 years of hands-on experience in AI/ML testing and quality assurance.\n\nSkills\nMachine Learning, Python, LLM, Artificial Intelligence, Generative AI, Agentic AI","description_format":"text","description_chars":2345,"description_truncated":false,"requirements":{"experience_years_min":3,"management_years_min":null,"team_size_min":null,"manages_managers":false,"education":null,"security_clearance":false,"languages":[]},"benefits":[],"hiring_locations":[{"name":"India","iso":"IN","kind":"country"}],"hiring_excludes":[],"relocation_offered":false,"industries":["Recruiting & Staffing"],"lifecycle":[{"event":"open","at":"2026-10-17T04:49:54Z"}],"visa":[],"liveness":{"score":5,"band":"cold","label":"Long shot","p_open":0.4,"p_active":0.491,"p_room":0.28,"age_days":61,"expected_fill_days":28,"reasons":["seen:61","velocity","win:tail","crowd:"],"computed_at":"2026-10-20T05:45:00Z"},"pay":null,"html_url":"https://alion.io/job/value-lane-consulting-ai-quality-evaluation-engineer-llm-architecture","json_url":"https://alion.io/job/value-lane-consulting-ai-quality-evaluation-engineer-llm-architecture.json","meta":{"generated_at":"2026-10-20T10:17:47Z","cache_seconds":300,"methodology":"https://alion.io/methodology","terms":"https://alion.io/terms","contact":"https://alion.io/contact","api":"https://alion.io/developers","about":"Alion is a live layer of people, companies and AI agents: who they are, whether they are real and active right now, what they do and how to work with them, readable by people and by agents and paid per call.","catalog":"https://alion.io/catalog.json","usage":{"tier":"crawler_verified","counted_by":"address","units_charged":1,"used_today":11290,"day_limit":null,"remaining_today":null,"minute_limit":300,"resets_at":"2026-10-21T00:00:00Z"}},"offers":[{"id":"company.slices","title":"One company in depth, by slice","status":"live","price":{"credits":0.02,"usd":0.002,"plus_per_slice":{"credits":0.05,"usd":0.005}},"unit":"per company, plus each slice with data","note":"the employer in depth","call":{"mcp_tool":"get_company","arguments":{"id":3800676},"rest":"https://alion.io/mcp/rest/get_company?id=3800676"},"human":"https://alion.io/catalog?offer=company.slices&for=job%2Fvalue-lane-consulting-ai-quality-evaluation-engineer-llm-architecture"},{"id":"market.stats","title":"A market slice: pay, demand and time to fill","status":"live","price":{"credits":1,"usd":0.1},"unit":"per slice","note":"pay, demand and time to fill for this role and place","call":{"mcp_tool":"market_stats"},"human":"https://alion.io/catalog?offer=market.stats&for=job%2Fvalue-lane-consulting-ai-quality-evaluation-engineer-llm-architecture"},{"id":"job.search","title":"Open jobs by role, technology, place, pay and visa","status":"live","price":{"credits":0.02,"usd":0.002},"unit":"per posting in a list","note":"similar open postings","call":{"mcp_tool":"search_jobs"},"human":"https://alion.io/catalog?offer=job.search&for=job%2Fvalue-lane-consulting-ai-quality-evaluation-engineer-llm-architecture"},{"id":"company.verify","title":"Is this company real and active right now","status":"pilot","price":null,"unit":"per company","request":{"url":"https://alion.io/catalog/request","method":"POST","body":"{\"offer\": \"company.verify\", \"for\": \"job/value-lane-consulting-ai-quality-evaluation-engineer-llm-architecture\", \"note\": \"what you need it for\"}"},"human":"https://alion.io/catalog?offer=company.verify&for=job%2Fvalue-lane-consulting-ai-quality-evaluation-engineer-llm-architecture"}]}