{"id":2595739,"url":"https://alion.io/job/rubrik-senior-ai-evals-engineer","title":"Senior AI Evals Engineer","company":{"id":1145,"name":"Rubrik","domain":"rubrik.com","url":"https://alion.io/company/rubrik","size_band":null,"is_staffing_agency":false,"employer_type":"direct","is_intermediary":false,"listed_via":null,"ats_vendor":"Greenhouse","truth_index":{"grade":"A","score":88,"open_postings":23,"ghost_share":0,"stale_share":0.478,"repost_share":0,"time_to_fill_p50_days":45,"computed_at":"2026-10-19T05:45:15Z"}},"role":"AI/ML","role_family":"AI/ML","seniority":"senior","employment_type":null,"work_mode":"on_site","remote_scope":null,"remote_scope_basis":null,"remote_working_hours":null,"hiring_geo_confidence":"structured","locations":["Bengaluru, India"],"countries":["IN"],"hiring_countries":[],"hiring_countries_total":0,"salary":null,"salary_estimate":{"min_usd":31000,"max_usd":68000,"period":"year","method":"role_seniority_country_remote_cell","sample_n":29},"experience_years_min":8,"visa_sponsorship":false,"relocation_package":false,"has_equity":false,"technologies":[{"name":"Agentic Workflows","optional":false},{"name":"AI Agents","optional":false},{"name":"CI/CD","optional":false},{"name":"Datadog","optional":false},{"name":"DeepEval","optional":false},{"name":"Function Calling","optional":false},{"name":"GCP","optional":false},{"name":"Gemini","optional":false},{"name":"Git","optional":false},{"name":"GitHub Actions","optional":false},{"name":"Google ADK","optional":false},{"name":"Grafana","optional":false},{"name":"Hallucination","optional":false},{"name":"Hugging Face","optional":false},{"name":"Jenkins","optional":false},{"name":"Jira","optional":false},{"name":"Kubernetes","optional":false},{"name":"LangSmith","optional":false},{"name":"LLM","optional":false},{"name":"LLM Evaluation","optional":false},{"name":"Prompt Engineering","optional":false},{"name":"Promptfoo","optional":false},{"name":"Pytest","optional":false},{"name":"Python","optional":false},{"name":"RAG","optional":false},{"name":"Tool Use","optional":false},{"name":"Vertex AI","optional":false},{"name":"Agile","optional":true},{"name":"Anthropic","optional":true},{"name":"AWS","optional":true},{"name":"Azure","optional":true},{"name":"Claude","optional":true},{"name":"Fine-tuning","optional":true},{"name":"Human-in-the-Loop","optional":true},{"name":"Java","optional":true},{"name":"LangChain","optional":true},{"name":"LangGraph","optional":true},{"name":"LLM Guardrails","optional":true},{"name":"Machine Learning","optional":true},{"name":"NLP","optional":true},{"name":"OpenAI","optional":true},{"name":"OpenAI Agents SDK","optional":true},{"name":"Pandas","optional":true},{"name":"Playwright","optional":true},{"name":"Scrum","optional":true},{"name":"Selenium","optional":true},{"name":"SQL","optional":true},{"name":"TypeScript","optional":true}],"status":"live","first_seen_at":"2026-10-15T05:43:23Z","employer_posted_date":"2026-10-15","last_verified_at":"2026-10-19T14:13:24Z","board_verified":true,"closed_at":null,"days_open":4,"trust":{"level":"ok","repost_count":null,"flags":[],"days_open":4},"description":"About the Role\nTheSenior AI Evals Test Engineer is a hands-on technical role responsible for defining, developing, and executing robust evaluation frameworks and test strategies for AI agents built using the Google Agent Development Kit (ADK) or similar sort of agentic frameworks. This role bridges AI development and reliable deployment - ensuring our agents are safe, ethical, effective, and meet high-quality performance standards. A significant focus of the role is on Responsible AI and Safety Evals, including adversarial testing, red-teaming, bias detection, and toxicity measurement. The engineer will design scalable evaluation pipelines, build synthetic test environments, and work closely with AI/ML Engineers, Researchers, and policy teams to ensure agents are production-ready and compliant with regulatory and ethical standards. The role is 70% Automation and 30% Manual Testing, and sits within the Applied AI: Test Engineering team as a key technical authority.\nKey Responsibilities\nDesign and build agent evaluation frameworks - create end-to-end infrastructure for evaluating LLM-based agents across correctness, latency, cost, safety, and task completion rate\n Build synthetic testing environments and simulation strategies to stress-test AI agents under diverse real-world conditions, with specific focus on agents built on Google ADK and related Google AI/ML services (e.g., Vertex AI, Gemini models)\n Build golden datasets and test harnesses - curate domain-specific evaluation sets for Rubrik's product verticals (data security, SaaS protection, cyber recovery, compliance)\nEstablish eval-as-a-service - build and enhance centralized evaluation platform that any IT team members can self-serve to test their agents against standardized and custom benchmarks\nAutomate regression and drift detection - implement CI/CD-integrated eval pipelines that catch regressions on every model or prompt change before production deployment\n Define eval taxonomies and rubrics - establish structured scoring criteria for agentic workflows (tool use accuracy, multi-step reasoning, hallucination rate, policy adherence, error recovery)\nInstrument production observability - build real-time monitoring for agent quality in production: success rates, user corrections, fallback triggers, escalation patterns\n Benchmark and select models - run structured model comparisons (cost, quality, latency trade-offs) to inform model selection decisions across agent use cases\nAnalyze and interpret evaluation results, providing clear, actionable quality and safety reports to stakeholders and development teams, with specific focus on risk metrics, trend analysis, and mitigation recommendations\n Define comprehensive end-to-end QA strategies - functional, integration, regression, and user acceptance testing (UAT) - tailored for conversational and goal-oriented AI agents with tool-use, memory, and multi-turn reasoning capabilities\n Develop and maintain detailed test artifacts (test plans, test cases, test scenarios, automated test suites) covering agent features, tool invocation, memory persistence, and reasoning accuracy across evolving model versions\nIdentify, document, prioritize, and track defects, performance regressions, and alignment failures using Jira, collaborating closely with Applied AI Engineers to analyze root causes and validate fixes\n Evaluate new technologies and patterns (e.g., new tools for Agent Evals), run proofs-of-concept, and influence the platform roadmap\nPartner with policy and legal teams to ensure agent evaluations systematically cover regulatory compliance, fairness objectives, and emerging AI governance requirements\nProvide technical leadership to engineers, including solutioning, design reviews, code reviews, pair programming, and mentoring of junior and contract engineers\nCollaborate cross-functionally - work closely with ML engineers, product teams, platform engineers, and security to embed eval-driven quality gates into the agent development lifecycle\nRequired Qualifications\nEducation & Experience\nBachelor's or Master's degree in Computer Science, Information Systems, Engineering, or equivalent practical experience\n 8+ years of overall software Test engineering experience, with 5+ years focused on test engineering, QA automation, or developer tooling, and demonstrated ability to build evaluation systems across multiple AI/ML domains, conversational agents, or Large Language Models (LLMs)\n2+ years working with ML/AI systems, building, testing, or evaluating models in production environments, including hands-on experience with LLM-based applications, prompt engineering, RAG pipelines, or agentic workflows\n2+ years as a senior/lead, owning evaluation strategy, quality frameworks, and technical decisions for complex AI agent systems in an enterprise product environment\n1+ years in designing or operating evaluation frameworks specifically for generative AI / LLM agents, including benchmarking, red-teaming, and production quality monitoring in the AI ADLC model\nTechnical Skills\nDeep hands-on experience with LLM evaluation - familiarity with frameworks and libraries such as LangSmith, DeepEval,LLM as a Judge, Promptfoo, RAGAS, Hugging Face evaluation tooling, custom eval harnesses\n Strong understanding of agentic AI patterns - tool use, function calling, multi-step planning, ReAct/CoT reasoning chains, retrieval-augmented generation (RAG)\n Production experience with eval pipelines - building automated, CI-integrated evaluation infrastructure (not just notebooks)\nProficiency in Python - strong scripting and tooling skills; experience with pytest, async patterns, and data processing at scale\n Familiarity with Google Cloud Platform (GCP) or any Cloud Service relevant to AI/ML (e.g., Vertex AI) and experience integrating testing into MLOps workflows\n Statistical literacy - ability to design experiments, interpret eval metrics with appropriate confidence, and avoid common pitfalls (overfitting to benchmarks, metric gaming)\n Enterprise product sensibility - understanding of how AI quality translates to customer trust in a B2B/enterprise context (security, compliance, reliability SLAs)\nExcellent communication - ability to present eval results and quality trade-offs to engineering leadership, articulate risks and safety findings to non-technical audiences, and drive alignment on ship/no-ship decisions\nHands-on experience with CI/CD pipelines and Git-based workflows, including automated testing, static analysis, deployment automation, and environment configuration for our DevTestOps\nTech Stack:\n Languages: Python(Must), TypeScript, Java (Optional)\nTest Automation:Selenium, Playwright\nLLM Providers: Anthropic Claude, OpenAI, Azure OpenAI, open-source models\n Eval Tooling: DeepEvals(Must), LLM as a Judge (Must), Promptfoo/RAGAS/LangSmith/ Custom harnesses\nAgent Evals Frameworks/libraries: LangGraph, Claude Agent SDK, custom orchestrators\n Infrastructure: DevOps (Basic), Kubernetes, CI/CD (Jenkins/GitHub Actions), observability stacks (Datadog/Grafana)\nPreferred Qualifications\nDeep Learning/NLP Knowledge: Fundamental understanding of how LLMs work, including concepts like RAG, prompting, and the technical challenges related to alignment\n Data Analysis: Experience with data analysis and visualization tools (e.g., Pandas, SQL) for analyzing evaluation and safety results\nPolicy Background: Familiarity with AI ethics principles and industry standards for responsible AI development\n Garbage-in/Garbage-out (GIGO) Tools: Any tool or framework that specializes in testing the model's response to highly ambiguous, misleading, or malformed inputs to test for robustness and safety failures\nExperience testing complex APIs, especially for tool-integration and function-calling scenarios in agentic workflows\n Experience with prompt injection (Must) and data poisoning skills is a plus\n Certifications in AI/ML, cloud platforms (GCP, AWS), AWS Machine Learning Specialty, Google Professional ML Engineer, DeepLearning.AI specializations and software testing methodologies\nSoft Skills\nProven ability to lead without direct authority, influencing cross-functional teams and driving technical decisions through clarity and data rather than hierarchy.\nExperience working in agile environments, collaborating closely with product owners, scrum teams, and distributed stakeholders across time zones.\nExcellent communication and documentation skills, comfortable presenting solutions and trade-offs to engineers, managers, and senior business stakeholders.\nStrong architectural thinking and problem-solving skills, including the ability to decompose complex business processes into simple solutions\nNice to have\nExperience evaluating agents in cybersecurity, data protection, or infrastructure domains\nFamiliarity with Rubrik's product portfolio or similar enterprise data management platforms\nHands-on with human-in-the-loop eval workflows (annotation platforms, inter-rater reliability)\nExperience with model fine-tuning evaluation (pre/post quality comparison)\nBackground in building internal developer platforms or tooling teams\nContributions to open-source eval frameworks or published work on AI evaluation methodology\nDrive red-teaming and adversarial testing - design adversarial evaluation suites for jailbreak resistance, data leakage prevention, and boundary enforcement in enterprise-grade agents\nJoin Us in Securing and Accelerating the World's AI Transformation\nRubrik (RBRK), the Security and AI Operations Company, leads at the intersection of data protection, cyber resilience, and enterprise AI acceleration. Rubrik Security Cloud delivers complete cyber resilience by securing, monitoring, and recovering data, identities, and workloads across clouds. Rubrik Agent Cloud accelerates trusted AI agent deployments at scale by monitoring and auditing agentic actions, enforcing real-time guardrails, fine-tuning for accuracy and undoing agentic mistakes. \nLinkedin | X (formerly Twitter) | Instagram | Rubrik.com\nInclusion @ Rubrik\nAt Rubrik, we are dedicated to fostering a culture where people from all backgrounds are valued, feel they belong, and believe they can succeed. Our commitment to inclusion is at the heart of our mission to secure the world’s data.\nOur goal is to hire and promote the best talent, regardless of background. We continually review our hiring practices to ensure fairness and strive to create an environment where every employee has equal access to opportunities for growth and excellence. We believe in empowering everyone to bring their authentic selves to work and achieve their fullest potential.\nOur inclusion strategy focuses on three core areas of our business and culture:\nOur Company: We are committed to building a merit-based organization that offers equal access to growth and success for all employees globally. Your potential is limitless here.\n\nOur Culture: We strive to create an inclusive atmosphere where individuals from all backgrounds feel a strong sense of belonging, can thrive, and do their best work. Your contributions help us innovate and break boundaries.\n\nOur Communities: We are dedicated to expanding our engagement with the communities we operate in, creating opportunities for underrepresented talent and driving greater innovation for our clients. Your impact extends beyond Rubrik, contributing to safer and stronger communities.\n\nEqual Opportunity Employer/Veterans/Disabled\nRubrik is an Equal Opportunity Employer. All qualified applicants will receive consideration for employment without regard to race, color, religion, sex, sexual orientation, gender identity, national origin, age, disability, genetic information, protected veteran status, or any other status protected by applicable law. Rubrik provides equal employment opportunities to employees and applicants in all terms and conditions of employment, including recruiting, hiring, promotion, compensation, training, leave, and termination.\nRubrik provides reasonable a...","description_format":"text","description_chars":12453,"description_truncated":true,"requirements":{"experience_years_min":8,"management_years_min":null,"team_size_min":null,"manages_managers":false,"education":{"level":"bachelor","optional":false},"security_clearance":false,"languages":[]},"benefits":[],"hiring_locations":[],"hiring_excludes":[],"relocation_offered":false,"industries":["Cloud Security","Data Loss Prevention","Backup & Disaster Recovery"],"lifecycle":[{"event":"open","at":"2026-10-15T07:10:00Z"}],"visa":[],"liveness":{"score":90,"band":"hot","label":"Hiring now","p_open":1,"p_active":0.903,"p_room":1,"age_days":4,"expected_fill_days":45,"reasons":["conf:3","velocity","win:early","comp:brand"],"computed_at":"2026-10-19T05:45:15Z"},"pay":null,"html_url":"https://alion.io/job/rubrik-senior-ai-evals-engineer","json_url":"https://alion.io/job/rubrik-senior-ai-evals-engineer.json","meta":{"generated_at":"2026-10-19T16:38:12Z","cache_seconds":300,"methodology":"https://alion.io/methodology","terms":"https://alion.io/terms","contact":"https://alion.io/contact","api":"https://alion.io/developers","about":"Alion is a live layer of people, companies and AI agents: who they are, whether they are real and active right now, what they do and how to work with them, readable by people and by agents and paid per call.","catalog":"https://alion.io/catalog.json","usage":{"tier":"crawler_verified","counted_by":"address","units_charged":1,"used_today":12614,"day_limit":null,"remaining_today":null,"minute_limit":300,"resets_at":"2026-10-20T00:00:00Z"}},"offers":[{"id":"company.slices","title":"One company in depth, by slice","status":"live","price":{"credits":0.02,"usd":0.002,"plus_per_slice":{"credits":0.05,"usd":0.005}},"unit":"per company, plus each slice with data","note":"the employer in depth","call":{"mcp_tool":"get_company","arguments":{"id":1145},"rest":"https://alion.io/mcp/rest/get_company?id=1145"},"human":"https://alion.io/catalog?offer=company.slices&for=job%2Frubrik-senior-ai-evals-engineer"},{"id":"market.stats","title":"A market slice: pay, demand and time to fill","status":"live","price":{"credits":1,"usd":0.1},"unit":"per slice","note":"pay, demand and time to fill for this role and place","call":{"mcp_tool":"market_stats"},"human":"https://alion.io/catalog?offer=market.stats&for=job%2Frubrik-senior-ai-evals-engineer"},{"id":"job.search","title":"Open jobs by role, technology, place, pay and visa","status":"live","price":{"credits":0.02,"usd":0.002},"unit":"per posting in a list","note":"similar open postings","call":{"mcp_tool":"search_jobs"},"human":"https://alion.io/catalog?offer=job.search&for=job%2Frubrik-senior-ai-evals-engineer"},{"id":"company.verify","title":"Is this company real and active right now","status":"pilot","price":null,"unit":"per company","request":{"url":"https://alion.io/catalog/request","method":"POST","body":"{\"offer\": \"company.verify\", \"for\": \"job/rubrik-senior-ai-evals-engineer\", \"note\": \"what you need it for\"}"},"human":"https://alion.io/catalog?offer=company.verify&for=job%2Frubrik-senior-ai-evals-engineer"}]}