{"id":2042444,"url":"https://alion.io/job/wolters-kluwer-senior-product-software-engineer-ai-quality-engineering","title":"Senior Product Software Engineer - AI Quality Engineering","company":{"id":2555,"name":"Wolters Kluwer","domain":"wolterskluwer.com","url":"https://alion.io/company/wolters-kluwer","size_band":"5000+","is_staffing_agency":false,"employer_type":"direct","is_intermediary":false,"listed_via":null,"ats_vendor":"Workday","truth_index":{"grade":"B","score":82,"open_postings":61,"ghost_share":0.016,"stale_share":0.656,"repost_share":0.049,"time_to_fill_p50_days":36,"computed_at":"2026-11-05T05:54:30Z"}},"role":"QA","role_family":"QA","seniority":"senior","employment_type":"full_time","work_mode":"hybrid","remote_scope":null,"remote_scope_basis":null,"remote_working_hours":null,"hiring_geo_confidence":"structured","locations":["Cape Girardeau, United States"],"countries":["US"],"hiring_countries":[],"hiring_countries_total":0,"salary":null,"salary_estimate":{"min_usd":71000,"max_usd":141000,"period":"year","method":"role_seniority_country_remote_cell","sample_n":648},"experience_years_min":5,"visa_sponsorship":false,"relocation_package":false,"has_equity":false,"technologies":[{"name":"A/B Testing","optional":false},{"name":"Agentic Workflows","optional":false},{"name":"Agile","optional":false},{"name":"Arize Phoenix","optional":false},{"name":"Azure","optional":false},{"name":"CI/CD","optional":false},{"name":"Datadog","optional":false},{"name":"Docker","optional":false},{"name":"ElasticSearch","optional":false},{"name":"Embeddings","optional":false},{"name":"Git","optional":false},{"name":"Google AI Studio","optional":false},{"name":"Grafana","optional":false},{"name":"Hallucination","optional":false},{"name":"Human-in-the-Loop","optional":false},{"name":"JMeter","optional":false},{"name":"k6","optional":false},{"name":"Kubernetes","optional":false},{"name":"LangChain","optional":false},{"name":"Langfuse","optional":false},{"name":"LangSmith","optional":false},{"name":"LlamaIndex","optional":false},{"name":"LLM","optional":false},{"name":"LLM Evaluation","optional":false},{"name":"LLMOps","optional":false},{"name":"Locust","optional":false},{"name":"OpenAI","optional":false},{"name":"OpenTelemetry","optional":false},{"name":"Pinecone","optional":false},{"name":"Prometheus","optional":false},{"name":"Pytest","optional":false},{"name":"Python","optional":false},{"name":"RAG","optional":false},{"name":"Rest API","optional":false},{"name":"Semantic Kernel","optional":false},{"name":"TruLens","optional":false},{"name":"Weaviate","optional":false}],"status":"live","first_seen_at":"2026-10-07T18:24:02Z","employer_posted_date":"2026-10-07","last_verified_at":"2026-11-05T21:55:38Z","board_verified":true,"closed_at":null,"days_open":29,"trust":{"level":"ok","repost_count":null,"flags":[],"days_open":29},"description":"We are seeking an experienced Senior AI Quality Engineer to own and implement quality strategy, evaluation systems, and reliability practices for AI-powered product experiences in Wolters Kluwer Tax and Accounting. In this role, you will architect, build, and maintain evaluation systems that measure retrieval accuracy, citation correctness, response quality, safety, performance, and overall alignment with customer intent. You will work hands-on across test automation, AI evaluation, APIs, data pipelines, observability, and release readiness while partnering with software engineers, product managers, UX, and stakeholders to ensure our generative AI systems are trustworthy, observable, secure, and ready for production use in professional tax and accounting workflows.\nThis role goes beyond validating that features function as designed. You will own the practical quality engineering implementation for non-deterministic AI systems: defining how quality is measured, building automated evaluation and regression capabilities, implementing quality gates in delivery pipelines, driving release-readiness decisions, mentoring team members in AI quality practices, and establishing the standards customers can trust.\nKey Responsibilities:\nOwn and implement the quality engineering approach for enterprise-scale generative AI applications, including RAG pipelines, LLM orchestration, agentic workflows, APIs, ingestion pipelines, and AI-driven user experiences.\nArchitect and implement evaluation harnesses that measure retrieval accuracy, citation correctness, answer relevance, groundedness, hallucination rate, safety, latency, cost, and end-to-end system behavior.\nDesign automated regression suites that detect quality drift across prompts, embeddings, chunking strategies, model versions, retrieval configuration, orchestration logic, and application workflows.\nPartner with developers to validate AI system architecture, service integrations, API behavior, data ingestion, observability, resilience, security controls, and production readiness.\nLead root-cause analysis for AI quality failures using logs, traces, evaluation outputs, feedback signals, customer scenarios, and domain-specific acceptance criteria.\nDefine, implement, and maintain measurable quality gates for AI releases, and advise product and engineering leaders on risk, readiness, and tradeoffs across accuracy, reliability, latency, cost, and user trust.\nDevelop test automation in Python and related frameworks for APIs, services, chat workflows, data pipelines, and AI evaluation workflows within CI/CD pipelines.\nBuild and maintain observability practices and dashboards that track model behavior, retrieval performance, operational health, quality trends, and customer-impacting failure modes.\nContribute to responsible AI practices including bias and fairness checks, content safety validation, prompt injection testing, data privacy validation, and compliance-oriented evidence collection.\nCollaborate with product managers, UX, subject matter experts, and stakeholders to translate customer workflows and business requirements into testable AI quality criteria.\nMentor engineers and quality team members in AI evaluation methods, non-deterministic testing strategies, automation design, and production-quality engineering practices.\nStay current with emerging AI quality, LLM evaluation, RAG assessment, and LLMOps practices through research, experimentation, and community engagement.\nKey Requirements:\n5+ years of experience in quality engineering, SDET, software engineering, test automation, reliability engineering, or equivalent engineering roles.\n2+ years of hands-on experience testing, evaluating, or validating AI/ML, LLM, RAG, or other non-deterministic systems in production or production-like environments.\nStrong proficiency with Python test automation frameworks such as Pytest, unittest, or equivalent, including experience building reusable test harnesses and regression suites.\nExperience validating REST APIs, microservices, service integrations, data ingestion pipelines, and end-to-end web or chat-based workflows.\nPractical understanding of RAG systems, vector search, embeddings, prompt behavior, model evaluation, groundedness, citation quality, and AI failure modes.\nExperience defining quality metrics and success criteria where outputs are probabilistic rather than strictly deterministic.\nKnowledge of CI/CD pipelines, Git-based workflows, automated regression testing, release gates, and modern engineering practices.\nExperience with observability, logging, tracing, dashboards, and metrics used to monitor production behavior and diagnose system failures.\nAbility to navigate ambiguity, decompose complex AI behaviors into testable risks, and communicate findings clearly to technical and non-technical stakeholders.\nStrong collaboration skills in Agile environments, with demonstrated ability to influence engineering quality practices across product, design, development, and domain expert teams.\nCommitment to responsible AI quality practices, including trust, transparency, security, privacy, fairness, and customer impact.\nDesired Qualifications:\nExperience with LLM observability, tracing, and evaluation platforms such as LangSmith, Langfuse, TruLens, Arize/Phoenix, or custom evaluation and monitoring tooling.\nExperience testing RAG pipelines using vector databases or search platforms such as Azure AI Search, Pinecone, Weaviate, Elasticsearch, or similar technologies.\nExperience with LLM application frameworks such as LangChain, LlamaIndex, Semantic Kernel, or custom orchestration approaches.\nExperience with Azure OpenAI Service, Azure AI Studio, Azure cloud services, or enterprise AI deployments.\nExperience with containerized environments and cloud-native delivery practices using Docker, Kubernetes, and related tooling.\nExperience with performance, load, or resiliency testing tools such as Locust, k6, JMeter, or equivalent.\nExperience with production system observability tooling such as Azure Application Insights, Grafana, Prometheus, Datadog, OpenTelemetry, or equivalent.\nKnowledge of LLMOps, model monitoring, A/B testing, human-in-the-loop evaluation, or production feedback-loop design.\nExperience validating AI applications that handle sensitive, regulated, or professional-domain data.\nBackground in tax, accounting, legal, financial, or other domains where answer accuracy and explainability are critical.\nOur Interview Practices\nTo maintain a fair and genuine hiring process, we kindly ask that all candidates participate in interviews without the assistance of AI tools or external prompts. Our interview process is designed to assess your individual skills, experiences, and communication style. We value authenticity and want to ensure we’re getting to know you-not a digital assistant. To help maintain this integrity, we ask to remove virtual backgrounds and include in-person interviews in our hiring process. Please note that use of AI-generated responses or third-party support during interviews will be grounds for disqualification from the recruitment process.\nApplicants may be required to appear onsite at a Wolters Kluwer office as part of the recruitment process.\nCompensation:\n$83,400.00 - $145,650.00 USDThis role is eligible for Bonus.Compensation range listed is based on primary location of the position. Actual base salary offer is influenced by a wide array of factors including but not limited to skills, experience and actual hiring location. Your recruiter can share more information about the specific offer for the job location during the hiring process.\nAdditional Information:\nWolters Kluwer offers a wide variety of competitive benefits and programs to help meet your needs and balance your work and personal life, including but not limited to: Medical, Dental, & Vision Plans, 401(k), FSA/HSA, Commuter Benefits, Tuition Assistance Plan, Vacation and Sick Time, and Paid Parental Leave. Full details of our benefits are available upon request.","description_format":"text","description_chars":8023,"description_truncated":false,"requirements":{"experience_years_min":5,"management_years_min":null,"team_size_min":null,"manages_managers":false,"education":null,"security_clearance":false,"languages":[]},"benefits":["Parental leave"],"hiring_locations":[{"name":"United States","iso":"US","kind":"country"}],"hiring_excludes":[],"relocation_offered":false,"industries":["Artificial Intelligence","Professional Services"],"lifecycle":[{"event":"open","at":"2026-10-07T18:24:02Z"}],"visa":[],"liveness":{"score":46,"band":"ok","label":"Likely open","p_open":1,"p_active":0.611,"p_room":0.75,"age_days":28,"expected_fill_days":36,"reasons":["conf:1","stale_co","velocity","win:late","comp:brand"],"computed_at":"2026-11-05T05:54:30Z"},"pay":null,"html_url":"https://alion.io/job/wolters-kluwer-senior-product-software-engineer-ai-quality-engineering","json_url":"https://alion.io/job/wolters-kluwer-senior-product-software-engineer-ai-quality-engineering.json","meta":{"generated_at":"2026-11-05T22:52:57Z","cache_seconds":300,"methodology":"https://alion.io/methodology","terms":"https://alion.io/terms","contact":"https://alion.io/contact","api":"https://alion.io/developers","about":"Alion is a live layer of people, companies and AI agents: who they are, whether they are real and active right now, what they do and how to work with them, readable by people and by agents and paid per call.","catalog":"https://alion.io/catalog.json","usage":{"tier":"crawler_verified","counted_by":"address","units_charged":1,"used_today":3900,"day_limit":null,"remaining_today":null,"minute_limit":300,"resets_at":"2026-11-06T00:00:00Z"}},"offers":[{"id":"company.slices","title":"One company in depth, by slice","status":"live","price":{"credits":0.02,"usd":0.002,"plus_per_slice":{"credits":0.05,"usd":0.005}},"unit":"per company, plus each slice with data","note":"the employer in depth","call":{"mcp_tool":"get_company","arguments":{"id":2555},"rest":"https://alion.io/mcp/rest/get_company?id=2555"},"human":"https://alion.io/catalog?offer=company.slices&for=job%2Fwolters-kluwer-senior-product-software-engineer-ai-quality-engineering"},{"id":"market.stats","title":"A market slice: pay, demand and time to fill","status":"live","price":{"credits":1,"usd":0.1},"unit":"per slice","note":"pay, demand and time to fill for this role and place","call":{"mcp_tool":"market_stats"},"human":"https://alion.io/catalog?offer=market.stats&for=job%2Fwolters-kluwer-senior-product-software-engineer-ai-quality-engineering"},{"id":"job.search","title":"Open jobs by role, technology, place, pay and visa","status":"live","price":{"credits":0.02,"usd":0.002},"unit":"per posting in a list","note":"similar open postings","call":{"mcp_tool":"search_jobs"},"human":"https://alion.io/catalog?offer=job.search&for=job%2Fwolters-kluwer-senior-product-software-engineer-ai-quality-engineering"},{"id":"company.verify","title":"Is this company real and active right now","status":"pilot","price":null,"unit":"per company","request":{"url":"https://alion.io/catalog/request","method":"POST","body":"{\"offer\": \"company.verify\", \"for\": \"job/wolters-kluwer-senior-product-software-engineer-ai-quality-engineering\", \"note\": \"what you need it for\"}"},"human":"https://alion.io/catalog?offer=company.verify&for=job%2Fwolters-kluwer-senior-product-software-engineer-ai-quality-engineering"}],"agent":{"mcp":"https://alion.io/mcp/jobs","mcp_open":"https://alion.io/mcp/open","next":[{"tool":"get_job","arguments":{"id":2042444}},{"tool":"search_jobs","arguments":{"company":"wolterskluwer.com","detail":"compact"}},{"tool":"match_people","arguments":{"job_id":2042444},"note":"people open to offers who fit this vacancy"}],"note":"The same data through MCP tools: https://alion.io/mcp/open needs no sign-in (compact searches, postings, people); https://alion.io/mcp with an Alion account has everything. Docs: https://alion.io/developers"}}