{"id":3155495,"url":"https://alion.io/job/madiff-senior-ai-engineer","title":"Senior AI Engineer","company":{"id":4374669,"name":"Madiff","domain":"madiff.eu","url":"https://alion.io/company/madiff-2","size_band":"51-200","is_staffing_agency":true,"employer_type":"agency","is_intermediary":false,"listed_via":null,"ats_vendor":"Zoho Recruit","truth_index":null},"role":"AI/ML","role_family":"AI/ML","seniority":"senior","employment_type":"full_time","work_mode":"hybrid","remote_scope":null,"remote_scope_basis":null,"remote_working_hours":null,"hiring_geo_confidence":"structured","locations":["Warsaw, Poland"],"countries":["PL"],"hiring_countries":[],"hiring_countries_total":0,"salary":null,"salary_estimate":{"min_usd":66000,"max_usd":110000,"period":"year","method":"role_seniority_country_remote_cell","sample_n":59},"experience_years_min":5,"visa_sponsorship":false,"relocation_package":false,"has_equity":false,"technologies":[{"name":"AI Agents","optional":false},{"name":"Braintrust","optional":false},{"name":"CI/CD","optional":false},{"name":"Claude Code","optional":false},{"name":"Context Engineering","optional":false},{"name":"Copilot","optional":false},{"name":"Cursor","optional":false},{"name":"Docker","optional":false},{"name":"Function Calling","optional":false},{"name":"Langfuse","optional":false},{"name":"LLM","optional":false},{"name":"LLM Guardrails","optional":false},{"name":"Prompt Engineering","optional":false},{"name":"Python","optional":false},{"name":"SQL","optional":false},{"name":"Model Context Protocol","optional":true},{"name":"Recommender Systems","optional":true}],"status":"live","first_seen_at":"2026-10-23T14:20:34Z","employer_posted_date":"2026-10-23","last_verified_at":"2026-10-29T06:39:28Z","board_verified":true,"closed_at":null,"days_open":5,"trust":{"level":"ok","repost_count":null,"flags":[],"days_open":5},"description":"We are looking for a Senior AI Engineer to lead the development and optimisation of an LLM evaluation platform supporting a production AI-powered shopping assistant. The role focuses on building LLM evaluation pipelines, LLM-as-a-judge systems, golden datasets, and automated regression tests using Python, SQL, and LLM observability tools such as Langfuse, Braintrust, or Weave, while analysing real user conversations to identify hallucinations, quality issues, and potential risks. The engineer will take technical ownership of the evaluation platform, collaborate with product and engineering teams, and improve AI agents, prompt and context engineering, guardrails, and monitoring to ensure reliable, measurable, and continuously improving LLM performance.\nResponsibilities\nDevelop and maintain LLM evaluation pipelines\nBuild LLM-as-a-judge solutions to assess AI conversations\nCreate evaluation datasets and golden test sets\nAnalyse real customer conversations and identify recurring problems\nMeasure AI response quality and compare model performance\nBuild automated regression tests for LLM applications\nDevelop monitoring and observability for AI systems\nIdentify hallucinations, incorrect responses and unusual user behaviour\nWork on AI guardrails, adversarial testing and abuse detection\nImprove prompts, context management and agent workflows\nWork closely with product managers and engineering teams\nTake technical ownership of the evaluation platform\nRequirements\nMust have\nMinimum 5 years of software engineering experience\nMinimum 2 years working with LLM systems in production\nStrong Python development\nHands-on experience evaluating LLM applications\nLLM-as-a-judge and evaluation frameworks\nPrompt engineering and context engineering\nExperience with AI agents, tool calling and multi-turn conversations\nLLM observability tools such as Langfuse, Braintrust, Weave or similar\nSQL and experience working with large datasets\nCI/CD, testing, Docker or other container technologies\nCloud infrastructure experience\nExperience using AI development tools such as Cursor, Claude Code or GitHub Copilot\nGood English and Polish\nNice to have\nHuman evaluation and calibration of LLM results\nGolden datasets and regression testing for generative AI\nAgent orchestration and automated workflows\nMCP and agent-to-agent integrations\nAI red-teaming, jailbreak testing and guardrails\nExperience improving LLM applications using production feedback\nE-commerce, search or recommendation systems\nExperience as a Technical Lead or Staff Engineer\nBenefits\nFlexible hybrid mode (1-2 days per week in the office in Warsaw/Kraków)\nSolid, competitive salary\nWork in a multinational environment on international projects\nComprehensive healthcare\nLong-term B2B contract with a stable project pipeline","description_format":"text","description_chars":2784,"description_truncated":false,"requirements":{"experience_years_min":5,"management_years_min":null,"team_size_min":null,"manages_managers":false,"education":null,"security_clearance":false,"languages":[]},"benefits":["Hybrid work"],"hiring_locations":[{"name":"Poland","iso":"PL","kind":"country"}],"hiring_excludes":[],"relocation_offered":false,"industries":["Commerce"],"lifecycle":[{"event":"open","at":"2026-10-23T14:20:34Z"}],"visa":[],"liveness":{"score":54,"band":"ok","label":"Likely open","p_open":1,"p_active":0.542,"p_room":1,"age_days":5,"expected_fill_days":36,"reasons":["conf:3","agency","velocity","win:early","comp:brand"],"computed_at":"2026-10-29T05:49:05Z"},"pay":null,"html_url":"https://alion.io/job/madiff-senior-ai-engineer","json_url":"https://alion.io/job/madiff-senior-ai-engineer.json","meta":{"generated_at":"2026-10-29T09:52:01Z","cache_seconds":300,"methodology":"https://alion.io/methodology","terms":"https://alion.io/terms","contact":"https://alion.io/contact","api":"https://alion.io/developers","about":"Alion is a live layer of people, companies and AI agents: who they are, whether they are real and active right now, what they do and how to work with them, readable by people and by agents and paid per call.","catalog":"https://alion.io/catalog.json","usage":{"tier":"crawler_verified","counted_by":"address","units_charged":1,"used_today":14951,"day_limit":null,"remaining_today":null,"minute_limit":300,"resets_at":"2026-10-30T00:00:00Z"}},"offers":[{"id":"company.slices","title":"One company in depth, by slice","status":"live","price":{"credits":0.02,"usd":0.002,"plus_per_slice":{"credits":0.05,"usd":0.005}},"unit":"per company, plus each slice with data","note":"the employer in depth","call":{"mcp_tool":"get_company","arguments":{"id":4374669},"rest":"https://alion.io/mcp/rest/get_company?id=4374669"},"human":"https://alion.io/catalog?offer=company.slices&for=job%2Fmadiff-senior-ai-engineer"},{"id":"market.stats","title":"A market slice: pay, demand and time to fill","status":"live","price":{"credits":1,"usd":0.1},"unit":"per slice","note":"pay, demand and time to fill for this role and place","call":{"mcp_tool":"market_stats"},"human":"https://alion.io/catalog?offer=market.stats&for=job%2Fmadiff-senior-ai-engineer"},{"id":"job.search","title":"Open jobs by role, technology, place, pay and visa","status":"live","price":{"credits":0.02,"usd":0.002},"unit":"per posting in a list","note":"similar open postings","call":{"mcp_tool":"search_jobs"},"human":"https://alion.io/catalog?offer=job.search&for=job%2Fmadiff-senior-ai-engineer"},{"id":"company.verify","title":"Is this company real and active right now","status":"pilot","price":null,"unit":"per company","request":{"url":"https://alion.io/catalog/request","method":"POST","body":"{\"offer\": \"company.verify\", \"for\": \"job/madiff-senior-ai-engineer\", \"note\": \"what you need it for\"}"},"human":"https://alion.io/catalog?offer=company.verify&for=job%2Fmadiff-senior-ai-engineer"}],"agent":{"mcp":"https://alion.io/mcp/jobs","mcp_open":"https://alion.io/mcp/open","next":[{"tool":"get_job","arguments":{"id":3155495}},{"tool":"search_jobs","arguments":{"company":"madiff.eu","detail":"compact"}},{"tool":"match_people","arguments":{"job_id":3155495},"note":"people open to offers who fit this vacancy"}],"note":"The same data through MCP tools: https://alion.io/mcp/open needs no sign-in (compact searches, postings, people); https://alion.io/mcp with an Alion account has everything. Docs: https://alion.io/developers"}}