{"id":2906381,"url":"https://alion.io/job/numentica-technologies-senior-databricks-engineer","title":"Senior Databricks Engineer","company":{"id":3808063,"name":"Numentica Technologies","domain":"numentica.com","url":"https://alion.io/company/numentica-technologies","size_band":null,"is_staffing_agency":false,"employer_type":"direct","is_intermediary":false,"listed_via":null,"ats_vendor":"Zoho Recruit","truth_index":null},"role":"Data Science","role_family":"Data Science","seniority":"senior","employment_type":"full_time","work_mode":"on_site","remote_scope":null,"remote_scope_basis":null,"remote_working_hours":null,"hiring_geo_confidence":"structured","locations":[],"countries":[],"hiring_countries":[],"hiring_countries_total":0,"salary":null,"salary_estimate":null,"experience_years_min":7,"visa_sponsorship":false,"relocation_package":false,"has_equity":false,"technologies":[{"name":"AWS","optional":false},{"name":"Azure","optional":false},{"name":"CI/CD","optional":false},{"name":"Claude","optional":false},{"name":"Databricks","optional":false},{"name":"Delta Lake","optional":false},{"name":"ETL/ELT","optional":false},{"name":"GCP","optional":false},{"name":"Git","optional":false},{"name":"pySpark","optional":false},{"name":"Python","optional":false},{"name":"Spark","optional":false},{"name":"SQL","optional":false},{"name":"AI Agents","optional":true},{"name":"Apache Kafka","optional":true},{"name":"AWS Glue","optional":true},{"name":"Azure Data Factory","optional":true}],"status":"live","first_seen_at":"2026-10-20T07:56:53Z","employer_posted_date":"2026-10-20","last_verified_at":"2026-10-27T16:33:37Z","board_verified":true,"closed_at":null,"days_open":7,"trust":{"level":"ok","repost_count":null,"flags":[],"days_open":7},"description":"We are looking for a Senior Databricks Engineer with 7+ years of experience in data engineering and strong hands-on expertise in Databricks, Apache Spark, and Claude Agent. The ideal candidate will design and build scalable data pipelines, optimize large-scale data processing, work with cloud-based data platforms, and leverage Claude Agent for AI-powered data engineering and workflow automation.\nKey Responsibilities\nDesign, develop, and maintain scalable data pipelines using Databricks and Apache Spark.\nDevelop and optimize PySpark/Spark jobs for large-scale data processing.\nBuild reliable ETL/ELT pipelines using Databricks workflows, Delta Lake, and SQL.\nImplement data transformations, data quality checks, and performance optimization.\nWork with cloud data platforms such as AWS, Azure, or GCP.\nOptimize Spark jobs, cluster configurations, partitioning, joins, and data processing performance.\nDesign, develop, and integrate Claude Agent-based solutions for AI-powered data engineering, workflow automation, and intelligent data processing.\nCollaborate with data engineers, data scientists, analysts, and business teams to deliver data solutions.\nImplement best practices for CI/CD, version control, monitoring, and production support.\nTroubleshoot data pipeline and performance issues in production environments.\nRequired Skills\n7+ years of experience in Data Engineering.\nStrong hands-on experience with Databricks.\nStrong expertise in Apache Spark / PySpark.\nExperience with Delta Lake, Databricks Workflows, and Spark SQL.\nStrong proficiency in Python and SQL.\nExperience building and optimizing large-scale ETL/ELT pipelines.\nExperience with at least one cloud platform: AWS, Azure, or GCP.\nGood understanding of data warehousing, data modeling, and distributed data processing.\nExperience with Git and CI/CD practices.\nPreferred Skills\nExperience with Unity Catalog and Databricks governance.\nExperience with Azure Data Factory, AWS Glue, or similar orchestration tools.\nExperience with Claude Agent / Claude-based AI agents for data engineering or workflow automation.\nKnowledge of streaming technologies such as Kafka or Spark Structured Streaming.\nExperience with Infrastructure as Code or cloud automation.\nDatabricks certifications are a plus.","description_format":"text","description_chars":2265,"description_truncated":false,"requirements":{"experience_years_min":7,"management_years_min":null,"team_size_min":null,"manages_managers":false,"education":null,"security_clearance":false,"languages":[]},"benefits":[],"hiring_locations":[],"hiring_excludes":[],"relocation_offered":false,"industries":["IT Consulting & Digital Transformation","Custom Software Development","IT Staffing & Staff Augmentation"],"lifecycle":[{"event":"open","at":"2026-10-20T07:56:53Z"}],"visa":[],"liveness":{"score":90,"band":"hot","label":"Hiring now","p_open":1,"p_active":0.903,"p_room":1,"age_days":6,"expected_fill_days":28,"reasons":["conf:2","velocity","win:early"],"computed_at":"2026-10-27T05:46:45Z"},"pay":null,"html_url":"https://alion.io/job/numentica-technologies-senior-databricks-engineer","json_url":"https://alion.io/job/numentica-technologies-senior-databricks-engineer.json","meta":{"generated_at":"2026-10-27T21:01:04Z","cache_seconds":300,"methodology":"https://alion.io/methodology","terms":"https://alion.io/terms","contact":"https://alion.io/contact","api":"https://alion.io/developers","about":"Alion is a live layer of people, companies and AI agents: who they are, whether they are real and active right now, what they do and how to work with them, readable by people and by agents and paid per call.","catalog":"https://alion.io/catalog.json","usage":{"tier":"crawler_verified","counted_by":"address","units_charged":1,"used_today":12117,"day_limit":null,"remaining_today":null,"minute_limit":300,"resets_at":"2026-10-28T00:00:00Z"}},"offers":[{"id":"company.slices","title":"One company in depth, by slice","status":"live","price":{"credits":0.02,"usd":0.002,"plus_per_slice":{"credits":0.05,"usd":0.005}},"unit":"per company, plus each slice with data","note":"the employer in depth","call":{"mcp_tool":"get_company","arguments":{"id":3808063},"rest":"https://alion.io/mcp/rest/get_company?id=3808063"},"human":"https://alion.io/catalog?offer=company.slices&for=job%2Fnumentica-technologies-senior-databricks-engineer"},{"id":"market.stats","title":"A market slice: pay, demand and time to fill","status":"live","price":{"credits":1,"usd":0.1},"unit":"per slice","note":"pay, demand and time to fill for this role and place","call":{"mcp_tool":"market_stats"},"human":"https://alion.io/catalog?offer=market.stats&for=job%2Fnumentica-technologies-senior-databricks-engineer"},{"id":"job.search","title":"Open jobs by role, technology, place, pay and visa","status":"live","price":{"credits":0.02,"usd":0.002},"unit":"per posting in a list","note":"similar open postings","call":{"mcp_tool":"search_jobs"},"human":"https://alion.io/catalog?offer=job.search&for=job%2Fnumentica-technologies-senior-databricks-engineer"},{"id":"company.verify","title":"Is this company real and active right now","status":"pilot","price":null,"unit":"per company","request":{"url":"https://alion.io/catalog/request","method":"POST","body":"{\"offer\": \"company.verify\", \"for\": \"job/numentica-technologies-senior-databricks-engineer\", \"note\": \"what you need it for\"}"},"human":"https://alion.io/catalog?offer=company.verify&for=job%2Fnumentica-technologies-senior-databricks-engineer"}]}