{"id":3939140,"url":"https://alion.io/job/exl-lead-databricks-data-engineer","title":"Lead Databricks Data Engineer","company":{"id":38016,"name":"EXL","domain":"exlservice.com","url":"https://alion.io/company/exl","size_band":"5000+","is_staffing_agency":false,"employer_type":"services","is_intermediary":false,"listed_via":null,"ats_vendor":"Oracle","truth_index":{"grade":"B","score":76,"open_postings":252,"ghost_share":0,"stale_share":0.972,"repost_share":0,"time_to_fill_p50_days":26,"computed_at":"2026-11-08T05:47:12Z"}},"role":"Data Science","role_family":"Data Science","seniority":"lead","employment_type":null,"work_mode":"on_site","remote_scope":null,"remote_scope_basis":null,"remote_working_hours":null,"hiring_geo_confidence":"structured","locations":["New York, United States"],"countries":["US"],"hiring_countries":[],"hiring_countries_total":0,"salary":null,"salary_estimate":null,"experience_years_min":12,"visa_sponsorship":false,"relocation_package":false,"has_equity":false,"technologies":[{"name":"Amazon Redshift","optional":false},{"name":"Amazon S3","optional":false},{"name":"Apache Kafka","optional":false},{"name":"AWS","optional":false},{"name":"AWS Lambda","optional":false},{"name":"Azure","optional":false},{"name":"Azure Data Factory","optional":false},{"name":"Azure DevOps","optional":false},{"name":"CI/CD","optional":false},{"name":"Data Vault","optional":false},{"name":"Databricks","optional":false},{"name":"Delta Lake","optional":false},{"name":"Dimensional Modeling","optional":false},{"name":"Docker","optional":false},{"name":"ETL/ELT","optional":false},{"name":"GCP","optional":false},{"name":"Git","optional":false},{"name":"GitHub Actions","optional":false},{"name":"Google BigQuery","optional":false},{"name":"IAM","optional":false},{"name":"Jenkins","optional":false},{"name":"Kubernetes","optional":false},{"name":"Master Data Management","optional":false},{"name":"Platform Engineering","optional":false},{"name":"pySpark","optional":false},{"name":"Python","optional":false},{"name":"Scala","optional":false},{"name":"Snowflake","optional":false},{"name":"Spark","optional":false},{"name":"SQL","optional":false},{"name":"Terraform","optional":false},{"name":"Alation","optional":true},{"name":"Collibra","optional":true},{"name":"Great Expectations","optional":true},{"name":"Informatica","optional":true},{"name":"Microsoft Fabric","optional":true}],"status":"live","first_seen_at":"2026-11-05T23:03:24Z","employer_posted_date":"2026-11-05","last_verified_at":"2026-11-08T11:48:47Z","board_verified":true,"closed_at":null,"days_open":2,"trust":{"level":"ok","repost_count":null,"flags":[],"days_open":2},"description":"Job Description: Lead Databricks Data Engineer\nJob Title: Lead Databricks Data Engineer\nExperience: 10-12 Years\nEmployment Type: Full-Time\nJob Summary\nWe are looking for a highly experienced Lead Databricks Data Engineer with 10-12 years of experience in Data Engineering, Big Data, and Cloud Data Platforms. The ideal candidate will lead the design, development, modernization, and optimization of enterprise-scale data platforms using Databricks Lakehouse architecture. This role requires strong technical leadership, solution architecture expertise, stakeholder management skills, and hands-on experience building scalable cloud-native data ecosystems.\nThe candidate will be responsible for defining data engineering standards, mentoring teams, driving modernization initiatives, and delivering high-performance data solutions that support analytics, AI/ML, and business intelligence initiatives.\nKey Responsibilities Data Engineering & Architecture\nDesign and implement enterprise-scale data platforms using Databricks Lakehouse architecture.\nArchitect and develop scalable ETL/ELT data pipelines for batch and real-time processing.\nDefine enterprise data models, ingestion frameworks, and transformation strategies.\nLead the implementation of Delta Lake, Unity Catalog, and data governance frameworks.\nBuild and optimize highly scalable Spark and PySpark solutions handling large-volume datasets.\nDrive cloud data platform modernization and migration initiatives.\nTechnical Leadership\nServe as the technical lead for Databricks and Data Engineering programs.\nEstablish engineering best practices, coding standards, and architecture guidelines.\nConduct architecture reviews, code reviews, and solution assessments.\nMentor and guide Data Engineers, Technical Leads, and junior team members.\nCollaborate with Enterprise Architects, Product Owners, Data Scientists, and Business Stakeholders.\nCloud & Platform Engineering\nDesign cloud-native data solutions on Azure, AWS, or GCP.\nImplement secure, scalable, and resilient data architectures.\nOptimize costs, performance, reliability, and availability of data platforms.\nLead CI/CD and Infrastructure-as-Code implementation for data workloads.\nDefine and implement monitoring, observability, and operational excellence practices.\nGovernance & Data Quality\nEstablish data governance, lineage, cataloging, and security controls.\nEnsure compliance with organizational and regulatory requirements.\nImplement data quality frameworks and monitoring solutions.\nCollaborate with security and compliance teams to enforce data protection standards.\nStakeholder & Delivery Management\nPartner with business leaders to understand strategic data requirements.\nTranslate business requirements into scalable technical solutions.\nDrive delivery planning, risk management, and resource coordination.\nPresent architecture decisions and technical recommendations to leadership teams.\nRequired Technical Skills Databricks\nDatabricks Lakehouse Platform\nDatabricks Workflows\nDelta Lake\nUnity Catalog\nStructured Streaming\nDatabricks SQL\nDelta Live Tables (DLT)\nDatabricks Asset Bundles\nBig Data Technologies\nApache Spark\nPySpark\nSpark SQL\nKafka\nReal-Time Streaming Architectures\nData Lake & Lakehouse Architectures\nCloud Platforms (Strong Experience in One or More) Azure\nAzure Databricks\nAzure Data Factory (ADF)\nADLS Gen2\nAzure Synapse Analytics\nAzure Key Vault\nEvent Hub\nAWS\nS3\nEMR\nGlue\nRedshift\nLambda\nIAM\nGCP\nBigQuery\nCloud Storage\nDataflow\nPub/Sub\nComposer\nProgramming\nPython\nSQL\nPySpark\nScala (Preferred)\nShell Scripting\nDevOps & Automation\nGit\nAzure DevOps\nGitHub Actions\nJenkins\nTerraform\nDocker\nKubernetes\nData Warehousing\nDimensional Modeling\nStar & Snowflake Schema\nData Vault\nMaster Data Management\nData Governance Frameworks\nRequired Experience\n12-15 years of overall IT experience.\nMinimum 6+ years of hands-on Databricks experience.\n7+ years of experience in Spark/PySpark-based data engineering.\nExperience designing enterprise-scale Data Lake and Lakehouse solutions.\nStrong experience leading large data transformation and modernization programs.\nExperience handling structured, semi-structured, and unstructured data processing.\nExperience supporting analytics, AI/ML, and reporting ecosystems.\nProven experience managing technical teams and driving architectural decisions.\nPreferred Qualifications\nBachelor's or Master's Degree in Computer Science, Engineering, Information Technology, or related field.\nDatabricks Certified Data Engineer Professional.\nAzure Data Engineer Associate.\nAWS Data Analytics Specialty.\nGoogle Professional Data Engineer Certification.\nTOGAF or Cloud Architecture certifications are a plus.\nNice to Have\nMLOps and AI Engineering exposure.\nSnowflake and Databricks integration experience.\nMicrosoft Fabric exposure.\nData Mesh implementation experience.\nData Governance tools such as Collibra, Alation, or Informatica.\nExpertise in Data Quality frameworks such as Great Expectations or Deequ.\nKey Competencies\nSolution Architecture\nTechnical Leadership\nStakeholder Management\nProgram Delivery\nData Strategy & Governance\nTeam Mentoring & Coaching\nProblem Solving & Decision Making\nCommunication & Executive Presentation Skills\nPerformance Optimization\nInnovation & Continuous Improvement\nTechnology Classification\nTechnology Tower: Data & AI\nPractice Area: Data Engineering\nPrimary Skill: Databricks Lakehouse Engineering\nSecondary Skills: Apache Spark, PySpark, Cloud Data Platforms (Azure/AWS/GCP), Data Architecture, Data Governance, MLOps.\n Management of DB Systems, Performance Tuning, Security, Compliance Standards, Graduate with 4 years BS or Professional Degree. 15 - 18 Years","description_format":"text","description_chars":5657,"description_truncated":false,"requirements":{"experience_years_min":12,"management_years_min":null,"team_size_min":null,"manages_managers":false,"education":{"level":"bachelor","optional":true},"security_clearance":false,"languages":[]},"benefits":[],"hiring_locations":[],"hiring_excludes":[],"relocation_offered":false,"industries":["Business Process Outsourcing (BPO)","AI Consulting & Integration","Analytics & BI Consulting"],"lifecycle":[{"event":"open","at":"2026-11-08T11:48:47Z"}],"visa":[{"country":"US","licensed_sponsor":true,"adverts_offering_sponsorship_24m":0,"sanctioned_now":false,"evidence":"H-1B filings in 12 months: 281 · green card filings: 17","filings_12m":281,"filings_prev_12m":395,"green_card_filings_12m":17,"median_offered_wage_usd":128641,"route":null,"cap_exempt":false,"checked_at":"2026-11-08T10:45:15+00:00","sources":["US Department of Labor: LCA disclosure data (H-1B, H-1B1, E-3)","US Department of Labor: PERM disclosure data (green cards)"],"filings_for_role_12m":125}],"liveness":{"score":86,"band":"hot","label":"Hiring now","p_open":1,"p_active":0.86,"p_room":1,"age_days":2,"expected_fill_days":26,"reasons":["conf:2","win:early","comp:brand"],"computed_at":"2026-11-08T14:22:04Z"},"pay":null,"html_url":"https://alion.io/job/exl-lead-databricks-data-engineer","json_url":"https://alion.io/job/exl-lead-databricks-data-engineer.json","meta":{"generated_at":"2026-11-08T14:22:04Z","cache_seconds":300,"methodology":"https://alion.io/methodology","terms":"https://alion.io/terms","contact":"https://alion.io/contact","api":"https://alion.io/developers","about":"Alion is a live layer of people, companies and AI agents: who they are, whether they are real and active right now, what they do and how to work with them, readable by people and by agents and paid per call.","catalog":"https://alion.io/catalog.json","usage":{"tier":"crawler_verified","counted_by":"address","units_charged":1,"used_today":30393,"day_limit":null,"remaining_today":null,"minute_limit":300,"resets_at":"2026-11-09T00:00:00Z"}},"offers":[{"id":"company.slices","title":"One company in depth, by slice","status":"live","price":{"credits":0.1,"usd":0.01,"plus_per_slice":{"credits":0.05,"usd":0.005}},"unit":"per company, plus each slice with data","note":"the employer in depth","call":{"mcp_tool":"get_company","arguments":{"id":38016},"rest":"https://alion.io/mcp/rest/get_company?id=38016"},"human":"https://alion.io/catalog?offer=company.slices&for=job%2Fexl-lead-databricks-data-engineer"},{"id":"market.stats","title":"A market slice: pay, demand and time to fill","status":"live","price":{"credits":2,"usd":0.2},"unit":"per slice","note":"pay, demand and time to fill for this role and place","call":{"mcp_tool":"market_stats"},"human":"https://alion.io/catalog?offer=market.stats&for=job%2Fexl-lead-databricks-data-engineer"},{"id":"job.search","title":"Open jobs by role, technology, place, pay and visa","status":"live","price":{"credits":0.1,"usd":0.01},"unit":"per posting in a list","note":"similar open postings","call":{"mcp_tool":"search_jobs"},"human":"https://alion.io/catalog?offer=job.search&for=job%2Fexl-lead-databricks-data-engineer"},{"id":"company.verify","title":"Is this company real and active right now","status":"pilot","price":null,"unit":"per company","request":{"url":"https://alion.io/catalog/request","method":"POST","body":"{\"offer\": \"company.verify\", \"for\": \"job/exl-lead-databricks-data-engineer\", \"note\": \"what you need it for\"}"},"human":"https://alion.io/catalog?offer=company.verify&for=job%2Fexl-lead-databricks-data-engineer"}],"agent":{"mcp":"https://alion.io/mcp/jobs","mcp_open":"https://alion.io/mcp/open","next":[{"tool":"get_job","arguments":{"id":3939140}},{"tool":"search_jobs","arguments":{"company":"exlservice.com","detail":"compact"}},{"tool":"match_people","arguments":{"job_id":3939140},"note":"people open to offers who fit this vacancy"}],"note":"The same data through MCP tools: https://alion.io/mcp/open needs no sign-in (compact searches, postings, people); https://alion.io/mcp with an Alion account has everything. Docs: https://alion.io/developers"}}