{"id":3664364,"url":"https://alion.io/job/sans-lead-data-engineer-databricks","title":"Lead Data Engineer Databricks","company":{"id":4416827,"name":"Sans","domain":"livesans.com","url":"https://alion.io/company/livesans","size_band":null,"is_staffing_agency":false,"employer_type":"direct","is_intermediary":false,"listed_via":null,"ats_vendor":"Workable","truth_index":null},"role":"Data Science","role_family":"Data Science","seniority":"lead","employment_type":"full_time","work_mode":"hybrid","remote_scope":null,"remote_scope_basis":null,"remote_working_hours":null,"hiring_geo_confidence":"structured","locations":["New York, United States"],"countries":["US"],"hiring_countries":[],"hiring_countries_total":0,"salary":null,"salary_estimate":null,"experience_years_min":6,"visa_sponsorship":false,"relocation_package":false,"has_equity":false,"technologies":[{"name":"CI/CD","optional":false},{"name":"Databricks","optional":false},{"name":"Delta Lake","optional":false},{"name":"Git","optional":false},{"name":"pySpark","optional":false},{"name":"Python","optional":false},{"name":"SQL","optional":false},{"name":"Airflow","optional":true},{"name":"Amazon S3","optional":true},{"name":"AWS","optional":true},{"name":"Spark","optional":true}],"status":"live","first_seen_at":"2026-11-02T21:50:12Z","employer_posted_date":null,"last_verified_at":"2026-11-02T21:50:12Z","board_verified":false,"closed_at":null,"days_open":2,"trust":{"level":"not_scored","repost_count":null,"flags":[],"days_open":2},"description":"Job Title: Lead Data Engineer Databricks Location: New York City, NY | Hybrid (3 days per week onsite)\nIdeal Experience Range: 6-12 Years\nInterview Process: 3 Rounds with One being onsite in NY/NJ\nClient: Media Streaming\nKey requirements for this role:\nHands-on Databricks experience within the last 2 years, with working knowledge of its features\nPrior data engineering experience, including building pipelines and integrating multiple data sources\nKnows data modeling\n Role Overview\nClient is seeking a Lead Data Engineer with deep hands-on expertise in Databricks, SQL, Python, and PySpark to design, develop, and optimize scalable data engineering solutions.\nThis role offers an opportunity to work on enterprise-scale data platforms supporting analytics, digital products, and emerging AI use cases. The candidate will drive Databricks engineering best practices, solve challenging technical problems, and provide technical leadership across multiple initiatives.\nKey Responsibilities\nDesign and develop scalable data pipelines on Databricks, following Medallion Architecture.\nBuild and optimize data ingestion, transformation, and orchestration workflows for batch and incremental processing.\nOptimize data processing workloads, SQL queries, and compute configurations for performance and cost efficiency.\nEnsure data quality, observability, lineage, and reliability across production pipelines, including troubleshooting complex issues.\nProvide hands-on technical leadership through solution design, code reviews, reusable engineering patterns, and mentoring other engineers.\nRequired Qualifications\n6+ years of Data Engineering experience, with substantial hands-on experience delivering production solutions on Databricks.\nStrong proficiency in SQL, Python, and PySpark, including distributed data processing and performance optimization.\nStrong understanding of Databricks Lakehouse architecture, Delta Lake, Unity Catalog, and workflow orchestration.\nFamiliarity with Git, CI/CD, and production deployment practices.\nStrong troubleshooting, problem-solving, and communication skills.\nPreferred Qualifications\nDatabricks Certified Data Engineer Associate or Professional certification.\nExperience working with AWS cloud services, particularly S3 and cloud-based data integrations.\nExperience with advanced ingestion patterns such as Auto Loader, Structured Streaming, or Change Data Capture (CDC).\nFamiliarity with Databricks Asset Bundles or Apache Airflow.\nExperience with large-scale media datasets: streaming, clickstream, or advertising data.\nThanks And Regards,\nAmit Lakhotia","description_format":"text","description_chars":2591,"description_truncated":false,"requirements":{"experience_years_min":6,"management_years_min":null,"team_size_min":null,"manages_managers":false,"education":null,"security_clearance":false,"languages":[]},"benefits":[],"hiring_locations":[],"hiring_excludes":[],"relocation_offered":false,"industries":[],"lifecycle":[{"event":"open","at":"2026-11-03T04:54:05Z"}],"visa":[{"country":"US","licensed_sponsor":true,"evidence":"H-1B filings in 12 months: 5","filings_12m":5,"filings_prev_12m":12,"green_card_filings_12m":0,"median_offered_wage_usd":107500,"route":null,"cap_exempt":false,"checked_at":"2026-10-29T16:27:21+00:00","sources":["US Department of Labor: LCA disclosure data (H-1B, H-1B1, E-3)"],"filings_for_role_12m":0}],"liveness":{"score":90,"band":"hot","label":"Hiring now","p_open":1,"p_active":0.903,"p_room":1,"age_days":1,"expected_fill_days":28,"reasons":["seen:1","velocity","win:early"],"computed_at":"2026-11-04T06:34:25Z"},"pay":null,"html_url":"https://alion.io/job/sans-lead-data-engineer-databricks","json_url":"https://alion.io/job/sans-lead-data-engineer-databricks.json","meta":{"generated_at":"2026-11-04T22:09:02Z","cache_seconds":300,"methodology":"https://alion.io/methodology","terms":"https://alion.io/terms","contact":"https://alion.io/contact","api":"https://alion.io/developers","about":"Alion is a live layer of people, companies and AI agents: who they are, whether they are real and active right now, what they do and how to work with them, readable by people and by agents and paid per call.","catalog":"https://alion.io/catalog.json","usage":{"tier":"crawler_verified","counted_by":"address","units_charged":1,"used_today":304,"day_limit":null,"remaining_today":null,"minute_limit":300,"resets_at":"2026-11-05T00:00:00Z"}},"offers":[{"id":"company.slices","title":"One company in depth, by slice","status":"live","price":{"credits":0.02,"usd":0.002,"plus_per_slice":{"credits":0.05,"usd":0.005}},"unit":"per company, plus each slice with data","note":"the employer in depth","call":{"mcp_tool":"get_company","arguments":{"id":4416827},"rest":"https://alion.io/mcp/rest/get_company?id=4416827"},"human":"https://alion.io/catalog?offer=company.slices&for=job%2Fsans-lead-data-engineer-databricks"},{"id":"market.stats","title":"A market slice: pay, demand and time to fill","status":"live","price":{"credits":1,"usd":0.1},"unit":"per slice","note":"pay, demand and time to fill for this role and place","call":{"mcp_tool":"market_stats"},"human":"https://alion.io/catalog?offer=market.stats&for=job%2Fsans-lead-data-engineer-databricks"},{"id":"job.search","title":"Open jobs by role, technology, place, pay and visa","status":"live","price":{"credits":0.02,"usd":0.002},"unit":"per posting in a list","note":"similar open postings","call":{"mcp_tool":"search_jobs"},"human":"https://alion.io/catalog?offer=job.search&for=job%2Fsans-lead-data-engineer-databricks"},{"id":"company.verify","title":"Is this company real and active right now","status":"pilot","price":null,"unit":"per company","request":{"url":"https://alion.io/catalog/request","method":"POST","body":"{\"offer\": \"company.verify\", \"for\": \"job/sans-lead-data-engineer-databricks\", \"note\": \"what you need it for\"}"},"human":"https://alion.io/catalog?offer=company.verify&for=job%2Fsans-lead-data-engineer-databricks"}],"agent":{"mcp":"https://alion.io/mcp/jobs","mcp_open":"https://alion.io/mcp/open","next":[{"tool":"get_job","arguments":{"id":3664364}},{"tool":"search_jobs","arguments":{"company":"livesans.com","detail":"compact"}},{"tool":"match_people","arguments":{"job_id":3664364},"note":"people open to offers who fit this vacancy"}],"note":"The same data through MCP tools: https://alion.io/mcp/open needs no sign-in (compact searches, postings, people); https://alion.io/mcp with an Alion account has everything. Docs: https://alion.io/developers"}}