{"id":4014245,"url":"https://alion.io/job/shyftlabs-senior-data-engineer-databricks-2","title":"Senior Data Engineer - Databricks","company":{"id":685982,"name":"Shyftlabs","domain":"shyftlabs.io","url":"https://alion.io/company/shyftlabs","size_band":"201-500","is_staffing_agency":false,"employer_type":"direct","is_intermediary":false,"listed_via":null,"ats_vendor":"Lever","truth_index":{"grade":"A","score":87,"open_postings":21,"ghost_share":0,"stale_share":0.333,"repost_share":0,"time_to_fill_p50_days":60,"computed_at":"2026-11-10T06:52:35Z"}},"role":"Data Science","role_family":"Data Science","seniority":"senior","employment_type":"full_time","work_mode":"hybrid","remote_scope":null,"remote_scope_basis":null,"remote_working_hours":null,"hiring_geo_confidence":"structured","locations":["Coimbatore, India"],"countries":["IN"],"hiring_countries":[],"hiring_countries_total":0,"salary":null,"salary_estimate":null,"experience_years_min":7,"visa_sponsorship":false,"relocation_package":false,"has_equity":false,"technologies":[{"name":"Amazon S3","optional":false},{"name":"CI/CD","optional":false},{"name":"Databricks","optional":false},{"name":"Delta Lake","optional":false},{"name":"ETL/ELT","optional":false},{"name":"Git","optional":false},{"name":"pySpark","optional":false},{"name":"Python","optional":false},{"name":"Rest API","optional":false},{"name":"Spark","optional":false},{"name":"SQL","optional":false}],"status":"live","first_seen_at":"2026-11-10T09:16:57Z","employer_posted_date":"2026-11-10","last_verified_at":"2026-11-10T11:56:06Z","board_verified":true,"closed_at":null,"days_open":0,"trust":{"level":"ok","repost_count":null,"flags":[],"days_open":0},"description":"Position Overview\nWe are looking for a Senior Data Engineer with strong hands-on experience in Databricks and a solid understanding of Data Governance concepts. The ideal candidate should have 7+ years of overall experience, including a minimum of 5 years of Databricks experience, covering data engineering and/or Databricks infrastructure, along with knowledge of data governance principles and practices.\nShyftLabs is a growing data product company that was founded in early 2020 and works primarily with Fortune 500 companies. We deliver digital solutions built to help accelerate the growth of businesses in various industries, by focusing on creating value through innovation.\nJob Responsibilities:\nDesign, develop, and maintain scalable ETL/ELT pipelines using Databricks, PySpark, Python, and SQL.\nWork on Databricks infrastructure, including workspace configuration, clusters, compute resources, access controls, and performance optimization.\nApply Data Governance principles, including data quality, data lineage, metadata management, data cataloging, and access controls.\nWork with Unity Catalog to manage data permissions, governance policies, and data assets.\nDevelop and manage Delta Lake tables using Medallion Architecture (Bronze, Silver, and Gold).\nIntegrate data from multiple sources, including databases, Amazon S3, files, and REST APIs.\nCreate, schedule, monitor, and optimize Databricks Jobs and Workflows.\nImplement data validation, error handling, logging, monitoring, and data quality checks.\nOptimize Spark workloads, SQL queries, and data processing pipelines for performance and scalability.\nSupport data discovery, metadata documentation, and governance standards across data platforms.\nCollaborate with cross-functional teams to ensure data accuracy, consistency, security, and compliance.\nTroubleshoot data pipeline and infrastructure issues and recommend improvements to platform reliability.\nContribute to enterprise data management and governance best practices.\nBasic Qualification:\n7+ years of overall experience in Data Engineering or a related data technology role.\nMinimum 5 years of hands-on Databricks experience, including data engineering and/or Databricks infrastructure.\nMandatory understanding of Data Governance concepts, including data quality, metadata management, data lineage, data cataloging, data ownership, and access controls.\nPractical knowledge of data governance principles and their application in data platforms.\nUnderstanding of data governance frameworks, policies, standards, and data management best practices.\nExperience or knowledge of implementing data quality checks, data lineage tracking, metadata documentation, and governed data access.\nStrong expertise in Python, PySpark, and Advanced SQL.\nHands-on experience with the Databricks Lakehouse Platform, Delta Lake, Unity Catalog, Workflows, Jobs, Clusters, and Notebooks.\nExperience developing ETL/ELT pipelines, data transformations, batch processing, and incremental data loading.\nGood understanding of data warehousing and data modeling concepts.\nKnowledge of data security, access management, and governance practices within data platforms.\nExperience with Spark performance tuning, partitioning, file optimization, and query optimization.\nFamiliarity with Git, CI/CD practices, and production deployment processes.\nStrong analytical, problem-solving, and communication skills.","description_format":"text","description_chars":3403,"description_truncated":false,"requirements":{"experience_years_min":7,"management_years_min":null,"team_size_min":null,"manages_managers":false,"education":null,"security_clearance":false,"languages":[]},"benefits":[],"hiring_locations":[{"name":"India","iso":"IN","kind":"country"}],"hiring_excludes":[],"relocation_offered":false,"industries":["AI Consulting & Integration"],"lifecycle":[{"event":"open","at":"2026-11-10T11:56:06Z"}],"visa":[],"liveness":{"score":86,"band":"hot","label":"Hiring now","p_open":1,"p_active":0.86,"p_room":1,"age_days":0,"expected_fill_days":60,"reasons":["conf:2","win:early"],"computed_at":"2026-11-10T14:26:53Z"},"pay":null,"html_url":"https://alion.io/job/shyftlabs-senior-data-engineer-databricks-2","json_url":"https://alion.io/job/shyftlabs-senior-data-engineer-databricks-2.json","meta":{"generated_at":"2026-11-10T14:26:53Z","cache_seconds":300,"methodology":"https://alion.io/methodology","terms":"https://alion.io/terms","contact":"https://alion.io/contact","api":"https://alion.io/developers","about":"Alion is a live layer of people, companies and AI agents: who they are, whether they are real and active right now, what they do and how to work with them, readable by people and by agents and paid per call.","catalog":"https://alion.io/catalog.json","usage":{"tier":"crawler_verified","counted_by":"address","units_charged":1,"used_today":33492,"day_limit":null,"remaining_today":null,"minute_limit":300,"resets_at":"2026-11-11T00:00:00Z"}},"offers":[{"id":"company.slices","title":"One company in depth, by slice","status":"live","price":{"credits":0.1,"usd":0.01,"plus_per_slice":{"credits":0.05,"usd":0.005}},"unit":"per company, plus each slice with data","note":"the employer in depth","call":{"mcp_tool":"get_company","arguments":{"id":685982},"rest":"https://alion.io/mcp/rest/get_company?id=685982"},"human":"https://alion.io/catalog?offer=company.slices&for=job%2Fshyftlabs-senior-data-engineer-databricks-2"},{"id":"market.stats","title":"A market slice: pay, demand and time to fill","status":"live","price":{"credits":2,"usd":0.2},"unit":"per slice","note":"pay, demand and time to fill for this role and place","call":{"mcp_tool":"market_stats"},"human":"https://alion.io/catalog?offer=market.stats&for=job%2Fshyftlabs-senior-data-engineer-databricks-2"},{"id":"job.search","title":"Open jobs by role, technology, place, pay and visa","status":"live","price":{"credits":0.1,"usd":0.01},"unit":"per posting in a list","note":"similar open postings","call":{"mcp_tool":"search_jobs"},"human":"https://alion.io/catalog?offer=job.search&for=job%2Fshyftlabs-senior-data-engineer-databricks-2"},{"id":"company.verify","title":"Is this company real and active right now","status":"pilot","price":null,"unit":"per company","request":{"url":"https://alion.io/catalog/request","method":"POST","body":"{\"offer\": \"company.verify\", \"for\": \"job/shyftlabs-senior-data-engineer-databricks-2\", \"note\": \"what you need it for\"}"},"human":"https://alion.io/catalog?offer=company.verify&for=job%2Fshyftlabs-senior-data-engineer-databricks-2"}],"agent":{"mcp":"https://alion.io/mcp/jobs","mcp_open":"https://alion.io/mcp/open","next":[{"tool":"get_job","arguments":{"id":4014245}},{"tool":"search_jobs","arguments":{"company":"shyftlabs.io","detail":"compact"}},{"tool":"match_people","arguments":{"job_id":4014245},"note":"people open to offers who fit this vacancy"}],"note":"The same data through MCP tools: https://alion.io/mcp/open needs no sign-in (compact searches, postings, people); https://alion.io/mcp with an Alion account has everything. Docs: https://alion.io/developers"}}