{"id":2685446,"url":"https://alion.io/job/shyftlabs-senior-data-engineer-databricks","title":"Senior Data Engineer -Databricks","company":{"id":685982,"name":"Shyftlabs","domain":"shyftlabs.io","url":"https://alion.io/company/shyftlabs","size_band":null,"is_staffing_agency":false,"employer_type":"direct","is_intermediary":false,"listed_via":null,"ats_vendor":"Lever","truth_index":{"grade":"C","score":69,"open_postings":23,"ghost_share":0.522,"stale_share":0,"repost_share":0,"time_to_fill_p50_days":36,"computed_at":"2026-10-17T05:46:00Z"}},"role":"Data Science","role_family":"Data Science","seniority":"senior","employment_type":"full_time","work_mode":"hybrid","remote_scope":null,"remote_scope_basis":null,"remote_working_hours":null,"hiring_geo_confidence":"structured","locations":["Coimbatore, India"],"countries":["IN"],"hiring_countries":[],"hiring_countries_total":0,"salary":null,"salary_estimate":{"min_usd":20000,"max_usd":41000,"period":"year","method":"role_seniority_country_remote_cell","sample_n":59},"experience_years_min":4,"visa_sponsorship":false,"relocation_package":false,"has_equity":false,"technologies":[{"name":"Amazon S3","optional":false},{"name":"AWS","optional":false},{"name":"CI/CD","optional":false},{"name":"Databricks","optional":false},{"name":"Delta Lake","optional":false},{"name":"Dimensional Modeling","optional":false},{"name":"ETL/ELT","optional":false},{"name":"Git","optional":false},{"name":"pySpark","optional":false},{"name":"Python","optional":false},{"name":"Rest API","optional":false},{"name":"Spark","optional":false},{"name":"SQL","optional":false},{"name":"Apache Kafka","optional":true},{"name":"dbt","optional":true}],"status":"live","first_seen_at":"2026-10-16T06:07:53Z","employer_posted_date":"2026-10-16","last_verified_at":"2026-10-18T02:57:57Z","board_verified":true,"closed_at":null,"days_open":1,"trust":{"level":"ok","repost_count":0,"flags":["company_stale"],"days_open":1},"description":"Position Overview\nWe are looking for a Data Engineer with hands-on experience in building scalable data pipelines and data engineering solutions on the Databricks Lakehouse Platform. The ideal candidate should have strong expertise in Python, PySpark, SQL, Databricks, AWS, and REST API integrations for data ingestion, managing large volumes of data, and data export\nShyftLabs is a growing data product company that was founded in early 2020 and works primarily with Fortune 500 companies. We deliver digital solutions built to help accelerate the growth of businesses in various industries, by focusing on creating value through innovation.\nJob Responsibilities:\nDesign, develop, and maintain scalable ETL/ELT pipelines using Databricks,\nPySpark, and SQL.\nIntegrate data from multiple sources, including databases, Amazon S3, files, and REST APIs.\nBuild data pipelines with Databricks Unity Catalog.\nImplement business logic, data transformations, and dimensional data models.\nCreate, schedule, monitor, and optimize Databricks Jobs and Workflows.\nDesign and manage Delta Lake tables using Medallion Architecture (Bronze, Silver,Gold).\nEnsure data quality through validations, error handling, logging, and monitoring.\nOptimize Spark workloads for performance, scalability, and reliability.\nCollaborate with cross-functional teams to deliver production-ready data solutions.\nBasic Qualification:\nStrong expertise in Python, PySpark, and Advanced SQL.\nHands-on experience with the Databricks Lakehouse Platform.\nGood understanding of Unity Catalog, Delta Lake, Databricks Workflows/Jobs,\nClusters, Notebooks, Repos, and Medallion Architecture.\nExperience integrating with REST APIs for data ingestion and data export.\nStrong knowledge of ETL/ELT development, batch processing, incremental loading, and data transformation.\nData Governance experience would be an added advantage.\nExperience with data modeling (Star Schema, Snowflake Schema, Fact & Dimension tables, SCD concepts).\nUnderstanding of data warehousing concepts and best practices.\nExperience working with structured and semi-structured data (CSV, JSON, Parquet, Delta).\nKnowledge of partitioning, file optimization, Spark performance tuning, and query optimization.\nExperience with Git and CI/CD best practices\nPreferred Qualifications:\n4+ years of experience in Data Engineering with 2+ years of hands-on Databricks\nexperience. Data Governance experience would be an added advantage.\nExperience with Auto Loader, Spark Declarative pipelines, Kafka, Airflow, or dbt is a plus.\nDatabricks certification is an added advantage.","description_format":"text","description_chars":2587,"description_truncated":false,"requirements":{"experience_years_min":4,"management_years_min":null,"team_size_min":null,"manages_managers":false,"education":null,"security_clearance":false,"languages":[]},"benefits":[],"hiring_locations":[{"name":"India","iso":"IN","kind":"country"}],"hiring_excludes":[],"relocation_offered":false,"industries":["AI Consulting & Integration"],"lifecycle":[{"event":"open","at":"2026-10-16T07:31:58Z"}],"visa":[],"liveness":{"score":60,"band":"ok","label":"Likely open","p_open":1,"p_active":0.602,"p_room":1,"age_days":0,"expected_fill_days":36,"reasons":["conf:0","stale_co","win:early"],"computed_at":"2026-10-17T05:46:00Z"},"pay":null,"html_url":"https://alion.io/job/shyftlabs-senior-data-engineer-databricks","json_url":"https://alion.io/job/shyftlabs-senior-data-engineer-databricks.json","meta":{"generated_at":"2026-10-18T04:28:58Z","cache_seconds":300,"methodology":"https://alion.io/methodology","terms":"https://alion.io/terms","contact":"https://alion.io/contact","api":"https://alion.io/developers","about":"Alion is a live layer of people, companies and AI agents: who they are, whether they are real and active right now, what they do and how to work with them, readable by people and by agents and paid per call.","catalog":"https://alion.io/catalog.json","usage":{"tier":"crawler_verified","counted_by":"address","units_charged":1,"used_today":6681,"day_limit":null,"remaining_today":null,"minute_limit":300,"resets_at":"2026-10-19T00:00:00Z"}},"offers":[{"id":"company.slices","title":"One company in depth, by slice","status":"live","price":{"credits":0.02,"usd":0.002,"plus_per_slice":{"credits":0.05,"usd":0.005}},"unit":"per company, plus each slice with data","note":"the employer in depth","call":{"mcp_tool":"get_company","arguments":{"id":685982},"rest":"https://alion.io/mcp/rest/get_company?id=685982"},"human":"https://alion.io/catalog?offer=company.slices&for=job%2Fshyftlabs-senior-data-engineer-databricks"},{"id":"market.stats","title":"A market slice: pay, demand and time to fill","status":"live","price":{"credits":1,"usd":0.1},"unit":"per slice","note":"pay, demand and time to fill for this role and place","call":{"mcp_tool":"market_stats"},"human":"https://alion.io/catalog?offer=market.stats&for=job%2Fshyftlabs-senior-data-engineer-databricks"},{"id":"job.search","title":"Open jobs by role, technology, place, pay and visa","status":"live","price":{"credits":0.02,"usd":0.002},"unit":"per posting in a list","note":"similar open postings","call":{"mcp_tool":"search_jobs"},"human":"https://alion.io/catalog?offer=job.search&for=job%2Fshyftlabs-senior-data-engineer-databricks"},{"id":"company.verify","title":"Is this company real and active right now","status":"pilot","price":null,"unit":"per company","request":{"url":"https://alion.io/catalog/request","method":"POST","body":"{\"offer\": \"company.verify\", \"for\": \"job/shyftlabs-senior-data-engineer-databricks\", \"note\": \"what you need it for\"}"},"human":"https://alion.io/catalog?offer=company.verify&for=job%2Fshyftlabs-senior-data-engineer-databricks"}]}