{"id":1221147,"url":"https://alion.io/job/birlasoft-data-engg-with-databricks-pyspark-sr-technical-lead-data-engg","title":"Data Engg with Databricks+ PySpark -Sr Technical Lead-Data Engg","company":{"id":59310,"name":"Birlasoft","domain":"birlasoft.com","url":"https://alion.io/company/birlasoft","size_band":"1001-5000","is_staffing_agency":false,"employer_type":"direct","is_intermediary":false,"listed_via":null,"ats_vendor":"SuccessFactors","truth_index":{"grade":"A","score":100,"open_postings":10,"ghost_share":0,"stale_share":0,"repost_share":0,"time_to_fill_p50_days":19,"computed_at":"2026-10-27T05:46:45Z"}},"role":"Data Science","role_family":"Data Science","seniority":"lead","employment_type":null,"work_mode":"on_site","remote_scope":null,"remote_scope_basis":null,"remote_working_hours":null,"hiring_geo_confidence":"structured","locations":["Pune, India"],"countries":["IN"],"hiring_countries":[],"hiring_countries_total":0,"salary":null,"salary_estimate":{"min_usd":27000,"max_usd":59000,"period":"year","method":"role_seniority_country_remote_cell","sample_n":20},"experience_years_min":null,"visa_sponsorship":false,"relocation_package":false,"has_equity":false,"technologies":[{"name":"Agile","optional":false},{"name":"Azure","optional":false},{"name":"Azure Data Factory","optional":false},{"name":"CI/CD","optional":false},{"name":"Databricks","optional":false},{"name":"Delta Lake","optional":false},{"name":"ETL/ELT","optional":false},{"name":"pySpark","optional":false},{"name":"Scrum","optional":false},{"name":"SQL","optional":false},{"name":"Python","optional":true},{"name":"Spark","optional":true}],"status":"live","first_seen_at":"2026-09-18T02:00:00Z","employer_posted_date":"2026-09-18","last_verified_at":"2026-10-28T03:03:36Z","board_verified":true,"closed_at":null,"days_open":40,"trust":{"level":"ok","repost_count":null,"flags":[],"days_open":40},"description":"Area(s) of responsibility\nRole Summary\nWe are seeking a seasoned Senior Azure Data Engineer / Sr. Technical Lead with extensive expertise in Azure Databricks, PySpark, ADF, SQL, and modern data engineering practices. This role requires strong technical leadership, hands on engineering capabilities, and the ability to design, architect, and deliver enterprise-scale data solutions. The ideal candidate will lead complex data initiatives, mentor engineering teams, and collaborate with cross-functional stakeholders to build a robust and scalable data ecosystem on Azure.\nKey Responsibilities\n1. Solution Architecture & Technical Leadership\nLead the design, development, and deployment of large-scale data engineering solutions using Azure Databricks, PySpark, SQL, ADF, and Azure Data Lake.\nArchitect end-to-end Modern Data Warehouse (MDW) and Lakehouse solutions, ensuring scalability, performance, security, and cost optimization.\nDefine technical standards, coding best practices, reusable frameworks, and architectural guidelines for engineering teams.\nProvide technical leadership across the project lifecycle-requirements analysis, solution blueprinting, estimation, development, and deployment.\n2. Data Pipeline Engineering\nBuild, optimize, and maintain scalable, high performance ELT/ETL pipelines to process large volumes of structured and unstructured data.\nSet up complex data ingestion frameworks, enabling seamless integration with on-premise systems, cloud services, APIs, and third-party sources.\nEnsure high availability, data reliability, and error-resilient orchestration workflows in Azure Data Factory.\n3. Azure Databricks & PySpark Expertise\nDesign and implement advanced transformation logic using PySpark on Databricks, ensuring efficient data processing and code modularity.\nUtilize Delta Lake capabilities-ACID transactions, schema evolution, versioning, time travel-to manage enterprise-grade datasets.\nPerform cluster-level tuning, optimization of shuffle operations, caching, partitioning, and job parallelization.\nManage Databricks job pipelines, notebooks, clusters, job scheduling, and integration with CI/CD pipelines.\n4. Data Governance, Quality & Documentation\nImplement data quality frameworks covering validation, reconciliation, error handling, and metadata management.\nEnforce best practices for security, access control, encryption, data lineage, and auditability.\nPrepare and maintain detailed documentation: technical specification documents, interface designs, architecture diagrams, and operational runbooks.\n5. Stakeholder Collaboration & Team Leadership\nCollaborate with BI, analytics, business teams, and architects to convert business requirements into scalable technical solutions.\nLead code reviews, provide mentoring and technical guidance to junior and mid-level engineers.\nWork closely with Scrum Masters and Product Owners within an Agile delivery model.","description_format":"text","description_chars":2908,"description_truncated":false,"requirements":{"experience_years_min":null,"management_years_min":null,"team_size_min":null,"manages_managers":false,"education":{"level":"master","optional":false},"security_clearance":false,"languages":[]},"benefits":[],"hiring_locations":[],"hiring_excludes":[],"relocation_offered":false,"industries":["IT Consulting & Digital Transformation","Enterprise Apps Implementation Partners","Cloud Consulting & Migration"],"lifecycle":[{"event":"open","at":"2026-09-25T12:00:09Z"}],"visa":[],"liveness":{"score":24,"band":"cold","label":"Long shot","p_open":1,"p_active":0.672,"p_room":0.35,"age_days":39,"expected_fill_days":19,"reasons":["conf:0","velocity","win:tail","comp:brand"],"computed_at":"2026-10-27T05:46:45Z"},"pay":null,"html_url":"https://alion.io/job/birlasoft-data-engg-with-databricks-pyspark-sr-technical-lead-data-engg","json_url":"https://alion.io/job/birlasoft-data-engg-with-databricks-pyspark-sr-technical-lead-data-engg.json","meta":{"generated_at":"2026-10-28T03:34:12Z","cache_seconds":300,"methodology":"https://alion.io/methodology","terms":"https://alion.io/terms","contact":"https://alion.io/contact","api":"https://alion.io/developers","about":"Alion is a live layer of people, companies and AI agents: who they are, whether they are real and active right now, what they do and how to work with them, readable by people and by agents and paid per call.","catalog":"https://alion.io/catalog.json","usage":{"tier":"crawler_verified","counted_by":"address","units_charged":1,"used_today":4709,"day_limit":null,"remaining_today":null,"minute_limit":300,"resets_at":"2026-10-29T00:00:00Z"}},"offers":[{"id":"company.slices","title":"One company in depth, by slice","status":"live","price":{"credits":0.02,"usd":0.002,"plus_per_slice":{"credits":0.05,"usd":0.005}},"unit":"per company, plus each slice with data","note":"the employer in depth","call":{"mcp_tool":"get_company","arguments":{"id":59310},"rest":"https://alion.io/mcp/rest/get_company?id=59310"},"human":"https://alion.io/catalog?offer=company.slices&for=job%2Fbirlasoft-data-engg-with-databricks-pyspark-sr-technical-lead-data-engg"},{"id":"market.stats","title":"A market slice: pay, demand and time to fill","status":"live","price":{"credits":1,"usd":0.1},"unit":"per slice","note":"pay, demand and time to fill for this role and place","call":{"mcp_tool":"market_stats"},"human":"https://alion.io/catalog?offer=market.stats&for=job%2Fbirlasoft-data-engg-with-databricks-pyspark-sr-technical-lead-data-engg"},{"id":"job.search","title":"Open jobs by role, technology, place, pay and visa","status":"live","price":{"credits":0.02,"usd":0.002},"unit":"per posting in a list","note":"similar open postings","call":{"mcp_tool":"search_jobs"},"human":"https://alion.io/catalog?offer=job.search&for=job%2Fbirlasoft-data-engg-with-databricks-pyspark-sr-technical-lead-data-engg"},{"id":"company.verify","title":"Is this company real and active right now","status":"pilot","price":null,"unit":"per company","request":{"url":"https://alion.io/catalog/request","method":"POST","body":"{\"offer\": \"company.verify\", \"for\": \"job/birlasoft-data-engg-with-databricks-pyspark-sr-technical-lead-data-engg\", \"note\": \"what you need it for\"}"},"human":"https://alion.io/catalog?offer=company.verify&for=job%2Fbirlasoft-data-engg-with-databricks-pyspark-sr-technical-lead-data-engg"}]}