{"id":586569,"url":"https://alion.io/job/logixhealth-senior-data-engineer","title":"Senior Data Engineer","company":{"id":454473,"name":"LogixHealth","domain":"logixhealth.com","url":"https://alion.io/company/logixhealth","size_band":null,"is_staffing_agency":false,"employer_type":"direct","is_intermediary":false,"listed_via":null,"ats_vendor":"Rippling","truth_index":null},"role":"Data Science","role_family":"Data Science","seniority":"senior","employment_type":null,"work_mode":"on_site","remote_scope":null,"remote_scope_basis":null,"remote_working_hours":null,"hiring_geo_confidence":"structured","locations":["Bengaluru, India"],"countries":["IN"],"hiring_countries":[],"hiring_countries_total":0,"salary":null,"salary_estimate":{"min_usd":21000,"max_usd":46000,"period":"year","method":"role_seniority_country_remote_cell","sample_n":75},"experience_years_min":3,"visa_sponsorship":false,"relocation_package":false,"has_equity":false,"technologies":[{"name":"Azure","optional":false},{"name":"Azure Data Factory","optional":false},{"name":"Azure DevOps","optional":false},{"name":"CI/CD","optional":false},{"name":"Databricks","optional":false},{"name":"Datadog","optional":false},{"name":"Delta Lake","optional":false},{"name":"ETL/ELT","optional":false},{"name":"Git","optional":false},{"name":"GitHub Actions","optional":false},{"name":"Jenkins","optional":false},{"name":"Power BI","optional":false},{"name":"pySpark","optional":false},{"name":"Python","optional":false},{"name":"Scala","optional":false},{"name":"Spark","optional":false},{"name":"SQL","optional":false},{"name":"Tableau","optional":false},{"name":"Terraform","optional":false},{"name":"Airflow","optional":true},{"name":"MS SQL","optional":true},{"name":"MySQL","optional":true},{"name":"PostgreSQL","optional":true}],"status":"live","first_seen_at":"2026-09-10T05:06:18Z","employer_posted_date":null,"last_verified_at":"2026-09-10T05:06:18Z","board_verified":false,"closed_at":null,"days_open":49,"trust":{"level":"not_scored","repost_count":null,"flags":[],"days_open":49},"description":"We're looking for a Senior Data Engineer with 3+ years of hands-on experience in Apache Spark and Databricks to design and build scalable data pipelines and distributed data processing solutions on Azure. If you're strong in PySpark, Spark optimization, Delta Lake, Databricks, and modern data engineering, and enjoy working with large-scale data, this role is for you.\nResponsibilities:\nDesign, develop, and maintain high-performance batch and streaming data pipelines using Databricks, Apache Spark, PySpark, SQL, Python, and Delta Lake.\nBuild scalable data solutions within the Azure ecosystem, leveraging services such as Azure Blob Storage, Azure Data Factory, and Event Hubs.\nImplement Medallion Architecture (Bronze/Silver/Gold) and robust ETL/ELT frameworks.\nDevelop incremental processing and CDC pipelines for large-scale datasets.\nDesign and optimize Spark workloads handling TB+ datasets.\nApply Spark performance optimization techniques, including partitioning and repartitioning, efficient joins and broadcast joins, caching and persistence, adaptive query execution (AQE), data skew handling, Query and job optimization.\nBuild and manage Delta Lake solutions, including ACID transactions, schema evolution, optimization, and reliable data pipelines.\nDevelop Delta Live Tables (DLT) pipelines with strong data quality and pipeline design practices.\nUse Databricks Jobs and Workflows for production-grade orchestration.\nImplement data governance, access control, security, and lineage using Unity Catalog and RBAC.\nEstablish strong practices around data quality, testing, monitoring, maintainability, and reusability.\nIntegrate the data platform with Power BI, Tableau, APIs, and custom applications.\nContribute to CI/CD, infrastructure-as-code, automated testing, monitoring, and alerting using tools such as Git, Terraform, Azure DevOps, GitHub Actions, Jenkins, Azure Monitor, or Datadog.\nWork closely with engineering, product, and business teams to deliver reliable data solutions that support reporting and analytics.\nContribute to the development of a self-service data platform that enables teams to discover, access, and use trusted data efficiently.\nRequirements:\n3+ years of strong hands-on experience with Apache Spark and Databricks.\nExpert-level knowledge of PySpark / Spark, with Python or Scala.\nStrong understanding of Spark DataFrames, Spark SQL, Structured Streaming, Distributed Data Processing, and Large-Scale Data Engineering.\nDeep Databricks experience across Delta Lake, Delta Live Tables (DLT), Unity Catalog, Databricks Jobs & Workflows, and Databricks Notebooks.\nProven experience designing and tuning high-performance Spark jobs.\nHands-on experience working with large-scale datasets, preferably TB+.\nStrong understanding of batch and streaming architectures.\nExperience implementing Medallion Architecture.\nPractical experience with incremental processing and CDC patterns.\nStrong SQL and data engineering fundamentals.\nExperience working with Azure cloud services, particularly Azure Blob Storage.\nGood to Have:\nExperience integrating Databricks with Azure Data Factory.\nExperience with Azure Event Hubs and streaming pipelines.\nExperience with external orchestration tools such as Apache Airflow.\nExperience with NoSQL databases.\nExperience with MS SQL, PostgreSQL, or MySQL.\nKnowledge of data governance, security, compliance, RBAC, and data lineage.\nExperience with Terraform / Infrastructure as Code.\nExperience with Azure DevOps, GitHub Actions, or Jenkins.\nExperience with Azure Monitor or Datadog.\nExposure to Power BI, Tableau, APIs, or custom applications.\nYou'll be expected to:\nBuild for scale: Design Spark pipelines that perform reliably across TB+ datasets.\nBuild for reliability: Create pipelines with strong data quality, testing, monitoring, and failure-handling practices.\nBuild for change: Handle schema evolution, incremental loads, CDC, and continuously changing data sources.\nBuild for governance: Use Unity Catalog, RBAC, lineage, and security best practices to make data discoverable and trustworthy.\nBuild for reuse: Create maintainable frameworks and components that can be used across multiple data products.\nBuild for production: Follow CI/CD, Infrastructure as Code, deployment, observability, and operational best practices.\nTechnology Stack:\nCloud: Azure.\nData Platform: Databricks.\nProcessing: Apache Spark, PySpark, Spark SQL, Structured Streaming.\nStorage: Delta Lake, Azure Blob Storage.\nPipelines: DLT, Databricks Jobs, and Workflows.\nGovernance: Unity Catalog, RBAC, Data Lineage.\nIntegration: Azure Data Factory, Event Hubs, Airflow.\nDatabases: SQL Server, PostgreSQL, MySQL, NoSQL.\nDevOps: Git, Terraform, Azure DevOps, GitHub Actions, Jenkins.\nMonitoring: Azure Monitor, Datadog.\nAnalytics: Power BI, Tableau.","description_format":"text","description_chars":4792,"description_truncated":false,"requirements":{"experience_years_min":3,"management_years_min":null,"team_size_min":null,"manages_managers":false,"education":null,"security_clearance":false,"languages":[]},"benefits":[],"hiring_locations":[],"hiring_excludes":[],"relocation_offered":false,"industries":["Blockchain & Crypto"],"lifecycle":[{"event":"open","at":"2026-09-10T05:06:18Z"}],"visa":[],"liveness":{"score":11,"band":"cold","label":"Long shot","p_open":0.4,"p_active":0.589,"p_room":0.45,"age_days":49,"expected_fill_days":28,"reasons":["seen:49","win:tail"],"computed_at":"2026-10-29T05:49:05Z"},"pay":null,"html_url":"https://alion.io/job/logixhealth-senior-data-engineer","json_url":"https://alion.io/job/logixhealth-senior-data-engineer.json","meta":{"generated_at":"2026-10-30T01:19:31Z","cache_seconds":300,"methodology":"https://alion.io/methodology","terms":"https://alion.io/terms","contact":"https://alion.io/contact","api":"https://alion.io/developers","about":"Alion is a live layer of people, companies and AI agents: who they are, whether they are real and active right now, what they do and how to work with them, readable by people and by agents and paid per call.","catalog":"https://alion.io/catalog.json","usage":{"tier":"crawler_verified","counted_by":"address","units_charged":1,"used_today":1811,"day_limit":null,"remaining_today":null,"minute_limit":300,"resets_at":"2026-10-31T00:00:00Z"}},"offers":[{"id":"company.slices","title":"One company in depth, by slice","status":"live","price":{"credits":0.02,"usd":0.002,"plus_per_slice":{"credits":0.05,"usd":0.005}},"unit":"per company, plus each slice with data","note":"the employer in depth","call":{"mcp_tool":"get_company","arguments":{"id":454473},"rest":"https://alion.io/mcp/rest/get_company?id=454473"},"human":"https://alion.io/catalog?offer=company.slices&for=job%2Flogixhealth-senior-data-engineer"},{"id":"market.stats","title":"A market slice: pay, demand and time to fill","status":"live","price":{"credits":1,"usd":0.1},"unit":"per slice","note":"pay, demand and time to fill for this role and place","call":{"mcp_tool":"market_stats"},"human":"https://alion.io/catalog?offer=market.stats&for=job%2Flogixhealth-senior-data-engineer"},{"id":"job.search","title":"Open jobs by role, technology, place, pay and visa","status":"live","price":{"credits":0.02,"usd":0.002},"unit":"per posting in a list","note":"similar open postings","call":{"mcp_tool":"search_jobs"},"human":"https://alion.io/catalog?offer=job.search&for=job%2Flogixhealth-senior-data-engineer"},{"id":"company.verify","title":"Is this company real and active right now","status":"pilot","price":null,"unit":"per company","request":{"url":"https://alion.io/catalog/request","method":"POST","body":"{\"offer\": \"company.verify\", \"for\": \"job/logixhealth-senior-data-engineer\", \"note\": \"what you need it for\"}"},"human":"https://alion.io/catalog?offer=company.verify&for=job%2Flogixhealth-senior-data-engineer"}],"agent":{"mcp":"https://alion.io/mcp/jobs","mcp_open":"https://alion.io/mcp/open","next":[{"tool":"get_job","arguments":{"id":586569}},{"tool":"search_jobs","arguments":{"company":"logixhealth.com","detail":"compact"}},{"tool":"match_people","arguments":{"job_id":586569},"note":"people open to offers who fit this vacancy"}],"note":"The same data through MCP tools: https://alion.io/mcp/open needs no sign-in (compact searches, postings, people); https://alion.io/mcp with an Alion account has everything. Docs: https://alion.io/developers"}}