{"id":3146568,"url":"https://alion.io/job/random-trees-senior-data-engineer-pysparkazure-databricks","title":"Senior Data Engineer - PySpark/Azure Databricks","company":{"id":3801016,"name":"Random Trees","domain":"randomtrees.com","url":"https://alion.io/company/random-trees","size_band":"201-500","is_staffing_agency":false,"employer_type":"direct","is_intermediary":false,"listed_via":null,"ats_vendor":null,"truth_index":null},"role":"Data Science","role_family":"Data Science","seniority":"senior","employment_type":null,"work_mode":"on_site","remote_scope":null,"remote_scope_basis":null,"remote_working_hours":null,"hiring_geo_confidence":"structured","locations":["Hyderabad, India"],"countries":["IN"],"hiring_countries":[],"hiring_countries_total":0,"salary":null,"salary_estimate":{"min_usd":19000,"max_usd":42000,"period":"year","method":"role_seniority_country_remote_cell","sample_n":75},"experience_years_min":7,"visa_sponsorship":false,"relocation_package":false,"has_equity":false,"technologies":[{"name":"Azure","optional":false},{"name":"Azure Data Factory","optional":false},{"name":"Azure DevOps","optional":false},{"name":"CI/CD","optional":false},{"name":"Databricks","optional":false},{"name":"Delta Lake","optional":false},{"name":"ETL/ELT","optional":false},{"name":"Git","optional":false},{"name":"pySpark","optional":false},{"name":"Python","optional":false},{"name":"Spark","optional":false},{"name":"SQL","optional":false}],"status":"live","first_seen_at":"2026-10-23T10:17:41Z","employer_posted_date":null,"last_verified_at":"2026-10-23T10:17:41Z","board_verified":false,"closed_at":null,"days_open":5,"trust":{"level":"not_scored","repost_count":null,"flags":[],"days_open":5},"description":"About the Job : \n\nWe are looking for a skilled Senior Data Engineer with 7+ years of experience in designing, building, and maintaining scalable data pipelines.\n\nThe ideal candidate will have strong expertise in PySpark, SQL, Databricks, Azure, and Azure Data Factory (ADF).\n\nYou will be responsible for developing efficient data solutions, optimizing workflows, and ensuring data reliability to support business decisions.\n\nKey Responsibilities : \n\n- Design, develop, and deploy high-performance data pipelines using PySpark, Databricks, and Azure Data Factory (ADF).\n\n- Optimize and maintain SQL queries, stored procedures, and ETL processes for large-scale data processing.\n\n- Work with Azure cloud services (Azure Data Lake, Azure Synapse, Blob Storage, etc.) to build scalable data solutions.\n\n- Implement data integration, transformation, and warehousing solutions in Databricks.\n\n- Collaborate with data scientists, analysts, and business teams to understand data requirements and deliver actionable insights.\n\n- Ensure data quality, governance, and security best practices across all pipelines.\n\n- Troubleshoot and resolve data-related issues in production environments.\n\n- Automate workflows and improve efficiency using CI/CD pipelines.\n\n- Mentor junior engineers and provide technical guidance.\n\nRequired Skills & Qualifications : \n\n- 7+ years of hands-on experience in Data Engineering.\n\n- Strong expertise in PySpark for large-scale data processing.\n\n- Proficiency in SQL (query optimization, stored procedures, performance tuning).\n\n- Hands-on experience with Azure cloud services (ADF, Azure Data Lake, Synapse, Blob Storage, etc.)\n\n- Experience with Databricks (Delta Lake, Spark optimization, notebook workflows).\n\n- Knowledge of data modeling, ETL/ELT processes, and data warehousing.\n\n- Familiarity with Python, Shell scripting, and automation.\n\n- Understanding of CI/CD pipelines and DevOps practices (Git, Azure DevOps).\n\n- Strong problem-solving skills and ability to work in a fast-paced environment.\n\nSkills\nData Engineering, PySpark, Azure Databricks, Azure, Azure Data Factory, SQL, Data Warehousing, ETL, Python, Data Pipeline, Synapse Analytics","description_format":"text","description_chars":2172,"description_truncated":false,"requirements":{"experience_years_min":7,"management_years_min":null,"team_size_min":null,"manages_managers":false,"education":null,"security_clearance":false,"languages":[]},"benefits":[],"hiring_locations":[],"hiring_excludes":[],"relocation_offered":false,"industries":["LLM & Generative AI","AI Consulting & Integration"],"lifecycle":[{"event":"open","at":"2026-10-23T10:30:35Z"}],"visa":[],"liveness":{"score":90,"band":"hot","label":"Hiring now","p_open":1,"p_active":0.903,"p_room":1,"age_days":4,"expected_fill_days":36,"reasons":["seen:4","velocity","win:early"],"computed_at":"2026-10-28T05:59:10Z"},"pay":null,"html_url":"https://alion.io/job/random-trees-senior-data-engineer-pysparkazure-databricks","json_url":"https://alion.io/job/random-trees-senior-data-engineer-pysparkazure-databricks.json","meta":{"generated_at":"2026-10-28T10:53:24Z","cache_seconds":300,"methodology":"https://alion.io/methodology","terms":"https://alion.io/terms","contact":"https://alion.io/contact","api":"https://alion.io/developers","about":"Alion is a live layer of people, companies and AI agents: who they are, whether they are real and active right now, what they do and how to work with them, readable by people and by agents and paid per call.","catalog":"https://alion.io/catalog.json","usage":{"tier":"crawler_verified","counted_by":"address","units_charged":1,"used_today":14224,"day_limit":null,"remaining_today":null,"minute_limit":300,"resets_at":"2026-10-29T00:00:00Z"}},"offers":[{"id":"company.slices","title":"One company in depth, by slice","status":"live","price":{"credits":0.02,"usd":0.002,"plus_per_slice":{"credits":0.05,"usd":0.005}},"unit":"per company, plus each slice with data","note":"the employer in depth","call":{"mcp_tool":"get_company","arguments":{"id":3801016},"rest":"https://alion.io/mcp/rest/get_company?id=3801016"},"human":"https://alion.io/catalog?offer=company.slices&for=job%2Frandom-trees-senior-data-engineer-pysparkazure-databricks"},{"id":"market.stats","title":"A market slice: pay, demand and time to fill","status":"live","price":{"credits":1,"usd":0.1},"unit":"per slice","note":"pay, demand and time to fill for this role and place","call":{"mcp_tool":"market_stats"},"human":"https://alion.io/catalog?offer=market.stats&for=job%2Frandom-trees-senior-data-engineer-pysparkazure-databricks"},{"id":"job.search","title":"Open jobs by role, technology, place, pay and visa","status":"live","price":{"credits":0.02,"usd":0.002},"unit":"per posting in a list","note":"similar open postings","call":{"mcp_tool":"search_jobs"},"human":"https://alion.io/catalog?offer=job.search&for=job%2Frandom-trees-senior-data-engineer-pysparkazure-databricks"},{"id":"company.verify","title":"Is this company real and active right now","status":"pilot","price":null,"unit":"per company","request":{"url":"https://alion.io/catalog/request","method":"POST","body":"{\"offer\": \"company.verify\", \"for\": \"job/random-trees-senior-data-engineer-pysparkazure-databricks\", \"note\": \"what you need it for\"}"},"human":"https://alion.io/catalog?offer=company.verify&for=job%2Frandom-trees-senior-data-engineer-pysparkazure-databricks"}],"agent":{"mcp":"https://alion.io/mcp/jobs","mcp_open":"https://alion.io/mcp/open","next":[{"tool":"get_job","arguments":{"id":3146568}},{"tool":"search_jobs","arguments":{"company":"randomtrees.com","detail":"compact"}},{"tool":"match_people","arguments":{"job_id":3146568},"note":"people open to offers who fit this vacancy"}],"note":"The same data through MCP tools: https://alion.io/mcp/open needs no sign-in (compact searches, postings, people); https://alion.io/mcp with an Alion account has everything. Docs: https://alion.io/developers"}}