{"id":3061121,"url":"https://alion.io/job/yoda-technologies-databricks-consultant-pythonpyspark","title":"Databricks Consultant - Python/PySpark","company":{"id":3809823,"name":"Yoda Technologies","domain":"yoda-tech.com","url":"https://alion.io/company/yoda-technologies","size_band":null,"is_staffing_agency":false,"employer_type":"direct","is_intermediary":false,"listed_via":null,"ats_vendor":null,"truth_index":null},"role":"Data Science","role_family":"Data Science","seniority":"senior","employment_type":null,"work_mode":"on_site","remote_scope":null,"remote_scope_basis":null,"remote_working_hours":null,"hiring_geo_confidence":"structured","locations":["India"],"countries":["IN"],"hiring_countries":[],"hiring_countries_total":0,"salary":null,"salary_estimate":{"min_usd":19500,"max_usd":42000,"period":"year","method":"role_seniority_country_remote_cell","sample_n":75},"experience_years_min":8,"visa_sponsorship":false,"relocation_package":false,"has_equity":false,"technologies":[{"name":"AWS","optional":false},{"name":"Azure","optional":false},{"name":"Databricks","optional":false},{"name":"Delta Lake","optional":false},{"name":"ETL/ELT","optional":false},{"name":"GCP","optional":false},{"name":"pySpark","optional":false},{"name":"Python","optional":false},{"name":"Spark","optional":false},{"name":"SQL","optional":false},{"name":"Apache Kafka","optional":true},{"name":"AWS Glue","optional":true},{"name":"Azure Data Factory","optional":true},{"name":"CI/CD","optional":true}],"status":"live","first_seen_at":"2026-10-22T06:13:56Z","employer_posted_date":null,"last_verified_at":"2026-10-22T06:13:56Z","board_verified":false,"closed_at":null,"days_open":6,"trust":{"level":"not_scored","repost_count":null,"flags":[],"days_open":6},"description":"Yoda Tech is looking for an experienced Databricks Consultant to design, develop, and optimize scalable data engineering solutions using the Databricks platform.\n\nThe ideal candidate will have strong expertise in data engineering, Apache Spark, Python, SQL, and cloud-based data architectures.\n\nThe candidate will work with business stakeholders, data architects, and engineering teams to build reliable data pipelines, improve data processing performance, and deliver high-quality data solutions.\n\nKey Responsibilities :\n\n- Design, develop, and maintain scalable data pipelines using Databricks and Apache Spark.\n\n- Build and optimize ETL/ELT workflows for processing structured and semi-structured data.\n\n- Develop data transformation logic using Python, PySpark, and SQL.\n\n- Implement data ingestion, cleansing, transformation, and integration processes.\n\n- Work with the Lakehouse architecture and Delta Lake for reliable data storage and processing.\n\n- Optimize Spark jobs, cluster configurations, and data workflows to improve performance and cost efficiency.\n\n- Develop and manage batch and, where required, streaming data pipelines.\n\n- Implement data quality checks, error handling, logging, and monitoring mechanisms.\n\n- Work with cloud-based data services and integrate Databricks with existing data platforms.\n\n- Follow data security, access control, governance, and compliance best practices.\n\n- Collaborate with architects, analysts, and cross-functional teams to understand requirements and deliver solutions.\n\n- Troubleshoot production issues and ensure the reliability and scalability of data pipelines.\n\n- Participate in technical discussions, code reviews, solution design, and documentation.\n\nRequired Skills & Qualifications :\n\n- 8 - 11 years of experience in data engineering, data development, or related roles.\n\n- Strong hands-on experience with Databricks and its data engineering capabilities.\n\n- Proficiency in Python, PySpark, and SQL.\n\n- Strong understanding of Apache Spark, distributed data processing, and performance optimization.\n\n- Experience designing and implementing ETL/ELT pipelines.\n\n- Knowledge of Delta Lake, lakehouse architecture, and data lake concepts.\n\n- Experience working with cloud-based data platforms such as AWS, Microsoft Azure, or Google Cloud.\n\n- Understanding of data modelling, data integration, and data quality management.\n\n- Experience with version control, deployment processes, and data engineering best practices.\n\n- Strong analytical, troubleshooting, and problem-solving skills.\n\n- Excellent communication and collaboration skills.\n\nPreferred Skills :\n\n- Experience with Databricks Workflows and Delta Live Tables/Lakeflow capabilities.\n\n- Familiarity with Azure Data Factory, AWS Glue, or comparable data integration tools.\n\n- Experience with CI/CD practices for data engineering solutions.\n\n- Knowledge of data governance, cataloguing, and access management.\n\n- Exposure to streaming technologies such as Kafka or Structured Streaming.\n\n- Experience handling large-scale enterprise data platforms and complex data transformations.\n\nIdeal Candidate Profile :\n\n- The ideal candidate will be an experienced data engineering professional with strong hands-on Databricks expertise, capable of designing scalable data pipelines, optimizing distributed processing, and delivering reliable cloud-based data solutions.\nSkills\nData Engineering, Databricks, Data Pipeline, Apache Spark, Python, SQL, PySpark, ETL, AWS, Azure Data Factory","description_format":"text","description_chars":3494,"description_truncated":false,"requirements":{"experience_years_min":8,"management_years_min":null,"team_size_min":null,"manages_managers":false,"education":null,"security_clearance":false,"languages":[]},"benefits":[],"hiring_locations":[],"hiring_excludes":[],"relocation_offered":false,"industries":["Workflow & Process Automation (RPA, BPM)","Robotic Software & Control Systems"],"lifecycle":[{"event":"open","at":"2026-10-22T06:34:35Z"}],"visa":[],"liveness":{"score":90,"band":"hot","label":"Hiring now","p_open":1,"p_active":0.903,"p_room":1,"age_days":5,"expected_fill_days":28,"reasons":["seen:5","velocity","win:early"],"computed_at":"2026-10-28T05:59:10Z"},"pay":null,"html_url":"https://alion.io/job/yoda-technologies-databricks-consultant-pythonpyspark","json_url":"https://alion.io/job/yoda-technologies-databricks-consultant-pythonpyspark.json","meta":{"generated_at":"2026-10-28T10:36:37Z","cache_seconds":300,"methodology":"https://alion.io/methodology","terms":"https://alion.io/terms","contact":"https://alion.io/contact","api":"https://alion.io/developers","about":"Alion is a live layer of people, companies and AI agents: who they are, whether they are real and active right now, what they do and how to work with them, readable by people and by agents and paid per call.","catalog":"https://alion.io/catalog.json","usage":{"tier":"crawler_verified","counted_by":"address","units_charged":1,"used_today":13974,"day_limit":null,"remaining_today":null,"minute_limit":300,"resets_at":"2026-10-29T00:00:00Z"}},"offers":[{"id":"company.slices","title":"One company in depth, by slice","status":"live","price":{"credits":0.02,"usd":0.002,"plus_per_slice":{"credits":0.05,"usd":0.005}},"unit":"per company, plus each slice with data","note":"the employer in depth","call":{"mcp_tool":"get_company","arguments":{"id":3809823},"rest":"https://alion.io/mcp/rest/get_company?id=3809823"},"human":"https://alion.io/catalog?offer=company.slices&for=job%2Fyoda-technologies-databricks-consultant-pythonpyspark"},{"id":"market.stats","title":"A market slice: pay, demand and time to fill","status":"live","price":{"credits":1,"usd":0.1},"unit":"per slice","note":"pay, demand and time to fill for this role and place","call":{"mcp_tool":"market_stats"},"human":"https://alion.io/catalog?offer=market.stats&for=job%2Fyoda-technologies-databricks-consultant-pythonpyspark"},{"id":"job.search","title":"Open jobs by role, technology, place, pay and visa","status":"live","price":{"credits":0.02,"usd":0.002},"unit":"per posting in a list","note":"similar open postings","call":{"mcp_tool":"search_jobs"},"human":"https://alion.io/catalog?offer=job.search&for=job%2Fyoda-technologies-databricks-consultant-pythonpyspark"},{"id":"company.verify","title":"Is this company real and active right now","status":"pilot","price":null,"unit":"per company","request":{"url":"https://alion.io/catalog/request","method":"POST","body":"{\"offer\": \"company.verify\", \"for\": \"job/yoda-technologies-databricks-consultant-pythonpyspark\", \"note\": \"what you need it for\"}"},"human":"https://alion.io/catalog?offer=company.verify&for=job%2Fyoda-technologies-databricks-consultant-pythonpyspark"}],"agent":{"mcp":"https://alion.io/mcp/jobs","mcp_open":"https://alion.io/mcp/open","next":[{"tool":"get_job","arguments":{"id":3061121}},{"tool":"search_jobs","arguments":{"company":"yoda-tech.com","detail":"compact"}},{"tool":"match_people","arguments":{"job_id":3061121},"note":"people open to offers who fit this vacancy"}],"note":"The same data through MCP tools: https://alion.io/mcp/open needs no sign-in (compact searches, postings, people); https://alion.io/mcp with an Alion account has everything. Docs: https://alion.io/developers"}}