{"id":1279248,"url":"https://alion.io/job/ignitivesearch-consulting-python-spark-lead","title":"Python Spark Lead","company":{"id":3809820,"name":"Ignitivesearch Consulting","domain":"ignitivesearch.com","url":"https://alion.io/company/ignitivesearch-consulting","size_band":null,"is_staffing_agency":false,"employer_type":"direct","is_intermediary":false,"listed_via":null,"ats_vendor":null,"truth_index":null},"role":"Backend","role_family":"Backend","seniority":"lead","employment_type":null,"work_mode":"on_site","remote_scope":null,"remote_scope_basis":null,"remote_working_hours":null,"hiring_geo_confidence":"structured","locations":["Chennai, India","Bengaluru, India"],"countries":["IN"],"hiring_countries":[],"hiring_countries_total":0,"salary":null,"salary_estimate":{"min_usd":20000,"max_usd":50000,"period":"year","method":"role_seniority_country_remote_cell","sample_n":74},"experience_years_min":7,"visa_sponsorship":false,"relocation_package":false,"has_equity":false,"technologies":[{"name":"ETL/ELT","optional":false},{"name":"Java","optional":false},{"name":"pySpark","optional":false},{"name":"Python","optional":false},{"name":"Scala","optional":false},{"name":"Spark","optional":false},{"name":"SQL","optional":false}],"status":"live","first_seen_at":"2026-08-25T11:32:55Z","employer_posted_date":null,"last_verified_at":"2026-08-25T11:32:55Z","board_verified":false,"closed_at":null,"days_open":66,"trust":{"level":"not_scored","repost_count":null,"flags":[],"days_open":66},"description":"Job Description :\n\nWe are looking for a strong Python & Spark Lead with hands-on expertise in Python, PySpark, Scala, Apache Spark, and Big Data technologies. The ideal candidate should have strong coding skills and experience in building scalable data solutions, data pipelines, migration accelerators, and high-performance ETL/ELT frameworks. Candidates with experience in product development or Tier-1 organizations and strong engineering fundamentals are preferred.\n\nKey Responsibilities :\n\n- Lead the design and development of scalable, secure, and high-performance data pipelines using Python, PySpark, Scala, and Spark.\n\n- Develop robust ETL/ELT pipelines for large-scale data processing and transformation.\n\n- Design and implement data engineering solutions using Apache Spark and Big Data technologies.\n\n- Develop migration automation frameworks and accelerators to improve data migration efficiency.\n\n- Drive Dataiku-to-Python migration initiatives, including redesign, automation, and optimization of existing workflows.\n\n- Develop reusable frameworks and components for data ingestion, transformation, validation, and processing.\n\n- Work with Java and SQL for data processing, integration, and application development.\n\n- Design and implement data integration solutions across multiple systems and data sources.\n\n- Optimize Spark jobs, pipelines, queries, and applications for performance and scalability.\n\n- Implement appropriate data quality, validation, error-handling, and monitoring mechanisms.\n\n- Collaborate with architecture, product, QA, and engineering teams to deliver high-quality solutions.\n\n- Lead technical design discussions, code reviews, testing, deployment, and production support.\n\n- Mentor engineers and contribute to engineering best practices and coding standards.\n\nRequired Technical Skills :\n\n- 7-10 years of experience in software/data engineering.\n\n- Strong hands-on expertise in Python and PySpark.\n\n- Strong knowledge of Apache Spark and Big Data technologies.\n\n- Experience with Scala and good understanding of Java.\n\n- Strong proficiency in SQL.\n\n- Hands-on experience in ETL/ELT and data pipeline development.\n\n- Experience building scalable and high-performance data processing solutions.\n\n- Strong understanding of data integration, transformation, testing, and optimization.\n\n- Experience developing migration tools, automation frameworks, or accelerators.\nSkills\nData Engineering, Python, Apache Spark, ETL, Big Data, PySpark, Data Pipeline","description_format":"text","description_chars":2488,"description_truncated":false,"requirements":{"experience_years_min":7,"management_years_min":null,"team_size_min":null,"manages_managers":false,"education":null,"security_clearance":false,"languages":[]},"benefits":[],"hiring_locations":[],"hiring_excludes":[],"relocation_offered":false,"industries":[],"lifecycle":[{"event":"open","at":"2026-09-26T02:00:24Z"}],"visa":[],"liveness":{"score":6,"band":"cold","label":"Long shot","p_open":0.4,"p_active":0.511,"p_room":0.28,"age_days":65,"expected_fill_days":28,"reasons":["seen:65","win:tail","crowd:"],"computed_at":"2026-10-30T05:48:23Z"},"pay":null,"html_url":"https://alion.io/job/ignitivesearch-consulting-python-spark-lead","json_url":"https://alion.io/job/ignitivesearch-consulting-python-spark-lead.json","meta":{"generated_at":"2026-10-31T05:23:10Z","cache_seconds":300,"methodology":"https://alion.io/methodology","terms":"https://alion.io/terms","contact":"https://alion.io/contact","api":"https://alion.io/developers","about":"Alion is a live layer of people, companies and AI agents: who they are, whether they are real and active right now, what they do and how to work with them, readable by people and by agents and paid per call.","catalog":"https://alion.io/catalog.json","usage":{"tier":"crawler_verified","counted_by":"address","units_charged":1,"used_today":7762,"day_limit":null,"remaining_today":null,"minute_limit":300,"resets_at":"2026-11-01T00:00:00Z"}},"offers":[{"id":"company.slices","title":"One company in depth, by slice","status":"live","price":{"credits":0.02,"usd":0.002,"plus_per_slice":{"credits":0.05,"usd":0.005}},"unit":"per company, plus each slice with data","note":"the employer in depth","call":{"mcp_tool":"get_company","arguments":{"id":3809820},"rest":"https://alion.io/mcp/rest/get_company?id=3809820"},"human":"https://alion.io/catalog?offer=company.slices&for=job%2Fignitivesearch-consulting-python-spark-lead"},{"id":"market.stats","title":"A market slice: pay, demand and time to fill","status":"live","price":{"credits":1,"usd":0.1},"unit":"per slice","note":"pay, demand and time to fill for this role and place","call":{"mcp_tool":"market_stats"},"human":"https://alion.io/catalog?offer=market.stats&for=job%2Fignitivesearch-consulting-python-spark-lead"},{"id":"job.search","title":"Open jobs by role, technology, place, pay and visa","status":"live","price":{"credits":0.02,"usd":0.002},"unit":"per posting in a list","note":"similar open postings","call":{"mcp_tool":"search_jobs"},"human":"https://alion.io/catalog?offer=job.search&for=job%2Fignitivesearch-consulting-python-spark-lead"},{"id":"company.verify","title":"Is this company real and active right now","status":"pilot","price":null,"unit":"per company","request":{"url":"https://alion.io/catalog/request","method":"POST","body":"{\"offer\": \"company.verify\", \"for\": \"job/ignitivesearch-consulting-python-spark-lead\", \"note\": \"what you need it for\"}"},"human":"https://alion.io/catalog?offer=company.verify&for=job%2Fignitivesearch-consulting-python-spark-lead"}],"agent":{"mcp":"https://alion.io/mcp/jobs","mcp_open":"https://alion.io/mcp/open","next":[{"tool":"get_job","arguments":{"id":1279248}},{"tool":"search_jobs","arguments":{"company":"ignitivesearch.com","detail":"compact"}},{"tool":"match_people","arguments":{"job_id":1279248},"note":"people open to offers who fit this vacancy"}],"note":"The same data through MCP tools: https://alion.io/mcp/open needs no sign-in (compact searches, postings, people); https://alion.io/mcp with an Alion account has everything. Docs: https://alion.io/developers"}}