{"id":3151478,"url":"https://alion.io/job/softility-big-data-lead-hadoopapache-spark","title":"Big Data Lead - Hadoop/Apache Spark","company":{"id":4432027,"name":"Softility","domain":"softility.com","url":"https://alion.io/company/softility","size_band":"201-500","is_staffing_agency":false,"employer_type":"direct","is_intermediary":false,"listed_via":null,"ats_vendor":null,"truth_index":null},"role":"Data Science","role_family":"Data Science","seniority":"lead","employment_type":null,"work_mode":"on_site","remote_scope":null,"remote_scope_basis":null,"remote_working_hours":null,"hiring_geo_confidence":"structured","locations":["Hyderabad, India"],"countries":["IN"],"hiring_countries":[],"hiring_countries_total":0,"salary":null,"salary_estimate":{"min_usd":29000,"max_usd":61000,"period":"year","method":"role_seniority_country_remote_cell","sample_n":20},"experience_years_min":8,"visa_sponsorship":false,"relocation_package":false,"has_equity":false,"technologies":[{"name":"Airflow","optional":false},{"name":"Apache Kafka","optional":false},{"name":"AWS","optional":false},{"name":"CI/CD","optional":false},{"name":"Databricks","optional":false},{"name":"GCP","optional":false},{"name":"Git","optional":false},{"name":"Hadoop","optional":false},{"name":"HBase","optional":false},{"name":"Incident Management","optional":false},{"name":"Java","optional":false},{"name":"Python","optional":false},{"name":"Scala","optional":false},{"name":"Spark","optional":false},{"name":"SQL","optional":false}],"status":"live","first_seen_at":"2026-10-23T11:37:39Z","employer_posted_date":null,"last_verified_at":"2026-10-23T11:37:39Z","board_verified":false,"closed_at":null,"days_open":3,"trust":{"level":"not_scored","repost_count":null,"flags":[],"days_open":3},"description":"Role : Big Data Lead\n\nLocation : Hyderabad (Offshore)\n\nExperience : 8 - 10 Years\n\nShift Timings : Night Shift (5 : 00 PM - 2 : 00 AM IST) / US EST Primetime\n\nRole Overview : \n\nWe are looking for a Big Data Lead with 8 - 10 years of experience to design, scale, and manage our high-volume data pipelines and real-time streaming platforms.\n\nIn this role, you will serve as the primary technical point of contact for US clients, lead engineering teams, and ensure data systems run reliably during US overlap hours.\n\nLeadership & Strategy : \n\n- Guide architecture standards and data governance practices.\n\n- Mentor developers, train tech leads, and build a strong engineering culture.\n\nPipeline Development : \n\n- Build and optimize data processing applications using Hadoop, Spark, Kafka, and HBase.\n\n- Manage batch/streaming performance, low-latency querying, and partition strategies.\n\nClient Engagement : \n\n- Act as the technical anchor for client accounts.\n\n- Translate technical performance and architecture plans into clear updates for stakeholders.\n\nIncident Management : \n\n- Lead troubleshooting and root-cause analysis (RCA) during system outages or performance bottlenecks.\n\n- Set up monitoring and automated recovery routines.\n\nRequired Technical & Professional Skills : \n\n- Experience : 8 to 10 years in software engineering and big data development, including experience leading client-facing delivery teams.\n\n- Big Data Stack : Strong hands-on experience with Hadoop (HDFS, MapReduce, YARN), Apache Spark (Core, SQL, Streaming), Apache Kafka, and HBase.\n\n- Architecture : Deep understanding of distributed systems, real-time message streaming, NoSQL schema design (HBase compaction/region tuning), and data lake design.\n\n- Performance Tuning : Proven capability in tuning Spark jobs, memory allocation, execution plans, and Kafka partition throughput.\n\n- Programming : Proficiency in Scala, Java, or Python, along with strong SQL skills.\n\n- CI/CD & Orchestration : Hands-on experience with Git, CI/CD pipelines, and workflow managers like Airflow or Oozie.\n\n- Communication : Clear verbal and written communication with the flexibility to work night shifts and overlap with US hours.\n\nPreferred Certifications : \n\n- Databricks Certified Apache Spark Developer / Data Engineer Professional\n\n- AWS Certified Data Engineer or GCP Professional Data Engineer\n\nSkills\nBig Data, Hadoop, Apache Spark, HBase, Python, Java, SQL, Data Pipeline, Data Governance, Data Strategy, Scala","description_format":"text","description_chars":2482,"description_truncated":false,"requirements":{"experience_years_min":8,"management_years_min":null,"team_size_min":null,"manages_managers":false,"education":null,"security_clearance":false,"languages":[]},"benefits":[],"hiring_locations":[],"hiring_excludes":[],"relocation_offered":false,"industries":["Artificial Intelligence","Information Technology","AI Agents"],"lifecycle":[{"event":"open","at":"2026-10-23T12:29:10Z"}],"visa":[],"liveness":{"score":90,"band":"hot","label":"Hiring now","p_open":1,"p_active":0.903,"p_room":1,"age_days":2,"expected_fill_days":32,"reasons":["seen:2","velocity","win:early"],"computed_at":"2026-10-26T05:57:25Z"},"pay":null,"html_url":"https://alion.io/job/softility-big-data-lead-hadoopapache-spark","json_url":"https://alion.io/job/softility-big-data-lead-hadoopapache-spark.json","meta":{"generated_at":"2026-10-26T12:13:12Z","cache_seconds":300,"methodology":"https://alion.io/methodology","terms":"https://alion.io/terms","contact":"https://alion.io/contact","api":"https://alion.io/developers","about":"Alion is a live layer of people, companies and AI agents: who they are, whether they are real and active right now, what they do and how to work with them, readable by people and by agents and paid per call.","catalog":"https://alion.io/catalog.json","usage":{"tier":"crawler_verified","counted_by":"address","units_charged":1,"used_today":16596,"day_limit":null,"remaining_today":null,"minute_limit":300,"resets_at":"2026-10-27T00:00:00Z"}},"offers":[{"id":"company.slices","title":"One company in depth, by slice","status":"live","price":{"credits":0.02,"usd":0.002,"plus_per_slice":{"credits":0.05,"usd":0.005}},"unit":"per company, plus each slice with data","note":"the employer in depth","call":{"mcp_tool":"get_company","arguments":{"id":4432027},"rest":"https://alion.io/mcp/rest/get_company?id=4432027"},"human":"https://alion.io/catalog?offer=company.slices&for=job%2Fsoftility-big-data-lead-hadoopapache-spark"},{"id":"market.stats","title":"A market slice: pay, demand and time to fill","status":"live","price":{"credits":1,"usd":0.1},"unit":"per slice","note":"pay, demand and time to fill for this role and place","call":{"mcp_tool":"market_stats"},"human":"https://alion.io/catalog?offer=market.stats&for=job%2Fsoftility-big-data-lead-hadoopapache-spark"},{"id":"job.search","title":"Open jobs by role, technology, place, pay and visa","status":"live","price":{"credits":0.02,"usd":0.002},"unit":"per posting in a list","note":"similar open postings","call":{"mcp_tool":"search_jobs"},"human":"https://alion.io/catalog?offer=job.search&for=job%2Fsoftility-big-data-lead-hadoopapache-spark"},{"id":"company.verify","title":"Is this company real and active right now","status":"pilot","price":null,"unit":"per company","request":{"url":"https://alion.io/catalog/request","method":"POST","body":"{\"offer\": \"company.verify\", \"for\": \"job/softility-big-data-lead-hadoopapache-spark\", \"note\": \"what you need it for\"}"},"human":"https://alion.io/catalog?offer=company.verify&for=job%2Fsoftility-big-data-lead-hadoopapache-spark"}]}