{"id":3204471,"url":"https://alion.io/job/epsilon-lead-data-engineer","title":"Lead Data Engineer","company":{"id":410,"name":"Epsilon","domain":"epsilon.com","url":"https://alion.io/company/epsilon","size_band":"5000+","is_staffing_agency":false,"employer_type":"direct","is_intermediary":false,"listed_via":null,"ats_vendor":null,"truth_index":null},"role":"Data Science","role_family":"Data Science","seniority":"lead","employment_type":null,"work_mode":"on_site","remote_scope":null,"remote_scope_basis":null,"remote_working_hours":null,"hiring_geo_confidence":"structured","locations":["Bengaluru, India"],"countries":["IN"],"hiring_countries":[],"hiring_countries_total":0,"salary":null,"salary_estimate":{"min_usd":31000,"max_usd":67000,"period":"year","method":"role_seniority_country_remote_cell","sample_n":20},"experience_years_min":8,"visa_sponsorship":false,"relocation_package":false,"has_equity":false,"technologies":[{"name":"Agile","optional":false},{"name":"AWS","optional":false},{"name":"Databricks","optional":false},{"name":"Delta Lake","optional":false},{"name":"Docker","optional":false},{"name":"ElasticSearch","optional":false},{"name":"GCP","optional":false},{"name":"Grafana","optional":false},{"name":"Hadoop","optional":false},{"name":"Kibana","optional":false},{"name":"Kubernetes","optional":false},{"name":"Logstash","optional":false},{"name":"Power BI","optional":false},{"name":"Python","optional":false},{"name":"Scala","optional":false},{"name":"Scrum","optional":false},{"name":"Spark","optional":false},{"name":"SQL","optional":false},{"name":"Tableau","optional":false}],"status":"live","first_seen_at":"2026-10-24T11:36:30Z","employer_posted_date":null,"last_verified_at":"2026-10-24T11:36:30Z","board_verified":false,"closed_at":null,"days_open":2,"trust":{"level":"not_scored","repost_count":null,"flags":[],"days_open":2},"description":"As a Lead Engineer in the Epsilon Attribution/Forecasting Product Development team, you will play a key role in implementing and optimizing advanced data processing solutions using Scala, Spark, and Hadoop. You will collaborate with multi-functional teams to deploy scalable big data solutions on our on-premises and cloud infrastructure. Your responsibilities will include building, scheduling, and maintaining complex workflows, as well as performing data integration and transformation tasks. You will troubleshoot issues, document processes, and communicate technical concepts clearly to both technical and non-technical partners. Additionally, you will focus on continuously enhancing our attribution and forecasting engines, ensuring they effectively meet evolving business needs.\n\nRequirements: \nStrong written and verbal communication skills (in English) are required to facilitate work across multiple countries and time zones. Good understanding of Agile Methodologies - SCRUM.\nOver 8 years of strong experience in Scala programming and extensive use of Apache Spark for developing and maintaining scalable big data solutions on both on-premises and cloud environments, particularly AWS and GCP.\nProficient in performance tuning of Spark jobs, optimizing resource usage, shuffling, partitioning, and caching for maximum efficiency.\nSkilled in implementing scalable, fault-tolerant data pipelines with comprehensive monitoring and alerting.\nHands-on experience with Python for developing infrastructure modules.\nDeep understanding of the Hadoop ecosystem, including HDFS, YARN, and MapReduce.\nProficient in writing efficient SQL queries for handling large volumes of data in various database systems.\nExperienced in building, scheduling, and maintaining DAG workflows.\nFamiliar with data warehousing concepts and technologies.\nCapable of taking end-to-end ownership in defining, developing, and documenting software objectives and requirements in collaboration with stakeholders.\nExperienced with GIT or equivalent source control systems.\nProficient in developing and implementing unit test cases to ensure code quality and reliability and experienced in utilizing integration testing frameworks to validate system interactions.\nEffective collaborator with stakeholders and teams to understand requirements and develop solutions.\nAbility to work within tight deadlines, prioritize tasks effectively, and perform under pressure.\nExperience in mentoring junior staff.\n\nAdvantageous to have experience in the below: \nHands-on with Databricks for unified data analytics, including Databricks Notebooks, Delta Lake, and Catalogs.\nProficiency in using the ELK (Elasticsearch, Logstash, Kibana) stack for real-time search, log analysis, and visualization.\nStrong background in analytics, including the ability to derive actionable insights from large datasets and support data-driven decision-making.\nExperience with data visualization tools like Tableau, Power BI, or Grafana.\nFamiliarity with Docker for containerization and Kubernetes for orchestration.\nSkills\napache spark, hadoop, hdfs, mapreduce, scala, spark, yarn","description_format":"text","description_chars":3123,"description_truncated":false,"requirements":{"experience_years_min":8,"management_years_min":null,"team_size_min":null,"manages_managers":false,"education":null,"security_clearance":false,"languages":[]},"benefits":[],"hiring_locations":[],"hiring_excludes":[],"relocation_offered":false,"industries":["Marketing Automation","Email Marketing"],"lifecycle":[{"event":"open","at":"2026-10-24T11:36:30Z"}],"visa":[],"liveness":{"score":90,"band":"hot","label":"Hiring now","p_open":1,"p_active":0.903,"p_room":1,"age_days":2,"expected_fill_days":24,"reasons":["seen:2","velocity","win:early"],"computed_at":"2026-10-27T05:46:45Z"},"pay":null,"html_url":"https://alion.io/job/epsilon-lead-data-engineer","json_url":"https://alion.io/job/epsilon-lead-data-engineer.json","meta":{"generated_at":"2026-10-27T07:10:13Z","cache_seconds":300,"methodology":"https://alion.io/methodology","terms":"https://alion.io/terms","contact":"https://alion.io/contact","api":"https://alion.io/developers","about":"Alion is a live layer of people, companies and AI agents: who they are, whether they are real and active right now, what they do and how to work with them, readable by people and by agents and paid per call.","catalog":"https://alion.io/catalog.json","usage":{"tier":"crawler_verified","counted_by":"address","units_charged":1,"used_today":7563,"day_limit":null,"remaining_today":null,"minute_limit":300,"resets_at":"2026-10-28T00:00:00Z"}},"offers":[{"id":"company.slices","title":"One company in depth, by slice","status":"live","price":{"credits":0.02,"usd":0.002,"plus_per_slice":{"credits":0.05,"usd":0.005}},"unit":"per company, plus each slice with data","note":"the employer in depth","call":{"mcp_tool":"get_company","arguments":{"id":410},"rest":"https://alion.io/mcp/rest/get_company?id=410"},"human":"https://alion.io/catalog?offer=company.slices&for=job%2Fepsilon-lead-data-engineer"},{"id":"market.stats","title":"A market slice: pay, demand and time to fill","status":"live","price":{"credits":1,"usd":0.1},"unit":"per slice","note":"pay, demand and time to fill for this role and place","call":{"mcp_tool":"market_stats"},"human":"https://alion.io/catalog?offer=market.stats&for=job%2Fepsilon-lead-data-engineer"},{"id":"job.search","title":"Open jobs by role, technology, place, pay and visa","status":"live","price":{"credits":0.02,"usd":0.002},"unit":"per posting in a list","note":"similar open postings","call":{"mcp_tool":"search_jobs"},"human":"https://alion.io/catalog?offer=job.search&for=job%2Fepsilon-lead-data-engineer"},{"id":"company.verify","title":"Is this company real and active right now","status":"pilot","price":null,"unit":"per company","request":{"url":"https://alion.io/catalog/request","method":"POST","body":"{\"offer\": \"company.verify\", \"for\": \"job/epsilon-lead-data-engineer\", \"note\": \"what you need it for\"}"},"human":"https://alion.io/catalog?offer=company.verify&for=job%2Fepsilon-lead-data-engineer"}]}