{"id":1452438,"url":"https://alion.io/job/simplify-hr-proprietary-limited-senior-data-engineer-spark-python-specialist","title":"Senior Data Engineer (Spark & Python Specialist)","company":{"id":1891733,"name":"Simplify HR","domain":"simplify.hr","url":"https://alion.io/company/simplify-hr","size_band":"1001-5000","is_staffing_agency":false,"employer_type":"direct","is_intermediary":false,"listed_via":null,"ats_vendor":null,"truth_index":{"grade":"B","score":75,"open_postings":369,"ghost_share":0,"stale_share":1,"repost_share":0,"time_to_fill_p50_days":49,"computed_at":"2026-10-01T05:45:00Z"}},"role":"Data Science","role_family":"Data Science","seniority":"senior","employment_type":null,"work_mode":"on_site","remote_scope":null,"remote_scope_basis":null,"remote_working_hours":null,"hiring_geo_confidence":"structured","locations":["South Africa"],"countries":["ZA"],"hiring_countries":[],"hiring_countries_total":0,"salary":{"min":840000,"max":840000,"currency":"ZAR","period":"year","gross":null,"usd_annual":51112},"salary_estimate":null,"experience_years_min":6,"visa_sponsorship":false,"relocation_package":false,"has_equity":false,"technologies":[{"name":"Azure","optional":false},{"name":"Azure Data Factory","optional":false},{"name":"Dagster","optional":false},{"name":"Delta Lake","optional":false},{"name":"Docker","optional":false},{"name":"ETL/ELT","optional":false},{"name":"MS SQL","optional":false},{"name":"pySpark","optional":false},{"name":"Pytest","optional":false},{"name":"Python","optional":false},{"name":"Spark","optional":false},{"name":"SQL","optional":false}],"status":"live","first_seen_at":"2026-03-19T00:00:00Z","employer_posted_date":"2026-03-19","last_verified_at":"2026-09-29T13:23:43Z","board_verified":false,"closed_at":null,"days_open":197,"trust":{"level":"stale","repost_count":0,"flags":["stale"],"days_open":196},"description":"An award-winning privacy-preserving data collaboration platform that enables companies to analyze and collaborate on consumer data to gain insights, build predictive models, and monetize data without sharing the raw data or compromising consumer privacy, is seeking a Senior Cloud Data Engineer who will be a lead technical contributor responsible for building and optimizing high-performance data processing engines.Responsibilities:\nSpark Optimization: Act as the internal SME for Spark internals; manage memory, shuffle tuning, and partitioning for cost-effective performance.\n\nCloud-Agnostic Development: Build pipelines using Python and Delta Lake, decoupling code from specific cloud providers and reducing reliance on GUI tools (e.g., ADF).\n\nRefactoring & Modernization: Migrate complex SQL-based ETL into modular, testable, and maintainable Python libraries.\n\nLakehouse Engineering: Manage Medallion Architecture (Bronze/Silver/Gold) using Delta Lake, ensuring storage performance via Z-Ordering and Vacuuming.\n\nCode-First Orchestration: Support the transition to code-centric patterns (Airflow, Dagster) to prioritize portability.\n\nTechnical Excellence: Lead code reviews, mentor junior engineers, and implement automated testing frameworks (Pytest).\n\nMinimum Requirements:\nEducation: Bachelor’s degree in Computer Science, Information Systems, Engineering, or a related field.\nSpark Mastery: 6+ years of Spark/PySpark experience; expert ability to diagnose bottlenecks via Spark UI and optimize complex DAGs.\n\nAdvanced Python: Proficiency in production-grade Python, including building reusable libraries and automated testing.\n\nAzure Ecosystem: Strong experience with Azure Synapse, Dedicated SQL Pools, and Data Factory.\n\nModern Data Stack: Hands-on experience with Delta Lake, Parquet, and containerization (Docker).\n\nMigration Skills: Solid T-SQL skills to interpret and migrate legacy logic into Python-centric environments.\n\nSecurity & Governance: Proven ability to implement high levels of security and compliance across data processes.\n\nBenefits:\nCompetitive salary based on experience (salary can potentially be more based on experience/skills)\nIF you meet the above requirements and want to make a career-changing move, apply today by emailing your CV to ","description_format":"text","description_chars":2303,"description_truncated":false,"requirements":{"experience_years_min":6,"management_years_min":null,"team_size_min":null,"manages_managers":false,"education":{"level":"bachelor","optional":false},"security_clearance":false,"languages":[]},"benefits":[],"hiring_locations":[],"hiring_excludes":[],"relocation_offered":false,"industries":["Professional Services","Human Resources","Recruiting Software & ATS"],"lifecycle":[{"event":"open","at":"2026-09-29T08:02:34Z"}],"liveness":{"score":9,"band":"cold","label":"Long shot","p_open":1,"p_active":0.314,"p_room":0.28,"age_days":196,"expected_fill_days":49,"reasons":["conf:40","stale_co","velocity","win:tail","crowd:brand"],"computed_at":"2026-10-01T05:45:00Z"},"pay":{"stated_usd_annual":51112,"is_top_pay":false},"html_url":"https://alion.io/job/simplify-hr-proprietary-limited-senior-data-engineer-spark-python-specialist","json_url":"https://alion.io/job/simplify-hr-proprietary-limited-senior-data-engineer-spark-python-specialist.json","meta":{"generated_at":"2026-10-02T01:45:17Z","cache_seconds":300,"methodology":"https://alion.io/methodology","terms":"https://alion.io/terms","contact":"https://alion.io/contact","api":"https://alion.io/developers","usage":{"tier":"crawler","counted_by":"address","units_charged":1,"used_today":2287,"day_limit":5000,"remaining_today":2713,"minute_limit":60,"resets_at":"2026-10-03T00:00:00Z"}}}