{"id":3148602,"url":"https://alion.io/job/citi-pyspark-engineer","title":"Pyspark Engineer","company":{"id":7841,"name":"Citi","domain":"citi.com","url":"https://alion.io/company/citi","size_band":"5000+","is_staffing_agency":false,"employer_type":"direct","is_intermediary":false,"listed_via":null,"ats_vendor":"Workday","truth_index":{"grade":"A","score":100,"open_postings":473,"ghost_share":0.006,"stale_share":0.129,"repost_share":0.078,"time_to_fill_p50_days":10,"computed_at":"2026-10-25T05:49:26Z"}},"role":"Data Science","role_family":"Data Science","seniority":"middle","employment_type":"full_time","work_mode":"hybrid","remote_scope":null,"remote_scope_basis":null,"remote_working_hours":null,"hiring_geo_confidence":"structured","locations":["India"],"countries":["IN"],"hiring_countries":[],"hiring_countries_total":0,"salary":null,"salary_estimate":{"min_usd":13500,"max_usd":35000,"period":"year","method":"role_seniority_country_remote_cell","sample_n":21},"experience_years_min":4,"visa_sponsorship":false,"relocation_package":false,"has_equity":false,"technologies":[{"name":"Airflow","optional":false},{"name":"Amazon CloudWatch","optional":false},{"name":"Amazon Kinesis","optional":false},{"name":"Amazon Redshift","optional":false},{"name":"Amazon S3","optional":false},{"name":"Apache Hudi","optional":false},{"name":"Apache Iceberg","optional":false},{"name":"Apache Kafka","optional":false},{"name":"AWS","optional":false},{"name":"Azure","optional":false},{"name":"CI/CD","optional":false},{"name":"Dagster","optional":false},{"name":"Data Vault","optional":false},{"name":"Databricks","optional":false},{"name":"Delta Lake","optional":false},{"name":"ETL/ELT","optional":false},{"name":"GCP","optional":false},{"name":"GDPR","optional":false},{"name":"Git","optional":false},{"name":"GitHub Actions","optional":false},{"name":"GitLab CI","optional":false},{"name":"Google BigQuery","optional":false},{"name":"Grafana","optional":false},{"name":"Great Expectations","optional":false},{"name":"Jenkins","optional":false},{"name":"Kubernetes","optional":false},{"name":"Prefect","optional":false},{"name":"Prometheus","optional":false},{"name":"pySpark","optional":false},{"name":"Pytest","optional":false},{"name":"Python","optional":false},{"name":"Rest API","optional":false},{"name":"Snowflake","optional":false},{"name":"Spark","optional":false},{"name":"SQL","optional":false},{"name":"CloudFormation","optional":true},{"name":"Hadoop","optional":true},{"name":"Java","optional":true},{"name":"MLFlow","optional":true},{"name":"Oracle","optional":true},{"name":"Scala","optional":true},{"name":"Terraform","optional":true},{"name":"Unix","optional":true}],"status":"live","first_seen_at":"2026-10-23T11:19:44Z","employer_posted_date":"2026-10-23","last_verified_at":"2026-10-24T08:16:14Z","board_verified":true,"closed_at":null,"days_open":2,"trust":{"level":"ok","repost_count":null,"flags":[],"days_open":2},"description":"Job Description\nWe are seeking an experienced and results-driven PySpark / Big Data Engineer to design, develop, and maintain large-scale batch and real-time data pipelines. In this role, you will be responsible for transforming complex raw data into actionable analytical data models, optimizing distributed computing workloads, and ensuring data integrity, scalability, and security across enterprise data platforms.\nThe ideal candidate possesses deep expertise in Apache Spark architecture, advanced Python programming, distributed systems optimization, modern cloud platforms, and data warehousing/lakehouse paradigms.\nKey Responsibilities\nData Pipeline Design & Development\nArchitect, build, and maintain robust, scalable ETL/ELT pipelines using PySpark and Python to ingest data from diverse batch and streaming sources (Kafka, relational databases, REST APIs, Object Storage).\nDesign and implement Lakehouse architectures using formats such as Delta Lake, Apache Iceberg, or Apache Hudi.\nBuild reusable frameworks and data ingestion libraries to standardize enterprise data processing.\nSpark Performance Tuning & Optimization\nDiagnose and tune Spark application performance, managing memory allocation, partitions, execution plans, broadcast joins, data skews, and garbage collection.\nOptimize query execution plans using Catalyst Optimizer and Tungsten execution engine.\nMonitor and troubleshoot production Spark jobs across cluster management environments (YARN, Kubernetes, or Databricks).\nData Modeling & Architecture\nCollaborate with Data Architects to design dimensional (Star/Snowflake), normalized, and Data Vault schemas optimized for analytical workloads.\nEnsure efficient data storage, partitioning strategies, and file format selection (Parquet, ORC, Avro).\nOrchestration, CI/CD & DevOps\nSchedule and orchestrate complex workflow DAGs using workflow orchestrators such as Apache Airflow, Prefect, or Dagster.\nImplement CI/CD pipelines (Git, Jenkins, GitLab CI, or GitHub Actions) for automated testing, packaging, and deployment of data pipelines.\nWrite unit tests, integration tests, and data validation routines (e.g., Great Expectations, PyTest).\nGovernance, Security & Quality\nImplement enterprise data governance, lineage tracking, metadata management, and data quality frameworks.\nEnsure strict compliance with data security, masking, tokenization, RBAC, and regulatory privacy standards (e.g., GDPR, CCPA).\nRequired Technical Qualifications & Skills\n Core Technologies - Apache Spark, PySpark (Spark SQL, DataFrames, Datasets, Structured Streaming), Python 3.x\nDatabase & Querying - Advanced SQL, Query optimization, Indexing, Window functions\nData Lake & Lakehouse - Delta Lake, Apache Iceberg, Apache Hudi, Parquet, Avro, ORC\nWorkflow & Orchestration - Apache Airflow, Luigi, Prefect, or Databricks Workflows\n Cloud Platforms - AWS (EMR, Glue, S3, Redshift), Azure (Synapse, ADLS, HDInsight), or GCP (Dataproc, BigQuery, GCS)\nStreaming Technologies - Apache Kafka, Spark Structured Streaming, Event Hubs, or Kinesis\nKey Competencies & Experience\n Distributed Computing Fundamentals: In-depth knowledge of Spark internals (RDD, Catalyst Optimizer, DAG execution, Shuffling, Narrow vs. Wide dependencies).\nProduction Experience: Minimum 4+ years of hands-on experience building and deploying production-grade PySpark applications at enterprise scale.\n Problem Solving: Proven track record in debugging memory bottlenecks, Out-Of-Memory (OOM) errors, and resolving data skew issues.\n Data Engineering Best Practices: Experience implementing modular code architectures, logging, monitoring (Prometheus, Grafana, CloudWatch), and alerting.\n5. Preferred Qualifications (Nice-to-Have)\nHands-on experience with Databricks platform (Delta Lake, Unity Catalog, Photon engine, MLflow).\nFamiliarity with Scala/Java in addition to Python.\nKnowledge of DataOps and Infrastructure as Code (Terraform, CloudFormation).\nExperience with modern analytical data warehouses (Snowflake, Google BigQuery, Amazon Redshift).\nRelevant certifications: Databricks Certified Data Engineer Associate / Professional AWS Certified Data Engineer / Azure Data Engineer Associate (DP-203)CCA Spark and Hadoop Developer (or equivalent)\n6. Educational Requirements\nBachelor’s or Master’s degree in Computer Science, Software Engineering, Information Technology, Data Science, or a related quantitative field (or equivalent practical experience).\nPrimary : Py Spark, Python\nSecondary : Bigdata, Abinitio, Unix, Oracle\n------------------------------------------------------\nJob Family Group:\nTechnology------------------------------------------------------\nJob Family:\nApplications Development------------------------------------------------------\nTime Type:\nFull time------------------------------------------------------\nMost Relevant Skills\nPlease see the requirements listed above.------------------------------------------------------\nOther Relevant Skills\nFor complementary skills, please see above and/or contact the recruiter.------------------------------------------------------\nCiti is an equal opportunity employer, and qualified candidates will receive consideration without regard to their race, color, religion, sex, sexual orientation, gender identity, national origin, disability, status as a protected veteran, or any other characteristic protected by law.\nIf you are a person with a disability and need a reasonable accommodation to use our search tools and/or apply for a career opportunity review Accessibility at Citi.\nView Citi’s EEO Policy Statement and the Know Your Rights poster.","description_format":"text","description_chars":5595,"description_truncated":false,"requirements":{"experience_years_min":4,"management_years_min":null,"team_size_min":null,"manages_managers":false,"education":{"level":"bachelor","optional":false},"security_clearance":false,"languages":[]},"benefits":[],"hiring_locations":[{"name":"India","iso":"IN","kind":"country"}],"hiring_excludes":[],"relocation_offered":false,"industries":["Cards & Card Issuing","Wealth Management & Financial Advisors","Payment Processing & Gateways","Investment Banking & M&A Advisory"],"lifecycle":[{"event":"open","at":"2026-10-23T11:19:44Z"}],"visa":[],"liveness":{"score":90,"band":"hot","label":"Hiring now","p_open":1,"p_active":0.897,"p_room":1,"age_days":1,"expected_fill_days":10,"reasons":["conf:21","velocity","win:early","comp:brand"],"computed_at":"2026-10-25T05:49:26Z"},"pay":null,"html_url":"https://alion.io/job/citi-pyspark-engineer","json_url":"https://alion.io/job/citi-pyspark-engineer.json","meta":{"generated_at":"2026-10-25T17:55:04Z","cache_seconds":300,"methodology":"https://alion.io/methodology","terms":"https://alion.io/terms","contact":"https://alion.io/contact","api":"https://alion.io/developers","about":"Alion is a live layer of people, companies and AI agents: who they are, whether they are real and active right now, what they do and how to work with them, readable by people and by agents and paid per call.","catalog":"https://alion.io/catalog.json","usage":{"tier":"crawler_verified","counted_by":"address","units_charged":1,"used_today":25633,"day_limit":null,"remaining_today":null,"minute_limit":300,"resets_at":"2026-10-26T00:00:00Z"}},"offers":[{"id":"company.slices","title":"One company in depth, by slice","status":"live","price":{"credits":0.02,"usd":0.002,"plus_per_slice":{"credits":0.05,"usd":0.005}},"unit":"per company, plus each slice with data","note":"the employer in depth","call":{"mcp_tool":"get_company","arguments":{"id":7841},"rest":"https://alion.io/mcp/rest/get_company?id=7841"},"human":"https://alion.io/catalog?offer=company.slices&for=job%2Fciti-pyspark-engineer"},{"id":"market.stats","title":"A market slice: pay, demand and time to fill","status":"live","price":{"credits":1,"usd":0.1},"unit":"per slice","note":"pay, demand and time to fill for this role and place","call":{"mcp_tool":"market_stats"},"human":"https://alion.io/catalog?offer=market.stats&for=job%2Fciti-pyspark-engineer"},{"id":"job.search","title":"Open jobs by role, technology, place, pay and visa","status":"live","price":{"credits":0.02,"usd":0.002},"unit":"per posting in a list","note":"similar open postings","call":{"mcp_tool":"search_jobs"},"human":"https://alion.io/catalog?offer=job.search&for=job%2Fciti-pyspark-engineer"},{"id":"company.verify","title":"Is this company real and active right now","status":"pilot","price":null,"unit":"per company","request":{"url":"https://alion.io/catalog/request","method":"POST","body":"{\"offer\": \"company.verify\", \"for\": \"job/citi-pyspark-engineer\", \"note\": \"what you need it for\"}"},"human":"https://alion.io/catalog?offer=company.verify&for=job%2Fciti-pyspark-engineer"}]}