{"id":3239388,"url":"https://alion.io/job/unionsys-technologies-pyspark-data-engineer","title":"PySpark Data Engineer","company":{"id":2984,"name":"UnionSys Technologies","domain":"unionsystechnologies.com","url":"https://alion.io/company/unionsys-technologies","size_band":null,"is_staffing_agency":false,"employer_type":"direct","is_intermediary":false,"listed_via":null,"ats_vendor":null,"truth_index":null},"role":"Data Science","role_family":"Data Science","seniority":"senior","employment_type":null,"work_mode":"on_site","remote_scope":null,"remote_scope_basis":null,"remote_working_hours":null,"hiring_geo_confidence":"structured","locations":["Bengaluru, India","Gurgaon, India","Pune, India"],"countries":["IN"],"hiring_countries":[],"hiring_countries_total":0,"salary":null,"salary_estimate":{"min_usd":21000,"max_usd":46000,"period":"year","method":"role_seniority_country_remote_cell","sample_n":75},"experience_years_min":5,"visa_sponsorship":false,"relocation_package":false,"has_equity":false,"technologies":[{"name":"pySpark","optional":false},{"name":"Python","optional":false},{"name":"Spark","optional":false},{"name":"SQL","optional":false},{"name":"Airflow","optional":true},{"name":"AWS","optional":true},{"name":"Azure","optional":true},{"name":"CI/CD","optional":true},{"name":"GCP","optional":true}],"status":"closed","first_seen_at":"2026-10-25T14:35:00Z","employer_posted_date":null,"last_verified_at":"2026-10-30T06:35:00Z","board_verified":false,"closed_at":"2026-10-30T06:35:00Z","days_open":4,"trust":{"level":"not_scored","repost_count":null,"flags":[],"days_open":4},"description":"We are seeking an experienced PySpark Data Engineer to design, develop, and maintain scalable, high-performance data pipelines. The role requires strong expertise in Apache Spark, large-scale data processing, and enterprise data integration within complex environments.\n\nResponsibilities: \nDesign, develop, and support scalable data pipelines using PySpark in production environments.\nBuild and maintain robust data ingestion frameworks, integrating data from multiple heterogeneous source systems.\nProcess and transform large datasets (500GB+), ensuring data quality, consistency, and performance optimisation.\nOptimise Spark jobs by leveraging a deep understanding of Spark architecture (DAG execution, partitioning, shuffles, caching, executors, cluster resource management).\nWork extensively with HDFS and structured/semi-structured data formats such as Parquet, ORC, Avro, and CSV.\nPerform data cleansing, standardisation, and transformation to deliver curated datasets for downstream consumption.\nDevelop and maintain Python and shell scripts to automate workflows and reduce manual intervention.\nCollaborate with cross-functional teams, including Data Architects, Business Analysts, and DevOps teams.\nEnsure adherence to enterprise data governance, security, and compliance standards.\nTroubleshoot and resolve performance bottlenecks and production issues.\n\nRequirements: \n5+ years of experience in data engineering with strong hands-on expertise in PySpark and Apache Spark.\nSolid understanding of distributed data processing and big data ecosystems.\nStrong programming skills in Python and advanced SQL (Oracle preferred).\nHands-on experience with HDFS and large-scale data environments.\nProven ability to handle and optimise high-volume data processing workloads.\nStrong analytical and problem-solving skills.\n\nPreferred Skills: \nExperience with cloud platforms (AWS/Azure/GCP).\nExposure to scheduling/orchestration tools such as Airflow, Control-M, or Oozie.\nFamiliarity with CI/CD practices and DevOps integration.\nExperience working in enterprise environments with strict SLAs and compliance requirements.\n\nSoft Skills: \nStrong communication and stakeholder management skills.\nAbility to work in structured, process-driven environments.\nProactive approach to issue resolution and continuous improvement.\nSkills\nApache Spark, Oracle, PySpark, Python, SQL, Spark","description_format":"text","description_chars":2373,"description_truncated":false,"requirements":{"experience_years_min":5,"management_years_min":null,"team_size_min":null,"manages_managers":false,"education":null,"security_clearance":false,"languages":[]},"benefits":[],"hiring_locations":[],"hiring_excludes":[],"relocation_offered":false,"industries":["IT Consulting & Digital Transformation","Custom Software Development"],"lifecycle":[{"event":"open","at":"2026-10-25T14:35:00Z"},{"event":"close","at":"2026-10-30T06:35:00Z"}],"visa":[],"liveness":null,"pay":null,"html_url":"https://alion.io/job/unionsys-technologies-pyspark-data-engineer","json_url":"https://alion.io/job/unionsys-technologies-pyspark-data-engineer.json","meta":{"generated_at":"2026-11-04T02:45:32Z","cache_seconds":300,"methodology":"https://alion.io/methodology","terms":"https://alion.io/terms","contact":"https://alion.io/contact","api":"https://alion.io/developers","about":"Alion is a live layer of people, companies and AI agents: who they are, whether they are real and active right now, what they do and how to work with them, readable by people and by agents and paid per call.","catalog":"https://alion.io/catalog.json","usage":{"tier":"crawler_verified","counted_by":"address","units_charged":1,"used_today":4221,"day_limit":null,"remaining_today":null,"minute_limit":300,"resets_at":"2026-11-05T00:00:00Z"}},"offers":[{"id":"company.slices","title":"One company in depth, by slice","status":"live","price":{"credits":0.02,"usd":0.002,"plus_per_slice":{"credits":0.05,"usd":0.005}},"unit":"per company, plus each slice with data","note":"the employer in depth","call":{"mcp_tool":"get_company","arguments":{"id":2984},"rest":"https://alion.io/mcp/rest/get_company?id=2984"},"human":"https://alion.io/catalog?offer=company.slices&for=job%2Funionsys-technologies-pyspark-data-engineer"},{"id":"market.stats","title":"A market slice: pay, demand and time to fill","status":"live","price":{"credits":1,"usd":0.1},"unit":"per slice","note":"pay, demand and time to fill for this role and place","call":{"mcp_tool":"market_stats"},"human":"https://alion.io/catalog?offer=market.stats&for=job%2Funionsys-technologies-pyspark-data-engineer"},{"id":"job.search","title":"Open jobs by role, technology, place, pay and visa","status":"live","price":{"credits":0.02,"usd":0.002},"unit":"per posting in a list","note":"similar open postings","call":{"mcp_tool":"search_jobs"},"human":"https://alion.io/catalog?offer=job.search&for=job%2Funionsys-technologies-pyspark-data-engineer"},{"id":"company.verify","title":"Is this company real and active right now","status":"pilot","price":null,"unit":"per company","request":{"url":"https://alion.io/catalog/request","method":"POST","body":"{\"offer\": \"company.verify\", \"for\": \"job/unionsys-technologies-pyspark-data-engineer\", \"note\": \"what you need it for\"}"},"human":"https://alion.io/catalog?offer=company.verify&for=job%2Funionsys-technologies-pyspark-data-engineer"}],"agent":{"mcp":"https://alion.io/mcp/jobs","mcp_open":"https://alion.io/mcp/open","next":[{"tool":"get_job","arguments":{"id":3239388}},{"tool":"search_jobs","arguments":{"company":"unionsystechnologies.com","detail":"compact"}},{"tool":"match_people","arguments":{"job_id":3239388},"note":"people open to offers who fit this vacancy"}],"note":"The same data through MCP tools: https://alion.io/mcp/open needs no sign-in (compact searches, postings, people); https://alion.io/mcp with an Alion account has everything. Docs: https://alion.io/developers"}}