{"id":4227592,"url":"https://alion.io/job/iris-software-data-engineer-pysparkaws","title":"Data Engineer - PySpark/AWS","company":{"id":3800524,"name":"Iris Software","domain":"irissoftware.com","url":"https://alion.io/company/iris-software","size_band":"1001-5000","is_staffing_agency":false,"employer_type":"services","is_intermediary":false,"listed_via":null,"ats_vendor":null,"truth_index":null},"role":"Data Science","role_family":"Data Science","seniority":"senior","employment_type":null,"work_mode":"on_site","remote_scope":null,"remote_scope_basis":null,"remote_working_hours":null,"hiring_geo_confidence":"structured","locations":["Delhi, India"],"countries":["IN"],"hiring_countries":[],"hiring_countries_total":0,"salary":null,"salary_estimate":null,"experience_years_min":6,"visa_sponsorship":false,"relocation_package":false,"has_equity":false,"technologies":[{"name":"Airflow","optional":false},{"name":"Amazon Redshift","optional":false},{"name":"Amazon S3","optional":false},{"name":"AWS","optional":false},{"name":"AWS Lambda","optional":false},{"name":"CI/CD","optional":false},{"name":"ETL/ELT","optional":false},{"name":"Git","optional":false},{"name":"pySpark","optional":false},{"name":"Python","optional":false},{"name":"SQL","optional":false},{"name":"Docker","optional":true},{"name":"Kubernetes","optional":true},{"name":"Spark","optional":true}],"status":"live","first_seen_at":"2026-11-13T06:47:30Z","employer_posted_date":null,"last_verified_at":"2026-11-13T06:47:30Z","board_verified":false,"closed_at":null,"days_open":0,"trust":{"level":"not_scored","repost_count":null,"flags":[],"days_open":0},"description":"About the Role :\n\nWe are looking for an experienced Data Engineer with strong expertise in PySpark, Python, AWS and Apache Airflow to design, develop and maintain scalable data engineering solutions. The ideal candidate will have hands-on experience building robust ETL/ELT pipelines, processing large datasets and developing cloud-based data platforms.\n\nKey Responsibilities :\n\n- Design, develop and maintain scalable data pipelines using Python and PySpark.\n\n- Build and optimize ETL/ELT workflows for processing structured and unstructured data.\n\n- Develop and manage data processing solutions on AWS cloud services.\n\n- Design, implement and orchestrate data workflows using Apache Airflow.\n\n- Work with AWS services such as S3, Glue, EMR, Lambda, Redshift and Athena.\n\n- Perform data transformation, cleansing, validation and integration across multiple sources.\n\n- Optimize PySpark jobs for performance, scalability and cost efficiency.\n\n- Implement data quality checks, monitoring and error-handling mechanisms.\n\n- Collaborate with data architects, analysts and business stakeholders to understand data requirements.\n\n- Ensure data security, governance and adherence to engineering best practices.\n\n- Troubleshoot pipeline failures and resolve data processing issues.\n\nRequired Skills & Qualifications:\n\n- 6 - 10 years of experience in Data Engineering.\n\n- Strong hands-on experience with PySpark and Python.\n\n- Expertise in building ETL/ELT pipelines and large-scale data processing solutions.\n\n- Strong experience with AWS data services, particularly S3, Glue, EMR, Redshift and Athena.\n\n- Hands-on experience with Apache Airflow for workflow orchestration and scheduling.\n\n- Strong understanding of SQL, data warehousing and database concepts.\n\n- Experience working with distributed computing and big data processing frameworks.\n\n- Knowledge of data modelling, data integration and pipeline optimization.\n\n- Experience with Git, CI/CD and deployment practices.\n\n- Strong analytical, problem-solving and debugging skills.\n\nPreferred Skills:\n\n- Exposure to AWS data architecture and cloud-native data engineering solutions.\n\n- Knowledge of Docker, Kubernetes and infrastructure automation.\n\n- Understanding of data lake and lakehouse architectures.\n\n- Experience implementing automated data quality and monitoring frameworks.\n\nSkills\nData Engineering, PySpark, AWS, ETL, SQL, Data Quality, Apache Airflow, Data Validation, Data Governance, Python, Data Services","description_format":"text","description_chars":2468,"description_truncated":false,"requirements":{"experience_years_min":6,"management_years_min":null,"team_size_min":null,"manages_managers":false,"education":null,"security_clearance":false,"languages":[]},"benefits":[],"hiring_locations":[],"hiring_excludes":[],"relocation_offered":false,"industries":["IT Consulting & Digital Transformation","IT Outsourcing & Dedicated Teams","Software Testing & QA","Custom Software Development"],"lifecycle":[{"event":"open","at":"2026-11-13T08:29:55Z"}],"visa":[],"liveness":{"score":86,"band":"hot","label":"Hiring now","p_open":1,"p_active":0.86,"p_room":1,"age_days":0,"expected_fill_days":45,"reasons":["seen:0","win:early","comp:brand"],"computed_at":"2026-11-13T09:31:40Z"},"pay":null,"html_url":"https://alion.io/job/iris-software-data-engineer-pysparkaws","json_url":"https://alion.io/job/iris-software-data-engineer-pysparkaws.json","meta":{"generated_at":"2026-11-13T09:31:40Z","cache_seconds":300,"methodology":"https://alion.io/methodology","terms":"https://alion.io/terms","contact":"https://alion.io/contact","api":"https://alion.io/developers","about":"Alion is a live layer of people, companies and AI agents: who they are, whether they are real and active right now, what they do and how to work with them, readable by people and by agents and paid per call.","catalog":"https://alion.io/catalog.json","usage":{"tier":"crawler_verified","counted_by":"address","units_charged":1,"used_today":19647,"day_limit":null,"remaining_today":null,"minute_limit":300,"resets_at":"2026-11-14T00:00:00Z"}},"offers":[{"id":"company.slices","title":"One company in depth, by slice","status":"live","price":{"credits":0.1,"usd":0.01,"plus_per_slice":{"credits":0.05,"usd":0.005}},"unit":"per company, plus each slice with data","note":"the employer in depth","call":{"mcp_tool":"get_company","arguments":{"id":3800524},"rest":"https://alion.io/mcp/rest/get_company?id=3800524"},"human":"https://alion.io/catalog?offer=company.slices&for=job%2Firis-software-data-engineer-pysparkaws"},{"id":"market.stats","title":"A market slice: pay, demand and time to fill","status":"free","price":{"credits":0,"usd":0},"unit":"per slice","note":"pay, demand and time to fill for this role and place","call":{"mcp_tool":"market_stats"},"human":"https://alion.io/catalog?offer=market.stats&for=job%2Firis-software-data-engineer-pysparkaws"},{"id":"job.search","title":"Open jobs by role, technology, place, pay and visa","status":"live","price":{"credits":0.1,"usd":0.01},"unit":"per posting in a list","note":"similar open postings","call":{"mcp_tool":"search_jobs"},"human":"https://alion.io/catalog?offer=job.search&for=job%2Firis-software-data-engineer-pysparkaws"},{"id":"company.verify","title":"Is this company real and active right now","status":"pilot","price":null,"unit":"per company","request":{"url":"https://alion.io/catalog/request","method":"POST","body":"{\"offer\": \"company.verify\", \"for\": \"job/iris-software-data-engineer-pysparkaws\", \"note\": \"what you need it for\"}"},"human":"https://alion.io/catalog?offer=company.verify&for=job%2Firis-software-data-engineer-pysparkaws"}],"agent":{"mcp":"https://alion.io/mcp/jobs","mcp_open":"https://alion.io/mcp/open","next":[{"tool":"get_job","arguments":{"id":4227592}},{"tool":"search_jobs","arguments":{"company":"irissoftware.com","detail":"compact"}},{"tool":"match_people","arguments":{"job_id":4227592},"note":"people open to offers who fit this vacancy"}],"note":"The same data through MCP tools: https://alion.io/mcp/open needs no sign-in (compact searches, postings, people); https://alion.io/mcp with an Alion account has everything. Docs: https://alion.io/developers"}}