{"id":2903769,"url":"https://alion.io/job/npci-data-engineer-real-time-streaming","title":"Data Engineer (Real-time Streaming)","company":{"id":59461,"name":"NPCI","domain":"npci.org.in","url":"https://alion.io/company/npci","size_band":"501-1000","is_staffing_agency":false,"employer_type":"direct","is_intermediary":false,"listed_via":null,"ats_vendor":"Zoho Recruit","truth_index":{"grade":"B","score":75,"open_postings":22,"ghost_share":0,"stale_share":1,"repost_share":0,"time_to_fill_p50_days":null,"computed_at":"2026-10-22T06:10:10Z"}},"role":"Data Science","role_family":"Data Science","seniority":"middle","employment_type":"full_time","work_mode":"on_site","remote_scope":null,"remote_scope_basis":null,"remote_working_hours":null,"hiring_geo_confidence":"structured","locations":["Hyderabad, India"],"countries":["IN"],"hiring_countries":[],"hiring_countries_total":0,"salary":null,"salary_estimate":{"min_usd":14000,"max_usd":35000,"period":"year","method":"role_seniority_country_remote_cell","sample_n":21},"experience_years_min":3,"visa_sponsorship":false,"relocation_package":false,"has_equity":false,"technologies":[{"name":"Airflow","optional":true},{"name":"Amazon S3","optional":true},{"name":"Apache Hudi","optional":true},{"name":"Apache Iceberg","optional":true},{"name":"Apache Kafka","optional":true},{"name":"Cassandra","optional":true},{"name":"CI/CD","optional":true},{"name":"Dagster","optional":true},{"name":"Flink","optional":true},{"name":"GitHub Actions","optional":true},{"name":"Java","optional":true},{"name":"Jenkins","optional":true},{"name":"Linux","optional":true},{"name":"MinIO","optional":true},{"name":"Python","optional":true},{"name":"Redis","optional":true},{"name":"Scala","optional":true},{"name":"Spark","optional":true},{"name":"SQL","optional":true},{"name":"Superset","optional":true},{"name":"Tableau","optional":true},{"name":"Trino","optional":true}],"status":"live","first_seen_at":"2026-06-15T00:00:00Z","employer_posted_date":"2026-06-15","last_verified_at":"2026-10-22T13:42:04Z","board_verified":true,"closed_at":null,"days_open":129,"trust":{"level":"stale","repost_count":0,"flags":["stale"],"days_open":129},"description":"The Opportunity To design and implement real-time data streaming pipelines ensuring 24×7 availability of high-volume transactional data. Your role includes building fault-tolerant, scalable systems using modern big data technologies, integrating data from multiple sources into data lakes/lakehouses, and enabling downstream analytics. You will collaborate with internal teams (Data Science, Product, and Technology) to deliver reliable data solutions, optimize performance, and maintain compliance with NPCI standards. Job Details Job Title: Data Engineer (Streaming) Division: Data Analytics Years of Experience: 3-8 yearsEducation: Graduation in Computer Science/IT (preferably BE/B.Tech) or equivalent; advanced degrees are a plus\nEmployment Type: Full-time, Permanent Location: Mumbai & Hyderabad\nKey\nResponsibilities Design and develop real-time data pipelines using Apache Kafka and stream processing frameworks (Spark Structured Streaming / Apache Flink). Ensure 24×7 data availability with fault-tolerant, highly reliable systems. Implement ingestion, transformation, and load (ITL) patterns for data lakes/lakehouses (S3/MinIO, HDFS). Work with table formats like Iceberg, Hudi, or Paimon for ACID transactions and schema evolution. Optimize SQL queries on Trino/Hive for large-scale analytics. Develop orchestration workflows using DBT, Dagster, or Airflow for data transformations. Write efficient code in Python, Scala, and Java for data processing and automation. Collaborate with cross-functional teams to understand requirements and deliver high-quality data solutions. Monitor pipeline health, manage checkpoints, and implement observability for streaming jobs. Ensure compliance with security, governance, and audit standards.\nRequirements Key Skills and\nExperience Required Mandatory\nTechnical Skills: Apache Kafka (topics, partitions, offsets, reliability) Stream processing (Spark Structured Streaming or Apache Flink) Preferred\nTechnical Skills: Data Lake / Lakehouse (S3, MinIO, HDFS) Table formats: Iceberg, Hudi, Paimon SQL engines: Trino, Hive Orchestration tools: DBT, Dagster, Airflow Programming: Python, Scala, Java NoSQL databases (MongoDB, Cassandra, Redis) CI/CD (Jenkins, GitHub Actions), Linux basics BI tools: Superset, Tableau Data Quality frameworks and observability practices Other\nRequirements: Strong analytical and problem-solving skills Ability to work in 24×7 environments Quick adaptability to new technologies","description_format":"text","description_chars":2456,"description_truncated":false,"requirements":{"experience_years_min":3,"management_years_min":null,"team_size_min":null,"manages_managers":false,"education":null,"security_clearance":false,"languages":[]},"benefits":[],"hiring_locations":[],"hiring_excludes":[],"relocation_offered":false,"industries":["Financial Software & Embedded Finance","Payment Processing & Gateways"],"lifecycle":[{"event":"open","at":"2026-10-20T07:20:44Z"}],"visa":[],"liveness":{"score":6,"band":"cold","label":"Long shot","p_open":1,"p_active":0.209,"p_room":0.28,"age_days":129,"expected_fill_days":30,"reasons":["conf:3","stale_co","win:tail","crowd:"],"computed_at":"2026-10-22T06:10:10Z"},"pay":null,"html_url":"https://alion.io/job/npci-data-engineer-real-time-streaming","json_url":"https://alion.io/job/npci-data-engineer-real-time-streaming.json","meta":{"generated_at":"2026-10-22T17:29:17Z","cache_seconds":300,"methodology":"https://alion.io/methodology","terms":"https://alion.io/terms","contact":"https://alion.io/contact","api":"https://alion.io/developers","about":"Alion is a live layer of people, companies and AI agents: who they are, whether they are real and active right now, what they do and how to work with them, readable by people and by agents and paid per call.","catalog":"https://alion.io/catalog.json","usage":{"tier":"crawler_verified","counted_by":"address","units_charged":1,"used_today":18459,"day_limit":null,"remaining_today":null,"minute_limit":300,"resets_at":"2026-10-23T00:00:00Z"}},"offers":[{"id":"company.slices","title":"One company in depth, by slice","status":"live","price":{"credits":0.02,"usd":0.002,"plus_per_slice":{"credits":0.05,"usd":0.005}},"unit":"per company, plus each slice with data","note":"the employer in depth","call":{"mcp_tool":"get_company","arguments":{"id":59461},"rest":"https://alion.io/mcp/rest/get_company?id=59461"},"human":"https://alion.io/catalog?offer=company.slices&for=job%2Fnpci-data-engineer-real-time-streaming"},{"id":"market.stats","title":"A market slice: pay, demand and time to fill","status":"live","price":{"credits":1,"usd":0.1},"unit":"per slice","note":"pay, demand and time to fill for this role and place","call":{"mcp_tool":"market_stats"},"human":"https://alion.io/catalog?offer=market.stats&for=job%2Fnpci-data-engineer-real-time-streaming"},{"id":"job.search","title":"Open jobs by role, technology, place, pay and visa","status":"live","price":{"credits":0.02,"usd":0.002},"unit":"per posting in a list","note":"similar open postings","call":{"mcp_tool":"search_jobs"},"human":"https://alion.io/catalog?offer=job.search&for=job%2Fnpci-data-engineer-real-time-streaming"},{"id":"company.verify","title":"Is this company real and active right now","status":"pilot","price":null,"unit":"per company","request":{"url":"https://alion.io/catalog/request","method":"POST","body":"{\"offer\": \"company.verify\", \"for\": \"job/npci-data-engineer-real-time-streaming\", \"note\": \"what you need it for\"}"},"human":"https://alion.io/catalog?offer=company.verify&for=job%2Fnpci-data-engineer-real-time-streaming"}]}