{"id":2968,"url":"https://alion.io/job/arcana-senior-data-engineer","title":"Senior Data Engineer","company":{"id":1050,"name":"Arcana","domain":"arcana.io","url":"https://alion.io/company/arcana","size_band":"51-200","is_staffing_agency":false,"employer_type":"direct","is_intermediary":false,"listed_via":null,"ats_vendor":"Greenhouse","truth_index":{"grade":"D","score":50,"open_postings":25,"ghost_share":0.84,"stale_share":0,"repost_share":0,"time_to_fill_p50_days":null,"computed_at":"2026-10-17T05:46:00Z"}},"role":"Data Science","role_family":"Data Science","seniority":"senior","employment_type":null,"work_mode":"hybrid","remote_scope":null,"remote_scope_basis":null,"remote_working_hours":null,"hiring_geo_confidence":"structured","locations":["Bengaluru, India"],"countries":["IN"],"hiring_countries":[],"hiring_countries_total":0,"salary":null,"salary_estimate":{"min_usd":23000,"max_usd":46000,"period":"year","method":"role_seniority_country_remote_cell","sample_n":59},"experience_years_min":3,"visa_sponsorship":false,"relocation_package":false,"has_equity":false,"technologies":[{"name":"Amazon CloudWatch","optional":false},{"name":"Amazon Kinesis","optional":false},{"name":"Amazon Redshift","optional":false},{"name":"Amazon S3","optional":false},{"name":"AWS","optional":false},{"name":"AWS Lambda","optional":false},{"name":"Cassandra","optional":false},{"name":"ETL/ELT","optional":false},{"name":"Git","optional":false},{"name":"GitHub","optional":false},{"name":"Hadoop","optional":false},{"name":"HBase","optional":false},{"name":"IAM","optional":false},{"name":"pySpark","optional":false},{"name":"Python","optional":false},{"name":"SQL","optional":false},{"name":"Spark","optional":true}],"status":"live","first_seen_at":"2025-06-13T10:32:54Z","employer_posted_date":"2026-07-21","last_verified_at":"2026-10-17T13:51:18Z","board_verified":true,"closed_at":null,"days_open":491,"trust":{"level":"ghost","repost_count":0,"flags":["stale","company_stale"],"days_open":490},"description":"Join our dynamic team at the forefront of cutting-edge technology as we seek a seasoned Senior Data Engineer. Embark on a journey where your deep-rooted expertise in computer science fundamentals, alongside an intricate understanding of data structures, algorithms, and system design, becomes the cornerstone of innovative solutions. This pivotal role not only demands your proficiency in developing and elevating compute and I/O-intensive applications but also ensures their peak performance and unwavering reliability.\nResponsibilities:\nDevelop and implement real-time data ingestion and processing systems.\nDesign, build, and operationalize large-scale enterprise data solutions and applications.\nCreate and manage production data pipelines, from ingestion to consumption, within a big data architecture using PySpark.\nLeverage expertise in Python, Airflow, SQL, and cloud platforms to build robust solutions (strong knowledge of these is essential).\nTranslate complex business challenges into scalable and efficient technical solutions.\nWork collaboratively with a high-performing data engineering team, owning the full lifecycle of solution implementation.\nRequirements:\nBachelor's Degree in Computer Science, Information Technology, or a similar discipline.\n3 - 8 years of professional experience in data engineering or related fields.\nProven experience with ETL processes, data integration, and handling large-scale datasets using PySpark.\nStrong understanding of data engineering concepts like ETL, near/real-time streaming, data structures, and workflow management.\nExperience with version control tools like Git/GitHub.\nProficiency in SQL and Data Warehouse concepts.\nExperience working with SQL or NoSQL databases like Cassandra, MongoDB, or HBase.\nKnowledge of AWS technologies such as EMR, RedShift, Kinesis, Lambda, Glue, S3 IAM, CloudWatch, and big data tools like Hadoop/EMR, Hive, and Sqoop.","description_format":"text","description_chars":1908,"description_truncated":false,"requirements":{"experience_years_min":3,"management_years_min":null,"team_size_min":null,"manages_managers":false,"education":{"level":"bachelor","optional":false},"security_clearance":false,"languages":[]},"benefits":[],"hiring_locations":[{"name":"India","iso":"IN","kind":"country"}],"hiring_excludes":[],"relocation_offered":false,"industries":["Financial Software & Embedded Finance","Financial Data & Market Intelligence"],"lifecycle":[{"event":"open","at":"2025-06-13T10:32:54Z"}],"visa":[],"liveness":{"score":5,"band":"cold","label":"Long shot","p_open":1,"p_active":0.194,"p_room":0.28,"age_days":490,"expected_fill_days":37,"reasons":["conf:1","stale_co","ghost","win:tail","crowd:"],"computed_at":"2026-10-17T05:46:00Z"},"pay":null,"html_url":"https://alion.io/job/arcana-senior-data-engineer","json_url":"https://alion.io/job/arcana-senior-data-engineer.json","meta":{"generated_at":"2026-10-17T16:35:18Z","cache_seconds":300,"methodology":"https://alion.io/methodology","terms":"https://alion.io/terms","contact":"https://alion.io/contact","api":"https://alion.io/developers","about":"Alion is a live layer of people, companies and AI agents: who they are, whether they are real and active right now, what they do and how to work with them, readable by people and by agents and paid per call.","catalog":"https://alion.io/catalog.json","usage":{"tier":"crawler_verified","counted_by":"address","units_charged":1,"used_today":22781,"day_limit":null,"remaining_today":null,"minute_limit":300,"resets_at":"2026-10-18T00:00:00Z"}},"offers":[{"id":"company.slices","title":"One company in depth, by slice","status":"live","price":{"credits":0.02,"usd":0.002,"plus_per_slice":{"credits":0.05,"usd":0.005}},"unit":"per company, plus each slice with data","note":"the employer in depth","call":{"mcp_tool":"get_company","arguments":{"id":1050},"rest":"https://alion.io/mcp/rest/get_company?id=1050"},"human":"https://alion.io/catalog?offer=company.slices&for=job%2Farcana-senior-data-engineer"},{"id":"market.stats","title":"A market slice: pay, demand and time to fill","status":"live","price":{"credits":1,"usd":0.1},"unit":"per slice","note":"pay, demand and time to fill for this role and place","call":{"mcp_tool":"market_stats"},"human":"https://alion.io/catalog?offer=market.stats&for=job%2Farcana-senior-data-engineer"},{"id":"job.search","title":"Open jobs by role, technology, place, pay and visa","status":"live","price":{"credits":0.02,"usd":0.002},"unit":"per posting in a list","note":"similar open postings","call":{"mcp_tool":"search_jobs"},"human":"https://alion.io/catalog?offer=job.search&for=job%2Farcana-senior-data-engineer"},{"id":"company.verify","title":"Is this company real and active right now","status":"pilot","price":null,"unit":"per company","request":{"url":"https://alion.io/catalog/request","method":"POST","body":"{\"offer\": \"company.verify\", \"for\": \"job/arcana-senior-data-engineer\", \"note\": \"what you need it for\"}"},"human":"https://alion.io/catalog?offer=company.verify&for=job%2Farcana-senior-data-engineer"}]}