{"id":3776359,"url":"https://alion.io/job/astek-data-engineer-apache-airflow","title":"Data Engineer- Apache Airflow","company":{"id":4374840,"name":"Astek","domain":"astek.com","url":"https://alion.io/company/astek-4","size_band":null,"is_staffing_agency":false,"employer_type":"direct","is_intermediary":false,"listed_via":null,"ats_vendor":null,"truth_index":null},"role":"Data Science","role_family":"Data Science","seniority":null,"employment_type":"full_time","work_mode":"on_site","remote_scope":null,"remote_scope_basis":null,"remote_working_hours":null,"hiring_geo_confidence":"structured","locations":["Kochi, India"],"countries":["IN"],"hiring_countries":[],"hiring_countries_total":0,"salary":null,"salary_estimate":null,"experience_years_min":null,"visa_sponsorship":false,"relocation_package":false,"has_equity":false,"technologies":[{"name":"Airflow","optional":false},{"name":"Amazon S3","optional":false},{"name":"Ansible","optional":false},{"name":"AWS","optional":false},{"name":"Git","optional":false},{"name":"GitLab","optional":false},{"name":"Hadoop","optional":false},{"name":"MS SQL","optional":false},{"name":"pySpark","optional":false},{"name":"Python","optional":false},{"name":"Snowflake","optional":false},{"name":"Spark","optional":false},{"name":"SSIS","optional":false}],"status":"live","first_seen_at":"2026-11-04T17:24:20Z","employer_posted_date":null,"last_verified_at":"2026-11-04T17:24:20Z","board_verified":false,"closed_at":null,"days_open":2,"trust":{"level":"not_scored","repost_count":null,"flags":[],"days_open":2},"description":"As a Senior Data Engineer you will play a key role in designing, developing, and maintaining data workflows and pipelines, ensuring the efficient and reliable movement of data across various systems. You will work with modern data technologies including Apache Airflow, PySpark, Hive, AWS, and Snowflake, while collaborating with cross-functional teams to deliver high-quality data solutions. This role requires a strong focus on data quality, performance optimization, and adherence to security and compliance standards.\nWhat You'll Do\n• Design, implement, and manage complex DAGs and SubDAGs in Apache Airflow.\n• Utilize Operators, macros, connections, variables, and XCom for efficient task execution.\n• Conduct unit testing to ensure the robustness of Airflow workflows.\n• Perform file operations and data movement between HDFS and S3 (and vice versa).\n• Manage Hive external tables, update table DDL, and handle partition operations.\n• Execute basic Hadoop commands and navigate/traverse HDFS. • Work with Hive on Tez for optimized query execution.\n• Utilize AWS command line tools for tasks such as listing and copying files from S3.\n• Implement and manage secrets using tools like Ansible Vault and AWS Secrets Manager.\n• Develop and optimize data processing tasks using PySpark.\n• Understand and leverage Livy for efficient Spark job execution.\n• Import and export data between Microsoft SQL Server and HDFS using Sqoop.\n• Apply dimension modeling principles, define facts and dimensions, and manage surrogate keys. • Implement effective branching strategies in Git.\n• Participate in the process of raising and reviewing Merge Requests in GitLab.\n• Utilize SSIS for migrating data workflows, ensuring a smooth transition.\n• Model data in Snowflake, differentiating between external and internal tables.\n• Understand the use of views and materialized views in Snowflake.\nWhat You'll Need\n• Bachelor's degree in Computer Science, Information Technology, or a related field.\n• Proficient in Python (both Python 2.7 and 3.6+).\n• Experience in unit testing and test-driven development. \n• Familiarity with Secrets Management tools and techniques.\n• Knowledge of data modeling principles and practices.\n• Strong understanding of Git version control and GitLab workflows.\n• Excellent problem-solving and communication skills.\n• Awareness or knowledge of IT security best practices as defined by ISO / SOC or similar standards. • Experience with SSIS for data migration workflows.\n• Experience with Snowflake (external/internal tables, views, materialized views).\n• Familiarity with Hive on Tez for optimized query execution.\n• Experience with AWS command line tools and S3 operations.\n• Knowledge of Livy for efficient Spark job execution.","description_format":"text","description_chars":2740,"description_truncated":false,"requirements":{"experience_years_min":null,"management_years_min":null,"team_size_min":null,"manages_managers":false,"education":null,"security_clearance":false,"languages":[]},"benefits":[],"hiring_locations":[],"hiring_excludes":[],"relocation_offered":false,"industries":["Design & Creative","Media & Entertainment","Creative Agencies"],"lifecycle":[{"event":"open","at":"2026-11-04T23:09:45Z"}],"visa":[],"liveness":{"score":90,"band":"hot","label":"Hiring now","p_open":1,"p_active":0.897,"p_room":1,"age_days":1,"expected_fill_days":5,"reasons":["seen:1","velocity","win:early"],"computed_at":"2026-11-06T05:58:05Z"},"pay":null,"html_url":"https://alion.io/job/astek-data-engineer-apache-airflow","json_url":"https://alion.io/job/astek-data-engineer-apache-airflow.json","meta":{"generated_at":"2026-11-06T19:03:40Z","cache_seconds":300,"methodology":"https://alion.io/methodology","terms":"https://alion.io/terms","contact":"https://alion.io/contact","api":"https://alion.io/developers","about":"Alion is a live layer of people, companies and AI agents: who they are, whether they are real and active right now, what they do and how to work with them, readable by people and by agents and paid per call.","catalog":"https://alion.io/catalog.json","usage":{"tier":"crawler_verified","counted_by":"address","units_charged":1,"used_today":2084,"day_limit":null,"remaining_today":null,"minute_limit":300,"resets_at":"2026-11-07T00:00:00Z"}},"offers":[{"id":"company.slices","title":"One company in depth, by slice","status":"live","price":{"credits":0.02,"usd":0.002,"plus_per_slice":{"credits":0.05,"usd":0.005}},"unit":"per company, plus each slice with data","note":"the employer in depth","call":{"mcp_tool":"get_company","arguments":{"id":4374840},"rest":"https://alion.io/mcp/rest/get_company?id=4374840"},"human":"https://alion.io/catalog?offer=company.slices&for=job%2Fastek-data-engineer-apache-airflow"},{"id":"market.stats","title":"A market slice: pay, demand and time to fill","status":"live","price":{"credits":1,"usd":0.1},"unit":"per slice","note":"pay, demand and time to fill for this role and place","call":{"mcp_tool":"market_stats"},"human":"https://alion.io/catalog?offer=market.stats&for=job%2Fastek-data-engineer-apache-airflow"},{"id":"job.search","title":"Open jobs by role, technology, place, pay and visa","status":"live","price":{"credits":0.02,"usd":0.002},"unit":"per posting in a list","note":"similar open postings","call":{"mcp_tool":"search_jobs"},"human":"https://alion.io/catalog?offer=job.search&for=job%2Fastek-data-engineer-apache-airflow"},{"id":"company.verify","title":"Is this company real and active right now","status":"pilot","price":null,"unit":"per company","request":{"url":"https://alion.io/catalog/request","method":"POST","body":"{\"offer\": \"company.verify\", \"for\": \"job/astek-data-engineer-apache-airflow\", \"note\": \"what you need it for\"}"},"human":"https://alion.io/catalog?offer=company.verify&for=job%2Fastek-data-engineer-apache-airflow"}],"agent":{"mcp":"https://alion.io/mcp/jobs","mcp_open":"https://alion.io/mcp/open","next":[{"tool":"get_job","arguments":{"id":3776359}},{"tool":"search_jobs","arguments":{"company":"astek.com","detail":"compact"}},{"tool":"match_people","arguments":{"job_id":3776359},"note":"people open to offers who fit this vacancy"}],"note":"The same data through MCP tools: https://alion.io/mcp/open needs no sign-in (compact searches, postings, people); https://alion.io/mcp with an Alion account has everything. Docs: https://alion.io/developers"}}