{"id":2460040,"url":"https://alion.io/job/vega-intellisoft-lead-data-engineer","title":"Lead Data engineer","company":{"id":3848052,"name":"Vega Intellisoft","domain":"vegaintellisoft.com","url":"https://alion.io/company/vega-intellisoft-pvt-ltd","size_band":null,"is_staffing_agency":false,"employer_type":"direct","is_intermediary":false,"listed_via":null,"ats_vendor":null,"truth_index":null},"role":"Data Science","role_family":"Data Science","seniority":"lead","employment_type":"full_time","work_mode":"hybrid","remote_scope":null,"remote_scope_basis":null,"remote_working_hours":null,"hiring_geo_confidence":"structured","locations":["Cary, United States"],"countries":["US"],"hiring_countries":[],"hiring_countries_total":0,"salary":null,"salary_estimate":{"min_usd":113000,"max_usd":221000,"period":"year","method":"role_seniority_country_remote_cell","sample_n":390},"experience_years_min":12,"visa_sponsorship":false,"relocation_package":false,"has_equity":false,"technologies":[{"name":"Agentic Workflows","optional":false},{"name":"AI Agents","optional":false},{"name":"Apache Kafka","optional":false},{"name":"Azure","optional":false},{"name":"Azure Data Factory","optional":false},{"name":"Azure DevOps","optional":false},{"name":"CI/CD","optional":false},{"name":"Databricks","optional":false},{"name":"Delta Lake","optional":false},{"name":"Embeddings","optional":false},{"name":"ETL/ELT","optional":false},{"name":"Knowledge Graph","optional":false},{"name":"LangChain","optional":false},{"name":"LangGraph","optional":false},{"name":"LlamaIndex","optional":false},{"name":"LLM","optional":false},{"name":"OpenAI","optional":false},{"name":"pySpark","optional":false},{"name":"Python","optional":false},{"name":"RAG","optional":false},{"name":"Scala","optional":false},{"name":"Spark","optional":false},{"name":"SQL","optional":false},{"name":"Terraform","optional":false},{"name":"Tool Use","optional":false},{"name":"dbt","optional":true},{"name":"Great Expectations","optional":true},{"name":"LLMOps","optional":true}],"status":"live","first_seen_at":"2026-10-13T14:02:33Z","employer_posted_date":null,"last_verified_at":"2026-10-13T14:02:33Z","board_verified":false,"closed_at":null,"days_open":2,"trust":{"level":"not_scored","repost_count":null,"flags":[],"days_open":2},"description":"HIRING: Lead Data Engineer – AI / Azure Databricks\nLocation: Cary, NC (On-site / Hybrid)\nEmployment Type: Full-Time\nExperience: 12–18 Years\nAny Visa\nABOUT THE ROLE\nWe are looking for a hands-on Lead Data Engineer to architect and build an enterprise-scale, AI-first Data Hub using Azure Databricks. The role combines advanced data engineering, lakehouse architecture, production-grade AI/LLM solutions, and technical leadership.\nKEY RESPONSIBILITIES\n• Design enterprise lakehouse architecture using Azure Databricks, ADLS Gen2, Delta Lake, and Bronze/Silver/Gold layers.\n• Build metadata-driven ingestion frameworks and scalable ETL pipelines using Python, Scala, and PySpark.\n• Optimize Spark jobs, cluster performance, partitioning, and processing costs.\n• Develop batch, CDC, and streaming pipelines using Azure Data Factory, Event Hubs, Kafka, and Spark Structured Streaming.\n• Build AI-assisted source-to-canonical mapping, automated data quality, anomaly detection, and reconciliation solutions.\n• Design production LLM applications using RAG, agentic workflows, LangChain, LangGraph, or LlamaIndex.\n• Implement semantic layers, knowledge graphs, and conversational data access where applicable.\n• Establish CI/CD, automated testing, governance, security, and data quality standards.\n• Lead architecture reviews, mentor engineers, and contribute hands-on production code.\nREQUIRED SKILLS\n• 12–18 years of data engineering or data platform experience.\n• Expert Python, Scala, PySpark, SQL, and Apache Spark.\n• Strong Azure Databricks, Delta Lake, Unity Catalog, and ADLS Gen2.\n• Hands-on Azure Data Factory, Azure Event Hubs, and metadata-driven ingestion.\n• 3+ years building production LLM solutions, including RAG, retrieval, embeddings, and agentic/tool-calling workflows.\n• LangChain, LangGraph, or LlamaIndex with Azure OpenAI, OpenAI, or Databricks Model Serving.\n• Azure DevOps, Terraform, Databricks Asset Bundles, and CI/CD.\n• Strong Azure security, governance, data modelling, and Spark performance tuning.\n• Proven enterprise-scale lakehouse architecture and technical leadership.\nPREFERRED SKILLS\nKnowledge graphs, text-to-SQL, AI-driven anomaly detection, LLMOps/MLOps, dbt, Great Expectations, and insurance or financial services experience.\nInterested candidates, please share your resume at:","description_format":"text","description_chars":2313,"description_truncated":false,"requirements":{"experience_years_min":12,"management_years_min":null,"team_size_min":null,"manages_managers":false,"education":null,"security_clearance":false,"languages":[]},"benefits":[],"hiring_locations":[],"hiring_excludes":[],"relocation_offered":false,"industries":["Custom Software Development"],"lifecycle":[{"event":"open","at":"2026-10-13T16:52:00Z"}],"visa":[],"liveness":{"score":86,"band":"hot","label":"Hiring now","p_open":1,"p_active":0.86,"p_room":1,"age_days":1,"expected_fill_days":24,"reasons":["seen:1","win:early"],"computed_at":"2026-10-15T05:46:00Z"},"pay":null,"html_url":"https://alion.io/job/vega-intellisoft-lead-data-engineer","json_url":"https://alion.io/job/vega-intellisoft-lead-data-engineer.json","meta":{"generated_at":"2026-10-15T21:44:37Z","cache_seconds":300,"methodology":"https://alion.io/methodology","terms":"https://alion.io/terms","contact":"https://alion.io/contact","api":"https://alion.io/developers","about":"Alion is a live layer of people, companies and AI agents: who they are, whether they are real and active right now, what they do and how to work with them, readable by people and by agents and paid per call.","catalog":"https://alion.io/catalog.json","usage":{"tier":"crawler_verified","counted_by":"address","units_charged":1,"used_today":32970,"day_limit":null,"remaining_today":null,"minute_limit":300,"resets_at":"2026-10-16T00:00:00Z"}},"offers":[{"id":"company.slices","title":"One company in depth, by slice","status":"live","price":{"credits":0.02,"usd":0.002,"plus_per_slice":{"credits":0.05,"usd":0.005}},"unit":"per company, plus each slice with data","note":"the employer in depth","call":{"mcp_tool":"get_company","arguments":{"id":3848052},"rest":"https://alion.io/mcp/rest/get_company?id=3848052"},"human":"https://alion.io/catalog?offer=company.slices&for=job%2Fvega-intellisoft-lead-data-engineer"},{"id":"market.stats","title":"A market slice: pay, demand and time to fill","status":"live","price":{"credits":1,"usd":0.1},"unit":"per slice","note":"pay, demand and time to fill for this role and place","call":{"mcp_tool":"market_stats"},"human":"https://alion.io/catalog?offer=market.stats&for=job%2Fvega-intellisoft-lead-data-engineer"},{"id":"job.search","title":"Open jobs by role, technology, place, pay and visa","status":"live","price":{"credits":0.02,"usd":0.002},"unit":"per posting in a list","note":"similar open postings","call":{"mcp_tool":"search_jobs"},"human":"https://alion.io/catalog?offer=job.search&for=job%2Fvega-intellisoft-lead-data-engineer"},{"id":"company.verify","title":"Is this company real and active right now","status":"pilot","price":null,"unit":"per company","request":{"url":"https://alion.io/catalog/request","method":"POST","body":"{\"offer\": \"company.verify\", \"for\": \"job/vega-intellisoft-lead-data-engineer\", \"note\": \"what you need it for\"}"},"human":"https://alion.io/catalog?offer=company.verify&for=job%2Fvega-intellisoft-lead-data-engineer"}]}