{"id":3751520,"url":"https://alion.io/job/highbrow-technology-data-engineer-pyspark","title":"Data Engineer – PySpark","company":{"id":3808937,"name":"Highbrow Technology","domain":"highbrowtechnology.com","url":"https://alion.io/company/highbrow-technology-private-limited","size_band":null,"is_staffing_agency":false,"employer_type":"direct","is_intermediary":false,"listed_via":null,"ats_vendor":null,"truth_index":null},"role":"Data Science","role_family":"Data Science","seniority":null,"employment_type":"full_time","work_mode":"hybrid","remote_scope":null,"remote_scope_basis":null,"remote_working_hours":null,"hiring_geo_confidence":"structured","locations":["Durham, United States"],"countries":["US"],"hiring_countries":[],"hiring_countries_total":0,"salary":null,"salary_estimate":null,"experience_years_min":null,"visa_sponsorship":false,"relocation_package":false,"has_equity":false,"technologies":[{"name":"Airflow","optional":false},{"name":"AWS","optional":false},{"name":"Azure","optional":false},{"name":"CI/CD","optional":false},{"name":"Databricks","optional":false},{"name":"Delta Lake","optional":false},{"name":"ETL/ELT","optional":false},{"name":"pySpark","optional":false},{"name":"Snowflake","optional":false},{"name":"Spark","optional":false},{"name":"SQL","optional":false},{"name":"Python","optional":true}],"status":"closed","first_seen_at":"2026-11-04T15:20:24Z","employer_posted_date":null,"last_verified_at":"2026-11-04T21:35:15Z","board_verified":false,"closed_at":"2026-11-04T21:35:15Z","days_open":0,"trust":{"level":"not_scored","repost_count":null,"flags":[],"days_open":0},"description":"Highbrow Technology Inc is seeking a Data Engineer to design, develop, and maintain scalable data engineering solutions. The role focuses on PySpark-based data pipelines, ETL/ELT workflows, distributed data processing, and support for data lakes and data warehouses.\n\nResponsibilities\nDesign, develop, and maintain scalable data pipelines using PySpark and Python\nBuild and optimize ETL/ELT workflows for processing large volumes of structured and unstructured data\nDevelop data transformation and data processing solutions using Apache Spark\nWork with distributed data processing frameworks and big data technologies\nPerform data cleansing, validation, integration, and transformation\nOptimize Spark jobs for performance, scalability, and reliability\nCollaborate with data architects, analysts, and other engineering teams\nTroubleshoot data pipeline issues and ensure data quality and consistency\nSupport data lake and data warehouse solutions\n\nSkills\n6+ years of experience in Data Engineering or related roles\nStrong hands-on experience with PySpark and Python\nStrong experience with Apache Spark and distributed data processing\nExperience developing and maintaining ETL/ELT data pipelines\nStrong SQL skills and experience working with large datasets\nKnowledge of data warehousing and data lake concepts\nExperience with data transformation, optimization, and performance tuning\nAbility to work independently and collaboratively in a hybrid work environment\nVisa-independent work authorization\nExperience with cloud platforms such as AWS, Azure, or GCP\nKnowledge of Databricks, Delta Lake, or similar data platforms\nExperience with Snowflake or other cloud data warehouses\nFamiliarity with Airflow or other workflow orchestration tools\nExperience with CI/CD and data engineering best practices\n\nBenefits\nHybrid – Onsite\n\nCompany Overview\nHighbrow Technology Inc. is a global technology solutions provider, currently assisting clients in the Generative AI (GenAI) space. It was founded in 2011, and is headquartered in Cary, North Carolina, USA, with a workforce of 201-500 employees. Its website is https://highbrowtechnology.com.","description_format":"text","description_chars":2132,"description_truncated":false,"requirements":{"experience_years_min":null,"management_years_min":null,"team_size_min":null,"manages_managers":false,"education":null,"security_clearance":false,"languages":[]},"benefits":["Hybrid work"],"hiring_locations":[],"hiring_excludes":[],"relocation_offered":false,"industries":["IT Consulting & Digital Transformation","IT Staffing & Staff Augmentation"],"lifecycle":[{"event":"open","at":"2026-11-04T15:20:58Z"},{"event":"close","at":"2026-11-04T21:35:15Z"}],"visa":[{"country":"US","licensed_sponsor":true,"evidence":"H-1B filings in 12 months: 32","filings_12m":32,"filings_prev_12m":27,"green_card_filings_12m":0,"median_offered_wage_usd":92500,"route":null,"cap_exempt":false,"checked_at":"2026-10-29T16:27:21+00:00","sources":["US Department of Labor: LCA disclosure data (H-1B, H-1B1, E-3)","US Department of Labor: PERM disclosure data (green cards)"],"filings_for_role_12m":7}],"liveness":null,"pay":null,"html_url":"https://alion.io/job/highbrow-technology-data-engineer-pyspark","json_url":"https://alion.io/job/highbrow-technology-data-engineer-pyspark.json","meta":{"generated_at":"2026-11-06T09:11:15Z","cache_seconds":300,"methodology":"https://alion.io/methodology","terms":"https://alion.io/terms","contact":"https://alion.io/contact","api":"https://alion.io/developers","about":"Alion is a live layer of people, companies and AI agents: who they are, whether they are real and active right now, what they do and how to work with them, readable by people and by agents and paid per call.","catalog":"https://alion.io/catalog.json","usage":{"tier":"crawler_verified","counted_by":"address","units_charged":1,"used_today":16632,"day_limit":null,"remaining_today":null,"minute_limit":300,"resets_at":"2026-11-07T00:00:00Z"}},"offers":[{"id":"company.slices","title":"One company in depth, by slice","status":"live","price":{"credits":0.02,"usd":0.002,"plus_per_slice":{"credits":0.05,"usd":0.005}},"unit":"per company, plus each slice with data","note":"the employer in depth","call":{"mcp_tool":"get_company","arguments":{"id":3808937},"rest":"https://alion.io/mcp/rest/get_company?id=3808937"},"human":"https://alion.io/catalog?offer=company.slices&for=job%2Fhighbrow-technology-data-engineer-pyspark"},{"id":"market.stats","title":"A market slice: pay, demand and time to fill","status":"live","price":{"credits":1,"usd":0.1},"unit":"per slice","note":"pay, demand and time to fill for this role and place","call":{"mcp_tool":"market_stats"},"human":"https://alion.io/catalog?offer=market.stats&for=job%2Fhighbrow-technology-data-engineer-pyspark"},{"id":"job.search","title":"Open jobs by role, technology, place, pay and visa","status":"live","price":{"credits":0.02,"usd":0.002},"unit":"per posting in a list","note":"similar open postings","call":{"mcp_tool":"search_jobs"},"human":"https://alion.io/catalog?offer=job.search&for=job%2Fhighbrow-technology-data-engineer-pyspark"},{"id":"company.verify","title":"Is this company real and active right now","status":"pilot","price":null,"unit":"per company","request":{"url":"https://alion.io/catalog/request","method":"POST","body":"{\"offer\": \"company.verify\", \"for\": \"job/highbrow-technology-data-engineer-pyspark\", \"note\": \"what you need it for\"}"},"human":"https://alion.io/catalog?offer=company.verify&for=job%2Fhighbrow-technology-data-engineer-pyspark"}],"agent":{"mcp":"https://alion.io/mcp/jobs","mcp_open":"https://alion.io/mcp/open","next":[{"tool":"get_job","arguments":{"id":3751520}},{"tool":"search_jobs","arguments":{"company":"highbrowtechnology.com","detail":"compact"}},{"tool":"match_people","arguments":{"job_id":3751520},"note":"people open to offers who fit this vacancy"}],"note":"The same data through MCP tools: https://alion.io/mcp/open needs no sign-in (compact searches, postings, people); https://alion.io/mcp with an Alion account has everything. Docs: https://alion.io/developers"}}