{"id":2290350,"url":"https://alion.io/job/awisee-senior-data-infrastructure-engineer-scraping-scale","title":"Senior Data Infrastructure Engineer (Scraping & Scale)","company":{"id":2336065,"name":"Awisee","domain":"awisee.agency","url":"https://alion.io/company/awisee","size_band":null,"is_staffing_agency":false,"employer_type":"direct","is_intermediary":false,"listed_via":null,"ats_vendor":null,"truth_index":null},"role":"Data Science","role_family":"Data Science","seniority":"senior","employment_type":"full_time","work_mode":"remote","remote_scope":"stated_countries","remote_scope_basis":"inferred_company_offices","remote_working_hours":null,"hiring_geo_confidence":"inferred","locations":[],"countries":[],"hiring_countries":["US"],"hiring_countries_total":1,"salary":null,"salary_estimate":{"min_usd":87000,"max_usd":197000,"period":"year","method":"global_role_seniority_cell","sample_n":2324},"experience_years_min":4,"visa_sponsorship":false,"relocation_package":false,"has_equity":false,"technologies":[{"name":"Akamai","optional":false},{"name":"Amazon S3","optional":false},{"name":"Apache Iceberg","optional":false},{"name":"Apache Kafka","optional":false},{"name":"Celery","optional":false},{"name":"Cloudflare","optional":false},{"name":"Playwright","optional":false},{"name":"Python","optional":false},{"name":"Ray","optional":false},{"name":"Redis","optional":false},{"name":"Scrapy","optional":false},{"name":"Selenium","optional":false},{"name":"Snowflake","optional":false},{"name":"SQL","optional":false}],"status":"live","first_seen_at":"2026-10-11T16:03:40Z","employer_posted_date":null,"last_verified_at":"2026-10-11T16:03:40Z","board_verified":false,"closed_at":null,"days_open":3,"trust":{"level":"not_scored","repost_count":null,"flags":[],"days_open":3},"description":"This is a remote position.We are looking for a Senior Data Infrastructure Engineer specializing in web scraping, anti-bot evasion, and large-scale data ingestion. In this role, you will build and maintain the core ingestion engine powering our social media data lab. You will overcome complex platform defenses to deliver millions of profile, post, and video records daily with near-zero downtime. Responsibilities\n\nAnti-Bot Evasion Architecture: Build and manage stealth scraping clusters using residential proxy networks, TLS fingerprinting, headful/headless browser farms (Playwright, Puppeteer), and session rotation. \n\nHigh-Throughput Pipelines: Build fault-tolerant, scalable web-scraping pipelines that extract data from Instagram, TikTok, YouTube, X, and web sources. \n\nPipeline Orchestration: Design distributed queues and workflow engines (Temporal, Ray, Apache Kafka, Celery) to manage millions of asynchronous scraping tasks daily. \n\nStorage & Data Lake Management: Architect structured and unstructured storage environments (Parquet, Apache Iceberg, Snowflake, S3) for downstream AI modeling. \n\nMonitoring & Evasion Recovery: Implement automated alerts for platform UI/API changes, blocking patterns, and proxy failures. \n\nRequirements\n\n4+ years of experience in high-volume web scraping, data engineering, or reverse engineering. \n\nDeep experience defeating advanced anti-bot providers (Cloudflare, Akamai, PerimeterX) via TLS impersonation, browser automation, and proxy management. \n\nMastery of Python or Go, with deep knowledge of Playwright, Puppeteer, Scrapy, or Selenium. \n\nExperience with distributed systems and task queues (Temporal, Ray, Kafka, Redis). \n\nStrong SQL skills and experience with modern analytical data lakes / data warehouses. \n\nPreferred Qualifications\nDirect experience extracting short-form video content and user metadata from TikTok, Instagram, and YouTube. \n\nExperience integrating scraping outputs directly into vector databases and real-time AI processing queues. \n\nOriginally posted on Himalayas","description_format":"text","description_chars":2042,"description_truncated":false,"requirements":{"experience_years_min":4,"management_years_min":null,"team_size_min":null,"manages_managers":false,"education":null,"security_clearance":false,"languages":[]},"benefits":[],"hiring_locations":[{"name":"United States","iso":"US","kind":"country"}],"hiring_excludes":[],"relocation_offered":false,"industries":[],"lifecycle":[{"event":"open","at":"2026-10-11T17:28:30Z"}],"visa":[],"liveness":{"score":86,"band":"hot","label":"Hiring now","p_open":1,"p_active":0.86,"p_room":1,"age_days":3,"expected_fill_days":29,"reasons":["seen:3","win:early"],"computed_at":"2026-10-15T05:46:00Z"},"pay":null,"html_url":"https://alion.io/job/awisee-senior-data-infrastructure-engineer-scraping-scale","json_url":"https://alion.io/job/awisee-senior-data-infrastructure-engineer-scraping-scale.json","meta":{"generated_at":"2026-10-15T11:14:16Z","cache_seconds":300,"methodology":"https://alion.io/methodology","terms":"https://alion.io/terms","contact":"https://alion.io/contact","api":"https://alion.io/developers","about":"Alion is a live layer of people, companies and AI agents: who they are, whether they are real and active right now, what they do and how to work with them, readable by people and by agents and paid per call.","catalog":"https://alion.io/catalog.json","usage":{"tier":"crawler_verified","counted_by":"address","units_charged":1,"used_today":17336,"day_limit":null,"remaining_today":null,"minute_limit":300,"resets_at":"2026-10-16T00:00:00Z"}},"offers":[{"id":"company.slices","title":"One company in depth, by slice","status":"live","price":{"credits":0.02,"usd":0.002,"plus_per_slice":{"credits":0.05,"usd":0.005}},"unit":"per company, plus each slice with data","note":"the employer in depth","call":{"mcp_tool":"get_company","arguments":{"id":2336065},"rest":"https://alion.io/mcp/rest/get_company?id=2336065"},"human":"https://alion.io/catalog?offer=company.slices&for=job%2Fawisee-senior-data-infrastructure-engineer-scraping-scale"},{"id":"market.stats","title":"A market slice: pay, demand and time to fill","status":"live","price":{"credits":1,"usd":0.1},"unit":"per slice","note":"pay, demand and time to fill for this role and place","call":{"mcp_tool":"market_stats"},"human":"https://alion.io/catalog?offer=market.stats&for=job%2Fawisee-senior-data-infrastructure-engineer-scraping-scale"},{"id":"job.search","title":"Open jobs by role, technology, place, pay and visa","status":"live","price":{"credits":0.02,"usd":0.002},"unit":"per posting in a list","note":"similar open postings","call":{"mcp_tool":"search_jobs"},"human":"https://alion.io/catalog?offer=job.search&for=job%2Fawisee-senior-data-infrastructure-engineer-scraping-scale"},{"id":"company.verify","title":"Is this company real and active right now","status":"pilot","price":null,"unit":"per company","request":{"url":"https://alion.io/catalog/request","method":"POST","body":"{\"offer\": \"company.verify\", \"for\": \"job/awisee-senior-data-infrastructure-engineer-scraping-scale\", \"note\": \"what you need it for\"}"},"human":"https://alion.io/catalog?offer=company.verify&for=job%2Fawisee-senior-data-infrastructure-engineer-scraping-scale"}]}