{"id":1280019,"url":"https://alion.io/job/diligentminds-consulting-senior-web-scrapingdata-extraction-engineer","title":"Senior Web Scraping/Data Extraction Engineer","company":{"id":3809029,"name":"DILIGENTMINDS CONSULTING","domain":"diligentmindsconsulting.com","url":"https://alion.io/company/diligentminds-consulting","size_band":null,"is_staffing_agency":false,"employer_type":"direct","is_intermediary":false,"listed_via":null,"ats_vendor":null,"truth_index":null},"role":"Backend","role_family":"Backend","seniority":"senior","employment_type":null,"work_mode":"on_site","remote_scope":null,"remote_scope_basis":null,"remote_working_hours":null,"hiring_geo_confidence":"structured","locations":["Bengaluru, India"],"countries":["IN"],"hiring_countries":[],"hiring_countries_total":0,"salary":null,"salary_estimate":{"min_usd":18000,"max_usd":42000,"period":"year","method":"role_seniority_country_remote_cell","sample_n":159},"experience_years_min":6,"visa_sponsorship":false,"relocation_package":false,"has_equity":false,"technologies":[{"name":"AWS","optional":false},{"name":"AWS Bedrock","optional":false},{"name":"LLM","optional":false},{"name":"Playwright","optional":false}],"status":"live","first_seen_at":"2026-08-21T06:54:17Z","employer_posted_date":null,"last_verified_at":"2026-08-21T06:54:17Z","board_verified":false,"closed_at":null,"days_open":60,"trust":{"level":"not_scored","repost_count":null,"flags":[],"days_open":60},"description":"Job Description:\n\nWeb Scraping and Crawling, AWS Bedrock (AI Development For Scraping)\n\nRole Overview:\n\nWe are looking for an experienced Senior Web Scraping / Data Extraction Engineer with 6+ years of experience owning scraping platforms end-to-end. The ideal candidate should have strong expertise in web crawling, web scraping, data extraction, and scraping frameworks such as Scrapy, Playwright, and Crawlee.\n\nThe role will also involve mentoring engineers, establishing crawling standards, reviewing engineering practices, and leveraging LLM-assisted extraction using AWS Bedrock in a cost-controlled manner.\n\nKey Responsibilities:\n\n- Own and manage the web scraping platform end-to-end, from architecture and development to optimization and maintenance.\n\n- Design, develop, and maintain scalable web crawling and scraping solutions.\n\n- Establish and maintain standards and best practices for web crawling and data extraction.\n\n- Review engineering practices, code, and scraping approaches to ensure quality and scalability.\n\n- Mentor engineers and provide technical guidance on scraping and data extraction practices.\n\n- Develop and optimize scraping solutions using Scrapy, Playwright, and Crawlee.\n\n- Contribute to open-source projects and libraries related to Scrapy, Playwright, or data parsing.\n\n- Implement LLM-assisted data extraction and verification solutions.\n\n- Leverage AWS Bedrock for LLM-based verification while maintaining control over operational costs.\n\n- Identify opportunities to improve scraping efficiency, reliability, and data quality.\n\n- Collaborate with engineering and data teams to build robust data extraction solutions.\n\nRequired Skills & Experience:\n\n- 6+ years of experience owning and managing web scraping/crawling platforms end-to-end.\n\n- Strong experience with Web Scraping and Web Crawling.\n\n- Hands-on expertise with:\n\n1. Scrapy\n\n2. Playwright\n\n3. Crawlee\n\n4. Data parsing and extraction\n\n- Experience mentoring engineers and establishing technical standards.\n\n- Strong understanding of scraping architecture, crawling practices, and data extraction.\n\n- Experience with open-source contributions to Scrapy, Playwright, or parsing libraries.\n\n- Experience with LLM-assisted extraction.\n\n- Hands-on experience with AWS Bedrock for LLM-based verification.\n\n- Strong focus on controlling operational costs while implementing LLM-based solutions.\n\nKey Skills:\n\nScrapy, Playwright, Crawlee, Web Scraping, Web Crawling, Data Scraping, Data Extraction, Data Engineering, AWS Bedrock, LLM, LLM-Assisted Extraction, Data Parsing, Scraping Platform, Open-Source Contributions.\nSkills\nWeb Scraping, AWS Bedrock, Scrapy, Web Crawling, LLM, AI Integration","description_format":"text","description_chars":2685,"description_truncated":false,"requirements":{"experience_years_min":6,"management_years_min":null,"team_size_min":null,"manages_managers":false,"education":null,"security_clearance":false,"languages":[]},"benefits":[],"hiring_locations":[],"hiring_excludes":[],"relocation_offered":false,"industries":["Artificial Intelligence","Commerce","Digital Goods"],"lifecycle":[{"event":"open","at":"2026-09-26T02:00:24Z"}],"visa":[],"liveness":{"score":8,"band":"cold","label":"Long shot","p_open":0.4,"p_active":0.559,"p_room":0.35,"age_days":59,"expected_fill_days":27,"reasons":["seen:59","win:tail"],"computed_at":"2026-10-20T05:45:00Z"},"pay":null,"html_url":"https://alion.io/job/diligentminds-consulting-senior-web-scrapingdata-extraction-engineer","json_url":"https://alion.io/job/diligentminds-consulting-senior-web-scrapingdata-extraction-engineer.json","meta":{"generated_at":"2026-10-21T00:02:30Z","cache_seconds":300,"methodology":"https://alion.io/methodology","terms":"https://alion.io/terms","contact":"https://alion.io/contact","api":"https://alion.io/developers","about":"Alion is a live layer of people, companies and AI agents: who they are, whether they are real and active right now, what they do and how to work with them, readable by people and by agents and paid per call.","catalog":"https://alion.io/catalog.json","usage":{"tier":"crawler_verified","counted_by":"address","units_charged":1,"used_today":51,"day_limit":null,"remaining_today":null,"minute_limit":300,"resets_at":"2026-10-22T00:00:00Z"}},"offers":[{"id":"company.slices","title":"One company in depth, by slice","status":"live","price":{"credits":0.02,"usd":0.002,"plus_per_slice":{"credits":0.05,"usd":0.005}},"unit":"per company, plus each slice with data","note":"the employer in depth","call":{"mcp_tool":"get_company","arguments":{"id":3809029},"rest":"https://alion.io/mcp/rest/get_company?id=3809029"},"human":"https://alion.io/catalog?offer=company.slices&for=job%2Fdiligentminds-consulting-senior-web-scrapingdata-extraction-engineer"},{"id":"market.stats","title":"A market slice: pay, demand and time to fill","status":"live","price":{"credits":1,"usd":0.1},"unit":"per slice","note":"pay, demand and time to fill for this role and place","call":{"mcp_tool":"market_stats"},"human":"https://alion.io/catalog?offer=market.stats&for=job%2Fdiligentminds-consulting-senior-web-scrapingdata-extraction-engineer"},{"id":"job.search","title":"Open jobs by role, technology, place, pay and visa","status":"live","price":{"credits":0.02,"usd":0.002},"unit":"per posting in a list","note":"similar open postings","call":{"mcp_tool":"search_jobs"},"human":"https://alion.io/catalog?offer=job.search&for=job%2Fdiligentminds-consulting-senior-web-scrapingdata-extraction-engineer"},{"id":"company.verify","title":"Is this company real and active right now","status":"pilot","price":null,"unit":"per company","request":{"url":"https://alion.io/catalog/request","method":"POST","body":"{\"offer\": \"company.verify\", \"for\": \"job/diligentminds-consulting-senior-web-scrapingdata-extraction-engineer\", \"note\": \"what you need it for\"}"},"human":"https://alion.io/catalog?offer=company.verify&for=job%2Fdiligentminds-consulting-senior-web-scrapingdata-extraction-engineer"}]}