{"id":1142317,"url":"https://alion.io/job/gramian-consulting-group-ai-quality-assurance-evaluation-specialist","title":"AI Quality Assurance & Evaluation Specialist","company":{"id":5702,"name":"Gramian Consulting Group","domain":"gramianconsultinggroup.com","url":"https://alion.io/company/gramianconsultinggroup","size_band":"1001-5000","is_staffing_agency":false,"is_intermediary":false,"ats_vendor":"Workable","truth_index":{"grade":"A","score":100,"open_postings":3,"ghost_share":0,"stale_share":0,"repost_share":0,"time_to_fill_p50_days":16,"computed_at":"2026-09-23T05:45:00Z"}},"role":"QA","role_family":"QA","seniority":null,"employment_type":"contractor","work_mode":"remote","remote_scope":"stated_countries","hiring_geo_confidence":"inferred","locations":[],"countries":[],"hiring_countries":["AR","BR","CL","PE","UY"],"hiring_countries_total":5,"salary":null,"salary_estimate":null,"experience_years_min":null,"visa_sponsorship":false,"relocation_package":false,"has_equity":false,"technologies":[{"name":"AI Agents","optional":false},{"name":"Python","optional":false},{"name":"SQL","optional":false}],"status":"live","first_seen_at":"2026-09-23T12:51:27Z","employer_posted_date":"2026-09-23","last_verified_at":"2026-09-23T19:27:32Z","board_verified":true,"closed_at":null,"days_open":0,"trust":{"level":"ok","repost_count":null,"flags":[],"days_open":0},"description":"About Gramian\nGramian Consultancy is a boutique consultancy specializing in IT professional services and engineering talent solutions. With a strong background in software engineering and leadership, we help companies build high-performing teams by matching them with professionals who truly fit their needs.\nAbout the Role\nWe are seeking an AI Quality & Evaluation Specialist to validate task quality, review AI agent performance, and audit grading logic. You will analyze execution traces, tool calls, reference solutions, and evaluation criteria to identify task defects, grading errors, and unjustified model failures. The ideal candidate combines technical fluency with strong analytical judgment and the ability to provide clear, evidence-based feedback.\nCONTRACT: Contractor (Hourly)\nCOMMITMENT: 40 hours per week\nLOCATION: Remote - Latin America (LATAM)\nResponsibilities\nValidate task instructions, source materials, reference solutions, and evaluation criteria for consistency and completeness.\nReview AI agent execution traces, tool calls, and generated deliverables to assess whether outcomes are justified.\nAudit grading logic to identify brittle checks, incorrect expected answers, and unsupported rubric criteria.\nIdentify cases where valid alternative solutions are unfairly penalized.\nInvestigate discrepancies between model performance, task defects, grader errors, and environment or tool failures.\nIndependently assess automated QC findings rather than accepting them without verification.\nDocument concise, evidence-backed findings and actionable recommendations.\nClearly communicate uncertainty and distinguish confirmed issues from potential problems.\nVerify that task revisions resolve previously identified defects.\nCollaborate with relevant teams to improve evaluation quality and reliability.\nRequirements\nExperience in technical QA, AI evaluation, data analysis, software testing, or a related technical field.\nComfortable reading and interpreting Python, SQL, shell scripts, structured data, and execution logs.\nStrong analytical skills, including the ability to validate calculations and reconcile conflicting evidence.\nExperience assessing the correctness and completeness of technical or professional deliverables.\nStrong written English and the ability to provide specific, reproducible feedback.\nHigh attention to detail when reviewing task requirements, grading behavior, and model outputs.\nAbility to independently investigate issues and make evidence-based decisions.","description_format":"text","description_chars":2503,"description_truncated":false,"requirements":{"experience_years_min":null,"management_years_min":null,"team_size_min":null,"manages_managers":false,"education":{"level":"bachelor","optional":false},"security_clearance":false,"languages":[{"language":"English","level":"All levels","optional":false}]},"benefits":[],"hiring_locations":[{"name":"Argentina","iso":"AR","kind":"country"},{"name":"Brazil","iso":"BR","kind":"country"},{"name":"Colombia","iso":null,"kind":"country"},{"name":"Chile","iso":"CL","kind":"country"},{"name":"Peru","iso":"PE","kind":"country"},{"name":"Uruguay","iso":"UY","kind":"country"}],"hiring_excludes":[],"relocation_offered":false,"industries":["Professional Services"],"lifecycle":[{"event":"open","at":"2026-09-23T12:51:27Z"}],"liveness":{"score":86,"band":"hot","label":"Hiring now","p_open":1,"p_active":0.86,"p_room":1,"age_days":0,"expected_fill_days":16,"reasons":["conf:0","win:early"],"computed_at":"2026-09-23T19:50:37Z"},"pay":null,"html_url":"https://alion.io/job/gramian-consulting-group-ai-quality-assurance-evaluation-specialist","json_url":"https://alion.io/job/gramian-consulting-group-ai-quality-assurance-evaluation-specialist.json","meta":{"generated_at":"2026-09-23T19:50:37Z","cache_seconds":300,"methodology":"https://alion.io/methodology","terms":"https://alion.io/terms","contact":"https://alion.io/contact","api":"https://alion.io/developers"}}