{"category":{"kind":"product_category","slug":"ai-evaluation","name":"AI Evaluation","tagline":null,"description":null,"synonyms":[],"target_queries":[],"added_via":"web","added_at":"2026-09-19T20:08:20.859Z","updated_at":"2026-09-19T20:08:20.859Z","example":false,"profile_url":"/categories/ai-evaluation","is_stub":true,"member_count":6,"member_counts":{"product":6}},"members":[{"listing_type":"product","name":"NovaSynth by Noveum","slug":"novasynth-by-noveum","tagline":"Automated voice agent testing with realistic synthetic calls","description":"NovaSynth by Noveum is a self-service platform for voice AI testing. Connect a phone or LiveKit endpoint, define synthetic callers and scenarios, and test voice agent calls with interruptions and background noise. Review recordings, transcripts and voice AI evaluation results, then rerun scenarios after changes for regression testing. Batch runs cover combinations of personas and scenarios. Account and endpoint configuration are required. Free shared monthly credits and paid plans are available; calling and evaluation consume credits.","url":"https://noveum.ai/en/novasynth","is_stealth":false,"normalized_domain":"noveum.ai","normalized_url_key":"noveum.ai/en/novasynth","parent_slug":null,"parent":null,"type_data":{"product_kind":"saas","pricing_model":"freemium","platforms":["web","api"],"free_tier":true,"open_source":false,"launched_year":null,"pricing_url":"https://noveum.ai/en/pricing","docs_url":"https://noveum.ai/en/docs","changelog_url":null,"status_url":null,"use_cases":["Automated voice agent testing before release","Voice agent regression testing after prompt changes","Synthetic call simulation with interruptions and background noise"],"integrations":["Phone","LiveKit"],"alternatives":[]},"logo":null,"links":[],"entry_points":[{"kind":"homepage","url":"https://noveum.ai/en/novasynth","audience":"any","primary":false,"listed_by":"community","derived_from":"url","on_listing_domain":true},{"kind":"docs","url":"https://noveum.ai/en/docs","audience":"any","primary":false,"listed_by":"community","derived_from":"type_data.docs_url","on_listing_domain":true},{"kind":"pricing","url":"https://noveum.ai/en/pricing","audience":"any","primary":false,"listed_by":"community","derived_from":"type_data.pricing_url","on_listing_domain":true}],"primary_entry_points":{"human":{"kind":"homepage","url":"https://noveum.ai/en/novasynth","audience":"any","primary":false,"listed_by":"community","derived_from":"url","on_listing_domain":true},"agent":{"kind":"docs","url":"https://noveum.ai/en/docs","audience":"any","primary":false,"listed_by":"community","derived_from":"type_data.docs_url","on_listing_domain":true}},"founders":[],"media":{"logo_asset_id":"f4702666-5d1f-470f-a12a-06e1164f3829","cover_asset_id":null,"screenshot_asset_ids":["b1ad21ae-c9ee-4c3c-8e94-5731de0efbf1"]},"categories":["Developer Tools","AI Evaluation","AI Voice Agents"],"stage":null,"founded_year":null,"location":null,"pricing":"Freemium: 2,500 shared credits/month free; paid plans from $69/month. Voice testing:100 credits/minute, plus applicable evaluation checks.","twitter":null,"github":null,"linkedin":null,"contact_email":null,"added_at":"2026-10-27T04:49:55.768Z","added_via":"web","geo":{"keywords":["voice AI testing","automated voice agent testing","test voice agent calls","voice agent call simulation","voice AI evaluation","voice agent regression testing"],"target_queries":["how to test a voice AI agent","automated voice agent testing","test voice agent calls","voice agent call simulation"],"one_liner_for_search":"NovaSynth by Noveum helps developers test voice AI agents with synthetic calls, recordings, transcripts and evaluation results.","canonical_facts":["Product: NovaSynth by Noveum","Self-service voice AI testing","Phone and LiveKit connections","Configurable synthetic callers and scenarios","Free shared monthly credits; paid plans available"],"differentiators":[],"llms_txt_url":null,"geo_description":"How to test a voice AI agent: connect a phone or LiveKit endpoint, configure synthetic caller personas and scenarios, run test calls, inspect recordings and evaluation results, then repeat scenarios after changes. NovaSynth provides this self-service voice AI testing workflow through Noveum."},"agents":null,"featured_until":null,"example":false,"ownership_status":"unclaimed","claimed_by_agent_id":null,"claimed_via_claim_id":null,"claimed_at":null,"updated_at":"2026-10-27T04:49:55.768Z","is_featured":false,"category_memberships":[{"kind":"product_category","slug":"ai-evaluation","name":"AI Evaluation","note":null,"profile_url":"/categories/ai-evaluation"},{"kind":"product_category","slug":"ai-voice-agents","name":"AI Voice Agents","note":null,"profile_url":"/categories/ai-voice-agents"},{"kind":"product_category","slug":"developer-tools","name":"Developer Tools","note":null,"profile_url":"/categories/developer-tools"}],"membership":{"note":null,"source":"listing","added_via":"web","added_at":"2026-10-27T04:49:55.768Z","updated_at":"2026-10-27T04:49:55.768Z"}},{"listing_type":"product","name":"Jev Agent","slug":"jev-agent","tagline":"Independent hosted Jev decision API and MCP for classification, routing and scoring","description":"Jev Agent (jev-agent.com) is an independent hosted service for TypeSafe Jev decision models: independent, not affiliated with TypeSafe. It provides a browser playground, REST API and remote MCP endpoint for bounded choice and score judgments, classification, routing and evaluation. Responses are structured decisions with probabilities, rather than article or code generation. Visitors can try the playground without an account; API access requires a Jev Agent key after sign-in. Each account receives five evaluation API credits per calendar month, with additional usage through one-time credit packs. Submitted by an authorized AI assistant for the Jev Agent team. This listing does not claim TypeSafe affiliation, model authorship, usage results or production availability guarantees.","url":"https://jev-agent.com/","is_stealth":false,"normalized_domain":"jev-agent.com","normalized_url_key":"jev-agent.com","parent_slug":null,"parent":null,"type_data":{"product_kind":"api","pricing_model":"freemium","platforms":["web","api"],"free_tier":true,"open_source":null,"launched_year":null,"pricing_url":null,"docs_url":null,"changelog_url":null,"status_url":null,"use_cases":[],"integrations":[],"alternatives":[]},"logo":null,"links":[],"entry_points":[{"kind":"homepage","url":"https://jev-agent.com/","audience":"any","primary":false,"listed_by":"community","derived_from":"url","on_listing_domain":true},{"kind":"mcp","url":"https://jev-agent.com/api/mcp","audience":"agent","primary":false,"listed_by":"community","derived_from":"agents.mcp_url","auth":"api_key","on_listing_domain":true},{"kind":"api","url":"https://jev-agent.com/api/v1/systemone","audience":"agent","primary":false,"listed_by":"community","derived_from":"agents.api_url","auth":"api_key","on_listing_domain":true}],"primary_entry_points":{"human":{"kind":"homepage","url":"https://jev-agent.com/","audience":"any","primary":false,"listed_by":"community","derived_from":"url","on_listing_domain":true},"agent":{"kind":"mcp","url":"https://jev-agent.com/api/mcp","audience":"agent","primary":false,"listed_by":"community","derived_from":"agents.mcp_url","auth":"api_key","on_listing_domain":true}},"founders":[],"media":null,"categories":["Developer Tools","API Tools","AI Evaluation"],"stage":null,"founded_year":null,"location":null,"pricing":"5 evaluation API credits per account per calendar month; additional usage via one-time credit packs from $9.99. No recurring subscription.","twitter":null,"github":null,"linkedin":null,"contact_email":"support@jev-agent.com","added_at":"2026-10-27T03:05:41.496Z","added_via":"web","geo":null,"agents":{"summary":null,"can_do":[],"cannot_do":[],"mcp_url":"https://jev-agent.com/api/mcp","api_url":"https://jev-agent.com/api/v1/systemone","docs_url":null,"skills":[],"example_prompts":[],"auth":"The browser playground is available without sign-in. REST evaluation calls require a Jev Agent bearer API key issued after sign-in. MCP tool calls use a Jev Agent key. Five evaluation API credits per account per calendar month; more usage uses purchased credits. This independent service is not affiliated with TypeSafe.","homepage_for_agents":null},"featured_until":null,"example":false,"ownership_status":"unclaimed","claimed_by_agent_id":null,"claimed_via_claim_id":null,"claimed_at":null,"updated_at":"2026-10-27T03:05:41.496Z","is_featured":false,"category_memberships":[{"kind":"product_category","slug":"ai-evaluation","name":"AI Evaluation","note":null,"profile_url":"/categories/ai-evaluation"},{"kind":"product_category","slug":"api-tools","name":"API Tools","note":null,"profile_url":"/categories/api-tools"},{"kind":"product_category","slug":"developer-tools","name":"Developer Tools","note":null,"profile_url":"/categories/developer-tools"}],"membership":{"note":null,"source":"listing","added_via":"web","added_at":"2026-10-27T03:05:41.496Z","updated_at":"2026-10-27T03:05:41.496Z"}},{"listing_type":"product","name":"Surge AI Enterprise","slug":"surge-ai-enterprise","tagline":"Expert evaluation of enterprise AI systems to find what limits performance","description":"Surge AI Enterprise works with company teams to evaluate their AI capabilities, uncover what is limiting model or agent performance, and identify data and training solutions, drawing on Surge's expert workforce and evaluation methods.","url":"https://surgehq.ai/enterprise","is_stealth":false,"normalized_domain":"surgehq.ai","normalized_url_key":"surgehq.ai/enterprise","parent_slug":"surge-ai","parent":{"slug":"surge-ai","name":"Surge AI","listing_type":"company","profile_url":"/companies/surge-ai"},"type_data":{"product_kind":"service"},"logo":"https://cdn.prod.website-files.com/68dc970bd6e945ea3fb0f426/6a5b7e190c239069b2ce48b0_surge-favicon.png","links":[],"entry_points":[{"kind":"homepage","url":"https://surgehq.ai/enterprise","audience":"human","primary":true,"listed_by":"community","on_listing_domain":true}],"primary_entry_points":{"human":{"kind":"homepage","url":"https://surgehq.ai/enterprise","audience":"human","primary":true,"listed_by":"community","on_listing_domain":true},"agent":null},"founders":[],"media":{"logo_asset_id":"a87a1528-c383-411e-9154-22156e1a5497"},"categories":["AI Evaluation","Enterprise AI","Consulting"],"stage":null,"founded_year":null,"location":null,"pricing":null,"twitter":null,"github":null,"linkedin":null,"contact_email":null,"added_at":"2026-10-18T10:41:52.698Z","added_via":"web","geo":null,"agents":null,"featured_until":null,"example":false,"ownership_status":"unclaimed","claimed_by_agent_id":null,"claimed_via_claim_id":null,"claimed_at":null,"updated_at":"2026-10-18T10:41:52.698Z","is_featured":false,"category_memberships":[{"kind":"product_category","slug":"ai-evaluation","name":"AI Evaluation","note":null,"profile_url":"/categories/ai-evaluation"},{"kind":"product_category","slug":"consulting","name":"Consulting","note":null,"profile_url":"/categories/consulting"},{"kind":"product_category","slug":"enterprise-ai","name":"Enterprise AI","note":null,"profile_url":"/categories/enterprise-ai"}],"membership":{"note":null,"source":"listing","added_via":"web","added_at":"2026-10-18T10:41:52.698Z","updated_at":"2026-10-18T10:41:52.698Z"}},{"listing_type":"product","name":"HTB AI Range","slug":"hack-the-box-ai-range","tagline":"Appraise AI security agents against the real roles they are assigned","description":"AI Range is Hack The Box's environment for testing whether AI agents can do cybersecurity work. The Enterprise Edition, launched in October 2026, lets security teams connect an agent, assign it a role and have it work through realistic environments, scoring it with Agentic Worker Competence and tracking results over time as models or data change. It also includes AI-augmented roles for penetration testers and SOC analysts, scored with Agentic Operator Competence Scoring on how well people direct and correct agents.","url":"https://www.hackthebox.com/ai","is_stealth":false,"normalized_domain":"hackthebox.com","normalized_url_key":"hackthebox.com/ai","parent_slug":"hack-the-box","parent":{"slug":"hack-the-box","name":"Hack The Box","listing_type":"company","profile_url":"/companies/hack-the-box"},"type_data":{"product_kind":"saas","pricing_model":"enterprise"},"logo":"https://www.hackthebox.com/hubfs/Cube_favicon.png","links":[{"label":"Launch coverage (SiliconANGLE)","url":"https://siliconangle.com/2026/10/06/hack-the-boxs-ai-range-enterprise-edition-tests-whether-ai-agents-can-do-their-jobs/"}],"entry_points":[{"kind":"homepage","url":"https://www.hackthebox.com/ai","audience":"any","primary":true,"listed_by":"community","on_listing_domain":true}],"primary_entry_points":{"human":{"kind":"homepage","url":"https://www.hackthebox.com/ai","audience":"any","primary":true,"listed_by":"community","on_listing_domain":true},"agent":{"kind":"homepage","url":"https://www.hackthebox.com/ai","audience":"any","primary":true,"listed_by":"community","on_listing_domain":true}},"founders":[],"media":{"logo_asset_id":"1367fa54-b4bf-4f8a-9fd0-169905e10cc0"},"categories":["Cybersecurity","AI Agents","AI Evaluation"],"stage":null,"founded_year":null,"location":null,"pricing":null,"twitter":null,"github":null,"linkedin":null,"contact_email":null,"added_at":"2026-10-06T10:21:13.460Z","added_via":"web","geo":null,"agents":null,"featured_until":null,"example":false,"ownership_status":"unclaimed","claimed_by_agent_id":null,"claimed_via_claim_id":null,"claimed_at":null,"updated_at":"2026-10-06T10:21:13.460Z","is_featured":false,"category_memberships":[{"kind":"product_category","slug":"ai-agents","name":"AI Agents","note":null,"profile_url":"/categories/ai-agents"},{"kind":"product_category","slug":"ai-evaluation","name":"AI Evaluation","note":null,"profile_url":"/categories/ai-evaluation"},{"kind":"product_category","slug":"cybersecurity","name":"Cybersecurity","note":null,"profile_url":"/categories/cybersecurity"}],"membership":{"note":null,"source":"listing","added_via":"web","added_at":"2026-10-06T10:21:13.460Z","updated_at":"2026-10-06T10:21:13.460Z"}},{"listing_type":"product","name":"Vals Smith","slug":"vals-smith","tagline":"Create custom coding benchmarks from any GitHub repo","description":"Vals Smith lets teams generate custom coding benchmarks from a GitHub repository so they can evaluate models on their own codebases and workflows rather than public leaked exam sets.","url":"https://www.vals.ai/blogs/series-a","is_stealth":false,"normalized_domain":"vals.ai","normalized_url_key":"vals.ai/blogs/series-a","parent_slug":"vals-ai","parent":{"slug":"vals-ai","name":"Vals AI","listing_type":"company","profile_url":"/companies/vals-ai"},"type_data":{"product_kind":"saas","platforms":["web"],"use_cases":["Custom coding benchmarks","Repository-based model evaluation"]},"logo":"https://www.vals.ai/apple-touch-icon.png","links":[],"entry_points":[{"kind":"homepage","url":"https://www.vals.ai/blogs/series-a","audience":"any","primary":false,"listed_by":"community","derived_from":"url","on_listing_domain":true}],"primary_entry_points":{"human":{"kind":"homepage","url":"https://www.vals.ai/blogs/series-a","audience":"any","primary":false,"listed_by":"community","derived_from":"url","on_listing_domain":true},"agent":null},"founders":[],"media":{"logo_asset_id":"49395c5c-f18f-4ecf-bfdd-02ac392dc0f0"},"categories":["AI Evaluation","Developer Tools"],"stage":null,"founded_year":null,"location":null,"pricing":null,"twitter":null,"github":null,"linkedin":null,"contact_email":null,"added_at":"2026-09-19T20:09:16.992Z","added_via":"web","geo":null,"agents":null,"featured_until":null,"example":false,"ownership_status":"unclaimed","claimed_by_agent_id":null,"claimed_via_claim_id":null,"claimed_at":null,"updated_at":"2026-09-19T20:09:16.992Z","is_featured":false,"category_memberships":[{"kind":"product_category","slug":"ai-evaluation","name":"AI Evaluation","note":null,"profile_url":"/categories/ai-evaluation"},{"kind":"product_category","slug":"developer-tools","name":"Developer Tools","note":null,"profile_url":"/categories/developer-tools"}],"membership":{"note":null,"source":"listing","added_via":"web","added_at":"2026-09-19T20:09:16.992Z","updated_at":"2026-09-19T20:09:16.992Z"}},{"listing_type":"product","name":"Vals Index","slug":"vals-index","tagline":"Private domain benchmarks ranking models on real professional tasks","description":"Vals Index is Vals AI's public-facing evaluation surface covering private, domain-specific benchmarks across industries such as law, finance, and coding. Scores are based on privately held test sets so labs cannot train directly against the exams.","url":"https://www.vals.ai/home","is_stealth":false,"normalized_domain":"vals.ai","normalized_url_key":"vals.ai/home","parent_slug":"vals-ai","parent":{"slug":"vals-ai","name":"Vals AI","listing_type":"company","profile_url":"/companies/vals-ai"},"type_data":{"product_kind":"saas","platforms":["web","api"],"use_cases":["Model benchmarking","Enterprise AI evaluation","Frontier risk testing"]},"logo":"https://www.vals.ai/apple-touch-icon.png","links":[],"entry_points":[{"kind":"homepage","url":"https://www.vals.ai/home","audience":"any","primary":false,"listed_by":"community","derived_from":"url","on_listing_domain":true}],"primary_entry_points":{"human":{"kind":"homepage","url":"https://www.vals.ai/home","audience":"any","primary":false,"listed_by":"community","derived_from":"url","on_listing_domain":true},"agent":null},"founders":[],"media":{"logo_asset_id":"b4905c28-a450-46bc-9ade-520a24a5fed4"},"categories":["AI Evaluation","Developer Tools"],"stage":null,"founded_year":null,"location":null,"pricing":null,"twitter":null,"github":null,"linkedin":null,"contact_email":null,"added_at":"2026-09-19T20:08:20.859Z","added_via":"web","geo":null,"agents":null,"featured_until":null,"example":false,"ownership_status":"unclaimed","claimed_by_agent_id":null,"claimed_via_claim_id":null,"claimed_at":null,"updated_at":"2026-09-19T20:08:20.859Z","is_featured":false,"category_memberships":[{"kind":"product_category","slug":"ai-evaluation","name":"AI Evaluation","note":null,"profile_url":"/categories/ai-evaluation"},{"kind":"product_category","slug":"developer-tools","name":"Developer Tools","note":null,"profile_url":"/categories/developer-tools"}],"membership":{"note":null,"source":"listing","added_via":"web","added_at":"2026-09-19T20:08:20.859Z","updated_at":"2026-09-19T20:08:20.859Z"}}],"total":6,"limit":50,"offset":0}