{"id":44585,"name":"evalchemy","description":"Automatic evals for LLMs","url":"https://github.com/mlfoundations/evalchemy","last_synced_at":"2025-09-05T10:13:51.673Z","repository":{"id":263697862,"uuid":"889248736","full_name":"mlfoundations/evalchemy","owner":"mlfoundations","description":"Automatic evals for LLMs","archived":false,"fork":false,"pushed_at":"2025-06-27T12:13:45.000Z","size":56575,"stargazers_count":522,"open_issues_count":28,"forks_count":62,"subscribers_count":17,"default_branch":"main","last_synced_at":"2025-09-01T03:47:08.840Z","etag":null,"topics":[],"latest_commit_sha":null,"homepage":"","language":"HTML","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":null,"status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/mlfoundations.png","metadata":{"files":{"readme":"README.md","changelog":null,"contributing":"CONTRIBUTING.md","funding":null,"license":null,"code_of_conduct":null,"threat_model":null,"audit":null,"citation":"CITATION.cff","codeowners":null,"security":null,"support":null,"governance":null,"roadmap":null,"authors":null,"dei":null,"publiccode":null,"codemeta":null,"zenodo":null}},"created_at":"2024-11-15T22:43:55.000Z","updated_at":"2025-08-27T23:40:10.000Z","dependencies_parsed_at":"2025-05-28T19:57:42.854Z","dependency_job_id":null,"html_url":"https://github.com/mlfoundations/evalchemy","commit_stats":null,"previous_names":["mlfoundations/evalchemy"],"tags_count":0,"template":false,"template_full_name":null,"purl":"pkg:github/mlfoundations/evalchemy","repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/mlfoundations%2Fevalchemy","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/mlfoundations%2Fevalchemy/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/mlfoundations%2Fevalchemy/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/mlfoundations%2Fevalchemy/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/mlfoundations","download_url":"https://codeload.github.com/mlfoundations/evalchemy/tar.gz/refs/heads/main","sbom_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/mlfoundations%2Fevalchemy/sbom","scorecard":null,"host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":273741853,"owners_count":25159680,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2022-07-04T15:15:14.044Z","status":"online","status_checked_at":"2025-09-05T02:00:09.113Z","response_time":402,"last_error":null,"robots_txt_status":"success","robots_txt_updated_at":"2025-07-24T06:49:26.215Z","robots_txt_url":"https://github.com/robots.txt","online":true,"can_crawl_api":true,"host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"owner":{"login":"mlfoundations","name":"mlfoundations","uuid":"87461581","kind":"organization","description":"","email":null,"website":"https://people.csail.mit.edu/ludwigs/","location":null,"twitter":null,"company":null,"icon_url":"https://avatars.githubusercontent.com/u/87461581?v=4","repositories_count":12,"last_synced_at":"2023-03-04T11:59:23.724Z","metadata":{"has_sponsors_listing":false},"html_url":"https://github.com/mlfoundations","funding_links":[],"total_stars":null,"followers":null,"following":null,"created_at":"2022-11-14T18:20:51.200Z","updated_at":"2023-03-04T11:59:23.747Z","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/mlfoundations","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/mlfoundations/repositories"},"packages":[],"commits":{"id":10277939,"full_name":"mlfoundations/evalchemy","default_branch":"master","committers":null,"total_commits":null,"total_committers":null,"total_bot_commits":null,"total_bot_committers":null,"mean_commits":null,"dds":null,"past_year_committers":null,"past_year_total_commits":null,"past_year_total_committers":null,"past_year_total_bot_commits":null,"past_year_total_bot_committers":null,"past_year_mean_commits":null,"past_year_dds":null,"last_synced_at":null,"last_synced_commit":null,"created_at":"2025-06-09T17:07:32.729Z","updated_at":"2025-06-09T17:07:32.729Z","commits_url":"https://commits.ecosyste.ms/api/v1/hosts/GitHub/repositories/mlfoundations%2Fevalchemy/commits","host":{"name":"GitHub","url":"https://github.com","kind":"github","last_synced_at":"2025-09-05T00:00:10.343Z","repositories_count":5480019,"commits_count":853389012,"contributors_count":31098138,"owners_count":906558,"icon_url":"https://github.com/github.png","host_url":"https://commits.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://commits.ecosyste.ms/api/v1/hosts/GitHub/repositories"}},"issues_stats":{"full_name":"mlfoundations/evalchemy","html_url":"https://github.com/mlfoundations/evalchemy","last_synced_at":"2025-09-03T08:06:32.521Z","status":null,"issues_count":24,"pull_requests_count":163,"avg_time_to_close_issue":1184981.4285714286,"avg_time_to_close_pull_request":347119.0,"issues_closed_count":7,"pull_requests_closed_count":141,"pull_request_authors_count":23,"issue_authors_count":19,"avg_comments_per_issue":0.375,"avg_comments_per_pull_request":0.6748466257668712,"merged_pull_requests_count":131,"bot_issues_count":0,"bot_pull_requests_count":0,"past_year_issues_count":24,"past_year_pull_requests_count":163,"past_year_avg_time_to_close_issue":1184981.4285714286,"past_year_avg_time_to_close_pull_request":347119.0,"past_year_issues_closed_count":7,"past_year_pull_requests_closed_count":141,"past_year_pull_request_authors_count":23,"past_year_issue_authors_count":19,"past_year_avg_comments_per_issue":0.375,"past_year_avg_comments_per_pull_request":0.6748466257668712,"past_year_bot_issues_count":0,"past_year_bot_pull_requests_count":0,"past_year_merged_pull_requests_count":131,"created_at":"2025-06-09T17:07:37.362Z","updated_at":"2025-09-03T08:06:32.523Z","repository_url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/repositories/mlfoundations%2Fevalchemy","issues_url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/repositories/mlfoundations%2Fevalchemy/issues","issue_labels_count":{},"pull_request_labels_count":{},"issue_author_associations_count":{"NONE":18,"MEMBER":4,"COLLABORATOR":2},"pull_request_author_associations_count":{"COLLABORATOR":102,"MEMBER":22,"CONTRIBUTOR":21,"NONE":18},"issue_authors":{"RyanMarten":4,"penfever":2,"sravan500":2,"luckyfan-cs":1,"EtashGuha":1,"TundeAtSN":1,"richardbaihe":1,"aashay-sarvam":1,"Siki-cloud":1,"chanansh":1,"marianna13":1,"noobimp":1,"BeastyZ":1,"juyongjiang":1,"slimfrkha":1,"richardzhuang0412":1,"GeorgiosSmyrnis":1,"hrdkbhatnagar":1,"Mugariya":1},"pull_request_authors":{"neginraoof":32,"sedrick-keh-tri":24,"RyanMarten":22,"jmercat":21,"EtashGuha":11,"penfever":6,"marianna13":6,"ssu53":6,"GeorgiosSmyrnis":4,"asad-aali":4,"Zayne-sprague":3,"Hritikbansal":3,"reinhardh":3,"jonsaadfalcon":2,"younesbelkada":2,"anand-sarvam":2,"john-b-yang":2,"ihebchaa":2,"jannalulu":2,"SGEthan":2,"asuvarna31":2,"dkimds":1,"slimfrkha":1},"host":{"name":"GitHub","url":"https://github.com","kind":"github","last_synced_at":"2025-09-04T00:00:25.939Z","repositories_count":10074573,"issues_count":31266531,"pull_requests_count":95697262,"authors_count":10690890,"icon_url":"https://github.com/github.png","host_url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/repositories","owners_url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/owners","authors_url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors"},"past_year_issue_labels_count":{},"past_year_pull_request_labels_count":{},"past_year_issue_author_associations_count":{"NONE":18,"MEMBER":4,"COLLABORATOR":2},"past_year_pull_request_author_associations_count":{"COLLABORATOR":102,"MEMBER":22,"CONTRIBUTOR":21,"NONE":18},"past_year_issue_authors":{"RyanMarten":4,"sravan500":2,"penfever":2,"TundeAtSN":1,"slimfrkha":1,"Siki-cloud":1,"richardzhuang0412":1,"richardbaihe":1,"noobimp":1,"Mugariya":1,"marianna13":1,"luckyfan-cs":1,"juyongjiang":1,"hrdkbhatnagar":1,"GeorgiosSmyrnis":1,"EtashGuha":1,"chanansh":1,"BeastyZ":1,"aashay-sarvam":1},"past_year_pull_request_authors":{"neginraoof":32,"sedrick-keh-tri":24,"RyanMarten":22,"jmercat":21,"EtashGuha":11,"ssu53":6,"marianna13":6,"penfever":6,"asad-aali":4,"GeorgiosSmyrnis":4,"Zayne-sprague":3,"reinhardh":3,"Hritikbansal":3,"asuvarna31":2,"SGEthan":2,"anand-sarvam":2,"jannalulu":2,"younesbelkada":2,"john-b-yang":2,"ihebchaa":2,"jonsaadfalcon":2,"slimfrkha":1,"dkimds":1},"maintainers":[{"login":"neginraoof","count":32,"url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors/neginraoof"},{"login":"RyanMarten","count":26,"url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors/RyanMarten"},{"login":"sedrick-keh-tri","count":24,"url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors/sedrick-keh-tri"},{"login":"jmercat","count":21,"url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors/jmercat"},{"login":"EtashGuha","count":12,"url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors/EtashGuha"},{"login":"GeorgiosSmyrnis","count":5,"url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors/GeorgiosSmyrnis"},{"login":"reinhardh","count":3,"url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors/reinhardh"},{"login":"Hritikbansal","count":3,"url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors/Hritikbansal"},{"login":"jonsaadfalcon","count":2,"url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors/jonsaadfalcon"},{"login":"SGEthan","count":2,"url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors/SGEthan"}],"active_maintainers":[{"login":"neginraoof","count":32,"url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors/neginraoof"},{"login":"RyanMarten","count":26,"url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors/RyanMarten"},{"login":"sedrick-keh-tri","count":24,"url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors/sedrick-keh-tri"},{"login":"jmercat","count":21,"url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors/jmercat"},{"login":"EtashGuha","count":12,"url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors/EtashGuha"},{"login":"GeorgiosSmyrnis","count":5,"url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors/GeorgiosSmyrnis"},{"login":"reinhardh","count":3,"url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors/reinhardh"},{"login":"Hritikbansal","count":3,"url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors/Hritikbansal"},{"login":"SGEthan","count":2,"url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors/SGEthan"},{"login":"jonsaadfalcon","count":2,"url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors/jonsaadfalcon"}]},"events":{"total":{"CreateEvent":61,"IssuesEvent":32,"WatchEvent":425,"DeleteEvent":37,"IssueCommentEvent":100,"MemberEvent":5,"PushEvent":619,"PublicEvent":1,"PullRequestReviewCommentEvent":23,"PullRequestReviewEvent":55,"PullRequestEvent":148,"ForkEvent":59},"last_year":{"CreateEvent":61,"IssuesEvent":32,"WatchEvent":425,"DeleteEvent":37,"IssueCommentEvent":100,"MemberEvent":5,"PushEvent":619,"PublicEvent":1,"PullRequestReviewCommentEvent":23,"PullRequestReviewEvent":55,"PullRequestEvent":148,"ForkEvent":59}},"keywords":[],"dependencies":[{"ecosystem":"docker","filepath":"eval/chat_benchmarks/MTBench/docker/Dockerfile","sha":null,"kind":"manifest","created_at":"2024-11-20T00:37:59.741Z","updated_at":"2024-11-20T00:37:59.741Z","repository_link":"https://github.com/mlfoundations/evalchemy/blob/main/eval/chat_benchmarks/MTBench/docker/Dockerfile","dependencies":[{"id":20809316572,"package_name":"nvidia/cuda","ecosystem":"docker","requirements":"12.2.0-runtime-ubuntu20.04","direct":true,"kind":"build","optional":false}]},{"ecosystem":"docker","filepath":"eval/chat_benchmarks/MTBench/docker/docker-compose.yml","sha":null,"kind":"manifest","created_at":"2024-11-20T00:38:00.826Z","updated_at":"2024-11-20T00:38:00.826Z","repository_link":"https://github.com/mlfoundations/evalchemy/blob/main/eval/chat_benchmarks/MTBench/docker/docker-compose.yml","dependencies":[{"id":20809317717,"package_name":"fastchat","ecosystem":"docker","requirements":"latest","direct":true,"kind":"runtime","optional":false}]},{"ecosystem":"pypi","filepath":"eval/chat_benchmarks/IFEval/requirements.txt","sha":null,"kind":"manifest","created_at":"2024-11-20T00:38:01.943Z","updated_at":"2024-11-20T00:38:01.943Z","repository_link":"https://github.com/mlfoundations/evalchemy/blob/main/eval/chat_benchmarks/IFEval/requirements.txt","dependencies":[{"id":20809320723,"package_name":"absl","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":20809321933,"package_name":"langdetect","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":20809321934,"package_name":"nltk","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":20809321935,"package_name":"immutabledict","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false}]},{"ecosystem":"pypi","filepath":"eval/chat_benchmarks/MTBench/pyproject.toml","sha":null,"kind":"manifest","created_at":"2024-11-20T00:38:04.286Z","updated_at":"2024-11-20T00:38:04.286Z","repository_link":"https://github.com/mlfoundations/evalchemy/blob/main/eval/chat_benchmarks/MTBench/pyproject.toml","dependencies":[{"id":20809322960,"package_name":"aiohttp","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":20809323236,"package_name":"fastapi","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":20809323237,"package_name":"httpx","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":20809323277,"package_name":"markdown2","ecosystem":"pypi","requirements":"[all]","direct":true,"kind":"runtime","optional":false},{"id":20809323278,"package_name":"nh3","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":20809323279,"package_name":"numpy","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":20809323280,"package_name":"prompt_toolkit","ecosystem":"pypi","requirements":"\u003e=3.0.0","direct":true,"kind":"runtime","optional":false},{"id":20809323281,"package_name":"pydantic","ecosystem":"pypi","requirements":"\u003c3,\u003e=2.0.0","direct":true,"kind":"runtime","optional":false},{"id":20809323595,"package_name":"pydantic-settings","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":20809323596,"package_name":"psutil","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":20809323597,"package_name":"requests","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":20809323598,"package_name":"rich","ecosystem":"pypi","requirements":"\u003e=10.0.0","direct":true,"kind":"runtime","optional":false},{"id":20809323599,"package_name":"shortuuid","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":20809324344,"package_name":"tiktoken","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":20809324852,"package_name":"uvicorn","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false}]},{"ecosystem":"pypi","filepath":"eval/chat_benchmarks/MixEval/setup.py","sha":null,"kind":"manifest","created_at":"2024-11-20T00:38:07.468Z","updated_at":"2024-11-20T00:38:07.468Z","repository_link":"https://github.com/mlfoundations/evalchemy/blob/main/eval/chat_benchmarks/MixEval/setup.py","dependencies":[{"id":20809325273,"package_name":"transformers","ecosystem":"pypi","requirements":"\u003e=4.43.1","direct":true,"kind":"runtime","optional":false},{"id":20809325276,"package_name":"tiktoken","ecosystem":"pypi","requirements":"\u003e=0.6.0","direct":true,"kind":"runtime","optional":false},{"id":20809325278,"package_name":"SentencePiece","ecosystem":"pypi","requirements":"\u003e=0.2.0","direct":true,"kind":"runtime","optional":false},{"id":20809326380,"package_name":"accelerate","ecosystem":"pypi","requirements":"\u003e=0.30.1","direct":true,"kind":"runtime","optional":false},{"id":20809326381,"package_name":"pandas","ecosystem":"pypi","requirements":"\u003e=2.2.2","direct":true,"kind":"runtime","optional":false},{"id":20809326382,"package_name":"scikit-learn","ecosystem":"pypi","requirements":"\u003e=1.5.0","direct":true,"kind":"runtime","optional":false},{"id":20809326383,"package_name":"hf_transfer","ecosystem":"pypi","requirements":"\u003e=0.1.6","direct":true,"kind":"runtime","optional":false},{"id":20809327390,"package_name":"openai","ecosystem":"pypi","requirements":"\u003e=1.30.5","direct":true,"kind":"runtime","optional":false},{"id":20809327391,"package_name":"httpx","ecosystem":"pypi","requirements":"\u003e=0.27.0","direct":true,"kind":"runtime","optional":false},{"id":20809327392,"package_name":"nltk","ecosystem":"pypi","requirements":"\u003e=3.8.1","direct":true,"kind":"runtime","optional":false},{"id":20809327393,"package_name":"numpy","ecosystem":"pypi","requirements":"\u003e=1.26.3","direct":true,"kind":"runtime","optional":false},{"id":20809327394,"package_name":"tqdm","ecosystem":"pypi","requirements":"\u003e=4.66.4","direct":true,"kind":"runtime","optional":false},{"id":20809327395,"package_name":"python-dotenv","ecosystem":"pypi","requirements":"\u003e=1.0.1","direct":true,"kind":"runtime","optional":false},{"id":20809327396,"package_name":"prettytable","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":20809327397,"package_name":"Andere","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false}]},{"ecosystem":"pypi","filepath":"eval/chat_benchmarks/RepoBench/requirements.txt","sha":null,"kind":"manifest","created_at":"2024-11-20T00:38:09.587Z","updated_at":"2024-11-20T00:38:09.587Z","repository_link":"https://github.com/mlfoundations/evalchemy/blob/main/eval/chat_benchmarks/RepoBench/requirements.txt","dependencies":[{"id":20809330099,"package_name":"openai","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":20809330101,"package_name":"transformers","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":20809330102,"package_name":"torch","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":20809330104,"package_name":"fuzzywuzzy","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":20809331388,"package_name":"difflib","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":20809333051,"package_name":"tqdm","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":20809333053,"package_name":"fire","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":20809333054,"package_name":"codebleu","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":20809333055,"package_name":"python-Levenshtein","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":20809333056,"package_name":"tree-sitter-python","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":20809333057,"package_name":"tree-sitter-java","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false}]},{"ecosystem":"pypi","filepath":"eval/chat_benchmarks/WildBench/requirements.txt","sha":null,"kind":"manifest","created_at":"2024-11-20T00:38:12.905Z","updated_at":"2024-11-20T00:38:12.905Z","repository_link":"https://github.com/mlfoundations/evalchemy/blob/main/eval/chat_benchmarks/WildBench/requirements.txt","dependencies":[{"id":20809335611,"package_name":"openai","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":20809335660,"package_name":"datasets","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":20809335667,"package_name":"tenacity","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":20809335671,"package_name":"google-generativeai","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":20809335675,"package_name":"cohere","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":20809335678,"package_name":"mistralai","ecosystem":"pypi","requirements":"==0.4.2","direct":true,"kind":"runtime","optional":false},{"id":20809335681,"package_name":"anthropic","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":20809335682,"package_name":"reka-api","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":20809335761,"package_name":"jsonlines","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":20809335762,"package_name":"together","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":20809335763,"package_name":"fire","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false}]},{"ecosystem":"pypi","filepath":"eval/chat_benchmarks/alpaca_eval/requirements.txt","sha":null,"kind":"manifest","created_at":"2024-11-20T00:38:14.118Z","updated_at":"2024-11-20T00:38:14.118Z","repository_link":"https://github.com/mlfoundations/evalchemy/blob/main/eval/chat_benchmarks/alpaca_eval/requirements.txt","dependencies":[{"id":20809335927,"package_name":"tqdm","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":20809335928,"package_name":"datasets","ecosystem":"pypi","requirements":"\u003e=2.20.0","direct":true,"kind":"runtime","optional":false},{"id":20809335929,"package_name":"openai","ecosystem":"pypi","requirements":"\u003e=1.5.0","direct":true,"kind":"runtime","optional":false},{"id":20809335930,"package_name":"pandas","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":20809335931,"package_name":"tiktoken","ecosystem":"pypi","requirements":"\u003e=0.3.2","direct":true,"kind":"runtime","optional":false},{"id":20809335932,"package_name":"fire","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":20809335934,"package_name":"scipy","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":20809335936,"package_name":"patsy","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":20809335937,"package_name":"scikit-learn","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false}]},{"ecosystem":"pypi","filepath":"eval/chat_benchmarks/alpaca_eval/setup.py","sha":null,"kind":"manifest","created_at":"2024-11-20T00:38:14.539Z","updated_at":"2024-11-20T00:38:14.539Z","repository_link":"https://github.com/mlfoundations/evalchemy/blob/main/eval/chat_benchmarks/alpaca_eval/setup.py","dependencies":[{"id":20809336149,"package_name":"python-dotenv","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":20809336150,"package_name":"datasets","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":20809336151,"package_name":"openai","ecosystem":"pypi","requirements":"\u003e=1.5.0","direct":true,"kind":"runtime","optional":false},{"id":20809336152,"package_name":"pandas","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":20809336153,"package_name":"tiktoken","ecosystem":"pypi","requirements":"\u003e=0.3.2","direct":true,"kind":"runtime","optional":false},{"id":20809336154,"package_name":"fire","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":20809336155,"package_name":"scipy","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":20809336156,"package_name":"huggingface_hub","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":20809336157,"package_name":"patsy","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":20809336158,"package_name":"scikit-learn","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false}]},{"ecosystem":"pypi","filepath":"eval/chat_benchmarks/zeroeval/requirements.txt","sha":null,"kind":"manifest","created_at":"2024-11-20T00:38:15.139Z","updated_at":"2024-11-20T00:38:15.139Z","repository_link":"https://github.com/mlfoundations/evalchemy/blob/main/eval/chat_benchmarks/zeroeval/requirements.txt","dependencies":[{"id":20809336759,"package_name":"openai","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":20809336760,"package_name":"datasets","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":20809336761,"package_name":"tenacity","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":20809336762,"package_name":"google-generativeai","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":20809336763,"package_name":"cohere","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":20809336764,"package_name":"mistralai","ecosystem":"pypi","requirements":"\u003c1.0","direct":true,"kind":"runtime","optional":false},{"id":20809336765,"package_name":"anthropic","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":20809336766,"package_name":"reka-api","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":20809336768,"package_name":"jsonlines","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":20809336769,"package_name":"together","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":20809336770,"package_name":"fire","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false}]},{"ecosystem":"pypi","filepath":"pyproject.toml","sha":null,"kind":"manifest","created_at":"2024-12-06T10:33:32.883Z","updated_at":"2024-12-06T10:33:32.883Z","repository_link":"https://github.com/mlfoundations/evalchemy/blob/main/pyproject.toml","dependencies":[{"id":21058806252,"package_name":"torch","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058806253,"package_name":"torchvision","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058806254,"package_name":"transformers","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058806255,"package_name":"accelerate","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058806256,"package_name":"peft","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058806257,"package_name":"trl","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058806258,"package_name":"bitsandbytes","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058806259,"package_name":"sentencepiece","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058806260,"package_name":"sentence-transformers","ecosystem":"pypi","requirements":"\u003e=2.2.2","direct":true,"kind":"runtime","optional":false},{"id":21058806261,"package_name":"optimum","ecosystem":"pypi","requirements":"==1.12.0","direct":true,"kind":"runtime","optional":false},{"id":21058806262,"package_name":"numpy","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058806263,"package_name":"pandas","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058806264,"package_name":"scipy","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058806265,"package_name":"scikit-learn","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058806266,"package_name":"faiss-cpu","ecosystem":"pypi","requirements":"==1.7.4","direct":true,"kind":"runtime","optional":false},{"id":21058806267,"package_name":"wandb","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058806268,"package_name":"tensorboard","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058806269,"package_name":"evaluate","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058806270,"package_name":"nltk","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058806271,"package_name":"fasttext-wheel","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058806290,"package_name":"codebleu","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058806494,"package_name":"sacrebleu","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058806495,"package_name":"langdetect","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058806496,"package_name":"fuzzywuzzy","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058806497,"package_name":"python-Levenshtein","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058806498,"package_name":"datasets","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058806499,"package_name":"protobuf","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058806500,"package_name":"bs4","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058806501,"package_name":"lxml","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058806502,"package_name":"fastavro","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058806503,"package_name":"jsonpickle","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058806504,"package_name":"hjson","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058806505,"package_name":"msgpack","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058806506,"package_name":"openai","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058806507,"package_name":"anthropic","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058806508,"package_name":"cohere","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058806509,"package_name":"google-generativeai","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058806510,"package_name":"mistralai","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058807128,"package_name":"reka-api","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058807139,"package_name":"together","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058807322,"package_name":"fschat","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058807716,"package_name":"dashscope","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058809643,"package_name":"gcsfs","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058809644,"package_name":"google-cloud-aiplatform","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058809645,"package_name":"google-auth","ecosystem":"pypi","requirements":"==2.25.1","direct":true,"kind":"runtime","optional":false},{"id":21058809646,"package_name":"boto3","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058809647,"package_name":"botocore","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058809648,"package_name":"sagemaker","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058809649,"package_name":"fsspec","ecosystem":"pypi","requirements":"==2024.6.1","direct":true,"kind":"runtime","optional":false},{"id":21058809650,"package_name":"hf-transfer","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058810051,"package_name":"huggingface_hub","ecosystem":"pypi","requirements":"[cli]","direct":true,"kind":"runtime","optional":false},{"id":21058810052,"package_name":"black","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058810053,"package_name":"tree-sitter-python","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058810613,"package_name":"tree-sitter-java","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058810884,"package_name":"tree_sitter","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058810896,"package_name":"asttokens","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058810897,"package_name":"mypy_extensions","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058810898,"package_name":"fastapi","ecosystem":"pypi","requirements":"\u003e=0.101.0","direct":true,"kind":"runtime","optional":false},{"id":21058810899,"package_name":"uvicorn","ecosystem":"pypi","requirements":"\u003e=0.23.0","direct":true,"kind":"runtime","optional":false},{"id":21058810900,"package_name":"aiohttp","ecosystem":"pypi","requirements":"[speedups]\u003e=3.8","direct":true,"kind":"runtime","optional":false},{"id":21058810901,"package_name":"httpx","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058810902,"package_name":"httpx-sse","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058810903,"package_name":"requests","ecosystem":"pypi","requirements":"\u003e=2.28","direct":true,"kind":"runtime","optional":false},{"id":21058810904,"package_name":"websocket","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058811397,"package_name":"aiofiles","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058811398,"package_name":"sqlalchemy","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058811399,"package_name":"psycopg","ecosystem":"pypi","requirements":"[binary]","direct":true,"kind":"runtime","optional":false},{"id":21058811400,"package_name":"psycopg2-binary","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058811401,"package_name":"sqlitedict","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058811425,"package_name":"tqdm","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058811426,"package_name":"python-box","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058811427,"package_name":"fire","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058811428,"package_name":"tiktoken","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058811429,"package_name":"ray","ecosystem":"pypi","requirements":"[default]","direct":true,"kind":"runtime","optional":false},{"id":21058811430,"package_name":"backoff","ecosystem":"pypi","requirements":"\u003e=2.2","direct":true,"kind":"runtime","optional":false},{"id":21058811431,"package_name":"docker-pycreds","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058811432,"package_name":"sentry-sdk","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058811433,"package_name":"loguru","ecosystem":"pypi","requirements":"\u003e=0.7","direct":true,"kind":"runtime","optional":false},{"id":21058811434,"package_name":"python-dotenv","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058811435,"package_name":"shortuuid","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058811436,"package_name":"tenacity","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058811437,"package_name":"prettytable","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058811438,"package_name":"portalocker","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058811439,"package_name":"tabulate","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058811440,"package_name":"colorama","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058811441,"package_name":"termcolor","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058811442,"package_name":"distro","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058811443,"package_name":"typing_inspect","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058811705,"package_name":"py-cpuinfo","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058811707,"package_name":"jiter","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058811709,"package_name":"patsy","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058811711,"package_name":"joblib","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058811713,"package_name":"threadpoolctl","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058811715,"package_name":"jmespath","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21058811717,"package_name":"immutabledict","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false}]},{"ecosystem":"actions","filepath":".github/workflows/black.yaml","sha":null,"kind":"manifest","created_at":"2024-12-28T08:33:02.966Z","updated_at":"2024-12-28T08:33:02.966Z","repository_link":"https://github.com/mlfoundations/evalchemy/blob/main/.github/workflows/black.yaml","dependencies":[{"id":21300341079,"package_name":"actions/checkout","ecosystem":"actions","requirements":"v4","direct":true,"kind":"composite","optional":false},{"id":21300341080,"package_name":"psf/black","ecosystem":"actions","requirements":"stable","direct":true,"kind":"composite","optional":false}]},{"ecosystem":"docker","filepath":"eval/chat_benchmarks/BigCodeBench/docker/Dockerfile","sha":null,"kind":"manifest","created_at":"2025-01-28T08:31:30.166Z","updated_at":"2025-01-28T08:31:30.166Z","repository_link":"https://github.com/mlfoundations/evalchemy/blob/main/eval/chat_benchmarks/BigCodeBench/docker/Dockerfile","dependencies":[{"id":21643609026,"package_name":"nvcr.io/nvidia/pytorch","ecosystem":"docker","requirements":"24.09-py3","direct":true,"kind":"build","optional":false}]},{"ecosystem":"docker","filepath":"eval/chat_benchmarks/MultiPLE/docker/Dockerfile","sha":null,"kind":"manifest","created_at":"2025-01-28T08:31:30.803Z","updated_at":"2025-01-28T08:31:30.803Z","repository_link":"https://github.com/mlfoundations/evalchemy/blob/main/eval/chat_benchmarks/MultiPLE/docker/Dockerfile","dependencies":[{"id":21643609036,"package_name":"nvcr.io/nvidia/pytorch","ecosystem":"docker","requirements":"24.09-py3","direct":true,"kind":"build","optional":false}]},{"ecosystem":"pypi","filepath":"eval/chat_benchmarks/BigCodeBench/requirements/requirements-eval.txt","sha":null,"kind":"manifest","created_at":"2025-01-28T08:31:30.870Z","updated_at":"2025-01-28T08:31:30.870Z","repository_link":"https://github.com/mlfoundations/evalchemy/blob/main/eval/chat_benchmarks/BigCodeBench/requirements/requirements-eval.txt","dependencies":[{"id":21643609037,"package_name":"beautifulsoup4","ecosystem":"pypi","requirements":"==4.8.2","direct":true,"kind":"runtime","optional":false},{"id":21643609038,"package_name":"blake3","ecosystem":"pypi","requirements":"==0.4.1","direct":true,"kind":"runtime","optional":false},{"id":21643609041,"package_name":"chardet","ecosystem":"pypi","requirements":"==5.2.0","direct":true,"kind":"runtime","optional":false},{"id":21643609042,"package_name":"cryptography","ecosystem":"pypi","requirements":"==38.0.0","direct":true,"kind":"runtime","optional":false},{"id":21643609043,"package_name":"datetime","ecosystem":"pypi","requirements":"==5.5","direct":true,"kind":"runtime","optional":false},{"id":21643609044,"package_name":"Django","ecosystem":"pypi","requirements":"==4.2.7","direct":true,"kind":"runtime","optional":false},{"id":21643609045,"package_name":"dnspython","ecosystem":"pypi","requirements":"==2.6.1","direct":true,"kind":"runtime","optional":false},{"id":21643609046,"package_name":"docxtpl","ecosystem":"pypi","requirements":"==0.11.5","direct":true,"kind":"runtime","optional":false},{"id":21643609062,"package_name":"Faker","ecosystem":"pypi","requirements":"==20.1.0","direct":true,"kind":"runtime","optional":false},{"id":21643609063,"package_name":"flask_login","ecosystem":"pypi","requirements":"==0.6.3","direct":true,"kind":"runtime","optional":false},{"id":21643609076,"package_name":"flask_restful","ecosystem":"pypi","requirements":"==0.3.10","direct":true,"kind":"runtime","optional":false},{"id":21643609110,"package_name":"flask_wtf","ecosystem":"pypi","requirements":"==1.2.1","direct":true,"kind":"runtime","optional":false},{"id":21643609130,"package_name":"Flask-Mail","ecosystem":"pypi","requirements":"==0.9.1","direct":true,"kind":"runtime","optional":false},{"id":21643609177,"package_name":"flask","ecosystem":"pypi","requirements":"==3.0.3","direct":true,"kind":"runtime","optional":false},{"id":21643609178,"package_name":"folium","ecosystem":"pypi","requirements":"==0.16.0","direct":true,"kind":"runtime","optional":false},{"id":21643609179,"package_name":"gensim","ecosystem":"pypi","requirements":"==4.3.2","direct":true,"kind":"runtime","optional":false},{"id":21643609180,"package_name":"geopandas","ecosystem":"pypi","requirements":"==0.13.2","direct":true,"kind":"runtime","optional":false},{"id":21643609181,"package_name":"geopy","ecosystem":"pypi","requirements":"==2.4.1","direct":true,"kind":"runtime","optional":false},{"id":21643609182,"package_name":"holidays","ecosystem":"pypi","requirements":"==0.29","direct":true,"kind":"runtime","optional":false},{"id":21643609185,"package_name":"keras","ecosystem":"pypi","requirements":"==2.11.0","direct":true,"kind":"runtime","optional":false},{"id":21643609186,"package_name":"Levenshtein","ecosystem":"pypi","requirements":"==0.25.0","direct":true,"kind":"runtime","optional":false},{"id":21643609187,"package_name":"librosa","ecosystem":"pypi","requirements":"==0.10.1","direct":true,"kind":"runtime","optional":false},{"id":21643609188,"package_name":"lxml","ecosystem":"pypi","requirements":"==4.9.3","direct":true,"kind":"runtime","optional":false},{"id":21643609189,"package_name":"matplotlib","ecosystem":"pypi","requirements":"==3.7.0","direct":true,"kind":"runtime","optional":false},{"id":21643609190,"package_name":"mechanize","ecosystem":"pypi","requirements":"==0.4.9","direct":true,"kind":"runtime","optional":false},{"id":21643609191,"package_name":"natsort","ecosystem":"pypi","requirements":"==7.1.1","direct":true,"kind":"runtime","optional":false},{"id":21643609202,"package_name":"networkx","ecosystem":"pypi","requirements":"==2.6.3","direct":true,"kind":"runtime","optional":false},{"id":21643609203,"package_name":"nltk","ecosystem":"pypi","requirements":"==3.8","direct":true,"kind":"runtime","optional":false},{"id":21643609204,"package_name":"numba","ecosystem":"pypi","requirements":"==0.55.0","direct":true,"kind":"runtime","optional":false},{"id":21643609205,"package_name":"numpy","ecosystem":"pypi","requirements":"==1.21.2","direct":true,"kind":"runtime","optional":false},{"id":21643609206,"package_name":"opencv-python-headless","ecosystem":"pypi","requirements":"==4.9.0.80","direct":true,"kind":"runtime","optional":false},{"id":21643609207,"package_name":"openpyxl","ecosystem":"pypi","requirements":"==3.1.2","direct":true,"kind":"runtime","optional":false},{"id":21643609208,"package_name":"pandas","ecosystem":"pypi","requirements":"==2.0.3","direct":true,"kind":"runtime","optional":false},{"id":21643609209,"package_name":"Pillow","ecosystem":"pypi","requirements":"==10.3.0","direct":true,"kind":"runtime","optional":false},{"id":21643609210,"package_name":"prettytable","ecosystem":"pypi","requirements":"==3.10.0","direct":true,"kind":"runtime","optional":false},{"id":21643609211,"package_name":"psutil","ecosystem":"pypi","requirements":"==5.9.5","direct":true,"kind":"runtime","optional":false},{"id":21643609212,"package_name":"pycryptodome","ecosystem":"pypi","requirements":"==3.14.1","direct":true,"kind":"runtime","optional":false},{"id":21643609213,"package_name":"pyfakefs","ecosystem":"pypi","requirements":"==5.4.1","direct":true,"kind":"runtime","optional":false},{"id":21643609215,"package_name":"pyquery","ecosystem":"pypi","requirements":"==1.4.3","direct":true,"kind":"runtime","optional":false},{"id":21643609220,"package_name":"pytesseract","ecosystem":"pypi","requirements":"==0.3.10","direct":true,"kind":"runtime","optional":false},{"id":21643609221,"package_name":"pytest","ecosystem":"pypi","requirements":"==8.2.0","direct":true,"kind":"runtime","optional":false},{"id":21643609222,"package_name":"python_http_client","ecosystem":"pypi","requirements":"==3.3.7","direct":true,"kind":"runtime","optional":false},{"id":21643609231,"package_name":"python-dateutil","ecosystem":"pypi","requirements":"==2.9.0","direct":true,"kind":"runtime","optional":false},{"id":21643609232,"package_name":"python-docx","ecosystem":"pypi","requirements":"==1.1.0","direct":true,"kind":"runtime","optional":false},{"id":21643609233,"package_name":"python-Levenshtein-wheels","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21643609235,"package_name":"pytz","ecosystem":"pypi","requirements":"==2023.3.post1","direct":true,"kind":"runtime","optional":false},{"id":21643609236,"package_name":"PyYAML","ecosystem":"pypi","requirements":"==6.0.1","direct":true,"kind":"runtime","optional":false},{"id":21643609237,"package_name":"requests_mock","ecosystem":"pypi","requirements":"==1.11.0","direct":true,"kind":"runtime","optional":false},{"id":21643609257,"package_name":"requests","ecosystem":"pypi","requirements":"==2.31.0","direct":true,"kind":"runtime","optional":false},{"id":21643609258,"package_name":"Requests","ecosystem":"pypi","requirements":"==2.31.0","direct":true,"kind":"runtime","optional":false},{"id":21643609297,"package_name":"rsa","ecosystem":"pypi","requirements":"==4.9","direct":true,"kind":"runtime","optional":false},{"id":21643609298,"package_name":"scikit-image","ecosystem":"pypi","requirements":"==0.18.0","direct":true,"kind":"runtime","optional":false},{"id":21643609299,"package_name":"scikit-learn","ecosystem":"pypi","requirements":"==1.3.1","direct":true,"kind":"runtime","optional":false},{"id":21643609300,"package_name":"scipy","ecosystem":"pypi","requirements":"==1.7.2","direct":true,"kind":"runtime","optional":false},{"id":21643609301,"package_name":"seaborn","ecosystem":"pypi","requirements":"==0.13.2","direct":true,"kind":"runtime","optional":false},{"id":21643609302,"package_name":"selenium","ecosystem":"pypi","requirements":"==4.15","direct":true,"kind":"runtime","optional":false},{"id":21643609303,"package_name":"sendgrid","ecosystem":"pypi","requirements":"==6.11.0","direct":true,"kind":"runtime","optional":false},{"id":21643609304,"package_name":"shapely","ecosystem":"pypi","requirements":"==2.0.4","direct":true,"kind":"runtime","optional":false},{"id":21643609305,"package_name":"soundfile","ecosystem":"pypi","requirements":"==0.12.1","direct":true,"kind":"runtime","optional":false},{"id":21643609306,"package_name":"statsmodels","ecosystem":"pypi","requirements":"==0.14.0","direct":true,"kind":"runtime","optional":false},{"id":21643609307,"package_name":"sympy","ecosystem":"pypi","requirements":"==1.12","direct":true,"kind":"runtime","optional":false},{"id":21643609308,"package_name":"tensorflow","ecosystem":"pypi","requirements":"==2.11.0","direct":true,"kind":"runtime","optional":false},{"id":21643609309,"package_name":"textblob","ecosystem":"pypi","requirements":"==0.18.0","direct":true,"kind":"runtime","optional":false},{"id":21643609310,"package_name":"texttable","ecosystem":"pypi","requirements":"==1.7.0","direct":true,"kind":"runtime","optional":false},{"id":21643609311,"package_name":"Werkzeug","ecosystem":"pypi","requirements":"==3.0.1","direct":true,"kind":"runtime","optional":false},{"id":21643609312,"package_name":"wikipedia","ecosystem":"pypi","requirements":"==1.4.0","direct":true,"kind":"runtime","optional":false},{"id":21643609313,"package_name":"wordcloud","ecosystem":"pypi","requirements":"==1.9.3","direct":true,"kind":"runtime","optional":false},{"id":21643609314,"package_name":"wordninja","ecosystem":"pypi","requirements":"==2.0.0","direct":true,"kind":"runtime","optional":false},{"id":21643609315,"package_name":"WTForms","ecosystem":"pypi","requirements":"==3.1.2","direct":true,"kind":"runtime","optional":false},{"id":21643609316,"package_name":"xlrd","ecosystem":"pypi","requirements":"==2.0.1","direct":true,"kind":"runtime","optional":false},{"id":21643609317,"package_name":"xlwt","ecosystem":"pypi","requirements":"==1.3.0","direct":true,"kind":"runtime","optional":false},{"id":21643609318,"package_name":"xmltodict","ecosystem":"pypi","requirements":"==0.13.0","direct":true,"kind":"runtime","optional":false}]},{"ecosystem":"pypi","filepath":"eval/chat_benchmarks/BigCodeBench/requirements/requirements.txt","sha":null,"kind":"manifest","created_at":"2025-01-28T08:31:35.479Z","updated_at":"2025-01-28T08:31:35.479Z","repository_link":"https://github.com/mlfoundations/evalchemy/blob/main/eval/chat_benchmarks/BigCodeBench/requirements/requirements.txt","dependencies":[{"id":21643609410,"package_name":"appdirs","ecosystem":"pypi","requirements":"\u003e=1.4.4","direct":true,"kind":"runtime","optional":false},{"id":21643609413,"package_name":"multipledispatch","ecosystem":"pypi","requirements":"\u003e=0.6.0","direct":true,"kind":"runtime","optional":false},{"id":21643609497,"package_name":"pqdm","ecosystem":"pypi","requirements":"\u003e=0.2.0","direct":true,"kind":"runtime","optional":false},{"id":21643610928,"package_name":"tempdir","ecosystem":"pypi","requirements":"\u003e=0.7.1","direct":true,"kind":"runtime","optional":false},{"id":21643610932,"package_name":"tqdm","ecosystem":"pypi","requirements":"\u003e=4.56.0","direct":true,"kind":"runtime","optional":false},{"id":21643610933,"package_name":"tree_sitter_languages","ecosystem":"pypi","requirements":"\u003e=1.10.2","direct":true,"kind":"runtime","optional":false},{"id":21643610934,"package_name":"tree-sitter","ecosystem":"pypi","requirements":"==0.21.3","direct":true,"kind":"runtime","optional":false},{"id":21643610936,"package_name":"wget","ecosystem":"pypi","requirements":"\u003e=3.2","direct":true,"kind":"runtime","optional":false}]},{"ecosystem":"pypi","filepath":"eval/chat_benchmarks/LiveBench/livebench/if_runner/instruction_following_eval/requirements.txt","sha":null,"kind":"manifest","created_at":"2025-02-12T21:32:17.680Z","updated_at":"2025-02-12T21:32:17.680Z","repository_link":"https://github.com/mlfoundations/evalchemy/blob/main/eval/chat_benchmarks/LiveBench/livebench/if_runner/instruction_following_eval/requirements.txt","dependencies":[{"id":21853841765,"package_name":"absl-py","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21853841766,"package_name":"langdetect","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21853841767,"package_name":"nltk","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21853841768,"package_name":"immutabledict","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false}]},{"ecosystem":"pypi","filepath":"eval/chat_benchmarks/LiveBench/pyproject.toml","sha":null,"kind":"manifest","created_at":"2025-02-12T21:32:18.298Z","updated_at":"2025-02-12T21:32:18.298Z","repository_link":"https://github.com/mlfoundations/evalchemy/blob/main/eval/chat_benchmarks/LiveBench/pyproject.toml","dependencies":[{"id":21853841770,"package_name":"accelerate","ecosystem":"pypi","requirements":"\u003e=0.21","direct":true,"kind":"runtime","optional":false},{"id":21853841771,"package_name":"aiohttp","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21853841772,"package_name":"anthropic","ecosystem":"pypi","requirements":"\u003e=0.3","direct":true,"kind":"runtime","optional":false},{"id":21853841773,"package_name":"antlr4-python3-runtime","ecosystem":"pypi","requirements":"==4.11","direct":true,"kind":"runtime","optional":false},{"id":21853841774,"package_name":"datasets","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21853841775,"package_name":"fastapi","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21853841776,"package_name":"httpx","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21853841777,"package_name":"immutabledict","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21853841778,"package_name":"langchain","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21853841779,"package_name":"langdetect","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21853841780,"package_name":"levenshtein","ecosystem":"pypi","requirements":"\u003e=0.20.4","direct":true,"kind":"runtime","optional":false},{"id":21853841781,"package_name":"lxml","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21853841782,"package_name":"markdown2","ecosystem":"pypi","requirements":"[all]","direct":true,"kind":"runtime","optional":false},{"id":21853841784,"package_name":"nh3","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21853841785,"package_name":"nltk","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21853841786,"package_name":"numpy","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21853841787,"package_name":"openai","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21853841788,"package_name":"packaging","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21853841789,"package_name":"pandas","ecosystem":"pypi","requirements":"==2.2.2","direct":true,"kind":"runtime","optional":false},{"id":21853841790,"package_name":"peft","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21853841791,"package_name":"prompt_toolkit","ecosystem":"pypi","requirements":"\u003e=3.0.0","direct":true,"kind":"runtime","optional":false},{"id":21853841792,"package_name":"protobuf","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21853841793,"package_name":"pydantic","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21853841794,"package_name":"pyext","ecosystem":"pypi","requirements":"==0.7","direct":true,"kind":"runtime","optional":false},{"id":21853841796,"package_name":"psutil","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21853841797,"package_name":"ray","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21853841799,"package_name":"requests","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21853841800,"package_name":"rich","ecosystem":"pypi","requirements":"\u003e=10.0.0","direct":true,"kind":"runtime","optional":false},{"id":21853841801,"package_name":"sentencepiece","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21853841802,"package_name":"shortuuid","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21853841803,"package_name":"sympy","ecosystem":"pypi","requirements":"\u003e=1.12","direct":true,"kind":"runtime","optional":false},{"id":21853841804,"package_name":"tiktoken","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21853841805,"package_name":"torch","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21853841806,"package_name":"transformers","ecosystem":"pypi","requirements":"\u003e=4.31.0","direct":true,"kind":"runtime","optional":false},{"id":21853841807,"package_name":"tqdm","ecosystem":"pypi","requirements":"\u003e=4.62.1","direct":true,"kind":"runtime","optional":false},{"id":21853841808,"package_name":"uvicorn","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21853841809,"package_name":"wheel","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21853841810,"package_name":"fschat","ecosystem":"pypi","requirements":"@git+https://github.com/lm-sys/FastChat#egg=c5223e34babd24c3f9b08205e6751ea6e42c9684","direct":true,"kind":"runtime","optional":false},{"id":21853841812,"package_name":"tenacity","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":21853841813,"package_name":"lark","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false}]},{"ecosystem":"pypi","filepath":"eval/chat_benchmarks/HMMT/matharena/pyproject.toml","sha":null,"kind":"manifest","created_at":"2025-05-28T19:57:41.637Z","updated_at":"2025-05-28T19:57:41.637Z","repository_link":"https://github.com/mlfoundations/evalchemy/blob/main/eval/chat_benchmarks/HMMT/matharena/pyproject.toml","dependencies":[{"id":23349385847,"package_name":"anthropic","ecosystem":"pypi","requirements":"\u003e=0.49.0","direct":true,"kind":"runtime","optional":false},{"id":23349385855,"package_name":"beautifulsoup4","ecosystem":"pypi","requirements":"\u003e=4.13.1","direct":true,"kind":"runtime","optional":false},{"id":23349385861,"package_name":"google-genai","ecosystem":"pypi","requirements":"\u003e=1.11.0","direct":true,"kind":"runtime","optional":false},{"id":23349385865,"package_name":"loguru","ecosystem":"pypi","requirements":"\u003e=0.7.3","direct":true,"kind":"runtime","optional":false},{"id":23349385868,"package_name":"openai","ecosystem":"pypi","requirements":"\u003e=1.67.0","direct":true,"kind":"runtime","optional":false},{"id":23349385869,"package_name":"python-fasthtml","ecosystem":"pypi","requirements":"\u003e=0.12.1","direct":true,"kind":"runtime","optional":false},{"id":23349385871,"package_name":"regex","ecosystem":"pypi","requirements":"\u003e=2024.11.6","direct":true,"kind":"runtime","optional":false},{"id":23349385873,"package_name":"requests","ecosystem":"pypi","requirements":"\u003e=2.32.3","direct":true,"kind":"runtime","optional":false},{"id":23349385876,"package_name":"together","ecosystem":"pypi","requirements":"\u003e=1.3.14","direct":true,"kind":"runtime","optional":false},{"id":23349386675,"package_name":"antlr4-python3-runtime","ecosystem":"pypi","requirements":"==4.11","direct":true,"kind":"runtime","optional":false},{"id":23349386676,"package_name":"sympy","ecosystem":"pypi","requirements":"\u003e=1.13.1","direct":true,"kind":"runtime","optional":false},{"id":23349386677,"package_name":"pandas","ecosystem":"pypi","requirements":"\u003e=2.2.3","direct":true,"kind":"runtime","optional":false},{"id":23349386678,"package_name":"seaborn","ecosystem":"pypi","requirements":"\u003e=0.13.2","direct":true,"kind":"runtime","optional":false},{"id":23349386679,"package_name":"matplotlib","ecosystem":"pypi","requirements":"\u003e=3.9.3","direct":true,"kind":"runtime","optional":false},{"id":23349386680,"package_name":"json5","ecosystem":"pypi","requirements":"\u003e=0.10.0","direct":true,"kind":"runtime","optional":false},{"id":23349386681,"package_name":"datasets","ecosystem":"pypi","requirements":"\u003e= 3.5.0","direct":true,"kind":"runtime","optional":false}]},{"ecosystem":"pypi","filepath":"requirements.txt","sha":null,"kind":"manifest","created_at":"2025-05-28T19:57:42.754Z","updated_at":"2025-05-28T19:57:42.754Z","repository_link":"https://github.com/mlfoundations/evalchemy/blob/main/requirements.txt","dependencies":[{"id":23349386849,"package_name":"fastapi","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":23349386850,"package_name":"uvicorn","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":23349386851,"package_name":"requests","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":23349386852,"package_name":"sqlalchemy","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":23349386853,"package_name":"asyncpg","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":23349386854,"package_name":"aiohttp","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":23349386855,"package_name":"python-json-logger","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":23349386856,"package_name":"psycopg2-binary","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":23349386857,"package_name":"antlr4-python3-runtime","ecosystem":"pypi","requirements":"==4.11","direct":true,"kind":"runtime","optional":false}]}],"score":null,"created_at":"2025-09-04T15:51:19.875Z","updated_at":"2025-10-07T08:17:56.938Z","avatar_url":"https://github.com/mlfoundations.png","language":"HTML","category":null,"sub_category":null,"monthly_downloads":0,"funding_links":[],"readme_doi_urls":[],"works":{},"citation_counts":{},"total_citations":0,"keywords_from_contributors":[],"project_url":"https://science.ecosyste.ms/api/v1/projects/44585","html_url":"https://science.ecosyste.ms/projects/44585","bibtex_url":"https://science.ecosyste.ms/projects/44585/export.bibtex","apalike_url":"https://science.ecosyste.ms/projects/44585/export.apalike"}