{"id":146682,"name":null,"description":"A framework for the evaluation of autoregressive code generation language models.","url":"https://github.com/bigcode-project/bigcode-evaluation-harness","last_synced_at":"2025-09-09T14:40:19.515Z","repository":{"id":60283294,"uuid":"522950827","full_name":"bigcode-project/bigcode-evaluation-harness","owner":"bigcode-project","description":"A framework for the evaluation of autoregressive code generation language models.","archived":false,"fork":false,"pushed_at":"2025-07-01T09:16:37.000Z","size":887,"stargazers_count":951,"open_issues_count":92,"forks_count":244,"subscribers_count":11,"default_branch":"main","last_synced_at":"2025-07-01T10:29:23.668Z","etag":null,"topics":[],"latest_commit_sha":null,"homepage":null,"language":"Python","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":"apache-2.0","status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/bigcode-project.png","metadata":{"files":{"readme":"README.md","changelog":null,"contributing":"CONTRIBUTING.md","funding":null,"license":"LICENSE","code_of_conduct":null,"threat_model":null,"audit":null,"citation":null,"codeowners":null,"security":null,"support":null,"governance":null,"roadmap":null,"authors":null,"dei":null,"publiccode":null,"codemeta":null}},"created_at":"2022-08-09T12:58:56.000Z","updated_at":"2025-07-01T09:16:41.000Z","dependencies_parsed_at":"2023-02-19T09:55:17.009Z","dependency_job_id":"2f654e39-ef8b-4410-a205-0f84ffb4270c","html_url":"https://github.com/bigcode-project/bigcode-evaluation-harness","commit_stats":null,"previous_names":[],"tags_count":1,"template":false,"template_full_name":null,"purl":"pkg:github/bigcode-project/bigcode-evaluation-harness","repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/bigcode-project%2Fbigcode-evaluation-harness","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/bigcode-project%2Fbigcode-evaluation-harness/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/bigcode-project%2Fbigcode-evaluation-harness/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/bigcode-project%2Fbigcode-evaluation-harness/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/bigcode-project","download_url":"https://codeload.github.com/bigcode-project/bigcode-evaluation-harness/tar.gz/refs/heads/main","sbom_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/bigcode-project%2Fbigcode-evaluation-harness/sbom","scorecard":null,"host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":274314153,"owners_count":25262570,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2022-07-04T15:15:14.044Z","status":"online","status_checked_at":"2025-09-09T02:00:10.223Z","response_time":80,"last_error":null,"robots_txt_status":"success","robots_txt_updated_at":"2025-07-24T06:49:26.215Z","robots_txt_url":"https://github.com/robots.txt","online":true,"can_crawl_api":true,"host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"owner":{"login":"bigcode-project","name":"BigCode Project","uuid":"110470554","kind":"organization","description":"BigCode Project is an open scientific collaboration run by Hugging Face and ServiceNow Research, focused on open and responsible development of LLMs for code.","email":"contact@bigcode-project.org","website":"https://www.bigcode-project.org/","location":null,"twitter":"BigCodeProject","company":null,"icon_url":"https://avatars.githubusercontent.com/u/110470554?v=4","repositories_count":26,"last_synced_at":"2024-04-28T01:28:43.383Z","metadata":{"has_sponsors_listing":false},"html_url":"https://github.com/bigcode-project","funding_links":[],"total_stars":10426,"followers":1353,"following":0,"created_at":"2023-02-14T20:55:16.318Z","updated_at":"2024-04-28T01:28:53.518Z","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/bigcode-project","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/bigcode-project/repositories"},"packages":[{"id":10352291,"name":"github.com/bigcode-project/bigcode-evaluation-harness","ecosystem":"go","description":null,"homepage":null,"licenses":"apache-2.0","normalized_licenses":["Apache-2.0"],"repository_url":"https://github.com/bigcode-project/bigcode-evaluation-harness","keywords_array":[],"namespace":null,"versions_count":1,"first_release_published_at":"2023-05-24T10:12:04.000Z","latest_release_published_at":"2023-05-24T10:12:04.000Z","latest_release_number":"v0.1.0","last_synced_at":"2025-09-02T22:46:44.252Z","created_at":"2024-06-11T15:43:44.593Z","updated_at":"2025-09-02T22:46:44.252Z","registry_url":"https://pkg.go.dev/github.com/bigcode-project/bigcode-evaluation-harness","install_command":"go get github.com/bigcode-project/bigcode-evaluation-harness","documentation_url":"https://pkg.go.dev/github.com/bigcode-project/bigcode-evaluation-harness#section-documentation","metadata":{},"repo_metadata":{"id":60283294,"uuid":"522950827","full_name":"bigcode-project/bigcode-evaluation-harness","owner":"bigcode-project","description":"A framework for the evaluation of autoregressive code generation language models.","archived":false,"fork":false,"pushed_at":"2024-05-05T22:31:13.000Z","size":780,"stargazers_count":641,"open_issues_count":58,"forks_count":155,"subscribers_count":13,"default_branch":"main","last_synced_at":"2024-05-05T23:28:54.357Z","etag":null,"topics":[],"latest_commit_sha":null,"homepage":null,"language":"Python","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":"apache-2.0","status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/bigcode-project.png","metadata":{"files":{"readme":"README.md","changelog":null,"contributing":"CONTRIBUTING.md","funding":null,"license":"LICENSE","code_of_conduct":null,"threat_model":null,"audit":null,"citation":null,"codeowners":null,"security":null,"support":null,"governance":null,"roadmap":null,"authors":null,"dei":null,"publiccode":null,"codemeta":null}},"created_at":"2022-08-09T12:58:56.000Z","updated_at":"2024-05-30T01:10:02.530Z","dependencies_parsed_at":"2023-02-19T09:55:17.009Z","dependency_job_id":"2f654e39-ef8b-4410-a205-0f84ffb4270c","html_url":"https://github.com/bigcode-project/bigcode-evaluation-harness","commit_stats":null,"previous_names":[],"tags_count":1,"template":false,"template_full_name":null,"repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/bigcode-project%2Fbigcode-evaluation-harness","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/bigcode-project%2Fbigcode-evaluation-harness/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/bigcode-project%2Fbigcode-evaluation-harness/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/bigcode-project%2Fbigcode-evaluation-harness/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/bigcode-project","download_url":"https://codeload.github.com/bigcode-project/bigcode-evaluation-harness/tar.gz/refs/heads/main","host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":206658505,"owners_count":14607145,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2022-07-04T15:15:14.044Z","host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"repo_metadata_updated_at":"2024-06-12T04:46:29.198Z","dependent_packages_count":0,"downloads":null,"downloads_period":null,"dependent_repos_count":0,"rankings":{"downloads":null,"dependent_repos_count":6.966061736475957,"dependent_packages_count":6.527826853133711,"stargazers_count":null,"forks_count":null,"docker_downloads_count":null,"average":6.746944294804834},"purl":"pkg:golang/github.com/bigcode-project/bigcode-evaluation-harness","advisories":[],"docker_usage_url":"https://docker.ecosyste.ms/usage/go/github.com/bigcode-project/bigcode-evaluation-harness","docker_dependents_count":null,"docker_downloads_count":null,"usage_url":"https://repos.ecosyste.ms/usage/go/github.com/bigcode-project/bigcode-evaluation-harness","dependent_repositories_url":"https://repos.ecosyste.ms/api/v1/usage/go/github.com/bigcode-project/bigcode-evaluation-harness/dependencies","status":null,"funding_links":[],"critical":null,"issue_metadata":null,"versions_url":"https://packages.ecosyste.ms/api/v1/registries/proxy.golang.org/packages/github.com%2Fbigcode-project%2Fbigcode-evaluation-harness/versions","version_numbers_url":"https://packages.ecosyste.ms/api/v1/registries/proxy.golang.org/packages/github.com%2Fbigcode-project%2Fbigcode-evaluation-harness/version_numbers","dependent_packages_url":"https://packages.ecosyste.ms/api/v1/registries/proxy.golang.org/packages/github.com%2Fbigcode-project%2Fbigcode-evaluation-harness/dependent_packages","related_packages_url":"https://packages.ecosyste.ms/api/v1/registries/proxy.golang.org/packages/github.com%2Fbigcode-project%2Fbigcode-evaluation-harness/related_packages","maintainers":[],"registry":{"name":"proxy.golang.org","url":"https://proxy.golang.org","ecosystem":"go","default":true,"packages_count":1954770,"maintainers_count":0,"namespaces_count":741920,"keywords_count":109374,"github":"golang","metadata":{"funded_packages_count":49379},"icon_url":"https://github.com/golang.png","created_at":"2022-04-04T15:19:22.939Z","updated_at":"2025-09-09T06:45:13.065Z","packages_url":"https://packages.ecosyste.ms/api/v1/registries/proxy.golang.org/packages","maintainers_url":"https://packages.ecosyste.ms/api/v1/registries/proxy.golang.org/maintainers","namespaces_url":"https://packages.ecosyste.ms/api/v1/registries/proxy.golang.org/namespaces"}}],"commits":{"message":"Repository syncing started."},"issues_stats":{"full_name":"bigcode-project/bigcode-evaluation-harness","html_url":"https://github.com/bigcode-project/bigcode-evaluation-harness","last_synced_at":"2025-08-31T10:20:59.481Z","status":null,"issues_count":7,"pull_requests_count":11,"avg_time_to_close_issue":37737400.0,"avg_time_to_close_pull_request":17514949.0,"issues_closed_count":1,"pull_requests_closed_count":4,"pull_request_authors_count":10,"issue_authors_count":7,"avg_comments_per_issue":0.0,"avg_comments_per_pull_request":0.9090909090909091,"merged_pull_requests_count":1,"bot_issues_count":0,"bot_pull_requests_count":0,"past_year_issues_count":6,"past_year_pull_requests_count":9,"past_year_avg_time_to_close_issue":null,"past_year_avg_time_to_close_pull_request":35.0,"past_year_issues_closed_count":0,"past_year_pull_requests_closed_count":2,"past_year_pull_request_authors_count":8,"past_year_issue_authors_count":6,"past_year_avg_comments_per_issue":0.0,"past_year_avg_comments_per_pull_request":0.0,"past_year_bot_issues_count":0,"past_year_bot_pull_requests_count":0,"past_year_merged_pull_requests_count":0,"created_at":"2025-07-16T12:08:18.062Z","updated_at":"2025-09-03T04:41:21.761Z","repository_url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/repositories/bigcode-project%2Fbigcode-evaluation-harness","issues_url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/repositories/bigcode-project%2Fbigcode-evaluation-harness/issues","issue_labels_count":{},"pull_request_labels_count":{},"issue_author_associations_count":{"NONE":69,"CONTRIBUTOR":2},"pull_request_author_associations_count":{"NONE":44,"CONTRIBUTOR":11,"COLLABORATOR":7},"issue_authors":{"shuaiwang2022":3,"virt9":3,"Vipitis":3,"ALLISWELL8":2,"phqtuyen":2,"alat-rights":2,"icoderzqliu":2,"RylanSchaeffer":2,"noforit":2,"burger-pb":2,"kimheeyori":1,"zongyf02":1,"showlibia":1,"edgan8":1,"yifan-bao":1,"Extirpater":1,"fxnie":1,"pengzhangzhi":1,"manthan0227":1,"s-natsubori":1,"wwngh1233":1,"cosmo3769":1,"liw8hz":1,"ganler":1,"xinghuang2050":1,"guanqun-yang":1,"DonteFlynn":1,"zxiangx":1,"Liuzzyg":1,"toptechie156":1,"JohnneyQin":1,"Santhoshkumar-p":1,"kbmlcoding":1,"login256":1,"ankush13r":1,"sadaisystems":1,"Randl":1,"shehrozek-cerebras":1,"yxchng":1,"SummCoder":1,"Gene-su-GGN":1,"YangZhou08":1,"marybloodyzz":1,"kn0wn-cyber":1,"jessyford":1,"anil-gurbuz":1,"aladinggit":1,"Hambaobao":1,"Sunt-ing":1,"moyi-qwq":1,"inigopm":1,"Elfsong":1,"nielstron":1,"zhimin-z":1,"hitesh-1997":1,"Reeleon":1,"PeiqinSun":1,"berserank":1},"pull_request_authors":{"loubnabnl":6,"arjunguha":5,"pawelknes":3,"ganler":3,"Muennighoff":3,"Vipitis":2,"gameofby":2,"shehrozek-cerebras":2,"nielstron":1,"q-rz":1,"keyboardAnt":1,"ab-10":1,"meher-m":1,"younesbelkada":1,"seobeomjin":1,"zkcpku":1,"kbmlcoding":1,"tarahjjeon":1,"abhinavnmagic":1,"sad-mathematician":1,"sxjscience":1,"TK-21st":1,"Anindyadeep":1,"ahmedashrafy":1,"maxmatical":1,"ggcr":1,"ArtemisDicoTiar":1,"IQ17":1,"envsp":1,"koking0":1,"AnitaLiu98":1,"zawedcvg":1,"hrshtv":1,"nmanthan":1,"anil-gurbuz":1,"phuonglvh":1,"hitesh-1997":1,"mostafaelhoushi":1,"hmellor":1,"Abhineetsoccer":1,"michaelfeil":1,"neehar18":1,"TechxGenus":1,"zoya-hammad":1},"host":{"name":"GitHub","url":"https://github.com","kind":"github","last_synced_at":"2025-09-03T00:00:08.273Z","repositories_count":10061379,"issues_count":31059648,"pull_requests_count":94180017,"authors_count":10686068,"icon_url":"https://github.com/github.png","host_url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/repositories","owners_url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/owners","authors_url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors"},"past_year_issue_labels_count":{},"past_year_pull_request_labels_count":{},"past_year_issue_author_associations_count":{"NONE":17},"past_year_pull_request_author_associations_count":{"NONE":20,"CONTRIBUTOR":2},"past_year_issue_authors":{"ankush13r":1,"Extirpater":1,"fxnie":1,"JohnneyQin":1,"kbmlcoding":1,"kn0wn-cyber":1,"Liuzzyg":1,"login256":1,"manthan0227":1,"marybloodyzz":1,"nielstron":1,"pengzhangzhi":1,"RylanSchaeffer":1,"showlibia":1,"yxchng":1,"zongyf02":1,"zxiangx":1},"past_year_pull_request_authors":{"pawelknes":3,"arjunguha":2,"gameofby":2,"zkcpku":1,"zoya-hammad":1,"envsp":1,"ahmedashrafy":1,"TK-21st":1,"younesbelkada":1,"ArtemisDicoTiar":1,"michaelfeil":1,"nmanthan":1,"neehar18":1,"tarahjjeon":1,"sad-mathematician":1,"ggcr":1,"hrshtv":1,"zawedcvg":1},"maintainers":[{"login":"loubnabnl","count":6,"url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors/loubnabnl"},{"login":"maxmatical","count":1,"url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors/maxmatical"}],"active_maintainers":[]},"events":{"total":{"IssuesEvent":20,"WatchEvent":180,"IssueCommentEvent":19,"PushEvent":3,"PullRequestEvent":28,"PullRequestReviewEvent":4,"ForkEvent":45,"CreateEvent":1},"last_year":{"IssuesEvent":20,"WatchEvent":180,"IssueCommentEvent":19,"PushEvent":3,"PullRequestEvent":28,"PullRequestReviewEvent":4,"ForkEvent":45,"CreateEvent":1}},"keywords":[],"dependencies":[{"ecosystem":"actions","filepath":".github/workflows/ci.yml","sha":null,"kind":"manifest","created_at":"2023-02-19T09:55:16.879Z","updated_at":"2023-02-19T09:55:16.879Z","repository_link":"https://github.com/bigcode-project/bigcode-evaluation-harness/blob/main/.github/workflows/ci.yml","dependencies":[{"id":7784194226,"package_name":"actions/checkout","ecosystem":"actions","requirements":"v3","direct":true,"kind":"composite","optional":false},{"id":7784194227,"package_name":"actions/setup-python","ecosystem":"actions","requirements":"v3","direct":true,"kind":"composite","optional":false}]},{"ecosystem":"pypi","filepath":"requirements.txt","sha":null,"kind":"manifest","created_at":"2022-09-27T16:00:26.804Z","updated_at":"2022-09-27T16:00:26.804Z","repository_link":"https://github.com/bigcode-project/bigcode-evaluation-harness/blob/main/requirements.txt","dependencies":[{"id":6180018875,"package_name":"transformers","ecosystem":"pypi","requirements":"==4.20.0","direct":true,"kind":"runtime","optional":false},{"id":6180018876,"package_name":"accelerate","ecosystem":"pypi","requirements":"==0.11.0","direct":true,"kind":"runtime","optional":false},{"id":6180018877,"package_name":"datasets","ecosystem":"pypi","requirements":"==2.3.2","direct":true,"kind":"runtime","optional":false},{"id":6180018878,"package_name":"evaluate","ecosystem":"pypi","requirements":"==0.1.2","direct":true,"kind":"runtime","optional":false},{"id":6180018879,"package_name":"pyext","ecosystem":"pypi","requirements":"==0.7","direct":true,"kind":"runtime","optional":false},{"id":6180018880,"package_name":"mosestokenizer","ecosystem":"pypi","requirements":"==1.0.0","direct":true,"kind":"runtime","optional":false}]}],"score":null,"created_at":"2025-09-09T00:22:41.829Z","updated_at":"2025-10-07T08:36:44.074Z","avatar_url":"https://github.com/bigcode-project.png","language":"Python","category":null,"sub_category":null,"monthly_downloads":0,"funding_links":[],"readme_doi_urls":[],"works":{},"citation_counts":{},"total_citations":0,"keywords_from_contributors":[],"project_url":"https://science.ecosyste.ms/api/v1/projects/146682","html_url":"https://science.ecosyste.ms/projects/146682"}