{"id":97996,"name":null,"description":"LLM-KG-Bench is a Framework and task collection for automated benchmarking of Large Language Models (LLMs) on Knowledge Graph (KG) related tasks.","url":"https://github.com/aksw/llm-kg-bench","last_synced_at":"2025-09-09T02:12:23.022Z","repository":{"id":178638151,"uuid":"644904022","full_name":"AKSW/LLM-KG-Bench","owner":"AKSW","description":"LLM-KG-Bench is a Framework and task collection for automated benchmarking of Large Language Models (LLMs) on Knowledge Graph (KG) related tasks.","archived":false,"fork":false,"pushed_at":"2025-06-20T07:10:49.000Z","size":21740,"stargazers_count":42,"open_issues_count":9,"forks_count":5,"subscribers_count":26,"default_branch":"main","last_synced_at":"2025-08-31T11:56:16.275Z","etag":null,"topics":["knowledge-graph","large-language-models","llm","llm-benchmarking","rdf","sparql"],"latest_commit_sha":null,"homepage":"","language":"Python","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":"mpl-2.0","status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/AKSW.png","metadata":{"files":{"readme":"README.md","changelog":"CHANGELOG.md","contributing":null,"funding":null,"license":"LICENSE","code_of_conduct":null,"threat_model":null,"audit":null,"citation":null,"codeowners":null,"security":null,"support":null,"governance":null,"roadmap":null,"authors":null,"dei":null,"publiccode":null,"codemeta":null,"zenodo":null}},"created_at":"2023-05-24T13:49:02.000Z","updated_at":"2025-08-15T08:31:06.000Z","dependencies_parsed_at":"2023-11-28T11:26:44.050Z","dependency_job_id":"d1e57680-a6c2-4311-8d7d-e59db7b96ab6","html_url":"https://github.com/AKSW/LLM-KG-Bench","commit_stats":null,"previous_names":["aksw/llm-kg-bench"],"tags_count":6,"template":false,"template_full_name":null,"purl":"pkg:github/AKSW/LLM-KG-Bench","repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/AKSW%2FLLM-KG-Bench","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/AKSW%2FLLM-KG-Bench/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/AKSW%2FLLM-KG-Bench/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/AKSW%2FLLM-KG-Bench/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/AKSW","download_url":"https://codeload.github.com/AKSW/LLM-KG-Bench/tar.gz/refs/heads/main","sbom_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/AKSW%2FLLM-KG-Bench/sbom","scorecard":null,"host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":274231860,"owners_count":25245858,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2022-07-04T15:15:14.044Z","status":"online","status_checked_at":"2025-09-09T02:00:10.223Z","response_time":80,"last_error":null,"robots_txt_status":"success","robots_txt_updated_at":"2025-07-24T06:49:26.215Z","robots_txt_url":"https://github.com/robots.txt","online":true,"can_crawl_api":true,"host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"owner":{"login":"AKSW","name":"AKSW Research Group @ University of Leipzig","uuid":"770163","kind":"organization","description":"","email":null,"website":"http://aksw.org","location":"Leipzig","twitter":null,"company":null,"icon_url":"https://avatars.githubusercontent.com/u/770163?v=4","repositories_count":358,"last_synced_at":"2024-05-21T03:54:44.689Z","metadata":{"has_sponsors_listing":false},"html_url":"https://github.com/AKSW","funding_links":[],"total_stars":1311,"followers":31,"following":0,"created_at":"2022-11-03T01:32:23.671Z","updated_at":"2024-05-21T03:54:47.899Z","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/AKSW","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/AKSW/repositories"},"packages":[],"commits":{"message":"Repository syncing started."},"issues_stats":{"full_name":"AKSW/LLM-KG-Bench","html_url":"https://github.com/AKSW/LLM-KG-Bench","last_synced_at":"2025-08-31T08:34:48.853Z","status":null,"issues_count":4,"pull_requests_count":2,"avg_time_to_close_issue":17566986.5,"avg_time_to_close_pull_request":105.5,"issues_closed_count":2,"pull_requests_closed_count":2,"pull_request_authors_count":1,"issue_authors_count":3,"avg_comments_per_issue":0.75,"avg_comments_per_pull_request":0.0,"merged_pull_requests_count":2,"bot_issues_count":0,"bot_pull_requests_count":0,"past_year_issues_count":3,"past_year_pull_requests_count":2,"past_year_avg_time_to_close_issue":215173.0,"past_year_avg_time_to_close_pull_request":105.5,"past_year_issues_closed_count":1,"past_year_pull_requests_closed_count":2,"past_year_pull_request_authors_count":1,"past_year_issue_authors_count":2,"past_year_avg_comments_per_issue":0.6666666666666666,"past_year_avg_comments_per_pull_request":0.0,"past_year_bot_issues_count":0,"past_year_bot_pull_requests_count":0,"past_year_merged_pull_requests_count":2,"created_at":"2025-08-29T13:04:23.798Z","updated_at":"2025-09-03T05:31:21.279Z","repository_url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/repositories/AKSW%2FLLM-KG-Bench","issues_url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/repositories/AKSW%2FLLM-KG-Bench/issues","issue_labels_count":{"enhancement":2,"documentation":2,"bug":1},"pull_request_labels_count":{},"issue_author_associations_count":{"MEMBER":8,"NONE":1,"CONTRIBUTOR":1},"pull_request_author_associations_count":{"MEMBER":5,"COLLABORATOR":3},"issue_authors":{"lpmeyer":7,"nishadi":1,"JJ-Author":1,"white-gecko":1},"pull_request_authors":{"lpmeyer":7,"deheim":1},"host":{"name":"GitHub","url":"https://github.com","kind":"github","last_synced_at":"2025-09-09T00:00:11.778Z","repositories_count":10174310,"issues_count":31919157,"pull_requests_count":100070755,"authors_count":10734448,"icon_url":"https://github.com/github.png","host_url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/repositories","owners_url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/owners","authors_url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors"},"past_year_issue_labels_count":{"documentation":2,"bug":1},"past_year_pull_request_labels_count":{},"past_year_issue_author_associations_count":{"MEMBER":7,"NONE":1},"past_year_pull_request_author_associations_count":{"MEMBER":5,"COLLABORATOR":1},"past_year_issue_authors":{"lpmeyer":7,"nishadi":1},"past_year_pull_request_authors":{"lpmeyer":5,"deheim":1},"maintainers":[{"login":"lpmeyer","count":14,"url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors/lpmeyer"},{"login":"deheim","count":1,"url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors/deheim"},{"login":"white-gecko","count":1,"url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors/white-gecko"}],"active_maintainers":[{"login":"lpmeyer","count":12,"url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors/lpmeyer"},{"login":"deheim","count":1,"url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors/deheim"}]},"events":{"total":{"CreateEvent":5,"ReleaseEvent":3,"IssuesEvent":14,"WatchEvent":22,"DeleteEvent":1,"IssueCommentEvent":9,"PushEvent":143,"PullRequestEvent":10,"ForkEvent":2},"last_year":{"CreateEvent":5,"ReleaseEvent":3,"IssuesEvent":14,"WatchEvent":22,"DeleteEvent":1,"IssueCommentEvent":9,"PushEvent":143,"PullRequestEvent":10,"ForkEvent":2}},"keywords":["knowledge-graph","large-language-models","llm","llm-benchmarking","rdf","sparql"],"dependencies":[{"ecosystem":"pypi","filepath":"poetry.lock","sha":null,"kind":"lockfile","created_at":"2023-09-22T13:39:06.090Z","updated_at":"2023-09-22T13:39:06.090Z","repository_link":"https://github.com/AKSW/LLM-KG-Bench/blob/main/poetry.lock","dependencies":[{"id":13875080646,"package_name":"aiohttp","ecosystem":"pypi","requirements":"3.8.5","direct":false,"kind":"runtime","optional":false},{"id":13875080647,"package_name":"aiosignal","ecosystem":"pypi","requirements":"1.3.1","direct":false,"kind":"runtime","optional":false},{"id":13875080648,"package_name":"anthropic","ecosystem":"pypi","requirements":"0.3.7","direct":false,"kind":"runtime","optional":false},{"id":13875080649,"package_name":"anyio","ecosystem":"pypi","requirements":"3.7.1","direct":false,"kind":"runtime","optional":false},{"id":13875080650,"package_name":"async-timeout","ecosystem":"pypi","requirements":"4.0.2","direct":false,"kind":"runtime","optional":false},{"id":13875080651,"package_name":"attrs","ecosystem":"pypi","requirements":"23.1.0","direct":false,"kind":"runtime","optional":false},{"id":13875080652,"package_name":"backoff","ecosystem":"pypi","requirements":"2.2.1","direct":false,"kind":"runtime","optional":false},{"id":13875080653,"package_name":"certifi","ecosystem":"pypi","requirements":"2023.7.22","direct":false,"kind":"runtime","optional":false},{"id":13875080654,"package_name":"charset-normalizer","ecosystem":"pypi","requirements":"3.2.0","direct":false,"kind":"runtime","optional":false},{"id":13875080655,"package_name":"colorama","ecosystem":"pypi","requirements":"0.4.6","direct":false,"kind":"runtime","optional":false},{"id":13875080656,"package_name":"contourpy","ecosystem":"pypi","requirements":"1.1.0","direct":false,"kind":"runtime","optional":false},{"id":13875080657,"package_name":"cycler","ecosystem":"pypi","requirements":"0.11.0","direct":false,"kind":"runtime","optional":false},{"id":13875080658,"package_name":"distro","ecosystem":"pypi","requirements":"1.8.0","direct":false,"kind":"runtime","optional":false},{"id":13875080659,"package_name":"exceptiongroup","ecosystem":"pypi","requirements":"1.1.2","direct":false,"kind":"runtime","optional":false},{"id":13875080660,"package_name":"fonttools","ecosystem":"pypi","requirements":"4.41.1","direct":false,"kind":"runtime","optional":false},{"id":13875080661,"package_name":"frozenlist","ecosystem":"pypi","requirements":"1.4.0","direct":false,"kind":"runtime","optional":false},{"id":13875080662,"package_name":"gitdb","ecosystem":"pypi","requirements":"4.0.10","direct":false,"kind":"runtime","optional":false},{"id":13875080663,"package_name":"gitpython","ecosystem":"pypi","requirements":"3.1.32","direct":false,"kind":"runtime","optional":false},{"id":13875080664,"package_name":"gpt4all","ecosystem":"pypi","requirements":"1.0.8","direct":false,"kind":"runtime","optional":false},{"id":13875080665,"package_name":"h11","ecosystem":"pypi","requirements":"0.14.0","direct":false,"kind":"runtime","optional":false},{"id":13875080666,"package_name":"httpcore","ecosystem":"pypi","requirements":"0.17.3","direct":false,"kind":"runtime","optional":false},{"id":13875080667,"package_name":"httpx","ecosystem":"pypi","requirements":"0.24.1","direct":false,"kind":"runtime","optional":false},{"id":13875080668,"package_name":"idna","ecosystem":"pypi","requirements":"3.4","direct":false,"kind":"runtime","optional":false},{"id":13875080669,"package_name":"importlib-resources","ecosystem":"pypi","requirements":"6.0.0","direct":false,"kind":"runtime","optional":false},{"id":13875080670,"package_name":"isodate","ecosystem":"pypi","requirements":"0.6.1","direct":false,"kind":"runtime","optional":false},{"id":13875080671,"package_name":"jinja2","ecosystem":"pypi","requirements":"3.1.2","direct":false,"kind":"runtime","optional":false},{"id":13875080672,"package_name":"kiwisolver","ecosystem":"pypi","requirements":"1.4.4","direct":false,"kind":"runtime","optional":false},{"id":13875080673,"package_name":"markupsafe","ecosystem":"pypi","requirements":"2.1.3","direct":false,"kind":"runtime","optional":false},{"id":13875080674,"package_name":"matplotlib","ecosystem":"pypi","requirements":"3.7.2","direct":false,"kind":"runtime","optional":false},{"id":13875080675,"package_name":"multidict","ecosystem":"pypi","requirements":"6.0.4","direct":false,"kind":"runtime","optional":false},{"id":13875080676,"package_name":"numpy","ecosystem":"pypi","requirements":"1.25.2","direct":false,"kind":"runtime","optional":false},{"id":13875080677,"package_name":"openai","ecosystem":"pypi","requirements":"0.27.8","direct":false,"kind":"runtime","optional":false},{"id":13875080678,"package_name":"packaging","ecosystem":"pypi","requirements":"23.1","direct":false,"kind":"runtime","optional":false},{"id":13875080679,"package_name":"pandas","ecosystem":"pypi","requirements":"2.0.3","direct":false,"kind":"runtime","optional":false},{"id":13875080680,"package_name":"pillow","ecosystem":"pypi","requirements":"10.0.0","direct":false,"kind":"runtime","optional":false},{"id":13875080681,"package_name":"pydantic","ecosystem":"pypi","requirements":"1.10.12","direct":false,"kind":"runtime","optional":false},{"id":13875080682,"package_name":"pyparsing","ecosystem":"pypi","requirements":"3.0.9","direct":false,"kind":"runtime","optional":false},{"id":13875080683,"package_name":"python-dateutil","ecosystem":"pypi","requirements":"2.8.2","direct":false,"kind":"runtime","optional":false},{"id":13875080684,"package_name":"pytz","ecosystem":"pypi","requirements":"2023.3","direct":false,"kind":"runtime","optional":false},{"id":13875080685,"package_name":"pyyaml","ecosystem":"pypi","requirements":"6.0.1","direct":false,"kind":"runtime","optional":false},{"id":13875080686,"package_name":"rdflib","ecosystem":"pypi","requirements":"6.3.2","direct":false,"kind":"runtime","optional":false},{"id":13875080687,"package_name":"requests","ecosystem":"pypi","requirements":"2.31.0","direct":false,"kind":"runtime","optional":false},{"id":13875080688,"package_name":"seaborn","ecosystem":"pypi","requirements":"0.12.2","direct":false,"kind":"runtime","optional":false},{"id":13875080689,"package_name":"six","ecosystem":"pypi","requirements":"1.16.0","direct":false,"kind":"runtime","optional":false},{"id":13875080690,"package_name":"smmap","ecosystem":"pypi","requirements":"5.0.0","direct":false,"kind":"runtime","optional":false},{"id":13875080691,"package_name":"sniffio","ecosystem":"pypi","requirements":"1.3.0","direct":false,"kind":"runtime","optional":false},{"id":13875080692,"package_name":"tokenizers","ecosystem":"pypi","requirements":"0.13.3","direct":false,"kind":"runtime","optional":false},{"id":13875080693,"package_name":"tqdm","ecosystem":"pypi","requirements":"4.65.0","direct":false,"kind":"runtime","optional":false},{"id":13875080694,"package_name":"typing-extensions","ecosystem":"pypi","requirements":"4.7.1","direct":false,"kind":"runtime","optional":false},{"id":13875080695,"package_name":"tzdata","ecosystem":"pypi","requirements":"2023.3","direct":false,"kind":"runtime","optional":false},{"id":13875080696,"package_name":"urllib3","ecosystem":"pypi","requirements":"2.0.4","direct":false,"kind":"runtime","optional":false},{"id":13875080697,"package_name":"yarl","ecosystem":"pypi","requirements":"1.9.2","direct":false,"kind":"runtime","optional":false},{"id":13875080698,"package_name":"zipp","ecosystem":"pypi","requirements":"3.16.2","direct":false,"kind":"runtime","optional":false}]},{"ecosystem":"pypi","filepath":"pyproject.toml","sha":null,"kind":"manifest","created_at":"2023-09-22T13:39:07.548Z","updated_at":"2023-09-22T13:39:07.548Z","repository_link":"https://github.com/AKSW/LLM-KG-Bench/blob/main/pyproject.toml","dependencies":[{"id":13875081031,"package_name":"python","ecosystem":"pypi","requirements":"^3.9","direct":true,"kind":"runtime","optional":false},{"id":13875081032,"package_name":"rdflib","ecosystem":"pypi","requirements":"^6.3.2","direct":true,"kind":"runtime","optional":false},{"id":13875081033,"package_name":"openai","ecosystem":"pypi","requirements":"^0.27.8","direct":true,"kind":"runtime","optional":false},{"id":13875081034,"package_name":"backoff","ecosystem":"pypi","requirements":"^2.2.1","direct":true,"kind":"runtime","optional":false},{"id":13875081035,"package_name":"pyyaml","ecosystem":"pypi","requirements":"^6.0.1","direct":true,"kind":"runtime","optional":false},{"id":13875081036,"package_name":"anthropic","ecosystem":"pypi","requirements":"^0.3.6","direct":true,"kind":"runtime","optional":false},{"id":13875081037,"package_name":"gitpython","ecosystem":"pypi","requirements":"^3.1.32","direct":true,"kind":"runtime","optional":false},{"id":13875081038,"package_name":"pandas","ecosystem":"pypi","requirements":"^2.0.3","direct":true,"kind":"runtime","optional":false},{"id":13875081039,"package_name":"matplotlib","ecosystem":"pypi","requirements":"^3.7.2","direct":true,"kind":"runtime","optional":false},{"id":13875081040,"package_name":"seaborn","ecosystem":"pypi","requirements":"^0.12.2","direct":true,"kind":"runtime","optional":false},{"id":13875081041,"package_name":"gpt4all","ecosystem":"pypi","requirements":"^1.0.7","direct":true,"kind":"runtime","optional":false},{"id":13875081042,"package_name":"jinja2","ecosystem":"pypi","requirements":"^3.0.1","direct":true,"kind":"runtime","optional":false}]}],"score":null,"created_at":"2025-09-08T21:56:59.429Z","updated_at":"2025-10-07T08:30:43.649Z","avatar_url":"https://github.com/AKSW.png","language":"Python","category":null,"sub_category":null,"monthly_downloads":0,"funding_links":[],"readme_doi_urls":["https://doi.org/10.5281/zenodo.8251944","https://doi.org/10.48550/arXiv.2308.16622","https://doi.org/10.5281/zenodo.8250645","https://doi.org/10.5281/zenodo.8364535","https://doi.org/10.1007/978-3-031-78952-6_5","https://doi.org/10.5281/zenodo.10572907","https://doi.org/10.5281/zenodo.14206042","https://doi.org/10.5281/zenodo.15100645"],"works":{},"citation_counts":{},"total_citations":0,"keywords_from_contributors":[],"project_url":"https://science.ecosyste.ms/api/v1/projects/97996","html_url":"https://science.ecosyste.ms/projects/97996"}