{"id":163901,"name":null,"description":"This repo contains the code for 1D tokenizer and generator","url":"https://github.com/bytedance/1d-tokenizer","last_synced_at":"2025-09-09T19:15:46.401Z","repository":{"id":246277377,"uuid":"818342919","full_name":"bytedance/1d-tokenizer","owner":"bytedance","description":"This repo contains the code for 1D tokenizer and generator","archived":false,"fork":false,"pushed_at":"2025-03-20T17:30:54.000Z","size":29981,"stargazers_count":884,"open_issues_count":41,"forks_count":46,"subscribers_count":20,"default_branch":"main","last_synced_at":"2025-05-29T07:30:20.783Z","etag":null,"topics":["research"],"latest_commit_sha":null,"homepage":"","language":"Jupyter Notebook","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":"apache-2.0","status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/bytedance.png","metadata":{"files":{"readme":"README.md","changelog":null,"contributing":null,"funding":null,"license":"LICENSE","code_of_conduct":null,"threat_model":null,"audit":null,"citation":null,"codeowners":null,"security":null,"support":null,"governance":null,"roadmap":null,"authors":null,"dei":null,"publiccode":null,"codemeta":null,"zenodo":null}},"created_at":"2024-06-21T16:27:11.000Z","updated_at":"2025-05-28T16:19:24.000Z","dependencies_parsed_at":"2024-06-27T01:01:14.526Z","dependency_job_id":"2db67baf-4f62-45ac-8f21-599d66be02da","html_url":"https://github.com/bytedance/1d-tokenizer","commit_stats":null,"previous_names":["bytedance/1d-tokenizer"],"tags_count":0,"template":false,"template_full_name":null,"purl":"pkg:github/bytedance/1d-tokenizer","repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/bytedance%2F1d-tokenizer","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/bytedance%2F1d-tokenizer/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/bytedance%2F1d-tokenizer/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/bytedance%2F1d-tokenizer/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/bytedance","download_url":"https://codeload.github.com/bytedance/1d-tokenizer/tar.gz/refs/heads/main","sbom_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/bytedance%2F1d-tokenizer/sbom","scorecard":null,"host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":274348064,"owners_count":25268972,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2022-07-04T15:15:14.044Z","status":"online","status_checked_at":"2025-09-09T02:00:10.223Z","response_time":80,"last_error":null,"robots_txt_status":"success","robots_txt_updated_at":"2025-07-24T06:49:26.215Z","robots_txt_url":"https://github.com/robots.txt","online":true,"can_crawl_api":true,"host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"owner":{"login":"bytedance","name":"Bytedance Inc.","uuid":"4158466","kind":"organization","description":"","email":null,"website":"https://opensource.bytedance.com","location":"Singapore","twitter":"ByteDanceOSS","company":null,"icon_url":"https://avatars.githubusercontent.com/u/4158466?v=4","repositories_count":255,"last_synced_at":"2024-04-16T02:59:18.813Z","metadata":{"has_sponsors_listing":false},"html_url":"https://github.com/bytedance","funding_links":[],"total_stars":88648,"followers":6017,"following":0,"created_at":"2022-11-13T11:29:28.707Z","updated_at":"2024-04-16T03:00:36.055Z","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/bytedance","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/bytedance/repositories"},"packages":[],"commits":{"id":5836636,"full_name":"bytedance/1d-tokenizer","default_branch":"main","committers":[{"name":"Qihang Yu","email":"yucornetto@gmail.com","login":"yucornetto","count":18},{"name":"cornettoyu","email":"138716072+cornettoyu","login":"cornettoyu","count":17},{"name":"TACJu","email":"jhe47@jh.edu","login":"TACJu","count":13},{"name":"Niels","email":"niels.rogge1@gmail.com","login":"NielsRogge","count":6},{"name":"Ju He","email":"ju.he1@bytedance.com","login":null,"count":6}],"total_commits":60,"total_committers":5,"total_bot_commits":0,"total_bot_committers":0,"mean_commits":12.0,"dds":0.7,"past_year_committers":[{"name":"Qihang Yu","email":"yucornetto@gmail.com","login":"yucornetto","count":18},{"name":"cornettoyu","email":"138716072+cornettoyu","login":"cornettoyu","count":17},{"name":"TACJu","email":"jhe47@jh.edu","login":"TACJu","count":13},{"name":"Niels","email":"niels.rogge1@gmail.com","login":"NielsRogge","count":6},{"name":"Ju He","email":"ju.he1@bytedance.com","login":null,"count":6}],"past_year_total_commits":60,"past_year_total_committers":5,"past_year_total_bot_commits":0,"past_year_total_bot_committers":0,"past_year_mean_commits":12.0,"past_year_dds":0.7,"last_synced_at":"2025-05-29T07:25:06.360Z","last_synced_commit":"942a96fbdd873780179d1b78d5462911528bf8c8","created_at":"2024-11-15T13:15:05.635Z","updated_at":"2025-05-29T07:25:06.368Z","commits_url":"https://commits.ecosyste.ms/api/v1/hosts/GitHub/repositories/bytedance%2F1d-tokenizer/commits","host":{"name":"GitHub","url":"https://github.com","kind":"github","last_synced_at":"2025-09-09T00:17:06.873Z","repositories_count":5534301,"commits_count":872232494,"contributors_count":32156303,"owners_count":918527,"icon_url":"https://github.com/github.png","host_url":"https://commits.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://commits.ecosyste.ms/api/v1/hosts/GitHub/repositories"}},"issues_stats":{"full_name":"bytedance/1d-tokenizer","html_url":"https://github.com/bytedance/1d-tokenizer","last_synced_at":"2025-09-03T01:07:36.434Z","status":null,"issues_count":81,"pull_requests_count":46,"avg_time_to_close_issue":1462347.677419355,"avg_time_to_close_pull_request":11779.51282051282,"issues_closed_count":31,"pull_requests_closed_count":39,"pull_request_authors_count":9,"issue_authors_count":70,"avg_comments_per_issue":1.222222222222222,"avg_comments_per_pull_request":0.10869565217391304,"merged_pull_requests_count":36,"bot_issues_count":0,"bot_pull_requests_count":0,"past_year_issues_count":58,"past_year_pull_requests_count":39,"past_year_avg_time_to_close_issue":673111.7777777778,"past_year_avg_time_to_close_pull_request":2273.375,"past_year_issues_closed_count":18,"past_year_pull_requests_closed_count":32,"past_year_pull_request_authors_count":8,"past_year_issue_authors_count":48,"past_year_avg_comments_per_issue":1.0517241379310345,"past_year_avg_comments_per_pull_request":0.0,"past_year_bot_issues_count":0,"past_year_bot_pull_requests_count":0,"past_year_merged_pull_requests_count":29,"created_at":"2024-11-15T13:15:06.164Z","updated_at":"2025-09-03T05:51:44.448Z","repository_url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/repositories/bytedance%2F1d-tokenizer","issues_url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/repositories/bytedance%2F1d-tokenizer/issues","issue_labels_count":{},"pull_request_labels_count":{},"issue_author_associations_count":{"NONE":79,"CONTRIBUTOR":1},"pull_request_author_associations_count":{"CONTRIBUTOR":30,"NONE":17},"issue_authors":{"RohollahHS":5,"Sunnet314159":2,"jeasinema":2,"localbetascreening":2,"Peterande":2,"Doctor-James":2,"sorobedio":2,"torahoang":1,"Jason3900":1,"hp-l33":1,"Changlin-Lee":1,"NilanEkanayake":1,"juncongmoo":1,"dongzhuoyao":1,"Faded1022":1,"tanzheen":1,"HansPolo113":1,"JJJYmmm":1,"yfChang-cv":1,"thesby":1,"demour96":1,"tau-yihouxiang":1,"yuanzhi-zhu":1,"shashankvkt":1,"shun31y":1,"RJT1990":1,"S-aiueo32":1,"yingShen-ys":1,"Jyouhou":1,"xingjianleng":1,"SnakeOnex":1,"deJQK":1,"wAnnA2102":1,"XPLearner":1,"chs20":1,"Amos1109":1,"baoxiaoyi":1,"lxa9867":1,"ChuxiJ":1,"NieShenRuc":1,"kabachuha":1,"ChencongZJU":1,"eanson023":1,"TinyTigerPan":1,"beitong95":1,"YoojLee":1,"kxgong":1,"AlienKevin":1,"speedcell4":1,"ArsenalCheng":1,"qiuqc1":1,"NielsRogge":1,"lucasjinreal":1,"cucNaifuXue":1,"ThisisBillhe":1,"zichengsaber":1,"hasqdwuduw":1,"z-jiaming":1,"liusulin":1,"MikeWangWZHL":1,"xyfJASON":1,"GuotaoLiang":1,"elias-ramzi":1,"bargaincheese":1,"VitorGuizilini-TRI":1,"CoinCheung":1,"qyangcv":1,"zhangqingwu":1,"Yinhance":1,"wangybwz":1},"pull_request_authors":{"TACJu":21,"yucornetto":15,"Yubel426":2,"NielsRogge":2,"laiviet":2,"QY-H00":2,"rand0musername":1,"eltociear":1,"AlienKevin":1},"host":{"name":"GitHub","url":"https://github.com","kind":"github","last_synced_at":"2025-09-09T00:00:11.778Z","repositories_count":10197929,"issues_count":32275653,"pull_requests_count":101879419,"authors_count":10740965,"icon_url":"https://github.com/github.png","host_url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/repositories","owners_url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/owners","authors_url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors"},"past_year_issue_labels_count":{},"past_year_pull_request_labels_count":{},"past_year_issue_author_associations_count":{"NONE":55,"CONTRIBUTOR":1},"past_year_pull_request_author_associations_count":{"CONTRIBUTOR":24,"NONE":16},"past_year_issue_authors":{"RohollahHS":5,"localbetascreening":2,"Peterande":2,"Doctor-James":2,"Sunnet314159":2,"sorobedio":2,"AlienKevin":1,"NilanEkanayake":1,"qiuqc1":1,"qyangcv":1,"RJT1990":1,"S-aiueo32":1,"shun31y":1,"SnakeOnex":1,"tanzheen":1,"thesby":1,"torahoang":1,"VitorGuizilini-TRI":1,"wangybwz":1,"wAnnA2102":1,"xingjianleng":1,"xyfJASON":1,"yfChang-cv":1,"yingShen-ys":1,"z-jiaming":1,"Amos1109":1,"ArsenalCheng":1,"bargaincheese":1,"beitong95":1,"Changlin-Lee":1,"ChencongZJU":1,"chs20":1,"CoinCheung":1,"deJQK":1,"demour96":1,"dongzhuoyao":1,"eanson023":1,"elias-ramzi":1,"Faded1022":1,"GuotaoLiang":1,"HansPolo113":1,"hasqdwuduw":1,"hp-l33":1,"jeasinema":1,"Jyouhou":1,"lxa9867":1,"NielsRogge":1},"past_year_pull_request_authors":{"TACJu":21,"yucornetto":10,"QY-H00":2,"laiviet":2,"Yubel426":2,"rand0musername":1,"AlienKevin":1,"eltociear":1},"maintainers":[],"active_maintainers":[]},"events":{"total":{"IssuesEvent":65,"WatchEvent":493,"IssueCommentEvent":84,"PushEvent":16,"PullRequestEvent":34,"ForkEvent":44},"last_year":{"IssuesEvent":65,"WatchEvent":493,"IssueCommentEvent":84,"PushEvent":16,"PullRequestEvent":34,"ForkEvent":44}},"keywords":["research"],"dependencies":[{"ecosystem":"pypi","filepath":"requirements.txt","sha":null,"kind":"manifest","created_at":"2024-06-27T01:01:14.492Z","updated_at":"2024-06-27T01:01:14.492Z","repository_link":"https://github.com/bytedance/1d-tokenizer/blob/main/requirements.txt","dependencies":[{"id":18844880010,"package_name":"torch","ecosystem":"pypi","requirements":"\u003e=2.0.0","direct":true,"kind":"runtime","optional":false},{"id":18844880011,"package_name":"torchvision","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":18844880012,"package_name":"omegaconf","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":18844880013,"package_name":"transformers","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":18844880014,"package_name":"timm","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":18844880015,"package_name":"open_clip_torch","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":18844880016,"package_name":"einops","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":18844880017,"package_name":"scipy","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":18844880018,"package_name":"pillow","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":18844880019,"package_name":"accelerate","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":18844880020,"package_name":"gdown","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false},{"id":18844880021,"package_name":"huggingface-hub","ecosystem":"pypi","requirements":"*","direct":true,"kind":"runtime","optional":false}]}],"score":8.439231649946525,"created_at":"2025-09-09T01:06:27.171Z","updated_at":"2025-10-07T08:38:56.833Z","avatar_url":"https://github.com/bytedance.png","language":"Jupyter Notebook","category":null,"sub_category":null,"monthly_downloads":0,"funding_links":[],"readme_doi_urls":[],"works":{},"citation_counts":{},"total_citations":0,"keywords_from_contributors":[],"project_url":"https://science.ecosyste.ms/api/v1/projects/163901","html_url":"https://science.ecosyste.ms/projects/163901"}