BHARGAV REDDY commited on
Commit
da28a11
Β·
verified Β·
1 Parent(s): cc1b008

Upload push_dataset_to_hf.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. push_dataset_to_hf.py +145 -0
push_dataset_to_hf.py ADDED
@@ -0,0 +1,145 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """
3
+ Push LUNA datasets to HuggingFace dataset repo.
4
+
5
+ Targets: https://huggingface.co/datasets/ASTERIZER/Luna_Dataset
6
+
7
+ Uploads:
8
+ - Cleaned English 1B corpus (parquet files)
9
+ - LitData tokenized chunks (binary files + index.json)
10
+ - Build/cleaning reports
11
+ - SFT datasets (if present)
12
+
13
+ Usage:
14
+ HF_TOKEN=hf_xxx python push_dataset_to_hf.py
15
+ HF_TOKEN=hf_xxx python push_dataset_to_hf.py --include_litdata
16
+ HF_TOKEN=hf_xxx python push_dataset_to_hf.py --include_sft
17
+ """
18
+
19
+ import argparse
20
+ import os
21
+ from pathlib import Path
22
+
23
+ from huggingface_hub import HfApi
24
+
25
+
26
+ DATASET_REPO = "ASTERIZER/Luna_Dataset"
27
+
28
+
29
+ def parse_args():
30
+ parser = argparse.ArgumentParser(description="Push datasets to HuggingFace")
31
+ parser.add_argument("--include_litdata", action="store_true",
32
+ help="Also upload tokenized LitData chunks")
33
+ parser.add_argument("--include_sft", action="store_true",
34
+ help="Also upload SFT datasets")
35
+ parser.add_argument("--include_raw", action="store_true",
36
+ help="Also upload raw (pre-clean) corpus")
37
+ return parser.parse_args()
38
+
39
+
40
+ def upload_folder_safe(api, repo_id, folder, path_in_repo, token):
41
+ """Upload a folder, skipping if it doesn't exist."""
42
+ folder = Path(folder)
43
+ if not folder.exists():
44
+ print(f" SKIP (not found): {folder}")
45
+ return 0
46
+ files = list(folder.rglob("*"))
47
+ files = [f for f in files if f.is_file()]
48
+ if not files:
49
+ print(f" SKIP (empty): {folder}")
50
+ return 0
51
+ print(f" Uploading {len(files)} files from {folder} -> {path_in_repo}/")
52
+ api.upload_folder(
53
+ repo_id=repo_id,
54
+ repo_type="dataset",
55
+ folder_path=str(folder),
56
+ path_in_repo=path_in_repo,
57
+ )
58
+ return len(files)
59
+
60
+
61
+ def upload_file_safe(api, repo_id, filepath, path_in_repo, token):
62
+ """Upload a single file, skipping if it doesn't exist."""
63
+ if not os.path.exists(filepath):
64
+ print(f" SKIP (not found): {filepath}")
65
+ return 0
66
+ print(f" OK: {filepath} -> {path_in_repo}")
67
+ api.upload_file(
68
+ path_or_fileobj=filepath,
69
+ path_in_repo=path_in_repo,
70
+ repo_id=repo_id,
71
+ repo_type="dataset",
72
+ )
73
+ return 1
74
+
75
+
76
+ def main():
77
+ args = parse_args()
78
+ token = os.environ.get("HF_TOKEN")
79
+ if not token:
80
+ raise RuntimeError("Set HF_TOKEN environment variable")
81
+
82
+ api = HfApi(token=token)
83
+
84
+ # Create dataset repo
85
+ api.create_repo(
86
+ repo_id=DATASET_REPO,
87
+ repo_type="dataset",
88
+ token=token,
89
+ exist_ok=True,
90
+ private=False,
91
+ )
92
+ print(f"Dataset repo: https://huggingface.co/datasets/{DATASET_REPO}\n")
93
+
94
+ total = 0
95
+
96
+ # ── 1. Cleaned English 1B corpus ─────────────────────────────────────
97
+ print("── Cleaned English 1B corpus ──")
98
+ total += upload_folder_safe(api, DATASET_REPO,
99
+ "Base/data/english_1b_clean",
100
+ "english_1b_clean", token)
101
+
102
+ # ── 2. Raw corpus (if requested) ─────────────────────────────────────
103
+ if args.include_raw:
104
+ print("\n── Raw English 1B corpus ──")
105
+ total += upload_folder_safe(api, DATASET_REPO,
106
+ "Base/data/english_1b_raw",
107
+ "english_1b_raw", token)
108
+
109
+ # ── 3. LitData tokenized chunks (if requested) ───────────────────────
110
+ if args.include_litdata:
111
+ print("\n── LitData tokenized (english_1b) ──")
112
+ total += upload_folder_safe(api, DATASET_REPO,
113
+ "Base/data/litdata_english_1b",
114
+ "litdata_english_1b", token)
115
+ # Also upload the original pretrain data if it exists
116
+ print("\n── LitData tokenized (pretrain_final) ──")
117
+ total += upload_folder_safe(api, DATASET_REPO,
118
+ "Base/data/litdata_pretrain_final",
119
+ "litdata_pretrain_final", token)
120
+
121
+ # ── 4. SFT datasets (if requested) ───────────────────────────────────
122
+ if args.include_sft:
123
+ print("\n── SFT datasets ──")
124
+ total += upload_folder_safe(api, DATASET_REPO,
125
+ "Base/Datasets/rag_mcp_sft",
126
+ "rag_mcp_sft", token)
127
+ total += upload_folder_safe(api, DATASET_REPO,
128
+ "Base/Datasets/sft_clean",
129
+ "sft_clean", token)
130
+
131
+ # ── 5. Reports & metadata ────────────────────────────────────────────
132
+ print("\n── Reports & metadata ──")
133
+ for report_path, repo_path in [
134
+ ("Base/data/english_1b_raw/BUILD_REPORT.txt", "reports/BUILD_REPORT.txt"),
135
+ ("Base/data/english_1b_raw/build_stats.json", "reports/build_stats.json"),
136
+ ("Base/data/english_1b_clean/CLEANING_REPORT.txt", "reports/CLEANING_REPORT.txt"),
137
+ ("Base/data/english_1b_clean/clean_stats.json", "reports/clean_stats.json"),
138
+ ]:
139
+ total += upload_file_safe(api, DATASET_REPO, report_path, repo_path, token)
140
+
141
+ print(f"\nDone! Uploaded {total} items to https://huggingface.co/datasets/{DATASET_REPO}")
142
+
143
+
144
+ if __name__ == "__main__":
145
+ main()