File size: 4,510 Bytes
aae3ba1 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 |
import json
import os
from huggingface_hub import hf_hub_download
def deep_update(d1, d2):
"""Deep update d1 with d2, recursively merging nested dictionaries."""
for k, v in d2.items():
if isinstance(v, dict) and k in d1:
assert isinstance(d1[k], dict), f"Cannot merge dict with non-dict for key {k}"
deep_update(d1[k], d2[k])
else:
d1[k] = d2[k]
return d1
import json
import os
from huggingface_hub import hf_hub_download, list_repo_files
def deep_update(d1, d2):
"""Deep update d1 with d2, recursively merging nested dictionaries."""
for k, v in d2.items():
if isinstance(v, dict) and k in d1:
assert isinstance(d1[k], dict), f"Cannot merge dict with non-dict for key {k}"
deep_update(d1[k], d2[k])
else:
d1[k] = d2[k]
return d1
def load_config(config_file):
"""Load configuration file with support for parent configs and Hugging Face Hub."""
# Check if config_file is a Hugging Face repo (format: "username/repo-name:filename")
from_huggingface = False
if ":" in config_file and "/" in config_file.split(":")[0] and not os.path.exists(config_file):
# Parse Hugging Face repo format: "username/repo-name:config.json"
repo_id, filename = config_file.split(":", 1)
print(f"Loading config from Hugging Face Hub: {repo_id}/{filename}")
config_path = hf_hub_download(repo_id=repo_id, filename=f"{filename}")
from_huggingface = True
elif "/" in config_file and not os.path.exists(config_file):
# If format is "username/repo-name", check which file exists first
try:
repo_files = list_repo_files(repo_id=config_file)
if "config.json" in repo_files:
print(f"Loading config from Hugging Face Hub: {config_file}/config.json")
config_path = hf_hub_download(repo_id=config_file, filename="config.json")
elif "configs/config.json" in repo_files:
print(f"Loading config from Hugging Face Hub: {config_file}/configs/config.json")
config_path = hf_hub_download(repo_id=config_file, filename="configs/config.json")
else:
raise FileNotFoundError(f"Neither config.json nor configs/config.json found in {config_file}")
from_huggingface = True
except Exception as e:
raise RuntimeError(f"Failed to load config from Hugging Face Hub: {config_file}. Error: {e}")
else:
# Local file path
config_path = config_file
from_huggingface = False
with open(config_path) as f:
_config = json.load(f)
if from_huggingface:
_config["model_load_path"] = config_file
_config['statistics_path'] = config_file
config = {}
if _config.get("parent"):
deep_update(config, load_config(_config["parent"]))
deep_update(config, _config)
return config
# def load_config(config_file):
# """Load configuration file with support for parent configs and Hugging Face Hub."""
# # Check if config_file is a Hugging Face repo (format: "username/repo-name:filename")
# from_huggingface = False
# if ":" in config_file and "/" in config_file.split(":")[0] and not os.path.exists(config_file):
# # Parse Hugging Face repo format: "username/repo-name:config.json"
# repo_id, filename = config_file.split(":", 1)
# print(f"Loading config from Hugging Face Hub: {repo_id}/{filename}")
# config_path = hf_hub_download(repo_id=repo_id, filename=f"{filename}")
# from_huggingface = True
# elif "/" in config_file and not os.path.exists(config_file) and not config_file.endswith(".json"):
# # If format is "username/repo-name", default to "config.json"
# print(f"Loading config from Hugging Face Hub: {config_file}/configs/config.json")
# config_path = hf_hub_download(repo_id=config_file, filename="configs/config.json")
# from_huggingface = True
# else:
# # Local file path
# config_path = config_file
# from_huggingface = False
# with open(config_path) as f:
# _config = json.load(f)
# if from_huggingface:
# _config["model_load_path"] = config_file
# _config['statistics_path'] = config_file
# config = {}
# if _config.get("parent"):
# deep_update(config, load_config(_config["parent"]))
# deep_update(config, _config)
# return config |