init

Instance.py

@@ -1,12 +1,10 @@
 import os
 import requests
 import json
-import urllib.request
 import time
 from threading import Thread, Event
 from requests.exceptions import RequestException
 from tqdm import tqdm
-from indexer import indexer
 import logging
 
 CACHE_DIR = os.getenv("CACHE_DIR")
@@ -14,76 +12,58 @@ CACHE_DIR = os.getenv("CACHE_DIR")
 download_progress = {}
 
 class Instance:
-    def __init__(self, id, url, cache_dir, index_file, token, repo, load_balancer_api, max_retries=20, initial_delay=1):
+    def __init__(self, id, url, cache_dir, token, repo, load_balancer_api, max_retries=20, initial_delay=1):
         self.version = "0.0.0.1 Alpha"
         self.id = id
         self.url = url
         self.CACHE_DIR = cache_dir
-        self.INDEX_FILE = index_file
         self.TOKEN = token
         self.REPO = repo
-        self.FILM_STORE_JSON_PATH = os.path.join(cache_dir, "film_store.json")
-        self.TV_STORE_JSON_PATH = os.path.join(cache_dir, "tv_store.json")
+        self.FILM_STORE = {}
+        self.TV_STORE = {}
         self.download_threads = {}
         self.file_structure = None
         self.load_balancer_api = load_balancer_api
         self.max_retries = max_retries
         self.initial_delay = initial_delay
-        self.last_report_time = time.time()  # Initialize the last report time
+        self.last_report_time = time.time()
         self.re_register_event = Event()
 
         # Ensure CACHE_DIR exists
         if not os.path.exists(self.CACHE_DIR):
             os.makedirs(self.CACHE_DIR)
 
-        for path in [self.FILM_STORE_JSON_PATH, self.TV_STORE_JSON_PATH]:
-            if not os.path.exists(path):
-                with open(path, 'w') as json_file:
-                    json.dump({}, json_file)
-
-        # Index the file structure and load it
-        self.run_indexer_and_load()
-
-        # Start prefetching metadata and monitoring registration
         self.register_to_load_balancer()
+        self.reload_file_structure()
         registration_thread = Thread(target=self.monitor_registration)
         registration_thread.daemon = True
         registration_thread.start()
 
         # Start the thread to re-index every 2 minutes
-        indexer_thread = Thread(target=self.run_indexer_periodically)
+        indexer_thread = Thread(target=self.get_file_structure_periodically)
         indexer_thread.daemon = True
         indexer_thread.start()
 
-    def run_indexer_and_load(self):
+    def reload_file_structure(self):
         """Runs the indexer and loads the file structure from INDEX_FILE."""
-        indexer()
-        if not os.path.exists(self.INDEX_FILE):
-            raise FileNotFoundError(f"{self.INDEX_FILE} not found. Please make sure the file exists.")
-
-        with open(self.INDEX_FILE, 'r') as f:
-            self.file_structure = json.load(f)
+        self.file_structure = self.load_balancer_api.get_file_structure()
         logging.info("File structure reloaded successfully.")
 
-    def run_indexer_periodically(self):
+    def get_file_structure_periodically(self):
         """Periodically reruns the indexer and reloads the file structure."""
         while True:
-            time.sleep(120)  # Wait for 2 minutes
+            time.sleep(300)  # Wait for 5 minutes
             logging.info("Re-running indexer and reloading file structure.")
-            self.run_indexer_and_load()
+            self.reload_file_structure()
 
     def compile_report(self):
         self.last_report_time = time.time()  # Update the last report time
-        
-        film_store_path = os.path.join(self.CACHE_DIR, "film_store.json")
-        tv_store_path = os.path.join(self.CACHE_DIR, "tv_store.json")
         cache_size = self.get_cache_size()
-
         report = {
             "instance_id": self.id,
             "instance_url": self.url,
-            "film_store": self.read_json(film_store_path),
-            "tv_store": self.read_json(tv_store_path),
+            "film_store": self.FILM_STORE,
+            "tv_store": self.TV_STORE,
             "cache_size": cache_size
         }
         return report
@@ -118,26 +98,7 @@ class Instance:
                 return json.load(json_file)
         return {}
 
-    @staticmethod
-    def get_system_proxies():
-        """
-        Retrieves the system's HTTP and HTTPS proxies.
-
-        Returns:
-            dict: A dictionary containing the proxies.
-        """
-        try:
-            proxies = urllib.request.getproxies()
-            print("System proxies:", proxies)
-            return {
-                "http": proxies.get("http"),
-                "https": proxies.get("http")
-            }
-        except Exception as e:
-            print(f"Error getting system proxies: {e}")
-            return {}
-
-    def download_film(self, file_url, token, cache_path, proxies, film_id, title, chunk_size=100 * 1024 * 1024):
+    def download_film(self, file_url, token, cache_path, film_id, title, chunk_size=100 * 1024 * 1024):
         """
         Downloads a file from the specified URL and saves it to the cache path.
         Tracks the download progress.
@@ -146,15 +107,14 @@ class Instance:
             file_url (str): The URL of the file to download.
             token (str): The authorization token for the request.
             cache_path (str): The path to save the downloaded file.
-            proxies (dict): Proxies for the request.
             film_id (str): Unique identifier for the film download.
             title (str): The title of the film.
             chunk_size (int): Size of each chunk to download.
         """
-        print(f"Downloading file from URL: {file_url} to {cache_path} with proxies: {proxies}")
+        print(f"Downloading file from URL: {file_url} to {cache_path}")
         headers = {'Authorization': f'Bearer {token}'}
         try:
-            response = requests.get(file_url, headers=headers, proxies=proxies, stream=True)
+            response = requests.get(file_url, headers=headers, stream=True)
             response.raise_for_status()
             
             total_size = int(response.headers.get('content-length', 0))
@@ -168,7 +128,7 @@ class Instance:
                     download_progress[film_id]["downloaded"] += len(data)
             
             print(f'File cached to {cache_path} successfully.')
-            self.update_film_store_json(title, cache_path)
+            self.FILM_STORE[title] = cache_path
             download_progress[film_id]["status"] = "Completed"
         except RequestException as e:
             print(f"Error downloading file: {e}")
@@ -208,26 +168,9 @@ class Instance:
             return {"total": total, "downloaded": downloaded, "progress": progress, "status": status, "eta": eta}
         return {"total": 0, "downloaded": 0, "progress": 0, "status": "Not Found", "eta": None}
 
-    def update_film_store_json(self,title, cache_path):
-        """
-        Updates the film store JSON with the new file.
-
-        Args:
-            title (str): The title of the film.
-            cache_path (str): The local path where the file is saved.
-        """
-        film_store_data = {}
-        if os.path.exists(self.FILM_STORE_JSON_PATH):
-            with open(self.FILM_STORE_JSON_PATH, 'r') as json_file:
-                film_store_data = json.load(json_file)
-
-        film_store_data[title] = cache_path
 
-        with open(self.FILM_STORE_JSON_PATH, 'w') as json_file:
-            json.dump(film_store_data, json_file, indent=2)
-        print(f'Film store updated with {title}.')
 
-    def download_episode(self, file_url, token, cache_path, proxies, episode_id, title, chunk_size=100 * 1024 * 1024):
+    def download_episode(self, file_url, token, cache_path, episode_id, title, chunk_size=100 * 1024 * 1024):
         """
         Downloads a file from the specified URL and saves it to the cache path.
         Tracks the download progress.
@@ -236,15 +179,14 @@ class Instance:
             file_url (str): The URL of the file to download.
             token (str): The authorization token for the request.
             cache_path (str): The path to save the downloaded file.
-            proxies (dict): Proxies for the request.
             episode_id (str): Unique identifier for the film download.
             title (str): The title of the film.
             chunk_size (int): Size of each chunk to download.
         """
-        print(f"Downloading file from URL: {file_url} to {cache_path} with proxies: {proxies}")
+        print(f"Downloading file from URL: {file_url} to {cache_path}")
         headers = {'Authorization': f'Bearer {token}'}
         try:
-            response = requests.get(file_url, headers=headers, proxies=proxies, stream=True)
+            response = requests.get(file_url, headers=headers, stream=True)
             response.raise_for_status()
             
             total_size = int(response.headers.get('content-length', 0))
@@ -258,7 +200,7 @@ class Instance:
                     download_progress[episode_id]["downloaded"] += len(data)
             
             print(f'File cached to {cache_path} successfully.')
-            self.update_tv_store_json(title, cache_path)
+            self.update_tv_store(title, cache_path)
             download_progress[episode_id]["status"] = "Completed"
         except RequestException as e:
             print(f"Error downloading file: {e}")
@@ -270,7 +212,7 @@ class Instance:
             if download_progress[episode_id]["status"] != "Downloading":
                 download_progress[episode_id]["end_time"] = time.time()
 
-    def update_tv_store_json(self, title, cache_path):
+    def update_tv_store(self, title, cache_path):
         """
         Updates the TV store JSON with the new file, organizing by title, season, and episode.
 
@@ -278,27 +220,20 @@ class Instance:
             title (str): The title of the TV show.
             cache_path (str): The local path where the file is saved.
         """
-        tv_store_data = {}
-        if os.path.exists(self.TV_STORE_JSON_PATH):
-            with open(self.TV_STORE_JSON_PATH, 'r') as json_file:
-                tv_store_data = json.load(json_file)
 
         # Extract season and episode information from the cache_path
         season_part = os.path.basename(os.path.dirname(cache_path))  # Extracts 'Season 1'
         episode_part = os.path.basename(cache_path)  # Extracts 'Grand Blue Dreaming - S01E01 - Deep Blue HDTV-720p.mp4'
 
         # Create the structure if not already present
-        if title not in tv_store_data:
-            tv_store_data[title] = {}
+        if title not in self.TV_STORE:
+            self.TV_STORE[title] = {}
         
-        if season_part not in tv_store_data[title]:
-            tv_store_data[title][season_part] = {}
+        if season_part not in self.TV_STORE[title]:
+            self.TV_STORE[title][season_part] = {}
 
         # Assuming episode_part is unique for each episode within a season
-        tv_store_data[title][season_part][episode_part] = cache_path
-
-        with open(self.TV_STORE_JSON_PATH, 'w') as json_file:
-            json.dump(tv_store_data, json_file, indent=2)
+        self.TV_STORE[title][season_part][episode_part] = cache_path
 
         print(f'TV store updated with {title}, {season_part}, {episode_part}.')
 
@@ -351,37 +286,6 @@ class Instance:
     def encode_episodeid(self, title, season, episode):
         return f"{title}_{season}_{episode}"
 
-    def get_all_tv_shows(self):
-        """Get all TV shows from the indexed cache structure JSON file."""
-        tv_shows = {}
-        for directory in self.file_structure:
-            if directory['type'] == 'directory' and directory['path'] == 'tv':
-                for sub_directory in directory['contents']:
-                    if sub_directory['type'] == 'directory':
-                        show_title = sub_directory['path'].split('/')[-1]
-                        tv_shows[show_title] = []
-                        for season_directory in sub_directory['contents']:
-                            if season_directory['type'] == 'directory':
-                                season = season_directory['path'].split('/')[-1]
-                                for episode in season_directory['contents']:
-                                    if episode['type'] == 'file':
-                                        tv_shows[show_title].append({
-                                            "season": season,
-                                            "episode": episode['path'].split('/')[-1],
-                                            "path": episode['path']
-                                        })
-        return tv_shows
-
-    def get_all_films(self):
-        """Get all films from the indexed cache structure JSON file."""
-        films = []
-        for directory in self.file_structure:
-            if directory['type'] == 'directory' and directory['path'] == 'films':
-                for sub_directory in directory['contents']:
-                    if sub_directory['type'] == 'directory':
-                        films.append(sub_directory['path'])
-        return films
-
     def register_to_load_balancer(self):
         retries = 0
         delay = self.initial_delay

api.py

@@ -20,3 +20,14 @@ class LoadBalancerAPI:
         except requests.exceptions.RequestException as e:
             logging.error(f'Failed to register instance {instance_id} to load balancer: {e}')
             return None
+
+    def get_file_structure(self):
+        api_endpoint = f'{self.base_url}/api/get/file_structure'
+        try:
+            headers = {'Content-Type': 'application/json'}
+            response = requests.get(api_endpoint, headers=headers)
+            response.raise_for_status()
+            return response.json()  # Assuming the API returns JSON
+        except requests.exceptions.RequestException as e:
+            logging.error(f'Failed to file structure: {e}')
+            return None

app.py

@@ -1,4 +1,5 @@
 from fastapi import FastAPI
+from fastapi.responses import JSONResponse
 from Instance import Instance
 from api import LoadBalancerAPI
 import os
@@ -18,4 +19,19 @@ app = FastAPI()
 
 @app.get("/")
 async def index():
-    return instance.version
+    return instance.version
+
+@app.get("/api/get/report")
+async def get_report():
+    report=instance.compile_report()
+    return JSONResponse(report)
+
+@app.get('/api/get/tv/store')
+async def get_tv_store_api():
+    """Endpoint to get the TV store JSON."""
+    return JSONResponse(instance.TV_STORE)
+
+@app.get('/api/get/film/store')
+async def get_film_store_api():
+    """Endpoint to get the TV store JSON."""
+    return JSONResponse(instance.FILM_STORE)

hf_scrapper.py

@@ -1,35 +1,15 @@
 import os
 import requests
-import json
-import urllib.request
 import time
 from requests.exceptions import RequestException
 from tqdm import tqdm
+from app import instance
 
 CACHE_DIR = os.getenv("CACHE_DIR")
-CACHE_JSON_PATH = os.path.join(CACHE_DIR, "cached_films.json")
 
 download_progress = {}
 
-def get_system_proxies():
-    """
-    Retrieves the system's HTTP and HTTPS proxies.
-
-    Returns:
-        dict: A dictionary containing the proxies.
-    """
-    try:
-        proxies = urllib.request.getproxies()
-        print("System proxies:", proxies)
-        return {
-            "http": proxies.get("http"),
-            "https": proxies.get("http")
-        }
-    except Exception as e:
-        print(f"Error getting system proxies: {e}")
-        return {}
-
-def download_film(file_url, token, cache_path, proxies, film_id, title, chunk_size=100 * 1024 * 1024):
+def download_film(file_url, token, cache_path, film_id, title, chunk_size=100 * 1024 * 1024):
     """
     Downloads a file from the specified URL and saves it to the cache path.
     Tracks the download progress.
@@ -38,15 +18,14 @@ def download_film(file_url, token, cache_path, proxies, film_id, title, chunk_si
         file_url (str): The URL of the file to download.
         token (str): The authorization token for the request.
         cache_path (str): The path to save the downloaded file.
-        proxies (dict): Proxies for the request.
         film_id (str): Unique identifier for the film download.
         title (str): The title of the film.
         chunk_size (int): Size of each chunk to download.
     """
-    print(f"Downloading file from URL: {file_url} to {cache_path} with proxies: {proxies}")
+    print(f"Downloading file from URL: {file_url} to {cache_path}")
     headers = {'Authorization': f'Bearer {token}'}
     try:
-        response = requests.get(file_url, headers=headers, proxies=proxies, stream=True)
+        response = requests.get(file_url, headers=headers, stream=True)
         response.raise_for_status()
         
         total_size = int(response.headers.get('content-length', 0))
@@ -60,7 +39,7 @@ def download_film(file_url, token, cache_path, proxies, film_id, title, chunk_si
                 download_progress[film_id]["downloaded"] += len(data)
         
         print(f'File cached to {cache_path} successfully.')
-        update_film_store_json(title, cache_path)
+        update_film_store(title, cache_path)
         download_progress[film_id]["status"] = "Completed"
     except RequestException as e:
         print(f"Error downloading file: {e}")
@@ -99,7 +78,7 @@ def get_download_progress(id):
         return {"total": total, "downloaded": downloaded, "progress": progress, "status": status, "eta": eta}
     return {"total": 0, "downloaded": 0, "progress": 0, "status": "Not Found", "eta": None}
 
-def update_film_store_json(title, cache_path):
+def update_film_store(title, cache_path):
     """
     Updates the film store JSON with the new file.
 
@@ -107,22 +86,12 @@ def update_film_store_json(title, cache_path):
         title (str): The title of the film.
         cache_path (str): The local path where the file is saved.
     """
-    FILM_STORE_JSON_PATH = os.path.join(CACHE_DIR, "film_store.json")
-
-    film_store_data = {}
-    if os.path.exists(FILM_STORE_JSON_PATH):
-        with open(FILM_STORE_JSON_PATH, 'r') as json_file:
-            film_store_data = json.load(json_file)
-
-    film_store_data[title] = cache_path
-
-    with open(FILM_STORE_JSON_PATH, 'w') as json_file:
-        json.dump(film_store_data, json_file, indent=2)
+    instance.FILM_STORE[title] = cache_path
     print(f'Film store updated with {title}.')
 
 
 ###############################################################################
-def download_episode(file_url, token, cache_path, proxies, episode_id, title, chunk_size=100 * 1024 * 1024):
+def download_episode(file_url, token, cache_path, episode_id, title, chunk_size=100 * 1024 * 1024):
     """
     Downloads a file from the specified URL and saves it to the cache path.
     Tracks the download progress.
@@ -131,15 +100,14 @@ def download_episode(file_url, token, cache_path, proxies, episode_id, title, ch
         file_url (str): The URL of the file to download.
         token (str): The authorization token for the request.
         cache_path (str): The path to save the downloaded file.
-        proxies (dict): Proxies for the request.
         film_id (str): Unique identifier for the film download.
         title (str): The title of the film.
         chunk_size (int): Size of each chunk to download.
     """
-    print(f"Downloading file from URL: {file_url} to {cache_path} with proxies: {proxies}")
+    print(f"Downloading file from URL: {file_url} to {cache_path}")
     headers = {'Authorization': f'Bearer {token}'}
     try:
-        response = requests.get(file_url, headers=headers, proxies=proxies, stream=True)
+        response = requests.get(file_url, headers=headers, stream=True)
         response.raise_for_status()
         
         total_size = int(response.headers.get('content-length', 0))
@@ -153,7 +121,7 @@ def download_episode(file_url, token, cache_path, proxies, episode_id, title, ch
                 download_progress[episode_id]["downloaded"] += len(data)
         
         print(f'File cached to {cache_path} successfully.')
-        update_tv_store_json(title, cache_path)
+        update_tv_store(title, cache_path)
         download_progress[episode_id]["status"] = "Completed"
     except RequestException as e:
         print(f"Error downloading file: {e}")
@@ -166,7 +134,7 @@ def download_episode(file_url, token, cache_path, proxies, episode_id, title, ch
             download_progress[episode_id]["end_time"] = time.time()
 
 
-def update_tv_store_json(title, cache_path):
+def update_tv_store(title, cache_path):
     """
     Updates the TV store JSON with the new file, organizing by title, season, and episode.
 
@@ -174,76 +142,24 @@ def update_tv_store_json(title, cache_path):
         title (str): The title of the TV show.
         cache_path (str): The local path where the file is saved.
     """
-    TV_STORE_JSON_PATH = os.path.join(CACHE_DIR, "tv_store.json")
-
-    tv_store_data = {}
-    if os.path.exists(TV_STORE_JSON_PATH):
-        with open(TV_STORE_JSON_PATH, 'r') as json_file:
-            tv_store_data = json.load(json_file)
-
     # Extract season and episode information from the cache_path
     season_part = os.path.basename(os.path.dirname(cache_path))  # Extracts 'Season 1'
     episode_part = os.path.basename(cache_path)  # Extracts 'Grand Blue Dreaming - S01E01 - Deep Blue HDTV-720p.mp4'
 
     # Create the structure if not already present
-    if title not in tv_store_data:
-        tv_store_data[title] = {}
+    if title not in instance.TV_STORE:
+        instance.TV_STORE[title] = {}
     
-    if season_part not in tv_store_data[title]:
-        tv_store_data[title][season_part] = {}
+    if season_part not in instance.TV_STORE[title]:
+        instance.TV_STORE[title][season_part] = {}
 
     # Assuming episode_part is unique for each episode within a season
-    tv_store_data[title][season_part][episode_part] = cache_path
-
-    with open(TV_STORE_JSON_PATH, 'w') as json_file:
-        json.dump(tv_store_data, json_file, indent=2)
-
+    instance.TV_STORE[title][season_part][episode_part] = cache_path
     print(f'TV store updated with {title}, {season_part}, {episode_part}.')
 
-###############################################################################
-def get_file_structure(repo, token, path="", proxies=None):
-    """
-    Fetches the file structure of a specified Hugging Face repository.
-
-    Args:
-        repo (str): The name of the repository.
-        token (str): The authorization token for the request.
-        path (str, optional): The specific path in the repository. Defaults to "".
-        proxies (dict, optional): The proxies to use for the request. Defaults to None.
-
-    Returns:
-        list: A list of file structure information.
-    """
-    api_url = f"https://huggingface.co/api/models/{repo}/tree/main/{path}"
-    headers = {'Authorization': f'Bearer {token}'}
-    print(f"Fetching file structure from URL: {api_url} with proxies: {proxies}")
-    try:
-        response = requests.get(api_url, headers=headers, proxies=proxies)
-        response.raise_for_status()
-        return response.json()
-    except RequestException as e:
-        print(f"Error fetching file structure: {e}")
-        return []
-
-def write_file_structure_to_json(file_structure, file_path):
-    """
-    Writes the file structure to a JSON file.
-
-    Args:
-        file_structure (list): The file structure data.
-        file_path (str): The path where the JSON file will be saved.
-    """
-    try:
-        with open(file_path, 'w') as json_file:
-            json.dump(file_structure, json_file, indent=2)
-        print(f'File structure written to {file_path}')
-    except IOError as e:
-        print(f"Error writing file structure to JSON: {e}")
-
 if __name__ == "__main__":
     file_url = "https://huggingface.co/Unicone-Studio/jellyfin_media/resolve/main/films/Funky%20Monkey%202004/Funky%20Monkey%20(2004)%20Web-dl%201080p.mp4"
     token = os.getenv("TOKEN")
     cache_path = os.path.join(CACHE_DIR, "films/Funky Monkey 2004/Funky Monkey (2004) Web-dl 1080p.mp4")
-    proxies = get_system_proxies()
     film_id = "funky_monkey_2004"  # Unique identifier for the film download
-    download_film(file_url, token, cache_path, proxies=proxies, film_id=film_id)
+    download_film(file_url, token, cache_path, film_id=film_id)

indexer.py

@@ -1,32 +0,0 @@
-import json
-from hf_scrapper import get_system_proxies, get_file_structure, write_file_structure_to_json
-from dotenv import load_dotenv
-import os
-
-load_dotenv()
-
-def index_repository(token, repo, current_path="", proxies=None):
-    file_structure = get_file_structure(repo, token, current_path, proxies)
-    full_structure = []
-    for item in file_structure:
-        if item['type'] == 'directory':
-            sub_directory_structure = index_repository(token, repo, item['path'], proxies)
-            full_structure.append({
-                "type": "directory",
-                "path": item['path'],
-                "contents": sub_directory_structure
-            })
-        else:
-            full_structure.append(item)
-    return full_structure
-
-def indexer():
-    token = os.getenv("TOKEN")
-    repo = os.getenv("REPO")
-    output_path = os.getenv("INDEX_FILE")
-    
-    proxies = get_system_proxies()
-    full_structure = index_repository(token, repo, "", proxies)
-    write_file_structure_to_json(full_structure, output_path)
-    print(f"Full file structure for repository '{repo}' has been indexed and saved to {output_path}")
-