diff --git a/utilityscripts/cobblemon_spawn_csv_to_json.py b/utilityscripts/cobblemon_spawn_csv_to_json.py index 3246fe84af..794e022c47 100644 --- a/utilityscripts/cobblemon_spawn_csv_to_json.py +++ b/utilityscripts/cobblemon_spawn_csv_to_json.py @@ -30,12 +30,15 @@ def init_filters(): included_generations = [1, 2, 3, 4, 5, 6, 7, 8, 9] -def ui_init_filters(pokemon_nrs_min, pokemon_nrs_max, included_grps, known_cntxts, bucket_map): - global pokemon_numbers, included_groups, known_contexts, bucket_mapping, included_generations +def ui_init_filters(pokemon_nrs_min, pokemon_nrs_max, included_grps, known_cntxts, bucket_map, cstm_files, cstm_dirs): + global pokemon_numbers, included_groups, known_contexts, bucket_mapping, included_generations, custom_files, custom_dirs pokemon_numbers = range(pokemon_nrs_min, pokemon_nrs_max + 1) included_groups = included_grps known_contexts = known_cntxts bucket_mapping = bucket_map + custom_files = cstm_files + custom_dirs = cstm_dirs + included_generations = [] @@ -228,25 +231,43 @@ def main(pokemon_data_dir, spawn_spreadsheet_path="", only_update_existing_files os.makedirs(pokemon_data_dir, exist_ok=True) # Handle dataframes depending on the usage of the public template spreadsheet - csv_grouped_with_folder_file = [] + csvs_grouped_by_folder_and_or_file = [] # Create two separate DataFrames: one for rows with 'folder' and 'file_name' and another for rows without them if {'Folder', 'File Name'}.issubset(csv_df.columns): - csv_df_with_folder_file = csv_df.dropna(subset=['Folder', 'File Name']) + csv_df_with_folder_or_file = csv_df.dropna(subset=['Folder', 'File Name'], how='all') # Group the DataFrame with 'folder' and 'file_name' by these columns - csv_grouped_with_folder_file = csv_df_with_folder_file.groupby(['Folder', 'File Name']) - csv_df_without_folder_file = csv_df[csv_df['Folder'].isna() | csv_df['File Name'].isna()] - # Group the DataFrame without 'folder' and 'file_name' by dex number - csv_grouped_without_folder_file = csv_df_without_folder_file.groupby('No.') + if csv_df_with_folder_or_file['Folder'].isna().any(): + csv_folder_file = csv_df_with_folder_or_file[csv_df_with_folder_or_file['Folder'].isna()] + csv_grouped_by_folder_file = csv_folder_file.groupby('File Name') + csvs_grouped_by_folder_and_or_file.append(csv_grouped_by_folder_file) + # If 'File Name' is NaN, group by 'Folder' only + if csv_df_with_folder_or_file['File Name'].isna().any(): + csv_with_file_name = csv_df_with_folder_or_file[csv_df_with_folder_or_file['File Name'].isna()] + csv_grouped_by_folder_and_no = csv_with_file_name.groupby(['Folder', 'No.']) + csvs_grouped_by_folder_and_or_file.append(csv_grouped_by_folder_and_no) + # If there are any entries where neither is NaN, group those by both + if csv_df_with_folder_or_file[['Folder', 'File Name']].notna().all(axis=1).any(): + csv_folder_and_file = csv_df_with_folder_or_file.dropna(subset=['Folder', 'File Name'], how='any') + csv_grouped_by_folder_and_file = csv_folder_and_file.groupby(['Folder', 'File Name']) + csvs_grouped_by_folder_and_or_file.append(csv_grouped_by_folder_and_file) + # Only fill the with_folder_file DataFrame with rows that have 'folder' and 'file_name' columns empty if both filters include '' + if ('' in custom_files) and ('' in custom_dirs): + csv_df_without_folder_file = csv_df[csv_df['Folder'].isna() & csv_df['File Name'].isna()] + # Group the DataFrame without 'folder' and 'file_name' by dex number + csv_grouped_without_folder_file = csv_df_without_folder_file.groupby('No.') + else: + csv_grouped_without_folder_file = pd.DataFrame() else: csv_grouped_without_folder_file = csv_df.groupby('No.') print_warning("Modifying files...") # Processing each Pokémon group and converting it to JSON try: - dataframes_to_process = [(csv_grouped_with_folder_file, MERGED_TOGETHER), - (csv_grouped_without_folder_file, 'Default Groupings')] + dataframes_to_process = [(df, MERGED_TOGETHER) for df in csvs_grouped_by_folder_and_or_file] + dataframes_to_process.append((csv_grouped_without_folder_file, 'Default Groupings')) for df, id_type in dataframes_to_process: + print(df.head(10)) for id_value, group in tqdm(df, desc=f"Processing {id_type} ...", bar_format='\033[92m' + '{l_bar}\033[0m{bar:58}\033[92m{r_bar}\033[0m', colour='blue'): @@ -254,7 +275,9 @@ def main(pokemon_data_dir, spawn_spreadsheet_path="", only_update_existing_files # Determine the file_id based on the id_type if id_type == MERGED_TOGETHER: file_id = group['No.'].iloc[0] - file_name = group['File Name'].iloc[0] or None + file_name = group['File Name'].iloc[0] + if pd.isna(file_name): + file_name = None else: file_id = id_value file_name = None @@ -266,7 +289,6 @@ def main(pokemon_data_dir, spawn_spreadsheet_path="", only_update_existing_files else: # Handle the case when 'Folder' is null save_json_to_file(pokemon_json, file_id, group['Pokémon'].iloc[0], pokemon_data_dir, file_name) - print() finally: print_report() @@ -295,21 +317,38 @@ def validateAndFilterData(csv_df, only_update_existing_files=False, ignore_filte csv_df = csv_df[csv_df['Pokémon'].notna()] # Filter the data if not ignore_filters: - # Filter by pokemon number - if pokemon_numbers: - csv_df = csv_df[csv_df['No.'].isin(list(pokemon_numbers))] - # Filter by group - if included_groups: - csv_df = csv_df[csv_df['Group'].isin(included_groups)] - # Filter by context - if known_contexts: - csv_df = csv_df[csv_df['Context'].isin(known_contexts)] - # Filter by bucket - if bucket_mapping: - csv_df = csv_df[csv_df['Bucket'].isin(bucket_mapping)] - # Filter by generation - if included_generations: - csv_df = csv_df[csv_df['Gen'].isin(included_generations)] + # Define a dictionary mapping filter variables to DataFrame column names + filter_dict = { + 'pokemon_numbers': 'No.', + 'included_groups': 'Group', + 'known_contexts': 'Context', + 'bucket_mapping': 'Bucket', + 'included_generations': 'Gen', + 'custom_dirs': 'Folder', + 'custom_files': 'File Name' + } + + # Iterate over the dictionary and apply the filters + for filter_var, column_name in filter_dict.items(): + filter_values = globals()[filter_var] + if filter_values: + # Replace '' with None in filter_values + filter_values = [value if value != '' else None for value in filter_values] + if None in filter_values: + csv_df = csv_df[csv_df[column_name].isin(filter_values) | csv_df[column_name].isnull()] + else: + csv_df = csv_df[csv_df[column_name].isin(filter_values)] + + def is_continuous_range(lst): + try: + return lst == list(range(min(lst), max(lst)+1)) + except Exception as e: + return False + + if is_continuous_range(filter_values): + print(f"Filtering by {filter_var}: ({min(filter_values)} - {max(filter_values)})") + else: + print(f"Filtering by {filter_var}: {filter_values}") if only_update_existing_files: # extract the dex numbers from the filenames of the existing files existing_dex_numbers = [int(file.split('/')[-1].split('_')[0]) for file in os.listdir(default_pokemon_data_dir)] @@ -724,6 +763,7 @@ def write_to_sqlite(df, db_name, table_name): def save_json_to_file(json_data, file_id, pokemon_name, output_directory, file_name=None): + print(f"Saving {pokemon_name} to {output_directory}") # Remove any ' from the pokemon name pokemon_name = pokemon_name.replace("'", "") # If the pokemon name contains a space, remove everything after the space (because of basculin) diff --git a/utilityscripts/public_spawns_to_json.py b/utilityscripts/public_spawns_to_json.py index 099fb6f6b6..8c77d90b99 100644 --- a/utilityscripts/public_spawns_to_json.py +++ b/utilityscripts/public_spawns_to_json.py @@ -8,6 +8,8 @@ import cobblemon_spawn_csv_to_json groups = ["basic", "boss", "fossil"] contexts = ["grounded", "submerged", "seafloor", "surface"] buckets = ["common", "uncommon", "rare", "ultra-rare"] +folders = set() +file_names = set() process_button = None @@ -21,7 +23,9 @@ def process_file(input_file, output_directory, filters, pokemon_number_start, po cobblemon_spawn_csv_to_json.ui_init_filters(pokemon_number_start, pokemon_number_end, included_grps=filtered_filters['included_groups'], known_cntxts=filtered_filters['known_contexts'], - bucket_map=filtered_filters['bucket_mapping']) + bucket_map=filtered_filters['bucket_mapping'], + cstm_files=filtered_filters['file_name'], + cstm_dirs=filtered_filters['folder']) if not validate_number_range(pokemon_number_start, pokemon_number_end): messagebox.showinfo("Invalid Pokémon number range. Please provide a valid range.") @@ -34,8 +38,8 @@ def validate_number_range(a, b): def select_input_file(input_file_var, input_file_label, output_dir_var, pokemon_number_start_var, - pokemon_number_end_var, filter_vars, frame3): - global groups, contexts, buckets, process_button + pokemon_number_end_var, filter_vars, frame3, folder_var, file_name_var): + global groups, contexts, buckets, process_button, folders, file_names filepath = filedialog.askopenfilename() if filepath: input_file_var.set(filepath) @@ -57,10 +61,10 @@ def select_input_file(input_file_var, input_file_label, output_dir_var, pokemon_ # if the file is an excel file, read it with pandas if input_file_var.get().endswith('.xlsx'): csv_df = pd.read_excel(input_file_var.get(), engine='openpyxl', - dtype={'Pokémon': str, 'Entry': str, 'No.': int}) + dtype={'Pokémon': str, 'Entry': str, 'No.': int, 'Folder': str, 'File Name': str}) # if the file is a csv file, read it with pandas elif input_file_var.get().endswith('.csv'): - csv_df = pd.read_csv(input_file_var.get(), dtype={'Pokémon': str, 'Entry': str, 'No.': int}) + csv_df = pd.read_csv(input_file_var.get(), dtype={'Pokémon': str, 'Entry': str, 'No.': int, 'Folder': str, 'File Name': str}) else: # show message messagebox.showinfo("Invalid file format. Please provide a valid excel or csv file") @@ -75,8 +79,16 @@ def select_input_file(input_file_var, input_file_label, output_dir_var, pokemon_ buckets = csv_df['Bucket'].unique() buckets = [bucket if bucket == bucket else '' for bucket in buckets] + # Parse the Folder and File Name columns + folders = set(csv_df['Folder'].unique()) + folders = {folder if folder == folder else "" for folder in folders} + folder_var.set(next(iter(folders), "")) + file_names = set(csv_df['File Name'].unique()) + file_names = {file_name if file_name == file_name else "" for file_name in file_names} + file_name_var.set(next(iter(file_names), "")) + update_filter_ui(filter_vars, frame3, input_file_var, output_dir_var, pokemon_number_start_var, - pokemon_number_end_var) + pokemon_number_end_var, folder_var, file_name_var) # Remove the loading message loading_label.destroy() @@ -123,6 +135,8 @@ def setup_ui(root): output_dir_var = tk.StringVar() pokemon_number_start_var = tk.IntVar(value=0) # Default start of dex range pokemon_number_end_var = tk.IntVar(value=1111) # Default end of dex range + folder_var = tk.StringVar() + file_name_var = tk.StringVar() frame = ttk.Frame(root, padding="10") frame.grid() @@ -132,11 +146,10 @@ def setup_ui(root): frame3.grid(sticky="wens") filter_vars, frame3 = setup_filter_ui(frame3, input_file_var, output_dir_var, - pokemon_number_start_var, - pokemon_number_end_var) + pokemon_number_start_var, pokemon_number_end_var, folder_var, file_name_var) input_file_label, output_dir_label = setup_file_selection_ui(frame, input_file_var, output_dir_var, pokemon_number_start_var, pokemon_number_end_var, - filter_vars, frame3) + filter_vars, frame3, folder_var, file_name_var) # Display a warning popup to remind the user to backup their spawn.json files before they run the script messagebox.showwarning("Warning", @@ -147,12 +160,12 @@ def setup_ui(root): def setup_file_selection_ui(frame, input_file_var, output_dir_var, pokemon_number_start_var, - pokemon_number_end_var, filter_vars, frame3): + pokemon_number_end_var, filter_vars, frame3, folder_var, file_name_var): input_file_button = ttk.Button(frame, text="Select Input File", command=lambda: select_input_file(input_file_var, input_file_label, output_dir_var, pokemon_number_start_var, pokemon_number_end_var, - filter_vars, frame3)) + filter_vars, frame3, folder_var, file_name_var)) input_file_button.grid(column=0, row=0, pady=5) input_file_label = ttk.Label(frame, text="No file selected", foreground='red') @@ -170,7 +183,8 @@ def setup_file_selection_ui(frame, input_file_var, output_dir_var, pokemon_numbe def update_filter_ui(filter_vars, frame3, input_file_var, output_dir_var, pokemon_number_start_var, - pokemon_number_end_var): + pokemon_number_end_var, folder_var, file_name_var): + global folders, file_names # Clear the current filter UI for widget in frame3.winfo_children(): widget.destroy() @@ -179,12 +193,14 @@ def update_filter_ui(filter_vars, frame3, input_file_var, output_dir_var, pokemo filter_vars['included_groups'] = {name: tk.IntVar(value=1) for name in groups} filter_vars['known_contexts'] = {name: tk.IntVar(value=1) for name in contexts} filter_vars['bucket_mapping'] = {name: tk.IntVar(value=1) for name in buckets} + filter_vars['folder'] = {name: tk.IntVar(value=1) for name in folders} + filter_vars['file_name'] = {name: tk.IntVar(value=1) for name in file_names} # Recreate the filter UI - setup_filter_ui(frame3, input_file_var, output_dir_var, pokemon_number_start_var, pokemon_number_end_var) + setup_filter_ui(frame3, input_file_var, output_dir_var, pokemon_number_start_var, pokemon_number_end_var, folder_var, file_name_var) -def setup_filter_ui(frame3, input_file_var, output_dir_var, pokemon_number_start_var, pokemon_number_end_var): +def setup_filter_ui(frame3, input_file_var, output_dir_var, pokemon_number_start_var, pokemon_number_end_var, folder_var, file_name_var): global process_button # Label and entry for specifying the range of Pokémon numbers to include ttk.Label(frame3, text="Pokédex Number Range:").grid(column=0, row=0, columnspan=5, pady=5) @@ -196,6 +212,8 @@ def setup_filter_ui(frame3, input_file_var, output_dir_var, pokemon_number_start 'included_groups': {name: tk.IntVar(value=1) for name in groups}, 'known_contexts': {name: tk.IntVar(value=1) for name in contexts}, 'bucket_mapping': {name: tk.IntVar(value=1) for name in buckets}, + 'folder': {name: tk.IntVar(value=1) for name in folders}, + 'file_name': {name: tk.IntVar(value=1) for name in file_names}, } # Custom styles for toggle buttons @@ -208,6 +226,8 @@ def setup_filter_ui(frame3, input_file_var, output_dir_var, pokemon_number_start ('included_groups', "Groups"), ('known_contexts', "Contexts"), ('bucket_mapping', "Buckets"), + ('folder', "Folders"), + ('file_name', "File Names"), ] current_row = 2