Allowing any combination of Folder and File Name [skip-ci]

This commit is contained in:
Waldleufer committed 2024-04-10 21:47:26 +02:00
1 parent e6a487dc52
commit bf330c112b
2 files changed
+101 -41

No files matched your search

+67 -27
View File
@@ -30,12 +30,15 @@ def init_filters():
included_generations = [1, 2, 3, 4, 5, 6, 7, 8, 9]
def ui_init_filters(pokemon_nrs_min, pokemon_nrs_max, included_grps, known_cntxts, bucket_map):
global pokemon_numbers, included_groups, known_contexts, bucket_mapping, included_generations
def ui_init_filters(pokemon_nrs_min, pokemon_nrs_max, included_grps, known_cntxts, bucket_map, cstm_files, cstm_dirs):
global pokemon_numbers, included_groups, known_contexts, bucket_mapping, included_generations, custom_files, custom_dirs
pokemon_numbers = range(pokemon_nrs_min, pokemon_nrs_max + 1)
included_groups = included_grps
known_contexts = known_cntxts
bucket_mapping = bucket_map
custom_files = cstm_files
custom_dirs = cstm_dirs
included_generations = []
@@ -228,25 +231,43 @@ def main(pokemon_data_dir, spawn_spreadsheet_path="", only_update_existing_files
os.makedirs(pokemon_data_dir, exist_ok=True)
# Handle dataframes depending on the usage of the public template spreadsheet
csv_grouped_with_folder_file = []
csvs_grouped_by_folder_and_or_file = []
# Create two separate DataFrames: one for rows with 'folder' and 'file_name' and another for rows without them
if {'Folder', 'File Name'}.issubset(csv_df.columns):
csv_df_with_folder_file = csv_df.dropna(subset=['Folder', 'File Name'])
csv_df_with_folder_or_file = csv_df.dropna(subset=['Folder', 'File Name'], how='all')
# Group the DataFrame with 'folder' and 'file_name' by these columns
csv_grouped_with_folder_file = csv_df_with_folder_file.groupby(['Folder', 'File Name'])
csv_df_without_folder_file = csv_df[csv_df['Folder'].isna() | csv_df['File Name'].isna()]
# Group the DataFrame without 'folder' and 'file_name' by dex number
csv_grouped_without_folder_file = csv_df_without_folder_file.groupby('No.')
if csv_df_with_folder_or_file['Folder'].isna().any():
csv_folder_file = csv_df_with_folder_or_file[csv_df_with_folder_or_file['Folder'].isna()]
csv_grouped_by_folder_file = csv_folder_file.groupby('File Name')
csvs_grouped_by_folder_and_or_file.append(csv_grouped_by_folder_file)
# If 'File Name' is NaN, group by 'Folder' only
if csv_df_with_folder_or_file['File Name'].isna().any():
csv_with_file_name = csv_df_with_folder_or_file[csv_df_with_folder_or_file['File Name'].isna()]
csv_grouped_by_folder_and_no = csv_with_file_name.groupby(['Folder', 'No.'])
csvs_grouped_by_folder_and_or_file.append(csv_grouped_by_folder_and_no)
# If there are any entries where neither is NaN, group those by both
if csv_df_with_folder_or_file[['Folder', 'File Name']].notna().all(axis=1).any():
csv_folder_and_file = csv_df_with_folder_or_file.dropna(subset=['Folder', 'File Name'], how='any')
csv_grouped_by_folder_and_file = csv_folder_and_file.groupby(['Folder', 'File Name'])
csvs_grouped_by_folder_and_or_file.append(csv_grouped_by_folder_and_file)
# Only fill the with_folder_file DataFrame with rows that have 'folder' and 'file_name' columns empty if both filters include ''
if ('' in custom_files) and ('' in custom_dirs):
csv_df_without_folder_file = csv_df[csv_df['Folder'].isna() & csv_df['File Name'].isna()]
# Group the DataFrame without 'folder' and 'file_name' by dex number
csv_grouped_without_folder_file = csv_df_without_folder_file.groupby('No.')
else:
csv_grouped_without_folder_file = pd.DataFrame()
else:
csv_grouped_without_folder_file = csv_df.groupby('No.')
print_warning("Modifying files...")
# Processing each Pokémon group and converting it to JSON
try:
dataframes_to_process = [(csv_grouped_with_folder_file, MERGED_TOGETHER),
(csv_grouped_without_folder_file, 'Default Groupings')]
dataframes_to_process = [(df, MERGED_TOGETHER) for df in csvs_grouped_by_folder_and_or_file]
dataframes_to_process.append((csv_grouped_without_folder_file, 'Default Groupings'))
for df, id_type in dataframes_to_process:
print(df.head(10))
for id_value, group in tqdm(df, desc=f"Processing {id_type} ...",
bar_format='\033[92m' + '{l_bar}\033[0m{bar:58}\033[92m{r_bar}\033[0m',
colour='blue'):
@@ -254,7 +275,9 @@ def main(pokemon_data_dir, spawn_spreadsheet_path="", only_update_existing_files
# Determine the file_id based on the id_type
if id_type == MERGED_TOGETHER:
file_id = group['No.'].iloc[0]
file_name = group['File Name'].iloc[0] or None
file_name = group['File Name'].iloc[0]
if pd.isna(file_name):
file_name = None
else:
file_id = id_value
file_name = None
@@ -266,7 +289,6 @@ def main(pokemon_data_dir, spawn_spreadsheet_path="", only_update_existing_files
else:
# Handle the case when 'Folder' is null
save_json_to_file(pokemon_json, file_id, group['Pokémon'].iloc[0], pokemon_data_dir, file_name)
print()
finally:
print_report()
@@ -295,21 +317,38 @@ def validateAndFilterData(csv_df, only_update_existing_files=False, ignore_filte
csv_df = csv_df[csv_df['Pokémon'].notna()]
# Filter the data
if not ignore_filters:
# Filter by pokemon number
if pokemon_numbers:
csv_df = csv_df[csv_df['No.'].isin(list(pokemon_numbers))]
# Filter by group
if included_groups:
csv_df = csv_df[csv_df['Group'].isin(included_groups)]
# Filter by context
if known_contexts:
csv_df = csv_df[csv_df['Context'].isin(known_contexts)]
# Filter by bucket
if bucket_mapping:
csv_df = csv_df[csv_df['Bucket'].isin(bucket_mapping)]
# Filter by generation
if included_generations:
csv_df = csv_df[csv_df['Gen'].isin(included_generations)]
# Define a dictionary mapping filter variables to DataFrame column names
filter_dict = {
'pokemon_numbers': 'No.',
'included_groups': 'Group',
'known_contexts': 'Context',
'bucket_mapping': 'Bucket',
'included_generations': 'Gen',
'custom_dirs': 'Folder',
'custom_files': 'File Name'
}
# Iterate over the dictionary and apply the filters
for filter_var, column_name in filter_dict.items():
filter_values = globals()[filter_var]
if filter_values:
# Replace '' with None in filter_values
filter_values = [value if value != '' else None for value in filter_values]
if None in filter_values:
csv_df = csv_df[csv_df[column_name].isin(filter_values) | csv_df[column_name].isnull()]
else:
csv_df = csv_df[csv_df[column_name].isin(filter_values)]
def is_continuous_range(lst):
try:
return lst == list(range(min(lst), max(lst)+1))
except Exception as e:
return False
if is_continuous_range(filter_values):
print(f"Filtering by {filter_var}: ({min(filter_values)} - {max(filter_values)})")
else:
print(f"Filtering by {filter_var}: {filter_values}")
if only_update_existing_files:
# extract the dex numbers from the filenames of the existing files
existing_dex_numbers = [int(file.split('/')[-1].split('_')[0]) for file in os.listdir(default_pokemon_data_dir)]
@@ -724,6 +763,7 @@ def write_to_sqlite(df, db_name, table_name):
def save_json_to_file(json_data, file_id, pokemon_name, output_directory, file_name=None):
print(f"Saving {pokemon_name} to {output_directory}")
# Remove any ' from the pokemon name
pokemon_name = pokemon_name.replace("'", "")
# If the pokemon name contains a space, remove everything after the space (because of basculin)
+34 -14
View File
@@ -8,6 +8,8 @@ import cobblemon_spawn_csv_to_json
groups = ["basic", "boss", "fossil"]
contexts = ["grounded", "submerged", "seafloor", "surface"]
buckets = ["common", "uncommon", "rare", "ultra-rare"]
folders = set()
file_names = set()
process_button = None
@@ -21,7 +23,9 @@ def process_file(input_file, output_directory, filters, pokemon_number_start, po
cobblemon_spawn_csv_to_json.ui_init_filters(pokemon_number_start, pokemon_number_end,
included_grps=filtered_filters['included_groups'],
known_cntxts=filtered_filters['known_contexts'],
bucket_map=filtered_filters['bucket_mapping'])
bucket_map=filtered_filters['bucket_mapping'],
cstm_files=filtered_filters['file_name'],
cstm_dirs=filtered_filters['folder'])
if not validate_number_range(pokemon_number_start, pokemon_number_end):
messagebox.showinfo("Invalid Pokémon number range. Please provide a valid range.")
@@ -34,8 +38,8 @@ def validate_number_range(a, b):
def select_input_file(input_file_var, input_file_label, output_dir_var, pokemon_number_start_var,
pokemon_number_end_var, filter_vars, frame3):
global groups, contexts, buckets, process_button
pokemon_number_end_var, filter_vars, frame3, folder_var, file_name_var):
global groups, contexts, buckets, process_button, folders, file_names
filepath = filedialog.askopenfilename()
if filepath:
input_file_var.set(filepath)
@@ -57,10 +61,10 @@ def select_input_file(input_file_var, input_file_label, output_dir_var, pokemon_
# if the file is an excel file, read it with pandas
if input_file_var.get().endswith('.xlsx'):
csv_df = pd.read_excel(input_file_var.get(), engine='openpyxl',
dtype={'Pokémon': str, 'Entry': str, 'No.': int})
dtype={'Pokémon': str, 'Entry': str, 'No.': int, 'Folder': str, 'File Name': str})
# if the file is a csv file, read it with pandas
elif input_file_var.get().endswith('.csv'):
csv_df = pd.read_csv(input_file_var.get(), dtype={'Pokémon': str, 'Entry': str, 'No.': int})
csv_df = pd.read_csv(input_file_var.get(), dtype={'Pokémon': str, 'Entry': str, 'No.': int, 'Folder': str, 'File Name': str})
else:
# show message
messagebox.showinfo("Invalid file format. Please provide a valid excel or csv file")
@@ -75,8 +79,16 @@ def select_input_file(input_file_var, input_file_label, output_dir_var, pokemon_
buckets = csv_df['Bucket'].unique()
buckets = [bucket if bucket == bucket else '' for bucket in buckets]
# Parse the Folder and File Name columns
folders = set(csv_df['Folder'].unique())
folders = {folder if folder == folder else "" for folder in folders}
folder_var.set(next(iter(folders), ""))
file_names = set(csv_df['File Name'].unique())
file_names = {file_name if file_name == file_name else "" for file_name in file_names}
file_name_var.set(next(iter(file_names), ""))
update_filter_ui(filter_vars, frame3, input_file_var, output_dir_var, pokemon_number_start_var,
pokemon_number_end_var)
pokemon_number_end_var, folder_var, file_name_var)
# Remove the loading message
loading_label.destroy()
@@ -123,6 +135,8 @@ def setup_ui(root):
output_dir_var = tk.StringVar()
pokemon_number_start_var = tk.IntVar(value=0) # Default start of dex range
pokemon_number_end_var = tk.IntVar(value=1111) # Default end of dex range
folder_var = tk.StringVar()
file_name_var = tk.StringVar()
frame = ttk.Frame(root, padding="10")
frame.grid()
@@ -132,11 +146,10 @@ def setup_ui(root):
frame3.grid(sticky="wens")
filter_vars, frame3 = setup_filter_ui(frame3, input_file_var, output_dir_var,
pokemon_number_start_var,
pokemon_number_end_var)
pokemon_number_start_var, pokemon_number_end_var, folder_var, file_name_var)
input_file_label, output_dir_label = setup_file_selection_ui(frame, input_file_var, output_dir_var,
pokemon_number_start_var, pokemon_number_end_var,
filter_vars, frame3)
filter_vars, frame3, folder_var, file_name_var)
# Display a warning popup to remind the user to backup their spawn.json files before they run the script
messagebox.showwarning("Warning",
@@ -147,12 +160,12 @@ def setup_ui(root):
def setup_file_selection_ui(frame, input_file_var, output_dir_var, pokemon_number_start_var,
pokemon_number_end_var, filter_vars, frame3):
pokemon_number_end_var, filter_vars, frame3, folder_var, file_name_var):
input_file_button = ttk.Button(frame, text="Select Input File",
command=lambda: select_input_file(input_file_var, input_file_label,
output_dir_var,
pokemon_number_start_var, pokemon_number_end_var,
filter_vars, frame3))
filter_vars, frame3, folder_var, file_name_var))
input_file_button.grid(column=0, row=0, pady=5)
input_file_label = ttk.Label(frame, text="No file selected", foreground='red')
@@ -170,7 +183,8 @@ def setup_file_selection_ui(frame, input_file_var, output_dir_var, pokemon_numbe
def update_filter_ui(filter_vars, frame3, input_file_var, output_dir_var, pokemon_number_start_var,
pokemon_number_end_var):
pokemon_number_end_var, folder_var, file_name_var):
global folders, file_names
# Clear the current filter UI
for widget in frame3.winfo_children():
widget.destroy()
@@ -179,12 +193,14 @@ def update_filter_ui(filter_vars, frame3, input_file_var, output_dir_var, pokemo
filter_vars['included_groups'] = {name: tk.IntVar(value=1) for name in groups}
filter_vars['known_contexts'] = {name: tk.IntVar(value=1) for name in contexts}
filter_vars['bucket_mapping'] = {name: tk.IntVar(value=1) for name in buckets}
filter_vars['folder'] = {name: tk.IntVar(value=1) for name in folders}
filter_vars['file_name'] = {name: tk.IntVar(value=1) for name in file_names}
# Recreate the filter UI
setup_filter_ui(frame3, input_file_var, output_dir_var, pokemon_number_start_var, pokemon_number_end_var)
setup_filter_ui(frame3, input_file_var, output_dir_var, pokemon_number_start_var, pokemon_number_end_var, folder_var, file_name_var)
def setup_filter_ui(frame3, input_file_var, output_dir_var, pokemon_number_start_var, pokemon_number_end_var):
def setup_filter_ui(frame3, input_file_var, output_dir_var, pokemon_number_start_var, pokemon_number_end_var, folder_var, file_name_var):
global process_button
# Label and entry for specifying the range of Pokémon numbers to include
ttk.Label(frame3, text="Pokédex Number Range:").grid(column=0, row=0, columnspan=5, pady=5)
@@ -196,6 +212,8 @@ def setup_filter_ui(frame3, input_file_var, output_dir_var, pokemon_number_start
'included_groups': {name: tk.IntVar(value=1) for name in groups},
'known_contexts': {name: tk.IntVar(value=1) for name in contexts},
'bucket_mapping': {name: tk.IntVar(value=1) for name in buckets},
'folder': {name: tk.IntVar(value=1) for name in folders},
'file_name': {name: tk.IntVar(value=1) for name in file_names},
}
# Custom styles for toggle buttons
@@ -208,6 +226,8 @@ def setup_filter_ui(frame3, input_file_var, output_dir_var, pokemon_number_start
('included_groups', "Groups"),
('known_contexts', "Contexts"),
('bucket_mapping', "Buckets"),
('folder', "Folders"),
('file_name', "File Names"),
]
current_row = 2