How to scan for all files in Folders and Subfolders outside the Godot Project Directory? (Godot 3.0.4)

:information_source: Attention Topic was automatically imported from the old Question2Answer platform.
:bust_in_silhouette: Asked By Corruptinator

Warning, this is a very advanced question to ask and the reason for the question is mostly because I was utilizing Godot to develop a useful software app that makes use of the PCKPacker Class to pack files into a custom PCK for DLC and Patching purposes, and I’ve been running into a lot of problems:

First of all, the PCKPacker is mostly used in this code method (A.K.A. Autoload Script) to pack files into a PCK:

(Apologies for long explanation, trying to be clear and in detail in what I ran into):

extends Node

var pck_queue_list = []

func _ready():
	pck_queue_list.clear()
	pass

func test():
	print("Test!")

func _pck_Packager(dir_path,file_path):
	_pre_Pack()
	_pck_Queue(dir_path,file_path)
	_pck_Pack()
	pass

func _pre_Pack():
	pck_queue_list.clear()
	pass

func _pck_Queue(dir_path,file_path):

	var packaging = PCKPacker.new() #Starts up the PCKPacker
	
	#Initilizes the packer. Filepath is where you save the PCK somewhere on a computer using a system directory
	packaging.pck_start(file_path, 0)
	
	
	#During the process the add_file works like this:
	#add_file(System_Directory_Location,Imported_File).
	#This is where the Dir_Path holds the string data
	#for the location of the file to be compressed into PCK.
	
	#Example [C://folder/location/to/file/Sprite.png]
	#I need to separate the Dir_Path so the folders
	#can be placed in the System_Directory_Location
	#while the sprite.png is located in the Imported_File.
	
	#Insert Pseudo-code to split file and folder path from each other:
	
	var folder_Path = C://folder/location/to/file/
	var file = Sprite.png
	
	
	for i in range(0,dir_path.size()): 
		packaging.add_file(folder_path,file)
	
	packaging.flush(false)
	pass

func _pck_Pack():
	_post_Pack()
	pass

func _post_Pack():
	#Pseudo-code to restore/clean up the scene/game.
	pass

So that is how PCKPacker should work… I guess?

Second part is where I ran into infinite headaches. I was using the Directory Class to fetch the files needed to compress into PCK, based on this answered Q&A page: https://forum.godotengine.org/5175

func _on_FileDialog_dir_selected(dir):
	var files = []
	var directories = Directory.new()
	directories.open(dir)
	directories.list_dir_begin()

	while true:
		var file = directories.get_next()
		print(file)
		if file == "":
			break
		elif not file.begins_with("."):
			files.append(file)

	directories.list_dir_end()

This is where I hit the dead end, using this code depending if I fetched a folder from a directory, I would get these results:

C://folder/resources/sprites     <- Folder
C://folder/resources/icon.png     <- File
C://folder/resources/music.ogg     <-File

The filepaths are saved, but it does not scan through all the subfolders such as the Sprites Folder. What I wanted to aim for was this:

C://folder/resources/sprites/Sprite1.png     <- File in folder
C://folder/resources/sprites/Sprite2.png     <- File in folder
C://folder/resources/icon.png     <- File
C://folder/resources/music.ogg     <-File

So I tried to redo and or recode the entire subfolder scan but its barely functional:

# Main Subfolder Scanner  (DEFINITELY WORK IN PROGRESS UNLESS I EITHER REDO CODE OR:
# ANY SUGGESTIONS OR REMEDIES ARE WELCOME.)

func _on_FileDialog_dir_selected(dir):
	var files = []
	var sub
	var directories = Directory.new()
	if directories.open(dir) == OK:
		directories.list_dir_begin()
		var file_name = directories.get_next()
		
		while (file_name != ""):
		
			#checks if the file is an actual file or folder, if folder it goes into the next folder using
			#the _folder_access function
			
			if directories.dir_exists(file_name) == true and file_name.begins_with(".") == true:
				pass
			elif directories.dir_exists(file_name) == true and file_name.begins_with(".") == false:
				#print("Found directory: " + file_name)
				sub = file_name
				_folder_access(files,file_name,directories,dir,sub) #To continue searching through Sub-Folders
			else:
				pass
			
			if file_name.begins_with(".") == false:
				print("Found file: " + dir+"/"+file_name) # This is where the loop stops after using the _folder_access scan...
				files.append(dir+"/"+file_name)
			elif file_name.begins_with(".") == true:
				pass
			
			file_name = directories.get_next()
	else:
		print("An error occurred when trying to access the path.")


#Theoretical Function to perform while loops to enter other subfolders to find more hidden files:
		
func _folder_access(files,file_name,directories,dir,sub):
	directories.change_dir(file_name)
	while(file_name != ""):
	
	
		#checks if the file is an actual file or folder, if folder it goes into the next folder using
		#the _folder_access function, just like the main function:
	
		if directories.dir_exists(sub+"/"+file_name) == true and file_name.begins_with(".") == true:
			pass
		elif directories.dir_exists(sub+"/"+file_name) == true and file_name.begins_with(".") == false:
			sub = sub+"/"+file_name
			_folder_access(files,file_name,directories,dir,sub)
			#print("Found directory: " + file_name)
		
		if file_name.begins_with(".") == false:
			print("Found file: " + dir+"/"+file_name)
			files.append(dir+"/"+file_name)
		elif file_name.begins_with(".") == true:
			pass
		file_name = directories.get_next()
	return

I was hoping it would get me this result:

C://folder/resources/sprites/Sprite1.png     <- File in folder
C://folder/resources/sprites/Sprite2.png     <- File in folder
C://folder/resources/icon.png     <- File
C://folder/resources/music.ogg     <-File

But sadly, I’m still barely making progress.

At this point, I’m open for any idea or suggestions, but one thing is for certain is that I wanted to make a function to scan through all subfolders, hunting down all remaining files to be encoded into a PCK, and then save the PCK.

Feedback, help, and or comments are welcome! Thanks!

:bust_in_silhouette: Reply From: Ratty

Bit late but in case anyone else lands here (I did while writing this).
The code below traverses the given directory`s tree and returns an array containing the path to all files (not dirs, although you could add those):

func get_filelist(scan_dir : String) -> Array:
var my_files : Array = []
var dir := Directory.new()
if dir.open(scan_dir) != OK:
	printerr("Warning: could not open directory: ", scan_dir)
	return []

if dir.list_dir_begin(true, true) != OK:
	printerr("Warning: could not list contents of: ", scan_dir)
	return []

var file_name := dir.get_next()
while file_name != "":
	if dir.current_is_dir():
		my_files += get_filelist(dir.get_current_dir() + "/" + file_name)
	else:
		my_files.append(dir.get_current_dir() + "/" + file_name)
	
	file_name = dir.get_next()

return my_files

Better late than never

Adam_S | 2019-09-12 18:34

I was looking for this to find all files with specific extension/s and worked great!
Here’s my tweaked version to add a filter for any file extension/s you want.

func get_filelist(scan_dir : String, filter_exts : Array = []) -> Array:
var my_files : Array = []
var dir := Directory.new()
if dir.open(scan_dir) != OK:
	printerr("Warning: could not open directory: ", scan_dir)
	return []

if dir.list_dir_begin(true, true) != OK:
	printerr("Warning: could not list contents of: ", scan_dir)
	return []

var file_name := dir.get_next()
while file_name != "":
	if dir.current_is_dir():
		my_files += get_filelist(dir.get_current_dir() + "/" + file_name, filter_exts)
	else:
		if filter_exts.size() == 0:
			my_files.append(dir.get_current_dir() + "/" + file_name)
		else:
			for ext in filter_exts:
				if file_name.get_extension() == ext:
					my_files.append(dir.get_current_dir() + "/" + file_name)
	file_name = dir.get_next()
return my_files

Usage for (example) finding all .txt files in Desktop:

var files= get_filelist("/Users/user/Desktop", ["txt"]) # Extension *without* the dot

Hope this helps some wanderer.

Gianclgar | 2022-09-29 15:04