diff options
Diffstat (limited to 'methods.py')
| -rw-r--r-- | methods.py | 403 |
1 files changed, 349 insertions, 54 deletions
diff --git a/methods.py b/methods.py index 805ae256c3..725fb36caa 100644 --- a/methods.py +++ b/methods.py @@ -2,6 +2,16 @@ import os import re import glob import subprocess +from collections import OrderedDict + +# We need to define our own `Action` method to control the verbosity of output +# and whenever we need to run those commands in a subprocess on some platforms. +from SCons import Node +from SCons.Script import Action +from SCons.Script import ARGUMENTS +from SCons.Script import Glob +from SCons.Variables.BoolVariable import _text2bool +from platform_methods import run_in_subprocess def add_source_files(self, sources, files, warn_duplicates=True): @@ -91,7 +101,7 @@ def update_version(module_version_string=""): gitfolder = module_folder[8:] if os.path.isfile(os.path.join(gitfolder, "HEAD")): - head = open(os.path.join(gitfolder, "HEAD"), "r").readline().strip() + head = open(os.path.join(gitfolder, "HEAD"), "r", encoding="utf8").readline().strip() if head.startswith("ref: "): head = os.path.join(gitfolder, head[5:]) if os.path.isfile(head): @@ -137,39 +147,112 @@ def parse_cg_file(fname, uniforms, sizes, conditionals): fs.close() -def detect_modules(): +def get_cmdline_bool(option, default): + """We use `ARGUMENTS.get()` to check if options were manually overridden on the command line, + and SCons' _text2bool helper to convert them to booleans, otherwise they're handled as strings. + """ + cmdline_val = ARGUMENTS.get(option) + if cmdline_val is not None: + return _text2bool(cmdline_val) + else: + return default + + +def detect_modules(search_path, recursive=False): + """Detects and collects a list of C++ modules at specified path + + `search_path` - a directory path containing modules. The path may point to + a single module, which may have other nested modules. A module must have + "register_types.h", "SCsub", "config.py" files created to be detected. + + `recursive` - if `True`, then all subdirectories are searched for modules as + specified by the `search_path`, otherwise collects all modules under the + `search_path` directory. If the `search_path` is a module, it is collected + in all cases. + + Returns an `OrderedDict` with module names as keys, and directory paths as + values. If a path is relative, then it is a built-in module. If a path is + absolute, then it is a custom module collected outside of the engine source. + """ + modules = OrderedDict() + + def add_module(path): + module_name = os.path.basename(path) + module_path = path.replace("\\", "/") # win32 + modules[module_name] = module_path + + def is_engine(path): + # Prevent recursively detecting modules in self and other + # Godot sources when using `custom_modules` build option. + version_path = os.path.join(path, "version.py") + if os.path.exists(version_path): + with open(version_path) as f: + if 'short_name = "godot"' in f.read(): + return True + return False + + def get_files(path): + files = glob.glob(os.path.join(path, "*")) + # Sort so that `register_module_types` does not change that often, + # and plugins are registered in alphabetic order as well. + files.sort() + return files + + if not recursive: + if is_module(search_path): + add_module(search_path) + for path in get_files(search_path): + if is_engine(path): + continue + if is_module(path): + add_module(path) + else: + to_search = [search_path] + while to_search: + path = to_search.pop() + if is_module(path): + add_module(path) + for child in get_files(path): + if not os.path.isdir(child): + continue + if is_engine(child): + continue + to_search.insert(0, child) + return modules + + +def is_module(path): + if not os.path.isdir(path): + return False + must_exist = ["register_types.h", "SCsub", "config.py"] + for f in must_exist: + if not os.path.exists(os.path.join(path, f)): + return False + return True + - module_list = [] +def write_modules(modules): includes_cpp = "" + preregister_cpp = "" register_cpp = "" unregister_cpp = "" - preregister_cpp = "" - files = glob.glob("modules/*") - files.sort() # so register_module_types does not change that often, and also plugins are registered in alphabetic order - for x in files: - if not os.path.isdir(x): - continue - if not os.path.exists(x + "/config.py"): - continue - x = x.replace("modules/", "") # rest of world - x = x.replace("modules\\", "") # win32 - module_list.append(x) + for name, path in modules.items(): try: - with open("modules/" + x + "/register_types.h"): - includes_cpp += '#include "modules/' + x + '/register_types.h"\n' - register_cpp += "#ifdef MODULE_" + x.upper() + "_ENABLED\n" - register_cpp += "\tregister_" + x + "_types();\n" - register_cpp += "#endif\n" - preregister_cpp += "#ifdef MODULE_" + x.upper() + "_ENABLED\n" - preregister_cpp += "#ifdef MODULE_" + x.upper() + "_HAS_PREREGISTER\n" - preregister_cpp += "\tpreregister_" + x + "_types();\n" + with open(os.path.join(path, "register_types.h")): + includes_cpp += '#include "' + path + '/register_types.h"\n' + preregister_cpp += "#ifdef MODULE_" + name.upper() + "_ENABLED\n" + preregister_cpp += "#ifdef MODULE_" + name.upper() + "_HAS_PREREGISTER\n" + preregister_cpp += "\tpreregister_" + name + "_types();\n" preregister_cpp += "#endif\n" preregister_cpp += "#endif\n" - unregister_cpp += "#ifdef MODULE_" + x.upper() + "_ENABLED\n" - unregister_cpp += "\tunregister_" + x + "_types();\n" + register_cpp += "#ifdef MODULE_" + name.upper() + "_ENABLED\n" + register_cpp += "\tregister_" + name + "_types();\n" + register_cpp += "#endif\n" + unregister_cpp += "#ifdef MODULE_" + name.upper() + "_ENABLED\n" + unregister_cpp += "\tunregister_" + name + "_types();\n" unregister_cpp += "#endif\n" - except IOError: + except OSError: pass modules_cpp = """// register_module_types.gen.cpp @@ -202,13 +285,47 @@ void unregister_module_types() { with open("modules/register_module_types.gen.cpp", "w") as f: f.write(modules_cpp) - return module_list + +def convert_custom_modules_path(path): + if not path: + return path + path = os.path.realpath(os.path.expanduser(os.path.expandvars(path))) + err_msg = "Build option 'custom_modules' must %s" + if not os.path.isdir(path): + raise ValueError(err_msg % "point to an existing directory.") + if path == os.path.realpath("modules"): + raise ValueError(err_msg % "be a directory other than built-in `modules` directory.") + return path def disable_module(self): self.disabled_modules.append(self.current_module) +def module_check_dependencies(self, module, dependencies): + """ + Checks if module dependencies are enabled for a given module, + and prints a warning if they aren't. + Meant to be used in module `can_build` methods. + Returns a boolean (True if dependencies are satisfied). + """ + missing_deps = [] + for dep in dependencies: + opt = "module_{}_enabled".format(dep) + if not opt in self or not self[opt]: + missing_deps.append(dep) + + if missing_deps != []: + print( + "Disabling '{}' module as the following dependencies are not satisfied: {}".format( + module, ", ".join(missing_deps) + ) + ) + return False + else: + return True + + def use_windows_spawn_fix(self, platform=None): if os.name != "nt": @@ -217,7 +334,7 @@ def use_windows_spawn_fix(self, platform=None): # On Windows, due to the limited command line length, when creating a static library # from a very high number of objects SCons will invoke "ar" once per object file; # that makes object files with same names to be overwritten so the last wins and - # the library looses symbols defined by overwritten objects. + # the library loses symbols defined by overwritten objects. # By enabling quick append instead of the default mode (replacing), libraries will # got built correctly regardless the invocation strategy. # Furthermore, since SCons will rebuild the library from scratch when an object file @@ -361,7 +478,7 @@ def detect_visual_c_compiler_version(tools_env): # and not scons setup environment (env)... so make sure you call the right environment on it or it will fail to detect # the proper vc version that will be called - # There is no flag to give to visual c compilers to set the architecture, ie scons bits argument (32,64,ARM etc) + # There is no flag to give to visual c compilers to set the architecture, i.e. scons bits argument (32,64,ARM etc) # There are many different cl.exe files that are run, and each one compiles & links to a different architecture # As far as I know, the only way to figure out what compiler will be run when Scons calls cl.exe via Program() # is to check the PATH variable and figure out which one will be called first. Code below does that and returns: @@ -470,10 +587,39 @@ def generate_cpp_hint_file(filename): try: with open(filename, "w") as fd: fd.write("#define GDCLASS(m_class, m_inherits)\n") - except IOError: + except OSError: print("Could not write cpp.hint file.") +def glob_recursive(pattern, node="."): + results = [] + for f in Glob(str(node) + "/*", source=True): + if type(f) is Node.FS.Dir: + results += glob_recursive(pattern, f) + results += Glob(str(node) + "/" + pattern, source=True) + return results + + +def add_to_vs_project(env, sources): + for x in sources: + if type(x) == type(""): + fname = env.File(x).path + else: + fname = env.File(x)[0].path + pieces = fname.split(".") + if len(pieces) > 0: + basename = pieces[0] + basename = basename.replace("\\\\", "/") + if os.path.isfile(basename + ".h"): + env.vs_incs += [basename + ".h"] + elif os.path.isfile(basename + ".hpp"): + env.vs_incs += [basename + ".hpp"] + if os.path.isfile(basename + ".c"): + env.vs_srcs += [basename + ".c"] + elif os.path.isfile(basename + ".cpp"): + env.vs_srcs += [basename + ".cpp"] + + def generate_vs_project(env, num_jobs): batch_file = find_visual_c_batch_file(env) if batch_file: @@ -482,37 +628,44 @@ def generate_vs_project(env, num_jobs): common_build_prefix = [ 'cmd /V /C set "plat=$(PlatformTarget)"', '(if "$(PlatformTarget)"=="x64" (set "plat=x86_amd64"))', - 'set "tools=yes"', + 'set "tools=%s"' % env["tools"], '(if "$(Configuration)"=="release" (set "tools=no"))', 'call "' + batch_file + '" !plat!', ] - result = " ^& ".join(common_build_prefix + [commands]) + # Windows allows us to have spaces in paths, so we need + # to double quote off the directory. However, the path ends + # in a backslash, so we need to remove this, lest it escape the + # last double quote off, confusing MSBuild + common_build_postfix = [ + "--directory=\"$(ProjectDir.TrimEnd('\\'))\"", + "platform=windows", + "target=$(Configuration)", + "progress=no", + "tools=!tools!", + "-j%s" % num_jobs, + ] + + if env["custom_modules"]: + common_build_postfix.append("custom_modules=%s" % env["custom_modules"]) + + result = " ^& ".join(common_build_prefix + [" ".join([commands] + common_build_postfix)]) return result - env.AddToVSProject(env.core_sources) - env.AddToVSProject(env.main_sources) - env.AddToVSProject(env.modules_sources) - env.AddToVSProject(env.scene_sources) - env.AddToVSProject(env.servers_sources) - env.AddToVSProject(env.editor_sources) - - # windows allows us to have spaces in paths, so we need - # to double quote off the directory. However, the path ends - # in a backslash, so we need to remove this, lest it escape the - # last double quote off, confusing MSBuild - env["MSVSBUILDCOM"] = build_commandline( - "scons --directory=\"$(ProjectDir.TrimEnd('\\'))\" platform=windows progress=no target=$(Configuration) tools=!tools! -j" - + str(num_jobs) - ) - env["MSVSREBUILDCOM"] = build_commandline( - "scons --directory=\"$(ProjectDir.TrimEnd('\\'))\" platform=windows progress=no target=$(Configuration) tools=!tools! vsproj=yes -j" - + str(num_jobs) - ) - env["MSVSCLEANCOM"] = build_commandline( - "scons --directory=\"$(ProjectDir.TrimEnd('\\'))\" --clean platform=windows progress=no target=$(Configuration) tools=!tools! -j" - + str(num_jobs) - ) + add_to_vs_project(env, env.core_sources) + add_to_vs_project(env, env.drivers_sources) + add_to_vs_project(env, env.main_sources) + add_to_vs_project(env, env.modules_sources) + add_to_vs_project(env, env.scene_sources) + add_to_vs_project(env, env.servers_sources) + add_to_vs_project(env, env.editor_sources) + + for header in glob_recursive("**/*.h"): + env.vs_incs.append(str(header)) + + env["MSVSBUILDCOM"] = build_commandline("scons") + env["MSVSREBUILDCOM"] = build_commandline("scons vsproj=yes") + env["MSVSCLEANCOM"] = build_commandline("scons --clean") # This version information (Win32, x64, Debug, Release, Release_Debug seems to be # required for Visual Studio to understand that it needs to generate an NMAKE @@ -571,6 +724,14 @@ def CommandNoCache(env, target, sources, command, **args): return result +def Run(env, function, short_message, subprocess=True): + output_print = short_message if not env["verbose"] else "" + if not subprocess: + return Action(function, output_print) + else: + return Action(run_in_subprocess(function), output_print) + + def detect_darwin_sdk_path(platform, env): sdk_name = "" if platform == "osx": @@ -634,3 +795,137 @@ def using_gcc(env): def using_clang(env): return "clang" in os.path.basename(env["CC"]) + + +def show_progress(env): + import sys + from SCons.Script import Progress, Command, AlwaysBuild + + screen = sys.stdout + # Progress reporting is not available in non-TTY environments since it + # messes with the output (for example, when writing to a file) + show_progress = env["progress"] and sys.stdout.isatty() + node_count = 0 + node_count_max = 0 + node_count_interval = 1 + node_count_fname = str(env.Dir("#")) + "/.scons_node_count" + + import time, math + + class cache_progress: + # The default is 1 GB cache and 12 hours half life + def __init__(self, path=None, limit=1073741824, half_life=43200): + self.path = path + self.limit = limit + self.exponent_scale = math.log(2) / half_life + if env["verbose"] and path != None: + screen.write( + "Current cache limit is {} (used: {})\n".format( + self.convert_size(limit), self.convert_size(self.get_size(path)) + ) + ) + self.delete(self.file_list()) + + def __call__(self, node, *args, **kw): + nonlocal node_count, node_count_max, node_count_interval, node_count_fname, show_progress + if show_progress: + # Print the progress percentage + node_count += node_count_interval + if node_count_max > 0 and node_count <= node_count_max: + screen.write("\r[%3d%%] " % (node_count * 100 / node_count_max)) + screen.flush() + elif node_count_max > 0 and node_count > node_count_max: + screen.write("\r[100%] ") + screen.flush() + else: + screen.write("\r[Initial build] ") + screen.flush() + + def delete(self, files): + if len(files) == 0: + return + if env["verbose"]: + # Utter something + screen.write("\rPurging %d %s from cache...\n" % (len(files), len(files) > 1 and "files" or "file")) + [os.remove(f) for f in files] + + def file_list(self): + if self.path is None: + # Nothing to do + return [] + # Gather a list of (filename, (size, atime)) within the + # cache directory + file_stat = [(x, os.stat(x)[6:8]) for x in glob.glob(os.path.join(self.path, "*", "*"))] + if file_stat == []: + # Nothing to do + return [] + # Weight the cache files by size (assumed to be roughly + # proportional to the recompilation time) times an exponential + # decay since the ctime, and return a list with the entries + # (filename, size, weight). + current_time = time.time() + file_stat = [(x[0], x[1][0], (current_time - x[1][1])) for x in file_stat] + # Sort by the most recently accessed files (most sensible to keep) first + file_stat.sort(key=lambda x: x[2]) + # Search for the first entry where the storage limit is + # reached + sum, mark = 0, None + for i, x in enumerate(file_stat): + sum += x[1] + if sum > self.limit: + mark = i + break + if mark is None: + return [] + else: + return [x[0] for x in file_stat[mark:]] + + def convert_size(self, size_bytes): + if size_bytes == 0: + return "0 bytes" + size_name = ("bytes", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB") + i = int(math.floor(math.log(size_bytes, 1024))) + p = math.pow(1024, i) + s = round(size_bytes / p, 2) + return "%s %s" % (int(s) if i == 0 else s, size_name[i]) + + def get_size(self, start_path="."): + total_size = 0 + for dirpath, dirnames, filenames in os.walk(start_path): + for f in filenames: + fp = os.path.join(dirpath, f) + total_size += os.path.getsize(fp) + return total_size + + def progress_finish(target, source, env): + nonlocal node_count, progressor + with open(node_count_fname, "w") as f: + f.write("%d\n" % node_count) + progressor.delete(progressor.file_list()) + + try: + with open(node_count_fname) as f: + node_count_max = int(f.readline()) + except Exception: + pass + + cache_directory = os.environ.get("SCONS_CACHE") + # Simple cache pruning, attached to SCons' progress callback. Trim the + # cache directory to a size not larger than cache_limit. + cache_limit = float(os.getenv("SCONS_CACHE_LIMIT", 1024)) * 1024 * 1024 + progressor = cache_progress(cache_directory, cache_limit) + Progress(progressor, interval=node_count_interval) + + progress_finish_command = Command("progress_finish", [], progress_finish) + AlwaysBuild(progress_finish_command) + + +def dump(env): + # Dumps latest build information for debugging purposes and external tools. + from json import dump + + def non_serializable(obj): + return "<<non-serializable: %s>>" % (type(obj).__qualname__) + + with open(".scons_env.json", "w") as f: + dump(env.Dictionary(), f, indent=4, default=non_serializable) |