#!/usr/bin/env python3 # # This file is part of the MicroPython project, http://micropython.org/ # # The MIT License (MIT) # # Copyright (c) 2019 Damien P. George # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in # all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. from __future__ import print_function import sys import os import subprocess # Always use the mpy-cross from this repo. sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../mpy-cross")) import mpy_cross import manifestfile VARS = {} class FreezeError(Exception): pass def system(cmd): try: output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) return 0, output except subprocess.CalledProcessError as er: return -1, er.output def get_timestamp(path, default=None): try: stat = os.stat(path) return stat.st_mtime except OSError: if default is None: raise FreezeError("cannot stat {}".format(path)) return default def mkdir(filename): path = os.path.dirname(filename) if not os.path.isdir(path): os.makedirs(path) # Formerly make-frozen.py. # This generates: # - MP_FROZEN_STR_NAMES macro # - mp_frozen_str_sizes # - mp_frozen_str_content def generate_frozen_str_content(modules): output = [ b"#include \n", b"#define MP_FROZEN_STR_NAMES \\\n", ] for _, target_path in modules: print("STR", target_path) output.append(b'"%s\\0" \\\n' % target_path.encode()) output.append(b"\n") output.append(b"const uint32_t mp_frozen_str_sizes[] = { ") for full_path, _ in modules: st = os.stat(full_path) output.append(b"%d, " % st.st_size) output.append(b"0 };\n") output.append(b"const char mp_frozen_str_content[] = {\n") for full_path, _ in modules: with open(full_path, "rb") as f: data = f.read() # We need to properly escape the script data to create a C string. # When C parses hex characters of the form \x00 it keeps parsing the hex # data until it encounters a non-hex character. Thus one must create # strings of the form "data\x01" "abc" to properly encode this kind of # data. We could just encode all characters as hex digits but it's nice # to be able to read the resulting C code as ASCII when possible. data = bytearray(data) # so Python2 extracts each byte as an integer esc_dict = {ord("\n"): b"\\n", ord("\r"): b"\\r", ord('"'): b'\\"', ord("\\"): b"\\\\"} output.append(b'"') break_str = False for c in data: try: output.append(esc_dict[c]) except KeyError: if 32 <= c <= 126: if break_str: output.append(b'" "') break_str = False output.append(chr(c).encode()) else: output.append(b"\\x%02x" % c) break_str = True output.append(b'\\0"\n') output.append(b'"\\0"\n};\n\n') return b"".join(output) def main(): # Parse arguments import argparse cmd_parser = argparse.ArgumentParser( description="A tool to generate frozen content in MicroPython firmware images." ) cmd_parser.add_argument("-o", "--output", help="output path") cmd_parser.add_argument("-b", "--build-dir", help="output path") cmd_parser.add_argument( "-f", "--mpy-cross-flags", default="", help="flags to pass to mpy-cross" ) cmd_parser.add_argument("-v", "--var", action="append", help="variables to substitute") cmd_parser.add_argument("--mpy-tool-flags", default="", help="flags to pass to mpy-tool") cmd_parser.add_argument("files", nargs="+", help="input manifest list") args = cmd_parser.parse_args() # Extract variables for substitution. for var in args.var: name, value = var.split("=", 1) if os.path.exists(value): value = os.path.abspath(value) VARS[name] = value if "MPY_DIR" not in VARS or "PORT_DIR" not in VARS: print("MPY_DIR and PORT_DIR variables must be specified") sys.exit(1) # Get paths to tools MPY_CROSS = VARS["MPY_DIR"] + "/mpy-cross/build/mpy-cross" if sys.platform == "win32": MPY_CROSS += ".exe" MPY_CROSS = os.getenv("MICROPY_MPYCROSS", MPY_CROSS) MPY_TOOL = VARS["MPY_DIR"] + "/tools/mpy-tool.py" # Ensure mpy-cross is built if not os.path.exists(MPY_CROSS): print("mpy-cross not found at {}, please build it first".format(MPY_CROSS)) sys.exit(1) manifest = manifestfile.ManifestFile(manifestfile.MODE_FREEZE, VARS) # Include top-level inputs, to generate the manifest for input_manifest in args.files: try: manifest.execute(input_manifest) except manifestfile.ManifestFileError as er: print('freeze error executing "{}": {}'.format(input_manifest, er.args[0])) sys.exit(1) # Process the manifest str_paths = [] mpy_files = [] ts_newest = 0 for result in manifest.files(): if result.kind == manifestfile.KIND_FREEZE_AS_STR: str_paths.append( ( result.full_path, result.target_path, ) ) ts_outfile = result.timestamp elif result.kind == manifestfile.KIND_FREEZE_AS_MPY: outfile = "{}/frozen_mpy/{}.mpy".format(args.build_dir, result.target_path[:-3]) ts_outfile = get_timestamp(outfile, 0) if result.timestamp >= ts_outfile: print("MPY", result.target_path) mkdir(outfile) # Add __version__ to the end of the file before compiling. with manifestfile.tagged_py_file(result.full_path, result.metadata) as tagged_path: try: mpy_cross.compile( tagged_path, dest=outfile, src_path=result.target_path, opt=result.opt, mpy_cross=MPY_CROSS, extra_args=args.mpy_cross_flags.split(), ) except mpy_cross.CrossCompileError as ex: print("error compiling {}:".format(result.target_path)) print(ex.args[0]) raise SystemExit(1) ts_outfile = get_timestamp(outfile) mpy_files.append(outfile) else: assert result.kind == manifestfile.KIND_FREEZE_MPY mpy_files.append(result.full_path) ts_outfile = result.timestamp ts_newest = max(ts_newest, ts_outfile) # Check if output file needs generating if ts_newest < get_timestamp(args.output, 0): # No files are newer than output file so it does not need updating return # Freeze paths as strings output_str = generate_frozen_str_content(str_paths) # Freeze .mpy files if mpy_files: res, output_mpy = system( [ sys.executable, MPY_TOOL, "-f", "-q", args.build_dir + "/genhdr/qstrdefs.preprocessed.h", ] + args.mpy_tool_flags.split() + mpy_files ) if res != 0: print("error freezing mpy {}:".format(mpy_files)) print(output_mpy.decode()) sys.exit(1) else: output_mpy = ( b'#include "py/emitglue.h"\n' b"extern const qstr_pool_t mp_qstr_const_pool;\n" b"const qstr_pool_t mp_qstr_frozen_const_pool = {\n" b" #if MICROPY_QSTR_BYTES_IN_HASH\n" b" (qstr_pool_t*)&mp_qstr_const_pool, MP_QSTRnumber_of, 0, 0, 0, NULL, NULL, {},\n" b" #else\n" b" (qstr_pool_t*)&mp_qstr_const_pool, MP_QSTRnumber_of, 0, 0, 0, NULL, {},\n" b" #endif\n" b"};\n" b'const char mp_frozen_names[] = { MP_FROZEN_STR_NAMES "\\0"};\n' b"const mp_raw_code_t *const mp_frozen_mpy_content[] = {NULL};\n" ) # Generate output print("GEN", args.output) mkdir(args.output) with open(args.output, "wb") as f: f.write(b"//\n// Content for MICROPY_MODULE_FROZEN_STR\n//\n") f.write(output_str) f.write(b"//\n// Content for MICROPY_MODULE_FROZEN_MPY\n//\n") f.write(output_mpy) if __name__ == "__main__": main()