#!/usr/bin/python # -*- coding: UTF-8 -*- import os import json import sys # sys.argv import getopt # getopt import hashlib # md5 sha1 import configparser # ini (python 3.0 use configparser) import fcntl # file lock import datetime # format file modify time import shutil # which import subprocess gDefaultPath = "../execution/glsl" gOutputHeadFile = "../shaders/AllShader.h" gOutputSourceFile = "AllShader.cpp" gGeneratedShaderDir = os.path.join(".cache", "shader", "generated") gEnableSpirvOpt = True gSpirvOptArgs = ["-O"] FP32_HEADER = """#version 450 core layout(std430) buffer; #define FLOAT float #define FLOAT2 vec2 #define FLOAT4 vec4 """ FP16_HEADER = """#version 450 core layout(std430) buffer; #extension GL_EXT_shader_explicit_arithmetic_types_float16 : require #extension GL_EXT_shader_16bit_storage : require #define FLOAT float16_t #define FLOAT2 f16vec2 #define FLOAT4 f16vec4 """ def _ensure_dir(path): if path and not os.path.exists(path): os.makedirs(path, exist_ok=True) def _run_cmd(args, stdout_path=None): if stdout_path is not None: with open(stdout_path, "w") as f: proc = subprocess.run(args, stdout=f, stderr=subprocess.STDOUT, text=True) return proc.returncode proc = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) if proc.stdout: print(proc.stdout) return proc.returncode def _spirv_opt_available(): return shutil.which("spirv-opt") is not None def _spirv_opt_tag(): if not gEnableSpirvOpt: return "" if os.environ.get("MNN_VULKAN_DISABLE_SPIRV_OPT", "") == "1": return "" if not _spirv_opt_available(): return "" return " ".join(gSpirvOptArgs) def _build_generated_shader_path(raw, macro, precision): fileName = os.path.basename(raw) if macro: fileName = fileName.replace(".comp", "_" + macro + ".comp") if precision == 'fp16': fileName = fileName.replace(".comp", "_FP16" + ".comp") return os.path.join(gGeneratedShaderDir, fileName) def findAllShader(path): output = [] extensions = [".comp", ".frag", ".vert"] for root, dirs, files in os.walk(path): for file in files: if any(file.endswith(ext) for ext in extensions): output.append(os.path.join(root, file)) return output def getName(fileName): s1 = fileName.replace("/", "_") s1 = s1.replace(".", "_") s1=s1.replace("__","_") return s1 def generateFileAsm(headfile, sourcefile, asmdirs): output = [] for root, dirs, files in os.walk(asmdirs): for file in files: if file.endswith(".spirv"): output.append(os.path.join(root, file)) h = "#ifndef SPRIV_SHADER_AUTO_GENERATE_H\n#define SPRIV_SHADER_AUTO_GENERATE_H\n" cpp = "#include \"" + headfile + "\"\n" for s in output: name = getName(s) print(name) subprocess.run(["spirv-as", s, "-o", "tempspv", "--target-env", "vulkan1.0"]) h += "extern const unsigned char " + name + "[];\n" h += 'extern unsigned int ' + name + '_len;\n' with open("temp.spv.cpp", "w") as tmp_cpp: subprocess.run(["xxd", "-i", "tempspv"], stdout=tmp_cpp) with open('temp.spv.cpp') as f: allContent = f.read().replace('tempspv', name) cpp += 'const ' + allContent + '\n' h += "#endif" with open(headfile, "w") as f: f.write(h) with open(sourcefile, "w") as f: f.write(cpp) if os.path.exists('temp.spv.cpp'): os.remove('temp.spv.cpp') if os.path.exists('tempspv'): os.remove('tempspv') class ShaderFile: def __init__(self, shader='', ref=False, raw='', precision='fp32'): self.shaderFile = shader # string , source .comp file self.refFile = ref # (Deprecated, always True) bool , reference file , org file not exists ,auto generated by script self.rawRefFile = raw # string , if isRefFile this value point out raw file name else empty self.spirvFile = '' # string, spir-v file self.spirvCacheFile = '' # string, spir-v cache file self.appendMacro = '' # string, append macro content if isRefFile is true self.precision = precision def getShaderFile(self): return self.shaderFile def getRawRefFile(self): if (None == self.rawRefFile): return '' return self.rawRefFile def getFileName(self): logicalName = self.rawRefFile if self.appendMacro: logicalName = logicalName.replace(".comp", "_" + self.appendMacro + ".comp") if self.precision == 'fp16': logicalName = logicalName.replace(".comp", "_FP16.comp") normalized = os.path.normpath(logicalName) parts = normalized.split(os.sep) if "glsl" in parts: parts = parts[parts.index("glsl"):] else: parts = [parts[-1]] s1 = "_".join(parts) s1 = s1.replace(".", "_").replace("-", "_") while "__" in s1: s1 = s1.replace("__", "_") return s1 def getPrecision(self): return self.precision def isRefFile(self): return self.refFile def setMacro(self, macro): self.appendMacro = macro def getMacro(self): return self.appendMacro def getSpirvFile(self): return self.spirvFile def setSpirvFile(self,f): self.spirvFile = f def getSpirvCacheFile(self): return self.spirvCacheFile def setSpirvCacheFile(self,f): self.spirvCacheFile = f class ShaderCache: # make dirs def __checkDir__(self,dir): if not os.path.exists(dir) : os.makedirs(dir) if not os.path.isdir(dir) : print('cache dir %s exists but it is a file not a directory, please check! \n' % dir) return False return True def __checkCacheDirs__(self): if not self.__checkDir__(self.root_dir): return False dir = os.path.join(self.root_dir, self.shader_dir) if not self.__checkDir__(dir): return False dir = os.path.join(self.root_dir, self.shader_dir,self.cache_dir) if not self.__checkDir__(dir): return False return True def __calcFileCheckInformation__(self,f): f = f.encode('utf-8') ck_size = os.path.getsize(f) ck_size = '%d' %ck_size ck_mtime = os.path.getmtime(f) ck_lastmodify = datetime.datetime.fromtimestamp(ck_mtime).strftime('%Y-%m-%d %H:%M:%S %f') ck_md5 = hashlib.md5(f) ck_sha1 = hashlib.sha1(f) return ck_size,ck_lastmodify,ck_md5.hexdigest(),ck_sha1.hexdigest() def __readShaderFileConfigInformation__(self,shader): sha1 = '' md5 = '' size = 0 lastmodify = '' spirv_opt = '' spirv_file = '' spirv_size = 0 spirv_sha1 = '' spirv_md5 = '' spirv_lastmodify = '' if self.configParser.has_section(shader) : sha1 = self.configParser.get(shader,'sha1') md5 = self.configParser.get(shader,'md5') size = self.configParser.getint(shader, 'size') lastmodify = self.configParser.get(shader, 'lastmodify') if self.configParser.has_option(shader, 'spirv_opt'): spirv_opt = self.configParser.get(shader, 'spirv_opt') spirv_file = self.configParser.get(shader, 'spirv_file') spirv_size = self.configParser.getint(shader, 'spirv_size') spirv_sha1 = self.configParser.get(shader, 'spirv_sha1') spirv_md5 = self.configParser.get(shader, 'spirv_md5') spirv_lastmodify = self.configParser.get(shader, 'spirv_lastmodify') return sha1, md5, size, lastmodify, spirv_opt, spirv_file, spirv_size, spirv_sha1, spirv_md5, spirv_lastmodify def __readConfigInformation__(self): self.cache_valid = False sh_section = self.scriptName if self.configParser.has_section(sh_section) : self.sha1 = self.configParser.get(sh_section,'sha1') self.md5 = self.configParser.get(sh_section,'md5') self.size = self.configParser.getint(sh_section, 'size') self.lastmodify = self.configParser.get(sh_section, 'lastmodify') # check information sc_size,lastmodify,sc_md5,sc_sha1 = self.__calcFileCheckInformation__(sh_section) if (sc_size == self.size) and (lastmodify == self.lastmodify) and (sc_md5 == self.md5) and (sc_sha1 == self.sha1) : self.cache_valid = True def __loadConfigFile__(self): if not self.__checkCacheDirs__() : return False if os.path.isfile(self.configFile): self.__readConfigInformation__() return True def initlizateShaderCache(self): if not self.__loadConfigFile__(): return False return True def __setupSpirvCacheFiles__(self,objs): for obj in objs: if obj.isRefFile(): shader = obj.getRawRefFile() cache_name = obj.getShaderFile() name, ext = os.path.splitext(os.path.basename(cache_name)) else : shader = obj.getShaderFile() name, ext = os.path.splitext(os.path.basename(shader)) cache_path = os.path.join(self.root_dir, self.shader_dir, self.cache_dir, name + '.spv') obj.setSpirvFile('') obj.setSpirvCacheFile(cache_path) def setupShaderCache(self,objs): if not self.use_cache : return # cache invalid ,we setup cache file path # if not self.cache_valid : # self.__setupSpirvCacheFiles__(objs) # return for obj in objs: shader = obj.getShaderFile() refFile = shader if obj.isRefFile(): cache_name = obj.getShaderFile() name, ext = os.path.splitext(os.path.basename(cache_name)) refFile = obj.getRawRefFile() else : name, ext = os.path.splitext(os.path.basename(shader)) cache_path = os.path.join(self.root_dir, self.shader_dir, self.cache_dir, name + '.spv') sha1, md5, size, lastmodify, spirv_opt, spirv_file, spirv_size, spirv_sha1, spirv_md5, spirv_lastmodify = self.__readShaderFileConfigInformation__(shader) if spirv_opt != self.spirv_opt_tag: obj.setSpirvFile('') obj.setSpirvCacheFile(cache_path) continue if len(sha1)<=0 or len(md5)<=0 or size<=0 or len(spirv_file) <= 0 or spirv_size <=0 or len(spirv_md5) <= 0 \ or len(spirv_lastmodify) <= 0 or (not os.path.isfile(refFile)) or (not os.path.isfile(spirv_file)) : obj.setSpirvFile('') obj.setSpirvCacheFile(cache_path) else : ck_size, ck_lastmodify, ck_md5, ck_sha1 = self.__calcFileCheckInformation__(refFile) sp_ck_size, sp_ck_lastmodify, sp_ck_md5, sp_ck_sha1 = self.__calcFileCheckInformation__(spirv_file) # print("Check begin") # print(sha1 == ck_sha1) # print(md5 == ck_md5) # print(size == ck_size) # print(lastmodify == ck_lastmodify) # print(sp_ck_sha1 == spirv_sha1) # print(sp_ck_size == spirv_size) # print(sp_ck_lastmodify == spirv_lastmodify) # print(sp_ck_md5 == spirv_md5) # print("Check End") if (sha1 == ck_sha1) and (md5 == ck_md5) and (lastmodify == ck_lastmodify) \ and (sp_ck_lastmodify == spirv_lastmodify) and (sp_ck_md5 == spirv_md5) \ and (sp_ck_sha1 == spirv_sha1) : obj.setSpirvFile(spirv_file) obj.setSpirvCacheFile('') # print("Cache Match: ", len(obj.getSpirvFile())) else : obj.setSpirvFile('') obj.setSpirvCacheFile(cache_path) def __writeShaderFileConfigInformation__(self,section,shader,spirv): if not self.configParser.has_section(section): self.configParser.add_section(section) ck_size, ck_lastmodify, ck_md5, ck_sha1 = self.__calcFileCheckInformation__(shader) self.configParser.set(section,'sha1',ck_sha1) self.configParser.set(section,'md5',ck_md5) self.configParser.set(section, 'size', ck_size) self.configParser.set(section, 'lastmodify', ck_lastmodify) self.configParser.set(section, 'spirv_opt', self.spirv_opt_tag) self.configParser.set(section, 'spirv_file', spirv) ck_size, ck_lastmodify, ck_md5, ck_sha1 = self.__calcFileCheckInformation__(spirv) self.configParser.set(section, 'spirv_size', ck_size) self.configParser.set(section, 'spirv_sha1', ck_sha1) self.configParser.set(section, 'spirv_md5', ck_md5) self.configParser.set(section, 'spirv_lastmodify', ck_lastmodify) def updateShaderCache(self,objs): if not self.use_cache : return write = False # update self sh_section = self.scriptName if not self.configParser.has_section(sh_section): self.configParser.add_section(sh_section) sc_size, lastmodify, sc_md5, sc_sha1 = self.__calcFileCheckInformation__(sh_section) if (sc_size != self.size) or (lastmodify != self.lastmodify) or (sc_md5 != self.md5) or (sc_sha1 != self.sha1) : self.configParser.set(sh_section, 'sha1', sc_sha1) self.configParser.set(sh_section, 'md5', sc_md5) self.configParser.set(sh_section, 'size', sc_size) self.configParser.set(sh_section, 'lastmodify', lastmodify) write = True for obj in objs: section = obj.getShaderFile() shader = section if obj.isRefFile() : shader = obj.getRawRefFile() spirv_file = obj.getSpirvCacheFile() if len(spirv_file) > 0 : self.__writeShaderFileConfigInformation__(section,shader,spirv_file) write = True if write : with open(self.configFile, 'w') as cfg: self.configParser.write(cfg) def __init__(self,use, spirv_opt_tag): self.root_dir = '.cache' self.shader_dir = 'shader' self.cache_dir = 'cache' self.configFile = os.path.join(self.root_dir, self.shader_dir,'config') # config path self.configParser = configparser.ConfigParser() self.configParser.read(self.configFile) self.scriptName = os.path.basename(__file__) # makeshader.py self.cache_valid = False # bool, is cache valid self.config_sha1 = '' # sha1 for '.cache/shader/config' ,check if '.cache/shader/config' is changed self.use_cache = use self.spirv_opt_tag = spirv_opt_tag self.sha1 = '' # string, self sha1 self.md5 = '' # string, self md5 self.size = '' # number, self size self.lastmodify = '' # string, self lastmodify def genShaderFileObjs(shaders, macros): shaderObjs = [] _ensure_dir(gGeneratedShaderDir) for fileName in shaders: simplename = os.path.basename(fileName) cfg = macros.get(simplename, {}) if isinstance(cfg, dict): macro_list = cfg.get("macro", []) or [] support_fp16 = cfg.get("useFP16", False) else: macro_list = cfg if isinstance(cfg, list) else [] support_fp16 = False variants = [(None, 'fp32')] for macro in macro_list: variants.append((macro, 'fp32')) if support_fp16: variants.append((None, 'fp16')) for macro in macro_list: variants.append((macro, 'fp16')) for macro, precision in variants: generated = _build_generated_shader_path(fileName, macro, precision) obj = ShaderFile(shader=generated, ref=True, raw=fileName, precision=precision) if macro: obj.setMacro(macro) shaderObjs.append(obj) return shaderObjs def genJsonHeadFile(): writeHeadFile = False with open("glsl/headfile.json") as f: originShaders = json.loads(f.read()) for s in shaders: if not s in originShaders: print("Write Head File") writeHeadFile = True break if (writeHeadFile): allShaders = {} for s in shaders: allShaders[s] = "glsl" with open("glsl/headfile.json", "w") as f: f.write(json.dumps(allShaders, indent=4)) def genRefCompFiles(objs): for obj in objs: if obj.isRefFile(): raw = obj.getRawRefFile() dst = obj.getShaderFile() if len(raw) > 0 and len(dst) > 0: header = FP16_HEADER if obj.getPrecision() == 'fp16' else FP32_HEADER macro_line = '' if len(obj.getMacro()) > 0: macro_line = "#define " + obj.getMacro() + "\n" with open(raw) as f: body = f.read() _ensure_dir(os.path.dirname(dst)) with open(dst, 'w') as wf: wf.write(header) wf.write("\n") if macro_line: wf.write(macro_line) if body and not body.startswith("\n"): wf.write("\n") wf.write(body) def removeRefCompFiles(objs): removed = False for obj in objs: if obj.isRefFile(): s = obj.getShaderFile() if os.path.exists(s) and os.path.isfile(s): os.remove(s) removed = True if removed and os.path.isdir(gGeneratedShaderDir) and not os.listdir(gGeneratedShaderDir): os.rmdir(gGeneratedShaderDir) def genMapFile(objs): mapFile = "VulkanShaderMap.cpp" cpp = '/*Auto Generated File, Don\' Modified.*/\n' cpp += "#include \"VulkanShaderMap.hpp\"\n" cpp += "#include \"AllShader.h\"\n" cpp += 'namespace MNN {\n' cpp += 'void VulkanShaderMap::init() {\n' for obj in objs: name = obj.getFileName() cpp += 'mMaps.insert(std::make_pair(\"'+ name + '", std::make_pair(' + name + ',' + name + '_len' ')));\n' cpp += '}\n' cpp += '}\n' with open(mapFile, 'w') as f: f.write(cpp) def genCppFile(objs, inc, dst): cpp = "#include \"" + inc + "\"\n" genRefCompFiles(objs) for obj in objs: s = obj.getShaderFile() name = obj.getFileName() #print name out = 'tempspv' rm = True spirv_cache = obj.getSpirvFile() # print("cache:", len(spirv_cache)) if len(spirv_cache) <= 0 : spirv_save = obj.getSpirvCacheFile() if len(spirv_save) > 0: out = spirv_save rm = False extra_flags = "" target_env_args = [] try: with open(s, "r") as rf: body = rf.read() if "GL_KHR_shader_subgroup" in body: target_env_args = ["--target-env", "vulkan1.1"] except: target_env_args = [] spirv_opt_tag = _spirv_opt_tag() enable_spirv_opt = len(spirv_opt_tag) > 0 raw_out = out need_remove_raw = False if enable_spirv_opt: raw_out = out + ".raw" need_remove_raw = True _run_cmd(["glslangValidator", "-V"] + target_env_args + [s, "-o", raw_out]) if enable_spirv_opt: ret = _run_cmd(["spirv-opt"] + gSpirvOptArgs + [raw_out, "-o", out]) if ret != 0: print("spirv-opt failed, fallback to unoptimized spirv: " + s) os.replace(raw_out, out) need_remove_raw = False if need_remove_raw and os.path.exists(raw_out) and os.path.isfile(raw_out): os.remove(raw_out) else: out = spirv_cache rm = False cpp_tmp_file = 'temp.spv.cpp' _run_cmd(["xxd", "-i", out], stdout_path=cpp_tmp_file) with open(cpp_tmp_file) as f: rep = out.replace(os.sep,'_') rep = rep.replace('.','_') allContent = f.read().replace(rep, name) cpp += 'const ' + allContent + '\n' if os.path.exists(cpp_tmp_file) and os.path.isfile(cpp_tmp_file) : os.remove(cpp_tmp_file) if rm and os.path.exists(out) and os.path.isfile(out) : os.remove(out) with open(dst, "w") as f: f.write(cpp) removeRefCompFiles(objs) def genHppFile(objs, fileName): h = "#ifndef VK_GLSL_SHADER_AUTO_GENERATE_H\n#define VK_GLSL_SHADER_AUTO_GENERATE_H\n" for obj in objs: name = obj.getFileName() print(name) h += "extern const unsigned char " + name + "[];\n"; h += 'extern unsigned int ' + name + '_len;\n' h += "#endif" with open(fileName, "w") as f: f.write(h) def generateFile(headfile, sourcefile, shaders, macros, cache): #genJsonHeadFile() print(macros) fileObjs = genShaderFileObjs(shaders, macros) cache.setupShaderCache(fileObjs) genHppFile(fileObjs, headfile) genCppFile(fileObjs, headfile, sourcefile) cache.updateShaderCache(fileObjs) genMapFile(fileObjs) def parseArgs(argv): try: opts, args = getopt.getopt(argv, "hf", ["force="]) except getopt.GetoptError: print('makeshader.py [-h,-f]') sys.exit(2) for opt, arg in opts: if opt == '-h': print('makeshader.py [-h,-f,--force] <-h> help <-f,--force> disable cache') sys.exit() elif opt in ("-f", "--force"): return False return True if __name__ == '__main__': use_cache = parseArgs(sys.argv[1:]) spirv_opt_tag = _spirv_opt_tag() if gEnableSpirvOpt and os.environ.get("MNN_VULKAN_DISABLE_SPIRV_OPT", "") != "1" and not _spirv_opt_available(): print("spirv-opt not found, skip -O optimization (install spirv-tools to enable).") shaderCache = ShaderCache(use_cache, spirv_opt_tag) if use_cache : if not shaderCache.initlizateShaderCache() : print("cache init failed,do't use cache") shaders = findAllShader(gDefaultPath) jsonFile = open(gDefaultPath +'/macro.json', 'r') print(gDefaultPath) macros = json.loads(jsonFile.read()) jsonFile.close() generateFile(gOutputHeadFile, gOutputSourceFile, shaders, macros, shaderCache) # generateFileAsm("AllShader_asm.h", "AllShader_asm.cpp", "spirv");