Convert build scripts to Python #31

Merged
MartinFouilleul merged 24 commits from build-python into main 2023-08-08 09:38:44 +00:00
3 changed files with 370 additions and 281 deletions
Showing only changes of commit 10351e45cd - Show all commits

View File

@ -1,131 +1,135 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
import sys import sys
if len(sys.argv) < 2: def bindgen(apiName, cdir):
print("bindgen require an api name\n") inPath = cdir + '/bindgen_' + apiName + '_api.txt'
exit(-1); outPath = cdir + '/bindgen_' + apiName + '_api.c'
apiName = sys.argv[1] inFile = open(inPath, 'r')
cdir = '' outFile = open(outPath, 'w')
if len(sys.argv) > 2: stubs = []
cdir = sys.argv[2] links = []
inPath = cdir + '/bindgen_' + apiName + '_api.txt' def gen_stub(name, sig, native_name):
outPath = cdir + '/bindgen_' + apiName + '_api.c' if native_name == None:
native_name = name
inFile = open(inPath, 'r') src = 'const void* ' + name + '_stub(IM3Runtime runtime, IM3ImportContext _ctx, uint64_t * _sp, void * _mem)\n'
outFile = open(outPath, 'w') src += '{\n'
spIndex = 0
stubs = [] parsingRet = True
links = [] retCount = 0
argCount = 0
retString = ''
argString = ''
def gen_stub(name, sig, native_name): for index, c in enumerate(sig):
if native_name == None: if parsingRet:
native_name = name if retCount > 1:
print('unsupported multiple return types\n')
src = 'const void* ' + name + '_stub(IM3Runtime runtime, IM3ImportContext _ctx, uint64_t * _sp, void * _mem)\n' break
src += '{\n' if c == '(':
spIndex = 0 parsingRet = False
continue
parsingRet = True elif c == ')':
retCount = 0 print('unexpected ) while parsing return type\n')
argCount = 0 break;
retString = '' elif c == 'v':
argString = '' continue
elif c == 'i':
for index, c in enumerate(sig): retString = '*((i32*)&_sp[0]) = '
if parsingRet: elif c == 'I':
if retCount > 1: retString = '*((i64*)&_sp[0]) = '
print('unsupported multiple return types\n') elif c == 'f':
break retString = '*((f32*)&_sp[0]) = '
if c == '(': elif c == 'F':
parsingRet = False retString = '*((f64*)&_sp[0]) = '
continue elif c == 'p':
elif c == ')': print('returning pointers is not supported yet\n')
print('unexpected ) while parsing return type\n') break
break; else:
elif c == 'v': print('unrecognized type ' + c + ' in procedure return\n')
continue break
elif c == 'i': retCount += 1
retString = '*((i32*)&_sp[0]) = '
elif c == 'I':
retString = '*((i64*)&_sp[0]) = '
elif c == 'f':
retString = '*((f32*)&_sp[0]) = '
elif c == 'F':
retString = '*((f64*)&_sp[0]) = '
elif c == 'p':
print('returning pointers is not supported yet\n')
break
else: else:
print('unrecognized type ' + c + ' in procedure return\n') argIndex = argCount + retCount
break if c == ')':
retCount += 1 break
else: elif c == 'v':
argIndex = argCount + retCount break
if c == ')': elif c == 'i':
break argString += '*(i32*)&_sp[' + str(argIndex) + ']'
elif c == 'v': elif c == 'I':
break argString += '*(i64*)&_sp[' + str(argIndex) + ']'
elif c == 'i': elif c == 'f':
argString += '*(i32*)&_sp[' + str(argIndex) + ']' argString += '*(f32*)&_sp[' + str(argIndex) + ']'
elif c == 'I': elif c == 'F':
argString += '*(i64*)&_sp[' + str(argIndex) + ']' argString += '*(f64*)&_sp[' + str(argIndex) + ']'
elif c == 'f': elif c == 'p':
argString += '*(f32*)&_sp[' + str(argIndex) + ']' argString += '(void*)((char*)_mem + *(i32*)&_sp[' + str(argIndex) + '])'
elif c == 'F': else:
argString += '*(f64*)&_sp[' + str(argIndex) + ']' print('unrecognized type ' + c + ' in procedure signature\n')
elif c == 'p': break
argString += '(void*)((char*)_mem + *(i32*)&_sp[' + str(argIndex) + '])'
if index+2 < len(sig):
argString += ', '
argCount += 1
src += '\t' + retString + native_name + '(' + argString + ');\n'
src += '\treturn(0);\n'
src += '}\n'
stubs.append(src)
def gen_link(name, sig):
m3_Sig = ''
for c in sig:
if c == 'p':
m3_Sig += 'i'
else: else:
print('unrecognized type ' + c + ' in procedure signature\n') m3_Sig += c
break
if index+2 < len(sig): src = '\tres = m3_LinkRawFunction(module, "*", "' + name + '", "' + m3_Sig + '", ' + name + '_stub);\n'
argString += ', ' src += '\tif(res != m3Err_none && res != m3Err_functionLookupFailed) { log_error("error: %s\\n", res); return(-1); }\n\n'
argCount += 1 links.append(src)
src += '\t' + retString + native_name + '(' + argString + ');\n' for line in inFile:
src += '\treturn(0);\n' if line.isspace():
src += '}\n' continue
stubs.append(src) desc = line.split()
def gen_link(name, sig): gen_stub(desc[0], desc[1], desc[2] if len(desc) > 2 else None)
m3_Sig = '' gen_link(desc[0], desc[1])
for c in sig:
if c == 'p':
m3_Sig += 'i'
else:
m3_Sig += c
src = '\tres = m3_LinkRawFunction(module, "*", "' + name + '", "' + m3_Sig + '", ' + name + '_stub);\n' linkProc = 'int bindgen_link_' + apiName + '_api(IM3Module module)\n'
src += '\tif(res != m3Err_none && res != m3Err_functionLookupFailed) { log_error("error: %s\\n", res); return(-1); }\n\n' linkProc += '{\n'
links.append(src) linkProc += '\tM3Result res;\n'
for line in inFile: for link in links:
if line.isspace(): linkProc += link
continue
desc = line.split()
gen_stub(desc[0], desc[1], desc[2] if len(desc) > 2 else None) linkProc += '\treturn(0);\n'
gen_link(desc[0], desc[1]) linkProc += '}\n'
linkProc = 'int bindgen_link_' + apiName + '_api(IM3Module module)\n' for stub in stubs:
linkProc += '{\n' outFile.write(stub)
linkProc += '\tM3Result res;\n'
for link in links: outFile.write('\n')
linkProc += link outFile.write(linkProc)
linkProc += '\treturn(0);\n' inFile.close()
linkProc += '}\n' outFile.close()
for stub in stubs: if __name__ == "__main__":
outFile.write(stub) if len(sys.argv) < 2:
print("bindgen require an api name\n")
exit(-1)
outFile.write('\n') apiName = sys.argv[1]
outFile.write(linkProc) cdir = ''
inFile.close() if len(sys.argv) > 2:
outFile.close() cdir = sys.argv[2]
bindgen(apiName, cdir)

View File

@ -3,31 +3,6 @@
from argparse import ArgumentParser from argparse import ArgumentParser
import json import json
parser = ArgumentParser(prog='bindgen.py')
parser.add_argument('api')
parser.add_argument('spec')
parser.add_argument('-g', '--guest-stubs')
parser.add_argument('--guest-include')
parser.add_argument('--wasm3-bindings')
args = parser.parse_args()
apiName = args.api
spec = args.spec
guest_stubs_path = args.guest_stubs
if guest_stubs_path == None:
guest_stubs_path = 'bindgen_' + apiName + '_guest_stubs.c'
wasm3_bindings_path = args.wasm3_bindings
if wasm3_bindings_path == None:
wasm3_bindings_path = 'bindgen_' + apiName + '_wasm3_bindings.c'
host_bindings = open(wasm3_bindings_path, 'w')
guest_bindings = None
specFile = open(spec, 'r')
data = json.load(specFile)
def needs_arg_ptr_stub(decl): def needs_arg_ptr_stub(decl):
res = (decl['ret']['tag'] == 'S') res = (decl['ret']['tag'] == 'S')
for arg in decl['args']: for arg in decl['args']:
@ -35,167 +10,205 @@ def needs_arg_ptr_stub(decl):
res = True res = True
return(res) return(res)
for decl in data: def bindgen2(apiName, spec, **kwargs):
if needs_arg_ptr_stub(decl): guest_stubs_path = kwargs["guest_stubs"]
guest_bindings = open(guest_stubs_path, 'w') guest_include = kwargs.get("guest-include")
if args.guest_include != None: wasm3_bindings_path = kwargs["wasm3_bindings"]
s = '#include"' + args.guest_include + '"\n\n'
print(s, file=guest_bindings)
break
for decl in data: host_bindings = open(wasm3_bindings_path, 'w')
guest_bindings = None
name = decl['name'] specFile = open(spec, 'r')
cname = decl.get('cname', name) data = json.load(specFile)
if needs_arg_ptr_stub(decl): for decl in data:
argPtrStubName = name + '_argptr_stub' if needs_arg_ptr_stub(decl):
# pointer arg stub declaration guest_bindings = open(guest_stubs_path, 'w')
s = '' if guest_include != None:
if decl['ret']['tag'] == 'S': s = '#include"' + guest_include + '"\n\n'
s += 'void' print(s, file=guest_bindings)
else: break
s += decl['ret']['name']
s += ' ORCA_IMPORT(' + argPtrStubName + ') (' for decl in data:
if decl['ret']['tag'] == 'S': name = decl['name']
s += decl['ret']['name'] + '* __retArg' cname = decl.get('cname', name)
if len(decl['args']) > 0:
s += ', '
for i, arg in enumerate(decl['args']): if needs_arg_ptr_stub(decl):
s += arg['type']['name'] argPtrStubName = name + '_argptr_stub'
if arg['type']['tag'] == 'S': # pointer arg stub declaration
s += '*' s = ''
s += ' ' + arg['name'] if decl['ret']['tag'] == 'S':
if i+1 < len(decl['args']): s += 'void'
s += ', '
s += ');\n\n'
# forward function to pointer arg stub declaration
s += decl['ret']['name'] + ' ' + name + '('
for i, arg in enumerate(decl['args']):
s += arg['type']['name'] + ' ' + arg['name']
if i+1 < len(decl['args']):
s += ', '
s += ')\n'
s += '{\n'
s += '\t'
if decl['ret']['tag'] == 'S':
s += decl['ret']['name'] + ' __ret;\n\t'
elif decl['ret']['tag'] != 'v':
s += decl['ret']['name']
s += ' __ret = '
s += argPtrStubName + '('
if decl['ret']['tag'] == 'S':
s += '&__ret'
if len(decl['args']) > 0:
s += ', '
for i, arg in enumerate(decl['args']):
if arg['type']['tag'] == 'S':
s += '&'
s += arg['name']
if i+1 < len(decl['args']):
s += ', '
s += ');\n'
if decl['ret']['tag'] != 'v':
s += '\treturn(__ret);\n'
s += '}\n\n'
print(s, file=guest_bindings)
# host-side stub
s = 'const void* ' + cname + '_stub(IM3Runtime runtime, IM3ImportContext _ctx, uint64_t* _sp, void* _mem)'
gen_stub = decl.get('gen_stub', True)
if gen_stub == False:
s += ';\n\n'
else:
s += '\n{\n\t'
retTag = decl['ret']['tag']
if retTag == 'i':
s += '*((i32*)&_sp[0]) = '
elif retTag == 'I':
s += '*((i64*)&_sp[0]) = '
elif retTag == 'f':
s += '*((f32*)&_sp[0]) = '
elif retTag == 'F':
s += '*((f64*)&_sp[0]) = '
elif retTag == 'S':
retTypeName = decl['ret']['name']
retTypeCName = decl['ret'].get('cname', retTypeName)
s += '*(' + retTypeCName + '*)((char*)_mem + *(i32*)&_sp[0]) = '
s += cname + '('
firstArgIndex = 0
if retTag != 'v':
firstArgIndex = 1
for i, arg in enumerate(decl['args']):
typeName = arg['type']['name']
typeCName = arg['type'].get('cname', typeName)
argTag = arg['type']['tag']
if argTag == 'i':
s += '*(i32*)&_sp[' + str(firstArgIndex + i) + ']'
elif argTag == 'I':
s += '*(i64*)&_sp[' + str(firstArgIndex + i) + ']'
elif argTag == 'f':
s += '*(f32*)&_sp[' + str(firstArgIndex + i) + ']'
elif argTag == 'F':
s += '*(f64*)&_sp[' + str(firstArgIndex + i) + ']'
elif argTag == 'p':
s += '(void*)((char*)_mem + *(i32*)&_sp[' + str(firstArgIndex + i) + '])'
elif argTag == 'S':
s += '*(' + typeCName + '*)((char*)_mem + *(i32*)&_sp[' + str(firstArgIndex + i) + '])'
else: else:
print('unrecognized type ' + c + ' in procedure signature\n') s += decl['ret']['name']
break
if i+1 < len(decl['args']): s += ' ORCA_IMPORT(' + argPtrStubName + ') ('
s += ', '
s += ');\n\treturn(0);\n}\n\n' if decl['ret']['tag'] == 'S':
s += decl['ret']['name'] + '* __retArg'
if len(decl['args']) > 0:
s += ', '
for i, arg in enumerate(decl['args']):
s += arg['type']['name']
if arg['type']['tag'] == 'S':
s += '*'
s += ' ' + arg['name']
if i+1 < len(decl['args']):
s += ', '
s += ');\n\n'
# forward function to pointer arg stub declaration
s += decl['ret']['name'] + ' ' + name + '('
for i, arg in enumerate(decl['args']):
s += arg['type']['name'] + ' ' + arg['name']
if i+1 < len(decl['args']):
s += ', '
s += ')\n'
s += '{\n'
s += '\t'
if decl['ret']['tag'] == 'S':
s += decl['ret']['name'] + ' __ret;\n\t'
elif decl['ret']['tag'] != 'v':
s += decl['ret']['name']
s += ' __ret = '
s += argPtrStubName + '('
if decl['ret']['tag'] == 'S':
s += '&__ret'
if len(decl['args']) > 0:
s += ', '
for i, arg in enumerate(decl['args']):
if arg['type']['tag'] == 'S':
s += '&'
s += arg['name']
if i+1 < len(decl['args']):
s += ', '
s += ');\n'
if decl['ret']['tag'] != 'v':
s += '\treturn(__ret);\n'
s += '}\n\n'
print(s, file=guest_bindings)
# host-side stub
s = 'const void* ' + cname + '_stub(IM3Runtime runtime, IM3ImportContext _ctx, uint64_t* _sp, void* _mem)'
gen_stub = decl.get('gen_stub', True)
if gen_stub == False:
s += ';\n\n'
else:
s += '\n{\n\t'
retTag = decl['ret']['tag']
if retTag == 'i':
s += '*((i32*)&_sp[0]) = '
elif retTag == 'I':
s += '*((i64*)&_sp[0]) = '
elif retTag == 'f':
s += '*((f32*)&_sp[0]) = '
elif retTag == 'F':
s += '*((f64*)&_sp[0]) = '
elif retTag == 'S':
retTypeName = decl['ret']['name']
retTypeCName = decl['ret'].get('cname', retTypeName)
s += '*(' + retTypeCName + '*)((char*)_mem + *(i32*)&_sp[0]) = '
s += cname + '('
firstArgIndex = 0
if retTag != 'v':
firstArgIndex = 1
for i, arg in enumerate(decl['args']):
typeName = arg['type']['name']
typeCName = arg['type'].get('cname', typeName)
argTag = arg['type']['tag']
if argTag == 'i':
s += '*(i32*)&_sp[' + str(firstArgIndex + i) + ']'
elif argTag == 'I':
s += '*(i64*)&_sp[' + str(firstArgIndex + i) + ']'
elif argTag == 'f':
s += '*(f32*)&_sp[' + str(firstArgIndex + i) + ']'
elif argTag == 'F':
s += '*(f64*)&_sp[' + str(firstArgIndex + i) + ']'
elif argTag == 'p':
s += '(void*)((char*)_mem + *(i32*)&_sp[' + str(firstArgIndex + i) + '])'
elif argTag == 'S':
s += '*(' + typeCName + '*)((char*)_mem + *(i32*)&_sp[' + str(firstArgIndex + i) + '])'
else:
print('unrecognized type ' + c + ' in procedure signature\n')
break
if i+1 < len(decl['args']):
s += ', '
s += ');\n\treturn(0);\n}\n\n'
print(s, file=host_bindings)
# link function
s = 'int bindgen_link_' + apiName + '_api(IM3Module module)\n{\n\t'
s += 'M3Result res;\n'
for decl in data:
name = decl['name']
cname = decl.get('cname', name)
if needs_arg_ptr_stub(decl):
name = name + '_argptr_stub'
m3Sig = ''
if decl['ret']['tag'] == 'S':
m3Sig += 'v'
else:
m3Sig += decl['ret']['tag']
m3Sig += '('
if decl['ret']['tag'] == 'S':
m3Sig += 'i'
for arg in decl['args']:
tag = arg['type']['tag']
if tag == 'p' or tag == 'S':
tag = 'i'
m3Sig += tag
m3Sig += ')'
s += '\tres = m3_LinkRawFunction(module, "*", "' + name + '", "' + m3Sig + '", ' + cname + '_stub);\n'
s += '\tif(res != m3Err_none && res != m3Err_functionLookupFailed) { log_error("error: %s\\n", res); return(-1); }\n\n'
s += '\treturn(0);\n}\n'
print(s, file=host_bindings) print(s, file=host_bindings)
# link function
s = 'int bindgen_link_' + apiName + '_api(IM3Module module)\n{\n\t'
s += 'M3Result res;\n'
for decl in data: if __name__ == "__main__":
name = decl['name'] parser = ArgumentParser(prog='bindgen.py')
cname = decl.get('cname', name) parser.add_argument('api')
parser.add_argument('spec')
parser.add_argument('-g', '--guest-stubs')
parser.add_argument('--guest-include')
parser.add_argument('--wasm3-bindings')
if needs_arg_ptr_stub(decl): args = parser.parse_args()
name = name + '_argptr_stub'
m3Sig = '' apiName = args.api
if decl['ret']['tag'] == 'S': spec = args.spec
m3Sig += 'v' guest_stubs_path = args.guest_stubs
else: if guest_stubs_path == None:
m3Sig += decl['ret']['tag'] guest_stubs_path = 'bindgen_' + apiName + '_guest_stubs.c'
m3Sig += '(' wasm3_bindings_path = args.wasm3_bindings
if decl['ret']['tag'] == 'S': if wasm3_bindings_path == None:
m3Sig += 'i' wasm3_bindings_path = 'bindgen_' + apiName + '_wasm3_bindings.c'
for arg in decl['args']:
tag = arg['type']['tag'] bindgen2(apiName, spec,
if tag == 'p' or tag == 'S': guest_stubs_path=guest_stubs_path,
tag = 'i' guest_include=args.guest_include,
m3Sig += tag wasm3_bindings_path=wasm3_bindings_path,
m3Sig += ')' )
s += '\tres = m3_LinkRawFunction(module, "*", "' + name + '", "' + m3Sig + '", ' + cname + '_stub);\n'
s += '\tif(res != m3Err_none && res != m3Err_functionLookupFailed) { log_error("error: %s\\n", res); return(-1); }\n\n'
s += '\treturn(0);\n}\n'
print(s, file=host_bindings)

View File

@ -9,6 +9,8 @@ import subprocess
from zipfile import ZipFile from zipfile import ZipFile
import checksum import checksum
from bindgen import bindgen
from bindgen2 import bindgen2
from log import * from log import *
from utils import pushd, removeall from utils import pushd, removeall
@ -28,6 +30,7 @@ def build_runtime(args):
build_milepost("lib", args.release) build_milepost("lib", args.release)
build_wasm3(args.release) build_wasm3(args.release)
build_orca(args.release)
def build_milepost(target, release): def build_milepost(target, release):
@ -213,6 +216,75 @@ def build_wasm3_lib_win(release):
], check=True) ], check=True)
def build_orca(release):
print("Building Orca...")
os.makedirs("bin", exist_ok=True)
if platform.system() == "Windows":
build_orca_win(release)
elif platform.system() == "Darwin":
raise "can't yet build Orca on Mac"
else:
log_error(f"can't build Orca for unknown platform '{platform.system()}'")
exit(1)
def build_orca_win(release):
pthread_dir = "..\\vcpkg\\packages\\pthreads_x64-windows"
# copy libraries
shutil.copy("milepost\\bin\\milepost.dll", "bin")
shutil.copy("milepost\\bin\\milepost.dll.lib", "bin")
shutil.copy(os.path.join(pthread_dir, "bin\\pthreadVC3.dll"), "bin")
# generate wasm3 api bindings
bindgen("core", "src")
bindgen("gles", "src")
bindgen2("canvas", "src\\canvas_api.json",
guest_stubs="sdk\\orca_surface.c",
guest_include="graphics.h",
wasm3_bindings="src\\canvas_api_bind_gen.c",
)
bindgen2("clock", "src\\clock_api.json",
guest_stubs="sdk\\orca_clock.c",
guest_include="platform_clock.h",
wasm3_bindings="src\\clock_api_bind_gen.c",
)
bindgen2("io", "src\\io_api.json",
guest_stubs="sdk\\io_stubs.c",
wasm3_bindings="src\\io_api_bind_gen.c",
)
# compile orca
pthread_include = os.path.join(pthread_dir, "include")
includes = [
"/I", "src",
"/I", "sdk",
"/I", "ext\wasm3\source",
"/I", "milepost\src",
"/I", "milepost\ext",
"/I", pthread_include,
]
pthread_lib = os.path.join(pthread_dir, "lib")
libs = [
"/LIBPATH:bin",
f"/LIBPATH:{pthread_lib}",
"milepost.dll.lib",
"wasm3.lib",
"pthreadVC3.lib",
]
subprocess.run([
"cl",
"/Zi", "/Zc:preprocessor", "/std:c11",
*includes,
"src\\main.c",
"/link", *libs,
"/out:bin\\orca.exe",
], check=True)
def ensure_programs(): def ensure_programs():
if platform.system() == "Windows": if platform.system() == "Windows":
try: try: