regenByteCode.py

#!/usr/bin/env python
#-------------------------------------------------------------------------------------------------------
# Copyright (C) Microsoft. All rights reserved.
# Copyright (c) ChakraCore Project Contributors. All rights reserved.
# Licensed under the MIT license. See LICENSE.txt file in the project root for full license information.
#-------------------------------------------------------------------------------------------------------

# Regenerate embedded bytecode headers. This script must be run when:
# a) Ahy changes have been made to the javascript in lib/Runtime/Library/InJavascript
# b) Any changes have been made to the bytecode emission process including edits to JnDirectFields.h
# NOTEs:
# 1. this script relies on forcing 64bit CC builds to produce 32bit bytecode - this could break due to future
#    changes to CC. If this facility breaks it will need to be fixed or this script will need to be updated to
#    use 32 bit builds as well as 64 bit
# 2. Options:
#    --skip-build  Don't build CC just generate bytecode with already built binaries
#    --jit only generate bytecode for CC with jit (default is to do both jit and noJit)
#    --noJit only generate bytecode for CC with noJit (default is to do both jit and noJit)
#    --verify throw an error if bytecode changes detected - intended for use in CI
#    --binary=path provide a path to a binary to use, requires either --jit or --noJit to be set
#    --x86 specify that provided binary is an x86 build, will generate x86 bytecode only - requires pre-built binary
# 3. Python version - this script is designed to run on both Python 2.x and 3.x

import subprocess
import sys
import uuid
import os

# Parse provided parameters
verification_mode = False
skip_build = False
noJit = True
jit = True
x86 = False
overide_binary = ""

for param in sys.argv:
    if param == '--skip-build':
        skip_build = True
    elif param == '--verify':
        # welcome message for CI
        print('######### Verifying generated bytecode #########')
        verification_mode = True
    elif param == '--noJit':
        jit = False
    elif param == '--jit':
        noJit = False
    elif param == '--x86':
        x86 = True
    elif param[:9] == '--binary=':
        overide_binary = param[9:]
        skip_build = True

# Detect OS
windows = False
if os.name == 'posix':
    print('OS is Linux or macOS')
else:
    print('OS is Windows')
    windows = True

# If Jit or noJit flag has been give print a message also if both flags given revert to default behaviour
if jit == False:
    if noJit == True:
        print('Regenerating bytecode for no-jit build only')
    else:
        noJit = True
        jit = True
elif noJit == False:
    print('Regenerating bytecode for jit build only')

if x86 == True:
    if overide_binary == "":
        print('x86 build can only be used when pre-built and provided with the --binary command line parameter')
        sys.exit(1)

# Adjust path for running from different locations
base_path = os.path.abspath(os.path.dirname(__file__))

# Compile ChakraCore both noJit and Jit variants (unless disabled by args)
def run_sub(message, commands, error):
    print(message)
    sub = subprocess.Popen(commands, shell=windows)
    sub.wait()
    if sub.returncode != 0:
        print(error)
        sys.exit(1)

if skip_build == False:
    # build for linux or macOS with build.sh script - could update to use cmake directly but this works for now
    if windows == False:
        if noJit == True:
            run_sub('Compiling ChakraCore with no Jit',
                [base_path + '/../build.sh', '--no-jit', '--debug', '--static', '--target-path=' + base_path + '/../out/noJit', '-j=8'], 
                'No Jit build failed - aborting bytecode generation')
        if jit == True:
            run_sub('Compiling ChakraCore with Jit',
                [base_path + '/../build.sh', '--debug', '--static', '--target-path=' + base_path + '/../out/Jit', '-j=8'], 
                'Jit build failed - aborting bytecode generation')
    # build for windows
    else:
        if noJit == True:
            run_sub('Compiling ChakraCore with no Jit',
                ['msbuild', '/P:platform=x64', '/P:configuration=debug', '/M', '/p:BuildJIT=false', base_path+ '/../Build/Chakra.Core.sln'],
                'No Jit build failed - aborting bytecode generation')
        if jit == True:
            run_sub('Compiling ChakraCore with Jit',
                ['msbuild', '/P:platform=x64', '/P:configuration=debug', '/M', base_path + '/../Build/Chakra.Core.sln'],
                'No Jit build failed - aborting bytecode generation')


# Generate the new bytecode checking for changes to each file
# First define variables and methods that will be used then the calls take place below
changes_detected = False

# this header text will be placed at the top of the generated files
header_text = '''//-------------------------------------------------------------------------------------------------------
// Copyright (C) Microsoft. All rights reserved.
// Copyright (c) ChakraCore Project Contributors. All rights reserved.
// Licensed under the MIT license. See LICENSE.txt file in the project root for full license information.
//-------------------------------------------------------------------------------------------------------

// Generated Bytecode Header, this file was created by tools/regenByteCode.py
// This file contains:
// a) bytecode for Intl library methods implemented in javascript and
// b) bytecode for other Js library methods, JsBuiltIns, implemented in javascript

#define JsBuiltIns(VALUE)'''

def append_bytecode(header, command, in_path, file_name, error):
    command_with_file = command[:]
    command_with_file.append(in_path + file_name)
    header.write('//Bytecode generated from ' + file_name + '\nconst char Library_Bytecode_')
    header.write(file_name[:-3])
    header.flush()
    job = subprocess.Popen(command_with_file, stdout=header)
    job.wait()
    if job.returncode != 0:
        print(error)
        print('Command line: ')
        print(*command_with_file)
        sys.exit(1)

# Load file and ensure line endings are '\n' if on windows
def load_file(path, mode):
    global windows
    if windows == True:
        if sys.version_info[0] < 3:
            return open(path, mode + 'b')
        else:
            return open(path, mode, newline='\n')
    else:
        return open(path, mode)

# Regenerate the bytecode
def bytecode_job(out_path, command, in_path, error):
    if verification_mode == True:
        print('Checking bytecode in file ' + out_path)
    else:
        print('Generating bytecode in file ' + out_path)

    old_version = ''
    global changes_detected
    if changes_detected == False:
        old_version = load_file(out_path, 'r').read()

    header = load_file(out_path, 'w')
    header.write(header_text)
    files = os.listdir(in_path)
    files.sort()
    filtered_files = []
    for file_name in files:
        if file_name.endswith('.js'):
            if file_name != 'Intl.js':
                without_extension = file_name[:-3]
                parts = without_extension.split('_')
                header.write(' \\\nVALUE(' + parts[0] + ', ' + parts[1] + ', ' +  parts[0] + parts[1].title() + ')')
                filtered_files.append(file_name)

    header.write('\n\nnamespace js\n{\n\n#ifdef ENABLE_JS_BUILTINS\n\n')

    # generate bytecode for JsBuiltins
    command_with_chakra_lib = command[:]
    command_with_chakra_lib.append('-LdChakraLib')
    command_with_chakra_lib.append('-JsBuiltIn')
    for file_name in filtered_files:
        append_bytecode(header, command_with_chakra_lib, in_path, file_name, error)

    # generate bytecode for Intl
    command.append('-Intl')
    header.write('#endif\n\n#ifdef ENABLE_INTL_OBJECT\n\n')
    append_bytecode(header, command, in_path, 'Intl.js', error)

    header.write('#endif\n\n}\n')
    header.close()
    if changes_detected == False:
        new_version = load_file(out_path, 'r').read()
        if new_version != old_version:
            changes_detected = True
            if verification_mode == True:
                new_lines = new_version.split('\n')
                old_lines = old_version.split('\n')
                max_lines = min(len(new_lines), len(old_lines))
                for i in range(0, max_lines):
                    if new_lines[i] != old_lines[i]:
                        print('Error found - output on line ' + str(i + 1) + ' is:')
                        print(new_lines[i].replace('\r', '\\r'))
                        print('Expected output was:')
                        print(old_lines[i].replace('\r', '\\r'))
                        break


# set paths for binaries - default paths based on build seteps above (different for windows to macOS and linux)
# OR overridden path provided on command line
noJitpath = base_path + "/../out/noJit/debug/ch"
jitPath = base_path + "/../out/jit/debug/ch"

if overide_binary != "":
    noJitpath = overide_binary
    jitPath = overide_binary
    if jit == True and noJit == True:
        print("Cannot use override binary option without specifying either jit or noJit")
        sys.exit(1)
elif windows == True:
    noJitpath = base_path + '/../Build/VcBuild.NoJIT/bin/x64_debug/ch.exe'
    jitPath = base_path + '/../Build/VcBuild/bin/x64_debug/ch.exe'

# Call the functions above to generate the bytecode
if noJit == True:
    commands = [noJitpath, '-GenerateLibraryByteCodeHeader']
    if x86 == False:
        bytecode_job(base_path + '/../lib/Runtime/Library/InJavascript/JsBuiltIn.nojit.bc.64b.h',
            commands, base_path + '/../lib/Runtime/Library/InJavascript/',
            'Failed to generate noJit 64bit Bytecode')
        commands.append('-Force32BitByteCode')

    bytecode_job(base_path + '/../lib/Runtime/Library/InJavascript/JsBuiltIn.nojit.bc.32b.h',
        commands, base_path + '/../lib/Runtime/Library/InJavascript/',
        'Failed to generate noJit 32bit JsBuiltin Bytecode')

if jit == True:
    commands = [jitPath, '-GenerateLibraryByteCodeHeader']
    if x86 == False:
        bytecode_job(base_path + '/../lib/Runtime/Library/InJavascript/JsBuiltIn.bc.64b.h',
            commands, base_path + '/../lib/Runtime/Library/InJavascript/',
            'Failed to generate 64bit JsBuiltin Bytecode')
        commands.append('-Force32BitByteCode')

    bytecode_job(base_path + '/../lib/Runtime/Library/InJavascript/JsBuiltIn.bc.32b.h',
        commands, base_path + '/../lib/Runtime/Library/InJavascript/',
        'Failed to generate 32bit JsBuiltin Bytecode')


# Bytecode regeneration complete - assess changes, report result AND if appropriate generate a new GUID
if changes_detected == True:
    if verification_mode == True:
        print('Bytecode changes detected - the generated bytecode files are not up to date please run tools/regenByteCode.py\n')
        sys.exit(1)
    if jit == False or noJit == False:
        print("Bytecode updated for one variant only - ensure you re-run for both variants before submitting code")
    else:
        print('Generating new GUID for new bytecode')
        guid_header = load_file(base_path + '/../lib/Runtime/Bytecode/ByteCodeCacheReleaseFileVersion.h', 'w')
        guid = str(uuid.uuid4())

        output_str = '''//-------------------------------------------------------------------------------------------------------
// Copyright (C) Microsoft. All rights reserved.
// Copyright (c) ChakraCore Project Contributors. All rights reserved.
// Licensed under the MIT license. See LICENSE.txt file in the project root for full license information.
//-------------------------------------------------------------------------------------------------------
// NOTE: If there is a merge conflict the correct fix is to make a new GUID.
// This file was generated with tools/regenByteCode.py

// {%s}
const GUID byteCodeCacheReleaseFileVersion =
{ 0x%s, 0x%s, 0x%s, {0x%s, 0x%s, 0x%s, 0x%s, 0x%s, 0x%s, 0x%s, 0x%s } };

''' % (guid,
            guid[:8], guid[9:13], guid[14:18], guid[19:21], guid[21:23], guid[24:26],
            guid[26:28], guid[28:30], guid[30:32], guid[32:34], guid[-2:])

        guid_header.write(output_str)

        print('Bytecode successfully regenerated. Please rebuild ChakraCore to incorporate it.')
else:
    if verification_mode == True:
        print('Bytecode is up to date\n')
    else:
        print('Bytecode update was not required')