From ee16d3f82cdae3331781e8d8715ea928bf3aa487 Mon Sep 17 00:00:00 2001 From: Tomas Zezula Date: Tue, 23 Jan 2024 12:58:34 +0100 Subject: [PATCH 01/13] [GR-48481] Deploy python as polyglot isolate. --- sdk/mx.sdk/mx_sdk_vm_impl.py | 43 +++++++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/sdk/mx.sdk/mx_sdk_vm_impl.py b/sdk/mx.sdk/mx_sdk_vm_impl.py index cfbbb915628d..1a25dd1f91f0 100644 --- a/sdk/mx.sdk/mx_sdk_vm_impl.py +++ b/sdk/mx.sdk/mx_sdk_vm_impl.py @@ -1991,6 +1991,29 @@ def getArchivableResults(self, use_relpath=True, single=False): def is_skipped(self): return _skip_libraries(self.native_image_config) +class PolyglotIsolateLibrary(GraalVmLibrary): + def __init__(self, component, name, resources, native_image_config, **kw_args): + deps = list(resources) + super(PolyglotIsolateLibrary, self).__init__(component, name, deps, native_image_config, **kw_args) + self.resources = resources + + def getBuildTask(self, args): + svm_support = _get_svm_support() + assert svm_support.is_supported(), "Needs svm to build " + str(self) + if not self.is_skipped(): + return PolyglotIsolateLibraryBuildTask(self, args, svm_support) + else: + return mx.NoOpTask(self, args) + + def getArchivableResults(self, use_relpath=True, single=False): + for e in super(PolyglotIsolateLibrary, self).getArchivableResults(use_relpath=use_relpath, single=single): + yield e + if single: + return + output_dir = dirname(self.output_file()) + resources_dir = join(output_dir, 'resources') + if exists(resources_dir): + yield resources_dir, 'resources' class GraalVmMiscLauncher(GraalVmLauncher): # pylint: disable=too-many-ancestors def __init__(self, component, native_image_config, stage1=False, **kw_args): @@ -2440,6 +2463,13 @@ class GraalVmLibraryBuildTask(GraalVmSVMNativeImageBuildTask): pass +class PolyglotIsolateLibraryBuildTask(GraalVmLibraryBuildTask): + def get_build_args(self): + build_args = super(PolyglotIsolateLibraryBuildTask, self).get_build_args() + build_args.extend(mx.get_runtime_jvm_args(self.subject.resources, exclude_names=['truffle:TRUFFLE_API'])) + return build_args + + class JmodModifier(mx.Project): def __init__(self, jmod_file, library_projects, jimage_project, **kw_args): """ @@ -3492,8 +3522,14 @@ def register_main_dist(dist, label): with_non_rebuildable_configs = True for library_config in _get_library_configs(component): library_project = None + with_polyglot_isolate = False if with_svm: - library_project = GraalVmLibrary(component, GraalVmNativeImage.project_name(library_config), [], library_config) + with_polyglot_isolate = has_component('tfle', stage1=True) and isinstance(library_config, mx_sdk.LanguageLibraryConfig) and library_config.isolate_library_layout_distribution + if with_polyglot_isolate: + resources_distributions = library_config.isolate_library_layout_distribution.get('resources', []) + library_project = PolyglotIsolateLibrary(component, GraalVmNativeImage.project_name(library_config), resources_distributions, library_config) + else: + library_project = GraalVmLibrary(component, GraalVmNativeImage.project_name(library_config), [], library_config) register_project(library_project) if _rebuildable_image(library_config): register_project(GraalVmNativeProperties(component, library_config)) @@ -3509,7 +3545,7 @@ def register_main_dist(dist, label): register_project(launcher_project) polyglot_config_project = PolyglotConfig(component, library_config) register_project(polyglot_config_project) - if with_svm and library_config.isolate_library_layout_distribution and not library_project.is_skipped() and has_component('tfle', stage1=True): + if with_polyglot_isolate and not library_project.is_skipped(): # Create a layout distribution with the resulting language library that can be consumed into the # isolate resources jar distribution, resource_base_folder = f'META-INF/resources/engine/{library_config.language}-isolate///libvm' @@ -3524,7 +3560,8 @@ def register_main_dist(dist, label): name=library_config.isolate_library_layout_distribution['name'], deps=[], layout={ - f'{resource_base_folder}/': f'dependency:{library_project.name}' + f'{resource_base_folder}/': f'dependency:{library_project.name}', + f'{resource_base_folder}/resources': f'dependency:{library_project.name}/resources', }, path=None, platformDependent=True, From d2d70a1ff5afcab0f03a53e053bf86dc81395cb6 Mon Sep 17 00:00:00 2001 From: Tomas Zezula Date: Tue, 30 Jan 2024 08:59:48 +0100 Subject: [PATCH 02/13] [GR-48481] Polyglot isolate builds work in unchained configuration. --- sdk/mx.sdk/mx_sdk_vm_impl.py | 44 +++++++---- truffle/mx.truffle/mx_truffle.py | 132 +++++++++++++++++++++++++------ 2 files changed, 139 insertions(+), 37 deletions(-) diff --git a/sdk/mx.sdk/mx_sdk_vm_impl.py b/sdk/mx.sdk/mx_sdk_vm_impl.py index 1a25dd1f91f0..bbad76be2fae 100644 --- a/sdk/mx.sdk/mx_sdk_vm_impl.py +++ b/sdk/mx.sdk/mx_sdk_vm_impl.py @@ -1992,10 +1992,18 @@ def is_skipped(self): return _skip_libraries(self.native_image_config) class PolyglotIsolateLibrary(GraalVmLibrary): - def __init__(self, component, name, resources, native_image_config, **kw_args): - deps = list(resources) - super(PolyglotIsolateLibrary, self).__init__(component, name, deps, native_image_config, **kw_args) - self.resources = resources + def __init__(self, target_suite, language, deps, build_args, **kw_args): + library_config = mx_sdk.LanguageLibraryConfig( + jar_distributions=deps, + build_args=[], + build_args_enterprise=build_args, + language=language, + ) + super(PolyglotIsolateLibrary, self).__init__(None, os.path.basename(library_config.destination) + ".image", + list(deps), library_config, **kw_args) + self.suite = target_suite + self.dir = target_suite.dir + def getBuildTask(self, args): svm_support = _get_svm_support() @@ -2015,6 +2023,9 @@ def getArchivableResults(self, use_relpath=True, single=False): if exists(resources_dir): yield resources_dir, 'resources' + def is_skipped(self): + return False + class GraalVmMiscLauncher(GraalVmLauncher): # pylint: disable=too-many-ancestors def __init__(self, component, native_image_config, stage1=False, **kw_args): super(GraalVmMiscLauncher, self).__init__(component, GraalVmLauncher.launcher_project_name(native_image_config, stage1=stage1), [], native_image_config, stage1=stage1, **kw_args) @@ -2465,8 +2476,19 @@ class GraalVmLibraryBuildTask(GraalVmSVMNativeImageBuildTask): class PolyglotIsolateLibraryBuildTask(GraalVmLibraryBuildTask): def get_build_args(self): - build_args = super(PolyglotIsolateLibraryBuildTask, self).get_build_args() - build_args.extend(mx.get_runtime_jvm_args(self.subject.resources, exclude_names=['truffle:TRUFFLE_API'])) + target = self.subject.native_image_name[:-len(_lib_suffix)] + build_args = [ + '-EJVMCI_VERSION_CHECK', # Propagate this env var when running native image from mx + '--parallelism=' + str(self.parallelism), + '--shared', + '-o', + target, + '--features=com.oracle.svm.enterprise.truffle.PolyglotIsolateGuestFeature', + '-H:APIFunctionPrefix=truffle_isolate_', + ] + svm_experimental_options([ + '-H:+BuildOutputPrefix', + '-H:+GenerateBuildArtifactsFile', # generate 'build-artifacts.json' + ]) + mx.get_runtime_jvm_args(self.subject.native_image_jar_distributions) return build_args @@ -3522,14 +3544,8 @@ def register_main_dist(dist, label): with_non_rebuildable_configs = True for library_config in _get_library_configs(component): library_project = None - with_polyglot_isolate = False if with_svm: - with_polyglot_isolate = has_component('tfle', stage1=True) and isinstance(library_config, mx_sdk.LanguageLibraryConfig) and library_config.isolate_library_layout_distribution - if with_polyglot_isolate: - resources_distributions = library_config.isolate_library_layout_distribution.get('resources', []) - library_project = PolyglotIsolateLibrary(component, GraalVmNativeImage.project_name(library_config), resources_distributions, library_config) - else: - library_project = GraalVmLibrary(component, GraalVmNativeImage.project_name(library_config), [], library_config) + library_project = GraalVmLibrary(component, GraalVmNativeImage.project_name(library_config), [], library_config) register_project(library_project) if _rebuildable_image(library_config): register_project(GraalVmNativeProperties(component, library_config)) @@ -3545,7 +3561,7 @@ def register_main_dist(dist, label): register_project(launcher_project) polyglot_config_project = PolyglotConfig(component, library_config) register_project(polyglot_config_project) - if with_polyglot_isolate and not library_project.is_skipped(): + if with_svm and library_config.isolate_library_layout_distribution and not library_project.is_skipped() and has_component('tfle', stage1=True): # Create a layout distribution with the resulting language library that can be consumed into the # isolate resources jar distribution, resource_base_folder = f'META-INF/resources/engine/{library_config.language}-isolate///libvm' diff --git a/truffle/mx.truffle/mx_truffle.py b/truffle/mx.truffle/mx_truffle.py index 6ed30989857f..a64a60b3ad4a 100644 --- a/truffle/mx.truffle/mx_truffle.py +++ b/truffle/mx.truffle/mx_truffle.py @@ -1205,34 +1205,121 @@ def read_file(path): "Make sure the grammar files are up to date with the generated code. You can regenerate the generated code using mx.") -def register_polyglot_isolate_distributions(register_distribution, language_id, language_distribution, isolate_library_layout_distribution, internal_resource_project): +def register_polyglot_isolate_distributions(language_suite, register_project, register_distribution, language_id, language_distribution, + internal_resource_project, isolate_build_options=None, additional_isolate_dependencies=None): """ - Registers the polyglot isolate resource distribution and isolate resource meta-POM distribution. + Creates and registers the polyglot isolate resource distribution and isolate resource meta-POM distribution. The created polyglot isolate resource distribution is named `_ISOLATE_RESOURCES`, inheriting the Maven group ID from the given `language_distribution`, and the Maven artifact ID is `-isolate`. The meta-POM distribution is named `_ISOLATE`, having the Maven group ID `org.graalvm.polyglot`, and the Maven artifact ID is `-isolate`. + :param language_suite: The language suite used to register generated projects and distributions to. + :param register_project: A callback to dynamically register the project, obtained as a parameter from `mx_register_dynamic_suite_constituents`. + :type register_project: (mx.Project) -> None :param register_distribution: A callback to dynamically register the distribution, obtained as a parameter from `mx_register_dynamic_suite_constituents`. :type register_distribution: (mx.Distribution) -> None :param language_id: The language ID. - :param language_distribution: The language distribution used to inherit distribution properties. - :param isolate_library_layout_distribution: The layout distribution with polyglot isolate library. + :param language_distribution: The language distribution used to inherit isolate distribution properties. :param internal_resource_project: The internal resource project used for unpacking the polyglot isolate library. + :param isolate_build_options: additional options passed to a native image to build the isolate library. + :param additional_isolate_dependencies: additional distributions used to build the isolate library. """ + assert language_suite + assert register_project + assert register_distribution + assert language_id assert language_distribution - assert isolate_library_layout_distribution assert internal_resource_project - owner_suite = language_distribution.suite - resources_dist_name = f'{language_id.upper()}_ISOLATE_RESOURCES' - isolate_dist_name = f'{language_id.upper()}_ISOLATE' - layout_dist_qualified_name = f'{isolate_library_layout_distribution.suite.name}:{isolate_library_layout_distribution.name}' - maven_group_id = language_distribution.maven_group_id() + + def _resolve_distribution(distribution_name): + distribution = mx.distribution(distribution_name, fatalIfMissing=False) + if not distribution: + simple_name = mx.splitqualname(distribution_name)[1] + filtered = [d for d in language_suite.dists if d.name == simple_name] + distribution = filtered[0] if len(filtered) > 0 else None + if not distribution: + mx.abort(f'Cannot resolve distribution {distribution_name}.') + return distribution + + def _resolve_project(project_name): + project = mx.project(project_name, fatalIfMissing=False) + if not project: + filtered = [p for p in language_suite.projects if p.name == project_name] + project = filtered[0] if len(filtered) > 0 else None + if not project: + mx.abort(f'Cannot resolve project {project_name}.') + return project + + def _qualname(distribution_name): + return language_suite.name + ':' + distribution_name if distribution_name.find(':') < 0 else distribution_name + + language_distribution = _qualname(language_distribution) + if isolate_build_options is None: + isolate_build_options = [] + if additional_isolate_dependencies: + additional_isolate_dependencies = [_qualname(d) for d in additional_isolate_dependencies] + else: + additional_isolate_dependencies = [] + language_id_upper_case = language_id.upper() + resolved_language_distribution = _resolve_distribution(language_distribution) + resolved_internal_resource_project = _resolve_project(internal_resource_project) + + # 1. Register a project building the isolate library + isolate_deps = [language_distribution, 'graal-enterprise:TRUFFLE_ENTERPRISE'] + additional_isolate_dependencies + import mx_sdk_vm_impl + build_library = mx_sdk_vm_impl.PolyglotIsolateLibrary(language_suite, language_id, isolate_deps, isolate_build_options) + register_project(build_library) + + # 2. Register layout distribution with isolate library and isolate resources + resource_base_folder = f'META-INF/resources/engine/{language_id}-isolate///libvm' + attrs = { + 'description': f'Contains {language_id} language library resources.', + 'hashEntry': f'{resource_base_folder}/sha256', + 'fileListEntry': f'{resource_base_folder}/files', + 'maven': False, + } + layout_dist = mx.LayoutDirDistribution( + suite=language_suite, + name=f'{language_id_upper_case}_ISOLATE_LAYOUT', + deps=[], + layout={ + f'{resource_base_folder}/': f'dependency:{build_library.name}', + f'{resource_base_folder}/resources': f'dependency:{build_library.name}/resources', + }, + path=None, + platformDependent=True, + theLicense=None, + platforms=[ + 'linux-amd64', + 'linux-aarch64', + 'darwin-amd64', + 'darwin-aarch64', + 'windows-amd64', + ], + **attrs + ) + register_distribution(layout_dist) + + # 3. Register Jar distribution containing the internal resource project and isolate library. + resources_dist_name = f'{language_id_upper_case}_ISOLATE_RESOURCES' + layout_dist_qualified_name = f'{layout_dist.suite.name}:{layout_dist.name}' + maven_group_id = resolved_language_distribution.maven_group_id() maven_artifact_id = f'{language_id}-isolate' - module_name = f'{get_module_name(language_distribution)}.isolate' + module_name = f'{get_module_name(resolved_language_distribution)}.isolate' licenses = set() - licenses.update(language_distribution.theLicense) - licenses.update(owner_suite.defaultLicense) + licenses.update(language_suite.defaultLicense) + if resolved_language_distribution.theLicense: + licenses.update(resolved_language_distribution.theLicense) + for additional_dep in additional_isolate_dependencies: + resolved_additional_distribution = _resolve_distribution(additional_dep) + if resolved_additional_distribution.theLicense: + licenses.update(resolved_additional_distribution.theLicense) + + # The graal-enterprise suite may not be fully loaded. + # We cannot look up the TRUFFLE_ENTERPRISE distribution to resolve its license + # We pass directly the license id + licenses.update(['GFTC']) attrs = { 'description': f'Polyglot isolate resources for {language_id}.', 'moduleInfo': { @@ -1247,25 +1334,28 @@ def register_polyglot_isolate_distributions(register_distribution, language_id, 'mavenNoSources': True, } isolate_library_dist = mx_jardistribution.JARDistribution( - suite=owner_suite, + suite=language_suite, name=resources_dist_name, - subDir=language_distribution.subDir, + subDir=resolved_language_distribution.subDir, path=None, sourcesPath=None, deps=[ - internal_resource_project.name, + resolved_internal_resource_project.name, layout_dist_qualified_name, ], mainClass=None, excludedLibs=[], distDependencies=['truffle:TRUFFLE_API'], - javaCompliance=str(internal_resource_project.javaCompliance)+'+', + javaCompliance=str(resolved_internal_resource_project.javaCompliance)+'+', platformDependent=True, - theLicense=list(licenses), + theLicense=sorted(list(licenses)), compress=True, **attrs ) register_distribution(isolate_library_dist) + + # 4. Register meta POM distribution for the isolate library jar file. + isolate_dist_name = f'{language_id_upper_case}_ISOLATE' attrs = { 'description': f'The {language_id} polyglot isolate.', 'maven': { @@ -1274,12 +1364,8 @@ def register_polyglot_isolate_distributions(register_distribution, language_id, 'tag': ['default', 'public'], }, } - # The graal-enterprise suite may not be fully loaded. - # We cannot look up the TRUFFLE_ENTERPRISE distribution to resolve its license - # We pass directly the license id - licenses.update(['GFTC']) meta_pom_dist = mx_pomdistribution.POMDistribution( - suite=owner_suite, + suite=language_suite, name=isolate_dist_name, distDependencies=[], runtimeDependencies=[ From 393b27e8295b5e5e2ef9286cdd3560b88835fa2c Mon Sep 17 00:00:00 2001 From: Tomas Zezula Date: Tue, 13 Feb 2024 19:25:53 +0100 Subject: [PATCH 03/13] [GR-48481] Removed the old way of registering polyglot isolates. --- sdk/mx.sdk/mx_sdk_vm.py | 4 +--- sdk/mx.sdk/mx_sdk_vm_impl.py | 25 ------------------------- 2 files changed, 1 insertion(+), 28 deletions(-) diff --git a/sdk/mx.sdk/mx_sdk_vm.py b/sdk/mx.sdk/mx_sdk_vm.py index 0a6e0937ef87..5bf52a43d8c3 100644 --- a/sdk/mx.sdk/mx_sdk_vm.py +++ b/sdk/mx.sdk/mx_sdk_vm.py @@ -205,11 +205,10 @@ def __init__(self, destination, jar_distributions, build_args, jvm_library=False class LanguageLibraryConfig(LibraryConfig): - def __init__(self, jar_distributions, build_args, language, main_class=None, is_sdk_launcher=True, launchers=None, option_vars=None, default_vm_args=None, headers=False, set_default_relative_home_path=True, isolate_library_layout_distribution=None, **kwargs): + def __init__(self, jar_distributions, build_args, language, main_class=None, is_sdk_launcher=True, launchers=None, option_vars=None, default_vm_args=None, headers=False, set_default_relative_home_path=True, **kwargs): """ :param str language :param str main_class - :param isolate_library_layout_distribution dict """ kwargs.pop('destination', None) super(LanguageLibraryConfig, self).__init__('lib/', jar_distributions, build_args, home_finder=True, headers=headers, **kwargs) @@ -228,7 +227,6 @@ def __init__(self, jar_distributions, build_args, language, main_class=None, is_ if set_default_relative_home_path: # Ensure the language launcher can always find the language home self.add_relative_home_path(language, relpath('.', dirname(self.destination))) - self.isolate_library_layout_distribution = isolate_library_layout_distribution class GraalVmComponent(object): def __init__(self, diff --git a/sdk/mx.sdk/mx_sdk_vm_impl.py b/sdk/mx.sdk/mx_sdk_vm_impl.py index bbad76be2fae..73e06b4fd951 100644 --- a/sdk/mx.sdk/mx_sdk_vm_impl.py +++ b/sdk/mx.sdk/mx_sdk_vm_impl.py @@ -3543,7 +3543,6 @@ def register_main_dist(dist, label): else: with_non_rebuildable_configs = True for library_config in _get_library_configs(component): - library_project = None if with_svm: library_project = GraalVmLibrary(component, GraalVmNativeImage.project_name(library_config), [], library_config) register_project(library_project) @@ -3561,30 +3560,6 @@ def register_main_dist(dist, label): register_project(launcher_project) polyglot_config_project = PolyglotConfig(component, library_config) register_project(polyglot_config_project) - if with_svm and library_config.isolate_library_layout_distribution and not library_project.is_skipped() and has_component('tfle', stage1=True): - # Create a layout distribution with the resulting language library that can be consumed into the - # isolate resources jar distribution, - resource_base_folder = f'META-INF/resources/engine/{library_config.language}-isolate///libvm' - attrs = { - 'description': f'Contains {library_config.language} language library resources', - 'hashEntry': f'{resource_base_folder}/sha256', - 'fileListEntry': f'{resource_base_folder}/files', - 'maven': False, - } - register_distribution(mx.LayoutDirDistribution( - suite=_suite, - name=library_config.isolate_library_layout_distribution['name'], - deps=[], - layout={ - f'{resource_base_folder}/': f'dependency:{library_project.name}', - f'{resource_base_folder}/resources': f'dependency:{library_project.name}/resources', - }, - path=None, - platformDependent=True, - theLicense=None, - platforms=library_config.isolate_library_layout_distribution['platforms'], - **attrs - )) if isinstance(component, mx_sdk.GraalVmLanguage) and component.support_distributions: ni_resources_components = dir_name_to_ni_resources_components.get(component.dir_name) if not ni_resources_components: From 4f4d4d0a1c507b6859273535bc952477c7d7165f Mon Sep 17 00:00:00 2001 From: Tomas Zezula Date: Tue, 20 Feb 2024 13:04:53 +0100 Subject: [PATCH 04/13] [GR-48481] register_polyglot_isolate_distributions takes a meta pom for module-path artifacts. --- truffle/mx.truffle/mx_truffle.py | 45 ++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/truffle/mx.truffle/mx_truffle.py b/truffle/mx.truffle/mx_truffle.py index a64a60b3ad4a..2264898075e7 100644 --- a/truffle/mx.truffle/mx_truffle.py +++ b/truffle/mx.truffle/mx_truffle.py @@ -1205,8 +1205,9 @@ def read_file(path): "Make sure the grammar files are up to date with the generated code. You can regenerate the generated code using mx.") -def register_polyglot_isolate_distributions(language_suite, register_project, register_distribution, language_id, language_distribution, - internal_resource_project, isolate_build_options=None, additional_isolate_dependencies=None): +def register_polyglot_isolate_distributions(language_suite, register_project, register_distribution, language_id, + language_pom_distribution, language_distribution, + internal_resource_project, isolate_build_options=None, platforms=None): """ Creates and registers the polyglot isolate resource distribution and isolate resource meta-POM distribution. The created polyglot isolate resource distribution is named `_ISOLATE_RESOURCES`, inheriting the Maven group ID @@ -1220,18 +1221,23 @@ def register_polyglot_isolate_distributions(language_suite, register_project, re :param register_distribution: A callback to dynamically register the distribution, obtained as a parameter from `mx_register_dynamic_suite_constituents`. :type register_distribution: (mx.Distribution) -> None :param language_id: The language ID. - :param language_distribution: The language distribution used to inherit isolate distribution properties. + :param language_pom_distribution: The language meta pom distribution used to set the image builder module-path + :param language_distribution: The language distribution used to inherit isolate distribution module name and maven coordinates :param internal_resource_project: The internal resource project used for unpacking the polyglot isolate library. :param isolate_build_options: additional options passed to a native image to build the isolate library. - :param additional_isolate_dependencies: additional distributions used to build the isolate library. + :param platforms: supported platforms, defaults to ['linux-amd64', 'linux-aarch64', 'darwin-amd64', 'darwin-aarch64', 'windows-amd64'] """ assert language_suite assert register_project assert register_distribution assert language_id + assert language_pom_distribution assert language_distribution assert internal_resource_project + if mx.get_env('POLYGLOT_ISOLATE_LIBRARY', 'false') != 'true': + return False + def _resolve_distribution(distribution_name): distribution = mx.distribution(distribution_name, fatalIfMissing=False) if not distribution: @@ -1254,19 +1260,24 @@ def _resolve_project(project_name): def _qualname(distribution_name): return language_suite.name + ':' + distribution_name if distribution_name.find(':') < 0 else distribution_name + language_pom_distribution = _qualname(language_pom_distribution) language_distribution = _qualname(language_distribution) if isolate_build_options is None: isolate_build_options = [] - if additional_isolate_dependencies: - additional_isolate_dependencies = [_qualname(d) for d in additional_isolate_dependencies] - else: - additional_isolate_dependencies = [] language_id_upper_case = language_id.upper() resolved_language_distribution = _resolve_distribution(language_distribution) resolved_internal_resource_project = _resolve_project(internal_resource_project) + if platforms is None: + platforms = [ + 'linux-amd64', + 'linux-aarch64', + 'darwin-amd64', + 'darwin-aarch64', + 'windows-amd64', + ] # 1. Register a project building the isolate library - isolate_deps = [language_distribution, 'graal-enterprise:TRUFFLE_ENTERPRISE'] + additional_isolate_dependencies + isolate_deps = [language_pom_distribution, 'graal-enterprise:TRUFFLE_ENTERPRISE'] import mx_sdk_vm_impl build_library = mx_sdk_vm_impl.PolyglotIsolateLibrary(language_suite, language_id, isolate_deps, isolate_build_options) register_project(build_library) @@ -1290,13 +1301,7 @@ def _qualname(distribution_name): path=None, platformDependent=True, theLicense=None, - platforms=[ - 'linux-amd64', - 'linux-aarch64', - 'darwin-amd64', - 'darwin-aarch64', - 'windows-amd64', - ], + platforms=platforms, **attrs ) register_distribution(layout_dist) @@ -1311,10 +1316,9 @@ def _qualname(distribution_name): licenses.update(language_suite.defaultLicense) if resolved_language_distribution.theLicense: licenses.update(resolved_language_distribution.theLicense) - for additional_dep in additional_isolate_dependencies: - resolved_additional_distribution = _resolve_distribution(additional_dep) - if resolved_additional_distribution.theLicense: - licenses.update(resolved_additional_distribution.theLicense) + resolved_language_pom_distribution = _resolve_distribution(language_pom_distribution) + if resolved_language_pom_distribution.theLicense: + licenses.update(resolved_language_pom_distribution.theLicense) # The graal-enterprise suite may not be fully loaded. # We cannot look up the TRUFFLE_ENTERPRISE distribution to resolve its license @@ -1375,6 +1379,7 @@ def _qualname(distribution_name): theLicense=sorted(list(licenses)), **attrs) register_distribution(meta_pom_dist) + return True class LibffiBuilderProject(mx.AbstractNativeProject, mx_native.NativeDependency): # pylint: disable=too-many-ancestors """Project for building libffi from source. From 6650f0f47ebf83dc7b0eb72959a5fbe98b4f908b Mon Sep 17 00:00:00 2001 From: Tomas Zezula Date: Wed, 6 Mar 2024 18:13:17 +0100 Subject: [PATCH 05/13] [GR-48481] Per-platform polyglot isolate distributions. --- truffle/mx.truffle/mx_truffle.py | 302 ++++++++++++------ .../polyglot_isolate_resource.template | 73 +++++ 2 files changed, 274 insertions(+), 101 deletions(-) create mode 100644 truffle/mx.truffle/polyglot_isolate_resource.template diff --git a/truffle/mx.truffle/mx_truffle.py b/truffle/mx.truffle/mx_truffle.py index 2264898075e7..0d6144db42c8 100644 --- a/truffle/mx.truffle/mx_truffle.py +++ b/truffle/mx.truffle/mx_truffle.py @@ -1205,9 +1205,91 @@ def read_file(path): "Make sure the grammar files are up to date with the generated code. You can regenerate the generated code using mx.") +class _PolyglotIsolateResourceProject(mx.JavaProject): + + def __init__(self, language_distribution, language_id, platform): + language_suite = language_distribution.suite + name = f'com.oracle.truffle.isolate.resource.{language_id}.{platform.replace("-", ".")}' + subDir = language_distribution.subDir + javaCompliance = language_distribution.javaCompliance + if not javaCompliance: + javaCompliance = '17+' + theLicense = language_distribution.theLicense + dir = mx.join(language_suite.dir, subDir, name) + deps = ['truffle:TRUFFLE_API'] + super(_PolyglotIsolateResourceProject, self).__init__(language_suite, name, subDir=subDir, srcDirs=[], deps=deps, + javaCompliance=javaCompliance, workingSets='Truffle', d=dir, theLicense=theLicense) + src_gen_dir = self.source_gen_dir() + self.srcDirs.append(src_gen_dir) + mx.ensure_dir_exists(src_gen_dir) + self.declaredAnnotationProcessors = ['truffle:TRUFFLE_DSL_PROCESSOR'] + self.language_id = language_id + self.platform = platform + + + def getBuildTask(self, args): + jdk = mx.get_jdk(self.javaCompliance, tag=mx.DEFAULT_JDK_TAG, purpose='building ' + self.name) + return _PolyglotIsolateResourceBuildTask(args, self, jdk) + + +class _PolyglotIsolateResourceBuildTask(mx.JavaBuildTask): + + def __str__(self): + return f'Generating {self.subject.name} internal resource and compiling it with {self._getCompiler().name()}' + + def needsBuild(self, newestInput): + return True, 'Fixme :-)' + + @staticmethod + def _target_file(root, pkg_name): + target_folder = os.path.join(root, pkg_name.replace('.', os.sep)) + target_file = os.path.join(target_folder, 'PolyglotIsolateResource.java') + return target_file + + @staticmethod + def _generate_internal_resource(root, pkg_name, language_id, platform): + source_file = os.path.join(_suite.mxDir, 'polyglot_isolate_resource.template') + with open(source_file, 'r', encoding='utf-8') as f: + file_content = f.read() + file_content = (file_content.replace('${package}', pkg_name) + .replace('${id}', language_id) + .replace('${platform}', platform)) + target_file = _PolyglotIsolateResourceBuildTask._target_file(root, pkg_name) + mx.ensure_dir_exists(mx.dirname(target_file)) + with open(target_file, 'w', encoding='utf-8') as f: + f.write(file_content) + + def _collect_files(self): + if self._javafiles is not None: + # already collected + return self + # collect project files first, then extend with generated resource + super(_PolyglotIsolateResourceBuildTask, self)._collect_files() + javafiles = self._javafiles + prj = self.subject + gen_src_dir = prj.source_gen_dir() + bin_dir = prj.output_dir() + pkg_name = prj.name + target_file = _PolyglotIsolateResourceBuildTask._target_file(gen_src_dir, pkg_name) + if not target_file in javafiles: + target_class = os.path.join(bin_dir, os.path.relpath(target_file, gen_src_dir)[:-len('.java')] + '.class') + javafiles[target_file] = target_class + return self + + def build(self): + prj = self.subject + gen_src_dir = prj.source_gen_dir() + pkg_name = prj.name + language_id = prj.language_id + platform = prj.platform + _PolyglotIsolateResourceBuildTask._generate_internal_resource(gen_src_dir, pkg_name, language_id, platform) + super(_PolyglotIsolateResourceBuildTask, self).build() + + + def register_polyglot_isolate_distributions(language_suite, register_project, register_distribution, language_id, language_pom_distribution, language_distribution, - internal_resource_project, isolate_build_options=None, platforms=None): + isolate_build_options=None, platforms=None): """ Creates and registers the polyglot isolate resource distribution and isolate resource meta-POM distribution. The created polyglot isolate resource distribution is named `_ISOLATE_RESOURCES`, inheriting the Maven group ID @@ -1223,7 +1305,6 @@ def register_polyglot_isolate_distributions(language_suite, register_project, re :param language_id: The language ID. :param language_pom_distribution: The language meta pom distribution used to set the image builder module-path :param language_distribution: The language distribution used to inherit isolate distribution module name and maven coordinates - :param internal_resource_project: The internal resource project used for unpacking the polyglot isolate library. :param isolate_build_options: additional options passed to a native image to build the isolate library. :param platforms: supported platforms, defaults to ['linux-amd64', 'linux-aarch64', 'darwin-amd64', 'darwin-aarch64', 'windows-amd64'] """ @@ -1233,7 +1314,6 @@ def register_polyglot_isolate_distributions(language_suite, register_project, re assert language_id assert language_pom_distribution assert language_distribution - assert internal_resource_project if mx.get_env('POLYGLOT_ISOLATE_LIBRARY', 'false') != 'true': return False @@ -1248,15 +1328,6 @@ def _resolve_distribution(distribution_name): mx.abort(f'Cannot resolve distribution {distribution_name}.') return distribution - def _resolve_project(project_name): - project = mx.project(project_name, fatalIfMissing=False) - if not project: - filtered = [p for p in language_suite.projects if p.name == project_name] - project = filtered[0] if len(filtered) > 0 else None - if not project: - mx.abort(f'Cannot resolve project {project_name}.') - return project - def _qualname(distribution_name): return language_suite.name + ':' + distribution_name if distribution_name.find(':') < 0 else distribution_name @@ -1266,7 +1337,6 @@ def _qualname(distribution_name): isolate_build_options = [] language_id_upper_case = language_id.upper() resolved_language_distribution = _resolve_distribution(language_distribution) - resolved_internal_resource_project = _resolve_project(internal_resource_project) if platforms is None: platforms = [ 'linux-amd64', @@ -1275,96 +1345,129 @@ def _qualname(distribution_name): 'darwin-aarch64', 'windows-amd64', ] + current_platform = mx_subst.string_substitutions.substitute('-') + if current_platform not in platforms: + mx.abort(f'Current platform {current_platform} is not in supported platforms {", ".join(platforms)}') + + platform_meta_poms = [] + for platform in platforms: + + # 1. Register a project generating and building an internal resource for polyglot isolate library + build_internal_resource = _PolyglotIsolateResourceProject(resolved_language_distribution, language_id, platform) + register_project(build_internal_resource) + resources_dist_dependencies = [ + build_internal_resource.name, + ] - # 1. Register a project building the isolate library - isolate_deps = [language_pom_distribution, 'graal-enterprise:TRUFFLE_ENTERPRISE'] - import mx_sdk_vm_impl - build_library = mx_sdk_vm_impl.PolyglotIsolateLibrary(language_suite, language_id, isolate_deps, isolate_build_options) - register_project(build_library) - - # 2. Register layout distribution with isolate library and isolate resources - resource_base_folder = f'META-INF/resources/engine/{language_id}-isolate///libvm' - attrs = { - 'description': f'Contains {language_id} language library resources.', - 'hashEntry': f'{resource_base_folder}/sha256', - 'fileListEntry': f'{resource_base_folder}/files', - 'maven': False, - } - layout_dist = mx.LayoutDirDistribution( - suite=language_suite, - name=f'{language_id_upper_case}_ISOLATE_LAYOUT', - deps=[], - layout={ - f'{resource_base_folder}/': f'dependency:{build_library.name}', - f'{resource_base_folder}/resources': f'dependency:{build_library.name}/resources', - }, - path=None, - platformDependent=True, - theLicense=None, - platforms=platforms, - **attrs - ) - register_distribution(layout_dist) - - # 3. Register Jar distribution containing the internal resource project and isolate library. - resources_dist_name = f'{language_id_upper_case}_ISOLATE_RESOURCES' - layout_dist_qualified_name = f'{layout_dist.suite.name}:{layout_dist.name}' - maven_group_id = resolved_language_distribution.maven_group_id() - maven_artifact_id = f'{language_id}-isolate' - module_name = f'{get_module_name(resolved_language_distribution)}.isolate' - licenses = set() - licenses.update(language_suite.defaultLicense) - if resolved_language_distribution.theLicense: - licenses.update(resolved_language_distribution.theLicense) - resolved_language_pom_distribution = _resolve_distribution(language_pom_distribution) - if resolved_language_pom_distribution.theLicense: - licenses.update(resolved_language_pom_distribution.theLicense) - - # The graal-enterprise suite may not be fully loaded. - # We cannot look up the TRUFFLE_ENTERPRISE distribution to resolve its license - # We pass directly the license id - licenses.update(['GFTC']) - attrs = { - 'description': f'Polyglot isolate resources for {language_id}.', - 'moduleInfo': { - 'name': module_name, - }, - 'maven': { - 'groupId': maven_group_id, - 'artifactId': maven_artifact_id, - 'tag': ['default', 'public'], - }, - 'mavenNoJavadoc': True, - 'mavenNoSources': True, - } - isolate_library_dist = mx_jardistribution.JARDistribution( - suite=language_suite, - name=resources_dist_name, - subDir=resolved_language_distribution.subDir, - path=None, - sourcesPath=None, - deps=[ - resolved_internal_resource_project.name, - layout_dist_qualified_name, - ], - mainClass=None, - excludedLibs=[], - distDependencies=['truffle:TRUFFLE_API'], - javaCompliance=str(resolved_internal_resource_project.javaCompliance)+'+', - platformDependent=True, - theLicense=sorted(list(licenses)), - compress=True, - **attrs - ) - register_distribution(isolate_library_dist) - - # 4. Register meta POM distribution for the isolate library jar file. + if platform == current_platform: + # 2. Register a project building the isolate library + isolate_deps = [language_pom_distribution, 'graal-enterprise:TRUFFLE_ENTERPRISE'] + import mx_sdk_vm_impl + build_library = mx_sdk_vm_impl.PolyglotIsolateLibrary(language_suite, language_id, isolate_deps, isolate_build_options) + register_project(build_library) + + # 3. Register layout distribution with isolate library and isolate resources + resource_base_folder = f'META-INF/resources/engine/{language_id}-isolate///libvm' + attrs = { + 'description': f'Contains {language_id} language library resources.', + 'hashEntry': f'{resource_base_folder}/sha256', + 'fileListEntry': f'{resource_base_folder}/files', + 'maven': False, + } + layout_dist = mx.LayoutDirDistribution( + suite=language_suite, + name=f'{language_id_upper_case}_ISOLATE_LAYOUT', + deps=[], + layout={ + f'{resource_base_folder}/': f'dependency:{build_library.name}', + f'{resource_base_folder}/resources': f'dependency:{build_library.name}/resources', + }, + path=None, + platformDependent=True, + theLicense=None, + platforms=platforms, + **attrs + ) + register_distribution(layout_dist) + layout_dist_qualified_name = f'{layout_dist.suite.name}:{layout_dist.name}' + resources_dist_dependencies.append(layout_dist_qualified_name) + + # 4. Register Jar distribution containing the internal resource project and isolate library for current platform. + # For other platforms, create a jar distribution with an internal resource only + resources_dist_name = f'{language_id_upper_case}_ISOLATE_RESOURCES_{platform.upper()}' + maven_group_id = resolved_language_distribution.maven_group_id() + maven_artifact_id = f'{language_id}-isolate-{platform}' + licenses = set() + licenses.update(language_suite.defaultLicense) + if resolved_language_distribution.theLicense: + licenses.update(resolved_language_distribution.theLicense) + resolved_language_pom_distribution = _resolve_distribution(language_pom_distribution) + if resolved_language_pom_distribution.theLicense: + licenses.update(resolved_language_pom_distribution.theLicense) + # The graal-enterprise suite may not be fully loaded. + # We cannot look up the TRUFFLE_ENTERPRISE distribution to resolve its license + # We pass directly the license id + licenses.update(['GFTC']) + attrs = { + 'description': f'Polyglot isolate resources for {language_id} for {platform}.', + 'moduleInfo': { + 'name': build_internal_resource.name, + }, + 'maven': { + 'groupId': maven_group_id, + 'artifactId': maven_artifact_id, + 'tag': ['default', 'public'], + }, + 'mavenNoJavadoc': True, + 'mavenNoSources': True, + } + isolate_library_dist = mx_jardistribution.JARDistribution( + suite=language_suite, + name=resources_dist_name, + subDir=resolved_language_distribution.subDir, + path=None, + sourcesPath=None, + deps=resources_dist_dependencies, + mainClass=None, + excludedLibs=[], + distDependencies=['truffle:TRUFFLE_API'], + javaCompliance=str(build_internal_resource.javaCompliance)+'+', + platformDependent=True, + theLicense=sorted(list(licenses)), + compress=True, + **attrs + ) + register_distribution(isolate_library_dist) + + # 5. Register meta POM distribution for the isolate library jar file for a specific platform. + isolate_dist_name = f'{language_id_upper_case}_ISOLATE_{platform.upper()}' + attrs = { + 'description': f'The {language_id} polyglot isolate for {platform}.', + 'maven': { + 'groupId': 'org.graalvm.polyglot', + 'artifactId': maven_artifact_id, + 'tag': ['default', 'public'], + }, + } + meta_pom_dist = mx_pomdistribution.POMDistribution( + suite=language_suite, + name=isolate_dist_name, + distDependencies=[], + runtimeDependencies=[ + resources_dist_name, + 'graal-enterprise:TRUFFLE_ENTERPRISE', + ], + theLicense=sorted(list(licenses)), + **attrs) + register_distribution(meta_pom_dist) + platform_meta_poms.append(meta_pom_dist) + # 6. Register meta POM distribution listing all platform specific meta-POMS. isolate_dist_name = f'{language_id_upper_case}_ISOLATE' attrs = { 'description': f'The {language_id} polyglot isolate.', 'maven': { 'groupId': 'org.graalvm.polyglot', - 'artifactId': maven_artifact_id, + 'artifactId': f'{language_id}-isolate', 'tag': ['default', 'public'], }, } @@ -1372,10 +1475,7 @@ def _qualname(distribution_name): suite=language_suite, name=isolate_dist_name, distDependencies=[], - runtimeDependencies=[ - resources_dist_name, - 'graal-enterprise:TRUFFLE_ENTERPRISE', - ], + runtimeDependencies=[pom.name for pom in platform_meta_poms], theLicense=sorted(list(licenses)), **attrs) register_distribution(meta_pom_dist) diff --git a/truffle/mx.truffle/polyglot_isolate_resource.template b/truffle/mx.truffle/polyglot_isolate_resource.template new file mode 100644 index 000000000000..a764498059a8 --- /dev/null +++ b/truffle/mx.truffle/polyglot_isolate_resource.template @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The Universal Permissive License (UPL), Version 1.0 + * + * Subject to the condition set forth below, permission is hereby granted to any + * person obtaining a copy of this software, associated documentation and/or + * data (collectively the "Software"), free of charge and under any and all + * copyright rights in the Software, and any and all patent rights owned or + * freely licensable by each licensor hereunder covering either (i) the + * unmodified Software as contributed to or provided by such licensor, or (ii) + * the Larger Works (as defined below), to deal in both + * + * (a) the Software, and + * + * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if + * one is included with the Software each a "Larger Work" to which the Software + * is contributed by such licensors), + * + * without restriction, including without limitation the rights to copy, create + * derivative works of, display, perform, and distribute the Software and make, + * use, sell, offer for sale, import, export, have made, and have sold the + * Software and the Larger Work(s), and to sublicense the foregoing rights on + * either these or other terms. + * + * This license is subject to the following condition: + * + * The above copyright notice and either this complete permission notice or at a + * minimum a reference to the UPL must be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package ${package}; + +import com.oracle.truffle.api.CompilerDirectives; +import com.oracle.truffle.api.InternalResource; + +import java.io.IOException; +import java.nio.file.Path; + +@InternalResource.Id(value = PolyglotIsolateResource.ID, componentId = "engine", optional = true) +public final class PolyglotIsolateResource implements InternalResource { + + static final String ID = "${id}-isolate-${platform}"; + + @Override + public void unpackFiles(Env env, Path targetDirectory) throws IOException { + Path base = basePath(env); + env.unpackResourceFiles(base.resolve("files"), targetDirectory, base); + } + + @Override + public String versionHash(Env env) { + try { + Path base = basePath(env); + return env.readResourceLines(base.resolve("sha256")).get(0); + } catch (IOException ioe) { + throw CompilerDirectives.shouldNotReachHere(ioe); + } + } + + private static Path basePath(Env env) { + return Path.of("META-INF", "resources", "engine", ID, env.getOS().toString(), env.getCPUArchitecture().toString(), "libvm"); + } +} From dbdcc1180f161cd80a75a6f3adf0245a47d5651d Mon Sep 17 00:00:00 2001 From: Tomas Zezula Date: Wed, 13 Mar 2024 13:18:17 +0100 Subject: [PATCH 06/13] [GR-48481] PolyglotIsolateResource is generated by mx. --- truffle/mx.truffle/mx_truffle.py | 133 ++++++++++-------- .../polyglot_isolate_resource.template | 4 +- ...polyglot_isolate_resource_invalid.template | 70 +++++++++ 3 files changed, 143 insertions(+), 64 deletions(-) create mode 100644 truffle/mx.truffle/polyglot_isolate_resource_invalid.template diff --git a/truffle/mx.truffle/mx_truffle.py b/truffle/mx.truffle/mx_truffle.py index 0d6144db42c8..c52aec2c01da 100644 --- a/truffle/mx.truffle/mx_truffle.py +++ b/truffle/mx.truffle/mx_truffle.py @@ -1207,24 +1207,24 @@ def read_file(path): class _PolyglotIsolateResourceProject(mx.JavaProject): - def __init__(self, language_distribution, language_id, platform): - language_suite = language_distribution.suite - name = f'com.oracle.truffle.isolate.resource.{language_id}.{platform.replace("-", ".")}' - subDir = language_distribution.subDir - javaCompliance = language_distribution.javaCompliance - if not javaCompliance: - javaCompliance = '17+' - theLicense = language_distribution.theLicense + def __init__(self, language_suite, subDir, language_id, resource_id, os_name, cpu_architecture, placeholder): + name = f'com.oracle.truffle.isolate.resource.{language_id}.{os_name}.{cpu_architecture}' + javaCompliance = str(mx.distribution('truffle:TRUFFLE_API').javaCompliance) + '+' dir = mx.join(language_suite.dir, subDir, name) deps = ['truffle:TRUFFLE_API'] + if placeholder: + deps += ['sdk:NATIVEIMAGE'] super(_PolyglotIsolateResourceProject, self).__init__(language_suite, name, subDir=subDir, srcDirs=[], deps=deps, - javaCompliance=javaCompliance, workingSets='Truffle', d=dir, theLicense=theLicense) + javaCompliance=javaCompliance, workingSets='Truffle', d=dir, + theLicense=_suite.defaultLicense) src_gen_dir = self.source_gen_dir() self.srcDirs.append(src_gen_dir) - mx.ensure_dir_exists(src_gen_dir) self.declaredAnnotationProcessors = ['truffle:TRUFFLE_DSL_PROCESSOR'] self.language_id = language_id - self.platform = platform + self.resource_id = resource_id + self.os_name = os_name + self.cpu_architecture = cpu_architecture + self.placeholder = placeholder def getBuildTask(self, args): @@ -1237,8 +1237,27 @@ class _PolyglotIsolateResourceBuildTask(mx.JavaBuildTask): def __str__(self): return f'Generating {self.subject.name} internal resource and compiling it with {self._getCompiler().name()}' + @staticmethod + def _template_file(placeholder): + name = 'polyglot_isolate_resource_invalid.template' if placeholder else 'polyglot_isolate_resource.template' + return os.path.join(_suite.mxDir, name) + def needsBuild(self, newestInput): - return True, 'Fixme :-)' + is_needed, reason = mx.ProjectBuildTask.needsBuild(self, newestInput) + if is_needed: + return True, reason + proj = self.subject + for outDir in [proj.output_dir(), proj.source_gen_dir()]: + if not os.path.exists(outDir): + return True, f"{outDir} does not exist" + template_ts = mx.TimeStampFile.newest([ + _PolyglotIsolateResourceBuildTask._template_file(False), + _PolyglotIsolateResourceBuildTask._template_file(True), + __file__ + ]) + if newestInput is None or newestInput.isOlderThan(template_ts): + newestInput = template_ts + return super().needsBuild(newestInput) @staticmethod def _target_file(root, pkg_name): @@ -1246,18 +1265,6 @@ def _target_file(root, pkg_name): target_file = os.path.join(target_folder, 'PolyglotIsolateResource.java') return target_file - @staticmethod - def _generate_internal_resource(root, pkg_name, language_id, platform): - source_file = os.path.join(_suite.mxDir, 'polyglot_isolate_resource.template') - with open(source_file, 'r', encoding='utf-8') as f: - file_content = f.read() - file_content = (file_content.replace('${package}', pkg_name) - .replace('${id}', language_id) - .replace('${platform}', platform)) - target_file = _PolyglotIsolateResourceBuildTask._target_file(root, pkg_name) - mx.ensure_dir_exists(mx.dirname(target_file)) - with open(target_file, 'w', encoding='utf-8') as f: - f.write(file_content) def _collect_files(self): if self._javafiles is not None: @@ -1268,27 +1275,37 @@ def _collect_files(self): javafiles = self._javafiles prj = self.subject gen_src_dir = prj.source_gen_dir() - bin_dir = prj.output_dir() pkg_name = prj.name target_file = _PolyglotIsolateResourceBuildTask._target_file(gen_src_dir, pkg_name) if not target_file in javafiles: + bin_dir = prj.output_dir() target_class = os.path.join(bin_dir, os.path.relpath(target_file, gen_src_dir)[:-len('.java')] + '.class') javafiles[target_file] = target_class + # Remove annotation processor generated files. + javafiles = {k: v for k, v in javafiles.items() if k == target_file} + self._javafiles = javafiles return self def build(self): prj = self.subject - gen_src_dir = prj.source_gen_dir() pkg_name = prj.name - language_id = prj.language_id - platform = prj.platform - _PolyglotIsolateResourceBuildTask._generate_internal_resource(gen_src_dir, pkg_name, language_id, platform) + with open(_PolyglotIsolateResourceBuildTask._template_file(prj.placeholder), 'r', encoding='utf-8') as f: + file_content = f.read() + file_content = (file_content.replace('${package}', pkg_name) + .replace('${languageId}', prj.language_id) + .replace('${resourceId}', prj.resource_id) + .replace('${os}', prj.os_name) + .replace('${arch}', prj.cpu_architecture)) + target_file = _PolyglotIsolateResourceBuildTask._target_file(prj.source_gen_dir(), pkg_name) + mx.ensure_dir_exists(mx.dirname(target_file)) + with mx_util.SafeFileCreation(target_file) as sfc, open(sfc.tmpPath, 'w', encoding='utf-8') as f: + f.write(file_content) super(_PolyglotIsolateResourceBuildTask, self).build() def register_polyglot_isolate_distributions(language_suite, register_project, register_distribution, language_id, - language_pom_distribution, language_distribution, + subDir, language_pom_distribution, maven_group_id, language_license, isolate_build_options=None, platforms=None): """ Creates and registers the polyglot isolate resource distribution and isolate resource meta-POM distribution. @@ -1297,46 +1314,42 @@ def register_polyglot_isolate_distributions(language_suite, register_project, re The meta-POM distribution is named `_ISOLATE`, having the Maven group ID `org.graalvm.polyglot`, and the Maven artifact ID is `-isolate`. - :param language_suite: The language suite used to register generated projects and distributions to. + :param Suite language_suite: The language suite used to register generated projects and distributions to. :param register_project: A callback to dynamically register the project, obtained as a parameter from `mx_register_dynamic_suite_constituents`. :type register_project: (mx.Project) -> None :param register_distribution: A callback to dynamically register the distribution, obtained as a parameter from `mx_register_dynamic_suite_constituents`. :type register_distribution: (mx.Distribution) -> None - :param language_id: The language ID. - :param language_pom_distribution: The language meta pom distribution used to set the image builder module-path - :param language_distribution: The language distribution used to inherit isolate distribution module name and maven coordinates - :param isolate_build_options: additional options passed to a native image to build the isolate library. - :param platforms: supported platforms, defaults to ['linux-amd64', 'linux-aarch64', 'darwin-amd64', 'darwin-aarch64', 'windows-amd64'] + :param str language_id: The language ID. + :param str subDir: a path relative to `suite.dir` in which the IDE project configuration for this distribution is generated + :param POMDistribution language_pom_distribution: The language meta pom distribution used to set the image builder module-path. + :param str maven_group_id: The maven language group id. + :param str | list | language_license: Language licence(s). + :param list isolate_build_options: additional options passed to a native image to build the isolate library. + :param list platforms: supported platforms, defaults to ['linux-amd64', 'linux-aarch64', 'darwin-amd64', 'darwin-aarch64', 'windows-amd64'] """ assert language_suite assert register_project assert register_distribution assert language_id + assert subDir assert language_pom_distribution - assert language_distribution + assert maven_group_id + assert language_license if mx.get_env('POLYGLOT_ISOLATE_LIBRARY', 'false') != 'true': return False - def _resolve_distribution(distribution_name): - distribution = mx.distribution(distribution_name, fatalIfMissing=False) - if not distribution: - simple_name = mx.splitqualname(distribution_name)[1] - filtered = [d for d in language_suite.dists if d.name == simple_name] - distribution = filtered[0] if len(filtered) > 0 else None - if not distribution: - mx.abort(f'Cannot resolve distribution {distribution_name}.') - return distribution + if type(language_license) is not list: + assert type(language_license) is str + language_license = [language_license] def _qualname(distribution_name): return language_suite.name + ':' + distribution_name if distribution_name.find(':') < 0 else distribution_name language_pom_distribution = _qualname(language_pom_distribution) - language_distribution = _qualname(language_distribution) if isolate_build_options is None: isolate_build_options = [] language_id_upper_case = language_id.upper() - resolved_language_distribution = _resolve_distribution(language_distribution) if platforms is None: platforms = [ 'linux-amd64', @@ -1351,15 +1364,18 @@ def _qualname(distribution_name): platform_meta_poms = [] for platform in platforms: - + build_for_current_platform = platform == current_platform + resource_id = f'{language_id}-isolate-{platform}' + os_name, cpu_architecture = platform.split('-') # 1. Register a project generating and building an internal resource for polyglot isolate library - build_internal_resource = _PolyglotIsolateResourceProject(resolved_language_distribution, language_id, platform) + build_internal_resource = _PolyglotIsolateResourceProject(language_suite, subDir, language_id, resource_id, + os_name, cpu_architecture, not build_for_current_platform) register_project(build_internal_resource) resources_dist_dependencies = [ build_internal_resource.name, ] - if platform == current_platform: + if build_for_current_platform: # 2. Register a project building the isolate library isolate_deps = [language_pom_distribution, 'graal-enterprise:TRUFFLE_ENTERPRISE'] import mx_sdk_vm_impl @@ -1367,7 +1383,7 @@ def _qualname(distribution_name): register_project(build_library) # 3. Register layout distribution with isolate library and isolate resources - resource_base_folder = f'META-INF/resources/engine/{language_id}-isolate///libvm' + resource_base_folder = f'META-INF/resources/engine/{resource_id}/libvm' attrs = { 'description': f'Contains {language_id} language library resources.', 'hashEntry': f'{resource_base_folder}/sha256', @@ -1395,15 +1411,8 @@ def _qualname(distribution_name): # 4. Register Jar distribution containing the internal resource project and isolate library for current platform. # For other platforms, create a jar distribution with an internal resource only resources_dist_name = f'{language_id_upper_case}_ISOLATE_RESOURCES_{platform.upper()}' - maven_group_id = resolved_language_distribution.maven_group_id() - maven_artifact_id = f'{language_id}-isolate-{platform}' - licenses = set() - licenses.update(language_suite.defaultLicense) - if resolved_language_distribution.theLicense: - licenses.update(resolved_language_distribution.theLicense) - resolved_language_pom_distribution = _resolve_distribution(language_pom_distribution) - if resolved_language_pom_distribution.theLicense: - licenses.update(resolved_language_pom_distribution.theLicense) + maven_artifact_id = resource_id + licenses = set(language_license) # The graal-enterprise suite may not be fully loaded. # We cannot look up the TRUFFLE_ENTERPRISE distribution to resolve its license # We pass directly the license id @@ -1424,7 +1433,7 @@ def _qualname(distribution_name): isolate_library_dist = mx_jardistribution.JARDistribution( suite=language_suite, name=resources_dist_name, - subDir=resolved_language_distribution.subDir, + subDir=subDir, path=None, sourcesPath=None, deps=resources_dist_dependencies, diff --git a/truffle/mx.truffle/polyglot_isolate_resource.template b/truffle/mx.truffle/polyglot_isolate_resource.template index a764498059a8..f03b955f609b 100644 --- a/truffle/mx.truffle/polyglot_isolate_resource.template +++ b/truffle/mx.truffle/polyglot_isolate_resource.template @@ -49,7 +49,7 @@ import java.nio.file.Path; @InternalResource.Id(value = PolyglotIsolateResource.ID, componentId = "engine", optional = true) public final class PolyglotIsolateResource implements InternalResource { - static final String ID = "${id}-isolate-${platform}"; + static final String ID = "${resourceId}"; @Override public void unpackFiles(Env env, Path targetDirectory) throws IOException { @@ -68,6 +68,6 @@ public final class PolyglotIsolateResource implements InternalResource { } private static Path basePath(Env env) { - return Path.of("META-INF", "resources", "engine", ID, env.getOS().toString(), env.getCPUArchitecture().toString(), "libvm"); + return Path.of("META-INF", "resources", "engine", ID, "libvm"); } } diff --git a/truffle/mx.truffle/polyglot_isolate_resource_invalid.template b/truffle/mx.truffle/polyglot_isolate_resource_invalid.template new file mode 100644 index 000000000000..f330924841e5 --- /dev/null +++ b/truffle/mx.truffle/polyglot_isolate_resource_invalid.template @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The Universal Permissive License (UPL), Version 1.0 + * + * Subject to the condition set forth below, permission is hereby granted to any + * person obtaining a copy of this software, associated documentation and/or + * data (collectively the "Software"), free of charge and under any and all + * copyright rights in the Software, and any and all patent rights owned or + * freely licensable by each licensor hereunder covering either (i) the + * unmodified Software as contributed to or provided by such licensor, or (ii) + * the Larger Works (as defined below), to deal in both + * + * (a) the Software, and + * + * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if + * one is included with the Software each a "Larger Work" to which the Software + * is contributed by such licensors), + * + * without restriction, including without limitation the rights to copy, create + * derivative works of, display, perform, and distribute the Software and make, + * use, sell, offer for sale, import, export, have made, and have sold the + * Software and the Larger Work(s), and to sublicense the foregoing rights on + * either these or other terms. + * + * This license is subject to the following condition: + * + * The above copyright notice and either this complete permission notice or at a + * minimum a reference to the UPL must be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package ${package}; + +import com.oracle.truffle.api.CompilerDirectives; +import com.oracle.truffle.api.InternalResource; + +import java.io.IOException; +import java.nio.file.Path; +import org.graalvm.nativeimage.ImageInfo; + +@InternalResource.Id(value = PolyglotIsolateResource.ID, componentId = "engine", optional = true) +public final class PolyglotIsolateResource implements InternalResource { + + static final String ID = "${resourceId}"; + + @Override + public void unpackFiles(Env env, Path targetDirectory) throws IOException { + if (!ImageInfo.inImageBuildtimeCode()) { + throw shouldNotReachHere(); + } + } + + @Override + public String versionHash(Env env) { + throw shouldNotReachHere(); + } + + private static RuntimeException shouldNotReachHere() { + throw CompilerDirectives.shouldNotReachHere("The polyglot isolate library was not built for the current platform."); + } +} From f4f209d05e3719fb7c996ea951d66f8e435a7b48 Mon Sep 17 00:00:00 2001 From: Tomas Zezula Date: Wed, 27 Mar 2024 19:15:42 +0100 Subject: [PATCH 07/13] [GR-48481] Resolved review comments. --- sdk/mx.sdk/mx_sdk_vm_impl.py | 19 +++++++++++++++++-- truffle/mx.truffle/mx_truffle.py | 11 +++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/sdk/mx.sdk/mx_sdk_vm_impl.py b/sdk/mx.sdk/mx_sdk_vm_impl.py index 73e06b4fd951..abd27e516e0b 100644 --- a/sdk/mx.sdk/mx_sdk_vm_impl.py +++ b/sdk/mx.sdk/mx_sdk_vm_impl.py @@ -1992,6 +1992,13 @@ def is_skipped(self): return _skip_libraries(self.native_image_config) class PolyglotIsolateLibrary(GraalVmLibrary): + """ + A native image project dedicated to constructing a language polyglot isolate library. + Despite being built upon the component supertype, it operates independently of the component system + and native-image macros. Its configuration relies solely on the module path and META-INF/native-image + configuration files. Instances are instantiated by mx_truffle::register_polyglot_isolate_distributions + when a language dynamically registers a polyglot isolate distribution. + """ def __init__(self, target_suite, language, deps, build_args, **kw_args): library_config = mx_sdk.LanguageLibraryConfig( jar_distributions=deps, @@ -2475,8 +2482,15 @@ class GraalVmLibraryBuildTask(GraalVmSVMNativeImageBuildTask): class PolyglotIsolateLibraryBuildTask(GraalVmLibraryBuildTask): + """ + A PolyglotIsolateLibrary build task building a language polyglot isolate library. + Despite being built upon the component supertype, it operates independently of the component system + and native-image macros. Its configuration relies solely on the module path and META-INF/native-image + configuration files. + """ def get_build_args(self): - target = self.subject.native_image_name[:-len(_lib_suffix)] + project = self.subject + target = project.native_image_name[:-len(_lib_suffix)] build_args = [ '-EJVMCI_VERSION_CHECK', # Propagate this env var when running native image from mx '--parallelism=' + str(self.parallelism), @@ -2488,7 +2502,8 @@ def get_build_args(self): ] + svm_experimental_options([ '-H:+BuildOutputPrefix', '-H:+GenerateBuildArtifactsFile', # generate 'build-artifacts.json' - ]) + mx.get_runtime_jvm_args(self.subject.native_image_jar_distributions) + ]) + mx.get_runtime_jvm_args(self.subject.native_image_jar_distributions) + \ + project.native_image_config.build_args + project.native_image_config.build_args_enterprise return build_args diff --git a/truffle/mx.truffle/mx_truffle.py b/truffle/mx.truffle/mx_truffle.py index c52aec2c01da..dde8d4cebbf3 100644 --- a/truffle/mx.truffle/mx_truffle.py +++ b/truffle/mx.truffle/mx_truffle.py @@ -1206,6 +1206,13 @@ def read_file(path): class _PolyglotIsolateResourceProject(mx.JavaProject): + """ + A Java project creating an internal resource implementation unpacking the polyglot isolate library. + The Java source code for internal resources is generated into the project's `source_gen_dir` before compilation, + using the `mx.truffle/polyglot_isolate_resource.template`. Configuration of the project follows these conventions: + The target package name is `com.oracle.truffle.isolate.resource...`, + and the resource ID is `-isolate--`. + """ def __init__(self, language_suite, subDir, language_id, resource_id, os_name, cpu_architecture, placeholder): name = f'com.oracle.truffle.isolate.resource.{language_id}.{os_name}.{cpu_architecture}' @@ -1233,6 +1240,10 @@ def getBuildTask(self, args): class _PolyglotIsolateResourceBuildTask(mx.JavaBuildTask): + """ + A _PolyglotIsolateResourceProject build task generating and building the internal resource unpacking + the polyglot isolate library. Refer to `_PolyglotIsolateResourceProject` documentation for more details. + """ def __str__(self): return f'Generating {self.subject.name} internal resource and compiling it with {self._getCompiler().name()}' From b5ee92e3ac22639f9371df2acd95c273e348d1fa Mon Sep 17 00:00:00 2001 From: Tomas Zezula Date: Mon, 8 Apr 2024 13:04:35 +0200 Subject: [PATCH 08/13] [GR-48481] Fixed gates. --- sdk/mx.sdk/mx_sdk_vm_impl.py | 2 +- truffle/mx.truffle/mx_truffle.py | 24 +++++++++++++----------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/sdk/mx.sdk/mx_sdk_vm_impl.py b/sdk/mx.sdk/mx_sdk_vm_impl.py index abd27e516e0b..a20e100af88b 100644 --- a/sdk/mx.sdk/mx_sdk_vm_impl.py +++ b/sdk/mx.sdk/mx_sdk_vm_impl.py @@ -2006,7 +2006,7 @@ def __init__(self, target_suite, language, deps, build_args, **kw_args): build_args_enterprise=build_args, language=language, ) - super(PolyglotIsolateLibrary, self).__init__(None, os.path.basename(library_config.destination) + ".image", + super(PolyglotIsolateLibrary, self).__init__(None, f'{language}.isolate.image', list(deps), library_config, **kw_args) self.suite = target_suite self.dir = target_suite.dir diff --git a/truffle/mx.truffle/mx_truffle.py b/truffle/mx.truffle/mx_truffle.py index dde8d4cebbf3..b1dbe5a519bd 100644 --- a/truffle/mx.truffle/mx_truffle.py +++ b/truffle/mx.truffle/mx_truffle.py @@ -48,7 +48,7 @@ import difflib import zipfile from argparse import Action, ArgumentParser, RawDescriptionHelpFormatter -from os.path import exists, isdir, join, abspath +from os.path import dirname, exists, isdir, join, abspath from urllib.parse import urljoin # pylint: disable=unused-import,no-name-in-module import mx @@ -1217,12 +1217,12 @@ class _PolyglotIsolateResourceProject(mx.JavaProject): def __init__(self, language_suite, subDir, language_id, resource_id, os_name, cpu_architecture, placeholder): name = f'com.oracle.truffle.isolate.resource.{language_id}.{os_name}.{cpu_architecture}' javaCompliance = str(mx.distribution('truffle:TRUFFLE_API').javaCompliance) + '+' - dir = mx.join(language_suite.dir, subDir, name) + project_dir = os.path.join(language_suite.dir, subDir, name) deps = ['truffle:TRUFFLE_API'] if placeholder: deps += ['sdk:NATIVEIMAGE'] super(_PolyglotIsolateResourceProject, self).__init__(language_suite, name, subDir=subDir, srcDirs=[], deps=deps, - javaCompliance=javaCompliance, workingSets='Truffle', d=dir, + javaCompliance=javaCompliance, workingSets='Truffle', d=project_dir, theLicense=_suite.defaultLicense) src_gen_dir = self.source_gen_dir() self.srcDirs.append(src_gen_dir) @@ -1308,7 +1308,7 @@ def build(self): .replace('${os}', prj.os_name) .replace('${arch}', prj.cpu_architecture)) target_file = _PolyglotIsolateResourceBuildTask._target_file(prj.source_gen_dir(), pkg_name) - mx.ensure_dir_exists(mx.dirname(target_file)) + mx_util.ensure_dir_exists(dirname(target_file)) with mx_util.SafeFileCreation(target_file) as sfc, open(sfc.tmpPath, 'w', encoding='utf-8') as f: f.write(file_content) super(_PolyglotIsolateResourceBuildTask, self).build() @@ -1350,8 +1350,8 @@ def register_polyglot_isolate_distributions(language_suite, register_project, re if mx.get_env('POLYGLOT_ISOLATE_LIBRARY', 'false') != 'true': return False - if type(language_license) is not list: - assert type(language_license) is str + if not isinstance(language_license, list): + assert isinstance(language_license, str) language_license = [language_license] def _qualname(distribution_name): @@ -1378,6 +1378,8 @@ def _qualname(distribution_name): build_for_current_platform = platform == current_platform resource_id = f'{language_id}-isolate-{platform}' os_name, cpu_architecture = platform.split('-') + os_name_upper_case = os_name.upper() + cpu_architecture_upper_case = cpu_architecture.upper() # 1. Register a project generating and building an internal resource for polyglot isolate library build_internal_resource = _PolyglotIsolateResourceProject(language_suite, subDir, language_id, resource_id, os_name, cpu_architecture, not build_for_current_platform) @@ -1403,7 +1405,7 @@ def _qualname(distribution_name): } layout_dist = mx.LayoutDirDistribution( suite=language_suite, - name=f'{language_id_upper_case}_ISOLATE_LAYOUT', + name=f'{language_id_upper_case}_ISOLATE_LAYOUT_{os_name_upper_case}_{cpu_architecture_upper_case}', deps=[], layout={ f'{resource_base_folder}/': f'dependency:{build_library.name}', @@ -1412,7 +1414,7 @@ def _qualname(distribution_name): path=None, platformDependent=True, theLicense=None, - platforms=platforms, + platforms=[platform], **attrs ) register_distribution(layout_dist) @@ -1421,7 +1423,7 @@ def _qualname(distribution_name): # 4. Register Jar distribution containing the internal resource project and isolate library for current platform. # For other platforms, create a jar distribution with an internal resource only - resources_dist_name = f'{language_id_upper_case}_ISOLATE_RESOURCES_{platform.upper()}' + resources_dist_name = f'{language_id_upper_case}_ISOLATE_RESOURCES_{os_name_upper_case}_{cpu_architecture_upper_case}' maven_artifact_id = resource_id licenses = set(language_license) # The graal-enterprise suite may not be fully loaded. @@ -1460,7 +1462,7 @@ def _qualname(distribution_name): register_distribution(isolate_library_dist) # 5. Register meta POM distribution for the isolate library jar file for a specific platform. - isolate_dist_name = f'{language_id_upper_case}_ISOLATE_{platform.upper()}' + isolate_dist_name = f'{language_id_upper_case}_ISOLATE_{os_name_upper_case}_{cpu_architecture_upper_case}' attrs = { 'description': f'The {language_id} polyglot isolate for {platform}.', 'maven': { @@ -1547,7 +1549,7 @@ class LibtoolNativeProject(mx.NativeProject, # pylint: disable=too-many-ancesto def getArchivableResults(self, use_relpath=True, single=False): for file_path, archive_path in super(LibtoolNativeProject, self).getArchivableResults(use_relpath): - path_in_lt_objdir = os.path.basename(os.path.dirname(file_path)) == '.libs' + path_in_lt_objdir = os.path.basename(dirname(file_path)) == '.libs' yield file_path, os.path.basename(archive_path) if path_in_lt_objdir else archive_path if single: assert path_in_lt_objdir, 'the first build result must be from LT_OBJDIR' From 42f9f2d4edea38e5d7ee067e06d9ade42bae511c Mon Sep 17 00:00:00 2001 From: Tomas Zezula Date: Thu, 11 Apr 2024 16:09:25 +0200 Subject: [PATCH 09/13] [GR-48481] Polyglot isolates are enabled by command line option. --- truffle/mx.truffle/mx_truffle.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/truffle/mx.truffle/mx_truffle.py b/truffle/mx.truffle/mx_truffle.py index b1dbe5a519bd..1e18fc49f9d1 100644 --- a/truffle/mx.truffle/mx_truffle.py +++ b/truffle/mx.truffle/mx_truffle.py @@ -1347,7 +1347,8 @@ def register_polyglot_isolate_distributions(language_suite, register_project, re assert maven_group_id assert language_license - if mx.get_env('POLYGLOT_ISOLATE_LIBRARY', 'false') != 'true': + polyglot_isolates_value = mx.get_opts().polyglot_isolates + if not polyglot_isolates_value or (polyglot_isolates_value != 'true' and language_id not in polyglot_isolates_value.split(',')): return False if not isinstance(language_license, list): @@ -1503,6 +1504,10 @@ def _qualname(distribution_name): register_distribution(meta_pom_dist) return True + +mx.add_argument('--polyglot-isolates', action='store', help='Comma-separated list of languages for which the polyglot isolate library should be built. Setting the value to `true` builds all polyglot isolate libraries.') + + class LibffiBuilderProject(mx.AbstractNativeProject, mx_native.NativeDependency): # pylint: disable=too-many-ancestors """Project for building libffi from source. From 4d62e3c88c210805e792d62a18ce14e4a46cea0e Mon Sep 17 00:00:00 2001 From: Tomas Zezula Date: Thu, 11 Apr 2024 18:15:34 +0200 Subject: [PATCH 10/13] [GR-48481] Enabled polyglot isolate build in deploy-vm-maven jobs. --- vm/ci/ci_common/common.jsonnet | 43 +++++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/vm/ci/ci_common/common.jsonnet b/vm/ci/ci_common/common.jsonnet index 53f81142934f..09348468565a 100644 --- a/vm/ci/ci_common/common.jsonnet +++ b/vm/ci/ci_common/common.jsonnet @@ -243,6 +243,27 @@ local devkits = graal_common.devkits; local ce_licenses = legacy_licenses + ',PSF-License,BSD-simplified,BSD-new,EPL-2.0'; ce_licenses, + platform_specific_distribution_name(base_name, os, arch):: + base_name + '_' + std.asciiUpper(os) + '_' + std.asciiUpper(arch), + + language_polyglot_isolate_distributions(language_id, current_os, current_arch, current_only=false):: + local id_upcase = std.asciiUpper(language_id); + local base_names = [id_upcase + '_ISOLATE', id_upcase + '_ISOLATE_RESOURCES']; + local oss = ['linux', 'darwin', 'windows']; + local archs = ['amd64', 'aarch64']; + [base_names[0]] + [self.platform_specific_distribution_name(base_name, os, arch), + for base_name in base_names + for os in oss for arch in archs + if os != 'windows' || arch != 'aarch64' + if !current_only || os == current_os && arch == current_arch + ], + + polyglot_isolate_distributions(language_ids, current_os, current_arch, current_only=false):: + std.flattenArrays([self.language_polyglot_isolate_distributions(id, current_os, current_arch, current_only) for id in language_ids]), + + local polyglot_isolate_languages = ['js', 'python'], + + legacy_mx_args:: ['--disable-installables=true'], # `['--force-bash-launcher=true', '--skip-libraries=true']` have been replaced by arguments from `vm.maven_deploy_base_functions.mx_args(os, arch)` mx_args(os, arch, reduced):: self.legacy_mx_args + vm.maven_deploy_base_functions.mx_args(os, arch, reduced), mx_cmd_base(os, arch, reduced):: ['mx'] + vm.maven_deploy_base_functions.dynamic_imports(os, arch) + self.mx_args(os, arch, reduced), @@ -300,8 +321,8 @@ local devkits = graal_common.devkits; if (vm.maven_deploy_base_functions.edition == 'ce') then self.deploy_ce(os, arch, false, dry_run, [remote_mvn_repo]) else - self.deploy_ee(os, arch, false, dry_run, ['--dummy-javadoc', remote_mvn_repo]) - + self.deploy_ee(os, arch, false, dry_run, ['--dummy-javadoc', '--only', 'JS_ISOLATE,JS_ISOLATE_RESOURCES,TOOLS_COMMUNITY,LANGUAGES_COMMUNITY', remote_mvn_repo], extra_mx_args=['--suite', 'graal-js']); + self.deploy_ee(os, arch, false, dry_run, ['--dummy-javadoc', '--skip', std.join(',', self.polyglot_isolate_distributions(polyglot_isolate_languages, os, arch)) + ',TOOLS_COMMUNITY,LANGUAGES_COMMUNITY', remote_mvn_repo]) + + self.deploy_ee(os, arch, false, dry_run, ['--dummy-javadoc', '--only', std.join(',', self.polyglot_isolate_distributions(polyglot_isolate_languages, os, arch, true)) + ',TOOLS_COMMUNITY,LANGUAGES_COMMUNITY', remote_mvn_repo], extra_mx_args=['--suite', 'graal-js', '--suite', 'graalpython-enterprise']); local mvn_bundle_snippet = [ @@ -315,8 +336,8 @@ local devkits = graal_common.devkits; if (vm.maven_deploy_base_functions.edition == 'ce') then self.deploy_ce(os, arch, false, dry_run, [local_repo, '${LOCAL_MAVEN_REPO_URL}']) else - self.deploy_ce(os, arch, false, dry_run, ['--dummy-javadoc', '--skip', 'JS_ISOLATE,JS_ISOLATE_RESOURCES,TOOLS_COMMUNITY,LANGUAGES_COMMUNITY', local_repo, '${LOCAL_MAVEN_REPO_URL}']) - + self.deploy_ee(os, arch, false, dry_run, ['--dummy-javadoc', '--only', 'JS_ISOLATE,JS_ISOLATE_RESOURCES,TOOLS_COMMUNITY,LANGUAGES_COMMUNITY', local_repo, '${LOCAL_MAVEN_REPO_URL}'], extra_mx_args=['--suite', 'graal-js']) + self.deploy_ce(os, arch, false, dry_run, ['--dummy-javadoc', '--skip', std.join(',', self.polyglot_isolate_distributions(polyglot_isolate_languages, os, arch)) + ',TOOLS_COMMUNITY,LANGUAGES_COMMUNITY', local_repo, '${LOCAL_MAVEN_REPO_URL}']) + + self.deploy_ee(os, arch, false, dry_run, ['--dummy-javadoc', '--only', std.join(',', self.polyglot_isolate_distributions(polyglot_isolate_languages, os, arch)) + ',TOOLS_COMMUNITY,LANGUAGES_COMMUNITY', local_repo, '${LOCAL_MAVEN_REPO_URL}'], extra_mx_args=['--suite', 'graal-js', '--suite', 'graalpython-enterprise']) + self.deploy_ee(os, arch, false, dry_run, ['--dummy-javadoc', local_repo, '${LOCAL_MAVEN_REPO_URL}']) ) + ( @@ -347,7 +368,7 @@ local devkits = graal_common.devkits; + ( # Locally deploy all relevant suites self.deploy_ce(os, arch, true, dry_run, ['--dummy-javadoc', '--only', vm.maven_deploy_base_functions.reduced_ce_dists, local_repo, '${LOCAL_MAVEN_REDUCED_REPO_URL}']) - + self.deploy_ee(os, arch, true, dry_run, ['--dummy-javadoc', '--only', vm.maven_deploy_base_functions.reduced_ee_dists, local_repo, '${LOCAL_MAVEN_REDUCED_REPO_URL}'], extra_mx_args=['--suite', 'graal-js']) + + self.deploy_ee(os, arch, true, dry_run, ['--dummy-javadoc', '--only', vm.maven_deploy_base_functions.reduced_ee_dists, local_repo, '${LOCAL_MAVEN_REDUCED_REPO_URL}'], extra_mx_args=['--suite', 'graal-js', '--suite', 'graalpython-enterprise']) ) + ( # Archive and deploy @@ -363,7 +384,9 @@ local devkits = graal_common.devkits; ); if (self.compose_platform(os, arch) == main_platform) then ( - [self.mx_cmd_base(os, arch, reduced=false) + ['restore-pd-layouts', self.pd_layouts_archive_name(platform)] for platform in other_platforms] + # The polyglot isolate layout distributions are not merged; each distribution exists solely for the current platform. + # Therefore, it's necessary to ignore these distributions for other platforms." + [self.mx_cmd_base(os, arch, reduced=false) + ['restore-pd-layouts', '--ignore-unknown-distributions', self.pd_layouts_archive_name(platform)] for platform in other_platforms] + ( if (mvn_artifacts || mvn_bundle) then multiplatform_build(reduced=false) @@ -389,12 +412,14 @@ local devkits = graal_common.devkits; [['echo', 'Skipping reduced Maven bundle']] ) ) else ( - self.build(os, arch, reduced=false, build_args=['--targets=' + self.only_native_dists + ',{PLATFORM_DEPENDENT_LAYOUT_DIR_DISTRIBUTIONS}']) - + ( + ( if (vm.maven_deploy_base_functions.edition == 'ce') then + self.build(os, arch, reduced=false, build_args=['--targets=' + self.only_native_dists + ',{PLATFORM_DEPENDENT_LAYOUT_DIR_DISTRIBUTIONS}']) + self.deploy_only_native(os, arch, reduced=false, dry_run=dry_run, extra_args=[remote_mvn_repo]) else - [['echo', 'Skipping the deployment of ' + self.only_native_dists + ': It is already deployed by the ce job']] + self.build(os, arch, reduced=false, build_args=['--targets=' + self.only_native_dists + ',' + std.join(',', self.polyglot_isolate_distributions(polyglot_isolate_languages, os, arch, true)) + ',{PLATFORM_DEPENDENT_LAYOUT_DIR_DISTRIBUTIONS}']) + + [['echo', 'Skipping the deployment of ' + self.only_native_dists + ': It is already deployed by the ce job']] + + self.deploy_ee(os, arch, false, dry_run, ['--dummy-javadoc', '--only', std.join(',', self.polyglot_isolate_distributions(polyglot_isolate_languages, os, arch, true)), remote_mvn_repo], extra_mx_args=['--suite', 'graal-js', '--suite', 'graalpython-enterprise']) ) + [self.mx_cmd_base(os, arch, reduced=false) + ['archive-pd-layouts', self.pd_layouts_archive_name(os + '-' + arch)]] ), From 80d2adf9b624f25ce2d3b7657ccc53574cf7b75e Mon Sep 17 00:00:00 2001 From: Tomas Zezula Date: Wed, 17 Apr 2024 13:14:14 +0200 Subject: [PATCH 11/13] [GR-48481] Resolved review comments. --- truffle/mx.truffle/mx_truffle.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/truffle/mx.truffle/mx_truffle.py b/truffle/mx.truffle/mx_truffle.py index 1e18fc49f9d1..cf89be89bf39 100644 --- a/truffle/mx.truffle/mx_truffle.py +++ b/truffle/mx.truffle/mx_truffle.py @@ -1348,6 +1348,8 @@ def register_polyglot_isolate_distributions(language_suite, register_project, re assert language_license polyglot_isolates_value = mx.get_opts().polyglot_isolates + if polyglot_isolates_value is None: + polyglot_isolates_value = os.getenv('POLYGLOT_ISOLATES') if not polyglot_isolates_value or (polyglot_isolates_value != 'true' and language_id not in polyglot_isolates_value.split(',')): return False From c3ed4e159129e4e0efa176acbbfa73b8fc19f8c4 Mon Sep 17 00:00:00 2001 From: Tomas Zezula Date: Mon, 22 Apr 2024 09:07:54 +0200 Subject: [PATCH 12/13] [GR-48481] Removed suite graalpython-enterprise parameter. --- vm/ci/ci_common/common.jsonnet | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/vm/ci/ci_common/common.jsonnet b/vm/ci/ci_common/common.jsonnet index 09348468565a..8ec0a712bafa 100644 --- a/vm/ci/ci_common/common.jsonnet +++ b/vm/ci/ci_common/common.jsonnet @@ -261,6 +261,8 @@ local devkits = graal_common.devkits; polyglot_isolate_distributions(language_ids, current_os, current_arch, current_only=false):: std.flattenArrays([self.language_polyglot_isolate_distributions(id, current_os, current_arch, current_only) for id in language_ids]), + # To add a polyglot isolate build for a language, ensure that the language is included in the `ee_suites` + # and add the language id to `polyglot_isolate_languages`. local polyglot_isolate_languages = ['js', 'python'], @@ -322,7 +324,7 @@ local devkits = graal_common.devkits; self.deploy_ce(os, arch, false, dry_run, [remote_mvn_repo]) else self.deploy_ee(os, arch, false, dry_run, ['--dummy-javadoc', '--skip', std.join(',', self.polyglot_isolate_distributions(polyglot_isolate_languages, os, arch)) + ',TOOLS_COMMUNITY,LANGUAGES_COMMUNITY', remote_mvn_repo]) - + self.deploy_ee(os, arch, false, dry_run, ['--dummy-javadoc', '--only', std.join(',', self.polyglot_isolate_distributions(polyglot_isolate_languages, os, arch, true)) + ',TOOLS_COMMUNITY,LANGUAGES_COMMUNITY', remote_mvn_repo], extra_mx_args=['--suite', 'graal-js', '--suite', 'graalpython-enterprise']); + + self.deploy_ee(os, arch, false, dry_run, ['--dummy-javadoc', '--only', std.join(',', self.polyglot_isolate_distributions(polyglot_isolate_languages, os, arch, true)) + ',TOOLS_COMMUNITY,LANGUAGES_COMMUNITY', remote_mvn_repo], extra_mx_args=['--suite', 'graal-js']); local mvn_bundle_snippet = [ @@ -337,7 +339,7 @@ local devkits = graal_common.devkits; self.deploy_ce(os, arch, false, dry_run, [local_repo, '${LOCAL_MAVEN_REPO_URL}']) else self.deploy_ce(os, arch, false, dry_run, ['--dummy-javadoc', '--skip', std.join(',', self.polyglot_isolate_distributions(polyglot_isolate_languages, os, arch)) + ',TOOLS_COMMUNITY,LANGUAGES_COMMUNITY', local_repo, '${LOCAL_MAVEN_REPO_URL}']) - + self.deploy_ee(os, arch, false, dry_run, ['--dummy-javadoc', '--only', std.join(',', self.polyglot_isolate_distributions(polyglot_isolate_languages, os, arch)) + ',TOOLS_COMMUNITY,LANGUAGES_COMMUNITY', local_repo, '${LOCAL_MAVEN_REPO_URL}'], extra_mx_args=['--suite', 'graal-js', '--suite', 'graalpython-enterprise']) + + self.deploy_ee(os, arch, false, dry_run, ['--dummy-javadoc', '--only', std.join(',', self.polyglot_isolate_distributions(polyglot_isolate_languages, os, arch)) + ',TOOLS_COMMUNITY,LANGUAGES_COMMUNITY', local_repo, '${LOCAL_MAVEN_REPO_URL}'], extra_mx_args=['--suite', 'graal-js']) + self.deploy_ee(os, arch, false, dry_run, ['--dummy-javadoc', local_repo, '${LOCAL_MAVEN_REPO_URL}']) ) + ( @@ -368,7 +370,7 @@ local devkits = graal_common.devkits; + ( # Locally deploy all relevant suites self.deploy_ce(os, arch, true, dry_run, ['--dummy-javadoc', '--only', vm.maven_deploy_base_functions.reduced_ce_dists, local_repo, '${LOCAL_MAVEN_REDUCED_REPO_URL}']) - + self.deploy_ee(os, arch, true, dry_run, ['--dummy-javadoc', '--only', vm.maven_deploy_base_functions.reduced_ee_dists, local_repo, '${LOCAL_MAVEN_REDUCED_REPO_URL}'], extra_mx_args=['--suite', 'graal-js', '--suite', 'graalpython-enterprise']) + + self.deploy_ee(os, arch, true, dry_run, ['--dummy-javadoc', '--only', vm.maven_deploy_base_functions.reduced_ee_dists, local_repo, '${LOCAL_MAVEN_REDUCED_REPO_URL}'], extra_mx_args=['--suite', 'graal-js']) ) + ( # Archive and deploy @@ -419,7 +421,7 @@ local devkits = graal_common.devkits; else self.build(os, arch, reduced=false, build_args=['--targets=' + self.only_native_dists + ',' + std.join(',', self.polyglot_isolate_distributions(polyglot_isolate_languages, os, arch, true)) + ',{PLATFORM_DEPENDENT_LAYOUT_DIR_DISTRIBUTIONS}']) + [['echo', 'Skipping the deployment of ' + self.only_native_dists + ': It is already deployed by the ce job']] + - self.deploy_ee(os, arch, false, dry_run, ['--dummy-javadoc', '--only', std.join(',', self.polyglot_isolate_distributions(polyglot_isolate_languages, os, arch, true)), remote_mvn_repo], extra_mx_args=['--suite', 'graal-js', '--suite', 'graalpython-enterprise']) + self.deploy_ee(os, arch, false, dry_run, ['--dummy-javadoc', '--only', std.join(',', self.polyglot_isolate_distributions(polyglot_isolate_languages, os, arch, true)), remote_mvn_repo], extra_mx_args=['--suite', 'graal-js']) ) + [self.mx_cmd_base(os, arch, reduced=false) + ['archive-pd-layouts', self.pd_layouts_archive_name(os + '-' + arch)]] ), From c8767ef6aa766b17439343ce97fe6f2d66f43f5b Mon Sep 17 00:00:00 2001 From: Tomas Zezula Date: Thu, 25 Apr 2024 08:53:35 +0200 Subject: [PATCH 13/13] [GR-48481] Resolved review comments. --- truffle/mx.truffle/mx_truffle.py | 20 +++++++++---------- .../polyglot_isolate_resource.template | 4 ++-- ...polyglot_isolate_resource_invalid.template | 4 ++-- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/truffle/mx.truffle/mx_truffle.py b/truffle/mx.truffle/mx_truffle.py index cf89be89bf39..69d4db858444 100644 --- a/truffle/mx.truffle/mx_truffle.py +++ b/truffle/mx.truffle/mx_truffle.py @@ -57,6 +57,7 @@ import mx_native import mx_sdk import mx_sdk_vm +import mx_sdk_vm_impl import mx_subst import mx_unittest import mx_jardistribution @@ -1302,11 +1303,13 @@ def build(self): pkg_name = prj.name with open(_PolyglotIsolateResourceBuildTask._template_file(prj.placeholder), 'r', encoding='utf-8') as f: file_content = f.read() - file_content = (file_content.replace('${package}', pkg_name) - .replace('${languageId}', prj.language_id) - .replace('${resourceId}', prj.resource_id) - .replace('${os}', prj.os_name) - .replace('${arch}', prj.cpu_architecture)) + subst_eng = mx_subst.SubstitutionEngine() + subst_eng.register_no_arg('package', pkg_name) + subst_eng.register_no_arg('languageId', prj.language_id) + subst_eng.register_no_arg('resourceId', prj.resource_id) + subst_eng.register_no_arg('os', prj.os_name) + subst_eng.register_no_arg('arch', prj.cpu_architecture) + file_content = subst_eng.substitute(file_content) target_file = _PolyglotIsolateResourceBuildTask._target_file(prj.source_gen_dir(), pkg_name) mx_util.ensure_dir_exists(dirname(target_file)) with mx_util.SafeFileCreation(target_file) as sfc, open(sfc.tmpPath, 'w', encoding='utf-8') as f: @@ -1347,10 +1350,8 @@ def register_polyglot_isolate_distributions(language_suite, register_project, re assert maven_group_id assert language_license - polyglot_isolates_value = mx.get_opts().polyglot_isolates - if polyglot_isolates_value is None: - polyglot_isolates_value = os.getenv('POLYGLOT_ISOLATES') - if not polyglot_isolates_value or (polyglot_isolates_value != 'true' and language_id not in polyglot_isolates_value.split(',')): + polyglot_isolates_value = mx_sdk_vm_impl._parse_cmd_arg('polyglot_isolates') + if not polyglot_isolates_value or not (polyglot_isolates_value is True or (isinstance(polyglot_isolates_value, list) and language_id in polyglot_isolates_value)): return False if not isinstance(language_license, list): @@ -1394,7 +1395,6 @@ def _qualname(distribution_name): if build_for_current_platform: # 2. Register a project building the isolate library isolate_deps = [language_pom_distribution, 'graal-enterprise:TRUFFLE_ENTERPRISE'] - import mx_sdk_vm_impl build_library = mx_sdk_vm_impl.PolyglotIsolateLibrary(language_suite, language_id, isolate_deps, isolate_build_options) register_project(build_library) diff --git a/truffle/mx.truffle/polyglot_isolate_resource.template b/truffle/mx.truffle/polyglot_isolate_resource.template index f03b955f609b..75cc2ee5a1ea 100644 --- a/truffle/mx.truffle/polyglot_isolate_resource.template +++ b/truffle/mx.truffle/polyglot_isolate_resource.template @@ -38,7 +38,7 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -package ${package}; +package ; import com.oracle.truffle.api.CompilerDirectives; import com.oracle.truffle.api.InternalResource; @@ -49,7 +49,7 @@ import java.nio.file.Path; @InternalResource.Id(value = PolyglotIsolateResource.ID, componentId = "engine", optional = true) public final class PolyglotIsolateResource implements InternalResource { - static final String ID = "${resourceId}"; + static final String ID = ""; @Override public void unpackFiles(Env env, Path targetDirectory) throws IOException { diff --git a/truffle/mx.truffle/polyglot_isolate_resource_invalid.template b/truffle/mx.truffle/polyglot_isolate_resource_invalid.template index f330924841e5..f1fcae2405e6 100644 --- a/truffle/mx.truffle/polyglot_isolate_resource_invalid.template +++ b/truffle/mx.truffle/polyglot_isolate_resource_invalid.template @@ -38,7 +38,7 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -package ${package}; +package ; import com.oracle.truffle.api.CompilerDirectives; import com.oracle.truffle.api.InternalResource; @@ -50,7 +50,7 @@ import org.graalvm.nativeimage.ImageInfo; @InternalResource.Id(value = PolyglotIsolateResource.ID, componentId = "engine", optional = true) public final class PolyglotIsolateResource implements InternalResource { - static final String ID = "${resourceId}"; + static final String ID = ""; @Override public void unpackFiles(Env env, Path targetDirectory) throws IOException {