diff --git a/sdk/mx.sdk/mx_sdk_vm.py b/sdk/mx.sdk/mx_sdk_vm.py index 0a6e0937ef87..5bf52a43d8c3 100644 --- a/sdk/mx.sdk/mx_sdk_vm.py +++ b/sdk/mx.sdk/mx_sdk_vm.py @@ -205,11 +205,10 @@ def __init__(self, destination, jar_distributions, build_args, jvm_library=False class LanguageLibraryConfig(LibraryConfig): - def __init__(self, jar_distributions, build_args, language, main_class=None, is_sdk_launcher=True, launchers=None, option_vars=None, default_vm_args=None, headers=False, set_default_relative_home_path=True, isolate_library_layout_distribution=None, **kwargs): + def __init__(self, jar_distributions, build_args, language, main_class=None, is_sdk_launcher=True, launchers=None, option_vars=None, default_vm_args=None, headers=False, set_default_relative_home_path=True, **kwargs): """ :param str language :param str main_class - :param isolate_library_layout_distribution dict """ kwargs.pop('destination', None) super(LanguageLibraryConfig, self).__init__('lib/', jar_distributions, build_args, home_finder=True, headers=headers, **kwargs) @@ -228,7 +227,6 @@ def __init__(self, jar_distributions, build_args, language, main_class=None, is_ if set_default_relative_home_path: # Ensure the language launcher can always find the language home self.add_relative_home_path(language, relpath('.', dirname(self.destination))) - self.isolate_library_layout_distribution = isolate_library_layout_distribution class GraalVmComponent(object): def __init__(self, diff --git a/sdk/mx.sdk/mx_sdk_vm_impl.py b/sdk/mx.sdk/mx_sdk_vm_impl.py index cfbbb915628d..a20e100af88b 100644 --- a/sdk/mx.sdk/mx_sdk_vm_impl.py +++ b/sdk/mx.sdk/mx_sdk_vm_impl.py @@ -1991,6 +1991,47 @@ def getArchivableResults(self, use_relpath=True, single=False): def is_skipped(self): return _skip_libraries(self.native_image_config) +class PolyglotIsolateLibrary(GraalVmLibrary): + """ + A native image project dedicated to constructing a language polyglot isolate library. + Despite being built upon the component supertype, it operates independently of the component system + and native-image macros. Its configuration relies solely on the module path and META-INF/native-image + configuration files. Instances are instantiated by mx_truffle::register_polyglot_isolate_distributions + when a language dynamically registers a polyglot isolate distribution. + """ + def __init__(self, target_suite, language, deps, build_args, **kw_args): + library_config = mx_sdk.LanguageLibraryConfig( + jar_distributions=deps, + build_args=[], + build_args_enterprise=build_args, + language=language, + ) + super(PolyglotIsolateLibrary, self).__init__(None, f'{language}.isolate.image', + list(deps), library_config, **kw_args) + self.suite = target_suite + self.dir = target_suite.dir + + + def getBuildTask(self, args): + svm_support = _get_svm_support() + assert svm_support.is_supported(), "Needs svm to build " + str(self) + if not self.is_skipped(): + return PolyglotIsolateLibraryBuildTask(self, args, svm_support) + else: + return mx.NoOpTask(self, args) + + def getArchivableResults(self, use_relpath=True, single=False): + for e in super(PolyglotIsolateLibrary, self).getArchivableResults(use_relpath=use_relpath, single=single): + yield e + if single: + return + output_dir = dirname(self.output_file()) + resources_dir = join(output_dir, 'resources') + if exists(resources_dir): + yield resources_dir, 'resources' + + def is_skipped(self): + return False class GraalVmMiscLauncher(GraalVmLauncher): # pylint: disable=too-many-ancestors def __init__(self, component, native_image_config, stage1=False, **kw_args): @@ -2440,6 +2481,32 @@ class GraalVmLibraryBuildTask(GraalVmSVMNativeImageBuildTask): pass +class PolyglotIsolateLibraryBuildTask(GraalVmLibraryBuildTask): + """ + A PolyglotIsolateLibrary build task building a language polyglot isolate library. + Despite being built upon the component supertype, it operates independently of the component system + and native-image macros. Its configuration relies solely on the module path and META-INF/native-image + configuration files. + """ + def get_build_args(self): + project = self.subject + target = project.native_image_name[:-len(_lib_suffix)] + build_args = [ + '-EJVMCI_VERSION_CHECK', # Propagate this env var when running native image from mx + '--parallelism=' + str(self.parallelism), + '--shared', + '-o', + target, + '--features=com.oracle.svm.enterprise.truffle.PolyglotIsolateGuestFeature', + '-H:APIFunctionPrefix=truffle_isolate_', + ] + svm_experimental_options([ + '-H:+BuildOutputPrefix', + '-H:+GenerateBuildArtifactsFile', # generate 'build-artifacts.json' + ]) + mx.get_runtime_jvm_args(self.subject.native_image_jar_distributions) + \ + project.native_image_config.build_args + project.native_image_config.build_args_enterprise + return build_args + + class JmodModifier(mx.Project): def __init__(self, jmod_file, library_projects, jimage_project, **kw_args): """ @@ -3491,7 +3558,6 @@ def register_main_dist(dist, label): else: with_non_rebuildable_configs = True for library_config in _get_library_configs(component): - library_project = None if with_svm: library_project = GraalVmLibrary(component, GraalVmNativeImage.project_name(library_config), [], library_config) register_project(library_project) @@ -3509,29 +3575,6 @@ def register_main_dist(dist, label): register_project(launcher_project) polyglot_config_project = PolyglotConfig(component, library_config) register_project(polyglot_config_project) - if with_svm and library_config.isolate_library_layout_distribution and not library_project.is_skipped() and has_component('tfle', stage1=True): - # Create a layout distribution with the resulting language library that can be consumed into the - # isolate resources jar distribution, - resource_base_folder = f'META-INF/resources/engine/{library_config.language}-isolate///libvm' - attrs = { - 'description': f'Contains {library_config.language} language library resources', - 'hashEntry': f'{resource_base_folder}/sha256', - 'fileListEntry': f'{resource_base_folder}/files', - 'maven': False, - } - register_distribution(mx.LayoutDirDistribution( - suite=_suite, - name=library_config.isolate_library_layout_distribution['name'], - deps=[], - layout={ - f'{resource_base_folder}/': f'dependency:{library_project.name}' - }, - path=None, - platformDependent=True, - theLicense=None, - platforms=library_config.isolate_library_layout_distribution['platforms'], - **attrs - )) if isinstance(component, mx_sdk.GraalVmLanguage) and component.support_distributions: ni_resources_components = dir_name_to_ni_resources_components.get(component.dir_name) if not ni_resources_components: diff --git a/truffle/mx.truffle/mx_truffle.py b/truffle/mx.truffle/mx_truffle.py index 6ed30989857f..69d4db858444 100644 --- a/truffle/mx.truffle/mx_truffle.py +++ b/truffle/mx.truffle/mx_truffle.py @@ -48,7 +48,7 @@ import difflib import zipfile from argparse import Action, ArgumentParser, RawDescriptionHelpFormatter -from os.path import exists, isdir, join, abspath +from os.path import dirname, exists, isdir, join, abspath from urllib.parse import urljoin # pylint: disable=unused-import,no-name-in-module import mx @@ -57,6 +57,7 @@ import mx_native import mx_sdk import mx_sdk_vm +import mx_sdk_vm_impl import mx_subst import mx_unittest import mx_jardistribution @@ -1205,90 +1206,309 @@ def read_file(path): "Make sure the grammar files are up to date with the generated code. You can regenerate the generated code using mx.") -def register_polyglot_isolate_distributions(register_distribution, language_id, language_distribution, isolate_library_layout_distribution, internal_resource_project): +class _PolyglotIsolateResourceProject(mx.JavaProject): """ - Registers the polyglot isolate resource distribution and isolate resource meta-POM distribution. + A Java project creating an internal resource implementation unpacking the polyglot isolate library. + The Java source code for internal resources is generated into the project's `source_gen_dir` before compilation, + using the `mx.truffle/polyglot_isolate_resource.template`. Configuration of the project follows these conventions: + The target package name is `com.oracle.truffle.isolate.resource...`, + and the resource ID is `-isolate--`. + """ + + def __init__(self, language_suite, subDir, language_id, resource_id, os_name, cpu_architecture, placeholder): + name = f'com.oracle.truffle.isolate.resource.{language_id}.{os_name}.{cpu_architecture}' + javaCompliance = str(mx.distribution('truffle:TRUFFLE_API').javaCompliance) + '+' + project_dir = os.path.join(language_suite.dir, subDir, name) + deps = ['truffle:TRUFFLE_API'] + if placeholder: + deps += ['sdk:NATIVEIMAGE'] + super(_PolyglotIsolateResourceProject, self).__init__(language_suite, name, subDir=subDir, srcDirs=[], deps=deps, + javaCompliance=javaCompliance, workingSets='Truffle', d=project_dir, + theLicense=_suite.defaultLicense) + src_gen_dir = self.source_gen_dir() + self.srcDirs.append(src_gen_dir) + self.declaredAnnotationProcessors = ['truffle:TRUFFLE_DSL_PROCESSOR'] + self.language_id = language_id + self.resource_id = resource_id + self.os_name = os_name + self.cpu_architecture = cpu_architecture + self.placeholder = placeholder + + + def getBuildTask(self, args): + jdk = mx.get_jdk(self.javaCompliance, tag=mx.DEFAULT_JDK_TAG, purpose='building ' + self.name) + return _PolyglotIsolateResourceBuildTask(args, self, jdk) + + +class _PolyglotIsolateResourceBuildTask(mx.JavaBuildTask): + """ + A _PolyglotIsolateResourceProject build task generating and building the internal resource unpacking + the polyglot isolate library. Refer to `_PolyglotIsolateResourceProject` documentation for more details. + """ + + def __str__(self): + return f'Generating {self.subject.name} internal resource and compiling it with {self._getCompiler().name()}' + + @staticmethod + def _template_file(placeholder): + name = 'polyglot_isolate_resource_invalid.template' if placeholder else 'polyglot_isolate_resource.template' + return os.path.join(_suite.mxDir, name) + + def needsBuild(self, newestInput): + is_needed, reason = mx.ProjectBuildTask.needsBuild(self, newestInput) + if is_needed: + return True, reason + proj = self.subject + for outDir in [proj.output_dir(), proj.source_gen_dir()]: + if not os.path.exists(outDir): + return True, f"{outDir} does not exist" + template_ts = mx.TimeStampFile.newest([ + _PolyglotIsolateResourceBuildTask._template_file(False), + _PolyglotIsolateResourceBuildTask._template_file(True), + __file__ + ]) + if newestInput is None or newestInput.isOlderThan(template_ts): + newestInput = template_ts + return super().needsBuild(newestInput) + + @staticmethod + def _target_file(root, pkg_name): + target_folder = os.path.join(root, pkg_name.replace('.', os.sep)) + target_file = os.path.join(target_folder, 'PolyglotIsolateResource.java') + return target_file + + + def _collect_files(self): + if self._javafiles is not None: + # already collected + return self + # collect project files first, then extend with generated resource + super(_PolyglotIsolateResourceBuildTask, self)._collect_files() + javafiles = self._javafiles + prj = self.subject + gen_src_dir = prj.source_gen_dir() + pkg_name = prj.name + target_file = _PolyglotIsolateResourceBuildTask._target_file(gen_src_dir, pkg_name) + if not target_file in javafiles: + bin_dir = prj.output_dir() + target_class = os.path.join(bin_dir, os.path.relpath(target_file, gen_src_dir)[:-len('.java')] + '.class') + javafiles[target_file] = target_class + # Remove annotation processor generated files. + javafiles = {k: v for k, v in javafiles.items() if k == target_file} + self._javafiles = javafiles + return self + + def build(self): + prj = self.subject + pkg_name = prj.name + with open(_PolyglotIsolateResourceBuildTask._template_file(prj.placeholder), 'r', encoding='utf-8') as f: + file_content = f.read() + subst_eng = mx_subst.SubstitutionEngine() + subst_eng.register_no_arg('package', pkg_name) + subst_eng.register_no_arg('languageId', prj.language_id) + subst_eng.register_no_arg('resourceId', prj.resource_id) + subst_eng.register_no_arg('os', prj.os_name) + subst_eng.register_no_arg('arch', prj.cpu_architecture) + file_content = subst_eng.substitute(file_content) + target_file = _PolyglotIsolateResourceBuildTask._target_file(prj.source_gen_dir(), pkg_name) + mx_util.ensure_dir_exists(dirname(target_file)) + with mx_util.SafeFileCreation(target_file) as sfc, open(sfc.tmpPath, 'w', encoding='utf-8') as f: + f.write(file_content) + super(_PolyglotIsolateResourceBuildTask, self).build() + + + +def register_polyglot_isolate_distributions(language_suite, register_project, register_distribution, language_id, + subDir, language_pom_distribution, maven_group_id, language_license, + isolate_build_options=None, platforms=None): + """ + Creates and registers the polyglot isolate resource distribution and isolate resource meta-POM distribution. The created polyglot isolate resource distribution is named `_ISOLATE_RESOURCES`, inheriting the Maven group ID from the given `language_distribution`, and the Maven artifact ID is `-isolate`. The meta-POM distribution is named `_ISOLATE`, having the Maven group ID `org.graalvm.polyglot`, and the Maven artifact ID is `-isolate`. + :param Suite language_suite: The language suite used to register generated projects and distributions to. + :param register_project: A callback to dynamically register the project, obtained as a parameter from `mx_register_dynamic_suite_constituents`. + :type register_project: (mx.Project) -> None :param register_distribution: A callback to dynamically register the distribution, obtained as a parameter from `mx_register_dynamic_suite_constituents`. :type register_distribution: (mx.Distribution) -> None - :param language_id: The language ID. - :param language_distribution: The language distribution used to inherit distribution properties. - :param isolate_library_layout_distribution: The layout distribution with polyglot isolate library. - :param internal_resource_project: The internal resource project used for unpacking the polyglot isolate library. + :param str language_id: The language ID. + :param str subDir: a path relative to `suite.dir` in which the IDE project configuration for this distribution is generated + :param POMDistribution language_pom_distribution: The language meta pom distribution used to set the image builder module-path. + :param str maven_group_id: The maven language group id. + :param str | list | language_license: Language licence(s). + :param list isolate_build_options: additional options passed to a native image to build the isolate library. + :param list platforms: supported platforms, defaults to ['linux-amd64', 'linux-aarch64', 'darwin-amd64', 'darwin-aarch64', 'windows-amd64'] """ - assert language_distribution - assert isolate_library_layout_distribution - assert internal_resource_project - owner_suite = language_distribution.suite - resources_dist_name = f'{language_id.upper()}_ISOLATE_RESOURCES' - isolate_dist_name = f'{language_id.upper()}_ISOLATE' - layout_dist_qualified_name = f'{isolate_library_layout_distribution.suite.name}:{isolate_library_layout_distribution.name}' - maven_group_id = language_distribution.maven_group_id() - maven_artifact_id = f'{language_id}-isolate' - module_name = f'{get_module_name(language_distribution)}.isolate' - licenses = set() - licenses.update(language_distribution.theLicense) - licenses.update(owner_suite.defaultLicense) - attrs = { - 'description': f'Polyglot isolate resources for {language_id}.', - 'moduleInfo': { - 'name': module_name, - }, - 'maven': { - 'groupId': maven_group_id, - 'artifactId': maven_artifact_id, - 'tag': ['default', 'public'], - }, - 'mavenNoJavadoc': True, - 'mavenNoSources': True, - } - isolate_library_dist = mx_jardistribution.JARDistribution( - suite=owner_suite, - name=resources_dist_name, - subDir=language_distribution.subDir, - path=None, - sourcesPath=None, - deps=[ - internal_resource_project.name, - layout_dist_qualified_name, - ], - mainClass=None, - excludedLibs=[], - distDependencies=['truffle:TRUFFLE_API'], - javaCompliance=str(internal_resource_project.javaCompliance)+'+', - platformDependent=True, - theLicense=list(licenses), - compress=True, - **attrs - ) - register_distribution(isolate_library_dist) + assert language_suite + assert register_project + assert register_distribution + assert language_id + assert subDir + assert language_pom_distribution + assert maven_group_id + assert language_license + + polyglot_isolates_value = mx_sdk_vm_impl._parse_cmd_arg('polyglot_isolates') + if not polyglot_isolates_value or not (polyglot_isolates_value is True or (isinstance(polyglot_isolates_value, list) and language_id in polyglot_isolates_value)): + return False + + if not isinstance(language_license, list): + assert isinstance(language_license, str) + language_license = [language_license] + + def _qualname(distribution_name): + return language_suite.name + ':' + distribution_name if distribution_name.find(':') < 0 else distribution_name + + language_pom_distribution = _qualname(language_pom_distribution) + if isolate_build_options is None: + isolate_build_options = [] + language_id_upper_case = language_id.upper() + if platforms is None: + platforms = [ + 'linux-amd64', + 'linux-aarch64', + 'darwin-amd64', + 'darwin-aarch64', + 'windows-amd64', + ] + current_platform = mx_subst.string_substitutions.substitute('-') + if current_platform not in platforms: + mx.abort(f'Current platform {current_platform} is not in supported platforms {", ".join(platforms)}') + + platform_meta_poms = [] + for platform in platforms: + build_for_current_platform = platform == current_platform + resource_id = f'{language_id}-isolate-{platform}' + os_name, cpu_architecture = platform.split('-') + os_name_upper_case = os_name.upper() + cpu_architecture_upper_case = cpu_architecture.upper() + # 1. Register a project generating and building an internal resource for polyglot isolate library + build_internal_resource = _PolyglotIsolateResourceProject(language_suite, subDir, language_id, resource_id, + os_name, cpu_architecture, not build_for_current_platform) + register_project(build_internal_resource) + resources_dist_dependencies = [ + build_internal_resource.name, + ] + + if build_for_current_platform: + # 2. Register a project building the isolate library + isolate_deps = [language_pom_distribution, 'graal-enterprise:TRUFFLE_ENTERPRISE'] + build_library = mx_sdk_vm_impl.PolyglotIsolateLibrary(language_suite, language_id, isolate_deps, isolate_build_options) + register_project(build_library) + + # 3. Register layout distribution with isolate library and isolate resources + resource_base_folder = f'META-INF/resources/engine/{resource_id}/libvm' + attrs = { + 'description': f'Contains {language_id} language library resources.', + 'hashEntry': f'{resource_base_folder}/sha256', + 'fileListEntry': f'{resource_base_folder}/files', + 'maven': False, + } + layout_dist = mx.LayoutDirDistribution( + suite=language_suite, + name=f'{language_id_upper_case}_ISOLATE_LAYOUT_{os_name_upper_case}_{cpu_architecture_upper_case}', + deps=[], + layout={ + f'{resource_base_folder}/': f'dependency:{build_library.name}', + f'{resource_base_folder}/resources': f'dependency:{build_library.name}/resources', + }, + path=None, + platformDependent=True, + theLicense=None, + platforms=[platform], + **attrs + ) + register_distribution(layout_dist) + layout_dist_qualified_name = f'{layout_dist.suite.name}:{layout_dist.name}' + resources_dist_dependencies.append(layout_dist_qualified_name) + + # 4. Register Jar distribution containing the internal resource project and isolate library for current platform. + # For other platforms, create a jar distribution with an internal resource only + resources_dist_name = f'{language_id_upper_case}_ISOLATE_RESOURCES_{os_name_upper_case}_{cpu_architecture_upper_case}' + maven_artifact_id = resource_id + licenses = set(language_license) + # The graal-enterprise suite may not be fully loaded. + # We cannot look up the TRUFFLE_ENTERPRISE distribution to resolve its license + # We pass directly the license id + licenses.update(['GFTC']) + attrs = { + 'description': f'Polyglot isolate resources for {language_id} for {platform}.', + 'moduleInfo': { + 'name': build_internal_resource.name, + }, + 'maven': { + 'groupId': maven_group_id, + 'artifactId': maven_artifact_id, + 'tag': ['default', 'public'], + }, + 'mavenNoJavadoc': True, + 'mavenNoSources': True, + } + isolate_library_dist = mx_jardistribution.JARDistribution( + suite=language_suite, + name=resources_dist_name, + subDir=subDir, + path=None, + sourcesPath=None, + deps=resources_dist_dependencies, + mainClass=None, + excludedLibs=[], + distDependencies=['truffle:TRUFFLE_API'], + javaCompliance=str(build_internal_resource.javaCompliance)+'+', + platformDependent=True, + theLicense=sorted(list(licenses)), + compress=True, + **attrs + ) + register_distribution(isolate_library_dist) + + # 5. Register meta POM distribution for the isolate library jar file for a specific platform. + isolate_dist_name = f'{language_id_upper_case}_ISOLATE_{os_name_upper_case}_{cpu_architecture_upper_case}' + attrs = { + 'description': f'The {language_id} polyglot isolate for {platform}.', + 'maven': { + 'groupId': 'org.graalvm.polyglot', + 'artifactId': maven_artifact_id, + 'tag': ['default', 'public'], + }, + } + meta_pom_dist = mx_pomdistribution.POMDistribution( + suite=language_suite, + name=isolate_dist_name, + distDependencies=[], + runtimeDependencies=[ + resources_dist_name, + 'graal-enterprise:TRUFFLE_ENTERPRISE', + ], + theLicense=sorted(list(licenses)), + **attrs) + register_distribution(meta_pom_dist) + platform_meta_poms.append(meta_pom_dist) + # 6. Register meta POM distribution listing all platform specific meta-POMS. + isolate_dist_name = f'{language_id_upper_case}_ISOLATE' attrs = { 'description': f'The {language_id} polyglot isolate.', 'maven': { 'groupId': 'org.graalvm.polyglot', - 'artifactId': maven_artifact_id, + 'artifactId': f'{language_id}-isolate', 'tag': ['default', 'public'], }, } - # The graal-enterprise suite may not be fully loaded. - # We cannot look up the TRUFFLE_ENTERPRISE distribution to resolve its license - # We pass directly the license id - licenses.update(['GFTC']) meta_pom_dist = mx_pomdistribution.POMDistribution( - suite=owner_suite, + suite=language_suite, name=isolate_dist_name, distDependencies=[], - runtimeDependencies=[ - resources_dist_name, - 'graal-enterprise:TRUFFLE_ENTERPRISE', - ], + runtimeDependencies=[pom.name for pom in platform_meta_poms], theLicense=sorted(list(licenses)), **attrs) register_distribution(meta_pom_dist) + return True + + +mx.add_argument('--polyglot-isolates', action='store', help='Comma-separated list of languages for which the polyglot isolate library should be built. Setting the value to `true` builds all polyglot isolate libraries.') + class LibffiBuilderProject(mx.AbstractNativeProject, mx_native.NativeDependency): # pylint: disable=too-many-ancestors """Project for building libffi from source. @@ -1336,7 +1556,7 @@ class LibtoolNativeProject(mx.NativeProject, # pylint: disable=too-many-ancesto def getArchivableResults(self, use_relpath=True, single=False): for file_path, archive_path in super(LibtoolNativeProject, self).getArchivableResults(use_relpath): - path_in_lt_objdir = os.path.basename(os.path.dirname(file_path)) == '.libs' + path_in_lt_objdir = os.path.basename(dirname(file_path)) == '.libs' yield file_path, os.path.basename(archive_path) if path_in_lt_objdir else archive_path if single: assert path_in_lt_objdir, 'the first build result must be from LT_OBJDIR' diff --git a/truffle/mx.truffle/polyglot_isolate_resource.template b/truffle/mx.truffle/polyglot_isolate_resource.template new file mode 100644 index 000000000000..75cc2ee5a1ea --- /dev/null +++ b/truffle/mx.truffle/polyglot_isolate_resource.template @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The Universal Permissive License (UPL), Version 1.0 + * + * Subject to the condition set forth below, permission is hereby granted to any + * person obtaining a copy of this software, associated documentation and/or + * data (collectively the "Software"), free of charge and under any and all + * copyright rights in the Software, and any and all patent rights owned or + * freely licensable by each licensor hereunder covering either (i) the + * unmodified Software as contributed to or provided by such licensor, or (ii) + * the Larger Works (as defined below), to deal in both + * + * (a) the Software, and + * + * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if + * one is included with the Software each a "Larger Work" to which the Software + * is contributed by such licensors), + * + * without restriction, including without limitation the rights to copy, create + * derivative works of, display, perform, and distribute the Software and make, + * use, sell, offer for sale, import, export, have made, and have sold the + * Software and the Larger Work(s), and to sublicense the foregoing rights on + * either these or other terms. + * + * This license is subject to the following condition: + * + * The above copyright notice and either this complete permission notice or at a + * minimum a reference to the UPL must be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package ; + +import com.oracle.truffle.api.CompilerDirectives; +import com.oracle.truffle.api.InternalResource; + +import java.io.IOException; +import java.nio.file.Path; + +@InternalResource.Id(value = PolyglotIsolateResource.ID, componentId = "engine", optional = true) +public final class PolyglotIsolateResource implements InternalResource { + + static final String ID = ""; + + @Override + public void unpackFiles(Env env, Path targetDirectory) throws IOException { + Path base = basePath(env); + env.unpackResourceFiles(base.resolve("files"), targetDirectory, base); + } + + @Override + public String versionHash(Env env) { + try { + Path base = basePath(env); + return env.readResourceLines(base.resolve("sha256")).get(0); + } catch (IOException ioe) { + throw CompilerDirectives.shouldNotReachHere(ioe); + } + } + + private static Path basePath(Env env) { + return Path.of("META-INF", "resources", "engine", ID, "libvm"); + } +} diff --git a/truffle/mx.truffle/polyglot_isolate_resource_invalid.template b/truffle/mx.truffle/polyglot_isolate_resource_invalid.template new file mode 100644 index 000000000000..f1fcae2405e6 --- /dev/null +++ b/truffle/mx.truffle/polyglot_isolate_resource_invalid.template @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The Universal Permissive License (UPL), Version 1.0 + * + * Subject to the condition set forth below, permission is hereby granted to any + * person obtaining a copy of this software, associated documentation and/or + * data (collectively the "Software"), free of charge and under any and all + * copyright rights in the Software, and any and all patent rights owned or + * freely licensable by each licensor hereunder covering either (i) the + * unmodified Software as contributed to or provided by such licensor, or (ii) + * the Larger Works (as defined below), to deal in both + * + * (a) the Software, and + * + * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if + * one is included with the Software each a "Larger Work" to which the Software + * is contributed by such licensors), + * + * without restriction, including without limitation the rights to copy, create + * derivative works of, display, perform, and distribute the Software and make, + * use, sell, offer for sale, import, export, have made, and have sold the + * Software and the Larger Work(s), and to sublicense the foregoing rights on + * either these or other terms. + * + * This license is subject to the following condition: + * + * The above copyright notice and either this complete permission notice or at a + * minimum a reference to the UPL must be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package ; + +import com.oracle.truffle.api.CompilerDirectives; +import com.oracle.truffle.api.InternalResource; + +import java.io.IOException; +import java.nio.file.Path; +import org.graalvm.nativeimage.ImageInfo; + +@InternalResource.Id(value = PolyglotIsolateResource.ID, componentId = "engine", optional = true) +public final class PolyglotIsolateResource implements InternalResource { + + static final String ID = ""; + + @Override + public void unpackFiles(Env env, Path targetDirectory) throws IOException { + if (!ImageInfo.inImageBuildtimeCode()) { + throw shouldNotReachHere(); + } + } + + @Override + public String versionHash(Env env) { + throw shouldNotReachHere(); + } + + private static RuntimeException shouldNotReachHere() { + throw CompilerDirectives.shouldNotReachHere("The polyglot isolate library was not built for the current platform."); + } +} diff --git a/vm/ci/ci_common/common.jsonnet b/vm/ci/ci_common/common.jsonnet index 53f81142934f..8ec0a712bafa 100644 --- a/vm/ci/ci_common/common.jsonnet +++ b/vm/ci/ci_common/common.jsonnet @@ -243,6 +243,29 @@ local devkits = graal_common.devkits; local ce_licenses = legacy_licenses + ',PSF-License,BSD-simplified,BSD-new,EPL-2.0'; ce_licenses, + platform_specific_distribution_name(base_name, os, arch):: + base_name + '_' + std.asciiUpper(os) + '_' + std.asciiUpper(arch), + + language_polyglot_isolate_distributions(language_id, current_os, current_arch, current_only=false):: + local id_upcase = std.asciiUpper(language_id); + local base_names = [id_upcase + '_ISOLATE', id_upcase + '_ISOLATE_RESOURCES']; + local oss = ['linux', 'darwin', 'windows']; + local archs = ['amd64', 'aarch64']; + [base_names[0]] + [self.platform_specific_distribution_name(base_name, os, arch), + for base_name in base_names + for os in oss for arch in archs + if os != 'windows' || arch != 'aarch64' + if !current_only || os == current_os && arch == current_arch + ], + + polyglot_isolate_distributions(language_ids, current_os, current_arch, current_only=false):: + std.flattenArrays([self.language_polyglot_isolate_distributions(id, current_os, current_arch, current_only) for id in language_ids]), + + # To add a polyglot isolate build for a language, ensure that the language is included in the `ee_suites` + # and add the language id to `polyglot_isolate_languages`. + local polyglot_isolate_languages = ['js', 'python'], + + legacy_mx_args:: ['--disable-installables=true'], # `['--force-bash-launcher=true', '--skip-libraries=true']` have been replaced by arguments from `vm.maven_deploy_base_functions.mx_args(os, arch)` mx_args(os, arch, reduced):: self.legacy_mx_args + vm.maven_deploy_base_functions.mx_args(os, arch, reduced), mx_cmd_base(os, arch, reduced):: ['mx'] + vm.maven_deploy_base_functions.dynamic_imports(os, arch) + self.mx_args(os, arch, reduced), @@ -300,8 +323,8 @@ local devkits = graal_common.devkits; if (vm.maven_deploy_base_functions.edition == 'ce') then self.deploy_ce(os, arch, false, dry_run, [remote_mvn_repo]) else - self.deploy_ee(os, arch, false, dry_run, ['--dummy-javadoc', remote_mvn_repo]) - + self.deploy_ee(os, arch, false, dry_run, ['--dummy-javadoc', '--only', 'JS_ISOLATE,JS_ISOLATE_RESOURCES,TOOLS_COMMUNITY,LANGUAGES_COMMUNITY', remote_mvn_repo], extra_mx_args=['--suite', 'graal-js']); + self.deploy_ee(os, arch, false, dry_run, ['--dummy-javadoc', '--skip', std.join(',', self.polyglot_isolate_distributions(polyglot_isolate_languages, os, arch)) + ',TOOLS_COMMUNITY,LANGUAGES_COMMUNITY', remote_mvn_repo]) + + self.deploy_ee(os, arch, false, dry_run, ['--dummy-javadoc', '--only', std.join(',', self.polyglot_isolate_distributions(polyglot_isolate_languages, os, arch, true)) + ',TOOLS_COMMUNITY,LANGUAGES_COMMUNITY', remote_mvn_repo], extra_mx_args=['--suite', 'graal-js']); local mvn_bundle_snippet = [ @@ -315,8 +338,8 @@ local devkits = graal_common.devkits; if (vm.maven_deploy_base_functions.edition == 'ce') then self.deploy_ce(os, arch, false, dry_run, [local_repo, '${LOCAL_MAVEN_REPO_URL}']) else - self.deploy_ce(os, arch, false, dry_run, ['--dummy-javadoc', '--skip', 'JS_ISOLATE,JS_ISOLATE_RESOURCES,TOOLS_COMMUNITY,LANGUAGES_COMMUNITY', local_repo, '${LOCAL_MAVEN_REPO_URL}']) - + self.deploy_ee(os, arch, false, dry_run, ['--dummy-javadoc', '--only', 'JS_ISOLATE,JS_ISOLATE_RESOURCES,TOOLS_COMMUNITY,LANGUAGES_COMMUNITY', local_repo, '${LOCAL_MAVEN_REPO_URL}'], extra_mx_args=['--suite', 'graal-js']) + self.deploy_ce(os, arch, false, dry_run, ['--dummy-javadoc', '--skip', std.join(',', self.polyglot_isolate_distributions(polyglot_isolate_languages, os, arch)) + ',TOOLS_COMMUNITY,LANGUAGES_COMMUNITY', local_repo, '${LOCAL_MAVEN_REPO_URL}']) + + self.deploy_ee(os, arch, false, dry_run, ['--dummy-javadoc', '--only', std.join(',', self.polyglot_isolate_distributions(polyglot_isolate_languages, os, arch)) + ',TOOLS_COMMUNITY,LANGUAGES_COMMUNITY', local_repo, '${LOCAL_MAVEN_REPO_URL}'], extra_mx_args=['--suite', 'graal-js']) + self.deploy_ee(os, arch, false, dry_run, ['--dummy-javadoc', local_repo, '${LOCAL_MAVEN_REPO_URL}']) ) + ( @@ -363,7 +386,9 @@ local devkits = graal_common.devkits; ); if (self.compose_platform(os, arch) == main_platform) then ( - [self.mx_cmd_base(os, arch, reduced=false) + ['restore-pd-layouts', self.pd_layouts_archive_name(platform)] for platform in other_platforms] + # The polyglot isolate layout distributions are not merged; each distribution exists solely for the current platform. + # Therefore, it's necessary to ignore these distributions for other platforms." + [self.mx_cmd_base(os, arch, reduced=false) + ['restore-pd-layouts', '--ignore-unknown-distributions', self.pd_layouts_archive_name(platform)] for platform in other_platforms] + ( if (mvn_artifacts || mvn_bundle) then multiplatform_build(reduced=false) @@ -389,12 +414,14 @@ local devkits = graal_common.devkits; [['echo', 'Skipping reduced Maven bundle']] ) ) else ( - self.build(os, arch, reduced=false, build_args=['--targets=' + self.only_native_dists + ',{PLATFORM_DEPENDENT_LAYOUT_DIR_DISTRIBUTIONS}']) - + ( + ( if (vm.maven_deploy_base_functions.edition == 'ce') then + self.build(os, arch, reduced=false, build_args=['--targets=' + self.only_native_dists + ',{PLATFORM_DEPENDENT_LAYOUT_DIR_DISTRIBUTIONS}']) + self.deploy_only_native(os, arch, reduced=false, dry_run=dry_run, extra_args=[remote_mvn_repo]) else - [['echo', 'Skipping the deployment of ' + self.only_native_dists + ': It is already deployed by the ce job']] + self.build(os, arch, reduced=false, build_args=['--targets=' + self.only_native_dists + ',' + std.join(',', self.polyglot_isolate_distributions(polyglot_isolate_languages, os, arch, true)) + ',{PLATFORM_DEPENDENT_LAYOUT_DIR_DISTRIBUTIONS}']) + + [['echo', 'Skipping the deployment of ' + self.only_native_dists + ': It is already deployed by the ce job']] + + self.deploy_ee(os, arch, false, dry_run, ['--dummy-javadoc', '--only', std.join(',', self.polyglot_isolate_distributions(polyglot_isolate_languages, os, arch, true)), remote_mvn_repo], extra_mx_args=['--suite', 'graal-js']) ) + [self.mx_cmd_base(os, arch, reduced=false) + ['archive-pd-layouts', self.pd_layouts_archive_name(os + '-' + arch)]] ), diff --git a/vm/mx.vm/suite.py b/vm/mx.vm/suite.py index 0e4cb8b6fb98..3a1a2f14e741 100644 --- a/vm/mx.vm/suite.py +++ b/vm/mx.vm/suite.py @@ -33,7 +33,7 @@ "name": "graal-nodejs", "subdir": True, "dynamic": True, - "version": "021a893d7d279686ed17adb211ba9dab2f170d61", + "version": "a54e3ea08c9d8783c3941430c8a8834d1481d6cf", "urls" : [ {"url" : "https://github.com/graalvm/graaljs.git", "kind" : "git"}, ] @@ -42,7 +42,7 @@ "name": "graal-js", "subdir": True, "dynamic": True, - "version": "021a893d7d279686ed17adb211ba9dab2f170d61", + "version": "a54e3ea08c9d8783c3941430c8a8834d1481d6cf", "urls": [ {"url": "https://github.com/graalvm/graaljs.git", "kind" : "git"}, ]