Skip to content

Commit

Permalink
Merge pull request #10451 from dra27/no-scripting-for-4.13
Browse files Browse the repository at this point in the history
Fix lack of iconv on Cygwin and build performance regression on Windows

(cherry picked from commit 3a0b1b1)
  • Loading branch information
dra27 committed Sep 1, 2021
1 parent f1bd190 commit 7e78d17
Show file tree
Hide file tree
Showing 12 changed files with 208 additions and 51 deletions.
2 changes: 2 additions & 0 deletions .gitattributes
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,8 @@ otherlibs/win32unix/readlink.c typo.long-line
otherlibs/win32unix/stat.c typo.long-line
otherlibs/win32unix/symlink.c typo.long-line

runtime/sak.c typo.non-ascii

stdlib/hashbang typo.white-at-eol typo.missing-lf

testsuite/tests/** typo.missing-header typo.long-line=may
Expand Down
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,8 @@ _build
/runtime/ld.conf
/runtime/.gdb_history
/runtime/.dep
/runtime/build_config.h
/runtime/sak
/runtime/domain_state32.inc
/runtime/domain_state64.inc

Expand Down
16 changes: 11 additions & 5 deletions Changes
Original file line number Diff line number Diff line change
Expand Up @@ -192,11 +192,11 @@ OCaml 4.13.0
(Gabriel Scherer, review by Nicolás Ojeda Bär, Alain Frisch, Xavier Leroy,
Daniel Bünzli and Stephen Dolan)

* #10169, #10270, #10301: Use capitalized module names in the Standard Library
prefixing scheme to match Dune, e.g. Stdlib__String instead of Stdlib__string.
This is a breaking change only to code which attempted to use the internal
names before. The Standard Library generated by the Dune rules is now
equivalent to the main build (the Dune rules still do not generate a
* #10169, #10270, #10301, #10451: Use capitalized module names in the Standard
Library prefixing scheme to match Dune, e.g. Stdlib__String instead of
Stdlib__string. This is a breaking change only to code which attempted to use
the internal names before. The Standard Library generated by the Dune rules is
now equivalent to the main build (the Dune rules still do not generate a
distributable compiler).
(David Allsopp and Mark Shinwell, review by Gabriel Scherer)

Expand Down Expand Up @@ -470,6 +470,12 @@ OCaml 4.13.0
which runtime to use while building the compilers (Sébastien Hinderer,
review by David Allsopp)

- #10451: Replace the use of iconv with a C utility to convert $(LIBDIR) to a
C string constant on Windows when building the runtime. Hardens the generation
of the constant on Unix for paths with backslashes, double-quotes and
newlines.
(David Allsopp, review by Florian Angeletti and Sébastien Hinderer)

### Bug fixes:

- #6654, #9774, #10401: make `include` and with `constraints` handle correctly
Expand Down
3 changes: 3 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -773,6 +773,9 @@ partialclean::

# The runtime system for the bytecode compiler

$(SAK):
$(MAKE) -C runtime sak$(EXE)

.PHONY: runtime
runtime: stdlib/libcamlrun.$(A)

Expand Down
12 changes: 12 additions & 0 deletions Makefile.common
Original file line number Diff line number Diff line change
Expand Up @@ -160,3 +160,15 @@ OCAMLYACCFLAGS ?=

%.ml %.mli: %.mly
$(OCAMLYACC) $(OCAMLYACCFLAGS) $<

SAK = $(ROOTDIR)/runtime/sak$(EXE)

# stdlib/StdlibModules cannot be include'd unless $(SAK) has been built. These
# two rules add that dependency. They have to be pattern rules since
# Makefile.common is included before default targets.
$(ROOTDIR)/%/sak$(EXE):
$(MAKE) -C $(ROOTDIR)/$* sak$(EXE)

ifneq "$(REQUIRES_CONFIGURATION)" ""
$(ROOTDIR)/%/StdlibModules: $(SAK) ;
endif
2 changes: 1 addition & 1 deletion api_docgen/Makefile.common
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@
ROOTDIR = ..
DOCGEN= $(ROOTDIR)/api_docgen

-include $(ROOTDIR)/stdlib/StdlibModules
include $(ROOTDIR)/Makefile.common
include $(ROOTDIR)/stdlib/StdlibModules
include $(ROOTDIR)/Makefile.best_binaries
include $(DOCGEN)/Makefile.docfiles

Expand Down
2 changes: 1 addition & 1 deletion manual/tests/Makefile
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
ROOTDIR = ../..
include $(ROOTDIR)/stdlib/StdlibModules
include $(ROOTDIR)/api_docgen/Makefile.docfiles
include $(ROOTDIR)/Makefile.common
include $(ROOTDIR)/stdlib/StdlibModules
include $(ROOTDIR)/Makefile.best_binaries
STDLIBFLAGS = -nostdlib -I $(ROOTDIR)/stdlib
OCAMLC ?= $(BEST_OCAMLC) $(STDLIBFLAGS)
Expand Down
42 changes: 15 additions & 27 deletions runtime/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ NATIVE_C_SOURCES := $(addsuffix .c, \
dynlink clambda_checks afl bigarray \
memprof domain skiplist codefrag)

GENERATED_HEADERS := caml/opnames.h caml/version.h caml/jumptbl.h
GENERATED_HEADERS := caml/opnames.h caml/version.h caml/jumptbl.h build_config.h
CONFIG_HEADERS := caml/m.h caml/s.h

ifeq "$(TOOLCHAIN)" "msvc"
Expand Down Expand Up @@ -105,25 +105,6 @@ libasmrunpic_OBJECTS := $(NATIVE_C_SOURCES:.c=.npic.$(O)) \

# General (non target-specific) assembler and compiler flags

# On Windows, OCAML_STDLIB_DIR needs to be defined dynamically

ifeq "$(UNIX_OR_WIN32)" "win32"
# OCAML_STDLIB_DIR needs to arrive in dynlink.c as a string which both gcc and
# msvc are willing parse without warning. This means we can't pass UTF-8
# directly since, as far as I can tell, cl can cope, but the pre-processor
# can't. So the string needs to be directly translated to L"" form. To do this,
# we take advantage of the fact that Cygwin uses GNU libiconv which includes a
# Java pseudo-encoding which translates any UTF-8 sequences to \uXXXX (and,
# unlike the C99 pseudo-encoding, emits two surrogate values when needed, rather
# than \UXXXXXXXX). The \u is then translated to \x in order to accommodate
# pre-Visual Studio 2013 compilers where \x is a non-standard alias for \u.
OCAML_STDLIB_DIR = $(shell echo $(LIBDIR)| iconv -t JAVA | sed -e 's/\\u/\\x/g')
STDLIB_CPP_FLAG = -DOCAML_STDLIB_DIR='L"$(OCAML_STDLIB_DIR)"'
else # Unix
OCAML_STDLIB_DIR = $(LIBDIR)
STDLIB_CPP_FLAG = -DOCAML_STDLIB_DIR='"$(OCAML_STDLIB_DIR)"'
endif

ifneq "$(CCOMPTYPE)" "msvc"
OC_CFLAGS += -g
endif
Expand Down Expand Up @@ -161,7 +142,8 @@ endif
# Build, install and clean targets

.PHONY: all
all: $(BYTECODE_STATIC_LIBRARIES) $(BYTECODE_SHARED_LIBRARIES) $(PROGRAMS)
all: $(BYTECODE_STATIC_LIBRARIES) $(BYTECODE_SHARED_LIBRARIES) $(PROGRAMS) \
sak$(EXE)

.PHONY: allopt
ifneq "$(NATIVE_COMPILER)" "false"
Expand Down Expand Up @@ -192,8 +174,8 @@ endif
.PHONY: clean
clean:
rm -f *.o *.obj *.a *.lib *.so *.dll ld.conf
rm -f ocamlrun ocamlrund ocamlruni ocamlruns
rm -f ocamlrun.exe ocamlrund.exe ocamlruni.exe ocamlruns.exe
rm -f ocamlrun ocamlrund ocamlruni ocamlruns sak
rm -f ocamlrun.exe ocamlrund.exe ocamlruni.exe ocamlruns.exe sak.exe
rm -f primitives primitives.new prims.c $(GENERATED_HEADERS)
rm -f domain_state*.inc
rm -rf $(DEPDIR)
Expand Down Expand Up @@ -261,6 +243,16 @@ caml/jumptbl.h : caml/instruct.h
caml/version.h : $(ROOTDIR)/tools/make-version-header.sh $(ROOTDIR)/VERSION
$^ > $@

sak$(EXE): sak.$(O)
$(call MKEXE_USING_COMPILER,$@,$^)

C_LITERAL = $(shell ./sak$(EXE) encode-C-literal '$(1)')

build_config.h: $(ROOTDIR)/Makefile.config sak$(EXE)
echo '/* This file is generated from $(ROOTDIR)/Makefile.config */' > $@
echo '#define OCAML_STDLIB_DIR $(call C_LITERAL,$(LIBDIR))' >> $@
echo '#define HOST "$(HOST)"' >> $@

# Libraries and programs

ocamlrun$(EXE): prims.$(O) libcamlrun.$(A)
Expand Down Expand Up @@ -365,10 +357,6 @@ endif
$(foreach object_type, $(object_types), \
$(eval $(call COMPILE_C_FILE,$(object_type),%)))

dynlink.%.$(O): OC_CPPFLAGS += $(STDLIB_CPP_FLAG)

startup_byt.%.$(O): OC_CPPFLAGS += $(STDLIB_CPP_FLAG) -DHOST='"$(HOST)"'

$(UNIX_OR_WIN32)_non_shared.%.$(O): OC_CPPFLAGS += -DBUILDING_LIBCAMLRUNS

$(eval $(call COMPILE_C_FILE,$(UNIX_OR_WIN32)_non_shared.%,$(UNIX_OR_WIN32)))
Expand Down
2 changes: 2 additions & 0 deletions runtime/dynlink.c
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@
#include "caml/prims.h"
#include "caml/signals.h"

#include "build_config.h"

#ifndef NATIVE_CODE

/* The table of primitives */
Expand Down
148 changes: 148 additions & 0 deletions runtime/sak.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,148 @@
/**************************************************************************/
/* */
/* OCaml */
/* */
/* David Allsopp, OCaml Labs, Cambridge. */
/* */
/* Copyright 2021 David Allsopp Ltd. */
/* */
/* All rights reserved. This file is distributed under the terms of */
/* the GNU Lesser General Public License version 2.1, with the */
/* special exception on linking described in the file LICENSE. */
/* */
/**************************************************************************/

/* Runtime Builder's Swiss Army Knife. This utility performs functions
previously delegated to classic Unix utilities but which ultimately seem to
cause more hassle for maintenance than the initial simplicity suggests.
This tool is a memorial to the many hours and PRs spent chasing down strange
locale issues, stray CR characters and fighting yet another incompatible
implementation of sed or awk. */

/* Borrow the Unicode *_os definitions and T() macro from misc.h */
#define CAML_INTERNALS
#include "caml/misc.h"

#include <stdio.h>
#include <string.h>
#include <ctype.h>

#ifdef _WIN32
#define strncmp_os wcsncmp
#define toupper_os towupper
#define printf_os wprintf
#else
#define strncmp_os strncmp
#define toupper_os toupper
#define printf_os printf
#endif

/* Operations
- encode-C-literal. Used for the OCAML_STDLIB_DIR macro in
runtime/build_config.h to ensure the LIBDIR make variable is correctly
represented as a C string literal.
On Unix, `sak encode-C-literal /usr/local/lib` returns `"/usr/local/lib"`
On Windows, `sak encode-C-literal "C:\OCaml🐫\lib"` returns
`L"C:\\OCaml\xd83d\xdc2b\\lib"`
- add-stdlib-prefix. Used in stdlib/StdlibModules to convert the list of
basenames given in STDLIB_MODULE_BASENAMES to the actual file basenames
in STDLIB_MODULES.
For example, `sak add-stdlib-prefix stdlib camlinternalAtomic Sys` returns
` stdlib camlinternalAtomic stdlib__Sys`
*/

void usage(void)
{
printf(
"OCaml Build System Swiss Army Knife\n"
"Usage: sak command\n"
"Commands:\n"
" * encode-C-literal path - encodes path as a C string literal\n"
" * add-stdlib-prefix name1 ... - prefix standard library module names\n"
);
}

/* Converts the supplied path (UTF-8 on Unix and UCS-2ish on Windows) to a valid
C string literal. On Windows, this is always a wchar_t* (L"..."). */
void encode_C_literal(char_os *path)
{
char_os c;

#ifdef _WIN32
putchar('L');
#endif
putchar('"');

while ((c = *path++) != 0) {
/* Escape \, " and \n */
if (c == '\\') {
printf("\\\\");
} else if (c == '"') {
printf("\\\"");
} else if (c == '\n') {
printf("\\n");
#ifndef _WIN32
/* On Unix, nothing else needs escaping */
} else {
putchar(c);
#else
/* On Windows, allow 7-bit printable characters to be displayed literally
and escape everything else (using the older \x notation for increased
compatibility, rather than the newer \U. */
} else if (c < 0x80 && iswprint(c)) {
putwchar(c);
} else {
printf("\\x%04x", c);
#endif
}
}

putchar('"');
}

/* Print the given array of module names to stdout. "stdlib" and names beginning
"camlinternal" are printed unaltered. All other names are prefixed "stdlib__"
with the original name capitalised (i.e. "foo" prints "stdlib__Foo"). */
void add_stdlib_prefix(int count, char_os **names)
{
int i;
char_os *name;

for (i = 0; i < count; i++) {
name = *names++;

/* "stdlib" and camlinternal* do not get changed. All other names get
capitalised and prefixed "stdlib__". */
if (strcmp_os(T("stdlib"), name) == 0
|| strncmp_os(T("camlinternal"), name, 12) == 0) {
printf_os(T(" %s"), name);
} else {
/* name is a null-terminated string, so an empty string simply has the
null-terminator "capitalised". */
*name = toupper_os(*name);
printf_os(T(" stdlib__%s"), name);
}
}
}

#ifdef _WIN32
int wmain(int argc, wchar_t **argv)
#else
int main(int argc, char **argv)
#endif
{
if (argc == 3 && !strcmp_os(argv[1], T("encode-C-literal"))) {
encode_C_literal(argv[2]);
} else if (argc > 1 && !strcmp_os(argv[1], T("add-stdlib-prefix"))) {
add_stdlib_prefix(argc - 2, &argv[2]);
} else {
usage();
return 1;
}

return 0;
}
2 changes: 2 additions & 0 deletions runtime/startup_byt.c
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,8 @@
#include "caml/startup_aux.h"
#include "caml/version.h"

#include "build_config.h"

#ifndef O_BINARY
#define O_BINARY 0
#endif
Expand Down
26 changes: 9 additions & 17 deletions stdlib/StdlibModules
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
#* *
#**************************************************************************

# This file must be self-contained.
# This file should be included after Makefile.common

# This file lists all standard library modules. It is used by:
# 1. stdlib/Makefile when building stdlib.cma
Expand All @@ -33,7 +33,7 @@

# Basenames of the source files for the standard library (i.e. unprefixed and
# with lowercase first letters). These must be listed in dependency order.
STDLIB_MODULE_BASENAMES=\
STDLIB_MODULE_BASENAMES = \
camlinternalFormatBasics camlinternalAtomic \
stdlib pervasives seq option either result bool char uchar \
sys list int bytes string unit marshal obj array float int32 int64 nativeint \
Expand All @@ -44,20 +44,12 @@ STDLIB_MODULE_BASENAMES=\
filename complex arrayLabels listLabels bytesLabels stringLabels moreLabels \
stdLabels bigarray

STDLIB_PREFIXED_MODULES=\
STDLIB_PREFIXED_MODULES = \
$(filter-out stdlib camlinternal%, $(STDLIB_MODULE_BASENAMES))

define add_stdlib_prefix_first
$(shell echo $1 | cut -c1 | tr '[:lower:]' '[:upper:]')
endef

# add stdlib__ as prefix to a module except for internal modules
# and the stdlib module itself
define add_stdlib_prefix
$(or $(filter-out $(STDLIB_PREFIXED_MODULES), $1), \
stdlib__$(call add_stdlib_prefix_first,$1)$(shell echo $1 | cut -c2-))
endef

STDLIB_MODULES:=\
$(foreach module, $(STDLIB_MODULE_BASENAMES), \
$(call add_stdlib_prefix,$(module)))
# The pattern FOO = $(eval FOO := $$(shell <cmd>)$(FOO) ensures that <cmd> is
# executed either once or not at all, giving us GNU make's equivalent of a
# string lazy_t.
STDLIB_MODULES = \
$(eval STDLIB_MODULES := $$(shell \
$(SAK) add-stdlib-prefix $(STDLIB_MODULE_BASENAMES)))$(STDLIB_MODULES)

0 comments on commit 7e78d17

Please sign in to comment.