Delete stdlib/StdlibModules

The link order for the Standard Library modules is derived from
stdlib/.depend using a small Makefile. The result of this is then fed to
runtime/sak which performs the required transformations to generate
stdlib/Makefile.stdlib_modules.

On Windows, the C auxiliary is considerably faster than the large number
of calls to tr and cut which were necessary in stdlib/StdlibModules.

.gitignore

@@ -211,6 +211,7 @@ _build
 /stdlib/labelled-*
 /stdlib/caml
 /stdlib/sys.ml
+/stdlib/Makefile.stdlib_modules
 
 /testsuite/**/*.result
 /testsuite/**/*.opt_result

Changes

@@ -215,11 +215,11 @@ OCaml 4.13.0
   (Gabriel Scherer, review by Nicolás Ojeda Bär, Alain Frisch, Xavier Leroy,
    Daniel Bünzli and Stephen Dolan)
 
-* #10169, #10270, #10301: Use capitalized module names in the Standard Library
-  prefixing scheme to match Dune, e.g. Stdlib__String instead of Stdlib__string.
-  This is a breaking change only to code which attempted to use the internal
-  names before. The Standard Library generated by the Dune rules is now
-  equivalent to the main build (the Dune rules still do not generate a
+* #10169, #10270, #10301, #10451: Use capitalized module names in the Standard
+  Library prefixing scheme to match Dune, e.g. Stdlib__String instead of
+  Stdlib__string. This is a breaking change only to code which attempted to use
+  the internal names before. The Standard Library generated by the Dune rules is
+  now equivalent to the main build (the Dune rules still do not generate a
   distributable compiler).
   (David Allsopp and Mark Shinwell, review by Gabriel Scherer)
 

Makefile

@@ -35,8 +35,6 @@ else
 LN = ln -sf
 endif
 
-include stdlib/StdlibModules
-
 CAMLC=$(BOOT_OCAMLC) -g -nostdlib -I boot -use-prims runtime/primitives
 CAMLOPT=$(OCAMLRUN) ./ocamlopt$(EXE) -g -nostdlib -I stdlib -I otherlibs/dynlink
 ARCHES=amd64 i386 arm arm64 power s390x riscv
@@ -75,6 +73,7 @@ TOPLEVELINIT=toplevel/toploop.cmo
 
 # This list is passed to expunge, which accepts both uncapitalized and
 # capitalized module names.
+-include stdlib/Makefile.stdlib_modules
 PERVASIVES=$(STDLIB_MODULES) outcometree topdirs toploop
 
 LIBFILES=stdlib.cma std_exit.cmo *.cmi camlheader

Makefile.common

@@ -160,3 +160,5 @@ OCAMLYACCFLAGS ?=
 
 %.ml %.mli: %.mly
 	$(OCAMLYACC) $(OCAMLYACCFLAGS) $<
+
+SAK = $(ROOTDIR)/runtime/sak$(EXE)

api_docgen/Makefile.common

@@ -15,7 +15,7 @@
 ROOTDIR = ..
 DOCGEN= $(ROOTDIR)/api_docgen
 
--include $(ROOTDIR)/stdlib/StdlibModules
+-include $(ROOTDIR)/stdlib/Makefile.stdlib_modules
 include $(ROOTDIR)/Makefile.common
 include $(ROOTDIR)/Makefile.best_binaries
 include $(DOCGEN)/Makefile.docfiles

manual/tests/Makefile

@@ -1,7 +1,7 @@
 ROOTDIR = ../..
-include $(ROOTDIR)/stdlib/StdlibModules
 include $(ROOTDIR)/api_docgen/Makefile.docfiles
 include $(ROOTDIR)/Makefile.common
+-include $(ROOTDIR)/stdlib/Makefile.stdlib_modules
 include $(ROOTDIR)/Makefile.best_binaries
 STDLIBFLAGS = -nostdlib -I $(ROOTDIR)/stdlib
 OCAMLC ?= $(BEST_OCAMLC) $(STDLIBFLAGS)

runtime/sak.c

@@ -25,10 +25,32 @@
 #include "caml/misc.h"
 
 #include <stdio.h>
+#include <errno.h>
+#include <fcntl.h>
 #include <string.h>
+#include <ctype.h>
+
+#ifdef HAS_UNISTD
+#include <unistd.h>
+#endif
 
 #ifdef _WIN32
 #include <windows.h>
+#include <io.h>
+#define lseek _lseeki64
+#endif
+
+#ifndef SEEK_SET
+#define SEEK_SET 0
+#define SEEK_CUR 1
+#define SEEK_END 2
+#endif
+
+#ifndef _O_BINARY
+#define _O_BINARY 0
+#endif
+#ifndef _O_TEXT
+#define _O_TEXT 0
 #endif
 
 void usage(void)
@@ -38,9 +60,87 @@ void usage(void)
     "Usage: sak command\n"
     "Commands:\n"
     " * translate path - converts path to a C string constant\n"
+    " * prefix - produce module lists for the standard library\n"
   );
 }
 
+void die(const char* format, ...)
+{
+  va_list args;
+  fputs("Fatal error: ", stderr);
+  va_start(args, format);
+  vfprintf(stderr, format, args);
+  va_end(args);
+  exit(1);
+}
+
+void *xmalloc(size_t size)
+{
+  void *block = malloc(size);
+  if (block == NULL)
+    die("out of memory");
+  return block;
+}
+
+/* Read the contents of path to a freshly allocated buffer. No CRLF translation
+   takes place on any platform. Returns NULL on any error (including malloc). */
+char *read_file(char_os *path)
+{
+  char *result = NULL, *buf;
+  int flags = O_RDONLY | _O_BINARY;
+  int fd, size, nbytes;
+
+  if ((fd = open_os(path, flags, 0)) == -1)
+    return NULL;
+
+  /* Determine the size of the file and allocate a buffer for it */
+  if ((size = lseek(fd, 0, SEEK_END)) != -1
+      && lseek(fd, 0, SEEK_SET) != -1
+      && (result = buf = (char *)malloc(size + 1)) != NULL) {
+    result[size] = 0;
+
+    while (size > 0) {
+      if ((nbytes = read(fd, buf, size)) == -1) {
+        if (errno != EAGAIN) {
+          free(result);
+          result = NULL;
+          size = 0;
+        }
+      } else {
+        size -= nbytes;
+        buf += nbytes;
+      }
+    }
+  }
+
+  close(fd);
+
+  return result;
+}
+
+/* Return a pointer either to the first character _after_ the next \n character
+   or the nul terminator at the end of the string if there isn't one. The \n or
+   the leftmost \r immediately preceding it is changed to nul thus after:
+   char *next = scan_to_eol(line);
+   line points to a CRLF-normalised single line of text and next is the
+   character at which to begin scanning for the next line. */
+char *scan_to_eol(char *p)
+{
+  char *q = p, *r;
+  while (*q != 0 && *q != '\n')
+    q++;
+  r = q;
+  /* If not at the end of the buffer, advance to next character */
+  if (*r != 0)
+    r++;
+  /* Backtrack over any previous CR characters */
+  while (q > p && *(q - 1) == '\r')
+    q--;
+  /* q now points to the first character of \r*\n at the end of the line */
+  *q = 0;
+  return r;
+}
+
 /* Converts the supplied path (UTF-8 on Unix and UCS-2ish on Windows) to a valid
    C string literal. On Windows, this is always a wchar_t (L"..."). */
 void emit_c_string(char_os *path)
@@ -79,6 +179,87 @@ void emit_c_string(char_os *path)
   putchar('"');
 }
 
+/* Prints a null-terminated list of strings as a Makefile definition. */
+void print_definition(const char *name, char *buffer)
+{
+  printf("%s =", name);
+
+  while (*buffer != 0) {
+    printf(" \\\n  %s", buffer);
+    buffer += strlen(buffer) + 1;
+  }
+  putchar('\n');
+}
+
+/* Process the list of modules in file to produce the definitions needed for
+   Makefile.stdlib_modules. */
+void prefix_stdlib_modules(char_os *file)
+{
+  char *content, *p, *q;
+  char *basenames, *prefixed, *p_basenames, *p_prefixed;
+  int sz;
+
+  if ((p = content = read_file(file)) == NULL)
+    die("unable to read input file");
+
+  sz = strlen(content) + 1;
+  p_basenames = basenames = (char *)xmalloc(sz);
+  p_prefixed = prefixed = (char *)xmalloc(sz);
+
+  /* The list of modules is converted into three lists in the Makefile:
+     - STDLIB_MODULE_BASENAMES - basenames, in dependency order, of all the
+         modules in the standard library
+     - STDLIB_PREFIXED_MODULES - just the namespaced modules of
+          $(STDLIB_MODULE_BASENAMES), i.e. without camlinternal* and stdlib.
+          Used in stdlib/Makefile to munge the dependencies.
+     - STDLIB_MODULES - full list, in prefixed form as appropriate. */
+  printf("STDLIB_MODULES =");
+
+  while (*p != 0) {
+    q = scan_to_eol(p);
+
+    /* Print the raw name for STDLIB_MODULES */
+    printf(" \\\n  %s", p);
+
+    /* Add the unprefixed form to STDLIB_MODULE_BASENAMES */
+    if (strncmp("stdlib__", p, 8) == 0) {
+      /* Prefixed name: skip over the stdlib__ */
+      p += 8;
+      sz = strlen(p) + 1;
+      /* stdlib__Arg -> stdlib__arg */
+      *p = tolower(*p);
+      /* arg copied to both buffers */
+      strcpy(basenames, p);
+      strcpy(prefixed, p);
+      basenames += sz;
+      /* Add the basename to STDLIB_PREFIXED_MODULES */
+      prefixed += sz;
+    } else {
+      /* Un-prefixed name: doesn't go in prefixed, therefore. */
+      strcpy(basenames, p);
+      basenames += strlen(p) + 1;
+    }
+
+    p = q;
+  }
+
+  putchar('\n');
+
+  *basenames = 0;
+  *prefixed = 0;
+
+  basenames = p_basenames;
+  prefixed = p_prefixed;
+
+  /* Emit STDLIB_MODULE_BASENAMES and STDLIB_PREFIXED_MODULES */
+  print_definition("STDLIB_MODULE_BASENAMES", basenames);
+  print_definition("STDLIB_PREFIXED_MODULES", prefixed);
+
+  free(content);
+  free(p_basenames);
+  free(p_prefixed);
+}
+
 #ifdef _WIN32
 int wmain(int argc, wchar_t **argv)
 #else
@@ -87,6 +268,8 @@ int main(int argc, char **argv)
 {
   if (argc == 3 && !strcmp_os(argv[1], T("translate"))) {
     emit_c_string(argv[2]);
+  } else if (argc == 3 && !strcmp_os(argv[1], T("prefix"))) {
+    prefix_stdlib_modules(argv[2]);
   } else {
     usage();
     return 1;