Merge pull request #53 from suborbital/connor/static-files

Runnable API to fetch static files from the bundle

README.md

@@ -6,7 +6,7 @@ Reactr runs functions called Runnables, and transparently spawns workers to proc
 
 ## Wasm
 
-Reactr has support for Wasm-packaged Runnables. The `rwasm` package contains a multi-tenant Wasm scheduler, an API to grant capabilities to Wasm Runnables, and support for several languages including Rust and Swift. See [wasm](./docs/wasm.md) and the [subo CLI](https://github.com/suborbital/subo) for details.
+Reactr has support for Wasm-packaged Runnables. The `rwasm` package contains a multi-tenant Wasm scheduler, an API to grant capabilities to Wasm Runnables, and support for several languages including Rust (stable) and Swift (experimental). See [wasm](./docs/wasm.md) and the [subo CLI](https://github.com/suborbital/subo) for details.
 
 ## FaaS
 

api/rust/suborbital/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "suborbital"
-version = "0.5.0"
+version = "0.6.0"
 authors = ["cohix <connorjhicks@gmail.com>"]
 edition = "2018"
 description = "Suborbital Wasm Runnable API"

api/rust/suborbital/src/lib.rs

@@ -376,6 +376,47 @@ pub mod log {
     }
 }
 
+pub mod file {
+    use std::slice;
+
+    extern {
+        fn get_static_file(name_ptr: *const u8, name_size: i32, dest_ptr: *const u8, dest_max_size: i32, ident: i32) -> i32;
+    }
+
+    pub fn get_static(name: &str) -> Option<Vec<u8>> {
+        let mut dest_pointer: *const u8;
+        let mut result_size: i32;
+        let mut capacity: i32 = 1024;
+
+        // make the request, and if the response size is greater than that of capacity, increase the capacity and try again
+        loop {
+            let cap = &mut capacity;
+
+            let mut dest_bytes = Vec::with_capacity(*cap as usize);
+            let dest_slice = dest_bytes.as_mut_slice();
+            dest_pointer = dest_slice.as_mut_ptr() as *const u8;
+
+            // do the request over FFI
+            result_size = unsafe { get_static_file(name.as_ptr(), name.len() as i32, dest_pointer, *cap, super::STATE.ident) };
+
+            if result_size < 0 {
+                return None;
+            } else if result_size > *cap {
+                super::log::info(format!("increasing capacity, need {}", result_size).as_str());
+                *cap = result_size;
+            } else {
+                break;
+            }
+        }
+
+        let result: &[u8] = unsafe {
+            slice::from_raw_parts(dest_pointer, result_size as usize)
+        };
+
+        Some(Vec::from(result))
+    }
+}
+
 pub mod util {
     pub fn to_string(input: Vec<u8>) -> String {
         String::from_utf8(input).unwrap()

api/swift/Sources/suborbital/lib.swift

@@ -16,6 +16,9 @@ func cache_get(key_pointer: UnsafeRawPointer, key_size: Int32, dest_pointer: Uns
 @_silgen_name("request_get_field_swift")
 func request_get_field(field_type: Int32, key_pointer: UnsafeRawPointer, key_size: Int32, dest_pointer: UnsafeRawPointer, dest_max_size: Int32, ident: Int32) -> Int32
 
+@_silgen_name("get_static_file_swift")
+func get_static_file(name_pointer: UnsafeRawPointer, name_size: Int32, dest_pointer: UnsafeRawPointer, dest_max_size: Int32, ident: Int32) -> Int32
+
 // keep track of the current ident
 var CURRENT_IDENT: Int32 = 0
 
@@ -213,6 +216,35 @@ func requestGetField(fieldType: Int32, key: String) -> String {
     return retVal
 }
 
+public func GetStaticFile(name: String) -> String {    
+    var maxSize: Int32 = 256000
+    var retVal = ""
+
+    // loop until the returned size is within the defined max size, increasing it as needed
+    var done = false
+    while !done {
+        toFFI(val: name, use: { (namePtr: UnsafePointer<Int8>, nameSize: Int32) in
+            let ptr = allocate(size: Int32(maxSize))
+
+            let resultSize = get_static_file(name_pointer: namePtr, name_size: nameSize, dest_pointer: ptr, dest_max_size: maxSize, ident: CURRENT_IDENT)
+
+            if resultSize == 0 {
+                done = true
+            } else if resultSize < 0 {
+                retVal = "failed to get file"
+                done = true
+            } else if resultSize > maxSize {
+                maxSize = resultSize
+            } else {
+                retVal = fromFFI(ptr: ptr, size: resultSize)
+                done = true
+            }
+        })
+    }
+
+    return retVal
+}
+
 @_cdecl("run_e")
 func run_e(pointer: UnsafeRawPointer, size: Int32, ident: Int32) {
     CURRENT_IDENT = ident

bundle/bundle.go

@@ -3,6 +3,7 @@ package bundle
 import (
 	"archive/zip"
 	"bytes"
+	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -12,10 +13,15 @@ import (
 	"github.com/suborbital/reactr/directive"
 )
 
+// FileFunc is a function that returns the contents of a requested file
+type FileFunc func(string) ([]byte, error)
+
 // Bundle represents a Runnable bundle
 type Bundle struct {
-	Directive *directive.Directive
-	Runnables []WasmModuleRef
+	filepath    string
+	Directive   *directive.Directive
+	Runnables   []WasmModuleRef
+	staticFiles map[string]bool
 }
 
 // WasmModuleRef is a reference to a Wasm module (either its filepath or its bytes)
@@ -25,9 +31,43 @@ type WasmModuleRef struct {
 	data     []byte
 }
 
+// StaticFile returns a static file from the bundle, if it exists
+func (b *Bundle) StaticFile(filePath string) ([]byte, error) {
+	if _, exists := b.staticFiles[filePath]; !exists {
+		return nil, os.ErrNotExist
+	}
+
+	r, err := zip.OpenReader(b.filepath)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to open bundle")
+	}
+
+	staticFilePath := ensurePrefix(filePath, "static/")
+
+	var contents []byte
+
+	for _, f := range r.File {
+		if f.Name == staticFilePath {
+			file, err := f.Open()
+			if err != nil {
+				return nil, errors.Wrap(err, "failed to Open static file")
+			}
+
+			contents, err = ioutil.ReadAll(file)
+			if err != nil {
+				return nil, errors.Wrap(err, "failed to ReadAll static file")
+			}
+
+			break
+		}
+	}
+
+	return contents, nil
+}
+
 // Write writes a runnable bundle
 // based loosely on https://golang.org/src/archive/zip/example_test.go
-func Write(directive *directive.Directive, files []os.File, targetPath string) error {
+func Write(directive *directive.Directive, files []os.File, staticFiles []os.File, targetPath string) error {
 	if directive == nil {
 		return errors.New("directive must be provided")
 	}
@@ -60,6 +100,19 @@ func Write(directive *directive.Directive, files []os.File, targetPath string) e
 		}
 	}
 
+	// Add static files to the archive.
+	for _, file := range staticFiles {
+		contents, err := ioutil.ReadAll(&file)
+		if err != nil {
+			return errors.Wrapf(err, "failed to read file %s", file.Name())
+		}
+
+		fileName := fmt.Sprintf("static/%s", filepath.Base(file.Name()))
+		if err := writeFile(w, fileName, contents); err != nil {
+			return errors.Wrap(err, "failed to writeFile into bundle")
+		}
+	}
+
 	if err := w.Close(); err != nil {
 		return errors.Wrap(err, "failed to close bundle writer")
 	}
@@ -110,11 +163,12 @@ func Read(path string) (*Bundle, error) {
 	defer r.Close()
 
 	bundle := &Bundle{
-		Runnables: []WasmModuleRef{},
+		filepath:    path,
+		Runnables:   []WasmModuleRef{},
+		staticFiles: map[string]bool{},
 	}
 
 	// Iterate through the files in the archive,
-
 	for _, f := range r.File {
 		if f.Name == "Directive.yaml" {
 			directive, err := readDirective(f)
@@ -124,6 +178,11 @@ func Read(path string) (*Bundle, error) {
 
 			bundle.Directive = directive
 			continue
+		} else if strings.HasPrefix(f.Name, "static/") {
+			// build up the list of available static files in the bundle for quick reference later
+			filePath := strings.TrimPrefix(f.Name, "static/")
+			bundle.staticFiles[filePath] = true
+			continue
 		} else if !strings.HasSuffix(f.Name, ".wasm") {
 			continue
 		}
@@ -197,3 +256,11 @@ func (w *WasmModuleRef) ModuleBytes() ([]byte, error) {
 
 	return w.data, nil
 }
+
+func ensurePrefix(val, prefix string) string {
+	if strings.HasPrefix(val, prefix) {
+		return val
+	}
+
+	return fmt.Sprintf("%s%s", prefix, val)
+}

bundle/bundlewritetester/main.go

@@ -12,8 +12,8 @@ import (
 
 func main() {
 	files := []os.File{}
-	for _, filename := range []string{"fetch.wasm", "log_example.wasm", "example.wasm"} {
-		path := filepath.Join("./", "wasm", "testdata", filename)
+	for _, filename := range []string{"fetch/fetch.wasm", "log/log.wasm", "hello-echo/hello-echo.wasm"} {
+		path := filepath.Join("./", "rwasm", "testdata", filename)
 
 		file, err := os.Open(path)
 		if err != nil {
@@ -23,6 +23,18 @@ func main() {
 		files = append(files, *file)
 	}
 
+	staticFiles := []os.File{}
+	for _, filename := range []string{"go.mod", "go.sum", "Makefile"} {
+		path := filepath.Join("./", filename)
+
+		file, err := os.Open(path)
+		if err != nil {
+			log.Fatal("failed to open file", err)
+		}
+
+		staticFiles = append(staticFiles, *file)
+	}
+
 	directive := &directive.Directive{
 		Identifier:  "dev.suborbital.appname",
 		AppVersion:  "v0.1.1",
@@ -33,11 +45,11 @@ func main() {
 				Namespace: "default",
 			},
 			{
-				Name:      "log_example",
+				Name:      "log",
 				Namespace: "default",
 			},
 			{
-				Name:      "example",
+				Name:      "hello-echo",
 				Namespace: "default",
 			},
 		},
@@ -56,13 +68,13 @@ func main() {
 								As: "ghData",
 							},
 							{
-								Fn: "log_example",
+								Fn: "log",
 							},
 						},
 					},
 					{
 						CallableFn: directive.CallableFn{
-							Fn: "example",
+							Fn: "hello-echo",
 							With: []string{
 								"data: ghData",
 							},
@@ -78,9 +90,21 @@ func main() {
 		log.Fatal("failed to validate directive: ", err)
 	}
 
-	if err := bundle.Write(directive, files, "./runnables.wasm.zip"); err != nil {
+	if err := bundle.Write(directive, files, staticFiles, "./runnables.wasm.zip"); err != nil {
 		log.Fatal("failed to WriteBundle", err)
 	}
 
+	bdl, err := bundle.Read("./runnables.wasm.zip")
+	if err != nil {
+		log.Fatal("failed to re-read bundle:", err)
+	}
+
+	file, err := bdl.StaticFile("go.mod")
+	if err != nil {
+		log.Fatal("failed to StaticFile:", err)
+	}
+
+	fmt.Println(string(file))
+
 	fmt.Println("done ✨")
 }

docs/wasm.md

@@ -1,18 +1,18 @@
-# Reactr ❤️ Wasm
+# Reactr ❤️ WebAssembly
 
-Reactr has first-class support for Wasm-packaged Runnables. Wasm is an incredibly useful modern portable binary format that allows multiple languages to be compiled into .wasm modules.
+Reactr has first-class support for WebAssembly-packaged Runnables. Wasm is an incredibly useful modern portable binary format that allows multiple languages to be compiled into .wasm modules.
 
-Wasm support in Reactr is powered by [Wasmer](https://github.com/wasmerio/wasmer-go), the hard work they've done to create a powerful Wasm runtime that is extensible has been very much appreciated, and it's been very cool seeing that project grow.
+Wasm support in Reactr is powered by [Wasmer](https://github.com/wasmerio/wasmer-go), the hard work they've done to create a powerful and extensible Wasm runtime has been very much appreciated, and it's been very cool seeing that project grow.
 
-The current supported languages are Rust and Swift, and the Runnable API is available for each. More languages such as AssemblyScript, Go, and C++ are coming soon!
+The current supported languages are Rust (stable) and Swift (experimental). The Runnable API is available for each. More languages such as AssemblyScript, Go, and C++ are coming soon!
 
 To create a Wasm runnable, check out the [subo CLI](https://github.com/suborbital/subo). Once you've generated a `.wasm` file, you can use it with Reactr just like any other Runnable!
 
-Due to the memory limitations of Wasm, Wasm runners accept bytes (rather than arbitrary input) and return bytes. Reactr will handle the conversion of inputs and outputs automatically. Wasm runners cannot currently schedule other jobs, though support for that is coming.
+A multitude of example Runnables can be found in the [testdata directory](../rwasm/testdata).
 
-Here's how to use Wasm Runnables:
+Due to the memory layout of WebAssembly, Wasm runners accept bytes (rather than arbitrary input) and return bytes. Reactr will handle the conversion of inputs and outputs automatically. Wasm runners cannot currently schedule other jobs.
 
-First, install Reactr's Wasm package `rwasm`:
+To get started with Wasm Runnables, install Reactr's WebAssembly package `rwasm`:
 ```bash
 go get github.com/suborbital/reactr/rwasm
 ```

request/request.go

@@ -93,6 +93,10 @@ func FromJSON(jsonBytes []byte) (*CoordinatedRequest, error) {
 		return nil, errors.Wrap(err, "failed to Unmarshal request")
 	}
 
+	if req.Method == "" || req.URL == "" || req.ID == "" {
+		return nil, errors.New("JSON is not CoordinatedRequest, required fields are empty")
+	}
+
 	return &req, nil
 }
 

rwasm/api_cache.go

@@ -1,4 +1,4 @@
-package wasm
+package rwasm
 
 import (
 	"github.com/pkg/errors"

rwasm/api_http.go

@@ -1,4 +1,4 @@
-package wasm
+package rwasm
 
 import (
 	"bytes"
@@ -120,3 +120,20 @@ func fetch_url(method int32, urlPointer int32, urlSize int32, bodyPointer int32,
 
 	return int32(len(respBytes))
 }
+
+func parseHTTPHeaders(urlParts []string) (*http.Header, error) {
+	headers := &http.Header{}
+
+	if len(urlParts) > 1 {
+		for _, p := range urlParts[1:] {
+			headerParts := strings.Split(p, ":")
+			if len(headerParts) != 2 {
+				return nil, errors.New("header was not formatted correctly")
+			}
+
+			headers.Add(headerParts[0], headerParts[1])
+		}
+	}
+
+	return headers, nil
+}