.github/ISSUE_TEMPLATE

@@ -1,12 +1,17 @@
+<!---
+This issue tracker is mainly for bugs and feature requests. Before asking a
+question, search online and try to investigate on your own.
+-->
+
 #### What versions are you running?
 
 <pre>
 $ go list -m github.com/chromedp/chromedp
-$ chromium --version
+$ google-chrome --version
 $ go version
 </pre>
 
-#### What did you do?
+#### What did you do? Include clear steps.
 
 
 #### What did you expect to see?

.gitignore

@@ -3,10 +3,10 @@ out*.txt
 old*.txt
 cdp-*.log
 cdp-*.txt
+*.out
 
-coverage.out
-massif.out
+/chromedp.test
+/chromedp.test.exe
 
-# headless_shell
-/headless_shell/
-/hs/
+/*.png
+/*.pdf

.travis.yml

@@ -5,6 +5,9 @@ language: go
 go:
   - 1.12.x
 
+env:
+  - GO111MODULE=on
+
 services:
   - docker
 
@@ -13,6 +16,6 @@ addons:
     chrome: stable
 
 script:
-  - go test
+  - go test -v
   - go test -c
-  - docker run --volume=$PWD:/chromedp --entrypoint=/chromedp/chromedp.test --workdir=/chromedp --env=PATH=/headless-shell chromedp/headless-shell:latest
+  - docker run --rm --volume=$PWD:/chromedp --entrypoint=/chromedp/chromedp.test --workdir=/chromedp --env=PATH=/headless-shell chromedp/headless-shell:latest -test.v

README.md

@@ -1,8 +1,8 @@
 # About chromedp [![GoDoc][1]][2] [![Build Status][3]][4]
 
 Package chromedp is a faster, simpler way to drive browsers supporting the
-[Chrome DevTools Protocol][5] in Go using the without external dependencies
-(like Selenium or PhantomJS).
+[Chrome DevTools Protocol][5] in Go without external dependencies (like
+Selenium or PhantomJS).
 
 ## Installing
 
@@ -12,17 +12,61 @@ Install in the usual Go way:
 
 ## Examples
 
-Refer to the [GoDoc page][7] for the documentation and examples. The
-[examples][6] repository contains more complex scenarios.
+Refer to the [GoDoc page][7] for the documentation and examples. Additionally,
+the [examples][6] repository contains more complex examples.
+
+## Frequently Asked Questions
+
+> I can't see any Chrome browser window
+
+By default, Chrome is run in headless mode. See `DefaultExecAllocatorOptions`, and
+[an example](https://godoc.org/github.com/chromedp/chromedp#example-ExecAllocator)
+to override the default options.
+
+> I'm seeing "context canceled" errors
+
+When the connection to the browser is lost, `chromedp` cancels the context, and
+it may result in this error. This occurs, for example, if the browser is closed
+manually, or if the browser process has been killed or otherwise terminated.
+
+> Chrome exits as soon as my Go program finishes
+
+On Linux, `chromedp` is configured to avoid leaking resources by force-killing
+any started Chrome child processes. If you need to launch a long-running Chrome
+instance, manually start Chrome and connect using `RemoteAllocator`.
+
+> Executing an action without `Run` results in "invalid context"
+
+By default, a `chromedp` context does not have an executor, however one can be
+specified manually if necessary; see [issue #326](https://github.com/chromedp/chromedp/issues/326)
+for an example.
+
+> I can't use an `Action` with `Run` because it returns many values
+
+Wrap it with an `ActionFunc`:
+
+```go
+chromedp.Run(ctx, chromedp.ActionFunc(func(ctx context.Context) error {
+	_, err := domain.SomeAction().Do(ctx)
+	return err
+}))
+```
+
+> I want to use chromedp on a headless environment
+
+The simplest way is to run the Go program that uses chromedp inside the
+[chromedp/headless-shell][8] image. That image contains `headless-shell`, a
+smaller headless build of Chrome, which `chromedp` is able to find out of the
+box.
 
 ## Resources
 
-* [chromedp: A New Way to Drive the Web][8] - GopherCon SG 2017 talk
+* [chromedp: A New Way to Drive the Web][9] - GopherCon SG 2017 talk
 * [Chrome DevTools Protocol][5] - Chrome DevTools Protocol Domain documentation
 * [chromedp examples][6] - various `chromedp` examples
-* [`github.com/chromedp/cdproto`][9] - GoDoc listing for the CDP domains used by `chromedp`
-* [`github.com/chromedp/cdproto-gen`][10] - tool used to generate `cdproto`
-* [`github.com/chromedp/chromedp-proxy`][11] - a simple CDP proxy for logging CDP clients and browsers
+* [`github.com/chromedp/cdproto`][10] - GoDoc listing for the CDP domains used by `chromedp`
+* [`github.com/chromedp/cdproto-gen`][11] - tool used to generate `cdproto`
+* [`github.com/chromedp/chromedp-proxy`][12] - a simple CDP proxy for logging CDP clients and browsers
 
 [1]: https://godoc.org/github.com/chromedp/chromedp?status.svg
 [2]: https://godoc.org/github.com/chromedp/chromedp
@@ -31,7 +75,8 @@ Refer to the [GoDoc page][7] for the documentation and examples. The
 [5]: https://chromedevtools.github.io/devtools-protocol/
 [6]: https://github.com/chromedp/examples
 [7]: https://godoc.org/github.com/chromedp/chromedp
-[8]: https://www.youtube.com/watch?v=_7pWCg94sKw
-[9]: https://godoc.org/github.com/chromedp/cdproto
-[10]: https://github.com/chromedp/cdproto-gen
-[11]: https://github.com/chromedp/chromedp-proxy
+[8]: https://hub.docker.com/r/chromedp/headless-shell/
+[9]: https://www.youtube.com/watch?v=_7pWCg94sKw
+[10]: https://godoc.org/github.com/chromedp/cdproto
+[11]: https://github.com/chromedp/cdproto-gen
+[12]: https://github.com/chromedp/chromedp-proxy

allocate.go

@@ -1,14 +1,13 @@
 package chromedp
 
 import (
-	"bufio"
+	"bytes"
 	"context"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
 	"os/exec"
-	"strings"
 	"sync"
 )
 
@@ -45,8 +44,9 @@ func setupExecAllocator(opts ...ExecAllocatorOption) *ExecAllocator {
 }
 
 // DefaultExecAllocatorOptions are the ExecAllocator options used by NewContext
-// if the given parent context doesn't have an allocator set up.
-var DefaultExecAllocatorOptions = []ExecAllocatorOption{
+// if the given parent context doesn't have an allocator set up. Do not modify
+// this global; instead, use NewExecAllocator. See ExampleExecAllocator.
+var DefaultExecAllocatorOptions = [...]ExecAllocatorOption{
 	NoFirstRun,
 	NoDefaultBrowserCheck,
 	Headless,
@@ -91,7 +91,7 @@ func NewExecAllocator(parent context.Context, opts ...ExecAllocatorOption) (cont
 }
 
 // ExecAllocatorOption is a exec allocator option.
-type ExecAllocatorOption func(*ExecAllocator)
+type ExecAllocatorOption = func(*ExecAllocator)
 
 // ExecAllocator is an Allocator which starts new browser processes on the host
 // machine.
@@ -100,6 +100,8 @@ type ExecAllocator struct {
 	initFlags map[string]interface{}
 
 	wg sync.WaitGroup
+
+	combinedOutputWriter io.Writer
 }
 
 // allocTempDir is used to group all ExecAllocator temporary user data dirs in
@@ -162,13 +164,14 @@ func (a *ExecAllocator) Allocate(ctx context.Context, opts ...BrowserOption) (*B
 	}()
 	allocateCmdOptions(cmd)
 
-	// We must start the cmd before calling cmd.Wait, as otherwise the two
-	// can run into a data race.
-	stderr, err := cmd.StderrPipe()
+	stdout, err := cmd.StdoutPipe()
 	if err != nil {
 		return nil, err
 	}
-	defer stderr.Close()
+	cmd.Stderr = cmd.Stdout
+
+	// We must start the cmd before calling cmd.Wait, as otherwise the two
+	// can run into a data race.
 	if err := cmd.Start(); err != nil {
 		return nil, err
 	}
@@ -179,6 +182,9 @@ func (a *ExecAllocator) Allocate(ctx context.Context, opts ...BrowserOption) (*B
 	case <-c.allocated: // for this browser's root context
 	}
 	a.wg.Add(1) // for the entire allocator
+	if a.combinedOutputWriter != nil {
+		a.wg.Add(1) // for the io.Copy in a separate goroutine
+	}
 	go func() {
 		<-ctx.Done()
 		// First wait for the process to be finished.
@@ -195,8 +201,14 @@ func (a *ExecAllocator) Allocate(ctx context.Context, opts ...BrowserOption) (*B
 		a.wg.Done()
 		close(c.allocated)
 	}()
-	wsURL, err := addrFromStderr(stderr)
+
+	wsURL, err := readOutput(stdout, a.combinedOutputWriter, a.wg.Done)
 	if err != nil {
+		if a.combinedOutputWriter != nil {
+			// There's no io.Copy goroutine to call the done func.
+			// TODO: a cleaner way to deal with this edge case?
+			a.wg.Done()
+		}
 		return nil, err
 	}
 
@@ -215,32 +227,50 @@ func (a *ExecAllocator) Allocate(ctx context.Context, opts ...BrowserOption) (*B
 	return browser, nil
 }
 
-// addrFromStderr finds the free port that Chrome selected for the debugging
-// protocol. This should be hooked up to a new Chrome process's Stderr pipe
-// right after it is started.
-func addrFromStderr(rc io.ReadCloser) (string, error) {
-	defer rc.Close()
-	url := ""
-	scanner := bufio.NewScanner(rc)
-	prefix := "DevTools listening on"
-
-	var lines []string
-	for scanner.Scan() {
-		line := scanner.Text()
-		if s := strings.TrimPrefix(line, prefix); s != line {
-			url = strings.TrimSpace(s)
-			break
+// readOutput grabs the websocket address from chrome's output, returning as
+// soon as it is found. All read output is forwarded to forward, if non-nil.
+// done is used to signal that the asynchronous io.Copy is done, if any.
+func readOutput(rc io.ReadCloser, forward io.Writer, done func()) (wsURL string, _ error) {
+	prefix := []byte("DevTools listening on")
+	var accumulated bytes.Buffer
+	var p [256]byte
+readLoop:
+	for {
+		n, err := rc.Read(p[:])
+		if err != nil {
+			return "", fmt.Errorf("chrome failed to start:\n%s",
+				accumulated.Bytes())
+		}
+		if forward != nil {
+			if _, err := forward.Write(p[:n]); err != nil {
+				return "", err
+			}
+		}
+
+		lines := bytes.Split(p[:n], []byte("\n"))
+		for _, line := range lines {
+			if bytes.HasPrefix(line, prefix) {
+				line = line[len(prefix):]
+				// use TrimSpace, to also remove \r on Windows
+				line = bytes.TrimSpace(line)
+				wsURL = string(line)
+				break readLoop
+			}
 		}
-		lines = append(lines, line)
+		accumulated.Write(p[:n])
 	}
-	if err := scanner.Err(); err != nil {
-		return "", err
+	if forward == nil {
+		// We don't need the process's output anymore.
+		rc.Close()
+	} else {
+		// Copy the rest of the output in a separate goroutine, as we
+		// need to return with the websocket URL.
+		go func() {
+			io.Copy(forward, rc)
+			done()
+		}()
 	}
-	if url == "" {
-		return "", fmt.Errorf("chrome stopped too early; stderr:\n%s",
-			strings.Join(lines, "\n"))
-	}
-	return url, nil
+	return wsURL, nil
 }
 
 // Wait satisfies the Allocator interface.
@@ -363,8 +393,17 @@ func DisableGPU(a *ExecAllocator) {
 	Flag("disable-gpu", true)(a)
 }
 
+// CombinedOutput is used to set an io.Writer where stdout and stderr
+// from the browser will be sent
+func CombinedOutput(w io.Writer) ExecAllocatorOption {
+	return func(a *ExecAllocator) {
+		a.combinedOutputWriter = w
+	}
+}
+
 // NewRemoteAllocator creates a new context set up with a RemoteAllocator,
-// suitable for use with NewContext.
+// suitable for use with NewContext. The url should point to the browser's
+// websocket address, such as "ws://127.0.0.1:$PORT/devtools/browser/...".
 func NewRemoteAllocator(parent context.Context, url string) (context.Context, context.CancelFunc) {
 	ctx, cancel := context.WithCancel(parent)
 	c := &Context{Allocator: &RemoteAllocator{

allocate_test.go

@@ -1,13 +1,15 @@
 package chromedp
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
 	"os"
 	"os/exec"
+	"regexp"
 	"strings"
 	"testing"
 	"time"
@@ -79,9 +81,9 @@ func TestExecAllocatorKillBrowser(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	kill := make(chan bool, 1)
+	kill := make(chan struct{}, 1)
 	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		kill <- true
+		kill <- struct{}{}
 		<-ctx.Done() // block until the end of the test
 	}))
 	defer s.Close()
@@ -153,7 +155,7 @@ func TestRemoteAllocator(t *testing.T) {
 	if err := cmd.Start(); err != nil {
 		t.Fatal(err)
 	}
-	wsURL, err := addrFromStderr(stderr)
+	wsURL, err := readOutput(stderr, nil, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -222,18 +224,69 @@ func TestExecAllocatorMissingWebsocketAddr(t *testing.T) {
 	t.Parallel()
 
 	allocCtx, cancel := NewExecAllocator(context.Background(),
-		// Use a bad listen addres, so both Chrome and headless-shell
-		// exit almost immediately.
+		// Use a bad listen address, so Chrome exits straight away.
 		append([]ExecAllocatorOption{Flag("remote-debugging-address", "_")},
 			allocOpts...)...)
 	defer cancel()
 
 	ctx, cancel := NewContext(allocCtx)
 	defer cancel()
 
-	want := "stopped too early"
+	want := regexp.MustCompile(`failed to start:\n.*Invalid devtools`)
 	got := fmt.Sprintf("%v", Run(ctx))
+	if !want.MatchString(got) {
+		t.Fatalf("want error to match %q, got %q", want, got)
+	}
+}
+
+func TestCombinedOutput(t *testing.T) {
+	t.Parallel()
+
+	buf := new(bytes.Buffer)
+	allocCtx, cancel := NewExecAllocator(context.Background(),
+		append([]ExecAllocatorOption{
+			CombinedOutput(buf),
+			Flag("enable-logging", true),
+		}, allocOpts...)...)
+	defer cancel()
+
+	taskCtx, _ := NewContext(allocCtx)
+	if err := Run(taskCtx,
+		Navigate(testdataDir+"/consolespam.html"),
+	); err != nil {
+		t.Fatal(err)
+	}
+	cancel()
+	if !strings.Contains(buf.String(), "DevTools listening on") {
+		t.Fatalf("failed to find websocket string in browser output test")
+	}
+	// Recent chrome versions have started replacing many "spam" messages
+	// with "spam 1", "spam 2", and so on. Search for the prefix only.
+	if want, got := 2000, strings.Count(buf.String(), `"spam`); want != got {
+		t.Fatalf("want %d spam console logs, got %d", want, got)
+	}
+}
+
+func TestCombinedOutputError(t *testing.T) {
+	t.Parallel()
+
+	// CombinedOutput used to hang the allocator if Chrome errored straight
+	// away, as there was no output to copy and the CombinedOutput would
+	// never signal it's done.
+	buf := new(bytes.Buffer)
+	allocCtx, cancel := NewExecAllocator(context.Background(),
+		// Use a bad listen address, so Chrome exits straight away.
+		append([]ExecAllocatorOption{
+			Flag("remote-debugging-address", "_"),
+			CombinedOutput(buf),
+		}, allocOpts...)...)
+	defer cancel()
+
+	ctx, cancel := NewContext(allocCtx)
+	defer cancel()
+	got := fmt.Sprint(Run(ctx))
+	want := "failed to start"
 	if !strings.Contains(got, want) {
-		t.Fatalf("want error to contain %q, got %q", want, got)
+		t.Fatalf("got %q, want %q", got, want)
 	}
 }

browser.go

@@ -5,8 +5,8 @@ import (
 	"fmt"
 	"log"
 	"net"
+	"net/url"
 	"os"
-	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -105,25 +105,25 @@ func NewBrowser(ctx context.Context, urlstr string, opts ...BrowserOption) (*Bro
 	return b, nil
 }
 
-// forceIP forces the host component in urlstr to be an IP address.
+// forceIP tries to force the host component in urlstr to be an IP address.
 //
 // Since Chrome 66+, Chrome DevTools Protocol clients connecting to a browser
 // must send the "Host:" header as either an IP address, or "localhost".
 func forceIP(urlstr string) string {
-	if i := strings.Index(urlstr, "://"); i != -1 {
-		scheme := urlstr[:i+3]
-		host, port, path := urlstr[len(scheme)+3:], "", ""
-		if i := strings.Index(host, "/"); i != -1 {
-			host, path = host[:i], host[i:]
-		}
-		if i := strings.Index(host, ":"); i != -1 {
-			host, port = host[:i], host[i:]
-		}
-		if addr, err := net.ResolveIPAddr("ip", host); err == nil {
-			urlstr = scheme + addr.IP.String() + port + path
-		}
+	u, err := url.Parse(urlstr)
+	if err != nil {
+		return urlstr
+	}
+	host, port, err := net.SplitHostPort(u.Host)
+	if err != nil {
+		return urlstr
+	}
+	addr, err := net.ResolveIPAddr("ip", host)
+	if err != nil {
+		return urlstr
 	}
-	return urlstr
+	u.Host = net.JoinHostPort(addr.IP.String(), port)
+	return u.String()
 }
 
 func (b *Browser) newExecutorForTarget(targetID target.ID, sessionID target.SessionID) *Target {
@@ -139,13 +139,10 @@ func (b *Browser) newExecutorForTarget(targetID target.ID, sessionID target.Sess
 		SessionID: sessionID,
 
 		messageQueue: make(chan *cdproto.Message, 1024),
-		waitQueue:    make(chan func() bool, 1024),
 		frames:       make(map[cdp.FrameID]*cdp.Frame),
 
 		logf: b.logf,
 		errf: b.errf,
-
-		tick: make(chan time.Time, 1),
 	}
 	// This send should be blocking, to ensure the tab is inserted into the
 	// map before any more target events are routed.
@@ -218,11 +215,7 @@ type decMessageString struct {
 }
 
 func (m *decMessageString) UnmarshalEasyJSON(l *jlexer.Lexer) {
-	if l.IsNull() {
-		l.Skip()
-	} else {
-		l.AddError(unmarshal(&m.lexer, l.UnsafeBytes(), &m.m))
-	}
+	l.AddError(unmarshal(&m.lexer, l.UnsafeBytes(), &m.m))
 }
 
 //easyjson:json
@@ -328,9 +321,6 @@ func (b *Browser) run(ctx context.Context) {
 	}()
 
 	pages := make(map[target.SessionID]*Target, 32)
-
-	ticker := time.NewTicker(2 * time.Millisecond)
-	defer ticker.Stop()
 	for {
 		select {
 		case <-ctx.Done():
@@ -362,28 +352,14 @@ func (b *Browser) run(ctx context.Context) {
 				delete(pages, tm.sessionID)
 			}
 
-		case t := <-ticker.C:
-			// Roughly once every 2ms, give every target a
-			// chance to run periodic work like checking if
-			// a wait function is complete.
-			//
-			// If a target hasn't picked up the previous
-			// tick, skip it.
-			for _, target := range pages {
-				select {
-				case target.tick <- t:
-				default:
-				}
-			}
-
 		case <-b.LostConnection:
 			return // to avoid "write: broken pipe" errors
 		}
 	}
 }
 
 // BrowserOption is a browser option.
-type BrowserOption func(*Browser)
+type BrowserOption = func(*Browser)
 
 // WithBrowserLogf is a browser option to specify a func to receive general logging.
 func WithBrowserLogf(f func(string, ...interface{})) BrowserOption {

chromedp.go

@@ -4,6 +4,9 @@
 //
 // chromedp requires no third-party dependencies, implementing the async Chrome
 // DevTools Protocol entirely in Go.
+//
+// This package includes a number of simple examples. Additionally,
+// https://github.com/chromedp/examples contains more complex examples.
 package chromedp
 
 import (
@@ -113,7 +116,7 @@ func NewContext(parent context.Context, opts ...ContextOption) (context.Context,
 		o(c)
 	}
 	if c.Allocator == nil {
-		c.Allocator = setupExecAllocator(DefaultExecAllocatorOptions...)
+		c.Allocator = setupExecAllocator(DefaultExecAllocatorOptions[:]...)
 	}
 
 	ctx = context.WithValue(ctx, contextKey{}, c)
@@ -220,45 +223,55 @@ func Run(ctx context.Context, actions ...Action) error {
 }
 
 func (c *Context) newTarget(ctx context.Context) error {
-	if c.targetID == "" && c.first {
-		tries := 0
-	retry:
-		// If we just allocated this browser, and it has a single page
-		// that's blank and not attached, use it.
-		infos, err := target.GetTargets().Do(cdp.WithExecutor(ctx, c.Browser))
-		if err != nil {
+	if c.targetID != "" {
+		if err := c.attachTarget(ctx, c.targetID); err != nil {
 			return err
 		}
-		pages := 0
-		for _, info := range infos {
-			if info.Type == "page" && info.URL == "about:blank" && !info.Attached {
-				c.targetID = info.TargetID
-				pages++
-			}
-		}
-		if pages < 1 {
-			// TODO: replace this polling with retries with a wait
-			// via Target.setDiscoverTargets after allocating a new
-			// browser.
-			if tries++; tries < 5 {
-				time.Sleep(10 * time.Millisecond)
-				goto retry
-			}
-			return fmt.Errorf("waited too long for page targets to show up")
-		}
-		if pages > 1 {
-			// Multiple blank pages; just in case, don't use any.
-			c.targetID = ""
+		// This new page might have already loaded its top-level frame
+		// already, in which case we wouldn't see the frameNavigated and
+		// documentUpdated events. Load them here.
+		tree, err := page.GetFrameTree().Do(cdp.WithExecutor(ctx, c.Target))
+		if err != nil {
+			return err
 		}
+		c.Target.cur = tree.Frame
+		c.Target.documentUpdated(ctx)
+		return nil
 	}
-
-	if c.targetID == "" {
+	if !c.first {
 		var err error
 		c.targetID, err = target.CreateTarget("about:blank").Do(cdp.WithExecutor(ctx, c.Browser))
 		if err != nil {
 			return err
 		}
+		return c.attachTarget(ctx, c.targetID)
+	}
+
+	// This is like WaitNewTarget, but for the entire browser.
+	ch := make(chan target.ID, 1)
+	lctx, cancel := context.WithCancel(ctx)
+	ListenBrowser(lctx, func(ev interface{}) {
+		var info *target.Info
+		switch ev := ev.(type) {
+		case *target.EventTargetCreated:
+			info = ev.TargetInfo
+		case *target.EventTargetInfoChanged:
+			info = ev.TargetInfo
+		default:
+			return
+		}
+		if info.Type == "page" && info.URL == "about:blank" {
+			ch <- info.TargetID
+			cancel()
+		}
+	})
+
+	// wait for the first blank tab to appear
+	action := target.SetDiscoverTargets(true)
+	if err := action.Do(cdp.WithExecutor(ctx, c.Browser)); err != nil {
+		return err
 	}
+	c.targetID = <-ch
 	return c.attachTarget(ctx, c.targetID)
 }
 
@@ -292,7 +305,7 @@ func (c *Context) attachTarget(ctx context.Context, targetID target.ID) error {
 }
 
 // ContextOption is a context option.
-type ContextOption func(*Context)
+type ContextOption = func(*Context)
 
 // WithTargetID sets up a context to be attached to an existing target, instead
 // of creating a new one.
@@ -329,19 +342,10 @@ func WithBrowserOption(opts ...BrowserOption) ContextOption {
 
 // Targets lists all the targets in the browser attached to the given context.
 func Targets(ctx context.Context) ([]*target.Info, error) {
-	// Don't rely on Run, as that needs to be able to call Targets, and we
-	// don't want cyclic func calls.
-	c := FromContext(ctx)
-	if c == nil || c.Allocator == nil {
-		return nil, ErrInvalidContext
-	}
-	if c.Browser == nil {
-		browser, err := c.Allocator.Allocate(ctx, c.browserOpts...)
-		if err != nil {
-			return nil, err
-		}
-		c.Browser = browser
+	if err := Run(ctx); err != nil {
+		return nil, err
 	}
+	c := FromContext(ctx)
 	return target.GetTargets().Do(cdp.WithExecutor(ctx, c.Browser))
 }
 
@@ -443,3 +447,34 @@ func ListenTarget(ctx context.Context, fn func(ev interface{})) {
 		c.targetListeners = append(c.targetListeners, cl)
 	}
 }
+
+// WaitNewTarget can be used to wait for the current target to open a new
+// target. Once fn matches a new unattached target, its target ID is sent via
+// the returned channel.
+func WaitNewTarget(ctx context.Context, fn func(*target.Info) bool) <-chan target.ID {
+	ch := make(chan target.ID, 1)
+	lctx, cancel := context.WithCancel(ctx)
+	ListenTarget(lctx, func(ev interface{}) {
+		var info *target.Info
+		switch ev := ev.(type) {
+		case *target.EventTargetCreated:
+			info = ev.TargetInfo
+		case *target.EventTargetInfoChanged:
+			info = ev.TargetInfo
+		default:
+			return
+		}
+		if info.OpenerID == "" {
+			return // not a child target
+		}
+		if info.Attached {
+			return // already attached; not a new target
+		}
+		if fn(info) {
+			ch <- info.TargetID
+			close(ch)
+			cancel()
+		}
+	})
+	return ch
+}

chromedp_test.go

@@ -1,11 +1,14 @@
 package chromedp
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"io/ioutil"
 	"log"
 	"net"
+	"net/http"
+	"net/http/httptest"
 	"os"
 	"path"
 	"strings"
@@ -14,6 +17,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/chromedp/cdproto/cdp"
 	"github.com/chromedp/cdproto/dom"
 	"github.com/chromedp/cdproto/page"
 	cdpruntime "github.com/chromedp/cdproto/runtime"
@@ -24,7 +28,7 @@ var (
 	// these are set up in init
 	execPath    string
 	testdataDir string
-	allocOpts   []ExecAllocatorOption
+	allocOpts   = DefaultExecAllocatorOptions[:]
 
 	// allocCtx is initialised in TestMain, to cancel before exiting.
 	allocCtx context.Context
@@ -45,9 +49,6 @@ func init() {
 		panic(fmt.Sprintf("could not create temp directory: %v", err))
 	}
 
-	// Build on top of the default options.
-	allocOpts = append(allocOpts, DefaultExecAllocatorOptions...)
-
 	// Disabling the GPU helps portability with some systems like Travis,
 	// and can slightly speed up the tests on other systems.
 	allocOpts = append(allocOpts, DisableGPU)
@@ -407,6 +408,36 @@ func TestLargeEventCount(t *testing.T) {
 	}
 }
 
+func TestLargeQuery(t *testing.T) {
+	t.Parallel()
+
+	ctx, cancel := testAllocate(t, "")
+	defer cancel()
+
+	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintf(w, "<html><body>\n")
+		for i := 0; i < 2000; i++ {
+			fmt.Fprintf(w, `<div>`)
+			fmt.Fprintf(w, `<a href="/%d">link %d</a>`, i, i)
+			fmt.Fprintf(w, `</div>`)
+		}
+		fmt.Fprintf(w, "</body></html>\n")
+	}))
+	defer s.Close()
+
+	// ByQueryAll queries thousands of events, which triggers thousands of
+	// DOM events. The target handler used to get into a deadlock, as the
+	// event queues would fill up and prevent the wait function from
+	// receiving any result.
+	var nodes []*cdp.Node
+	if err := Run(ctx,
+		Navigate(s.URL),
+		Nodes("a", &nodes, ByQueryAll),
+	); err != nil {
+		t.Fatal(err)
+	}
+}
+
 func TestDialTimeout(t *testing.T) {
 	t.Parallel()
 
@@ -483,3 +514,53 @@ func TestListenCancel(t *testing.T) {
 		t.Fatalf("want %d target events; got %d", want, targetCount)
 	}
 }
+
+func TestLogOptions(t *testing.T) {
+	t.Parallel()
+
+	var bufMu sync.Mutex
+	var buf bytes.Buffer
+	fn := func(format string, a ...interface{}) {
+		bufMu.Lock()
+		fmt.Fprintf(&buf, format, a...)
+		fmt.Fprintln(&buf)
+		bufMu.Unlock()
+	}
+
+	ctx, cancel := NewContext(context.Background(),
+		WithErrorf(fn),
+		WithLogf(fn),
+		WithDebugf(fn),
+	)
+	defer cancel()
+	if err := Run(ctx, Navigate(testdataDir+"/form.html")); err != nil {
+		t.Fatal(err)
+	}
+	cancel()
+
+	bufMu.Lock()
+	got := buf.String()
+	bufMu.Unlock()
+	for _, want := range []string{
+		"Page.navigate",
+		"Page.frameNavigated",
+	} {
+		if !strings.Contains(got, want) {
+			t.Errorf("expected output to contain %q", want)
+		}
+	}
+}
+
+func TestLargeOutboundMessages(t *testing.T) {
+	t.Parallel()
+
+	ctx, cancel := testAllocate(t, "")
+	defer cancel()
+
+	// ~50KiB of JS should fit just fine in our current buffer of 1MiB.
+	expr := fmt.Sprintf("//%s\n", strings.Repeat("x", 50<<10))
+	res := new([]byte)
+	if err := Run(ctx, Evaluate(expr, res)); err != nil {
+		t.Fatal(err)
+	}
+}

conn.go

@@ -15,6 +15,9 @@ import (
 )
 
 // Transport is the common interface to send/receive messages to a target.
+//
+// This interface is currently used internally by Browser, but it is exposed as
+// it will be useful as part of the public API in the future.
 type Transport interface {
 	Read(context.Context, *cdproto.Message) error
 	Write(context.Context, *cdproto.Message) error
@@ -40,6 +43,19 @@ type Conn struct {
 	dbgf func(string, ...interface{})
 }
 
+// Chrome doesn't support fragmentation of incoming websocket messages. To
+// compensate this, they support single-fragment messages of up to 100MiB.
+//
+// If our write buffer size is too small, large messages will get fragmented,
+// and Chrome will silently crash. And if it's too large, chromedp will require
+// more memory for all users.
+//
+// For now, make this a middle ground. 1MiB is large enough for practically any
+// outgoing message, but small enough to not take too much meomry.
+//
+// See https://github.com/ChromeDevTools/devtools-protocol/issues/175.
+const wsWriteBufferSize = 1 << 20
+
 // DialContext dials the specified websocket URL using gobwas/ws.
 func DialContext(ctx context.Context, urlstr string, opts ...DialOption) (*Conn, error) {
 	// connect
@@ -53,8 +69,9 @@ func DialContext(ctx context.Context, urlstr string, opts ...DialOption) (*Conn,
 
 	// apply opts
 	c := &Conn{
-		conn:   conn,
-		writer: *wsutil.NewWriterBufferSize(conn, ws.StateClientSide, ws.OpText, 4096),
+		conn: conn,
+		writer: *wsutil.NewWriterBufferSize(conn,
+			ws.StateClientSide, ws.OpText, wsWriteBufferSize),
 	}
 	for _, o := range opts {
 		o(c)
@@ -146,7 +163,7 @@ func (c *Conn) Write(_ context.Context, msg *cdproto.Message) error {
 }
 
 // DialOption is a dial option.
-type DialOption func(*Conn)
+type DialOption = func(*Conn)
 
 // WithConnDebugf is a dial option to set a protocol logger.
 func WithConnDebugf(f func(string, ...interface{})) DialOption {

device/gen.go

@@ -0,0 +1,139 @@
+// +build ignore
+
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os/exec"
+	"regexp"
+	"strings"
+)
+
+const deviceDescriptorsURL = "https://raw.githubusercontent.com/GoogleChrome/puppeteer/master/lib/DeviceDescriptors.js"
+
+var flagOut = flag.String("out", "device.go", "out")
+
+func main() {
+	flag.Parse()
+	if err := run(); err != nil {
+		log.Fatal(err)
+	}
+}
+
+type deviceDescriptor struct {
+	Name      string `json:"name"`
+	UserAgent string `json:"userAgent"`
+	Viewport  struct {
+		Width             int64   `json:"width"`
+		Height            int64   `json:"height"`
+		DeviceScaleFactor float64 `json:"deviceScaleFactor"`
+		IsMobile          bool    `json:"isMobile"`
+		HasTouch          bool    `json:"hasTouch"`
+		IsLandscape       bool    `json:"isLandscape"`
+	}
+}
+
+var cleanRE = regexp.MustCompile(`[^a-zA-Z0-9_]`)
+
+// run runs the program.
+func run() error {
+	var descriptors []deviceDescriptor
+	if err := get(&descriptors); err != nil {
+		return err
+	}
+
+	// add reset device
+	descriptors = append([]deviceDescriptor{{}}, descriptors...)
+
+	buf := new(bytes.Buffer)
+	fmt.Fprintf(buf, hdr, deviceDescriptorsURL)
+	fmt.Fprintln(buf, "\n// Devices.")
+	fmt.Fprintln(buf, "const (")
+	for i, d := range descriptors {
+		if i == 0 {
+			fmt.Fprintln(buf, "// Reset is the reset device.")
+			fmt.Fprintln(buf, "Reset infoType = iota\n")
+		} else {
+			name := cleanRE.ReplaceAllString(d.Name, "")
+			name = strings.ToUpper(name[0:1]) + name[1:]
+			fmt.Fprintf(buf, "// %s is the %q device.\n", name, d.Name)
+			fmt.Fprintf(buf, "%s\n\n", name)
+		}
+	}
+	fmt.Fprintln(buf, ")\n")
+
+	fmt.Fprintln(buf, "// devices is the list of devices.")
+	fmt.Fprintln(buf, "var devices = [...]Info{")
+	for _, d := range descriptors {
+		fmt.Fprintf(buf, "{%q, %q, %d, %d, %f, %t, %t, %t},\n",
+			d.Name, d.UserAgent,
+			d.Viewport.Width, d.Viewport.Height, d.Viewport.DeviceScaleFactor,
+			d.Viewport.IsLandscape, d.Viewport.IsMobile, d.Viewport.HasTouch,
+		)
+	}
+	fmt.Fprintln(buf, "}")
+
+	// write and  format
+	if err := ioutil.WriteFile(*flagOut, buf.Bytes(), 0644); err != nil {
+		return err
+	}
+	return exec.Command("gofmt", "-w", "-s", *flagOut).Run()
+}
+
+var (
+	startRE = regexp.MustCompile(`(?m)^module\.exports\s*=\s*\[`)
+	endRE   = regexp.MustCompile(`(?m)^\];$`)
+)
+
+// get retrieves and decodes the device descriptors.
+func get(v interface{}) error {
+	req, err := http.NewRequest("GET", deviceDescriptorsURL, nil)
+	if err != nil {
+		return err
+	}
+
+	// retrieve
+	cl := &http.Client{}
+	res, err := cl.Do(req)
+	if err != nil {
+		return err
+	}
+	defer res.Body.Close()
+	if res.StatusCode != 200 {
+		return fmt.Errorf("got status code %d", res.StatusCode)
+	}
+
+	buf, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return err
+	}
+
+	start := startRE.FindIndex(buf)
+	if start == nil {
+		return errors.New("could not find start")
+	}
+	buf = buf[start[1]-1:]
+
+	end := endRE.FindIndex(buf)
+	if end == nil {
+		return errors.New("could not find end")
+	}
+	buf = buf[:end[1]-1]
+	buf = bytes.Replace(buf, []byte("'"), []byte(`"`), -1)
+	return json.Unmarshal(buf, v)
+}
+
+const hdr = `package device
+
+// See: %s
+
+// Generated by gen.go. DO NOT EDIT.
+
+`

device/types.go

@@ -0,0 +1,56 @@
+// Package device contains device emulation definitions for use with chromedp's
+// Emulate action.
+package device
+
+//go:generate go run gen.go
+
+// Info holds device information for use with chromedp.Emulate.
+type Info struct {
+	// Name is the device name.
+	Name string
+
+	// UserAgent is the device user agent string.
+	UserAgent string
+
+	// Width is the viewport width.
+	Width int64
+
+	// Height is the viewport height.
+	Height int64
+
+	// Scale is the device viewport scale factor.
+	Scale float64
+
+	// Landscape indicates whether or not the device is in landscape mode or
+	// not.
+	Landscape bool
+
+	// Mobile indicates whether it is a mobile device or not.
+	Mobile bool
+
+	// Touch indicates whether the device has touch enabled.
+	Touch bool
+}
+
+// String satisfies fmt.Stringer.
+func (i Info) String() string {
+	return i.Name
+}
+
+// Device satisfies chromedp.Device.
+func (i Info) Device() Info {
+	return i
+}
+
+// infoType provides the enumerated device type.
+type infoType int
+
+// String satisfies fmt.Stringer.
+func (i infoType) String() string {
+	return devices[i].String()
+}
+
+// Device satisfies chromedp.Device.
+func (i infoType) Device() Info {
+	return devices[i]
+}

emulate.go

@@ -0,0 +1,123 @@
+package chromedp
+
+import (
+	"github.com/chromedp/cdproto/emulation"
+	"github.com/chromedp/chromedp/device"
+)
+
+// EmulateAction are actions that change the emulation settings for the
+// browser.
+type EmulateAction Action
+
+// EmulateViewport is an action to change the browser viewport.
+//
+// Wraps calls to emulation.SetDeviceMetricsOverride and emulation.SetTouchEmulationEnabled.
+//
+// Note: this has the effect of setting/forcing the screen orientation to
+// landscape, and will disable mobile and touch emulation by default. If this
+// is not the desired behavior, use the emulate viewport options
+// EmulateOrientation (or EmulateLandscape/EmulatePortrait), EmulateMobile, and
+// EmulateTouch, respectively.
+func EmulateViewport(width, height int64, opts ...EmulateViewportOption) EmulateAction {
+	p1 := emulation.SetDeviceMetricsOverride(width, height, 1.0, false)
+	p2 := emulation.SetTouchEmulationEnabled(false)
+	for _, o := range opts {
+		o(p1, p2)
+	}
+	return Tasks{p1, p2}
+}
+
+// EmulateViewportOption is the type for emulate viewport options.
+type EmulateViewportOption = func(*emulation.SetDeviceMetricsOverrideParams, *emulation.SetTouchEmulationEnabledParams)
+
+// EmulateScale is an emulate viewport option to set the device viewport scaling
+// factor.
+func EmulateScale(scale float64) EmulateViewportOption {
+	return func(p1 *emulation.SetDeviceMetricsOverrideParams, p2 *emulation.SetTouchEmulationEnabledParams) {
+		p1.DeviceScaleFactor = scale
+	}
+}
+
+// EmulateOrientation is an emulate viewport option to set the device viewport
+// screen orientation.
+func EmulateOrientation(orientation emulation.OrientationType, angle int64) EmulateViewportOption {
+	return func(p1 *emulation.SetDeviceMetricsOverrideParams, p2 *emulation.SetTouchEmulationEnabledParams) {
+		p1.ScreenOrientation = &emulation.ScreenOrientation{
+			Type:  orientation,
+			Angle: angle,
+		}
+	}
+}
+
+// EmulateLandscape is an emulate viewport option to set the device viewport
+// screen orientation in landscape primary mode and an angle of 90.
+func EmulateLandscape(p1 *emulation.SetDeviceMetricsOverrideParams, p2 *emulation.SetTouchEmulationEnabledParams) {
+	EmulateOrientation(emulation.OrientationTypeLandscapePrimary, 90)(p1, p2)
+}
+
+// EmulatePortrait is an emulate viewport option to set the device viewport
+// screen orentation in portrait primary mode and an angle of 0.
+func EmulatePortrait(p1 *emulation.SetDeviceMetricsOverrideParams, p2 *emulation.SetTouchEmulationEnabledParams) {
+	EmulateOrientation(emulation.OrientationTypePortraitPrimary, 0)(p1, p2)
+}
+
+// EmulateMobile is an emulate viewport option to toggle the device viewport to
+// display as a mobile device.
+func EmulateMobile(p1 *emulation.SetDeviceMetricsOverrideParams, p2 *emulation.SetTouchEmulationEnabledParams) {
+	p1.Mobile = true
+}
+
+// EmulateTouch is an emulate viewport option to enable touch emulation.
+func EmulateTouch(p1 *emulation.SetDeviceMetricsOverrideParams, p2 *emulation.SetTouchEmulationEnabledParams) {
+	p2.Enabled = true
+}
+
+// ResetViewport is an action to reset the browser viewport to the default
+// values the browser was started with.
+//
+// Note: does not modify / change the browser's emulated User-Agent, if any.
+func ResetViewport() EmulateAction {
+	return EmulateViewport(0, 0, EmulatePortrait)
+}
+
+// Device is the shared interface for known device types.
+//
+// See: github.com/chromedp/chromedp/device for a set of off-the-shelf devices
+// and modes.
+type Device interface {
+	// Device returns the device info.
+	Device() device.Info
+}
+
+// Emulate is an action to emulate a specific device.
+//
+// See: github.com/chromedp/chromedp/device for a set of off-the-shelf devices
+// and modes.
+func Emulate(device Device) EmulateAction {
+	d := device.Device()
+
+	var angle int64
+	orientation := emulation.OrientationTypePortraitPrimary
+	if d.Landscape {
+		orientation, angle = emulation.OrientationTypeLandscapePrimary, 90
+	}
+
+	return Tasks{
+		emulation.SetUserAgentOverride(d.UserAgent),
+		emulation.SetDeviceMetricsOverride(d.Width, d.Height, d.Scale, d.Mobile).
+			WithScreenOrientation(&emulation.ScreenOrientation{
+				Type:  orientation,
+				Angle: angle,
+			}),
+		emulation.SetTouchEmulationEnabled(d.Touch),
+	}
+}
+
+// EmulateReset is an action to reset the device emulation.
+//
+// Resets the browser's viewport, screen orientation, user-agent, and
+// mobile/touch emulation settings to the original values the browser was
+// started with.
+func EmulateReset() EmulateAction {
+	return Emulate(device.Reset)
+}

emulate_test.go

@@ -0,0 +1,33 @@
+package chromedp
+
+import (
+	"bytes"
+	"image/png"
+	"testing"
+
+	"github.com/chromedp/chromedp/device"
+)
+
+func TestEmulate(t *testing.T) {
+	t.Parallel()
+
+	ctx, cancel := testAllocate(t, "image.html")
+	defer cancel()
+
+	var buf []byte
+	if err := Run(ctx,
+		Emulate(device.IPhone7),
+		Screenshot(`#half-color`, &buf, ByID),
+	); err != nil {
+		t.Fatal(err)
+	}
+
+	img, err := png.Decode(bytes.NewReader(buf))
+	if err != nil {
+		t.Fatal(err)
+	}
+	size := img.Bounds().Size()
+	if size.X != 400 || size.Y != 400 {
+		t.Errorf("expected size 400x400, got: %dx%d", size.X, size.Y)
+	}
+}

eval.go

@@ -3,10 +3,15 @@ package chromedp
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 
 	"github.com/chromedp/cdproto/runtime"
 )
 
+// EvaluateAction are actions that evaluate Javascript expressions using
+// runtime.Evaluate.
+type EvaluateAction Action
+
 // Evaluate is an action to evaluate the Javascript expression, unmarshaling
 // the result of the script evaluation to res.
 //
@@ -21,7 +26,7 @@ import (
 // made to convert the result.
 //
 // Note: any exception encountered will be returned as an error.
-func Evaluate(expression string, res interface{}, opts ...EvaluateOption) Action {
+func Evaluate(expression string, res interface{}, opts ...EvaluateOption) EvaluateAction {
 	if res == nil {
 		panic("res cannot be nil")
 	}
@@ -59,6 +64,13 @@ func Evaluate(expression string, res interface{}, opts ...EvaluateOption) Action
 			return nil
 		}
 
+		if v.Type == "undefined" {
+			// The unmarshal above would fail with the cryptic
+			// "unexpected end of JSON input" error, so try to give
+			// a better one here.
+			return fmt.Errorf("encountered an undefined value")
+		}
+
 		// unmarshal
 		return json.Unmarshal(v.Value, res)
 	})
@@ -68,13 +80,15 @@ func Evaluate(expression string, res interface{}, opts ...EvaluateOption) Action
 // Chrome DevTools would, evaluating the expression in the "console" context,
 // and making the Command Line API available to the script.
 //
+// See Evaluate for more information on how script expressions are evaluated.
+//
 // Note: this should not be used with untrusted Javascript.
-func EvaluateAsDevTools(expression string, res interface{}, opts ...EvaluateOption) Action {
+func EvaluateAsDevTools(expression string, res interface{}, opts ...EvaluateOption) EvaluateAction {
 	return Evaluate(expression, res, append(opts, EvalObjectGroup("console"), EvalWithCommandLineAPI)...)
 }
 
-// EvaluateOption is the type for script evaluation options.
-type EvaluateOption func(*runtime.EvaluateParams) *runtime.EvaluateParams
+// EvaluateOption is the type for Javascript evaluation options.
+type EvaluateOption = func(*runtime.EvaluateParams) *runtime.EvaluateParams
 
 // EvalObjectGroup is a evaluate option to set the object group.
 func EvalObjectGroup(objectGroup string) EvaluateOption {
@@ -86,19 +100,21 @@ func EvalObjectGroup(objectGroup string) EvaluateOption {
 // EvalWithCommandLineAPI is an evaluate option to make the DevTools Command
 // Line API available to the evaluated script.
 //
+// See Evaluate for more information on how evaluate actions work.
+//
 // Note: this should not be used with untrusted Javascript.
 func EvalWithCommandLineAPI(p *runtime.EvaluateParams) *runtime.EvaluateParams {
 	return p.WithIncludeCommandLineAPI(true)
 }
 
-// EvalIgnoreExceptions is a evaluate option that will cause script evaluation
-// to ignore exceptions.
+// EvalIgnoreExceptions is a evaluate option that will cause Javascript
+// evaluation to ignore exceptions.
 func EvalIgnoreExceptions(p *runtime.EvaluateParams) *runtime.EvaluateParams {
 	return p.WithSilent(true)
 }
 
-// EvalAsValue is a evaluate option that will cause the evaluated script to
-// encode the result of the expression as a JSON-encoded value.
+// EvalAsValue is a evaluate option that will cause the evaluated Javascript
+// expression to encode the result of the expression as a JSON-encoded value.
 func EvalAsValue(p *runtime.EvaluateParams) *runtime.EvaluateParams {
 	return p.WithReturnByValue(true)
 }

event_test.go

@@ -112,6 +112,7 @@ func TestCloseDialog(t *testing.T) {
 	}
 
 	for _, test := range tests {
+		test := test
 		t.Run(test.name, func(t *testing.T) {
 			t.Parallel()
 
@@ -157,18 +158,14 @@ func TestCloseDialog(t *testing.T) {
 	}
 }
 
-func TestClickNewTab(t *testing.T) {
+func TestWaitNewTarget(t *testing.T) {
 	t.Parallel()
 
 	ctx, cancel := testAllocate(t, "newtab.html")
 	defer cancel()
 
-	ch := make(chan target.ID, 1)
-	ListenTarget(ctx, func(ev interface{}) {
-		if ev, ok := ev.(*target.EventTargetCreated); ok &&
-			ev.TargetInfo.OpenerID != "" {
-			ch <- ev.TargetInfo.TargetID
-		}
+	ch := WaitNewTarget(ctx, func(info *target.Info) bool {
+		return info.URL != ""
 	})
 	if err := Run(ctx, Click("#new-tab", ByID)); err != nil {
 		t.Fatal(err)
@@ -177,7 +174,10 @@ func TestClickNewTab(t *testing.T) {
 	defer cancel()
 
 	var urlstr string
-	if err := Run(blankCtx, Location(&urlstr)); err != nil {
+	if err := Run(blankCtx,
+		Location(&urlstr),
+		WaitVisible(`#form`, ByID),
+	); err != nil {
 		t.Fatal(err)
 	}
 	if !strings.HasSuffix(urlstr, "form.html") {

example_test.go

@@ -13,10 +13,13 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/chromedp/cdproto/cdp"
+	"github.com/chromedp/cdproto/dom"
 	"github.com/chromedp/cdproto/page"
-	cdpruntime "github.com/chromedp/cdproto/runtime"
+	"github.com/chromedp/cdproto/runtime"
 	"github.com/chromedp/cdproto/target"
 	"github.com/chromedp/chromedp"
+	"github.com/chromedp/chromedp/device"
 )
 
 func writeHTML(content string) http.Handler {
@@ -60,13 +63,10 @@ func ExampleExecAllocator() {
 	}
 	defer os.RemoveAll(dir)
 
-	opts := []chromedp.ExecAllocatorOption{
-		chromedp.NoFirstRun,
-		chromedp.NoDefaultBrowserCheck,
-		chromedp.Headless,
+	opts := append(chromedp.DefaultExecAllocatorOptions[:],
 		chromedp.DisableGPU,
 		chromedp.UserDataDir(dir),
-	}
+	)
 
 	allocCtx, cancel := chromedp.NewExecAllocator(context.Background(), opts...)
 	defer cancel()
@@ -139,7 +139,7 @@ func ExampleListenTarget_consoleLog() {
 
 	chromedp.ListenTarget(ctx, func(ev interface{}) {
 		switch ev := ev.(type) {
-		case *cdpruntime.EventConsoleAPICalled:
+		case *runtime.EventConsoleAPICalled:
 			fmt.Printf("console.%s call:\n", ev.Type)
 			for _, arg := range ev.Args {
 				fmt.Printf("%s - %s\n", arg.Type, arg.Value)
@@ -159,7 +159,7 @@ func ExampleListenTarget_consoleLog() {
 	// string - "hello js world"
 }
 
-func ExampleClickNewTab() {
+func ExampleWaitNewTarget() {
 	ctx, cancel := chromedp.NewContext(context.Background())
 	defer cancel()
 
@@ -171,15 +171,9 @@ func ExampleClickNewTab() {
 	ts := httptest.NewServer(mux)
 	defer ts.Close()
 
-	lctx, cancel := context.WithCancel(ctx)
-	ch := make(chan target.ID, 1)
-	chromedp.ListenTarget(lctx, func(ev interface{}) {
-		if ev, ok := ev.(*target.EventTargetCreated); ok &&
-			// if OpenerID == "", this is the first tab.
-			ev.TargetInfo.OpenerID != "" {
-			ch <- ev.TargetInfo.TargetID
-			cancel()
-		}
+	// Grab the first spawned tab that isn't blank.
+	ch := chromedp.WaitNewTarget(ctx, func(info *target.Info) bool {
+		return info.URL != ""
 	})
 	if err := chromedp.Run(ctx,
 		chromedp.Navigate(ts.URL+"/first"),
@@ -200,7 +194,7 @@ func ExampleClickNewTab() {
 	// new tab's path: /second
 }
 
-func ExampleAcceptAlert() {
+func ExampleListenTarget_acceptAlert() {
 	ctx, cancel := chromedp.NewContext(context.Background())
 	defer cancel()
 
@@ -234,3 +228,177 @@ func ExampleAcceptAlert() {
 	// Output:
 	// closing alert: alert text
 }
+
+func Example_retrieveHTML() {
+	ctx, cancel := chromedp.NewContext(context.Background())
+	defer cancel()
+
+	ts := httptest.NewServer(writeHTML(`
+<body>
+<p id="content" onclick="changeText()">Original content.</p>
+<script>
+function changeText() {
+	document.getElementById("content").textContent = "New content!"
+}
+</script>
+</body>
+	`))
+	defer ts.Close()
+
+	var outerBefore, outerAfter string
+	if err := chromedp.Run(ctx,
+		chromedp.Navigate(ts.URL),
+		chromedp.OuterHTML("#content", &outerBefore),
+		chromedp.Click("#content", chromedp.ByID),
+		chromedp.OuterHTML("#content", &outerAfter),
+	); err != nil {
+		panic(err)
+	}
+	fmt.Println("OuterHTML before clicking:")
+	fmt.Println(outerBefore)
+	fmt.Println("OuterHTML after clicking:")
+	fmt.Println(outerAfter)
+
+	// Output:
+	// OuterHTML before clicking:
+	// <p id="content" onclick="changeText()">Original content.</p>
+	// OuterHTML after clicking:
+	// <p id="content" onclick="changeText()">New content!</p>
+}
+
+func ExampleEmulate() {
+	ctx, cancel := chromedp.NewContext(context.Background())
+	defer cancel()
+
+	var buf []byte
+	if err := chromedp.Run(ctx,
+		chromedp.Emulate(device.IPhone7),
+		chromedp.Navigate(`https://google.com/`),
+		chromedp.WaitVisible(`#main`, chromedp.ByID),
+		chromedp.SendKeys(`input[name=q]`, "what's my user agent?\n"),
+		chromedp.WaitVisible(`#rso`, chromedp.ByID),
+		chromedp.CaptureScreenshot(&buf),
+	); err != nil {
+		panic(err)
+	}
+
+	if err := ioutil.WriteFile("google-iphone7.png", buf, 0644); err != nil {
+		panic(err)
+	}
+}
+
+func ExamplePrintToPDF() {
+	ctx, cancel := chromedp.NewContext(context.Background())
+	defer cancel()
+
+	var buf []byte
+	if err := chromedp.Run(ctx,
+		chromedp.Navigate(`https://godoc.org/github.com/chromedp/chromedp`),
+		chromedp.ActionFunc(func(ctx context.Context) error {
+			var err error
+			buf, _, err = page.PrintToPDF().
+				WithDisplayHeaderFooter(false).
+				WithLandscape(true).
+				Do(ctx)
+			return err
+		}),
+	); err != nil {
+		panic(err)
+	}
+
+	if err := ioutil.WriteFile("page.pdf", buf, 0644); err != nil {
+		panic(err)
+	}
+}
+
+func ExampleByJSPath() {
+	ctx, cancel := chromedp.NewContext(context.Background())
+	defer cancel()
+
+	ts := httptest.NewServer(writeHTML(`
+<body>
+	<div id="content">cool content</div>
+</body>
+	`))
+	defer ts.Close()
+
+	var ids []cdp.NodeID
+	var html string
+	if err := chromedp.Run(ctx,
+		chromedp.Navigate(ts.URL),
+		chromedp.NodeIDs(`document`, &ids, chromedp.ByJSPath),
+		chromedp.ActionFunc(func(ctx context.Context) error {
+			var err error
+			html, err = dom.GetOuterHTML().WithNodeID(ids[0]).Do(ctx)
+			return err
+		}),
+	); err != nil {
+		panic(err)
+	}
+
+	fmt.Println("Outer HTML:")
+	fmt.Println(html)
+
+	// Output:
+	// Outer HTML:
+	// <html><head></head><body>
+	// 	<div id="content">cool content</div>
+	// </body></html>
+}
+
+func Example_documentDump() {
+	ctx, cancel := chromedp.NewContext(context.Background())
+	defer cancel()
+
+	ts := httptest.NewServer(writeHTML(`<!doctype html>
+<html>
+<body>
+  <div id="content">the content</div>
+</body>
+</html>`))
+	defer ts.Close()
+
+	const expr = `(function(d, id, v) {
+		var b = d.querySelector('body');
+		var el = d.createElement('div');
+		el.id = id;
+		el.innerText = v;
+		b.insertBefore(el, b.childNodes[0]);
+	})(document, %q, %q);`
+
+	var nodes []*cdp.Node
+	if err := chromedp.Run(ctx,
+		chromedp.Navigate(ts.URL),
+		chromedp.Nodes(`document`, &nodes, chromedp.ByJSPath),
+		chromedp.WaitVisible(`#content`),
+		chromedp.ActionFunc(func(ctx context.Context) error {
+			s := fmt.Sprintf(expr, "thing", "a new thing!")
+			_, exp, err := runtime.Evaluate(s).Do(ctx)
+			if err != nil {
+				return err
+			}
+			if exp != nil {
+				return exp
+			}
+			return nil
+		}),
+		chromedp.WaitVisible(`#thing`),
+	); err != nil {
+		panic(err)
+	}
+
+	fmt.Println("Document tree:")
+	fmt.Print(nodes[0].Dump("  ", "  ", false))
+
+	// Output:
+	// Document tree:
+	//   #document <Document>
+	//     html <DocumentType>
+	//     html
+	//       head
+	//       body
+	//         div#thing
+	//           #text "a new thing!"
+	//         div#content
+	//           #text "the content"
+}

go.mod

@@ -3,10 +3,10 @@ module github.com/chromedp/chromedp
 go 1.11
 
 require (
-	github.com/chromedp/cdproto v0.0.0-20190429085128-1aa4f57ff2a9
+	github.com/chromedp/cdproto v0.0.0-20190812224334-39ef923dcb8d
 	github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee // indirect
 	github.com/gobwas/pool v0.2.0 // indirect
-	github.com/gobwas/ws v1.0.0
-	github.com/mailru/easyjson v0.0.0-20190403194419-1ea4449da983
-	golang.org/x/sys v0.0.0-20190509141414-a5b02f93d862 // indirect
+	github.com/gobwas/ws v1.0.2
+	github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e
+	golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a // indirect
 )

go.sum

@@ -1,15 +1,14 @@
-github.com/chromedp/cdproto v0.0.0-20190429085128-1aa4f57ff2a9 h1:ARnDd2vEk91rLNra8yk1hF40H8z+1HrD6juNpe7FsI0=
-github.com/chromedp/cdproto v0.0.0-20190429085128-1aa4f57ff2a9/go.mod h1:xquOK9dIGFlLaIGI4c6IyfLI/Gz0LiYYuJtzhsUODgI=
+github.com/chromedp/cdproto v0.0.0-20190812224334-39ef923dcb8d h1:00kLGv5nKzpFchNhGDXDRbKtYx/WoT983Ka2t8/pzRE=
+github.com/chromedp/cdproto v0.0.0-20190812224334-39ef923dcb8d/go.mod h1:0YChpVzuLJC5CPr+x3xkHN6Z8KOSXjNbL7qV8Wc4GW0=
 github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee h1:s+21KNqlpePfkah2I+gwHF8xmJWRjooY+5248k6m4A0=
 github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
 github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8=
 github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
-github.com/gobwas/ws v1.0.0 h1:1WdyfgUcImUfVBvYbsW2krIsnko+1QU2t45soaF8v1M=
-github.com/gobwas/ws v1.0.0/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
+github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo=
+github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
 github.com/knq/sysutil v0.0.0-20181215143952-f05b59f0f307 h1:vl4eIlySbjertFaNwiMjXsGrFVK25aOWLq7n+3gh2ls=
 github.com/knq/sysutil v0.0.0-20181215143952-f05b59f0f307/go.mod h1:BjPj+aVjl9FW/cCGiF3nGh5v+9Gd3VCgBQbod/GlMaQ=
-github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/mailru/easyjson v0.0.0-20190403194419-1ea4449da983 h1:wL11wNW7dhKIcRCHSm4sHKPWz0tt4mwBsVodG7+Xyqg=
-github.com/mailru/easyjson v0.0.0-20190403194419-1ea4449da983/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-golang.org/x/sys v0.0.0-20190509141414-a5b02f93d862 h1:rM0ROo5vb9AdYJi1110yjWGMej9ITfKddS89P3Fkhug=
-golang.org/x/sys v0.0.0-20190509141414-a5b02f93d862/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e h1:hB2xlXdHp/pmPZq0y3QnmWAArdw9PqbmotexnWx/FU8=
+github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ=
+golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

input.go

@@ -2,7 +2,6 @@ package chromedp
 
 import (
 	"context"
-	"fmt"
 
 	"github.com/chromedp/cdproto/cdp"
 	"github.com/chromedp/cdproto/dom"
@@ -11,53 +10,55 @@ import (
 	"github.com/chromedp/chromedp/kb"
 )
 
-// MouseAction is a mouse action.
-func MouseAction(typ input.MouseType, x, y int64, opts ...MouseOption) Action {
-	me := input.DispatchMouseEvent(typ, float64(x), float64(y))
+// MouseAction are mouse input event actions
+type MouseAction Action
 
+// MouseEvent is a mouse event action to dispatch the specified mouse event
+// type at coordinates x, y.
+func MouseEvent(typ input.MouseType, x, y float64, opts ...MouseOption) MouseAction {
+	p := input.DispatchMouseEvent(typ, x, y)
 	// apply opts
 	for _, o := range opts {
-		me = o(me)
+		p = o(p)
 	}
-
-	return me
+	return p
 }
 
-// MouseClickXY sends a left mouse button click (ie, mousePressed and
-// mouseReleased event) at the X, Y location.
-func MouseClickXY(x, y int64, opts ...MouseOption) Action {
+// MouseClickXY is an action that sends a left mouse button click (ie,
+// mousePressed and mouseReleased event) to the X, Y location.
+func MouseClickXY(x, y float64, opts ...MouseOption) MouseAction {
 	return ActionFunc(func(ctx context.Context) error {
-		me := &input.DispatchMouseEventParams{
+		p := &input.DispatchMouseEventParams{
 			Type:       input.MousePressed,
-			X:          float64(x),
-			Y:          float64(y),
+			X:          x,
+			Y:          y,
 			Button:     input.ButtonLeft,
 			ClickCount: 1,
 		}
 
 		// apply opts
 		for _, o := range opts {
-			me = o(me)
+			p = o(p)
 		}
 
-		if err := me.Do(ctx); err != nil {
+		if err := p.Do(ctx); err != nil {
 			return err
 		}
 
-		me.Type = input.MouseReleased
-		return me.Do(ctx)
+		p.Type = input.MouseReleased
+		return p.Do(ctx)
 	})
 }
 
-// MouseClickNode dispatches a mouse left button click event at the center of a
-// specified node.
+// MouseClickNode is an action that dispatches a mouse left button click event
+// at the center of a specified node.
 //
 // Note that the window will be scrolled if the node is not within the window's
 // viewport.
-func MouseClickNode(n *cdp.Node, opts ...MouseOption) Action {
+func MouseClickNode(n *cdp.Node, opts ...MouseOption) MouseAction {
 	return ActionFunc(func(ctx context.Context) error {
-		var pos []int
-		err := EvaluateAsDevTools(fmt.Sprintf(scrollIntoViewJS, n.FullXPath()), &pos).Do(ctx)
+		var pos []float64
+		err := EvaluateAsDevTools(snippet(scrollIntoViewJS, cashX(true), nil, n), &pos).Do(ctx)
 		if err != nil {
 			return err
 		}
@@ -72,20 +73,20 @@ func MouseClickNode(n *cdp.Node, opts ...MouseOption) Action {
 			return ErrInvalidDimensions
 		}
 
-		var x, y int64
+		var x, y float64
 		for i := 0; i < c; i += 2 {
-			x += int64(box.Content[i])
-			y += int64(box.Content[i+1])
+			x += box.Content[i]
+			y += box.Content[i+1]
 		}
-		x /= int64(c / 2)
-		y /= int64(c / 2)
+		x /= float64(c / 2)
+		y /= float64(c / 2)
 
 		return MouseClickXY(x, y, opts...).Do(ctx)
 	})
 }
 
 // MouseOption is a mouse action option.
-type MouseOption func(*input.DispatchMouseEventParams) *input.DispatchMouseEventParams
+type MouseOption = func(*input.DispatchMouseEventParams) *input.DispatchMouseEventParams
 
 // Button is a mouse action option to set the button to click from a string.
 func Button(btn string) MouseOption {
@@ -141,14 +142,20 @@ func ClickCount(n int) MouseOption {
 	}
 }
 
-// KeyAction will synthesize a keyDown, char, and keyUp event for each rune
-// contained in keys along with any supplied key options.
+// KeyAction are keyboard (key) input event actions.
+type KeyAction Action
+
+// KeyEvent is a key action that synthesizes a keyDown, char, and keyUp event
+// for each rune contained in keys along with any supplied key options.
 //
 // Only well-known, "printable" characters will have char events synthesized.
 //
-// Please see the chromedp/kb package for implementation details and the list
-// of well-known keys.
-func KeyAction(keys string, opts ...KeyOption) Action {
+// See the SendKeys action to synthesize key events for a specific element
+// node.
+//
+// See the chromedp/kb package for implementation details and list of
+// well-known keys.
+func KeyEvent(keys string, opts ...KeyOption) KeyAction {
 	return ActionFunc(func(ctx context.Context) error {
 		for _, r := range keys {
 			for _, k := range kb.Encode(r) {
@@ -162,20 +169,20 @@ func KeyAction(keys string, opts ...KeyOption) Action {
 	})
 }
 
-// KeyActionNode dispatches a key event on a node.
-func KeyActionNode(n *cdp.Node, keys string, opts ...KeyOption) Action {
+// KeyEventNode is a key action that dispatches a key event on a element node.
+func KeyEventNode(n *cdp.Node, keys string, opts ...KeyOption) KeyAction {
 	return ActionFunc(func(ctx context.Context) error {
 		err := dom.Focus().WithNodeID(n.NodeID).Do(ctx)
 		if err != nil {
 			return err
 		}
 
-		return KeyAction(keys, opts...).Do(ctx)
+		return KeyEvent(keys, opts...).Do(ctx)
 	})
 }
 
 // KeyOption is a key action option.
-type KeyOption func(*input.DispatchKeyEventParams) *input.DispatchKeyEventParams
+type KeyOption = func(*input.DispatchKeyEventParams) *input.DispatchKeyEventParams
 
 // KeyModifiers is a key action option to add additional modifiers on the key
 // press.

input_test.go

@@ -27,7 +27,7 @@ func TestMouseClickXY(t *testing.T) {
 		t.Fatal(err)
 	}
 	tests := []struct {
-		x, y int64
+		x, y float64
 	}{
 		{100, 100},
 		{0, 0},
@@ -44,23 +44,23 @@ func TestMouseClickXY(t *testing.T) {
 			t.Fatalf("test %d got error: %v", i, err)
 		}
 
-		x, err := strconv.ParseInt(xstr, 10, 64)
+		x, err := strconv.ParseFloat(xstr, 64)
 		if err != nil {
 			t.Fatalf("test %d got error: %v", i, err)
 		}
 		if x != test.x {
-			t.Fatalf("test %d expected x to be: %d, got: %d", i, test.x, x)
+			t.Fatalf("test %d expected x to be: %f, got: %f", i, test.x, x)
 		}
 		if err := Run(ctx, Value("#input2", &ystr, ByID)); err != nil {
 			t.Fatalf("test %d got error: %v", i, err)
 		}
 
-		y, err := strconv.ParseInt(ystr, 10, 64)
+		y, err := strconv.ParseFloat(ystr, 64)
 		if err != nil {
 			t.Fatalf("test %d got error: %v", i, err)
 		}
 		if y != test.y {
-			t.Fatalf("test %d expected y to be: %d, got: %d", i, test.y, y)
+			t.Fatalf("test %d expected y to be: %f, got: %f", i, test.y, y)
 		}
 	}
 }
@@ -73,12 +73,13 @@ func TestMouseClickNode(t *testing.T) {
 		opt      MouseOption
 		by       QueryOption
 	}{
-		{"button2", "foo", ButtonType(input.ButtonNone), ByID},
-		{"button2", "bar", ButtonType(input.ButtonLeft), ByID},
-		{"button2", "bar-middle", ButtonType(input.ButtonMiddle), ByID},
-		{"input3", "foo", ButtonModifiers(input.ModifierNone), ByID},
-		{"input3", "bar-right", ButtonType(input.ButtonRight), ByID},
-		{"input3", "bar-right", Button("right"), ByID},
+		{`button2`, "foo", ButtonType(input.ButtonNone), ByID},
+		{`button2`, "bar", ButtonType(input.ButtonLeft), ByID},
+		{`button2`, "bar-middle", ButtonType(input.ButtonMiddle), ByID},
+		{`input3`, "foo", ButtonModifiers(input.ModifierNone), ByID},
+		{`input3`, "bar-right", ButtonType(input.ButtonRight), ByID},
+		{`input3`, "bar-right", Button("right"), ByID},
+		{`document.querySelector('#input3')`, "bar-right", ButtonType(input.ButtonRight), ByJSPath},
 	}
 
 	for i, test := range tests {
@@ -119,9 +120,10 @@ func TestMouseClickOffscreenNode(t *testing.T) {
 		exp int
 		by  QueryOption
 	}{
-		{"#button3", 0, ByID},
-		{"#button3", 2, ByID},
-		{"#button3", 10, ByID},
+		{`#button3`, 0, ByID},
+		{`#button3`, 2, ByID},
+		{`#button3`, 10, ByID},
+		{`document.querySelector('#button3')`, 10, ByJSPath},
 	}
 
 	for i, test := range tests {
@@ -168,19 +170,20 @@ func TestMouseClickOffscreenNode(t *testing.T) {
 	}
 }
 
-func TestKeyAction(t *testing.T) {
+func TestKeyEvent(t *testing.T) {
 	t.Parallel()
 
 	tests := []struct {
 		sel, exp string
 		by       QueryOption
 	}{
-		{"#input4", "foo", ByID},
-		{"#input4", "foo and bar", ByID},
-		{"#input4", "1234567890", ByID},
-		{"#input4", "~!@#$%^&*()_+=[];'", ByID},
-		{"#input4", "你", ByID},
-		{"#input4", "\n\nfoo\n\nbar\n\n", ByID},
+		{`#input4`, "foo", ByID},
+		{`#input4`, "foo and bar", ByID},
+		{`#input4`, "1234567890", ByID},
+		{`#input4`, "~!@#$%^&*()_+=[];'", ByID},
+		{`#input4`, "你", ByID},
+		{`#input4`, "\n\nfoo\n\nbar\n\n", ByID},
+		{`document.querySelector('#input4')`, "\n\ntest\n\n", ByJSPath},
 	}
 
 	for i, test := range tests {
@@ -201,7 +204,7 @@ func TestKeyAction(t *testing.T) {
 			}
 			if err := Run(ctx,
 				Focus(test.sel, test.by),
-				KeyAction(test.exp),
+				KeyEvent(test.exp),
 			); err != nil {
 				t.Fatalf("got error: %v", err)
 			}
@@ -218,19 +221,20 @@ func TestKeyAction(t *testing.T) {
 	}
 }
 
-func TestKeyActionNode(t *testing.T) {
+func TestKeyEventNode(t *testing.T) {
 	t.Parallel()
 
 	tests := []struct {
 		sel, exp string
 		by       QueryOption
 	}{
-		{"#input4", "foo", ByID},
-		{"#input4", "foo and bar", ByID},
-		{"#input4", "1234567890", ByID},
-		{"#input4", "~!@#$%^&*()_+=[];'", ByID},
-		{"#input4", "你", ByID},
-		{"#input4", "\n\nfoo\n\nbar\n\n", ByID},
+		{`#input4`, "foo", ByID},
+		{`#input4`, "foo and bar", ByID},
+		{`#input4`, "1234567890", ByID},
+		{`#input4`, "~!@#$%^&*()_+=[];'", ByID},
+		{`#input4`, "你", ByID},
+		{`#input4`, "\n\nfoo\n\nbar\n\n", ByID},
+		{`document.querySelector('#input4')`, "\n\ntest\n\n", ByJSPath},
 	}
 
 	for i, test := range tests {
@@ -251,7 +255,7 @@ func TestKeyActionNode(t *testing.T) {
 			}
 			var value string
 			if err := Run(ctx,
-				KeyActionNode(nodes[0], test.exp),
+				KeyEventNode(nodes[0], test.exp),
 				Value(test.sel, &value, test.by),
 			); err != nil {
 				t.Fatalf("got error: %v", err)

js.go

@@ -1,5 +1,11 @@
 package chromedp
 
+import (
+	"fmt"
+
+	"github.com/chromedp/cdproto/cdp"
+)
+
 const (
 	// textJS is a javascript snippet that returns the concatenated textContent
 	// of all visible (ie, offsetParent !== null) children.
@@ -11,63 +17,94 @@ const (
 			}
 		}
 		return s;
-	})($x('%s/node()'))`
+	})(%s)`
 
 	// blurJS is a javscript snippet that blurs the specified element.
 	blurJS = `(function(a) {
-		a[0].blur();
+		a.blur();
 		return true;
-	})($x(%q))`
+	})(%s)`
 
 	// scrollIntoViewJS is a javascript snippet that scrolls the specified node
 	// into the window's viewport (if needed), returning the actual window x/y
 	// after execution.
 	scrollIntoViewJS = `(function(a) {
-		a[0].scrollIntoViewIfNeeded(true);
+		a.scrollIntoViewIfNeeded(true);
 		return [window.scrollX, window.scrollY];
-	})($x(%q))`
+	})(%s)`
 
 	// submitJS is a javascript snippet that will call the containing form's
 	// submit function, returning true or false if the call was successful.
 	submitJS = `(function(a) {
-		if (a[0].nodeName === 'FORM') {
-			a[0].submit();
+		if (a.nodeName === 'FORM') {
+			a.submit();
 			return true;
-		} else if (a[0].form !== null) {
-			a[0].form.submit();
+		} else if (a.form !== null) {
+			a.form.submit();
 			return true;
 		}
 		return false;
-	})($x(%q))`
+	})(%s)`
 
 	// resetJS is a javascript snippet that will call the containing form's
 	// reset function, returning true or false if the call was successful.
 	resetJS = `(function(a) {
-		if (a[0].nodeName === 'FORM') {
-			a[0].reset();
+		if (a.nodeName === 'FORM') {
+			a.reset();
 			return true;
-		} else if (a[0].form !== null) {
-			a[0].form.reset();
+		} else if (a.form !== null) {
+			a.form.reset();
 			return true;
 		}
 		return false;
-	})($x(%q))`
+	})(%s)`
 
 	// attributeJS is a javascript snippet that returns the attribute of a specified
 	// node.
 	attributeJS = `(function(a, n) {
-		return a[0][n];
-	})($x(%q), %q)`
+		return a[n];
+	})(%s, %q)`
 
 	// setAttributeJS is a javascript snippet that sets the value of the specified
 	// node, and returns the value.
 	setAttributeJS = `(function(a, n, v) {
-		return a[0][n] = v;
-	})($x(%q), %q, %q)`
+		return a[n] = v;
+	})(%s, %q, %q)`
 
 	// visibleJS is a javascript snippet that returns true or false depending
 	// on if the specified node's offsetParent is not null.
 	visibleJS = `(function(a) {
-		return a[0].offsetParent !== null;
-	})($x(%q))`
+		return a.offsetParent !== null;
+	})(%s)`
 )
+
+// snippet builds a Javascript expression snippet.
+func snippet(js string, f func(n *cdp.Node) string, sel interface{}, n *cdp.Node, v ...interface{}) string {
+	switch s := sel.(type) {
+	case *Selector:
+		if s != nil && s.raw {
+			return fmt.Sprintf(js, append([]interface{}{s.selAsString()}, v...)...)
+		}
+	}
+	return fmt.Sprintf(js, append([]interface{}{f(n)}, v...)...)
+}
+
+// cashX returns the $x() expression using the node's full xpath value.
+func cashX(flatten bool) func(*cdp.Node) string {
+	return func(n *cdp.Node) string {
+		if flatten {
+			return fmt.Sprintf(`$x(%q)[0]`, n.FullXPath())
+		}
+		return fmt.Sprintf(`$x(%q)`, n.FullXPath())
+	}
+}
+
+// cashXNode returns the $x(/node()) expression using the node's full xpath value.
+func cashXNode(flatten bool) func(*cdp.Node) string {
+	return func(n *cdp.Node) string {
+		if flatten {
+			return fmt.Sprintf(`$x(%q)[0]`, n.FullXPath()+"/node()")
+		}
+		return fmt.Sprintf(`$x(%q)`, n.FullXPath()+"/node()")
+	}
+}

kb/gen.go

@@ -372,7 +372,7 @@ func processKeys(keys map[rune]kb.Key) ([]byte, []byte, error) {
 		// add key definition
 		v := strings.TrimPrefix(fmt.Sprintf("%#v", key), "kb.")
 		v = nameRE.ReplaceAllString(v, "")
-		mapBuf.WriteString(fmt.Sprintf("%q: &%s,\n", s, v))
+		mapBuf.WriteString(fmt.Sprintf("'%s': &%s,\n", s, v))
 
 		// fix 'Quote' const
 		if s == `\'` {

nav.go

@@ -4,51 +4,54 @@ import (
 	"context"
 	"errors"
 
-	"github.com/chromedp/cdproto/cdp"
 	"github.com/chromedp/cdproto/page"
 )
 
-// Navigate navigates the current frame.
-func Navigate(urlstr string) Action {
+// NavigateAction are actions that manipulate the navigation of the browser.
+type NavigateAction Action
+
+// Navigate is an action that navigates the current frame.
+func Navigate(urlstr string) NavigateAction {
 	return ActionFunc(func(ctx context.Context) error {
-		ch := listenLoaded(ctx)
-		frameID, _, _, err := page.Navigate(urlstr).Do(ctx)
+		_, _, _, err := page.Navigate(urlstr).Do(ctx)
 		if err != nil {
 			return err
 		}
-		ch <- frameID
-		<-ch
-		return nil
+		return waitLoaded(ctx)
 	})
 }
 
-// listenLoaded sets up a listener before running an action that will load a
-// frame, so that later we can block until said frame has finished loading. A
-// channel is used to receive the frame ID to wait for and to block, since
-// page.Navigate returns the ID, but the listener must be set up before.
-func listenLoaded(ctx context.Context) chan cdp.FrameID {
-	ch := make(chan cdp.FrameID)
-	ctx, cancel := context.WithCancel(ctx)
-	var frameID cdp.FrameID
-	ListenTarget(ctx, func(ev interface{}) {
-		evs, ok := ev.(*page.EventFrameStoppedLoading)
-		if !ok {
-			return
-		}
-		if frameID == "" {
-			frameID = <-ch
-		}
-		if evs.FrameID == frameID {
+// waitLoaded blocks until a target receives a Page.loadEventFired.
+func waitLoaded(ctx context.Context) error {
+	// TODO: this function is inherently racy, as we don't run ListenTarget
+	// until after the navigate action is fired. For example, adding
+	// time.Sleep(time.Second) at the top of this body makes most tests hang
+	// forever, as they miss the load event.
+	//
+	// However, setting up the listener before firing the navigate action is
+	// also racy, as we might get a load event from a previous navigate.
+	//
+	// For now, the second race seems much more common in real scenarios, so
+	// keep the first approach. Is there a better way to deal with this?
+	ch := make(chan struct{})
+	lctx, cancel := context.WithCancel(ctx)
+	ListenTarget(lctx, func(ev interface{}) {
+		if _, ok := ev.(*page.EventLoadEventFired); ok {
 			cancel()
 			close(ch)
 		}
 	})
-	return ch
+	select {
+	case <-ch:
+		return nil
+	case <-ctx.Done():
+		return ctx.Err()
+	}
 }
 
-// NavigationEntries is an action to retrieve the page's navigation history
+// NavigationEntries is an action that retrieves the page's navigation history
 // entries.
-func NavigationEntries(currentIndex *int64, entries *[]*page.NavigationEntry) Action {
+func NavigationEntries(currentIndex *int64, entries *[]*page.NavigationEntry) NavigateAction {
 	if currentIndex == nil || entries == nil {
 		panic("currentIndex and entries cannot be nil")
 	}
@@ -62,21 +65,18 @@ func NavigationEntries(currentIndex *int64, entries *[]*page.NavigationEntry) Ac
 
 // NavigateToHistoryEntry is an action to navigate to the specified navigation
 // entry.
-func NavigateToHistoryEntry(entryID int64) Action {
+func NavigateToHistoryEntry(entryID int64) NavigateAction {
 	return ActionFunc(func(ctx context.Context) error {
-		ch := listenLoaded(ctx)
-		frameID := FromContext(ctx).Target.cur.ID
 		if err := page.NavigateToHistoryEntry(entryID).Do(ctx); err != nil {
 			return err
 		}
-		ch <- frameID
-		<-ch
-		return nil
+		return waitLoaded(ctx)
 	})
 }
 
-// NavigateBack navigates the current frame backwards in its history.
-func NavigateBack() Action {
+// NavigateBack is an action that navigates the current frame backwards in its
+// history.
+func NavigateBack() NavigateAction {
 	return ActionFunc(func(ctx context.Context) error {
 		cur, entries, err := page.GetNavigationHistory().Do(ctx)
 		if err != nil {
@@ -87,20 +87,17 @@ func NavigateBack() Action {
 			return errors.New("invalid navigation entry")
 		}
 
-		ch := listenLoaded(ctx)
-		frameID := FromContext(ctx).Target.cur.ID
 		entryID := entries[cur-1].ID
 		if err := page.NavigateToHistoryEntry(entryID).Do(ctx); err != nil {
 			return err
 		}
-		ch <- frameID
-		<-ch
-		return nil
+		return waitLoaded(ctx)
 	})
 }
 
-// NavigateForward navigates the current frame forwards in its history.
-func NavigateForward() Action {
+// NavigateForward is an action that navigates the current frame forwards in
+// its history.
+func NavigateForward() NavigateAction {
 	return ActionFunc(func(ctx context.Context) error {
 		cur, entries, err := page.GetNavigationHistory().Do(ctx)
 		if err != nil {
@@ -111,40 +108,36 @@ func NavigateForward() Action {
 			return errors.New("invalid navigation entry")
 		}
 
-		ch := listenLoaded(ctx)
-		frameID := FromContext(ctx).Target.cur.ID
 		entryID := entries[cur+1].ID
 		if err := page.NavigateToHistoryEntry(entryID).Do(ctx); err != nil {
 			return err
 		}
-		ch <- frameID
-		<-ch
-		return nil
+		return waitLoaded(ctx)
 	})
 }
 
-// Reload reloads the current page.
-func Reload() Action {
+// Reload is an action that reloads the current page.
+func Reload() NavigateAction {
 	return ActionFunc(func(ctx context.Context) error {
-		ch := listenLoaded(ctx)
-		frameID := FromContext(ctx).Target.cur.ID
 		if err := page.Reload().Do(ctx); err != nil {
 			return err
 		}
-		ch <- frameID
-		<-ch
-		return nil
+		return waitLoaded(ctx)
 	})
 }
 
-// Stop stops all navigation and pending resource retrieval.
-func Stop() Action {
+// Stop is an action that stops all navigation and pending resource retrieval.
+func Stop() NavigateAction {
 	return page.StopLoading()
 }
 
-// CaptureScreenshot captures takes a screenshot of the current viewport.
+// CaptureScreenshot is an action that captures/takes a screenshot of the
+// current browser viewport.
+//
+// See the Screenshot action to take a screenshot of a specific element.
 //
-// Note: this an alias for page.CaptureScreenshot.
+// See the 'screenshot' example in the https://github.com/chromedp/examples
+// project for an example of taking a screenshot of the entire page.
 func CaptureScreenshot(res *[]byte) Action {
 	if res == nil {
 		panic("res cannot be nil")
@@ -157,38 +150,18 @@ func CaptureScreenshot(res *[]byte) Action {
 	})
 }
 
-// AddOnLoadScript adds a script to evaluate on page load.
-/*func AddOnLoadScript(source string, id *page.ScriptIdentifier) Action {
-	if id == nil {
-		panic("id cannot be nil")
-	}
-
-	return ActionFunc(func(ctx context.Context) error {
-		var err error
-		*id, err = page.AddScriptToEvaluateOnLoad(source).Do(ctx)
-		return err
-	})
-}
-
-// RemoveOnLoadScript removes a script to evaluate on page load.
-func RemoveOnLoadScript(id page.ScriptIdentifier) Action {
-	return page.RemoveScriptToEvaluateOnLoad(id)
-}*/
-
-// Location retrieves the document location.
+// Location is an action that retrieves the document location.
 func Location(urlstr *string) Action {
 	if urlstr == nil {
 		panic("urlstr cannot be nil")
 	}
-
 	return EvaluateAsDevTools(`document.location.toString()`, urlstr)
 }
 
-// Title retrieves the document title.
+// Title is an action that retrieves the document title.
 func Title(title *string) Action {
 	if title == nil {
 		panic("title cannot be nil")
 	}
-
 	return EvaluateAsDevTools(`document.title`, title)
 }

nav_test.go

@@ -2,16 +2,17 @@ package chromedp
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"image"
 	_ "image/png"
+	"io"
 	"net/http"
 	"net/http/httptest"
 	"strings"
 	"testing"
 	"time"
 
-	"github.com/chromedp/cdproto/emulation"
 	"github.com/chromedp/cdproto/page"
 )
 
@@ -210,11 +211,12 @@ func TestCaptureScreenshot(t *testing.T) {
 	ctx, cancel := testAllocate(t, "image.html")
 	defer cancel()
 
+	const width, height = 650, 450
+
 	// set the viewport size, to know what screenshot size to expect
-	width, height := 650, 450
 	var buf []byte
 	if err := Run(ctx,
-		emulation.SetDeviceMetricsOverride(int64(width), int64(height), 1.0, false),
+		EmulateViewport(width, height),
 		CaptureScreenshot(&buf),
 	); err != nil {
 		t.Fatal(err)
@@ -233,94 +235,146 @@ func TestCaptureScreenshot(t *testing.T) {
 	}
 }
 
-/*func TestAddOnLoadScript(t *testing.T) {
+func TestLocation(t *testing.T) {
 	t.Parallel()
 
-	ctx, cancel := testAllocate(t, "")
+	ctx, cancel := testAllocate(t, "form.html")
 	defer cancel()
 
-	var scriptID page.ScriptIdentifier
-	if err := Run(ctx,
-		AddOnLoadScript(`window.alert("TEST")`, &scriptID),
-		Navigate(testdataDir+"/form.html"),
-	); err != nil {
+	var urlstr string
+	if err := Run(ctx, Location(&urlstr)); err != nil {
 		t.Fatal(err)
 	}
 
-	if scriptID == "" {
-		t.Fatal("got empty script ID")
+	if !strings.HasSuffix(urlstr, "form.html") {
+		t.Fatalf("expected to be on form.html, got %q", urlstr)
 	}
-	// TODO: Handle javascript dialog.
 }
 
-func TestRemoveOnLoadScript(t *testing.T) {
+func TestTitle(t *testing.T) {
 	t.Parallel()
 
-	ctx, cancel := testAllocate(t, "")
+	ctx, cancel := testAllocate(t, "image.html")
 	defer cancel()
 
-	var scriptID page.ScriptIdentifier
-	if err := Run(ctx, AddOnLoadScript(`window.alert("TEST")`, &scriptID)); err != nil {
+	var title string
+	if err := Run(ctx, Title(&title)); err != nil {
 		t.Fatal(err)
 	}
-	if scriptID == "" {
-		t.Fatal("got empty script ID")
+
+	exptitle := "this is title"
+	if title != exptitle {
+		t.Fatalf("expected title to be %q, got %q", exptitle, title)
 	}
+}
+
+func TestLoadIframe(t *testing.T) {
+	t.Parallel()
+
+	ctx, cancel := testAllocate(t, "iframe.html")
+	defer cancel()
 
 	if err := Run(ctx,
-		RemoveOnLoadScript(scriptID),
-		Navigate(testdataDir+"/form.html"),
+		// TODO: remove the sleep once we have better support for
+		// iframes.
+		Sleep(10*time.Millisecond),
+		// WaitVisible(`#form`, ByID), // for the nested form.html
+		WaitVisible(`#parent`, ByID), // for iframe.html
 	); err != nil {
 		t.Fatal(err)
 	}
-}*/
+}
 
-func TestLocation(t *testing.T) {
+func TestNavigateContextTimeout(t *testing.T) {
 	t.Parallel()
 
-	ctx, cancel := testAllocate(t, "form.html")
+	ctx, cancel := testAllocate(t, "")
 	defer cancel()
 
-	var urlstr string
-	if err := Run(ctx, Location(&urlstr)); err != nil {
+	// Serve the page, but cancel the context almost immediately after.
+	// Navigate shouldn't block waiting for the load to finish, which may
+	// not come as the target is cancelled.
+	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		go func() {
+			time.Sleep(time.Millisecond)
+			cancel()
+		}()
+	}))
+	defer s.Close()
+
+	if err := Run(ctx, Navigate(s.URL)); err != nil && err != context.Canceled {
 		t.Fatal(err)
 	}
+}
 
-	if !strings.HasSuffix(urlstr, "form.html") {
-		t.Fatalf("expected to be on form.html, got %q", urlstr)
-	}
+func writeHTML(content string) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/html")
+		io.WriteString(w, strings.TrimSpace(content))
+	})
 }
 
-func TestTitle(t *testing.T) {
+func TestClickNavigate(t *testing.T) {
 	t.Parallel()
 
-	ctx, cancel := testAllocate(t, "image.html")
+	ctx, cancel := testAllocate(t, "")
 	defer cancel()
 
+	mux := http.NewServeMux()
+	mux.Handle("/", writeHTML(`
+<img src="/img.jpg"></img>
+	`))
+	ch := make(chan struct{})
+	mux.Handle("/img.jpg", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		<-ch
+	}))
+	s := httptest.NewServer(mux)
+	defer s.Close()
+
+	// First, navigate to a page that starts loading, but doesn't finish.
+	// Then, tell the server to finish loading the page.
+	// Immediately after, navigate to another page.
+	// Finally, grab the page title, which should correspond with the last
+	// page.
+	//
+	// This has caused problems in the past. Because the first page might
+	// fire its load event just as we start the second navigate, the second
+	// navigate used to get confused, either blocking forever or not waiting
+	// for the right load event (the second).
 	var title string
-	if err := Run(ctx, Title(&title)); err != nil {
+	if err := Run(ctx,
+		ActionFunc(func(ctx context.Context) error {
+			_, _, _, err := page.Navigate(s.URL).Do(ctx)
+			return err
+		}),
+		ActionFunc(func(ctx context.Context) error {
+			ch <- struct{}{}
+			return nil
+		}),
+		Navigate(testdataDir+"/image.html"),
+		Title(&title),
+	); err != nil {
 		t.Fatal(err)
 	}
-
 	exptitle := "this is title"
 	if title != exptitle {
-		t.Fatalf("expected title to be %q, got %q", exptitle, title)
+		t.Errorf("want title to be %q, got %q", exptitle, title)
 	}
 }
 
-func TestLoadIframe(t *testing.T) {
+func TestNavigateWithoutWaitingForLoad(t *testing.T) {
 	t.Parallel()
 
-	ctx, cancel := testAllocate(t, "iframe.html")
+	ctx, cancel := testAllocate(t, "")
 	defer cancel()
 
-	if err := Run(ctx, Tasks{
-		// TODO: remove the sleep once we have better support for
-		// iframes.
-		Sleep(10 * time.Millisecond),
-		// WaitVisible(`#form`, ByID), // for the nested form.html
-		WaitVisible(`#parent`, ByID), // for iframe.html
-	}); err != nil {
+	if err := Run(ctx,
+		ActionFunc(func(ctx context.Context) error {
+			_, _, _, err := page.Navigate(testdataDir + "/form.html").Do(ctx)
+			return err
+		}),
+		WaitVisible(`#form`, ByID), // for form.html
+	); err != nil {
 		t.Fatal(err)
 	}
 }

target.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"sync"
 	"sync/atomic"
-	"time"
 
 	"github.com/mailru/easyjson"
 
@@ -24,43 +23,20 @@ type Target struct {
 	listenersMu sync.Mutex
 	listeners   []cancelableListener
 
-	waitQueue    chan func() bool
 	messageQueue chan *cdproto.Message
 
 	// frames is the set of encountered frames.
 	frames map[cdp.FrameID]*cdp.Frame
 
 	// cur is the current top level frame.
-	cur *cdp.Frame
+	cur   *cdp.Frame
+	curMu sync.RWMutex
 
 	// logging funcs
 	logf, errf func(string, ...interface{})
-
-	tick chan time.Time
 }
 
 func (t *Target) run(ctx context.Context) {
-	// tryWaits runs all wait functions on the current top-level frame, if
-	// neither are empty.
-	//
-	// This function is run after each DOM event, since those are the vast
-	// majority that we wait on, and approximately every 5ms. The periodic
-	// runs are necessary to wait for events such as a node no longer being
-	// visible in Chrome.
-	tryWaits := func() {
-		n := len(t.waitQueue)
-		if n == 0 || t.cur == nil {
-			return
-		}
-		for i := 0; i < n; i++ {
-			fn := <-t.waitQueue
-			if !fn() {
-				// try again later.
-				t.waitQueue <- fn
-			}
-		}
-	}
-
 	type eventValue struct {
 		method cdproto.MethodType
 		value  interface{}
@@ -115,10 +91,7 @@ func (t *Target) run(ctx context.Context) {
 				t.pageEvent(ev.value)
 			case "DOM":
 				t.domEvent(ctx, ev.value)
-				tryWaits()
 			}
-		case <-t.tick:
-			tryWaits()
 		}
 	}
 }
@@ -184,7 +157,9 @@ func (t *Target) Execute(ctx context.Context, method string, params easyjson.Mar
 // documentUpdated handles the document updated event, retrieving the document
 // root for the root frame.
 func (t *Target) documentUpdated(ctx context.Context) {
+	t.curMu.RLock()
 	f := t.cur
+	t.curMu.RUnlock()
 	if f == nil {
 		// TODO: This seems to happen on CI, when running the tests
 		// under the headless-shell Docker image. Figure out why.
@@ -224,7 +199,9 @@ func (t *Target) pageEvent(ev interface{}) {
 		if e.Frame.ParentID == "" {
 			// This frame is only the new top-level frame if it has
 			// no parent.
+			t.curMu.Lock()
 			t.cur = e.Frame
+			t.curMu.Unlock()
 		}
 		return
 
@@ -348,5 +325,3 @@ func (t *Target) domEvent(ctx context.Context, ev interface{}) {
 	op(n)
 	f.Unlock()
 }
-
-type TargetOption func(*Target)

testdata/form.html

@@ -18,4 +18,4 @@
     <input id="btn2" type="submit" value="Submit">
   </form>
 </body>
-</html>
+</html>

testdata/image.html

@@ -7,9 +7,9 @@
         display: inline-block;
         width: 200px;
         height: 200px;
-	position: relative;
-	top: 45px;
-	left: 45px;
+        position: relative;
+        top: 45px;
+        left: 45px;
         background: linear-gradient(to right, blue 50%, red 50%);
       }
     </style>