-
Notifications
You must be signed in to change notification settings - Fork 1.5k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* Dockerhub v2 detector * update package structure --------- Co-authored-by: Dustin Decker <dustin@trufflesec.com>
- Loading branch information
1 parent
f1abe2c
commit 6dbe808
Showing
5 changed files
with
330 additions
and
5 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,179 @@ | ||
package dockerhubv2 | ||
|
||
import ( | ||
"context" | ||
"encoding/json" | ||
"fmt" | ||
"io" | ||
"net/http" | ||
"strings" | ||
|
||
regexp "github.com/wasilibs/go-re2" | ||
|
||
"github.com/golang-jwt/jwt/v4" | ||
|
||
"github.com/trufflesecurity/trufflehog/v3/pkg/common" | ||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors" | ||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" | ||
) | ||
|
||
type Scanner struct { | ||
client *http.Client | ||
} | ||
|
||
func (s Scanner) Version() int { return 1 } | ||
|
||
// Ensure the Scanner satisfies the interface at compile time. | ||
var _ detectors.Detector = (*Scanner)(nil) | ||
var _ detectors.Versioner = (*Scanner)(nil) | ||
|
||
var ( | ||
// Can use email or username for login. | ||
usernamePat = regexp.MustCompile(`(?im)(?:user|usr|-u|id)\S{0,40}?[:=\s]{1,3}[ '"=]?([a-zA-Z0-9]{4,40})\b`) | ||
emailPat = regexp.MustCompile(`(` + common.EmailPattern + `)`) | ||
|
||
// Can use password or personal access token (PAT) for login, but this scanner will only check for PATs. | ||
accessTokenPat = regexp.MustCompile(detectors.PrefixRegex([]string{"docker"}) + `\b([a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})\b`) | ||
) | ||
|
||
// Keywords are used for efficiently pre-filtering chunks. | ||
// Use identifiers in the secret preferably, or the provider name. | ||
func (s Scanner) Keywords() []string { | ||
return []string{"docker"} | ||
} | ||
|
||
// FromData will find and optionally verify Dockerhub secrets in a given set of bytes. | ||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { | ||
dataStr := string(data) | ||
|
||
// Deduplicate results. | ||
tokens := make(map[string]struct{}) | ||
for _, matches := range accessTokenPat.FindAllStringSubmatch(dataStr, -1) { | ||
tokens[matches[1]] = struct{}{} | ||
} | ||
if len(tokens) == 0 { | ||
return | ||
} | ||
usernames := make(map[string]struct{}) | ||
for _, matches := range usernamePat.FindAllStringSubmatch(dataStr, -1) { | ||
usernames[matches[1]] = struct{}{} | ||
} | ||
for _, matches := range emailPat.FindAllStringSubmatch(dataStr, -1) { | ||
usernames[matches[1]] = struct{}{} | ||
} | ||
|
||
// Process results. | ||
for token := range tokens { | ||
s1 := detectors.Result{ | ||
DetectorType: s.Type(), | ||
Raw: []byte(token), | ||
} | ||
|
||
for username := range usernames { | ||
s1.RawV2 = []byte(fmt.Sprintf("%s:%s", username, token)) | ||
|
||
if verify { | ||
if s.client == nil { | ||
s.client = common.SaneHttpClient() | ||
} | ||
|
||
isVerified, extraData, verificationErr := s.verifyMatch(ctx, username, token) | ||
s1.Verified = isVerified | ||
s1.ExtraData = extraData | ||
s1.SetVerificationError(verificationErr) | ||
} | ||
|
||
results = append(results, s1) | ||
|
||
if s1.Verified { | ||
break | ||
} | ||
} | ||
|
||
// PAT matches without usernames cannot be verified but might still be useful. | ||
if len(usernames) == 0 { | ||
results = append(results, s1) | ||
} | ||
} | ||
return | ||
} | ||
|
||
func (s Scanner) verifyMatch(ctx context.Context, username string, password string) (bool, map[string]string, error) { | ||
payload := strings.NewReader(fmt.Sprintf(`{"username": "%s", "password": "%s"}`, username, password)) | ||
|
||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, "https://hub.docker.com/v2/users/login", payload) | ||
if err != nil { | ||
return false, nil, err | ||
} | ||
|
||
req.Header.Add("Content-Type", "application/json") | ||
res, err := s.client.Do(req) | ||
if err != nil { | ||
return false, nil, err | ||
} | ||
defer res.Body.Close() | ||
body, err := io.ReadAll(res.Body) | ||
if err != nil { | ||
return false, nil, err | ||
} | ||
|
||
if res.StatusCode == http.StatusOK { | ||
var tokenRes tokenResponse | ||
if err := json.Unmarshal(body, &tokenRes); (err != nil || tokenRes == tokenResponse{}) { | ||
return false, nil, err | ||
} | ||
|
||
parser := jwt.NewParser() | ||
token, _, err := parser.ParseUnverified(tokenRes.Token, &hubJwtClaims{}) | ||
if err != nil { | ||
return true, nil, err | ||
} | ||
|
||
if claims, ok := token.Claims.(*hubJwtClaims); ok { | ||
extraData := map[string]string{ | ||
"hub_username": username, | ||
"hub_email": claims.HubClaims.Email, | ||
"hub_scope": claims.Scope, | ||
} | ||
return true, extraData, nil | ||
} | ||
return true, nil, nil | ||
} else if res.StatusCode == http.StatusUnauthorized { | ||
// Valid credentials can still return a 401 status code if 2FA is enabled | ||
var mfaRes mfaRequiredResponse | ||
if err := json.Unmarshal(body, &mfaRes); err != nil || mfaRes.MfaToken == "" { | ||
return false, nil, nil | ||
} | ||
|
||
extraData := map[string]string{ | ||
"hub_username": username, | ||
"2fa_required": "true", | ||
} | ||
return true, extraData, nil | ||
} else { | ||
return false, nil, fmt.Errorf("unexpected response status %d", res.StatusCode) | ||
} | ||
} | ||
|
||
type tokenResponse struct { | ||
Token string `json:"token"` | ||
} | ||
|
||
type userClaims struct { | ||
Username string `json:"username"` | ||
Email string `json:"email"` | ||
} | ||
|
||
type hubJwtClaims struct { | ||
Scope string `json:"scope"` | ||
HubClaims userClaims `json:"https://hub.docker.com"` // not sure why this is a key, further investigation required. | ||
jwt.RegisteredClaims | ||
} | ||
|
||
type mfaRequiredResponse struct { | ||
MfaToken string `json:"login_2fa_token"` | ||
} | ||
|
||
func (s Scanner) Type() detectorspb.DetectorType { | ||
return detectorspb.DetectorType_Dockerhub | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,141 @@ | ||
//go:build detectors | ||
// +build detectors | ||
|
||
package dockerhubv2 | ||
|
||
import ( | ||
"context" | ||
"fmt" | ||
"testing" | ||
"time" | ||
|
||
"github.com/kylelemons/godebug/pretty" | ||
|
||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors" | ||
|
||
"github.com/trufflesecurity/trufflehog/v3/pkg/common" | ||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" | ||
) | ||
|
||
func TestDockerhub_FromChunk(t *testing.T) { | ||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) | ||
defer cancel() | ||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4") | ||
if err != nil { | ||
t.Fatalf("could not get test secrets from GCP: %s", err) | ||
} | ||
username := testSecrets.MustGetField("DOCKERHUB_USERNAME") | ||
email := testSecrets.MustGetField("DOCKERHUB_EMAIL") | ||
pat := testSecrets.MustGetField("DOCKERHUB_PAT") | ||
inactivePat := testSecrets.MustGetField("DOCKERHUB_INACTIVE_PAT") | ||
|
||
type args struct { | ||
ctx context.Context | ||
data []byte | ||
verify bool | ||
} | ||
tests := []struct { | ||
name string | ||
s Scanner | ||
args args | ||
want []detectors.Result | ||
wantErr bool | ||
}{ | ||
{ | ||
name: "found, verified", | ||
s: Scanner{}, | ||
args: args{ | ||
ctx: context.Background(), | ||
data: []byte(fmt.Sprintf("docker login -u %s -p %s", username, pat)), | ||
verify: true, | ||
}, | ||
want: []detectors.Result{ | ||
{ | ||
DetectorType: detectorspb.DetectorType_Dockerhub, | ||
Verified: true, | ||
}, | ||
}, | ||
wantErr: false, | ||
}, | ||
{ | ||
name: "found, verified (email)", | ||
s: Scanner{}, | ||
args: args{ | ||
ctx: context.Background(), | ||
data: []byte(fmt.Sprintf("docker login -u %s -p %s", email, pat)), | ||
verify: true, | ||
}, | ||
want: []detectors.Result{ | ||
{ | ||
DetectorType: detectorspb.DetectorType_Dockerhub, | ||
Verified: true, | ||
}, | ||
}, | ||
wantErr: false, | ||
}, | ||
{ | ||
name: "found, unverified", | ||
s: Scanner{}, | ||
args: args{ | ||
ctx: context.Background(), | ||
data: []byte(fmt.Sprintf("docker login -u %s -p %s", username, inactivePat)), // the secret would satisfy the regex but not pass validation | ||
verify: true, | ||
}, | ||
want: []detectors.Result{ | ||
{ | ||
DetectorType: detectorspb.DetectorType_Dockerhub, | ||
Verified: false, | ||
}, | ||
}, | ||
wantErr: false, | ||
}, | ||
{ | ||
name: "not found", | ||
s: Scanner{}, | ||
args: args{ | ||
ctx: context.Background(), | ||
data: []byte("You cannot find the secret within"), | ||
verify: true, | ||
}, | ||
want: nil, | ||
wantErr: false, | ||
}, | ||
} | ||
for _, tt := range tests { | ||
t.Run(tt.name, func(t *testing.T) { | ||
s := Scanner{} | ||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) | ||
if (err != nil) != tt.wantErr { | ||
t.Errorf("Dockerhub.FromData() error = %v, wantErr %v", err, tt.wantErr) | ||
return | ||
} | ||
for i := range got { | ||
if len(got[i].Raw) == 0 { | ||
t.Fatalf("no raw secret present: \n %+v", got[i]) | ||
} | ||
got[i].Raw = nil | ||
got[i].RawV2 = nil | ||
got[i].ExtraData = nil | ||
} | ||
if diff := pretty.Compare(got, tt.want); diff != "" { | ||
t.Errorf("Dockerhub.FromData() %s diff: (-got +want)\n%s", tt.name, diff) | ||
} | ||
}) | ||
} | ||
} | ||
|
||
func BenchmarkFromData(benchmark *testing.B) { | ||
ctx := context.Background() | ||
s := Scanner{} | ||
for name, data := range detectors.MustGetBenchmarkData() { | ||
benchmark.Run(name, func(b *testing.B) { | ||
b.ResetTimer() | ||
for n := 0; n < b.N; n++ { | ||
_, err := s.FromData(ctx, false, data) | ||
if err != nil { | ||
b.Fatal(err) | ||
} | ||
} | ||
}) | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters