Merge pull request #110 from euank/lazy-init

pass: only init on run, and do so lazily

.travis.yml

@@ -3,19 +3,26 @@
   sudo: required
   language: go
   dist: trusty
+  osx_image: xcode9
   os:
     - linux
     - osx
   notifications:
     email: false
   go:
-    - 1.8
+    - 1.8.1
-  install: make deps
+  before_install:
+    # work-around for issue https://github.com/travis-ci/travis-ci/issues/6307
+    # might not be necessary in the future
+    - command curl -sSL https://rvm.io/mpapis.asc | gpg --import -
+    - rvm get stable
   addons:
     apt:
       packages:
         - libsecret-1-dev
+        - pass
   before_script:
+    - make deps
     - "export DISPLAY=:99.0"
     - if [[ "$TRAVIS_OS_NAME" == "linux"  ]]; then sh ci/before_script_linux.sh; fi
     - make validate

CHANGELOG.md

@@ -4,6 +4,17 @@ This changelog tracks the releases of docker-credential-helpers.
 This project includes different binaries per platform.
 The platform released is identified after the tag name.
 
+## v0.6.0 (Go client, Linux)
+
+- New credential helper on Linux using `pass`
+- New entry point for passing environment variables when calling a credential helper
+- Add a Makefile rule generating a Windows release binary
+
+### Note
+
+`pass` needs to be configured for `docker-credential-pass` to work properly.
+It must be initialized with a `gpg2` key ID. Make sure your GPG key exists is in `gpg2` keyring as `pass` uses `gpg2` instead of the regular `gpg`.
+
 ## v0.5.2 (Mac OS X, Windows, Linux)
 
 - Add a `version` command to output the version

MAINTAINERS

@@ -11,18 +11,13 @@
 [Org]
 	[Org."Core maintainers"]
 		people = [
-			"aaronlehmann",
-			"calavera",
 			"coolljt0725",
 			"cpuguy83",
 			"crosbymichael",
 			"dnephin",
-			"dongluochen",
 			"duglin",
 			"estesp",
-			"icecrime",
 			"jhowardmsft",
-			"lk4d4",
 			"mavenugo",
 			"mhbauer",
 			"n4ss",
@@ -45,16 +40,6 @@
 
 	# ADD YOURSELF HERE IN ALPHABETICAL ORDER
 
-	[people.aaronlehmann]
-	Name = "Aaron Lehmann"
-	Email = "aaron.lehmann@docker.com"
-	GitHub = "aaronlehmann"
-
-	[people.calavera]
-	Name = "David Calavera"
-	Email = "david.calavera@gmail.com"
-	GitHub = "calavera"
-
 	[people.coolljt0725]
 	Name = "Lei Jitang"
 	Email = "leijitang@huawei.com"
@@ -75,11 +60,6 @@
 	Email = "dnephin@gmail.com"
 	GitHub = "dnephin"
 
-	[people.dongluochen]
-	Name = "Dongluo Chen"
-	Email = "dongluo.chen@docker.com"
-	GitHub = "dongluochen"
-
 	[people.duglin]
 	Name = "Doug Davis"
 	Email = "dug@us.ibm.com"
@@ -90,21 +70,11 @@
 	Email = "estesp@linux.vnet.ibm.com"
 	GitHub = "estesp"
 
-	[people.icecrime]
-	Name = "Arnaud Porterie"
-	Email = "arnaud@docker.com"
-	GitHub = "icecrime"
-
 	[people.jhowardmsft]
 	Name = "John Howard"
 	Email = "jhoward@microsoft.com"
 	GitHub = "jhowardmsft"
 
-	[people.lk4d4]
-	Name = "Alexander Morozov"
-	Email = "lk4d4@docker.com"
-	GitHub = "lk4d4"
-
 	[people.mavenugo]
 	Name = "Madhu Venugopal"
 	Email = "madhu@docker.com"

Makefile

@@ -1,4 +1,4 @@
-.PHONY: all deps osxkeychain secretservice test validate wincred
+.PHONY: all deps osxkeychain secretservice test validate wincred pass deb
 
 TRAVIS_OS_NAME ?= linux
 VERSION := $(shell grep 'const Version' credentials/version.go | awk -F'"' '{ print $$2 }')
@@ -6,14 +6,14 @@ VERSION := $(shell grep 'const Version' credentials/version.go | awk -F'"' '{ pr
 all: test
 
 deps:
-	go get github.com/golang/lint/golint
+	go get -u github.com/golang/lint/golint
 
 clean:
 	rm -rf bin
 	rm -rf release
 
 osxkeychain:
-	mkdir bin
+	mkdir -p bin
 	go build -ldflags -s -o bin/docker-credential-osxkeychain osxkeychain/cmd/main_darwin.go
 
 osxcodesign: osxkeychain
@@ -27,13 +27,22 @@ osxrelease: clean vet_osx lint fmt test osxcodesign
 	cd bin && tar cvfz ../release/docker-credential-osxkeychain-v$(VERSION)-amd64.tar.gz docker-credential-osxkeychain
 
 secretservice:
-	mkdir bin
+	mkdir -p bin
 	go build -o bin/docker-credential-secretservice secretservice/cmd/main_linux.go
 
+pass:
+	mkdir -p bin
+	go build -o bin/docker-credential-pass pass/cmd/main_linux.go
+
 wincred:
-	mkdir bin
+	mkdir -p bin
 	go build -o bin/docker-credential-wincred.exe wincred/cmd/main_windows.go
 
+winrelease: clean vet_win lint fmt test wincred
+	mkdir -p release
+	@echo "\nPackaging version ${VERSION}\n"
+	cd bin && zip ../release/docker-credential-wincred-v$(VERSION)-amd64.zip docker-credential-wincred.exe
+
 test:
 	# tests all packages except vendor
 	go test -v `go list ./... | grep -v /vendor/`
@@ -59,3 +68,15 @@ fmt:
 	gofmt -s -l `ls **/*.go | grep -v vendor`
 
 validate: vet lint fmt
+
+
+BUILDIMG:=docker-credential-secretservice-$(VERSION)
+deb:
+	mkdir -p release
+	docker build -f deb/Dockerfile \
+		--build-arg VERSION=$(VERSION) \
+		--build-arg DISTRO=xenial \
+		--tag $(BUILDIMG) \
+		.
+	docker run --rm --net=none $(BUILDIMG) tar cf - /release | tar xf -
+	docker rmi $(BUILDIMG)

README.md

@@ -55,6 +55,12 @@ You can see examples of each function in the [client](https://godoc.org/github.c
 1. osxkeychain: Provides a helper to use the OS X keychain as credentials store.
 2. secretservice: Provides a helper to use the D-Bus secret service as credentials store.
 3. wincred: Provides a helper to use Windows credentials manager as store.
+4. pass: Provides a helper to use `pass` as credentials store.
+
+#### Note
+
+`pass` needs to be configured for `docker-credential-pass` to work properly.
+It must be initialized with a `gpg2` key ID. Make sure your GPG key exists is in `gpg2` keyring as `pass` uses `gpg2` instead of the regular `gpg`.
 
 ## Development
 

appveyor.yml

@@ -2,13 +2,13 @@ image: Visual Studio 2015
 environment:
       GOPATH: c:\gopath
 
+stack: go 1.8.7
+
 clone_folder: c:\gopath\src\github.com\docker\docker-credential-helpers
 clone_depth: 10
 
 before_build:
   - set PATH=%PATH%;C:\MinGW\bin;
-  - set PATH=%PATH%;C:\go18\bin;
-  - set GOROOT=C:\go18
 
 build_script:
   - mingw32-make vet_win wincred

ci/before_script_linux.sh

@@ -2,3 +2,21 @@ set -ex
 
 sh -e /etc/init.d/xvfb start
 sleep 3 # give xvfb some time to start
+
+# init key for pass
+gpg --batch --gen-key <<-EOF
+%echo Generating a standard key
+Key-Type: DSA
+Key-Length: 1024
+Subkey-Type: ELG-E
+Subkey-Length: 1024
+Name-Real: Meshuggah Rocks
+Name-Email: meshuggah@example.com
+Expire-Date: 0
+# Do a commit here, so that we can later print "done" :-)
+%commit
+%echo done
+EOF
+
+key=$(gpg --no-auto-check-trustdb --list-secret-keys | grep ^sec | cut -d/ -f2 | cut -d" " -f1)
+pass init $key

client/command.go

@@ -1,6 +1,7 @@
 package client
 
 import (
+	"fmt"
 	"io"
 	"os"
 	"os/exec"
@@ -17,15 +18,26 @@ type ProgramFunc func(args ...string) Program
 
 // NewShellProgramFunc creates programs that are executed in a Shell.
 func NewShellProgramFunc(name string) ProgramFunc {
+	return NewShellProgramFuncWithEnv(name, nil)
+}
+
+// NewShellProgramFuncWithEnv creates programs that are executed in a Shell with environment variables
+func NewShellProgramFuncWithEnv(name string, env *map[string]string) ProgramFunc {
 	return func(args ...string) Program {
-		return &Shell{cmd: newCmdRedirectErr(name, args)}
+		return &Shell{cmd: createProgramCmdRedirectErr(name, args, env)}
 	}
 }
 
-func newCmdRedirectErr(name string, args []string) *exec.Cmd {
+func createProgramCmdRedirectErr(commandName string, args []string, env *map[string]string) *exec.Cmd {
-	newCmd := exec.Command(name, args...)
+	programCmd := exec.Command(commandName, args...)
-	newCmd.Stderr = os.Stderr
+	programCmd.Env = os.Environ()
-	return newCmd
+	if env != nil {
+		for k, v := range *env {
+			programCmd.Env = append(programCmd.Env, fmt.Sprintf("%s=%s", k, v))
+		}
+	}
+	programCmd.Stderr = os.Stderr
+	return programCmd
 }
 
 // Shell invokes shell commands to talk with a remote credentials helper.

credentials/version.go

@@ -1,4 +1,4 @@
 package credentials
 
 // Version holds a string describing the current version
-const Version = "0.5.2"
+const Version = "0.6.0"

deb/Dockerfile

@@ -0,0 +1,19 @@
+FROM ubuntu:xenial
+
+ARG VERSION
+ARG DISTRO
+
+RUN apt-get update && apt-get install -yy debhelper dh-make golang-go libsecret-1-dev
+RUN mkdir -p /build
+
+WORKDIR /build
+ENV GOPATH /build
+
+COPY Makefile .
+COPY credentials credentials
+COPY secretservice secretservice
+COPY pass pass
+COPY deb/debian ./debian
+COPY deb/build-deb .
+
+RUN /build/build-deb ${VERSION} ${DISTRO}

deb/build-deb

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+set -x
+set -e
+
+version=$1
+distro=$2
+
+maintainer=$(awk -F ': ' '$1 == "Maintainer" { print $2; exit }' debian/control)
+
+cat > "debian/changelog" <<-EOF
+docker-credential-helpers ($version) $DISTRO; urgency=low
+
+  * New upstream version
+
+ -- $maintainer  $(date --rfc-2822)
+EOF
+
+mkdir -p src/github.com/docker/docker-credential-helpers
+ln -s /build/credentials /build/src/github.com/docker/docker-credential-helpers/credentials
+ln -s /build/secretservice /build/src/github.com/docker/docker-credential-helpers/secretservice
+ln -s /build/pass /build/src/github.com/docker/docker-credential-helpers/pass
+
+dpkg-buildpackage -us -uc
+
+mkdir /release
+mv /docker-credential-* /release

deb/debian/compat

@@ -0,0 +1 @@
+9

deb/debian/control

@@ -0,0 +1,25 @@
+Source: docker-credential-helpers
+Section: admin
+Priority: optional
+Maintainer: Docker <support@docker.com>
+Homepage: https://dockerproject.org
+Standards-Version: 3.9.6
+Vcs-Browser: https://github.com/docker/docker-credential-helpers
+Vcs-Git: git://github.com/docker/docker-credential-helpers.git
+Build-Depends: debhelper
+  , dh-make
+  , libsecret-1-dev
+
+Package: docker-credential-secretservice
+Architecture: any
+Depends: libsecret-1-0
+  , ${misc:Depends}
+Description: docker-credential-secretservice is a credential helper backend
+ which uses libsecret to keep Docker credentials safe.
+
+Package: docker-credential-pass
+Architecture: any
+Depends: pass
+  , ${misc:Depends}
+Description: docker-credential-secretservice is a credential helper backend
+ which uses the pass utility to keep Docker credentials safe.

deb/debian/docker-credential-pass.install

@@ -0,0 +1 @@
+debian/tmp/usr/bin/docker-credential-pass

deb/debian/docker-credential-secretservice.install

@@ -0,0 +1 @@
+debian/tmp/usr/bin/docker-credential-secretservice

deb/debian/rules

@@ -0,0 +1,17 @@
+#!/usr/bin/make -f
+
+DESTDIR := $(CURDIR)/debian/tmp
+
+override_dh_auto_build:
+	make secretservice pass
+
+override_dh_auto_install:
+	install -D bin/docker-credential-secretservice $(DESTDIR)/usr/bin/docker-credential-secretservice
+	install -D bin/docker-credential-pass $(DESTDIR)/usr/bin/docker-credential-pass
+
+%:
+	dh $@
+
+override_dh_auto_test:
+	# no tests
+

pass/cmd/main_linux.go

@@ -0,0 +1,10 @@
+package main
+
+import (
+	"github.com/docker/docker-credential-helpers/credentials"
+	"github.com/docker/docker-credential-helpers/pass"
+)
+
+func main() {
+	credentials.Serve(pass.Pass{})
+}

pass/pass_linux.go

@@ -0,0 +1,207 @@
+// A `pass` based credential helper. Passwords are stored as arguments to pass
+// of the form: "$PASS_FOLDER/base64-url(serverURL)/username". We base64-url
+// encode the serverURL, because under the hood pass uses files and folders, so
+// /s will get translated into additional folders.
+package pass
+
+import (
+	"bytes"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path"
+	"strings"
+	"sync"
+
+	"github.com/docker/docker-credential-helpers/credentials"
+)
+
+const PASS_FOLDER = "docker-credential-helpers"
+
+// Pass handles secrets using Linux secret-service as a store.
+type Pass struct{}
+
+// Ideally these would be stored as members of Pass, but since all of Pass's
+// methods have value receivers, not pointer receivers, and changing that is
+// backwards incompatible, we assume that all Pass instances share the same configuration
+
+// initializationMutex is held while initializing so that only one 'pass'
+// round-tripping is done to check pass is functioning.
+var initializationMutex sync.Mutex
+var passInitialized bool
+
+// CheckInitialized checks whether the password helper can be used. It
+// internally caches and so may be safely called multiple times with no impact
+// on performance, though the first call may take longer.
+func (p Pass) CheckInitialized() bool {
+	return p.checkInitialized() == nil
+}
+
+func (p Pass) checkInitialized() error {
+	initializationMutex.Lock()
+	defer initializationMutex.Unlock()
+	if passInitialized {
+		return nil
+	}
+	// In principle, we could just run `pass init`. However, pass has a bug
+	// where if gpg fails, it doesn't always exit 1. Additionally, pass
+	// uses gpg2, but gpg is the default, which may be confusing. So let's
+	// just explictily check that pass actually can store and retreive a
+	// password.
+	password := "pass is initialized"
+	name := path.Join(getPassDir(), "docker-pass-initialized-check")
+
+	_, err := p.runPassHelper(password, "insert", "-f", "-m", name)
+	if err != nil {
+		return fmt.Errorf("error initializing pass: %v", err)
+	}
+
+	stored, err := p.runPassHelper("", "show", name)
+	if err != nil {
+		return fmt.Errorf("error fetching password during initialization: %v", err)
+	}
+	if stored != password {
+		return fmt.Errorf("error round-tripping password during initialization: %q != %q", password, stored)
+	}
+	passInitialized = true
+	return nil
+}
+
+func (p Pass) runPass(stdinContent string, args ...string) (string, error) {
+	if err := p.checkInitialized(); err != nil {
+		return "", err
+	}
+	return p.runPassHelper(stdinContent, args...)
+}
+
+func (p Pass) runPassHelper(stdinContent string, args ...string) (string, error) {
+	var stdout, stderr bytes.Buffer
+	cmd := exec.Command("pass", args...)
+	cmd.Stdin = strings.NewReader(stdinContent)
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+
+	err := cmd.Run()
+	if err != nil {
+		return "", fmt.Errorf("%s: %s", err, stderr.String())
+	}
+
+	// trim newlines; pass v1.7.1+ includes a newline at the end of `show` output
+	return strings.TrimRight(stdout.String(), "\n\r"), nil
+}
+
+// Add adds new credentials to the keychain.
+func (h Pass) Add(creds *credentials.Credentials) error {
+	if creds == nil {
+		return errors.New("missing credentials")
+	}
+
+	encoded := base64.URLEncoding.EncodeToString([]byte(creds.ServerURL))
+
+	_, err := h.runPass(creds.Secret, "insert", "-f", "-m", path.Join(PASS_FOLDER, encoded, creds.Username))
+	return err
+}
+
+// Delete removes credentials from the store.
+func (h Pass) Delete(serverURL string) error {
+	if serverURL == "" {
+		return errors.New("missing server url")
+	}
+
+	encoded := base64.URLEncoding.EncodeToString([]byte(serverURL))
+	_, err := h.runPass("", "rm", "-rf", path.Join(PASS_FOLDER, encoded))
+	return err
+}
+
+func getPassDir() string {
+	passDir := "$HOME/.password-store"
+	if envDir := os.Getenv("PASSWORD_STORE_DIR"); envDir != "" {
+		passDir = envDir
+	}
+	return os.ExpandEnv(passDir)
+}
+
+// listPassDir lists all the contents of a directory in the password store.
+// Pass uses fancy unicode to emit stuff to stdout, so rather than try
+// and parse this, let's just look at the directory structure instead.
+func listPassDir(args ...string) ([]os.FileInfo, error) {
+	passDir := getPassDir()
+	p := path.Join(append([]string{passDir, PASS_FOLDER}, args...)...)
+	contents, err := ioutil.ReadDir(p)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return []os.FileInfo{}, nil
+		}
+
+		return nil, err
+	}
+
+	return contents, nil
+}
+
+// Get returns the username and secret to use for a given registry server URL.
+func (h Pass) Get(serverURL string) (string, string, error) {
+	if serverURL == "" {
+		return "", "", errors.New("missing server url")
+	}
+
+	encoded := base64.URLEncoding.EncodeToString([]byte(serverURL))
+
+	if _, err := os.Stat(path.Join(getPassDir(), PASS_FOLDER, encoded)); err != nil {
+		if os.IsNotExist(err) {
+			return "", "", nil
+		}
+
+		return "", "", err
+	}
+
+	usernames, err := listPassDir(encoded)
+	if err != nil {
+		return "", "", err
+	}
+
+	if len(usernames) < 1 {
+		return "", "", fmt.Errorf("no usernames for %s", serverURL)
+	}
+
+	actual := strings.TrimSuffix(usernames[0].Name(), ".gpg")
+	secret, err := h.runPass("", "show", path.Join(PASS_FOLDER, encoded, actual))
+	return actual, secret, err
+}
+
+// List returns the stored URLs and corresponding usernames for a given credentials label
+func (h Pass) List() (map[string]string, error) {
+	servers, err := listPassDir()
+	if err != nil {
+		return nil, err
+	}
+
+	resp := map[string]string{}
+
+	for _, server := range servers {
+		if !server.IsDir() {
+			continue
+		}
+
+		serverURL, err := base64.URLEncoding.DecodeString(server.Name())
+		if err != nil {
+			return nil, err
+		}
+
+		usernames, err := listPassDir(server.Name())
+		if err != nil {
+			return nil, err
+		}
+
+		if len(usernames) < 1 {
+			return nil, fmt.Errorf("no usernames for %s", serverURL)
+		}
+
+		resp[string(serverURL)] = strings.TrimSuffix(usernames[0].Name(), ".gpg")
+	}
+
+	return resp, nil
+}

pass/pass_linux_test.go

@@ -0,0 +1,75 @@
+package pass
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/docker/docker-credential-helpers/credentials"
+)
+
+func TestPassHelper(t *testing.T) {
+	helper := Pass{}
+
+	creds := &credentials.Credentials{
+		ServerURL: "https://foobar.docker.io:2376/v1",
+		Username:  "nothing",
+		Secret:    "isthebestmeshuggahalbum",
+	}
+
+	helper.Add(creds)
+
+	creds.ServerURL = "https://foobar.docker.io:9999/v2"
+	helper.Add(creds)
+
+	credsList, err := helper.List()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for server, username := range credsList {
+		if !(strings.Contains(server, "2376") ||
+			strings.Contains(server, "9999")) {
+			t.Fatalf("invalid url: %s", creds.ServerURL)
+		}
+
+		if username != "nothing" {
+			t.Fatalf("invalid username: %v", username)
+		}
+
+		u, s, err := helper.Get(server)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if u != username {
+			t.Fatalf("invalid username %s", u)
+		}
+
+		if s != "isthebestmeshuggahalbum" {
+			t.Fatalf("invalid secret: %s", s)
+		}
+
+		err = helper.Delete(server)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		username, _, err = helper.Get(server)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if username != "" {
+			t.Fatalf("%s shouldn't exist any more", username)
+		}
+	}
+
+	credsList, err = helper.List()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(credsList) != 0 {
+		t.Fatal("didn't delete all creds?")
+	}
+}

wincred/wincred_windows.go

@@ -13,11 +13,12 @@ type Wincred struct{}
 
 // Add adds new credentials to the windows credentials manager.
 func (h Wincred) Add(creds *credentials.Credentials) error {
+	credsLabels := []byte(credentials.CredsLabel)
 	g := winc.NewGenericCredential(creds.ServerURL)
 	g.UserName = creds.Username
 	g.CredentialBlob = []byte(creds.Secret)
 	g.Persist = winc.PersistLocalMachine
-	g.Attributes = []winc.CredentialAttribute{{"label", []byte(credentials.CredsLabel)}}
+	g.Attributes = []winc.CredentialAttribute{{Keyword: "label", Value: credsLabels}}
 
 	return g.Write()
 }