Reorganize docker /Users/rfay/workspace/build-tools and try appveyor …

…for Docker-for-Windows testing (#61)

* Add appveyor config for Windows testing
* Reorganize/simplify build for Docker-ee compatibility (used in appveyor)

.appveyor.yml

@@ -2,49 +2,29 @@
 # Linux docker containers extensively, and appveyor is unable to use them.
 # So even building (not to mention testing) isn't possible.
 version: "{build}"
-image: Visual Studio 2017
+image: Windows Server 2019
 
-# Source Config
-clone_folder: c:\gopath\src\github.com\drud\build-tools
-
-# Build host
-
-environment:
-  GOPATH: c:\gopath
-  GOVERSION: 1.8.3
-  BASH: "C:/Program Files/git/bin/bash"
-
-init:
-  - git config --global core.autocrlf input
-
-# Build
 
 install:
-  - ps: Stop-Service docker
-  - ps: Remove-Item -Force -Recurse $env:ProgramFiles\docker
-  - ps: Invoke-WebRequest -Uri "https://download.docker.com/win/static/test/x86_64/docker-17.06.0-ce.zip" -OutFile "docker.zip"
-  - ps: Expand-Archive -Path "docker.zip" -DestinationPath $env:ProgramFiles -Force
-  - ps: Remove-Item docker.zip
-  - ps: Start-Service docker
-  - ps: Invoke-WebRequest "https://github.com/docker/compose/releases/download/1.14.0/docker-compose-Windows-x86_64.exe" -UseBasicParsing -OutFile $Env:ProgramFiles\docker\docker-compose.exe
-  - docker run -it -v C:/gopath:/junk busybox ls
-
-  - docker version
-  - docker-compose version
-  # Install the specific Go version.
-  - rmdir c:\go /s /q
-  - curl -fsS -o golang.msi https://storage.googleapis.com/golang/go%GOVERSION%.windows-amd64.msi
-  - msiexec /i golang.msi /q
-  - go version
-  - choco install -y git make
-  - make --version
+  - cinst make golang git
+  - refreshenv
+  - git config --global core.autocrlf false
 
 build_script:
-  - cd tests
-  - '"%BASH%" -c "make windows"'
-
+#  - docker run --rm -t  busybox ls /etc
+#  - docker run -t -v "C:\:/junk" busybox ls
+#  - docker run -t -v %cd%:/tmp/junk busybox ls /tmp/junk
+#  - docker run -t -v "%cd%:/tmp/junk" busybox ls /tmp/junk
+#  - ps: docker run --rm -it -v ${PWD}:/tmp/junk busybox ls /tmp/junk
+#  - where bash
+#  - bash --version
+#  - where make
+#  - make --version
+#  - go version
+  - docker version && docker-compose version
+  - cmd: where docker
+  - cmd: .autotests/test.cmd
+
+
+test: off
 deploy: false
-
-test_script:
-  - cd tests
-  - '"%BASH%" -c "make test"'

.buildkite/sanetestbot.sh → .autotests/sanetestbot.sh

@@ -9,10 +9,10 @@ set -o nounset
 
 
 # Check that required commands are available.
-for command in mysql git go make; do
+for command in git go make; do
     command -v $command >/dev/null || ( echo "Did not find command installed '$command'" && exit 2 )
 done
-docker run -t busybox ls
+docker run -t busybox ls >/dev/null
 
 if [ "$(go env GOOS)" = "windows"  -a "$(git config core.autocrlf)" != "false" ] ; then
  echo "git config core.autocrlf is not set to false on windows"

.buildkite/test.cmd → .autotests/test.cmd

@@ -1,5 +1,5 @@
-@echo "Building using bash and build.sh"
-"C:\Program Files\git\bin\bash" .buildkite/test.sh
+@echo "Building using bash and build.sh PWD=%cd%"
+"C:\Program Files\git\bin\bash" .autotests/test.sh
 
 if %ERRORLEVEL% EQU 0 (
    @echo Successful build

.autotests/test.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+# build drud/build-tools for automated testing (buildkite/appveyor/circleci)
+
+set -o errexit
+set -o pipefail
+set -o nounset
+#set -x
+
+# Make sure that everything remains readable. Go module cache is always getting
+# set to read-only, meanning it can't be cleaned up.
+function cleanup {
+	chmod -R u+w . || true
+}
+trap cleanup EXIT
+
+BUILD_OS=$(go env GOOS)
+echo "--- building at $(date) on $HOSTNAME for OS=$(go env GOOS) in $PWD"
+
+# Our testbot should now be sane, run the testbot checker to make sure.
+echo "--- Checking for sane testbot"
+./.autotests/sanetestbot.sh
+
+echo "--- make $BUILD_OS"
+cd tests
+rm -f windows darwin linux && time make
+echo "--- make test"
+time make test
+RV=$?
+echo "--- build.sh completed with status=$RV"
+exit $RV

.buildkite/macosdockerformac.yml

@@ -1,4 +1,4 @@
-  - command: ".buildkite/test.sh"
+  - command: ".autotests/test.sh"
     agents:
       - "os=macos"
       - "dockertype=dockerformac"

.buildkite/windows10dockerforwindows.yml

@@ -1,4 +1,4 @@
-  - command: ".buildkite/test.cmd"
+  - command: ".autotests/test.cmd"
     agents:
       - "os=windows"
       - "dockertype=dockerforwindows"

.buildkite/windows10dockertoolbox.yml

@@ -1,4 +1,4 @@
-  - command: ".buildkite/test.cmd"
+  - command: ".autotests/test.cmd"
     agents:
       - "os=windows"
       - "dockertype=toolbox"

.circleci/config.yml

@@ -4,15 +4,12 @@ stages:
     machine:
       image: circleci/classic:201711-01
 
-    environment:
-      GOPATH: /home/circleci/go
-
     steps:
       - run: sudo rm -rf /usr/local/go &&
-             wget -q -O /tmp/golang.tgz https://dl.google.com/go/go1.11.4.linux-amd64.tar.gz &&
+             wget -q -O /tmp/golang.tgz https://dl.google.com/go/go1.11.5.linux-amd64.tar.gz &&
              sudo tar -C /usr/local -xzf /tmp/golang.tgz
 
       - checkout
 
-      - run: cd tests && make test
+      - run: .autotests/test.sh
 

build_update.sh

@@ -40,7 +40,7 @@ tar -xf $local_file
 rm -rf build-tools/*
 cp -r $internal_name/ build-tools/
 rm -rf $internal_name/
-rm -rf build-tools/{tests,circle.yml,.circleci,.github,.appveyor.yml,.buildkite}
+rm -rf build-tools/{tests,circle.yml,.circleci,.github,.appveyor.yml,.buildkite,.autotests}
 touch build-tools/build-tools-VERSION-$tag.txt
 git add build-tools
 echo "Updated build-tools to $tag

makefile_components/base_build_go.mak

@@ -4,18 +4,21 @@
 ##### contents into ../Makefile and commenting out the include and adding a
 ##### comment about what you did and why.
 
+# The //workdir prevents docker and friends from trying to convert the thing to a non-unix path.
+# The $(S) in fromt of paths is a slash used only for Docker Toolbox, for the same reason.
 DOCKERBUILDCMD=docker run -t --rm -u $(shell id -u):$(shell id -g)                    \
-          	    -v "$(S)$$PWD/$(GOTMP):/go$(DOCKERMOUNTFLAG)"                                \
-          	    -v "$(S)$$PWD:/workdir$(DOCKERMOUNTFLAG)"                              \
+          	    -v "$(S)$(PWD):/workdir$(DOCKERMOUNTFLAG)"                              \
+          	    -v "$(S)$(PWD)/$(GOTMP)/bin:$(S)/go/bin" \
           	    -e CGO_ENABLED=0                  \
           	    -e GOOS=$@						  \
-          	    -w $(S)/workdir              \
+          	    -e GOPATH="//workdir/$(GOTMP)" \
+          	    -w //workdir              \
           	    $(BUILD_IMAGE)
 
 DOCKERTESTCMD=docker run -t --rm -u $(shell id -u):$(shell id -g)                    \
-          	    -v "$(S)$$PWD/$(GOTMP):/go$(DOCKERMOUNTFLAG)"                                \
-          	    -v "$(S)$$PWD:/workdir$(DOCKERMOUNTFLAG)"                              \
-          	    -w $(S)/workdir              \
+          	    -v "$(S)$(PWD):/workdir$(DOCKERMOUNTFLAG)"                              \
+          	    -e GOPATH="//workdir/$(GOTMP)" \
+          	    -w //workdir              \
           	    $(BUILD_IMAGE)
 
 .PHONY: all build test push clean container-clean bin-clean version static gofmt govet golint golangci-lint container
@@ -29,7 +32,7 @@ BUILD_OS = $(shell go env GOHOSTOS)
 
 BUILD_IMAGE ?= drud/golang-build-container:v1.11.4.2
 
-BUILD_BASE_DIR ?= $$PWD
+BUILD_BASE_DIR ?= $(PWD)
 
 # Expands SRC_DIRS into the common golang ./dir/... format for "all below"
 SRC_AND_UNDER = $(patsubst %,./%/...,$(SRC_DIRS))
@@ -39,7 +42,7 @@ GOMETALINTER_ARGS ?= --vendored-linters --disable-all --enable=gofmt --enable=ve
 GOLANGCI_LINT_ARGS ?= --out-format=line-number --disable-all --enable=gofmt --enable=govet --enable=golint --enable=errcheck --enable=staticcheck --enable=ineffassign --enable=varcheck --enable=deadcode
 
 COMMIT := $(shell git describe --tags --always --dirty)
-BUILDINFO = $(shell echo Built $$(date) $$(whoami)@$$(hostname) $(BUILD_IMAGE) )
+BUILDINFO = $(shell echo Built $$(date) $(BUILD_IMAGE) )
 
 VERSION_VARIABLES += VERSION COMMIT BUILDINFO
 
@@ -52,23 +55,33 @@ DOCKERMOUNTFLAG := :delegated
 # See https://github.com/golang/go/issues/27227
 USEMODVENDOR := $(shell if [ -d vendor ]; then echo "-mod=vendor"; fi)
 
-
-PWD=$(shell pwd)
-S =
 ifeq ($(BUILD_OS),windows)
     # On Windows docker toolbox, volume mounts oddly need a // at the beginning for things to work out, so
     # add that extra slash only on Windows.
+    DOCKERMOUNTFLAG=
+endif
+
+# On Docker Toolbox we can't use paths like C:\xxx\xxx, must use /C/xxx/xxx
+# However, on Docker-for-Windows/Appveyor we must use C:\xxx
+# On all other (macos/linux) $PWD will already be OK.
+ifneq ($(shell if [ "$(BUILD_OS)" = "windows" ] && [ -z "$(DOCKER_TOOLBOX_INSTALL_PATH)" ]; then echo true; fi),)
+	PWD=$(shell cmd //c "echo %cd%")
+endif
+ifneq ($(DOCKER_TOOLBOX_INSTALL_PATH),)
+	PWD=$(shell pwd)
     S=/
 endif
 
 build: $(BUILD_OS)
+pull:
+	@docker pull $(BUILD_IMAGE) >/dev/null 2>&1
 
-linux darwin windows: $(GOFILES)
+linux darwin windows: pull $(GOFILES)
 	@echo "building $@ from $(SRC_AND_UNDER)"
+	@echo $(shell if [ "$(BUILD_OS)" = "windows" ]; then echo "windows build: BUILD_OS=$(BUILD_OS)  DOCKER_TOOLBOX_INSTALL_PATH=$(DOCKER_TOOLBOX_INSTALL_PATH) PWD=$(PWD) S=$(S)"; fi )
 	@mkdir -p $(GOTMP)/{.cache,pkg,src,bin}
-	@$(DOCKERBUILDCMD) \
-        go install $(USEMODVENDOR) -installsuffix static -ldflags ' $(LDFLAGS) ' $(SRC_AND_UNDER)
-	@$(shell touch $@)
+	$(DOCKERBUILDCMD) \
+        go install -installsuffix static -ldflags ' $(LDFLAGS) ' $(SRC_AND_UNDER) && touch $@
 	$( shell if [ -d $(GOTMP) ]; then chmod -R u+w $(GOTMP); fi )
 	@echo $(VERSION) >VERSION.txt
 
@@ -79,7 +92,7 @@ gofmt:
 
 govet:
 	@echo "Checking go vet: "
-	@$(DOCKERTESTCMD) \
+	$(DOCKERTESTCMD) \
 		bash -c 'go vet $(SRC_AND_UNDER)'
 
 golint:

makefile_components/base_test_go.mak

@@ -9,16 +9,9 @@ TESTOS = $(BUILD_OS)
 test: build
 	@echo "Testing $(SRC_AND_UNDER) with TESTARGS=$(TESTARGS)"
 	@mkdir -p $(GOTMP)/{.cache,pkg,src,bin}
-	@docker run -t --rm  -u $(shell id -u):$(shell id -g)                 \
-	    -v $(PWD)/$(GOTMP):/go$(DOCKERMOUNTFLAG)                                                 \
-	    -v $(PWD):/go/src/$(PKG)$(DOCKERMOUNTFLAG)                                          \
-	    -v $(PWD)/bin/linux:/go/bin$(DOCKERMOUNTFLAG)                                     \
-	    -v $(PWD)/$(GOTMP)/std/linux:/usr/local/go/pkg/linux_amd64_static$(DOCKERMOUNTFLAG)  \
-	    -e CGO_ENABLED=0	\
-	    -w /go/src/$(PKG)                                                  \
-	    $(BUILD_IMAGE)                                                     \
+	@$(DOCKERTESTCMD) \
         go test $(USEMODVENDOR) -v -installsuffix static -ldflags '$(LDFLAGS)' $(SRC_AND_UNDER) $(TESTARGS)
-	@$(shell chmod -R u+w $(GOTMP))
+	$( shell if [ -d $(GOTMP) ]; then chmod -R u+w $(GOTMP); fi )
 
 # test_precompile allows a full compilation of _test.go files, without execution of the tests.
 # Setup and teardown in TestMain is still executed though, so this can cost some time.

tests/Makefile

@@ -13,14 +13,6 @@ UPSTREAM_REPO ?= golang:1.7.5-alpine3.5
 
 export WORKING_DIR = $(shell pwd)
 
-TESTDOCKERCMD=docker run -t --rm -u $(shell id -u):$(shell id -g)                    \
-          	    -v "$(S)$$PWD/$(GOTMP):/go$(DOCKERMOUNTFLAG)"                                \
-          	    -v "$(S)$$PWD:/workdir$(DOCKERMOUNTFLAG)"                              \
-          	    -v "$(S)$$PWD/$(GOTMP)/std/$@:/usr/local/go/pkg/$@_amd64_static$(DOCKERMOUNTFLAG)"  \
-          	    -w $(S)/workdir              \
-          	    $(BUILD_IMAGE)
-
-
 # Top-level directories to build - Only build the explicit complex fail stuff by default. Unit test will be overridden.
 SRC_DIRS := cmd pkg
 
@@ -76,6 +68,6 @@ test_precompile: test
 # COMMAND := govendor list
 container_cmd:
 	#@$(shell if [ -f ./go.mod ] ; then GOPATH=$$PWD/$(GOTMP) go get ./...;  fi; )
-	@$(TESTDOCKERCMD) \
+	$(DOCKERTESTCMD) \
 		$(COMMAND)