doc: getting-started

tested: ssh-docker

.devcontainer/devcontainer.json

@@ -1,10 +1,25 @@
 {
   "name": "Golang Dev",
   "image": "golang-dev:1.21-bookworm-user",
+  "mounts": [
+    {
+      "source": "WingmateGoPath",
+      "target": "/go",
+      "type": "volume"
+    },
+    {
+      "source": "WingmateGolangDevHome",
+      "target": "/home/golang",
+      "type": "volume"
+    }
+  ],
   "customizations": {
     "vscode": {
       "extensions": [
-        "golang.go"
+        "golang.go",
+        "ms-azuretools.vscode-docker",
+        "ms-vscode.makefile-tools",
+        "ms-vscode.cpptools-extension-pack"
       ]
     }
   }

.gitignore

@@ -1,2 +1,3 @@
-/.idea
+/cmd/wingmate/wingmate
-/wingmate
+/cmd/pidproxy/pidproxy
+/cmd/exec/exec

.idea/.gitignore

@@ -0,0 +1,8 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Editor-based HTTP Client requests
+/httpRequests/
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/wingmate.iml" filepath="$PROJECT_DIR$/.idea/wingmate.iml" />
+    </modules>
+  </component>
+</project>

.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="" vcs="Git" />
+  </component>
+</project>

.idea/wingmate.iml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="WEB_MODULE" version="4">
+  <component name="Go" enabled="true" />
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

.tool-versions

@@ -1 +1 @@
-golang 1.21.4
+golang 1.21.5

Makefile

@@ -0,0 +1,54 @@
+DESTDIR = /usr/local/bin
+
+installs = install-dir
+programs = wingmate pidproxy exec
+ifdef TEST_BUILD
+	programs += oneshot spawner starter dummy
+	installs += install-test
+endif
+
+all: ${programs}
+
+wingmate:
+	$(MAKE) -C cmd/wingmate all
+
+pidproxy:
+	$(MAKE) -C cmd/pidproxy all
+
+exec:
+	$(MAKE) -C cmd/exec all
+
+dummy:
+	$(MAKE) -C cmd/experiment/dummy all
+
+oneshot:
+	$(MAKE) -C cmd/experiment/oneshot all
+
+spawner:
+	$(MAKE) -C cmd/experiment/spawner all
+
+starter:
+	$(MAKE) -C cmd/experiment/starter all
+
+clean:
+	$(MAKE) -C cmd/wingmate clean
+	$(MAKE) -C cmd/pidproxy clean
+	$(MAKE) -C cmd/exec clean
+	$(MAKE) -C cmd/experiment/dummy clean
+	$(MAKE) -C cmd/experiment/oneshot clean
+	$(MAKE) -C cmd/experiment/spawner clean
+	$(MAKE) -C cmd/experiment/starter clean
+
+install: ${installs}
+	$(MAKE) -C cmd/wingmate DESTDIR=${DESTDIR} install
+	$(MAKE) -C cmd/pidproxy DESTDIR=${DESTDIR} install
+	$(MAKE) -C cmd/exec DESTDIR=${DESTDIR} install
+
+install-test:
+	$(MAKE) -C cmd/experiment/dummy DESTDIR=${DESTDIR} install
+	$(MAKE) -C cmd/experiment/oneshot DESTDIR=${DESTDIR} install
+	$(MAKE) -C cmd/experiment/spawner DESTDIR=${DESTDIR} install
+	$(MAKE) -C cmd/experiment/starter DESTDIR=${DESTDIR} install
+
+install-dir:
+	install -d ${DESTDIR}

README.md

@@ -1,2 +1,139 @@
 # wingmate
 
+Wingmate is a process manager for services. It works like init. It starts and restarts services.
+It also has cron feature. It is designed to run in a container/docker. 
+The Wingmate binary do not need any external dependency. 
+Just copy the binary, and exec from the entry point script.
+
+# Getting Started
+
+## Binaries
+
+There are three binaries in this project: `wingmate`, `wmpidproxy`, and `wmexec`.
+
+`wingmate` is the core binary. It reads config, starts, restarts services. It also
+runs cron. Read the [configuration](#configuration) section for files needed to run
+`wingmate`.
+
+`wmpidproxy` is a helper binary for monitoring _legacy style_ service (fork, exit
+initial proces, and continue in background). Read [here](#wingmate-pid-proxy-binary)
+for further details about `wmpidproxy`.
+
+`wmexec` is a helper binary for running process in different `user` or `group`.
+It also useful for setting the process as process group leader.
+Read [here](#wingmate-exec-binary) for further details about `wmexec`.
+
+## Building a container image based on wingmate image in Docker Hub
+
+Wingmate has no dependency other than `alpine` base image, so you just need to copy
+the binaries directly. If you have built your application into an `alpine` based image,
+all you need to do is copy whichever binary you need, crontab file (if you use cron)
+and add some shell script to glue them together. Here is a Dockerfile example.
+
+```Dockerfile
+# Dockerfile
+FROM suyono/wingmate:alpine as source
+
+FROM alpine:latest
+ADD --chmod=755 wingmate/ /etc/wingmate/
+ADD --chmod=755 entry.sh /usr/local/bin/entry.sh
+COPY --from=source /usr/local/bin/wingmate /usr/local/bin/wingmate
+COPY --from=source /usr/local/bin/wmpidproxy /usr/local/bin/wmpidproxy
+COPY --from=source /usr/local/bin/wmexec /usr/local/bin/wmexec
+ENTRYPOINT [ "/usr/local/bin/entry.sh" ]
+CMD [ "/usr/local/bin/wingmate" ]
+```
+You can find some examples for shell script in [alpine docker](docker/alpine) and 
+[bookworm docker](docker/bookworm).
+
+## Configuration
+
+When `wingmate` binary starts, it will look for some files. By default, it will
+try to read the content of `/etc/wingmate` directory. You can change the directory
+where it reads by setting `WINGMATE_CONFIG_PATH` environment variable. The structure
+inside the config path should look like this.
+
+```shell
+/etc
+ └── wingmate
+     ├── crontab
+     ├── crontab.d
+     │   ├── cron1.sh
+     │   ├── cron2.sh
+     │   └── cron3.sh
+     └── service
+         ├── one.sh
+         └── spawner.sh
+```
+
+First, `wingmate` will try to read the content of `service` directory. The content of
+this directory should be executables (either shell scripts or binaries). The wingmate
+will run every executable in `service` directory without going into any subdirectory.
+
+Next, `wingmate` will read the `crontab` file. `wingmate` expects the `crontab` file using
+common UNIX crontab file format. Something like this
+
+```shell
+ ┌───────────── minute (0–59)
+ │ ┌───────────── hour (0–23)
+ │ │ ┌───────────── day of the month (1–31)
+ │ │ │ ┌───────────── month (1–12)
+ │ │ │ │ ┌───────────── day of the week (0–6) (Sunday to Saturday)
+ │ │ │ │ │ 
+ │ │ │ │ │
+ │ │ │ │ │
+ * * * * * <commad or shell script or binary>
+```
+
+The command part only support simple command and arguments. Shell expression is not supported
+yet. It is recommended to write a shell script and put the path to shell script in
+the command part.
+
+# Appendix
+## Wingmate PID Proxy binary
+
+`wingmate` works by monitoring its direct children process. When it sees one of its children
+exited, it will start the child process again.
+
+Sometimes you find some services work by running in the background. It means it forks a new 
+process, disconnect the new child from terminal, exit the parent process, and continue 
+running in the child process. This kind of service usually write its background process
+PID in a pid file.
+
+To monitor the background services, `wingmate` utilizes `wmpidproxy`. `wmpidproxy` runs
+in foreground in-place of the background service. It also periodically check whether the
+background service is still running, in current implementation it checks every second.
+
+```shell
+wmpidproxy --pid-file <path to pid file> -- <background service binary/start script>
+```
+#### Example
+Running sshd background with `wingmate` and `wmpidproxy`: [here](example/ssh-docker)
+
+#### Note
+
+It is recommended to configure services to run in foreground if they support it. When services
+running in foreground, they are running as direct children process of `wingmate`.
+`wingmate` monitors children process effectively. Whenever a child process exited/terminated,
+`wingmate` will start it again quickly. Running in foreground also removes the overhead of running
+`wmpidproxy` together with the service.
+
+## Wingmate Exec binary
+
+`wingmate` runs all the services as its children using the same `uid`, `gid`, and in the
+same process group. You can use `wmexec` to run service in different `uid`, `gid`, or make
+the service process as its own process group leader.
+
+#### Syntax
+
+```shell
+wmexec [--user <uid>[:<gid>]] [--setsid] -- <target executable>
+```
+| Option   | Parameter | Description                                                                                              |
+|----------|-----------|----------------------------------------------------------------------------------------------------------|
+| --user   | uid[:gid] | Set the real user ID and the real group id. Uid and Gid can be either in numeric form or in name form    |
+| --setsid |           | Set the process become the leader of its own process group, effectively detaching from parent's terminal |
+
+#### Example
+
+You can find example for `wmexec` in [here](docker/alpine/etc/wingmate) and [here](docker/bookworm/etc/wingmate)

cmd/exec/Makefile

@@ -0,0 +1,8 @@
+all:
+	go build -v
+
+clean:
+	go clean -i -cache -testcache
+
+install:
+	install exec ${DESTDIR}/wmexec

cmd/exec/ent.go

@@ -0,0 +1,11 @@
+//go:build !(cgo && linux)
+
+package main
+
+func getUid(user string) (uint64, error) {
+	panic("not implemented")
+}
+
+func getGid(group string) (uint64, error) {
+	panic("not implemented")
+}

cmd/exec/ent_lin.go

@@ -0,0 +1,51 @@
+//go:build cgo && linux
+
+package main
+
+/*
+#include<errno.h>
+#include<string.h>
+#include<sys/types.h>
+#include<pwd.h>
+#include<grp.h>
+
+static uid_t getuid(const char* username) {
+	struct passwd local, *rv;
+	errno = 0;
+	rv = getpwnam(username);
+	if (errno != 0) {
+		return 0;
+	}
+
+	memcpy(&local, rv, sizeof(struct passwd));
+	return local.pw_uid;
+}
+static gid_t getgid(const char* groupname) {
+	struct group local, *rv;
+	errno = 0;
+	rv = getgrnam(groupname);
+	if (errno != 0) {
+		return 0;
+	}
+
+	memcpy(&local, rv, sizeof(struct group));
+	return local.gr_gid;
+}
+*/
+import "C"
+
+func getUid(user string) (uint64, error) {
+	u, err := C.getuid(C.CString(user))
+	if err != nil {
+		return 0, err
+	}
+	return uint64(u), nil
+}
+
+func getGid(group string) (uint64, error) {
+	g, err := C.getgid(C.CString(group))
+	if err != nil {
+		return 0, err
+	}
+	return uint64(g), nil
+}

cmd/exec/exec.go

@@ -0,0 +1,133 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"strconv"
+	"strings"
+
+	"gitea.suyono.dev/suyono/wingmate"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	"golang.org/x/sys/unix"
+)
+
+const (
+	setsidFlag = "setsid"
+	EnvSetsid  = "SETSID"
+	userFlag   = "user"
+	EnvUser    = "USER"
+)
+
+var (
+	rootCmd = &cobra.Command{
+		Use:  "wmexec",
+		RunE: execCmd,
+	}
+
+	childArgs []string
+)
+
+func main() {
+	var (
+		found    bool
+		i        int
+		arg      string
+		selfArgs []string
+	)
+
+	rootCmd.PersistentFlags().BoolP(setsidFlag, "s", false, "set to true to run setsid() before exec")
+	viper.BindPFlag(EnvSetsid, rootCmd.PersistentFlags().Lookup(setsidFlag))
+
+	rootCmd.PersistentFlags().StringP(userFlag, "u", "", "\"user:[group]\"")
+	viper.BindPFlag(EnvUser, rootCmd.PersistentFlags().Lookup(userFlag))
+
+	viper.SetEnvPrefix(wingmate.EnvPrefix)
+	viper.BindEnv(EnvUser)
+	viper.BindEnv(EnvSetsid)
+	viper.SetDefault(EnvSetsid, false)
+	viper.SetDefault(EnvUser, "")
+
+	found = false
+	for i, arg = range os.Args {
+		if arg == "--" {
+			found = true
+			if len(os.Args) <= i+1 {
+				log.Println("invalid argument")
+				os.Exit(1)
+			}
+			selfArgs = os.Args[1:i]
+			childArgs = os.Args[i+1:]
+			break
+		}
+	}
+	if !found {
+		log.Println("invalid argument")
+		os.Exit(1)
+	}
+
+	if len(childArgs) == 0 {
+		log.Println("invalid argument")
+		os.Exit(1)
+	}
+
+	rootCmd.SetArgs(selfArgs)
+	if err := rootCmd.Execute(); err != nil {
+		log.Println(err)
+		os.Exit(1)
+	}
+}
+
+func execCmd(cmd *cobra.Command, args []string) error {
+	if viper.GetBool(EnvSetsid) {
+		_, _ = unix.Setsid()
+	}
+
+	var (
+		uid  uint64
+		gid  uint64
+		err  error
+		path string
+	)
+
+	ug := viper.GetString(EnvUser)
+	if len(ug) > 0 {
+		user, group, ok := strings.Cut(ug, ":")
+		if ok {
+			if gid, err = strconv.ParseUint(group, 10, 32); err != nil {
+				if gid, err = getGid(group); err != nil {
+					return fmt.Errorf("cgo getgid: %w", err)
+				}
+			}
+			if err = unix.Setgid(int(gid)); err != nil {
+				return fmt.Errorf("setgid: %w", err)
+			}
+		}
+
+		uid, err = strconv.ParseUint(user, 10, 32)
+		if err != nil {
+			if uid, err = getUid(user); err != nil {
+				return fmt.Errorf("cgo getuid: %w", err)
+			}
+		}
+		if err = unix.Setuid(int(uid)); err != nil {
+			return fmt.Errorf("setuid: %w", err)
+		}
+
+	}
+
+	if path, err = exec.LookPath(childArgs[0]); err != nil {
+		if !errors.Is(err, exec.ErrDot) {
+			return fmt.Errorf("lookpath: %w", err)
+		}
+	}
+
+	if err = unix.Exec(path, childArgs, os.Environ()); err != nil {
+		return fmt.Errorf("exec: %w", err)
+	}
+
+	return nil
+}

cmd/experiment/.gitignore

@@ -0,0 +1,4 @@
+/dummy/dummy
+/starter/starter
+/oneshot/oneshot
+/spawner/spawner

cmd/experiment/dummy/Makefile

@@ -0,0 +1,8 @@
+all:
+	go build -v
+
+clean:
+	go clean -i -cache -testcache
+
+install:
+	install dummy ${DESTDIR}/wmdummy

cmd/experiment/dummy/dummy.go

@@ -0,0 +1,17 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"time"
+)
+
+func main() {
+	log.Println("using log.Println")
+	fmt.Println("using fmt.Println")
+
+	time.Sleep(time.Second * 5)
+
+	log.Println("log: finishing up")
+	fmt.Println("fmt: finishing up")
+}

cmd/experiment/oneshot/Makefile

@@ -0,0 +1,8 @@
+all:
+	go build -v
+
+clean:
+	go clean -i -cache -testcache
+
+install:
+	install oneshot ${DESTDIR}/wmoneshot

cmd/experiment/oneshot/oneshot.go

@@ -0,0 +1,78 @@
+package main
+
+import (
+	"log"
+	"math/rand"
+	"os"
+	"os/exec"
+
+	"gitea.suyono.dev/suyono/wingmate"
+	"github.com/spf13/viper"
+)
+
+const (
+	// DummyPath = "/workspaces/wingmate/cmd/experiment/dummy/dummy"
+	DummyPath            = "/usr/local/bin/wmdummy"
+	EnvDummyPath         = "DUMMY_PATH"
+	EnvLog               = "LOG"
+	EnvLogMessage        = "LOG_MESSAGE"
+	EnvDefaultLogMessage = "oneshot executed"
+	EnvInstanceNum       = "INSTANCE_NUM"
+	EnvDefaultInstances  = -1
+)
+
+func main() {
+	viper.SetEnvPrefix(wingmate.EnvPrefix)
+	viper.BindEnv(EnvDummyPath)
+	viper.BindEnv(EnvLog)
+	viper.BindEnv(EnvLogMessage)
+	viper.BindEnv(EnvInstanceNum)
+	viper.SetDefault(EnvDummyPath, DummyPath)
+	viper.SetDefault(EnvLogMessage, EnvDefaultLogMessage)
+	viper.SetDefault(EnvInstanceNum, EnvDefaultInstances)
+
+	exePath := viper.GetString(EnvDummyPath)
+
+	logPath := viper.GetString(EnvLog)
+	logMessage := viper.GetString(EnvLogMessage)
+	log.Println("log path:", logPath)
+	if logPath != "" {
+		var (
+			err  error
+			file *os.File
+		)
+
+		if file, err = os.OpenFile(logPath, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0o666); err == nil {
+			defer func() {
+				_ = file.Close()
+			}()
+
+			if err = wingmate.NewLog(file); err == nil {
+				wingmate.Log().Info().Msg(logMessage)
+			}
+		}
+	}
+
+	StartInstances(exePath)
+}
+
+func StartInstances(exePath string) {
+	num := (rand.Uint32() % 16) + 16
+
+	iNum := viper.GetInt(EnvInstanceNum)
+	if iNum > 0 {
+		num = uint32(iNum)
+	}
+
+	var (
+		ctr uint32
+		cmd *exec.Cmd
+		err error
+	)
+	for ctr = 0; ctr < num; ctr++ {
+		cmd = exec.Command(exePath)
+		if err = cmd.Start(); err != nil {
+			log.Printf("failed to run %s: %+v\n", exePath, err)
+		}
+	}
+}

cmd/experiment/spawner/Makefile

@@ -0,0 +1,8 @@
+all:
+	go build -v
+
+clean:
+	go clean -i -cache -testcache
+
+install:
+	install spawner ${DESTDIR}/wmspawner

cmd/experiment/spawner/spawner.go

@@ -0,0 +1,40 @@
+package main
+
+import (
+	"log"
+	"os/exec"
+	"time"
+
+	"gitea.suyono.dev/suyono/wingmate"
+	"github.com/spf13/viper"
+)
+
+const (
+	EnvOneShotPath = "ONESHOT_PATH"
+	OneShotPath    = "/usr/local/bin/wmoneshot"
+)
+
+func main() {
+	var (
+		cmd *exec.Cmd
+		err error
+		t   *time.Ticker
+	)
+	viper.SetEnvPrefix(wingmate.EnvPrefix)
+	viper.BindEnv(EnvOneShotPath)
+	viper.SetDefault(EnvOneShotPath, OneShotPath)
+
+	exePath := viper.GetString(EnvOneShotPath)
+
+	t = time.NewTicker(time.Second * 5)
+	for {
+		cmd = exec.Command(exePath)
+		if err = cmd.Run(); err != nil {
+			log.Printf("failed to run %s: %+v\n", exePath, err)
+		} else {
+			log.Printf("%s executed\n", exePath)
+		}
+
+		<-t.C
+	}
+}

cmd/experiment/starter/Makefile

@@ -0,0 +1,8 @@
+all:
+	go build -v
+
+clean:
+	go clean -i -cache -testcache
+
+install:
+	install starter ${DESTDIR}/wmstarter

cmd/experiment/starter/starter.go

@@ -0,0 +1,73 @@
+package main
+
+import (
+	"bufio"
+	"io"
+	"log"
+	"os/exec"
+	"sync"
+
+	"gitea.suyono.dev/suyono/wingmate"
+	"github.com/spf13/viper"
+)
+
+const (
+	// DummyPath = "/workspaces/wingmate/cmd/experiment/dummy/dummy"
+	DummyPath    = "/usr/local/bin/wmdummy"
+	EnvDummyPath = "DUMMY_PATH"
+)
+
+func main() {
+	var (
+		stdout  io.ReadCloser
+		stderr  io.ReadCloser
+		wg      *sync.WaitGroup
+		err     error
+		exePath string
+	)
+	viper.SetEnvPrefix(wingmate.EnvPrefix)
+	viper.BindEnv(EnvDummyPath)
+	viper.SetDefault(EnvDummyPath, DummyPath)
+
+	exePath = viper.GetString(EnvDummyPath)
+
+	cmd := exec.Command(exePath)
+
+	if stdout, err = cmd.StdoutPipe(); err != nil {
+		log.Panic(err)
+	}
+
+	if stderr, err = cmd.StderrPipe(); err != nil {
+		log.Panic(err)
+	}
+
+	wg = &sync.WaitGroup{}
+	wg.Add(2)
+	go pulley(wg, stdout, "stdout")
+	go pulley(wg, stderr, "stderr")
+
+	if err = cmd.Start(); err != nil {
+		log.Panic(err)
+	}
+	wg.Wait()
+
+	if err = cmd.Wait(); err != nil {
+		log.Printf("got error when Waiting for child process: %#v\n", err)
+	}
+}
+
+func pulley(wg *sync.WaitGroup, src io.ReadCloser, srcName string) {
+	defer wg.Done()
+
+	scanner := bufio.NewScanner(src)
+	for scanner.Scan() {
+		log.Printf("coming out from %s: %s\n", srcName, scanner.Text())
+	}
+
+	if err := scanner.Err(); err != nil {
+		log.Printf("got error whean reading from %s: %#v\n", srcName, err)
+	}
+
+	log.Printf("closing %s...\n", srcName)
+	_ = src.Close()
+}

cmd/pidproxy/Makefile

@@ -0,0 +1,8 @@
+all:
+	go build -v
+
+clean:
+	go clean -i -cache -testcache
+
+install:
+	install pidproxy ${DESTDIR}/wmpidproxy

cmd/pidproxy/pidproxy.go

@@ -0,0 +1,161 @@
+package main
+
+import (
+	"bufio"
+	"errors"
+	"log"
+	"os"
+	"os/exec"
+	"os/signal"
+	"strconv"
+	"syscall"
+	"time"
+
+	"gitea.suyono.dev/suyono/wingmate"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	"golang.org/x/sys/unix"
+)
+
+const (
+	pidFileFlag         = "pid-file"
+	EnvStartSecs        = "STARTSECS"
+	EnvDefaultStartSecs = 1
+)
+
+var (
+	rootCmd = &cobra.Command{
+		Use:  "wmpidproxy",
+		RunE: pidProxy,
+	}
+
+	childArgs []string
+)
+
+func main() {
+	var (
+		i        int
+		arg      string
+		selfArgs []string
+		found    bool
+	)
+
+	viper.SetEnvPrefix(wingmate.EnvPrefix)
+	viper.BindEnv(EnvStartSecs)
+	viper.SetDefault(EnvStartSecs, EnvDefaultStartSecs)
+
+	rootCmd.PersistentFlags().StringP(pidFileFlag, "p", "", "location of pid file")
+	rootCmd.MarkFlagRequired(pidFileFlag)
+	viper.BindPFlag(pidFileFlag, rootCmd.PersistentFlags().Lookup(pidFileFlag))
+
+	found = false
+	for i, arg = range os.Args {
+		if arg == "--" {
+			found = true
+			if len(os.Args) <= i+1 {
+				log.Println("invalid argument")
+				os.Exit(1)
+			}
+			selfArgs = os.Args[1:i]
+			childArgs = os.Args[i+1:]
+			break
+		}
+	}
+	if !found {
+		log.Println("invalid argument")
+		os.Exit(1)
+	}
+
+	if len(childArgs) == 0 {
+		log.Println("invalid argument")
+		os.Exit(1)
+	}
+
+	rootCmd.SetArgs(selfArgs)
+
+	if err := rootCmd.Execute(); err != nil {
+		log.Println(err)
+		os.Exit(1)
+	}
+}
+
+func pidProxy(cmd *cobra.Command, args []string) error {
+	pidfile := viper.GetString(pidFileFlag)
+	log.Printf("%s %v", pidfile, childArgs)
+	if len(childArgs) > 1 {
+		go startProcess(childArgs[0], childArgs[1:]...)
+	} else {
+		go startProcess(childArgs[0])
+	}
+	initialWait := viper.GetInt(EnvStartSecs)
+	time.Sleep(time.Second * time.Duration(initialWait))
+
+	var (
+		err error
+		pid int
+		sc  chan os.Signal
+		t   *time.Ticker
+	)
+
+	sc = make(chan os.Signal, 1)
+	signal.Notify(sc, unix.SIGTERM)
+
+	t = time.NewTicker(time.Second)
+
+check:
+	for {
+		if pid, err = readPid(pidfile); err != nil {
+			return err
+		}
+
+		if err = unix.Kill(pid, syscall.Signal(0)); err != nil {
+			return err
+		}
+
+		select {
+		case <-t.C:
+		case <-sc:
+			if pid, err = readPid(pidfile); err != nil {
+				return err
+			}
+
+			if err = unix.Kill(pid, unix.SIGTERM); err != nil {
+				return err
+			}
+			break check
+		}
+	}
+	return nil
+}
+
+func readPid(pidFile string) (int, error) {
+	var (
+		file  *os.File
+		err   error
+		pid64 int64
+	)
+
+	if file, err = os.Open(pidFile); err != nil {
+		return 0, err
+	}
+	defer func() {
+		_ = file.Close()
+	}()
+
+	scanner := bufio.NewScanner(file)
+	if scanner.Scan() {
+		if pid64, err = strconv.ParseInt(scanner.Text(), 10, 64); err != nil {
+			return 0, err
+		}
+		return int(pid64), nil
+	} else {
+		return 0, errors.New("invalid scanner")
+	}
+}
+
+func startProcess(arg0 string, args ...string) {
+	if err := exec.Command(arg0, args...).Run(); err != nil {
+		log.Println("exec:", err)
+		return
+	}
+}

cmd/wingmate/Makefile

@@ -0,0 +1,9 @@
+all:
+	go build -v
+
+clean:
+	go clean -i -cache -testcache
+
+install:
+	install wingmate ${DESTDIR}/wingmate
+	

config/config.go

@@ -7,6 +7,7 @@ import (
 
 	"gitea.suyono.dev/suyono/wingmate"
 	"github.com/spf13/viper"
+	"golang.org/x/sys/unix"
 )
 
 const (
@@ -48,13 +49,16 @@ func Read() (*Config, error) {
 	if len(dirent) > 0 {
 		for _, d := range dirent {
 			if d.Type().IsRegular() {
+				svcPath := filepath.Join(svcdir, d.Name())
+				if err = unix.Access(svcPath, unix.X_OK); err == nil {
 					serviceAvailable = true
-				outConfig.ServicePaths = append(outConfig.ServicePaths, filepath.Join(svcdir, d.Name()))
+					outConfig.ServicePaths = append(outConfig.ServicePaths, svcPath)
+				}
 			}
 		}
 	}
 	if err != nil {
-		wingmate.Log().Error().Msgf("encounter error when reading service directory %s: %#v", svcdir, err)
+		wingmate.Log().Error().Msgf("encounter error when reading service directory %s: %+v", svcdir, err)
 	}
 
 	crontabfile = filepath.Join(configPath, CrontabFileName)
@@ -64,7 +68,7 @@ func Read() (*Config, error) {
 		cronAvailable = true
 	}
 	if err != nil {
-		wingmate.Log().Error().Msgf("encounter error when reading crontab %s: %#v", crontabfile, err)
+		wingmate.Log().Error().Msgf("encounter error when reading crontab %s: %+v", crontabfile, err)
 	}
 
 	if !serviceAvailable && !cronAvailable {

config/config_test.go

@@ -0,0 +1,130 @@
+package config
+
+import (
+	"os"
+	"path"
+	"testing"
+
+	"gitea.suyono.dev/suyono/wingmate"
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRead(t *testing.T) {
+
+	type testEntry struct {
+		name     string
+		testFunc func(t *testing.T)
+	}
+
+	var (
+		configDir string
+		err       error
+	)
+
+	const serviceDir = "service"
+
+	setup := func(t *testing.T) {
+		if configDir, err = os.MkdirTemp("", "wingmate-*-test"); err != nil {
+			t.Fatal("setup", err)
+		}
+		viper.Set(EnvConfigPath, configDir)
+	}
+
+	tear := func(t *testing.T) {
+		if err = os.RemoveAll(configDir); err != nil {
+			t.Fatal("tear", err)
+		}
+	}
+
+	mkSvcDir := func(t *testing.T) {
+		if err := os.MkdirAll(path.Join(configDir, serviceDir), 0755); err != nil {
+			t.Fatal("create dir", err)
+		}
+	}
+
+	touchFile := func(t *testing.T, name string, perm os.FileMode) {
+		f, err := os.OpenFile(name, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, perm)
+		if err != nil {
+			t.Fatal("create file", err)
+		}
+		_ = f.Close()
+	}
+
+	_ = wingmate.NewLog(os.Stderr)
+	tests := []testEntry{
+		{
+			name: "positive",
+			testFunc: func(t *testing.T) {
+				mkSvcDir(t)
+				touchFile(t, path.Join(configDir, serviceDir, "one.sh"), 0755)
+				touchFile(t, path.Join(configDir, serviceDir, "two.sh"), 0755)
+
+				cfg, err := Read()
+				assert.Nil(t, err)
+				assert.ElementsMatch(
+					t,
+					cfg.ServicePaths,
+					[]string{
+						path.Join(configDir, serviceDir, "one.sh"),
+						path.Join(configDir, serviceDir, "two.sh"),
+					},
+				)
+			},
+		},
+		{
+			name: "with directory",
+			testFunc: func(t *testing.T) {
+				const subdir1 = "subdir1"
+				mkSvcDir(t)
+				assert.Nil(t, os.Mkdir(path.Join(configDir, serviceDir, subdir1), 0755))
+				touchFile(t, path.Join(configDir, serviceDir, subdir1, "one.sh"), 0755)
+				touchFile(t, path.Join(configDir, serviceDir, "two.sh"), 0755)
+				cfg, err := Read()
+				assert.Nil(t, err)
+				assert.ElementsMatch(
+					t,
+					cfg.ServicePaths,
+					[]string{
+						path.Join(configDir, serviceDir, "two.sh"),
+					},
+				)
+			},
+		},
+		{
+			name: "wrong mode",
+			testFunc: func(t *testing.T) {
+				mkSvcDir(t)
+				touchFile(t, path.Join(configDir, serviceDir, "one.sh"), 0755)
+				touchFile(t, path.Join(configDir, serviceDir, "two.sh"), 0644)
+
+				cfg, err := Read()
+				assert.Nil(t, err)
+				assert.ElementsMatch(
+					t,
+					cfg.ServicePaths,
+					[]string{
+						path.Join(configDir, serviceDir, "one.sh"),
+					},
+				)
+			},
+		},
+		{
+			name: "empty",
+			testFunc: func(t *testing.T) {
+				mkSvcDir(t)
+
+				_, err := Read()
+				assert.NotNil(t, err)
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			setup(t)
+			tt.testFunc(t)
+			tear(t)
+		})
+	}
+}

config/crontab.go

@@ -28,14 +28,6 @@ type CronTimeSpec interface {
 	Match(uint8) bool
 }
 
-// type Cron interface {
-// 	Minute() CronTimeSpec
-// 	Hour() CronTimeSpec
-// 	DayOfMonth() CronTimeSpec
-// 	Month() CronTimeSpec
-// 	DayOfWeek() CronTimeSpec
-// }
-
 type Cron struct {
 	minute  CronTimeSpec
 	hour    CronTimeSpec
@@ -97,27 +89,27 @@ func readCrontab(path string) ([]*Cron, error) {
 			hasRun: false,
 		}
 		if err = c.setField(minute, parts[1]); err != nil {
-			wingmate.Log().Error().Msgf("error parsing minute field %#v", err)
+			wingmate.Log().Error().Msgf("error parsing minute field %+v", err)
 			continue
 		}
 
 		if err = c.setField(hour, parts[2]); err != nil {
-			wingmate.Log().Error().Msgf("error parsing hour field %#v", err)
+			wingmate.Log().Error().Msgf("error parsing hour field %+v", err)
 			continue
 		}
 
 		if err = c.setField(dom, parts[3]); err != nil {
-			wingmate.Log().Error().Msgf("error parsing day of month field %#v", err)
+			wingmate.Log().Error().Msgf("error parsing day of month field %+v", err)
 			continue
 		}
 
 		if err = c.setField(month, parts[4]); err != nil {
-			wingmate.Log().Error().Msgf("error parsing month field %#v", err)
+			wingmate.Log().Error().Msgf("error parsing month field %+v", err)
 			continue
 		}
 
 		if err = c.setField(dow, parts[5]); err != nil {
-			wingmate.Log().Error().Msgf("error parsing day of week field %#v", err)
+			wingmate.Log().Error().Msgf("error parsing day of week field %+v", err)
 			continue
 		}
 
@@ -134,23 +126,26 @@ func (c *Cron) Command() string {
 }
 
 func (c *Cron) TimeToRun(now time.Time) bool {
-	if !c.hasRun {
-		c.lastRun = now
-		c.hasRun = true
-		return true
-	}
-
-	if now.Sub(c.lastRun) <= time.Minute && now.Minute() == c.lastRun.Minute() {
-		return false
-	}
-
 	if c.minute.Match(uint8(now.Minute())) &&
 		c.hour.Match(uint8(now.Hour())) &&
 		c.dom.Match(uint8(now.Day())) &&
 		c.month.Match(uint8(now.Month())) &&
 		c.dow.Match(uint8(now.Weekday())) {
+
+		if c.hasRun {
+			if now.Sub(c.lastRun) <= time.Minute && now.Minute() == c.lastRun.Minute() {
+				return false
+			} else {
+				c.lastRun = now
 				return true
 			}
+		} else {
+
+			c.lastRun = now
+			c.hasRun = true
+			return true
+		}
+	}
 
 	return false
 }
@@ -208,12 +203,12 @@ func (c *Cron) setField(field cronField, input string) error {
 		*cField = &specAny{}
 	} else if strings.HasPrefix(input, "*/") {
 		if parsed64, err = strconv.ParseUint(input[2:], 10, 8); err != nil {
-			return fmt.Errorf("error parse field %#v with input %s: %w", field, input, err)
+			return fmt.Errorf("error parse field %+v with input %s: %w", field, input, err)
 		}
 
 		parsed = uint8(parsed64)
-		if fr.valid(parsed) {
+		if !fr.valid(parsed) {
-			return fmt.Errorf("error parse field %#v with input %s: invalid value", field, input)
+			return fmt.Errorf("error parse field %+v with input %s parsed to %d: invalid value", field, input, parsed)
 		}
 		multi = make([]uint8, 0)
 		current = parsed
@@ -231,12 +226,12 @@ func (c *Cron) setField(field cronField, input string) error {
 			multi = make([]uint8, 0)
 			for _, s := range multiStr {
 				if parsed64, err = strconv.ParseUint(s, 10, 8); err != nil {
-					return fmt.Errorf("error parse field %#v with input %s: %w", field, input, err)
+					return fmt.Errorf("error parse field %+v with input %s: %w", field, input, err)
 				}
 
 				parsed = uint8(parsed64)
-				if fr.valid(parsed) {
+				if !fr.valid(parsed) {
-					return fmt.Errorf("error parse field %#v with input %s: invalid value", field, input)
+					return fmt.Errorf("error parse field %+v with input %s: invalid value", field, input)
 				}
 
 				multi = append(multi, parsed)
@@ -247,12 +242,12 @@ func (c *Cron) setField(field cronField, input string) error {
 			}
 		} else {
 			if parsed64, err = strconv.ParseUint(input, 10, 8); err != nil {
-				return fmt.Errorf("error parse field %#v with input %s: %w", field, input, err)
+				return fmt.Errorf("error parse field %+v with input %s: %w", field, input, err)
 			}
 
 			parsed = uint8(parsed64)
-			if fr.valid(parsed) {
+			if !fr.valid(parsed) {
-				return fmt.Errorf("error parse field %#v with input %s: invalid value", field, input)
+				return fmt.Errorf("error parse field %+v with input %s: invalid value", field, input)
 			}
 
 			*cField = &specExact{

docker/alpine/Dockerfile

@@ -0,0 +1,20 @@
+FROM golang:1.21-alpine as builder
+
+ADD . /root/wingmate
+WORKDIR /root/wingmate/
+ARG TEST_BUILD
+RUN apk add make build-base && CGO_ENABLED=1 make all && make DESTDIR=/usr/local/bin/wingmate install
+
+
+
+FROM alpine:3.18
+
+RUN apk add tzdata && ln -s /usr/share/zoneinfo/Australia/Sydney /etc/localtime && \
+    adduser -h /home/user1 -D -s /bin/sh user1 && \
+    adduser -h /home/user2 -D -s /bin/sh user2
+COPY --from=builder /usr/local/bin/wingmate/ /usr/local/bin/
+ADD --chmod=755 docker/alpine/entry.sh /usr/local/bin/entry.sh
+ADD --chmod=755 docker/alpine/etc /etc
+
+ENTRYPOINT [ "/usr/local/bin/entry.sh" ]
+CMD [ "/usr/local/bin/wingmate" ]

docker/alpine/entry.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+if [ $# -gt 0 ]; then
+    exec "$@"
+else
+    exec /usr/local/bin/wingmate
+fi

docker/alpine/etc/wingmate/crontab

@@ -0,0 +1,3 @@
+*/5 * * * * /etc/wingmate/crontab.d/cron1.sh
+17,42 */2 * * * /etc/wingmate/crontab.d/cron2.sh
+7,19,23,47 22 * * * /etc/wingmate/crontab.d/cron3.sh

docker/alpine/etc/wingmate/crontab.d/cron1.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+export WINGMATE_DUMMY_PATH=/usr/local/bin/wmdummy
+export WINGMATE_LOG=/var/log/cron1.log
+export WINGMATE_LOG_MESSAGE="cron executed in minute 5,10,15,20,25,30,35,40,45,50,55"
+
+echo "I'm runnig with dummy=$WINGMATE_DUMMY_PATH, log=$WINGMATE_LOG and mesage=$WINGMATE_LOG_MESSAGE" >> /var/log/debug-cron.log
+
+exec /usr/local/bin/wmoneshot

docker/alpine/etc/wingmate/crontab.d/cron2.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+export WINGMATE_DUMMY_PATH=/usr/local/bin/wmdummy
+export WINGMATE_LOG=/var/log/cron2.log
+export WINGMATE_LOG_MESSAGE="cron scheduled using 17,42 */2 * * *"
+
+exec /usr/local/bin/wmoneshot

docker/alpine/etc/wingmate/crontab.d/cron3.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+export WINGMATE_DUMMY_PATH=/usr/local/bin/wmdummy
+export WINGMATE_LOG=/var/log/cron3.log
+export WINGMATE_LOG_MESSAGE="cron entry: 7,19,23,47 22 * * * /etc/wingmate/crontab.d/cron3.sh"
+
+exec /usr/local/bin/wmoneshot

docker/alpine/etc/wingmate/service/one.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+export DUMMY_PATH=/usr/local/bin/wmdummy
+exec /usr/local/bin/wmexec --setsid --user user1:user1 -- /usr/local/bin/wmstarter

docker/alpine/etc/wingmate/service/spawner.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+export WINGMATE_ONESHOT_PATH=/usr/local/bin/wmoneshot
+export WINGMATE_DUMMY_PATH=/usr/local/bin/wmdummy
+exec /usr/local/bin/wmexec --user 1001 -- /usr/local/bin/wmspawner

docker/bookworm/Dockerfile

@@ -0,0 +1,20 @@
+FROM golang:1.21-bookworm as builder
+
+ADD . /root/wingmate
+WORKDIR /root/wingmate/
+ARG TEST_BUILD
+RUN make all && make DESTDIR=/usr/local/bin/wingmate install
+
+
+
+FROM debian:bookworm
+
+RUN ln -sf /usr/share/zoneinfo/Australia/Sydney /etc/localtime && \
+    apt update && apt install -y procps && \
+    useradd -m -s /bin/bash user1
+COPY --from=builder /usr/local/bin/wingmate/ /usr/local/bin/
+ADD --chmod=755 docker/bookworm/entry.sh /usr/local/bin/entry.sh
+ADD --chmod=755 docker/bookworm/etc /etc
+
+ENTRYPOINT [ "/usr/local/bin/entry.sh" ]
+CMD [ "/usr/local/bin/wingmate" ]

docker/bookworm/entry.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/bash
+
+if [ $# -gt 0 ]; then
+    exec "$@"
+else
+    exec /usr/local/bin/wingmate
+fi

docker/bookworm/etc/wingmate/crontab

@@ -0,0 +1,3 @@
+*/5 * * * * /etc/wingmate/crontab.d/cron1.sh
+17,42 */2 * * * /etc/wingmate/crontab.d/cron2.sh
+7,19,23,47 22 * * * /etc/wingmate/crontab.d/cron3.sh

docker/bookworm/etc/wingmate/crontab.d/cron1.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/bash
+
+export WINGMATE_DUMMY_PATH=/usr/local/bin/wmdummy
+export WINGMATE_LOG=/var/log/cron1.log
+export WINGMATE_LOG_MESSAGE="cron executed in minute 5,10,15,20,25,30,35,40,45,50,55"
+
+echo "I'm runnig with dummy=$WINGMATE_DUMMY_PATH, log=$WINGMATE_LOG and mesage=$WINGMATE_LOG_MESSAGE" >> /var/log/debug-cron.log
+
+exec /usr/local/bin/wmoneshot

docker/bookworm/etc/wingmate/crontab.d/cron2.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/bash
+
+export WINGMATE_DUMMY_PATH=/usr/local/bin/wmdummy
+export WINGMATE_LOG=/var/log/cron2.log
+export WINGMATE_LOG_MESSAGE="cron scheduled using 17,42 */2 * * *"
+
+exec /usr/local/bin/wmoneshot

docker/bookworm/etc/wingmate/crontab.d/cron3.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/bash
+
+export WINGMATE_DUMMY_PATH=/usr/local/bin/wmdummy
+export WINGMATE_LOG=/var/log/cron3.log
+export WINGMATE_LOG_MESSAGE="cron entry: 7,19,23,47 22 * * * /etc/wingmate/crontab.d/cron3.sh"
+
+exec /usr/local/bin/wmoneshot

docker/bookworm/etc/wingmate/service/one.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/bash
+
+export DUMMY_PATH=/usr/local/bin/wmdummy
+exec /usr/local/bin/wmexec --setsid --user user1:user1 -- /usr/local/bin/wmstarter

docker/bookworm/etc/wingmate/service/spawner.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/bash
+
+export WINGMATE_ONESHOT_PATH=/usr/local/bin/wmoneshot
+export WINGMATE_DUMMY_PATH=/usr/local/bin/wmdummy
+exec /usr/local/bin/wmexec --user 1200 -- /usr/local/bin/wmspawner

example/ssh-docker/Dockerfile

@@ -0,0 +1,14 @@
+FROM suyono/wingmate:alpine as source
+
+FROM alpine:3.19
+
+RUN apk update && apk add tzdata openssh-server && \
+    ln -s /usr/share/zoneinfo/Australia/Sydney /etc/localtime && ssh-keygen -A
+
+COPY --from=source /usr/local/bin/wingmate /usr/local/bin/
+COPY --from=source /usr/local/bin/wmpidproxy /usr/local/bin/
+ADD --chmod=755 example/ssh-docker/entry.sh /usr/local/bin/entry.sh
+ADD --chmod=755 example/ssh-docker/etc /etc
+
+ENTRYPOINT [ "/usr/local/bin/entry.sh" ]
+CMD [ "/usr/local/bin/wingmate" ]

example/ssh-docker/entry.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+if [ $# -gt 0 ]; then
+    exec "$@"
+else
+    exec /usr/local/bin/wingmate
+fi

example/ssh-docker/etc/wingmate/service/sshd.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+exec /usr/local/bin/wmpidproxy --pid-file /var/run/sshd.pid -- /usr/sbin/sshd

go.mod

@@ -3,8 +3,8 @@ module gitea.suyono.dev/suyono/wingmate
 go 1.21
 
 require (
-	github.com/rs/zerolog v1.30.0
+	github.com/rs/zerolog v1.31.0
-	github.com/spf13/cobra v1.7.0
+	github.com/spf13/cobra v1.8.0
 	github.com/spf13/viper v1.17.0
 	github.com/stretchr/testify v1.8.4
 	golang.org/x/sys v0.15.0
@@ -17,7 +17,7 @@ require (
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-isatty v0.0.19 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/pelletier/go-toml/v2 v2.1.0 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
@@ -28,6 +28,7 @@ require (
 	github.com/spf13/cast v1.6.0 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/stretchr/objx v0.5.1 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect

go.sum

@@ -48,6 +48,7 @@ github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnht
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -150,6 +151,8 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
 github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4=
@@ -166,6 +169,8 @@ github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZV
 github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/rs/zerolog v1.30.0 h1:SymVODrcRsaRaSInD9yQtKbtWqwsfoPcRff/oRXLj4c=
 github.com/rs/zerolog v1.30.0/go.mod h1:/tk+P47gFdPXq4QYjvCmT5/Gsug2nagsFWBWhAiSi1w=
+github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A=
+github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ=
 github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4=
@@ -183,6 +188,8 @@ github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0=
 github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
 github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
 github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
+github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
+github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
 github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
 github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
@@ -194,12 +201,15 @@ github.com/spf13/viper v1.17.0/go.mod h1:BmMMMLQXSbcHK6KAOiFLz0l5JHrU89OdIRHvsk0
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/objx v0.5.1 h1:4VhoImhV/Bm0ToFkXFi8hXNXwpDRZ/ynw3amt82mzq0=
+github.com/stretchr/objx v0.5.1/go.mod h1:/iHQpkQwBD6DLUmQ4pE+s1TXdob1mORJ4/UFdrifcy0=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=

init/cron.go

@@ -1,6 +1,7 @@
 package init
 
 import (
+	"io"
 	"os/exec"
 	"sync"
 	"time"
@@ -8,19 +9,61 @@ import (
 	"gitea.suyono.dev/suyono/wingmate"
 )
 
+const (
+	cronTag = "cron"
+)
+
 func (i *Init) cron(wg *sync.WaitGroup, cron Cron, exitFlag <-chan any) {
 	defer wg.Done()
 
-	ticker := time.NewTicker(time.Second * 20)
+	var (
+		iwg    *sync.WaitGroup
+		err    error
+		stdout io.ReadCloser
+		stderr io.ReadCloser
+	)
+
+	ticker := time.NewTicker(time.Second * 30)
 cron:
 	for {
 		if cron.TimeToRun(time.Now()) {
+			wingmate.Log().Info().Str(cronTag, cron.Command().Path()).Msg("executing")
 			cmd := exec.Command(cron.Command().Path())
-			if err := cmd.Run(); err != nil {
+			iwg = &sync.WaitGroup{}
-				wingmate.Log().Error().Msgf("running cron %s error %#v", cron.Command().Path(), err)
+
+			if stdout, err = cmd.StdoutPipe(); err != nil {
+				wingmate.Log().Error().Str(cronTag, cron.Command().Path()).Msgf("stdout pipe: %+v", err)
+				goto fail
+			}
+
+			if stderr, err = cmd.StderrPipe(); err != nil {
+				wingmate.Log().Error().Str(cronTag, cron.Command().Path()).Msgf("stderr pipe: %+v", err)
+				_ = stdout.Close()
+				goto fail
+			}
+
+			iwg.Add(1)
+			go i.pipeReader(iwg, stdout, cronTag, cron.Command().Path())
+
+			iwg.Add(1)
+			go i.pipeReader(iwg, stderr, cronTag, cron.Command().Path())
+
+			if err := cmd.Start(); err != nil {
+				wingmate.Log().Error().Msgf("starting cron %s error %+v", cron.Command().Path(), err)
+				_ = stdout.Close()
+				_ = stderr.Close()
+				iwg.Wait()
+				goto fail
+			}
+
+			iwg.Wait()
+
+			if err = cmd.Wait(); err != nil {
+				wingmate.Log().Error().Str(cronTag, cron.Command().Path()).Msgf("got error when waiting: %+v", err)
 			}
 		}
 
+	fail:
 		select {
 		case <-exitFlag:
 			ticker.Stop()

init/init.go

@@ -1,6 +1,7 @@
 package init
 
 import (
+	"os"
 	"sync"
 	"time"
 )
@@ -34,17 +35,19 @@ func (i *Init) Start() {
 		wg             *sync.WaitGroup
 		signalTrigger  chan any
 		sighandlerExit chan any
+		sigchld        chan os.Signal
 	)
 
 	signalTrigger = make(chan any)
 	sighandlerExit = make(chan any)
+	sigchld = make(chan os.Signal, 1)
 
 	wg = &sync.WaitGroup{}
 	wg.Add(1)
-	go i.waiter(wg, signalTrigger, sighandlerExit)
+	go i.waiter(wg, signalTrigger, sighandlerExit, sigchld)
 
 	wg.Add(1)
-	go i.sighandler(wg, signalTrigger, sighandlerExit)
+	go i.sighandler(wg, signalTrigger, sighandlerExit, sigchld)
 
 	for _, s := range i.config.Services() {
 		wg.Add(1)

init/service.go

@@ -1,26 +1,76 @@
 package init
 
 import (
+	"bufio"
+	"io"
 	"os/exec"
 	"sync"
+	"time"
 
 	"gitea.suyono.dev/suyono/wingmate"
 )
 
+const (
+	serviceTag = "service"
+)
+
 func (i *Init) service(wg *sync.WaitGroup, path Path, exitFlag <-chan any) {
 	defer wg.Done()
 
 	var (
 		err        error
+		iwg        *sync.WaitGroup
+		stderr     io.ReadCloser
+		stdout     io.ReadCloser
+		failStatus bool
 	)
 
+	defer func() {
+		wingmate.Log().Info().Str(serviceTag, path.Path()).Msg("stopped")
+	}()
+
 service:
 	for {
+		failStatus = false
 		cmd := exec.Command(path.Path())
-		if err = cmd.Run(); err != nil {
+		iwg = &sync.WaitGroup{}
+
+		if stdout, err = cmd.StdoutPipe(); err != nil {
+			wingmate.Log().Error().Str(serviceTag, path.Path()).Msgf("stdout pipe: %#v", err)
+			failStatus = true
+			goto fail
+		}
+		iwg.Add(1)
+		go i.pipeReader(iwg, stdout, serviceTag, path.Path())
+
+		if stderr, err = cmd.StderrPipe(); err != nil {
+			wingmate.Log().Error().Str(serviceTag, path.Path()).Msgf("stderr pipe: %#v", err)
+			_ = stdout.Close()
+			failStatus = true
+			goto fail
+		}
+		iwg.Add(1)
+		go i.pipeReader(iwg, stderr, serviceTag, path.Path())
+
+		if err = cmd.Start(); err != nil {
 			wingmate.Log().Error().Msgf("starting service %s error %#v", path.Path(), err)
+			failStatus = true
+			_ = stdout.Close()
+			_ = stderr.Close()
+			iwg.Wait()
+			goto fail
 		}
 
+		iwg.Wait()
+
+		if err = cmd.Wait(); err != nil {
+			wingmate.Log().Error().Str(serviceTag, path.Path()).Msgf("got error when waiting: %+v", err)
+		}
+	fail:
+		if failStatus {
+			time.Sleep(time.Second)
+			failStatus = false
+		}
 		select {
 		case <-exitFlag:
 			break service
@@ -29,3 +79,18 @@ service:
 	}
 
 }
+
+func (i *Init) pipeReader(wg *sync.WaitGroup, pipe io.ReadCloser, tag, serviceName string) {
+	defer wg.Done()
+
+	scanner := bufio.NewScanner(pipe)
+	for scanner.Scan() {
+		wingmate.Log().Info().Str(tag, serviceName).Msg(scanner.Text())
+	}
+
+	if err := scanner.Err(); err != nil {
+		wingmate.Log().Error().Str(tag, serviceName).Msgf("got error when reading pipe: %#v", err)
+	}
+
+	wingmate.Log().Info().Str(tag, serviceName).Msg("closing pipe")
+}

init/sighandler.go

@@ -5,11 +5,16 @@ import (
 	"os/signal"
 	"sync"
 
+	"gitea.suyono.dev/suyono/wingmate"
 	"golang.org/x/sys/unix"
 )
 
-func (i *Init) sighandler(wg *sync.WaitGroup, trigger chan<- any, selfExit <-chan any) {
+func (i *Init) sighandler(wg *sync.WaitGroup, trigger chan<- any, selfExit <-chan any, sigchld chan<- os.Signal) {
-	defer wg.Wait()
+	defer wg.Done()
+
+	defer func() {
+		wingmate.Log().Warn().Msg("signal handler: exiting")
+	}()
 
 	isOpen := true
 
@@ -23,14 +28,21 @@ signal:
 			switch s {
 			case unix.SIGTERM, unix.SIGINT:
 				if isOpen {
+					wingmate.Log().Info().Msg("initiating shutdown...")
 					close(trigger)
+					wg.Add(1)
+					go i.signalPump(wg, selfExit)
 					isOpen = false
 				}
 			case unix.SIGCHLD:
-				// do nothing
+				select {
+				case sigchld <- s:
+				default:
+				}
 			}
 
 		case <-selfExit:
+			wingmate.Log().Warn().Msg("signal handler received completion flag")
 			break signal
 		}
 	}

init/signal-pump.go

@@ -0,0 +1,71 @@
+package init
+
+import (
+	"sync"
+	"time"
+
+	"gitea.suyono.dev/suyono/wingmate"
+	"golang.org/x/sys/unix"
+)
+
+type status int
+
+const (
+	triggered status = iota
+	expired
+)
+
+func (i *Init) signalPump(wg *sync.WaitGroup, selfExit <-chan any) {
+	defer wg.Done()
+	defer func() {
+		wingmate.Log().Info().Msg("signal pump completed")
+	}()
+
+	if seStatus := i.sigTermPump(time.Now(), selfExit); seStatus == triggered {
+		return
+	}
+
+	i.sigKillPump(time.Now(), selfExit)
+}
+
+func (i *Init) sigKillPump(startTime time.Time, selfExit <-chan any) {
+	t := time.NewTicker(time.Millisecond * 200)
+	defer t.Stop()
+
+	wingmate.Log().Info().Msg("start pumping SIGKILL signal")
+	defer func() {
+		wingmate.Log().Info().Msg("stop pumping SIGKILL signal")
+	}()
+
+	for time.Since(startTime) < time.Second {
+		_ = unix.Kill(-1, unix.SIGKILL)
+
+		select {
+		case <-t.C:
+		case <-selfExit:
+			return
+		}
+	}
+}
+
+func (i *Init) sigTermPump(startTime time.Time, selfExit <-chan any) status {
+	t := time.NewTicker(time.Millisecond * 100)
+	defer t.Stop()
+
+	wingmate.Log().Info().Msg("start pumping SIGTERM signal")
+	defer func() {
+		wingmate.Log().Info().Msg("stop pumping SIGTERM signal")
+	}()
+
+	for time.Since(startTime) < time.Duration(time.Second*4) {
+		_ = unix.Kill(-1, unix.SIGTERM)
+
+		select {
+		case <-t.C:
+		case <-selfExit:
+			return triggered
+		}
+	}
+
+	return expired
+}

init/waiter.go

@@ -2,35 +2,68 @@ package init
 
 import (
 	"errors"
+	"os"
 	"sync"
 
+	"gitea.suyono.dev/suyono/wingmate"
 	"golang.org/x/sys/unix"
 )
 
-func (i *Init) waiter(wg *sync.WaitGroup, runningFlag <-chan any, sigHandlerFlag chan<- any) {
+func (i *Init) waiter(wg *sync.WaitGroup, runningFlag <-chan any, sigHandlerFlag chan<- any, sigchld <-chan os.Signal) {
 	var (
 		ws               unix.WaitStatus
-		// pid     int
 		err              error
 		running          bool
+		flagged          bool
+		waitingForSignal bool
 	)
+	defer wg.Done()
+
+	defer func() {
+		wingmate.Log().Info().Msg("waiter exiting...")
+	}()
 
 	running = true
+	flagged = true
+	waitingForSignal = true
 wait:
 	for {
+		if running {
+			if waitingForSignal {
 				select {
 				case <-runningFlag:
+					wingmate.Log().Info().Msg("waiter received shutdown signal...")
+					running = false
+				case <-sigchld:
+					waitingForSignal = false
+				}
+
+			} else {
+				select {
+				case <-runningFlag:
+					wingmate.Log().Info().Msg("waiter received shutdown signal...")
 					running = false
 				default:
 				}
 
+			}
+		}
+
 		if _, err = unix.Wait4(-1, &ws, 0, nil); err != nil {
 			if errors.Is(err, unix.ECHILD) {
 				if !running {
+					if flagged {
 						close(sigHandlerFlag)
+						flagged = false
+						wingmate.Log().Warn().Msg("waiter: inner flag")
+					}
+					wingmate.Log().Warn().Msg("waiter: no child left")
 					break wait
 				}
 			}
+
+			wingmate.Log().Warn().Msgf("Wait4 returns error: %+v", err)
+			waitingForSignal = true
 		}
 	}
 }

logger.go

@@ -1,9 +1,15 @@
 package wingmate
 
 import (
+	"io"
+	"time"
+
 	"gitea.suyono.dev/suyono/wingmate/logger"
 	"github.com/rs/zerolog"
-	"io"
+)
+
+const (
+	timeTag = "time"
 )
 
 var (
@@ -29,13 +35,28 @@ func Log() logger.Log {
 }
 
 func (w *wrapper) Info() logger.Content {
-	return w.log.Info()
+	return (*eventWrapper)(w.log.Info().Time(timeTag, time.Now()))
 }
 
 func (w *wrapper) Warn() logger.Content {
-	return w.log.Warn()
+	return (*eventWrapper)(w.log.Warn().Time(timeTag, time.Now()))
 }
 
 func (w *wrapper) Error() logger.Content {
-	return w.log.Error()
+	return (*eventWrapper)(w.log.Error().Time(timeTag, time.Now()))
+}
+
+type eventWrapper zerolog.Event
+
+func (w *eventWrapper) Msg(msg string) {
+	(*zerolog.Event)(w).Msg(msg)
+}
+
+func (w *eventWrapper) Msgf(format string, data ...any) {
+	(*zerolog.Event)(w).Msgf(format, data...)
+}
+
+func (w *eventWrapper) Str(key, value string) logger.Content {
+	rv := (*zerolog.Event)(w).Str(key, value)
+	return (*eventWrapper)(rv)
 }

logger/logger.go

@@ -3,6 +3,7 @@ package logger
 type Content interface {
 	Msg(string)
 	Msgf(string, ...any)
+	Str(string, string) Content
 }
 
 type Level interface {

types.go

@@ -6,4 +6,6 @@ const (
 	Any CronTimeType = iota
 	Exact
 	MultipleOccurrence
+
+	EnvPrefix = "WINGMATE"
 )