cmd/geth: Graceful shutdown if disk is full

cmd/utils/cmd.go

@@ -26,6 +26,7 @@ import (
 	"runtime"
 	"strings"
 	"syscall"
+	"time"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core"
@@ -40,7 +41,13 @@ import (
 )
 
 const (
-	importBatchSize = 2500
+	importBatchSize       = 2500
+	freeDiskSpaceWarning  = 1024 * 1024 * 1024
+	freeDiskSpaceCritical = 300 * 1024 * 1024
+)
+
+var (
+	sigtermCh = make(chan os.Signal, 1)
 )
 
 // Fatalf formats a message to standard error and exits the program.
@@ -68,14 +75,14 @@ func StartNode(stack *node.Node) {
 		Fatalf("Error starting protocol stack: %v", err)
 	}
 	go func() {
-		sigc := make(chan os.Signal, 1)
-		signal.Notify(sigc, syscall.SIGINT, syscall.SIGTERM)
-		defer signal.Stop(sigc)
-		<-sigc
+		signal.Notify(sigtermCh, syscall.SIGINT, syscall.SIGTERM)
+		defer signal.Stop(sigtermCh)
+		go monitorFreeDiskSpace(stack.InstanceDir())
+		<-sigtermCh
 		log.Info("Got interrupt, shutting down...")
 		go stack.Close()
 		for i := 10; i > 0; i-- {
-			<-sigc
+			<-sigtermCh
 			if i > 1 {
 				log.Warn("Already shutting down, interrupt more to panic.", "times", i-1)
 			}
@@ -85,6 +92,19 @@ func StartNode(stack *node.Node) {
 	}()
 }
 
+func monitorFreeDiskSpace(path string) {
+	for {
+		freeSpace := getFreeDiskSpace(path)
+		if freeSpace < freeDiskSpaceCritical {
+			log.Error("Low disk space. Gracefully shutting down Geth to prevent database corruption.", "available", freeSpace/1024/1024)
+			sigtermCh <- syscall.SIGTERM
+		} else if freeSpace < freeDiskSpaceWarning {
+			log.Warn("Disk space is running low. Geth will shutdown if disk space runs below critical level.", "available", freeSpace/1024/1024, "critical_level", freeDiskSpaceCritical/1024/1024)
+		}
+		time.Sleep(5 * time.Second)
+	}
+}
+
 func ImportChain(chain *core.BlockChain, fn string) error {
 	// Watch for Ctrl-C while the import is running.
 	// If a signal is received, the import will stop at the next batch.

cmd/utils/diskusage.go

@@ -0,0 +1,41 @@
+// Copyright 2020 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// +build !windows
+
+package utils
+
+import (
+	"syscall"
+
+	"github.com/ethereum/go-ethereum/log"
+	"golang.org/x/sys/unix"
+)
+
+var stat unix.Statfs_t
+
+func getFreeDiskSpace(path string) uint64 {
+
+	err := unix.Statfs(path, &stat)
+	if err != nil {
+		log.Warn("Failed to call Statfs", "path", path, "err", err)
+		sigtermCh <- syscall.SIGTERM
+		return 0
+	}
+
+	// Available blocks * size per block = available space in bytes
+	return stat.Bavail * uint64(stat.Bsize)
+}

cmd/utils/diskusage_windows.go

@@ -0,0 +1,41 @@
+// Copyright 2020 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package utils
+
+import (
+	"syscall"
+
+	"github.com/ethereum/go-ethereum/log"
+	"golang.org/x/sys/windows"
+)
+
+func getFreeDiskSpace(path string) uint64 {
+
+	cwd, err := windows.UTF16PtrFromString(path)
+	if err != nil {
+		log.Warn("Failed to call UTF16PtrFromString", "path", path, "err", err)
+		sigtermCh <- syscall.SIGTERM
+	}
+
+	var freeBytesAvailableToCaller, totalNumberOfBytes, totalNumberOfFreeBytes uint64
+	if err := windows.GetDiskFreeSpaceEx(cwd, &freeBytesAvailableToCaller, &totalNumberOfBytes, &totalNumberOfFreeBytes); err != nil {
+		log.Warn("Failed to call GetDiskFreeSpaceEx", "path", path, "err", err)
+		sigtermCh <- syscall.SIGTERM
+	}
+
+	return freeBytesAvailableToCaller
+}