Add tool to see size of column in parquet file

Author: cyriltovena

cmd/parquet-tool/main.go

@@ -0,0 +1,52 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/dustin/go-humanize"
+	"github.com/olekukonko/tablewriter"
+	"github.com/segmentio/parquet-go"
+)
+
+func main() {
+	f, err := os.Open("/tmp/test.parquet")
+	if err != nil {
+		panic(err)
+	}
+	defer f.Close()
+	stats, err := f.Stat()
+	if err != nil {
+		panic(err)
+	}
+	pf, err := parquet.OpenFile(f, stats.Size())
+	if err != nil {
+		panic(err)
+	}
+	fmt.Println("schema:", pf.Schema())
+	meta := pf.Metadata()
+	fmt.Println("Num Rows:", meta.NumRows)
+
+	for i, rg := range meta.RowGroups {
+		fmt.Println("\t Row group:", i)
+		fmt.Println("\t\t Row Count:", rg.NumRows)
+		fmt.Println("\t\t Row size:", humanize.Bytes(uint64(rg.TotalByteSize)))
+		fmt.Println("\t\t Columns:")
+		table := tablewriter.NewWriter(os.Stdout)
+		table.SetHeader([]string{"Col", "Type", "NumVal", "TotalCompressedSize", "TotalUncompressedSize", "Compression", "%"})
+		for _, ds := range rg.Columns {
+			table.Append(
+				[]string{
+					strings.Join(ds.MetaData.PathInSchema, "/"),
+					ds.MetaData.Type.String(),
+					fmt.Sprintf("%d", ds.MetaData.NumValues),
+					humanize.Bytes(uint64(ds.MetaData.TotalCompressedSize)),
+					humanize.Bytes(uint64(ds.MetaData.TotalUncompressedSize)),
+					fmt.Sprintf("%.2f", float64(ds.MetaData.TotalUncompressedSize-ds.MetaData.TotalCompressedSize)/float64(ds.MetaData.TotalCompressedSize)*100),
+					fmt.Sprintf("%.2f", float64(ds.MetaData.TotalCompressedSize)/float64(rg.TotalByteSize)*100),
+				})
+		}
+		table.Render()
+	}
+}

go.mod

@@ -23,6 +23,7 @@ require (
 	github.com/prometheus/common v0.35.0
 	github.com/prometheus/prometheus v1.8.2-0.20220315145411-881111fec433
 	github.com/pyroscope-io/pyroscope v0.18.0
+	github.com/segmentio/parquet-go v0.0.0-20220623195409-8b4d4260d8cb
 	github.com/stretchr/testify v1.7.5
 	github.com/thanos-io/objstore v0.0.0-20220324141029-c4f11442aa33
 	github.com/thanos-io/thanos v0.26.0
@@ -211,7 +212,6 @@ require (
 	github.com/scaleway/scaleway-sdk-go v1.0.0-beta.9 // indirect
 	github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect
 	github.com/segmentio/encoding v0.3.5 // indirect
-	github.com/segmentio/parquet-go v0.0.0-20220623195409-8b4d4260d8cb // indirect
 	github.com/sercand/kuberesolver v2.4.0+incompatible // indirect
 	github.com/shirou/gopsutil v3.21.4+incompatible // indirect
 	github.com/sirupsen/logrus v1.8.1 // indirect
@@ -262,6 +262,7 @@ require (
 
 replace (
 	github.com/polarsignals/frostdb => github.com/grafana/frostdb v0.0.0-20220623183226-4080ae2447c1
+	github.com/segmentio/parquet-go => github.com/cyriltovena/parquet-go v0.0.0-20220706111652-c477af2f3d29
 	google.golang.org/api => google.golang.org/api v0.70.0
 	google.golang.org/grpc => google.golang.org/grpc v1.44.0
 )

go.sum

@@ -414,6 +414,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
 github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4=
 github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
+github.com/cyriltovena/parquet-go v0.0.0-20220706111652-c477af2f3d29 h1:oTjhKKMrex0rLrUxgyAagfuA6ANOQATrT0L3iqpwoTc=
+github.com/cyriltovena/parquet-go v0.0.0-20220706111652-c477af2f3d29/go.mod h1:BuMbRhCCg3gFchup9zucJaUjQ4m6RxX+iVci37CoMPQ=
 github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ=
 github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s=
 github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5/go.mod h1:Eo87+Kg/IX2hfWJfwxMzLyuSZyxSoAug2nGa1G2QAi8=
@@ -1364,8 +1366,6 @@ github.com/seccomp/libseccomp-golang v0.9.2-0.20210429002308-3879420cc921/go.mod
 github.com/segmentio/asm v1.1.3/go.mod h1:Ld3L4ZXGNcSLRg4JBsZ3//1+f/TjYl0Mzen/DQy1EJg=
 github.com/segmentio/encoding v0.3.5 h1:UZEiaZ55nlXGDL92scoVuw00RmiRCazIEmvPSbSvt8Y=
 github.com/segmentio/encoding v0.3.5/go.mod h1:n0JeuIqEQrQoPDGsjo8UNd1iA0U8d8+oHAA4E3G3OxM=
-github.com/segmentio/parquet-go v0.0.0-20220623195409-8b4d4260d8cb h1:nGVNj4bt9Fn5rsEvLsuOoTuQGSdiSkPRH+BvKlIp5NY=
-github.com/segmentio/parquet-go v0.0.0-20220623195409-8b4d4260d8cb/go.mod h1:BuMbRhCCg3gFchup9zucJaUjQ4m6RxX+iVci37CoMPQ=
 github.com/sercand/kuberesolver v2.4.0+incompatible h1:WE2OlRf6wjLxHwNkkFLQGaZcVLEXjMjBPjjEU5vksH8=
 github.com/sercand/kuberesolver v2.4.0+incompatible/go.mod h1:lWF3GL0xptCB/vCiJPl/ZshwPsX/n4Y7u0CW9E7aQIQ=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=