|
| 1 | +package issues |
| 2 | + |
| 3 | +import ( |
| 4 | + "context" |
| 5 | + "fmt" |
| 6 | + "github.com/ClickHouse/clickhouse-go/v2/lib/driver" |
| 7 | + "github.com/google/uuid" |
| 8 | + "math/rand" |
| 9 | + "runtime" |
| 10 | + "strings" |
| 11 | + "testing" |
| 12 | + "time" |
| 13 | + |
| 14 | + "github.com/ClickHouse/clickhouse-go/v2" |
| 15 | + clickhouse_tests "github.com/ClickHouse/clickhouse-go/v2/tests" |
| 16 | + "github.com/stretchr/testify/require" |
| 17 | +) |
| 18 | + |
| 19 | +// test for https://github.com/ClickHouse/clickhouse-go/issues/1271 |
| 20 | +func Test1271(t *testing.T) { |
| 21 | + var ( |
| 22 | + conn, err = clickhouse_tests.GetConnection("issues", clickhouse.Settings{ |
| 23 | + "max_execution_time": 60, |
| 24 | + "allow_experimental_object_type": true, |
| 25 | + }, nil, &clickhouse.Compression{ |
| 26 | + Method: clickhouse.CompressionLZ4, |
| 27 | + }) |
| 28 | + ) |
| 29 | + defer func() { |
| 30 | + conn.Exec(context.Background(), "DROP TABLE flush_with_close_query_example") |
| 31 | + conn.Close() |
| 32 | + }() |
| 33 | + conn.Exec(context.Background(), "DROP TABLE IF EXISTS flush_with_close_query_example") |
| 34 | + ctx := context.Background() |
| 35 | + err = conn.Exec(ctx, ` |
| 36 | + CREATE TABLE IF NOT EXISTS flush_with_close_query_example ( |
| 37 | + Col1 UInt64 |
| 38 | + , Col2 String |
| 39 | + , Col3 FixedString(3) |
| 40 | + , Col4 UUID |
| 41 | + , Col5 Map(String, UInt64) |
| 42 | + , Col6 Array(String) |
| 43 | + , Col7 Tuple(String, UInt64, Array(Map(String, UInt64))) |
| 44 | + , Col8 DateTime |
| 45 | + ) Engine = MergeTree() ORDER BY tuple() |
| 46 | + `) |
| 47 | + require.NoError(t, err) |
| 48 | + |
| 49 | + batch, err := conn.PrepareBatch(ctx, "INSERT INTO flush_with_close_query_example", driver.WithCloseOnFlush()) |
| 50 | + require.NoError(t, err) |
| 51 | + // 1 million rows should only take < 1s on most desktops |
| 52 | + for i := 0; i < 100_000; i++ { |
| 53 | + require.NoError(t, batch.Append( |
| 54 | + uint64(i), |
| 55 | + RandAsciiString(5), |
| 56 | + RandAsciiString(3), |
| 57 | + uuid.New(), |
| 58 | + map[string]uint64{"key": uint64(i)}, // Map(String, UInt64) |
| 59 | + []string{RandAsciiString(1), RandAsciiString(1), RandAsciiString(1), RandAsciiString(1), RandAsciiString(1), RandAsciiString(1)}, // Array(String) |
| 60 | + []any{ // Tuple(String, UInt64, Array(Map(String, UInt64))) |
| 61 | + RandAsciiString(10), uint64(i), []map[string]uint64{ |
| 62 | + {"key": uint64(i)}, |
| 63 | + {"key": uint64(i + 1)}, |
| 64 | + {"key": uint64(i) + 2}, |
| 65 | + }, |
| 66 | + }, |
| 67 | + time.Now().Add(time.Duration(i)*time.Second), |
| 68 | + )) |
| 69 | + if i > 0 && i%10000 == 0 { |
| 70 | + require.NoError(t, batch.Flush()) |
| 71 | + PrintMemUsage() |
| 72 | + } |
| 73 | + } |
| 74 | + require.NoError(t, batch.Flush()) |
| 75 | + // confirm we have the right count |
| 76 | + var col1 uint64 |
| 77 | + require.NoError(t, conn.QueryRow(ctx, "SELECT count() FROM flush_with_close_query_example").Scan(&col1)) |
| 78 | + require.Equal(t, uint64(100_000), col1) |
| 79 | +} |
| 80 | + |
| 81 | +const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" |
| 82 | +const ( |
| 83 | + letterIdxBits = 6 // 6 bits to represent a letter index |
| 84 | + letterIdxMask = 1<<letterIdxBits - 1 // All 1-bits, as many as letterIdxBits |
| 85 | + letterIdxMax = 63 / letterIdxBits // # of letter indices fitting in 63 bits |
| 86 | +) |
| 87 | + |
| 88 | +func RandAsciiString(n int) string { |
| 89 | + return randString(n, letterBytes) |
| 90 | +} |
| 91 | + |
| 92 | +var src = rand.NewSource(time.Now().UnixNano()) |
| 93 | + |
| 94 | +func randString(n int, bytes string) string { |
| 95 | + sb := strings.Builder{} |
| 96 | + sb.Grow(n) |
| 97 | + // A src.Int63() generates 63 random bits, enough for letterIdxMax characters! |
| 98 | + for i, cache, remain := n-1, src.Int63(), letterIdxMax; i >= 0; { |
| 99 | + if remain == 0 { |
| 100 | + cache, remain = src.Int63(), letterIdxMax |
| 101 | + } |
| 102 | + if idx := int(cache & letterIdxMask); idx < len(bytes) { |
| 103 | + sb.WriteByte(bytes[idx]) |
| 104 | + i-- |
| 105 | + } |
| 106 | + cache >>= letterIdxBits |
| 107 | + remain-- |
| 108 | + } |
| 109 | + |
| 110 | + return sb.String() |
| 111 | +} |
| 112 | + |
| 113 | +// PrintMemUsage outputs the current, total and OS memory being used. As well as the number |
| 114 | +// of garage collection cycles completed. |
| 115 | +// thanks to https://golangcode.com/print-the-current-memory-usage/ |
| 116 | +func PrintMemUsage() { |
| 117 | + var m runtime.MemStats |
| 118 | + runtime.ReadMemStats(&m) |
| 119 | + // For info on each, see: https://golang.org/pkg/runtime/#MemStats |
| 120 | + fmt.Printf("Alloc = %v MiB", bToMb(m.Alloc)) |
| 121 | + fmt.Printf("\tTotalAlloc = %v MiB", bToMb(m.TotalAlloc)) |
| 122 | + fmt.Printf("\tSys = %v MiB", bToMb(m.Sys)) |
| 123 | + fmt.Printf("\tNumGC = %v\n", m.NumGC) |
| 124 | +} |
| 125 | + |
| 126 | +func bToMb(b uint64) uint64 { |
| 127 | + return b / 1024 / 1024 |
| 128 | +} |
0 commit comments