Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit 7612137

Browse filesBrowse files
agneumagneum
committed
Merge branch '152-mount-volumes' into 'master'
fix: mounting volumes in Postgres containers depends on Database Lab process environment (#152) Closes #152 See merge request postgres-ai/database-lab!145
2 parents aa065ba + 64afb83 commit 7612137
Copy full SHA for 7612137

File tree

8 files changed

+153
-90
lines changed
Filter options

8 files changed

+153
-90
lines changed

‎go.mod

Copy file name to clipboardExpand all lines: go.mod
+3Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,14 @@ require (
66
github.com/AlekSi/pointer v1.1.0
77
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect
88
github.com/Microsoft/go-winio v0.4.14 // indirect
9+
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
910
github.com/aws/aws-sdk-go v1.33.8
1011
github.com/containerd/containerd v1.3.4 // indirect
1112
github.com/docker/distribution v2.7.1+incompatible // indirect
1213
github.com/docker/docker v1.13.1
1314
github.com/docker/go-connections v0.4.0 // indirect
1415
github.com/docker/go-units v0.4.0 // indirect
16+
github.com/go-ole/go-ole v1.2.4 // indirect
1517
github.com/gogo/protobuf v1.3.1 // indirect
1618
github.com/gorilla/mux v1.7.3
1719
github.com/jessevdk/go-flags v1.4.1-0.20181221193153-c0795c8afcf4
@@ -23,6 +25,7 @@ require (
2325
github.com/robfig/cron/v3 v3.0.1
2426
github.com/rs/xid v1.2.1
2527
github.com/sergi/go-diff v1.1.0
28+
github.com/shirou/gopsutil v2.20.7+incompatible
2629
github.com/sirupsen/logrus v1.4.2
2730
github.com/stretchr/testify v1.5.1
2831
github.com/urfave/cli/v2 v2.1.1

‎go.sum

Copy file name to clipboardExpand all lines: go.sum
+6Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ
77
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
88
github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU=
99
github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
10+
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk=
11+
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
1012
github.com/aws/aws-sdk-go v1.33.8 h1:2/sOfb9oPHTRZ0lxinoaTPDcYwNa1H/SpKP4nVRBwmg=
1113
github.com/aws/aws-sdk-go v1.33.8/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
1214
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
@@ -30,6 +32,8 @@ github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD
3032
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
3133
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
3234
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
35+
github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI=
36+
github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM=
3337
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
3438
github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
3539
github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
@@ -80,6 +84,8 @@ github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0
8084
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
8185
github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
8286
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
87+
github.com/shirou/gopsutil v2.20.7+incompatible h1:Ymv4OD12d6zm+2yONe39VSmp2XooJe8za7ngOLW/o/w=
88+
github.com/shirou/gopsutil v2.20.7+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
8389
github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
8490
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
8591
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=

‎pkg/retrieval/engine/postgres/initialize/logical/dump.go

Copy file name to clipboardExpand all lines: pkg/retrieval/engine/postgres/initialize/logical/dump.go
+28-21Lines changed: 28 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -219,17 +219,12 @@ func (d *DumpJob) Run(ctx context.Context) (err error) {
219219
return errors.Wrap(err, "failed to scan pulling image response")
220220
}
221221

222-
cont, err := d.dockerClient.ContainerCreate(ctx,
223-
&container.Config{
224-
Env: d.getEnvironmentVariables(),
225-
Image: d.DockerImage,
226-
Healthcheck: health.GetConfig(),
227-
},
228-
&container.HostConfig{
229-
Mounts: d.getMountVolumes(),
230-
NetworkMode: d.getContainerNetworkMode(),
231-
},
232-
&network.NetworkingConfig{},
222+
hostConfig, err := d.buildHostConfig()
223+
if err != nil {
224+
return errors.Wrap(err, "failed to build container host config")
225+
}
226+
227+
cont, err := d.dockerClient.ContainerCreate(ctx, d.buildContainerConfig(), hostConfig, &network.NetworkingConfig{},
233228
d.dumpContainerName(),
234229
)
235230
if err != nil {
@@ -368,15 +363,30 @@ func (d *DumpJob) getEnvironmentVariables() []string {
368363
return envs
369364
}
370365

366+
func (d *DumpJob) buildContainerConfig() *container.Config {
367+
return &container.Config{
368+
Env: d.getEnvironmentVariables(),
369+
Image: d.DockerImage,
370+
Healthcheck: health.GetConfig(),
371+
}
372+
}
373+
374+
func (d *DumpJob) buildHostConfig() (*container.HostConfig, error) {
375+
hostConfig := &container.HostConfig{
376+
Mounts: d.getMountVolumes(),
377+
NetworkMode: d.getContainerNetworkMode(),
378+
}
379+
380+
if err := tools.AddVolumesToHostConfig(hostConfig, d.globalCfg.DataDir); err != nil {
381+
return nil, err
382+
}
383+
384+
return hostConfig, nil
385+
}
386+
371387
// getMountVolumes returns a list of mount volumes.
372388
func (d *DumpJob) getMountVolumes() []mount.Mount {
373-
mounts := []mount.Mount{
374-
{
375-
Type: mount.TypeBind,
376-
Source: d.globalCfg.DataDir,
377-
Target: d.globalCfg.DataDir,
378-
},
379-
}
389+
mounts := d.dumper.GetMounts()
380390

381391
if d.DumpOptions.DumpFile != "" {
382392
mounts = append(mounts, mount.Mount{
@@ -386,9 +396,6 @@ func (d *DumpJob) getMountVolumes() []mount.Mount {
386396
})
387397
}
388398

389-
// Add dump specific mounts.
390-
mounts = append(mounts, d.dumper.GetMounts()...)
391-
392399
return mounts
393400
}
394401

‎pkg/retrieval/engine/postgres/initialize/logical/logical.go

Copy file name to clipboardExpand all lines: pkg/retrieval/engine/postgres/initialize/logical/logical.go
-1Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@ func recalculateStats(ctx context.Context, dockerClient *client.Client, contID s
2222
exec, err := dockerClient.ContainerExecCreate(ctx, contID, types.ExecConfig{
2323
AttachStdout: true,
2424
AttachStderr: true,
25-
Tty: true,
2625
Cmd: analyzeCmd,
2726
})
2827

‎pkg/retrieval/engine/postgres/initialize/logical/restore.go

Copy file name to clipboardExpand all lines: pkg/retrieval/engine/postgres/initialize/logical/restore.go
+35-21Lines changed: 35 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -119,28 +119,14 @@ func (r *RestoreJob) Run(ctx context.Context) (err error) {
119119
return errors.Wrap(err, "failed to scan image pulling response")
120120
}
121121

122+
hostConfig, err := r.buildHostConfig()
123+
if err != nil {
124+
return errors.Wrap(err, "failed to build container host config")
125+
}
126+
122127
cont, err := r.dockerClient.ContainerCreate(ctx,
123-
&container.Config{
124-
Env: []string{
125-
"PGDATA=" + r.globalCfg.DataDir,
126-
},
127-
Image: r.RestoreOptions.DockerImage,
128-
Healthcheck: health.GetConfig(),
129-
},
130-
&container.HostConfig{
131-
Mounts: []mount.Mount{
132-
{
133-
Type: mount.TypeBind,
134-
Source: r.RestoreOptions.DumpFile,
135-
Target: r.RestoreOptions.DumpFile,
136-
},
137-
{
138-
Type: mount.TypeBind,
139-
Source: r.globalCfg.DataDir,
140-
Target: r.globalCfg.DataDir,
141-
},
142-
},
143-
},
128+
r.buildContainerConfig(),
129+
hostConfig,
144130
&network.NetworkingConfig{},
145131
r.restoreContainerName(),
146132
)
@@ -208,6 +194,34 @@ func (r *RestoreJob) Run(ctx context.Context) (err error) {
208194
return nil
209195
}
210196

197+
func (r *RestoreJob) buildContainerConfig() *container.Config {
198+
return &container.Config{
199+
Env: []string{
200+
"PGDATA=" + r.globalCfg.DataDir,
201+
},
202+
Image: r.RestoreOptions.DockerImage,
203+
Healthcheck: health.GetConfig(),
204+
}
205+
}
206+
207+
func (r *RestoreJob) buildHostConfig() (*container.HostConfig, error) {
208+
hostConfig := &container.HostConfig{
209+
Mounts: []mount.Mount{
210+
{
211+
Type: mount.TypeBind,
212+
Source: r.RestoreOptions.DumpFile,
213+
Target: r.RestoreOptions.DumpFile,
214+
},
215+
},
216+
}
217+
218+
if err := tools.AddVolumesToHostConfig(hostConfig, r.globalCfg.DataDir); err != nil {
219+
return nil, err
220+
}
221+
222+
return hostConfig, nil
223+
}
224+
211225
func (r *RestoreJob) markDatabase(ctx context.Context, contID string) error {
212226
dataStateAt, err := r.retrieveDataStateAt(ctx, contID)
213227
if err != nil {

‎pkg/retrieval/engine/postgres/initialize/physical/physical.go

Copy file name to clipboardExpand all lines: pkg/retrieval/engine/postgres/initialize/physical/physical.go
+24-22Lines changed: 24 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -236,14 +236,14 @@ func (r *RestoreJob) Run(ctx context.Context) (err error) {
236236
}
237237

238238
func (r *RestoreJob) startContainer(ctx context.Context, containerName string) (string, error) {
239+
hostConfig, err := r.buildHostConfig()
240+
if err != nil {
241+
return "", errors.Wrap(err, "failed to build container host config")
242+
}
243+
239244
syncInstance, err := r.dockerClient.ContainerCreate(ctx,
240-
&container.Config{
241-
Env: r.getEnvironmentVariables(),
242-
Image: r.DockerImage,
243-
},
244-
&container.HostConfig{
245-
Mounts: r.getMountVolumes(),
246-
},
245+
r.buildContainerConfig(),
246+
hostConfig,
247247
&network.NetworkingConfig{},
248248
containerName,
249249
)
@@ -355,21 +355,23 @@ func (r *RestoreJob) getEnvironmentVariables() []string {
355355
return envVariables
356356
}
357357

358-
func (r *RestoreJob) getMountVolumes() []mount.Mount {
359-
mounts := append(
360-
[]mount.Mount{
361-
{
362-
Type: mount.TypeBind,
363-
Source: r.globalCfg.DataDir,
364-
Target: r.globalCfg.DataDir,
365-
BindOptions: &mount.BindOptions{
366-
Propagation: mount.PropagationRShared,
367-
},
368-
},
369-
},
370-
r.restorer.GetMounts()...)
371-
372-
return mounts
358+
func (r *RestoreJob) buildContainerConfig() *container.Config {
359+
return &container.Config{
360+
Env: r.getEnvironmentVariables(),
361+
Image: r.DockerImage,
362+
}
363+
}
364+
365+
func (r *RestoreJob) buildHostConfig() (*container.HostConfig, error) {
366+
hostConfig := &container.HostConfig{
367+
Mounts: r.restorer.GetMounts(),
368+
}
369+
370+
if err := tools.AddVolumesToHostConfig(hostConfig, r.globalCfg.DataDir); err != nil {
371+
return nil, err
372+
}
373+
374+
return hostConfig, nil
373375
}
374376

375377
func (r *RestoreJob) markDatabaseData() error {

‎pkg/retrieval/engine/postgres/initialize/snapshot/physical.go

Copy file name to clipboardExpand all lines: pkg/retrieval/engine/postgres/initialize/snapshot/physical.go
+32-25Lines changed: 32 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@ import (
2121
"github.com/AlekSi/pointer"
2222
"github.com/docker/docker/api/types"
2323
"github.com/docker/docker/api/types/container"
24-
"github.com/docker/docker/api/types/mount"
2524
"github.com/docker/docker/api/types/network"
2625
"github.com/docker/docker/client"
2726
"github.com/pkg/errors"
@@ -306,6 +305,11 @@ func (p *PhysicalInitial) promoteInstance(ctx context.Context, clonePath string)
306305
return errors.Wrap(err, "failed to adjust recovery configuration")
307306
}
308307

308+
hostConfig, err := p.buildHostConfig(clonePath)
309+
if err != nil {
310+
return errors.Wrap(err, "failed to build container host config")
311+
}
312+
309313
promoteImage := fmt.Sprintf("postgresai/sync-instance:%s", pgVersion)
310314

311315
if err := tools.PullImage(ctx, p.dockerClient, promoteImage); err != nil {
@@ -314,30 +318,8 @@ func (p *PhysicalInitial) promoteInstance(ctx context.Context, clonePath string)
314318

315319
// Run promotion container.
316320
cont, err := p.dockerClient.ContainerCreate(ctx,
317-
&container.Config{
318-
Labels: map[string]string{"label": "dblab_control"},
319-
Env: []string{
320-
"PGDATA=" + clonePath,
321-
"POSTGRES_HOST_AUTH_METHOD=trust",
322-
},
323-
Image: promoteImage,
324-
Healthcheck: health.GetConfig(
325-
health.OptionInterval(hcPromoteInterval),
326-
health.OptionRetries(hcPromoteRetries),
327-
),
328-
},
329-
&container.HostConfig{
330-
Mounts: []mount.Mount{
331-
{
332-
Type: mount.TypeBind,
333-
Source: clonePath,
334-
Target: clonePath,
335-
BindOptions: &mount.BindOptions{
336-
Propagation: mount.PropagationRShared,
337-
},
338-
},
339-
},
340-
},
321+
p.buildContainerConfig(clonePath, promoteImage),
322+
hostConfig,
341323
&network.NetworkingConfig{},
342324
p.promoteContainerName(),
343325
)
@@ -470,6 +452,31 @@ func (p *PhysicalInitial) adjustRecoveryConfiguration(pgVersion, clonePGDataDir
470452
return nil
471453
}
472454

455+
func (p *PhysicalInitial) buildContainerConfig(clonePath, promoteImage string) *container.Config {
456+
return &container.Config{
457+
Labels: map[string]string{"label": "dblab_control"},
458+
Env: []string{
459+
"PGDATA=" + clonePath,
460+
"POSTGRES_HOST_AUTH_METHOD=trust",
461+
},
462+
Image: promoteImage,
463+
Healthcheck: health.GetConfig(
464+
health.OptionInterval(hcPromoteInterval),
465+
health.OptionRetries(hcPromoteRetries),
466+
),
467+
}
468+
}
469+
470+
func (p *PhysicalInitial) buildHostConfig(clonePath string) (*container.HostConfig, error) {
471+
hostConfig := &container.HostConfig{}
472+
473+
if err := tools.AddVolumesToHostConfig(hostConfig, clonePath); err != nil {
474+
return nil, err
475+
}
476+
477+
return hostConfig, nil
478+
}
479+
473480
func (p *PhysicalInitial) checkRecovery(ctx context.Context, containerID string) (string, error) {
474481
checkRecoveryCmd := []string{"psql", "-U", defaults.Username, "-XAtc", "select pg_is_in_recovery()"}
475482
log.Msg("Check recovery command", checkRecoveryCmd)

‎pkg/retrieval/engine/postgres/initialize/tools/tools.go

Copy file name to clipboardExpand all lines: pkg/retrieval/engine/postgres/initialize/tools/tools.go
+25Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,12 @@ import (
1919

2020
"github.com/AlekSi/pointer"
2121
"github.com/docker/docker/api/types"
22+
"github.com/docker/docker/api/types/container"
23+
"github.com/docker/docker/api/types/mount"
2224
"github.com/docker/docker/client"
2325
"github.com/docker/docker/pkg/stdcopy"
2426
"github.com/pkg/errors"
27+
"github.com/shirou/gopsutil/host"
2528

2629
"gitlab.com/postgres-ai/database-lab/pkg/log"
2730
)
@@ -74,6 +77,28 @@ func DetectPGVersion(dataDir string) (string, error) {
7477
return string(bytes.TrimSpace(version)), nil
7578
}
7679

80+
// AddVolumesToHostConfig adds volumes to container host configuration depends on process environment.
81+
func AddVolumesToHostConfig(hostConfig *container.HostConfig, dataDir string) error {
82+
hostInfo, err := host.Info()
83+
if err != nil {
84+
return errors.Wrap(err, "failed to get host info")
85+
}
86+
87+
log.Dbg("Virtualization system: ", hostInfo.VirtualizationSystem)
88+
89+
if hostInfo.VirtualizationRole == "guest" {
90+
hostConfig.VolumesFrom = []string{hostInfo.Hostname}
91+
} else {
92+
hostConfig.Mounts = append(hostConfig.Mounts, mount.Mount{
93+
Type: mount.TypeBind,
94+
Source: dataDir,
95+
Target: dataDir,
96+
})
97+
}
98+
99+
return nil
100+
}
101+
77102
// InspectCommandResponse inspects success of command execution.
78103
func InspectCommandResponse(ctx context.Context, dockerClient *client.Client, containerID, commandID string) error {
79104
inspect, err := dockerClient.ContainerExecInspect(ctx, commandID)

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.