Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit ef187f8

Browse filesBrowse files
agneumagneum
committed
fix: pre-release Docker container issues (#152)
1 parent d18a294 commit ef187f8
Copy full SHA for ef187f8

File tree

7 files changed

+140
-73
lines changed
Filter options

7 files changed

+140
-73
lines changed

‎pkg/retrieval/engine/postgres/initialize/logical/dump.go

Copy file name to clipboardExpand all lines: pkg/retrieval/engine/postgres/initialize/logical/dump.go
+25-11Lines changed: 25 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,7 @@ const (
3636
DumpJobType = "logical-dump"
3737

3838
// Defines dump options.
39-
dumpContainerName = "retrieval_logical_dump"
40-
dumpContainerDir = "/tmp"
39+
dumpContainerPrefix = "dblab_ld_"
4140

4241
// Defines dump source types.
4342
sourceTypeLocal = "local"
@@ -190,13 +189,17 @@ func (d *DumpJob) setupDumper() error {
190189
return errors.Errorf("unknown source type given: %v", d.Source.Type)
191190
}
192191

192+
func (d *DumpJob) dumpContainerName() string {
193+
return dumpContainerPrefix + d.globalCfg.InstanceID
194+
}
195+
193196
// Name returns a name of the job.
194197
func (d *DumpJob) Name() string {
195198
return d.name
196199
}
197200

198201
// Run starts the job.
199-
func (d *DumpJob) Run(ctx context.Context) error {
202+
func (d *DumpJob) Run(ctx context.Context) (err error) {
200203
log.Msg(fmt.Sprintf("Run job: %s. Options: %v", d.Name(), d.DumpOptions))
201204

202205
isEmpty, err := tools.IsEmptyDirectory(d.globalCfg.DataDir)
@@ -212,6 +215,10 @@ func (d *DumpJob) Run(ctx context.Context) error {
212215
log.Msg("The data directory is not empty. Existing data may be overwritten.")
213216
}
214217

218+
if err := tools.PullImage(ctx, d.dockerClient, d.DockerImage); err != nil {
219+
return errors.Wrap(err, "failed to scan pulling image response")
220+
}
221+
215222
cont, err := d.dockerClient.ContainerCreate(ctx,
216223
&container.Config{
217224
Env: d.getEnvironmentVariables(),
@@ -223,21 +230,27 @@ func (d *DumpJob) Run(ctx context.Context) error {
223230
NetworkMode: d.getContainerNetworkMode(),
224231
},
225232
&network.NetworkingConfig{},
226-
dumpContainerName,
233+
d.dumpContainerName(),
227234
)
228235
if err != nil {
229236
log.Err(err)
230237

231-
return errors.Wrap(err, "failed to create container")
238+
return errors.Wrapf(err, "failed to create container %q", d.dumpContainerName())
232239
}
233240

234241
defer tools.RemoveContainer(ctx, d.dockerClient, cont.ID, tools.StopTimeout)
235242

243+
defer func() {
244+
if err != nil {
245+
tools.PrintContainerLogs(ctx, d.dockerClient, d.dumpContainerName())
246+
}
247+
}()
248+
236249
if err := d.dockerClient.ContainerStart(ctx, cont.ID, types.ContainerStartOptions{}); err != nil {
237-
return errors.Wrap(err, "failed to start container")
250+
return errors.Wrapf(err, "failed to start container %q", d.dumpContainerName())
238251
}
239252

240-
log.Msg(fmt.Sprintf("Running container: %s. ID: %v", dumpContainerName, cont.ID))
253+
log.Msg(fmt.Sprintf("Running container: %s. ID: %v", d.dumpContainerName(), cont.ID))
241254

242255
if err := tools.CheckContainerReadiness(ctx, d.dockerClient, cont.ID); err != nil {
243256
return errors.Wrap(err, "failed to readiness check")
@@ -258,7 +271,7 @@ func (d *DumpJob) Run(ctx context.Context) error {
258271
})
259272

260273
if err != nil {
261-
return errors.Wrap(err, "failed to create an exec command")
274+
return errors.Wrap(err, "failed to create dump command")
262275
}
263276

264277
if len(d.Partial.Tables) > 0 {
@@ -343,11 +356,12 @@ func (d *DumpJob) getDumpContainerPath() string {
343356

344357
func (d *DumpJob) getEnvironmentVariables() []string {
345358
envs := []string{
346-
"PGDATA=" + pgDataContainerDir,
359+
"PGDATA=" + d.globalCfg.DataDir,
347360
"POSTGRES_HOST_AUTH_METHOD=trust",
348361
}
349362

350363
if d.DumpOptions.Source.Type == sourceTypeLocal && d.DumpOptions.Source.Connection.Port == defaults.Port {
364+
log.Msg(fmt.Sprintf("The default PostgreSQL port is busy, trying to use an alternative one: %d", reservePort))
351365
envs = append(envs, "PGPORT="+strconv.Itoa(reservePort))
352366
}
353367

@@ -360,15 +374,15 @@ func (d *DumpJob) getMountVolumes() []mount.Mount {
360374
{
361375
Type: mount.TypeBind,
362376
Source: d.globalCfg.DataDir,
363-
Target: pgDataContainerDir,
377+
Target: d.globalCfg.DataDir,
364378
},
365379
}
366380

367381
if d.DumpOptions.DumpFile != "" {
368382
mounts = append(mounts, mount.Mount{
369383
Type: mount.TypeBind,
370384
Source: filepath.Dir(d.DumpOptions.DumpFile),
371-
Target: dumpContainerDir,
385+
Target: filepath.Dir(d.DumpOptions.DumpFile),
372386
})
373387
}
374388

‎pkg/retrieval/engine/postgres/initialize/logical/restore.go

Copy file name to clipboardExpand all lines: pkg/retrieval/engine/postgres/initialize/logical/restore.go
+41-26Lines changed: 41 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -33,10 +33,8 @@ const (
3333
RestoreJobType = "logical-restore"
3434

3535
// const defines restore options.
36-
restoreContainerName = "retrieval_logical_restore"
37-
pgDataContainerDir = "/var/lib/postgresql/pgdata"
38-
dumpContainerFile = "/tmp/db.dump"
39-
defaultParallelJobs = 1
36+
restoreContainerPrefix = "dblab_lr_"
37+
defaultParallelJobs = 1
4038
)
4139

4240
// RestoreJob defines a logical restore job.
@@ -80,10 +78,6 @@ func NewJob(cfg config.JobConfig, docker *client.Client, globalCfg *dblabCfg.Glo
8078

8179
restoreJob.setDefaults()
8280

83-
if err := restoreJob.dbMarker.CreateConfig(); err != nil {
84-
return nil, errors.Wrap(err, "failed to create a DBMarker config of the database")
85-
}
86-
8781
return restoreJob, nil
8882
}
8983

@@ -94,32 +88,41 @@ func (r *RestoreJob) setDefaults() {
9488
}
9589
}
9690

91+
func (r *RestoreJob) restoreContainerName() string {
92+
return restoreContainerPrefix + r.globalCfg.InstanceID
93+
}
94+
9795
// Name returns a name of the job.
9896
func (r *RestoreJob) Name() string {
9997
return r.name
10098
}
10199

102100
// Run starts the job.
103-
func (r *RestoreJob) Run(ctx context.Context) error {
101+
func (r *RestoreJob) Run(ctx context.Context) (err error) {
104102
log.Msg(fmt.Sprintf("Run job: %s. Options: %v", r.Name(), r.RestoreOptions))
105103

106104
isEmpty, err := tools.IsEmptyDirectory(r.globalCfg.DataDir)
107105
if err != nil {
108-
return errors.Wrap(err, "failed to explore the data directory")
106+
return errors.Wrapf(err, "failed to explore the data directory %q", r.globalCfg.DataDir)
109107
}
110108

111109
if !isEmpty {
112110
if !r.ForceInit {
113-
return errors.New("the data directory is not empty. Use 'forceInit' or empty the data directory")
111+
return errors.Errorf("the data directory %q is not empty. Use 'forceInit' or empty the data directory",
112+
r.globalCfg.DataDir)
114113
}
115114

116-
log.Msg("The data directory is not empty. Existing data may be overwritten.")
115+
log.Msg(fmt.Sprintf("The data directory %q is not empty. Existing data may be overwritten.", r.globalCfg.DataDir))
116+
}
117+
118+
if err := tools.PullImage(ctx, r.dockerClient, r.RestoreOptions.DockerImage); err != nil {
119+
return errors.Wrap(err, "failed to scan image pulling response")
117120
}
118121

119122
cont, err := r.dockerClient.ContainerCreate(ctx,
120123
&container.Config{
121124
Env: []string{
122-
"PGDATA=" + pgDataContainerDir,
125+
"PGDATA=" + r.globalCfg.DataDir,
123126
},
124127
Image: r.RestoreOptions.DockerImage,
125128
Healthcheck: health.GetConfig(),
@@ -129,29 +132,35 @@ func (r *RestoreJob) Run(ctx context.Context) error {
129132
{
130133
Type: mount.TypeBind,
131134
Source: r.RestoreOptions.DumpFile,
132-
Target: dumpContainerFile,
135+
Target: r.RestoreOptions.DumpFile,
133136
},
134137
{
135138
Type: mount.TypeBind,
136139
Source: r.globalCfg.DataDir,
137-
Target: pgDataContainerDir,
140+
Target: r.globalCfg.DataDir,
138141
},
139142
},
140143
},
141144
&network.NetworkingConfig{},
142-
restoreContainerName,
145+
r.restoreContainerName(),
143146
)
144147
if err != nil {
145-
return errors.Wrap(err, "failed to create container")
148+
return errors.Wrapf(err, "failed to create container %q", r.restoreContainerName())
146149
}
147150

148151
defer tools.RemoveContainer(ctx, r.dockerClient, cont.ID, tools.StopTimeout)
149152

153+
defer func() {
154+
if err != nil {
155+
tools.PrintContainerLogs(ctx, r.dockerClient, r.restoreContainerName())
156+
}
157+
}()
158+
150159
if err := r.dockerClient.ContainerStart(ctx, cont.ID, types.ContainerStartOptions{}); err != nil {
151-
return errors.Wrap(err, "failed to start a container")
160+
return errors.Wrapf(err, "failed to start container %q", r.restoreContainerName())
152161
}
153162

154-
log.Msg(fmt.Sprintf("Running container: %s. ID: %v", restoreContainerName, cont.ID))
163+
log.Msg(fmt.Sprintf("Running container: %s. ID: %v", r.restoreContainerName(), cont.ID))
155164

156165
if err := r.markDatabase(ctx, cont.ID); err != nil {
157166
return errors.Wrap(err, "failed to mark the database")
@@ -172,19 +181,19 @@ func (r *RestoreJob) Run(ctx context.Context) error {
172181
})
173182

174183
if err != nil {
175-
return errors.Wrap(err, "failed to create an exec command")
184+
return errors.Wrap(err, "failed to create restore command")
176185
}
177186

178187
if len(r.Partial.Tables) > 0 {
179188
log.Msg("Partial restore will be run. Tables for restoring: ", strings.Join(r.Partial.Tables, ", "))
180189
}
181190

182191
if err := r.dockerClient.ContainerExecStart(ctx, execCommand.ID, types.ExecStartCheck{Tty: true}); err != nil {
183-
return errors.Wrap(err, "failed to run the exec command")
192+
return errors.Wrap(err, "failed to run restore command")
184193
}
185194

186195
if err := tools.InspectCommandResponse(ctx, r.dockerClient, cont.ID, execCommand.ID); err != nil {
187-
return errors.Wrap(err, "failed to exec the restore command")
196+
return errors.Wrap(err, "failed to exec restore command")
188197
}
189198

190199
if err := recalculateStats(ctx, r.dockerClient, cont.ID, buildAnalyzeCommand(Connection{
@@ -209,6 +218,10 @@ func (r *RestoreJob) markDatabase(ctx context.Context, contID string) error {
209218
r.dbMark.DataStateAt = dataStateAt
210219
}
211220

221+
if err := r.dbMarker.CreateConfig(); err != nil {
222+
return errors.Wrap(err, "failed to create a DBMarker config of the database")
223+
}
224+
212225
if err := r.dbMarker.SaveConfig(r.dbMark); err != nil {
213226
return errors.Wrap(err, "failed to mark the database")
214227
}
@@ -217,20 +230,22 @@ func (r *RestoreJob) markDatabase(ctx context.Context, contID string) error {
217230
}
218231

219232
func (r *RestoreJob) retrieveDataStateAt(ctx context.Context, contID string) (string, error) {
220-
restoreMetaCmd := []string{"sh", "-c", "pg_restore -l " + dumpContainerFile + " | head -n 10"}
233+
restoreMetaCmd := []string{"sh", "-c", "pg_restore -l " + r.RestoreOptions.DumpFile + " | head -n 10"}
234+
235+
log.Dbg("Running a restore metadata command: ", restoreMetaCmd)
221236

222237
execCommand, err := r.dockerClient.ContainerExecCreate(ctx, contID, types.ExecConfig{
223238
AttachStdout: true,
224239
AttachStderr: true,
225240
Cmd: restoreMetaCmd,
226241
})
227242
if err != nil {
228-
return "", errors.Wrap(err, "failed to create a restore meta command")
243+
return "", errors.Wrap(err, "failed to create a restore metadata command")
229244
}
230245

231246
execAttach, err := r.dockerClient.ContainerExecAttach(ctx, execCommand.ID, types.ExecStartCheck{})
232247
if err != nil {
233-
return "", errors.Wrap(err, "failed to exec a restore meta command")
248+
return "", errors.Wrap(err, "failed to exec a restore metadata command")
234249
}
235250

236251
defer execAttach.Close()
@@ -258,7 +273,7 @@ func (r *RestoreJob) buildLogicalRestoreCommand() []string {
258273
restoreCmd = append(restoreCmd, "-t", table)
259274
}
260275

261-
restoreCmd = append(restoreCmd, dumpContainerFile)
276+
restoreCmd = append(restoreCmd, r.RestoreOptions.DumpFile)
262277

263278
return restoreCmd
264279
}

‎pkg/retrieval/engine/postgres/initialize/logical/restore_test.go

Copy file name to clipboardExpand all lines: pkg/retrieval/engine/postgres/initialize/logical/restore_test.go
+3-1Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ func TestRestoreCommandBuilding(t *testing.T) {
2222
DBName: "testDB",
2323
ParallelJobs: 1,
2424
ForceInit: false,
25+
DumpFile: "/tmp/db.dump",
2526
},
2627
Command: []string{"pg_restore", "-U", "postgres", "-C", "-d", "testDB", "-j", "1", "/tmp/db.dump"},
2728
},
@@ -30,13 +31,14 @@ func TestRestoreCommandBuilding(t *testing.T) {
3031
ParallelJobs: 4,
3132
ForceInit: true,
3233
},
33-
Command: []string{"pg_restore", "-U", "postgres", "-C", "-d", "postgres", "--clean", "--if-exists", "-j", "4", "/tmp/db.dump"},
34+
Command: []string{"pg_restore", "-U", "postgres", "-C", "-d", "postgres", "--clean", "--if-exists", "-j", "4", ""},
3435
},
3536
{
3637
CopyOptions: RestoreOptions{
3738
DBName: "testDB",
3839
ParallelJobs: 1,
3940
Partial: Partial{Tables: []string{"test", "users"}},
41+
DumpFile: "/tmp/db.dump",
4042
},
4143
Command: []string{"pg_restore", "-U", "postgres", "-C", "-d", "testDB", "-j", "1", "-t", "test", "-t", "users", "/tmp/db.dump"},
4244
},

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.