Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit aea3573

Browse filesBrowse files
agneumagneum
committed
feat: create the initial snapshot (physical case) (#140)
1 parent 18ee555 commit aea3573
Copy full SHA for aea3573

File tree

25 files changed

+861
-188
lines changed
Filter options

25 files changed

+861
-188
lines changed

‎cmd/database-lab/main.go

Copy file name to clipboardExpand all lines: cmd/database-lab/main.go
+1-1Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ func main() {
8383
}
8484

8585
// Create a new retrieval service to prepare a data directory and start snapshotting.
86-
retrievalSvc, err := retrieval.New(cfg, dockerCLI, provisionSvc)
86+
retrievalSvc, err := retrieval.New(cfg, dockerCLI, provisionSvc.ThinCloneManager())
8787
if err != nil {
8888
log.Fatal("Failed to build a retrieval service:", err)
8989
}

‎configs/config.sample.yml

Copy file name to clipboardExpand all lines: configs/config.sample.yml
+7-1Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -183,6 +183,12 @@ retrieval:
183183
- name: logical-snapshot
184184
options:
185185
# It is possible to define a pre-precessing script.
186-
# preprocessingScript: "/tmp/scripts/custom.sh"
186+
# preprocessingScript: "/tmp/scripts/custom.sh"
187+
188+
- name: physical-snapshot
189+
options:
190+
promote: true
191+
# It is possible to define a pre-precessing script.
192+
# preprocessingScript: "/tmp/scripts/custom.sh"
187193

188194
debug: true

‎pkg/retrieval/dbmarker/dbmarker.go

Copy file name to clipboardExpand all lines: pkg/retrieval/dbmarker/dbmarker.go
+3-1Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,9 @@ const (
3838

3939
// LogicalDataType defines a logical data type.
4040
LogicalDataType = "logical"
41-
//PhysicalDataType = "physical"
41+
42+
// PhysicalDataType defines a physical data type.
43+
PhysicalDataType = "physical"
4244
)
4345

4446
// Init inits DB marker for the data directory.

‎pkg/retrieval/engine/engine.go

Copy file name to clipboardExpand all lines: pkg/retrieval/engine/engine.go
+4-3Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,14 +12,15 @@ import (
1212
"gitlab.com/postgres-ai/database-lab/pkg/config"
1313
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/components"
1414
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/engine/postgres"
15-
"gitlab.com/postgres-ai/database-lab/pkg/services/provision"
15+
"gitlab.com/postgres-ai/database-lab/pkg/services/provision/thinclones"
1616
)
1717

1818
// StageBuilder provides a new stage builder.
19-
func StageBuilder(globalCfg *config.Global, dockerCli *client.Client, prov provision.Provision) (components.StageBuilder, error) {
19+
func StageBuilder(globalCfg *config.Global, dockerCli *client.Client,
20+
cloneManager thinclones.Manager) (components.StageBuilder, error) {
2021
switch globalCfg.Engine {
2122
case postgres.EngineType:
22-
return postgres.NewStageBuilder(globalCfg, dockerCli, prov), nil
23+
return postgres.NewStageBuilder(globalCfg, dockerCli, cloneManager), nil
2324

2425
default:
2526
return nil, errors.New("failed to get engine")

‎pkg/retrieval/engine/postgres/initialize/logical/dump.go

Copy file name to clipboardExpand all lines: pkg/retrieval/engine/postgres/initialize/logical/dump.go
+11-54Lines changed: 11 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
package logical
66

77
import (
8-
"bytes"
98
"context"
109
"fmt"
1110
"io"
@@ -15,20 +14,20 @@ import (
1514
"strings"
1615
"time"
1716

18-
"github.com/AlekSi/pointer"
1917
"github.com/docker/docker/api/types"
2018
"github.com/docker/docker/api/types/container"
2119
"github.com/docker/docker/api/types/mount"
2220
"github.com/docker/docker/api/types/network"
2321
"github.com/docker/docker/client"
24-
"github.com/docker/docker/pkg/stdcopy"
2522
"github.com/pkg/errors"
2623

2724
dblabCfg "gitlab.com/postgres-ai/database-lab/pkg/config"
2825
"gitlab.com/postgres-ai/database-lab/pkg/log"
2926
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/config"
3027
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/dbmarker"
3128
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/engine/postgres/initialize/tools"
29+
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/engine/postgres/initialize/tools/defaults"
30+
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/engine/postgres/initialize/tools/health"
3231
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/options"
3332
)
3433

@@ -37,9 +36,8 @@ const (
3736
DumpJobType = "logical-dump"
3837

3938
// Defines dump options.
40-
dumpContainerName = "retrieval_logical_dump"
41-
dumpContainerDir = "/tmp"
42-
dumpContainerStopTimeout = 10 * time.Second
39+
dumpContainerName = "retrieval_logical_dump"
40+
dumpContainerDir = "/tmp"
4341

4442
// Defines dump source types.
4543
sourceTypeLocal = "local"
@@ -159,11 +157,11 @@ Either set 'numberOfJobs' equals to 1 or disable the restore section`)
159157
func (d *DumpJob) setDefaults() {
160158
// TODO: Default yaml values in tags.
161159
if d.DumpOptions.Source.Connection.Port == 0 {
162-
d.DumpOptions.Source.Connection.Port = defaultPort
160+
d.DumpOptions.Source.Connection.Port = defaults.Port
163161
}
164162

165163
if d.DumpOptions.Source.Connection.Username == "" {
166-
d.DumpOptions.Source.Connection.Username = defaultUsername
164+
d.DumpOptions.Source.Connection.Username = defaults.Username
167165
}
168166

169167
if d.DumpOptions.ParallelJobs == 0 {
@@ -222,7 +220,7 @@ func (d *DumpJob) Run(ctx context.Context) error {
222220
&container.Config{
223221
Env: d.getEnvironmentVariables(),
224222
Image: d.DockerImage,
225-
Healthcheck: getContainerHealthConfig(),
223+
Healthcheck: health.GetConfig(),
226224
},
227225
&container.HostConfig{
228226
Mounts: d.getMountVolumes(),
@@ -237,21 +235,7 @@ func (d *DumpJob) Run(ctx context.Context) error {
237235
return errors.Wrap(err, "failed to create container")
238236
}
239237

240-
defer func() {
241-
if err := d.dockerClient.ContainerStop(ctx, cont.ID, pointer.ToDuration(dumpContainerStopTimeout)); err != nil {
242-
log.Err("Failed to stop a dump container: ", err)
243-
}
244-
245-
if err := d.dockerClient.ContainerRemove(ctx, cont.ID, types.ContainerRemoveOptions{
246-
Force: true,
247-
}); err != nil {
248-
log.Err("Failed to remove container: ", err)
249-
250-
return
251-
}
252-
253-
log.Msg(fmt.Sprintf("Stop container: %s. ID: %v", dumpContainerName, cont.ID))
254-
}()
238+
defer tools.RemoveContainer(ctx, d.dockerClient, cont.ID, tools.StopTimeout)
255239

256240
if err := d.dockerClient.ContainerStart(ctx, cont.ID, types.ContainerStartOptions{}); err != nil {
257241
return errors.Wrap(err, "failed to start container")
@@ -341,34 +325,7 @@ func (d *DumpJob) performDumpCommand(ctx context.Context, cmdOutput io.Writer, c
341325
}
342326
defer execAttach.Close()
343327

344-
// read the cmd output
345-
var errBuf bytes.Buffer
346-
347-
outputDone := make(chan error)
348-
349-
go func() {
350-
// StdCopy de-multiplexes the stream into two writers.
351-
_, err = stdcopy.StdCopy(cmdOutput, &errBuf, execAttach.Reader)
352-
outputDone <- err
353-
}()
354-
355-
select {
356-
case err := <-outputDone:
357-
if err != nil {
358-
return errors.Wrap(err, "failed to copy output")
359-
}
360-
361-
break
362-
363-
case <-ctx.Done():
364-
return ctx.Err()
365-
}
366-
367-
if errBuf.Len() > 0 {
368-
return errors.New(errBuf.String())
369-
}
370-
371-
return nil
328+
return tools.ProcessAttachResponse(ctx, execAttach.Reader, cmdOutput)
372329
}
373330

374331
func (d *DumpJob) getDumpContainerPath() string {
@@ -381,7 +338,7 @@ func (d *DumpJob) getEnvironmentVariables() []string {
381338
"POSTGRES_HOST_AUTH_METHOD=trust",
382339
}
383340

384-
if d.DumpOptions.Source.Type == sourceTypeLocal && d.DumpOptions.Source.Connection.Port == defaultPort {
341+
if d.DumpOptions.Source.Type == sourceTypeLocal && d.DumpOptions.Source.Connection.Port == defaults.Port {
385342
envs = append(envs, "PGPORT="+strconv.Itoa(reservePort))
386343
}
387344

@@ -467,7 +424,7 @@ func (d *DumpJob) buildLogicalDumpCommand() []string {
467424
}
468425

469426
func (d *DumpJob) buildLogicalRestoreCommand() []string {
470-
restoreCmd := []string{"|", "pg_restore", "-U", defaultUsername, "-C", "-d", defaultDBName, "--no-privileges"}
427+
restoreCmd := []string{"|", "pg_restore", "-U", defaults.Username, "-C", "-d", defaults.DBName, "--no-privileges"}
471428

472429
if d.Restore.ForceInit {
473430
restoreCmd = append(restoreCmd, "--clean", "--if-exists")

‎pkg/retrieval/engine/postgres/initialize/logical/logical.go

Copy file name to clipboardExpand all lines: pkg/retrieval/engine/postgres/initialize/logical/logical.go
-29Lines changed: 0 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -4,32 +4,3 @@
44

55
// Package logical provides jobs for logical initial operations.
66
package logical
7-
8-
import (
9-
"time"
10-
11-
"github.com/docker/docker/api/types/container"
12-
)
13-
14-
const (
15-
// Default values.
16-
defaultPort = 5432
17-
defaultUsername = "postgres"
18-
defaultDBName = "postgres"
19-
20-
// Defines container health check options.
21-
hcInterval = 5 * time.Second
22-
hcTimeout = 2 * time.Second
23-
hcStartPeriod = 3 * time.Second
24-
hcRetries = 5
25-
)
26-
27-
func getContainerHealthConfig() *container.HealthConfig {
28-
return &container.HealthConfig{
29-
Test: []string{"CMD-SHELL", "pg_isready -U " + defaultUsername},
30-
Interval: hcInterval,
31-
Timeout: hcTimeout,
32-
StartPeriod: hcStartPeriod,
33-
Retries: hcRetries,
34-
}
35-
}

‎pkg/retrieval/engine/postgres/initialize/logical/restore.go

Copy file name to clipboardExpand all lines: pkg/retrieval/engine/postgres/initialize/logical/restore.go
+6-19Lines changed: 6 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@ import (
1111
"strconv"
1212
"strings"
1313

14-
"github.com/AlekSi/pointer"
1514
"github.com/docker/docker/api/types"
1615
"github.com/docker/docker/api/types/container"
1716
"github.com/docker/docker/api/types/mount"
@@ -24,6 +23,8 @@ import (
2423
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/config"
2524
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/dbmarker"
2625
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/engine/postgres/initialize/tools"
26+
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/engine/postgres/initialize/tools/defaults"
27+
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/engine/postgres/initialize/tools/health"
2728
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/options"
2829
)
2930

@@ -121,7 +122,7 @@ func (r *RestoreJob) Run(ctx context.Context) error {
121122
"PGDATA=" + pgDataContainerDir,
122123
},
123124
Image: r.RestoreOptions.DockerImage,
124-
Healthcheck: getContainerHealthConfig(),
125+
Healthcheck: health.GetConfig(),
125126
},
126127
&container.HostConfig{
127128
Mounts: []mount.Mount{
@@ -144,21 +145,7 @@ func (r *RestoreJob) Run(ctx context.Context) error {
144145
return errors.Wrap(err, "failed to create container")
145146
}
146147

147-
defer func() {
148-
if err := r.dockerClient.ContainerStop(ctx, cont.ID, pointer.ToDuration(dumpContainerStopTimeout)); err != nil {
149-
log.Err("Failed to stop a dump container: ", err)
150-
}
151-
152-
if err := r.dockerClient.ContainerRemove(ctx, cont.ID, types.ContainerRemoveOptions{
153-
Force: true,
154-
}); err != nil {
155-
log.Err("Failed to remove container: ", err)
156-
157-
return
158-
}
159-
160-
log.Msg(fmt.Sprintf("Stop container: %s. ID: %v", restoreContainerName, cont.ID))
161-
}()
148+
defer tools.RemoveContainer(ctx, r.dockerClient, cont.ID, tools.StopTimeout)
162149

163150
if err := r.dockerClient.ContainerStart(ctx, cont.ID, types.ContainerStartOptions{}); err != nil {
164151
return errors.Wrap(err, "failed to start a container")
@@ -251,10 +238,10 @@ func (r *RestoreJob) retrieveDataStateAt(ctx context.Context, contID string) (st
251238
}
252239

253240
func (r *RestoreJob) buildLogicalRestoreCommand() []string {
254-
restoreCmd := []string{"pg_restore", "-U", defaultUsername, "-C"}
241+
restoreCmd := []string{"pg_restore", "-U", defaults.Username, "-C"}
255242

256243
if r.ForceInit {
257-
restoreCmd = append(restoreCmd, "-d", defaultDBName, "--clean", "--if-exists")
244+
restoreCmd = append(restoreCmd, "-d", defaults.DBName, "--clean", "--if-exists")
258245
} else {
259246
restoreCmd = append(restoreCmd, "-d", r.RestoreOptions.DBName)
260247
}

‎pkg/retrieval/engine/postgres/initialize/physical/physical.go

Copy file name to clipboardExpand all lines: pkg/retrieval/engine/postgres/initialize/physical/physical.go
+16-1Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121
dblabCfg "gitlab.com/postgres-ai/database-lab/pkg/config"
2222
"gitlab.com/postgres-ai/database-lab/pkg/log"
2323
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/config"
24+
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/dbmarker"
2425
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/engine/postgres/initialize/tools"
2526
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/options"
2627
)
@@ -38,6 +39,7 @@ type RestoreJob struct {
3839
name string
3940
dockerClient *client.Client
4041
globalCfg *dblabCfg.Global
42+
dbMarker *dbmarker.Marker
4143
restorer restorer
4244
CopyOptions
4345
}
@@ -64,11 +66,12 @@ type restorer interface {
6466
}
6567

6668
// NewJob creates a new physical restore job.
67-
func NewJob(cfg config.JobConfig, docker *client.Client, global *dblabCfg.Global) (*RestoreJob, error) {
69+
func NewJob(cfg config.JobConfig, docker *client.Client, global *dblabCfg.Global, marker *dbmarker.Marker) (*RestoreJob, error) {
6870
physicalJob := &RestoreJob{
6971
name: cfg.Name,
7072
dockerClient: docker,
7173
globalCfg: global,
74+
dbMarker: marker,
7275
}
7376

7477
if err := options.Unmarshal(cfg.Options, &physicalJob.CopyOptions); err != nil {
@@ -181,6 +184,10 @@ func (r *RestoreJob) Run(ctx context.Context) error {
181184

182185
log.Msg("Restoring job has been finished")
183186

187+
if err := r.markDatabaseData(); err != nil {
188+
log.Err("Failed to mark database data: ", err)
189+
}
190+
184191
return nil
185192
}
186193

@@ -211,6 +218,14 @@ func (r *RestoreJob) getMountVolumes() []mount.Mount {
211218
return mounts
212219
}
213220

221+
func (r *RestoreJob) markDatabaseData() error {
222+
if err := r.dbMarker.CreateConfig(); err != nil {
223+
return errors.Wrap(err, "failed to create a DBMarker config of the database")
224+
}
225+
226+
return r.dbMarker.SaveConfig(&dbmarker.Config{DataType: dbmarker.PhysicalDataType})
227+
}
228+
214229
func waitForCommandResponse(ctx context.Context, attachResponse types.HijackedResponse) error {
215230
waitCommandCh := make(chan struct{})
216231

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.