Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit 691117a

Browse filesBrowse files
agneumagneum
committed
feat: create the initial snapshot (logical case) (#139)
1 parent 2438749 commit 691117a
Copy full SHA for 691117a

File tree

19 files changed

+461
-48
lines changed
Filter options

19 files changed

+461
-48
lines changed

‎.golangci.yml

Copy file name to clipboardExpand all lines: .golangci.yml
+1-1Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
run:
2-
timeout: 1m
2+
timeout: 2m
33
issues-exit-code: 1
44
tests: true
55
skip-dirs:

‎cmd/database-lab/main.go

Copy file name to clipboardExpand all lines: cmd/database-lab/main.go
+10-10Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -68,16 +68,22 @@ func main() {
6868
cfg.Provision.ModeLocal.DockerImage = opts.DockerImage
6969
}
7070

71+
ctx, cancel := context.WithCancel(context.Background())
72+
defer cancel()
73+
74+
// Create a cloning service to provision new clones.
75+
provisionSvc, err := provision.New(ctx, cfg.Provision)
76+
if err != nil {
77+
log.Fatalf(errors.WithMessage(err, `error in the "provision" section of the config`))
78+
}
79+
7180
dockerCLI, err := client.NewClientWithOpts(client.FromEnv)
7281
if err != nil {
7382
log.Fatal("Failed to create a Docker client:", err)
7483
}
7584

76-
ctx, cancel := context.WithCancel(context.Background())
77-
defer cancel()
78-
7985
// Create a new retrieval service to prepare a data directory and start snapshotting.
80-
retrievalSvc, err := retrieval.New(cfg, dockerCLI)
86+
retrievalSvc, err := retrieval.New(cfg, dockerCLI, provisionSvc)
8187
if err != nil {
8288
log.Fatal("Failed to build a retrieval service:", err)
8389
}
@@ -86,12 +92,6 @@ func main() {
8692
log.Fatal("Failed to run the data retrieval service:", err)
8793
}
8894

89-
// Create a cloning service to provision new clones.
90-
provisionSvc, err := provision.New(ctx, cfg.Provision)
91-
if err != nil {
92-
log.Fatalf(errors.WithMessage(err, `error in "provision" config`))
93-
}
94-
9595
cloningSvc, err := cloning.New(&cfg.Cloning, provisionSvc)
9696
if err != nil {
9797
log.Fatalf(errors.WithMessage(err, "failed to init a new cloning service"))

‎configs/config.sample.yml

Copy file name to clipboardExpand all lines: configs/config.sample.yml
+9-1Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,9 @@ retrieval:
116116
host: 127.0.0.1
117117
port: 5432
118118
username: postgres
119-
password: postgres # The environment variable PGPASSWORD can be used instead of this option.
119+
# Connection password. The environment variable PGPASSWORD can be used instead of this option.
120+
# The environment variable has a higher priority.
121+
password: postgres
120122

121123
# Optional definition of RDS data source.
122124
rds:
@@ -141,6 +143,7 @@ retrieval:
141143
- test
142144

143145
# The number of parallel jobs to get a dump.
146+
# It's ignored if "restore" is present because "pg_dump | pg_restore" is always single-threaded.
144147
parallelJobs: 1
145148

146149
# Options for direct restore to Database Lab Engine instance.
@@ -177,4 +180,9 @@ retrieval:
177180
backupName: LATEST
178181
credentialsFile: /tmp/sa.json # optional
179182

183+
- name: logical-snapshot
184+
options:
185+
# It is possible to define a pre-precessing script.
186+
# preprocessingScript: "/tmp/scripts/custom.sh"
187+
180188
debug: true

‎pkg/retrieval/dbmarker/dbmarker.go

Copy file name to clipboard
+107Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
/*
2+
2020 © Postgres.ai
3+
*/
4+
5+
// Package dbmarker provides a tool for marking database data.
6+
package dbmarker
7+
8+
import (
9+
"io/ioutil"
10+
"os"
11+
"path"
12+
13+
"github.com/pkg/errors"
14+
"gopkg.in/yaml.v2"
15+
)
16+
17+
// Marker marks database data depends on a retrieval process.
18+
type Marker struct {
19+
dataPath string
20+
}
21+
22+
// NewMarker creates a new DBMarker.
23+
func NewMarker(dataPath string) *Marker {
24+
return &Marker{
25+
dataPath: dataPath,
26+
}
27+
}
28+
29+
// Config describes marked data.
30+
type Config struct {
31+
DataStateAt string `yaml:"dataStateAt"`
32+
DataType string `yaml:"dataType"`
33+
}
34+
35+
const (
36+
configDir = ".dblab"
37+
configFilename = "dbmarker"
38+
39+
// LogicalDataType defines a logical data type.
40+
LogicalDataType = "logical"
41+
//PhysicalDataType = "physical"
42+
)
43+
44+
// Init inits DB marker for the data directory.
45+
func (m *Marker) initDBLabDirectory() error {
46+
dirname := path.Join(m.dataPath, configDir)
47+
if err := os.MkdirAll(dirname, 0755); err != nil {
48+
return errors.Wrapf(err, "cannot create a DBMarker directory %s", dirname)
49+
}
50+
51+
return nil
52+
}
53+
54+
// CreateConfig creates a new DBMarker config file.
55+
func (m *Marker) CreateConfig() error {
56+
if err := m.initDBLabDirectory(); err != nil {
57+
return errors.Wrap(err, "failed to init DBMarker")
58+
}
59+
60+
dbMarkerFile, err := os.OpenFile(m.buildFileName(), os.O_RDWR|os.O_CREATE, 0600)
61+
if err != nil {
62+
return err
63+
}
64+
65+
defer func() { _ = dbMarkerFile.Close() }()
66+
67+
return nil
68+
}
69+
70+
// GetConfig provides a loaded DBMarker config.
71+
func (m *Marker) GetConfig() (*Config, error) {
72+
configData, err := ioutil.ReadFile(m.buildFileName())
73+
if err != nil {
74+
return nil, err
75+
}
76+
77+
cfg := &Config{}
78+
79+
if len(configData) == 0 {
80+
return cfg, nil
81+
}
82+
83+
if err := yaml.Unmarshal(configData, cfg); err != nil {
84+
return nil, err
85+
}
86+
87+
return cfg, nil
88+
}
89+
90+
// SaveConfig stores a DBMarker config.
91+
func (m *Marker) SaveConfig(cfg *Config) error {
92+
configData, err := yaml.Marshal(cfg)
93+
if err != nil {
94+
return err
95+
}
96+
97+
if err := ioutil.WriteFile(m.buildFileName(), configData, 0600); err != nil {
98+
return err
99+
}
100+
101+
return nil
102+
}
103+
104+
// buildFileName builds a DBMarker config filename.
105+
func (m *Marker) buildFileName() string {
106+
return path.Join(m.dataPath, configDir, configFilename)
107+
}

‎pkg/retrieval/engine/engine.go

Copy file name to clipboardExpand all lines: pkg/retrieval/engine/engine.go
+3-2Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,13 +12,14 @@ import (
1212
"gitlab.com/postgres-ai/database-lab/pkg/config"
1313
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/components"
1414
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/engine/postgres"
15+
"gitlab.com/postgres-ai/database-lab/pkg/services/provision"
1516
)
1617

1718
// StageBuilder provides a new stage builder.
18-
func StageBuilder(globalCfg *config.Global, dockerCli *client.Client) (components.StageBuilder, error) {
19+
func StageBuilder(globalCfg *config.Global, dockerCli *client.Client, prov provision.Provision) (components.StageBuilder, error) {
1920
switch globalCfg.Engine {
2021
case postgres.EngineType:
21-
return postgres.NewStageBuilder(globalCfg, dockerCli), nil
22+
return postgres.NewStageBuilder(globalCfg, dockerCli, prov), nil
2223

2324
default:
2425
return nil, errors.New("failed to get engine")

‎pkg/retrieval/engine/postgres/initialize/logical/dump.go

Copy file name to clipboardExpand all lines: pkg/retrieval/engine/postgres/initialize/logical/dump.go
+32-6Lines changed: 32 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,9 @@ import (
1313
"path/filepath"
1414
"strconv"
1515
"strings"
16+
"time"
1617

18+
"github.com/AlekSi/pointer"
1719
"github.com/docker/docker/api/types"
1820
"github.com/docker/docker/api/types/container"
1921
"github.com/docker/docker/api/types/mount"
@@ -25,6 +27,7 @@ import (
2527
dblabCfg "gitlab.com/postgres-ai/database-lab/pkg/config"
2628
"gitlab.com/postgres-ai/database-lab/pkg/log"
2729
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/config"
30+
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/dbmarker"
2831
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/engine/postgres/initialize/tools"
2932
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/options"
3033
)
@@ -34,8 +37,9 @@ const (
3437
DumpJobType = "logical-dump"
3538

3639
// Defines dump options.
37-
dumpContainerName = "retriever_logical_dump"
38-
dumpContainerDir = "/tmp/dump"
40+
dumpContainerName = "retrieval_logical_dump"
41+
dumpContainerDir = "/tmp"
42+
dumpContainerStopTimeout = 10 * time.Second
3943

4044
// Defines dump source types.
4145
sourceTypeLocal = "local"
@@ -57,6 +61,8 @@ type DumpJob struct {
5761
globalCfg *dblabCfg.Global
5862
config dumpJobConfig
5963
dumper dumper
64+
dbMarker *dbmarker.Marker
65+
dbMark *dbmarker.Config
6066
DumpOptions
6167
}
6268

@@ -109,11 +115,15 @@ type DirectRestore struct {
109115
}
110116

111117
// NewDumpJob creates a new DumpJob.
112-
func NewDumpJob(cfg config.JobConfig, docker *client.Client, global *dblabCfg.Global) (*DumpJob, error) {
118+
func NewDumpJob(cfg config.JobConfig, docker *client.Client, global *dblabCfg.Global, marker *dbmarker.Marker) (*DumpJob, error) {
113119
dumpJob := &DumpJob{
114120
name: cfg.Name,
115121
dockerClient: docker,
116122
globalCfg: global,
123+
dbMarker: marker,
124+
dbMark: &dbmarker.Config{
125+
DataType: dbmarker.LogicalDataType,
126+
},
117127
}
118128

119129
if err := options.Unmarshal(cfg.Options, &dumpJob.DumpOptions); err != nil {
@@ -130,6 +140,10 @@ func NewDumpJob(cfg config.JobConfig, docker *client.Client, global *dblabCfg.Gl
130140
return nil, errors.Wrap(err, "failed to set up a dump helper")
131141
}
132142

143+
if err := dumpJob.dbMarker.CreateConfig(); err != nil {
144+
return nil, errors.Wrap(err, "failed to create a DBMarker config of the database")
145+
}
146+
133147
return dumpJob, nil
134148
}
135149

@@ -224,6 +238,10 @@ func (d *DumpJob) Run(ctx context.Context) error {
224238
}
225239

226240
defer func() {
241+
if err := d.dockerClient.ContainerStop(ctx, cont.ID, pointer.ToDuration(dumpContainerStopTimeout)); err != nil {
242+
log.Err("Failed to stop a dump container: ", err)
243+
}
244+
227245
if err := d.dockerClient.ContainerRemove(ctx, cont.ID, types.ContainerRemoveOptions{
228246
Force: true,
229247
}); err != nil {
@@ -292,6 +310,10 @@ func (d *DumpJob) Run(ctx context.Context) error {
292310
return errors.Wrap(err, "failed to exec the dump command")
293311
}
294312

313+
if err := d.dbMarker.SaveConfig(d.dbMark); err != nil {
314+
return errors.Wrap(err, "failed to mark the created dump")
315+
}
316+
295317
log.Msg("Dumping job has been finished")
296318

297319
return nil
@@ -309,6 +331,10 @@ func (d *DumpJob) setupConnectionOptions(ctx context.Context) error {
309331
}
310332

311333
func (d *DumpJob) performDumpCommand(ctx context.Context, cmdOutput io.Writer, commandID string) error {
334+
if d.DumpOptions.Restore != nil {
335+
d.dbMark.DataStateAt = time.Now().Format(tools.DataStateAtFormat)
336+
}
337+
312338
execAttach, err := d.dockerClient.ContainerExecAttach(ctx, commandID, types.ExecStartCheck{})
313339
if err != nil {
314340
return err
@@ -346,7 +372,7 @@ func (d *DumpJob) performDumpCommand(ctx context.Context, cmdOutput io.Writer, c
346372
}
347373

348374
func (d *DumpJob) getDumpContainerPath() string {
349-
return dumpContainerDir + strings.TrimPrefix(d.DumpFile, filepath.Dir(d.DumpFile))
375+
return d.DumpFile
350376
}
351377

352378
func (d *DumpJob) getEnvironmentVariables() []string {
@@ -413,7 +439,7 @@ func (d *DumpJob) getExecEnvironmentVariables() []string {
413439
}
414440

415441
func (d *DumpJob) buildLogicalDumpCommand() []string {
416-
dumpCmd := []string{"pg_dump", "-C"}
442+
dumpCmd := []string{"pg_dump", "-C", "-Fc"}
417443

418444
optionalArgs := map[string]string{
419445
"-h": d.config.db.Host,
@@ -441,7 +467,7 @@ func (d *DumpJob) buildLogicalDumpCommand() []string {
441467
}
442468

443469
func (d *DumpJob) buildLogicalRestoreCommand() []string {
444-
restoreCmd := []string{"-Fc", "|", "pg_restore", "-U", defaultUsername, "-C", "-d", defaultDBName, "--no-privileges"}
470+
restoreCmd := []string{"|", "pg_restore", "-U", defaultUsername, "-C", "-d", defaultDBName, "--no-privileges"}
445471

446472
if d.Restore.ForceInit {
447473
restoreCmd = append(restoreCmd, "--clean", "--if-exists")

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.