Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit 873d785

Browse filesBrowse files
agneumagneum
committed
Merge branch '201-asynchronous-sync-instance' into 'master'
feat: start sync instance asynchronously (#201) Closes #201 See merge request postgres-ai/database-lab!207
2 parents 7b2c323 + 36df49f commit 873d785
Copy full SHA for 873d785

File tree

Expand file treeCollapse file tree

6 files changed

+58
-30
lines changed
Filter options
Expand file treeCollapse file tree

6 files changed

+58
-30
lines changed

‎configs/config.example.physical_generic.yml

Copy file name to clipboardExpand all lines: configs/config.example.physical_generic.yml
+7Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -238,6 +238,13 @@ retrieval:
238238
# Limit defines how many snapshots should be hold.
239239
limit: 4
240240

241+
# Set environment variables here. See https://www.postgresql.org/docs/current/libpq-envars.html
242+
envs:
243+
PGUSER: "postgres"
244+
PGPASSWORD: "postgres"
245+
PGHOST: "source.hostname"
246+
PGPORT: 5432
247+
241248
cloning:
242249
# Host that will be specified in database connection info for all clones
243250
# Use public IP address if database connections are allowed from outside

‎configs/config.example.physical_walg.yml

Copy file name to clipboardExpand all lines: configs/config.example.physical_walg.yml
+5Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -228,6 +228,11 @@ retrieval:
228228
# Limit defines how many snapshots should be hold.
229229
limit: 4
230230

231+
# Passes custom environment variables to the promotion Docker container.
232+
envs:
233+
WALG_GS_PREFIX: "gs://{BUCKET}/{SCOPE}"
234+
GOOGLE_APPLICATION_CREDENTIALS: "/tmp/sa.json"
235+
231236
cloning:
232237
# Host that will be specified in database connection info for all clones
233238
# Use public IP address if database connections are allowed from outside

‎pkg/retrieval/engine/postgres/physical/physical.go

Copy file name to clipboardExpand all lines: pkg/retrieval/engine/postgres/physical/physical.go
+8-21Lines changed: 8 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -153,10 +153,12 @@ func (r *RestoreJob) Run(ctx context.Context) (err error) {
153153

154154
defer func() {
155155
if err == nil && r.CopyOptions.Sync.Enabled {
156-
if syncErr := r.runSyncInstance(ctx); syncErr != nil {
157-
log.Err("Failed to run sync instance: ", syncErr)
158-
tools.StopContainer(ctx, r.dockerClient, r.syncInstanceName(), time.Second)
159-
}
156+
go func() {
157+
if syncErr := r.runSyncInstance(ctx); syncErr != nil {
158+
log.Err("Failed to run sync instance: ", syncErr)
159+
tools.StopContainer(ctx, r.dockerClient, r.syncInstanceName(), time.Second)
160+
}
161+
}()
160162
}
161163
}()
162164

@@ -438,7 +440,7 @@ func (r *RestoreJob) adjustRecoveryConfiguration(pgVersion, pgDataDir string) er
438440
recoveryFilename = "recovery.conf"
439441
}
440442

441-
return appendConfigFile(path.Join(pgDataDir, recoveryFilename), r.restorer.GetRecoveryConfig())
443+
return fs.AppendFile(path.Join(pgDataDir, recoveryFilename), r.restorer.GetRecoveryConfig())
442444
}
443445

444446
func (r *RestoreJob) applyInitParams(ctx context.Context, contID, pgVersion, dataDir string) error {
@@ -521,20 +523,5 @@ func appendInitConfigs(initConfiguration map[string]string, pgDataDir string) er
521523
buffer.WriteString(fmt.Sprintf("%s = '%s'\n", key, value))
522524
}
523525

524-
return appendConfigFile(path.Join(pgDataDir, "postgresql.conf"), buffer.Bytes())
525-
}
526-
527-
func appendConfigFile(file string, data []byte) error {
528-
configFile, err := os.OpenFile(file, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
529-
if err != nil {
530-
return err
531-
}
532-
533-
defer func() { _ = configFile.Close() }()
534-
535-
if _, err := configFile.Write(data); err != nil {
536-
return err
537-
}
538-
539-
return nil
526+
return fs.AppendFile(path.Join(pgDataDir, "postgresql.conf"), buffer.Bytes())
540527
}

‎pkg/retrieval/engine/postgres/snapshot/physical.go

Copy file name to clipboardExpand all lines: pkg/retrieval/engine/postgres/snapshot/physical.go
+21-8Lines changed: 21 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ import (
99
"bytes"
1010
"context"
1111
"fmt"
12-
"io/ioutil"
1312
"os"
1413
"path"
1514
"strconv"
@@ -31,6 +30,7 @@ import (
3130
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/engine/postgres/tools"
3231
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/engine/postgres/tools/cont"
3332
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/engine/postgres/tools/defaults"
33+
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/engine/postgres/tools/fs"
3434
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/engine/postgres/tools/health"
3535
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/options"
3636
"gitlab.com/postgres-ai/database-lab/pkg/services/provision/databases/postgres/configuration"
@@ -80,6 +80,7 @@ type PhysicalOptions struct {
8080
PreprocessingScript string `yaml:"preprocessingScript"`
8181
Configs map[string]string `yaml:"configs"`
8282
Sysctls map[string]string `yaml:"sysctls"`
83+
Envs map[string]string `yaml:"envs"`
8384
Scheduler *Scheduler `yaml:"scheduler"`
8485
}
8586

@@ -529,11 +530,9 @@ func (p *PhysicalInitial) adjustRecoveryConfiguration(pgVersion, clonePGDataDir
529530
replicationFilename = "recovery.conf"
530531

531532
buffer.WriteString("standby_mode = 'on'\n")
532-
buffer.WriteString("primary_conninfo = ''\n")
533-
buffer.WriteString("restore_command = ''\n")
534533
}
535534

536-
if err := ioutil.WriteFile(path.Join(clonePGDataDir, replicationFilename), buffer.Bytes(), 0666); err != nil {
535+
if err := fs.AppendFile(path.Join(clonePGDataDir, replicationFilename), buffer.Bytes()); err != nil {
537536
return err
538537
}
539538

@@ -557,10 +556,7 @@ func (p *PhysicalInitial) buildContainerConfig(clonePath, promoteImage, password
557556
cont.DBLabControlLabel: cont.DBLabPromoteLabel,
558557
cont.DBLabInstanceIDLabel: p.globalCfg.InstanceID,
559558
},
560-
Env: []string{
561-
"PGDATA=" + clonePath,
562-
"POSTGRES_PASSWORD=" + password,
563-
},
559+
Env: p.getEnvironmentVariables(clonePath, password),
564560
Image: promoteImage,
565561
Healthcheck: health.GetConfig(
566562
p.globalCfg.Database.User(),
@@ -571,6 +567,20 @@ func (p *PhysicalInitial) buildContainerConfig(clonePath, promoteImage, password
571567
}
572568
}
573569

570+
func (p *PhysicalInitial) getEnvironmentVariables(clonePath, password string) []string {
571+
envVariables := []string{
572+
"PGDATA=" + clonePath,
573+
"POSTGRES_PASSWORD=" + password,
574+
}
575+
576+
// Add user-defined environment variables.
577+
for env, value := range p.options.Envs {
578+
envVariables = append(envVariables, fmt.Sprintf("%s=%s", env, value))
579+
}
580+
581+
return envVariables
582+
}
583+
574584
func (p *PhysicalInitial) buildHostConfig(ctx context.Context, clonePath string) (*container.HostConfig, error) {
575585
hostConfig := &container.HostConfig{
576586
Sysctls: p.options.Sysctls,
@@ -621,6 +631,9 @@ func (p *PhysicalInitial) runPromoteCommand(ctx context.Context, containerID, cl
621631
output, err := tools.ExecCommandWithOutput(ctx, p.dockerClient, containerID, types.ExecConfig{
622632
User: defaults.Username,
623633
Cmd: promoteCommand,
634+
Env: []string{
635+
fmt.Sprintf("PGCTLTIMEOUT=%d", p.options.Promotion.HealthCheck.MaxRetries*int(p.options.Promotion.HealthCheck.Interval)),
636+
},
624637
})
625638
if err != nil {
626639
return errors.Wrap(err, "failed to promote instance")

‎pkg/retrieval/engine/postgres/tools/fs/tools.go

Copy file name to clipboardExpand all lines: pkg/retrieval/engine/postgres/tools/fs/tools.go
+16Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,3 +57,19 @@ func copyFile(sourceFilename, destinationFilename string) error {
5757

5858
return nil
5959
}
60+
61+
// AppendFile appends data to a file.
62+
func AppendFile(file string, data []byte) error {
63+
configFile, err := os.OpenFile(file, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
64+
if err != nil {
65+
return err
66+
}
67+
68+
defer func() { _ = configFile.Close() }()
69+
70+
if _, err := configFile.Write(data); err != nil {
71+
return err
72+
}
73+
74+
return nil
75+
}

‎pkg/retrieval/engine/postgres/tools/tools.go

Copy file name to clipboardExpand all lines: pkg/retrieval/engine/postgres/tools/tools.go
+1-1Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -189,7 +189,7 @@ func CheckContainerReadiness(ctx context.Context, dockerClient *client.Client, c
189189
for {
190190
select {
191191
case <-ctx.Done():
192-
return nil
192+
return ctx.Err()
193193
default:
194194
}
195195

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.