Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit a400fb7

Browse filesBrowse files
agneumagneum
committed
feat: automated data retrieval - physical and logical restoring (#93)
1 parent 3fa22fc commit a400fb7
Copy full SHA for a400fb7

File tree

30 files changed

+1445
-18
lines changed
Filter options

30 files changed

+1445
-18
lines changed

‎.gitignore

Copy file name to clipboardExpand all lines: .gitignore
+1Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,3 +8,4 @@
88
/deploy/
99

1010
/configs/config.yml
11+
/configs/retrieval.yml

‎cmd/database-lab/main.go

Copy file name to clipboardExpand all lines: cmd/database-lab/main.go
+26-8Lines changed: 26 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,13 @@ import (
1414
"bytes"
1515
"context"
1616

17+
"github.com/docker/docker/client"
1718
"github.com/jessevdk/go-flags"
1819
"github.com/pkg/errors"
1920

2021
"gitlab.com/postgres-ai/database-lab/pkg/config"
2122
"gitlab.com/postgres-ai/database-lab/pkg/log"
23+
"gitlab.com/postgres-ai/database-lab/pkg/retrieval"
2224
"gitlab.com/postgres-ai/database-lab/pkg/services/cloning"
2325
"gitlab.com/postgres-ai/database-lab/pkg/services/platform"
2426
"gitlab.com/postgres-ai/database-lab/pkg/services/provision"
@@ -36,8 +38,6 @@ var opts struct {
3638
}
3739

3840
func main() {
39-
ctx := context.Background()
40-
4141
// Load CLI options.
4242
if _, err := parseArgs(); err != nil {
4343
if flags.WroteHelp(err) {
@@ -68,24 +68,41 @@ func main() {
6868
cfg.Provision.ModeLocal.DockerImage = opts.DockerImage
6969
}
7070

71-
provisionSvc, err := provision.NewProvision(ctx, cfg.Provision)
71+
dockerCLI, err := client.NewClientWithOpts(client.FromEnv)
72+
if err != nil {
73+
log.Fatal("Failed to create a Docker client:", err)
74+
}
75+
76+
ctx, cancel := context.WithCancel(context.Background())
77+
defer cancel()
78+
79+
// Create a new retrieval service to prepare a data directory and start snapshotting.
80+
retrievalSvc, err := retrieval.New(cfg, dockerCLI)
81+
if err != nil {
82+
log.Fatal("Failed to build a retrieval service:", err)
83+
}
84+
85+
if err := retrievalSvc.Run(ctx); err != nil {
86+
log.Fatal("Failed to run the data retrieval service:", err)
87+
}
88+
89+
// Create a cloning service to provision new clones.
90+
provisionSvc, err := provision.New(ctx, cfg.Provision)
7291
if err != nil {
7392
log.Fatalf(errors.WithMessage(err, `error in "provision" config`))
7493
}
7594

76-
cloningSvc, err := cloning.NewCloning(&cfg.Cloning, provisionSvc)
95+
cloningSvc, err := cloning.New(&cfg.Cloning, provisionSvc)
7796
if err != nil {
7897
log.Fatalf(errors.WithMessage(err, "failed to init a new cloning service"))
7998
}
8099

81-
ctx, cancel := context.WithCancel(context.Background())
82-
defer cancel()
83-
84100
if err = cloningSvc.Run(ctx); err != nil {
85101
log.Fatalf(err)
86102
}
87103

88-
platformSvc := platform.NewService(cfg.Platform)
104+
// Create a platform service to verify Platform tokens.
105+
platformSvc := platform.New(cfg.Platform)
89106
if err := platformSvc.Init(ctx); err != nil {
90107
log.Fatalf(errors.WithMessage(err, "failed to create a new platform service"))
91108
}
@@ -94,6 +111,7 @@ func main() {
94111
cfg.Server.VerificationToken = opts.VerificationToken
95112
}
96113

114+
// Start the Database Lab.
97115
server := srv.NewServer(&cfg.Server, cloningSvc, platformSvc)
98116
if err = server.Run(); err != nil {
99117
log.Fatalf(err)

‎configs/config.sample.yml

Copy file name to clipboardExpand all lines: configs/config.sample.yml
+31Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,11 @@ server:
1111
# HTTP server port. By default: 2345.
1212
port: 2345
1313

14+
global:
15+
engine: postgres
16+
dataDir: /var/lib/dblab
17+
dockerImage: "postgres:12-alpine"
18+
1419
# Postgres.ai Platform integration.
1520
platform:
1621
# API URL.
@@ -82,4 +87,30 @@ cloning:
8287
# 0 - disable automatic deletion.
8388
maxIdleMinutes: 120
8489

90+
retrieval:
91+
stages:
92+
- initialize
93+
94+
spec:
95+
initialize:
96+
jobs:
97+
# - name: logical-restore
98+
# options:
99+
# dumpFile: /tmp/db.dump
100+
# forceInit: false
101+
# dbName: test
102+
# partial:
103+
# tables:
104+
# - test
105+
- name: physical-restore
106+
options:
107+
tool: walg
108+
dockerImage: "postgresai/sync-instance:12"
109+
envs:
110+
WALG_GS_PREFIX: "gs://{BUCKET}/{SCOPE}"
111+
walg:
112+
storage: gcs
113+
backupName: LATEST
114+
credentialsFile: /tmp/sa.json # optional
115+
85116
debug: true

‎go.mod

Copy file name to clipboardExpand all lines: go.mod
+9Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,15 +4,20 @@ go 1.13
44

55
require (
66
github.com/AlekSi/pointer v1.1.0
7+
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect
78
github.com/Microsoft/go-winio v0.4.14 // indirect
9+
github.com/containerd/containerd v1.3.4 // indirect
810
github.com/docker/distribution v2.7.1+incompatible // indirect
911
github.com/docker/docker v1.13.1
1012
github.com/docker/go-connections v0.4.0 // indirect
1113
github.com/docker/go-units v0.4.0 // indirect
14+
github.com/gogo/protobuf v1.3.1 // indirect
1215
github.com/gorilla/mux v1.7.3
1316
github.com/jessevdk/go-flags v1.4.1-0.20181221193153-c0795c8afcf4
1417
github.com/lib/pq v1.3.0
18+
github.com/morikuni/aec v1.0.0 // indirect
1519
github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
20+
github.com/opencontainers/image-spec v1.0.1 // indirect
1621
github.com/pkg/errors v0.8.1
1722
github.com/rs/xid v1.2.1
1823
github.com/sergi/go-diff v1.1.0
@@ -22,5 +27,9 @@ require (
2227
golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9
2328
golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa // indirect
2429
golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 // indirect
30+
golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 // indirect
31+
google.golang.org/grpc v1.30.0 // indirect
2532
gopkg.in/yaml.v2 v2.2.7
2633
)
34+
35+
replace github.com/docker/docker v1.13.1 => github.com/docker/engine v17.12.0-ce-rc1.0.20200531234253-77e06fda0c94+incompatible

‎go.sum

Copy file name to clipboardExpand all lines: go.sum
+68-2Lines changed: 68 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,50 @@
1+
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
12
github.com/AlekSi/pointer v1.1.0 h1:SSDMPcXD9jSl8FPy9cRzoRaMJtm9g9ggGTxecRUbQoI=
23
github.com/AlekSi/pointer v1.1.0/go.mod h1:y7BvfRI3wXPWKXEBhU71nbnIEEZX0QTSB2Bj48UJIZE=
4+
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8=
5+
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
36
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
47
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
58
github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU=
69
github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
10+
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
11+
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
12+
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
13+
github.com/containerd/containerd v1.3.4 h1:3o0smo5SKY7H6AJCmJhsnCjR2/V2T8VmiHt7seN2/kI=
14+
github.com/containerd/containerd v1.3.4/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
715
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY=
816
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
917
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
1018
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
1119
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
1220
github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
1321
github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
14-
github.com/docker/docker v1.13.1 h1:IkZjBSIc8hBjLpqeAbeE5mca5mNgeatLHBy3GO78BWo=
15-
github.com/docker/docker v1.13.1/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
22+
github.com/docker/engine v17.12.0-ce-rc1.0.20200531234253-77e06fda0c94+incompatible h1:5uC2w1yhWUPlduFVWY/W3wiluc85Cv8nxIKWPSFk/WY=
23+
github.com/docker/engine v17.12.0-ce-rc1.0.20200531234253-77e06fda0c94+incompatible/go.mod h1:3CPr2caMgTHxxIAZgEMd3uLYPDlRvPqCpyeRf6ncPcY=
1624
github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
1725
github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
1826
github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
1927
github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
28+
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
29+
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
30+
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
31+
github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
32+
github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
33+
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
34+
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
35+
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
36+
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
37+
github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I=
38+
github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
39+
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
40+
github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
41+
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
2042
github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw=
2143
github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
2244
github.com/jessevdk/go-flags v1.4.1-0.20181221193153-c0795c8afcf4 h1:xKkUL6QBojwguhKKetf1SocCAKqc6W7S/mGm9xEGllo=
2345
github.com/jessevdk/go-flags v1.4.1-0.20181221193153-c0795c8afcf4/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
46+
github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
47+
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
2448
github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
2549
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
2650
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
@@ -30,12 +54,17 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
3054
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
3155
github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU=
3256
github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
57+
github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
58+
github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
3359
github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ=
3460
github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
61+
github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI=
62+
github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
3563
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
3664
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
3765
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
3866
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
67+
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
3968
github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
4069
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
4170
github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
@@ -57,9 +86,22 @@ github.com/urfave/cli/v2 v2.1.1/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2
5786
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
5887
golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9 h1:vEg9joUBmeBcK9iSJftGNf3coIG4HqZElCPehJsfAYM=
5988
golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
89+
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
90+
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
91+
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
92+
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
93+
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
94+
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
95+
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
96+
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
6097
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
6198
golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa h1:F+8P+gmewFQYRk6JoLQLwjBCTu3mcIURZfNkVweuRKA=
6299
golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
100+
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
101+
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
102+
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
103+
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
104+
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
63105
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
64106
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
65107
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -69,11 +111,35 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b h1:ag/x1USPSsqHud38I9BAC88qd
69111
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
70112
golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 h1:ogLJMz+qpzav7lGMh10LMvAkM/fAoGlaiiHYiFYdm80=
71113
golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
114+
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
72115
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
116+
golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 h1:NusfzzA6yGQ+ua51ck7E3omNUX/JuqbFSaRGqU8CcLI=
117+
golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
118+
golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
119+
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
120+
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
121+
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
122+
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
123+
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
124+
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
125+
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
126+
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
127+
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
128+
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE=
129+
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
130+
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
131+
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
132+
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
133+
google.golang.org/grpc v1.30.0 h1:M5a8xTlYTxwMn5ZFkwhRabsygDY5G8TYLyQDBxJNAxE=
134+
google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
73135
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
74136
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
75137
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
76138
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
77139
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
78140
gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo=
79141
gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
142+
gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
143+
gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
144+
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
145+
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

‎pkg/config/config.go

Copy file name to clipboardExpand all lines: pkg/config/config.go
+10Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"io/ioutil"
1111
"os/user"
1212

13+
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/config"
1314
"gitlab.com/postgres-ai/database-lab/pkg/services/cloning"
1415
"gitlab.com/postgres-ai/database-lab/pkg/services/platform"
1516
"gitlab.com/postgres-ai/database-lab/pkg/services/provision"
@@ -26,9 +27,18 @@ type Config struct {
2627
Provision provision.Config `yaml:"provision"`
2728
Cloning cloning.Config `yaml:"cloning"`
2829
Platform platform.Config `yaml:"platform"`
30+
Global Global `yaml:"global"`
31+
Retrieval config.Config `yaml:"retrieval"`
2932
Debug bool `yaml:"debug"`
3033
}
3134

35+
// Global contains global Database Lab configurations.
36+
type Global struct {
37+
Engine string `yaml:"engine"`
38+
DataDir string `yaml:"dataDir"`
39+
DockerImage string `yaml:"dockerImage"`
40+
}
41+
3242
// LoadConfig instances a new Config by configuration filename.
3343
func LoadConfig(name string) (*Config, error) {
3444
path, err := util.GetConfigPath(name)

‎pkg/retrieval/components/components.go

Copy file name to clipboard
+38Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
/*
2+
2020 © Postgres.ai
3+
*/
4+
5+
// Package components provides the key components of data retrieval.
6+
package components
7+
8+
import (
9+
"context"
10+
11+
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/config"
12+
)
13+
14+
// StageBuilder builds a StageRunner.
15+
type StageBuilder interface {
16+
BuildStageRunner(string) (StageRunner, error)
17+
}
18+
19+
// JobRunner performs a job.
20+
type JobRunner interface {
21+
// Name returns a job name.
22+
Name() string
23+
24+
// Run starts a job.
25+
Run(ctx context.Context) error
26+
}
27+
28+
// StageRunner declares stage content and performs stage jobs.
29+
type StageRunner interface {
30+
// AddJob applies jobs to the current stage.
31+
AddJob(JobRunner)
32+
33+
// BuildJob builds stage jobs.
34+
BuildJob(config.JobConfig) (JobRunner, error)
35+
36+
// Run starts stage jobs.
37+
Run(ctx context.Context) error
38+
}

‎pkg/retrieval/config/config.go

Copy file name to clipboard
+23Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
/*
2+
2020 © Postgres.ai
3+
*/
4+
5+
// Package config contains configuration options of the data retrieval.
6+
package config
7+
8+
// Config describes of data retrieval stages.
9+
type Config struct {
10+
Stages []string `yaml:"stages,flow"`
11+
StageSpec map[string]Spec `yaml:"spec"`
12+
}
13+
14+
// Spec describes a retrieval job.
15+
type Spec struct {
16+
Jobs []JobConfig `yaml:"jobs,flow"`
17+
}
18+
19+
// JobConfig contains details about a job.
20+
type JobConfig struct {
21+
Name string `yaml:"name"`
22+
Options map[string]interface{} `yaml:"options"`
23+
}

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.