Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit 0f0186d

Browse filesBrowse files
faisaltheparttimecoderfaisaltheparttimecoder
committed
2 parents 2f98890 + 3b74fb4 commit 0f0186d
Copy full SHA for 0f0186d

File tree

Expand file treeCollapse file tree

7 files changed

+97
-21
lines changed
Filter options
Expand file treeCollapse file tree

7 files changed

+97
-21
lines changed

‎README.md

Copy file name to clipboardExpand all lines: README.md
+3-3Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -130,9 +130,9 @@ For more examples how to use the tool, please check out the [wiki](https://githu
130130

131131
# Known Issues
132132

133-
1. If you have a unique index on a foreign key column then there are chance the constraint creation would fail, since mockd doesn't pick up unique value for foriegn key value it picks up random values from the reference table.
134-
2. CHECK constraints isn't supported , so recreating check constraints would fail, use `custom` subcommand to control the data being inserted
135-
3. On Greenplum Datbase tables are not supported (due to check constraint issues defined above), so use the `custom` sub command to define the data to be inserted to the column with check constraints
133+
1. If you have a composite unique index where one column is part of foreign key column then there are chances the constraint creation would fail.
134+
2. Fixing CHECK constraints isn't supported due to complexity, so recreating check constraints would fail, use `custom` subcommand to control the data being inserted
135+
3. On Greenplum Database partition tables are not supported (due to check constraint issues defined above), so use the `custom` sub command to define the data to be inserted to the column with check constraints
136136
4. Custom data types are not supported, use `custom` sub command to control the data for that custom data types
137137

138138
# Developers / Collaboration

‎cmd.go

Copy file name to clipboardExpand all lines: cmd.go
+4Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,10 @@ var tablesCmd = &cobra.Command{
132132
if IsStringEmpty(cmdOptions.Tab.SchemaName) {
133133
Fatalf("Cannot have the schema name empty, please check the arguments")
134134
}
135+
// If number of columns is greater than the limit, then error out
136+
if cmdOptions.Tab.MaxColumns > 1600 {
137+
Fatalf("Postgres cannot have more than 1600 columns, check the arguments")
138+
}
135139
},
136140
PostRun: func(cmd *cobra.Command, args []string) {
137141
Info("Successfully completed running the table sub command")

‎constraintsBackup.go

Copy file name to clipboardExpand all lines: constraintsBackup.go
+2-2Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -103,9 +103,9 @@ func RemoveConstraints(table string) {
103103
for _, c := range constraints {
104104
// Generate the DROP DDL command
105105
if c.Constrainttype == "index" { // if the constraint is a index
106-
statement = fmt.Sprintf("DROP INDEX %s CASCADE;", c.Constraintname)
106+
statement = fmt.Sprintf("DROP INDEX \"%s\" CASCADE;", c.Constraintname)
107107
} else { // if the constraint is a constraint
108-
statement = fmt.Sprintf("ALTER TABLE %s DROP CONSTRAINT %s CASCADE;", table, c.Constraintname)
108+
statement = fmt.Sprintf("ALTER TABLE %s DROP CONSTRAINT \"%s\" CASCADE;", table, c.Constraintname)
109109
}
110110

111111
// Execute the statement

‎constraintsRestore.go

Copy file name to clipboardExpand all lines: constraintsRestore.go
+45-8Lines changed: 45 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ func FixConstraints() {
3232
case v == "UNIQUE": // Run the same logic as primary key
3333
fixPKey(con)
3434
//case v == "CHECK": // TODO: Its hard to predict the check constraint ATM
35-
// fixCheck(db, con)
35+
// fixCheck(db, con)
3636
case v == "FOREIGN":
3737
fixFKey(con)
3838
}
@@ -196,12 +196,17 @@ func recreateAllConstraints() {
196196
_, err := ExecuteDB(content)
197197
if err != nil && !IgnoreErrorString(fmt.Sprintf("%s", err)) {
198198
Debugf("Error creating constraint %s, err: %v", content, err)
199-
err = WriteToFile(failedConstraintsFile, content+"\n")
200-
if err != nil {
201-
Fatalf("Error when saving the failed restore to file %s, err %v",
202-
failedConstraintsFile, err)
199+
// Try an attempt to recreate constraint again after deleting the
200+
// violating row
201+
successOrFailure := deleteViolatingPkOrUkConstriants(content)
202+
if !successOrFailure { // didn't succeed, ask the user to fix it manually
203+
err = WriteToFile(failedConstraintsFile, content+"\n")
204+
if err != nil {
205+
Fatalf("Error when saving the failed restore to file %s, err %v",
206+
failedConstraintsFile, err)
207+
}
208+
AnyError = true
203209
}
204-
AnyError = true
205210
}
206211
bar.Add(1)
207212
}
@@ -210,7 +215,39 @@ func recreateAllConstraints() {
210215
}
211216

212217
if AnyError {
213-
Warnf("There have been issue creating few constraints, all the "+
214-
"constraints that failed has been saved on to file: %s", failedConstraintsFile)
218+
Warnf("There have been issue creating few constraints and would need manual cleanup at your end, "+
219+
"all the constraints that failed has been saved on to file: %s", failedConstraintsFile)
215220
}
216221
}
222+
223+
// we tried to fix the primary key violation, but due to the nature
224+
// of how we fix the constraints like PK (or UK) followed by FK , there
225+
// are chances that we might inject duplicate keys again, for eg.s if
226+
// there is a PK ( or UK ) on a FK reference table. so the aim here
227+
// is, we will delete the rows that violates it and hoping that it will help in
228+
// recreating the constraints. Yes we will loose that row at least that help to
229+
// recreate constraints ( fingers crossed :) )
230+
func deleteViolatingPkOrUkConstriants(con string) bool {
231+
Debugf("Attempting to run the constraint command %s second time, after deleting violating rows", con)
232+
// does the DDL contain PK or UK keyword then do the following
233+
// rest send them back for user to fix it.
234+
if isSubStringAvailableOnString(con, "ADD CONSTRAINT.*PRIMARY KEY|ADD CONSTRAINT.*UNIQUE") {
235+
column, _ := ColExtractor(con, `\(.*?\)`)
236+
table, _ := ColExtractor(con, `ALTER TABLE(.*)ADD CONSTRAINT`)
237+
table = strings.Trim(strings.Trim(table, "ALTER TABLE"), "ADD CONSTRAINT")
238+
column = strings.Trim(column, "()")
239+
err := deleteViolatingConstraintKeys(table, column)
240+
if err != nil { // we failed to delete the the constraint violation rows
241+
Debugf("Error when deleting rows from the constraint violation rows: %v", err)
242+
return false
243+
}
244+
_, err = ExecuteDB(con) // retry to create the constraint again
245+
if err != nil { // we failed to recreate the constraint
246+
Debugf("Error when 2nd attempt to recreate constraint: %v", err)
247+
return false
248+
}
249+
// successfully cleaned it up
250+
return true
251+
}
252+
return false
253+
}

‎helpers.go

Copy file name to clipboardExpand all lines: helpers.go
+9Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -209,6 +209,15 @@ func BracketsExists(dt string) bool {
209209
}
210210
}
211211

212+
// Does the string contain the substring
213+
func isSubStringAvailableOnString(s string, criteria string) bool {
214+
var re = regexp.MustCompile(fmt.Sprintf("%s", criteria))
215+
if re.MatchString(s) {
216+
return true
217+
}
218+
return false
219+
}
220+
212221
// Built a method to find if the values exits with a slice
213222
func StringContains(item string, slice []string) bool {
214223
set := make(map[string]struct{}, len(slice))

‎sql.go

Copy file name to clipboardExpand all lines: sql.go
+33-7Lines changed: 33 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -244,9 +244,11 @@ func GetPGConstraintDDL(conntype string) []DBConstraints {
244244
Debugf("Extracting the DDL of the %s constraints", conntype)
245245
var result []DBConstraints
246246
query := `
247-
SELECT n.nspname
248-
|| '.'
249-
|| c.relname tablename,
247+
SELECT '"'
248+
|| n.nspname
249+
|| '"."'
250+
|| c.relname
251+
|| '"' tablename,
250252
con.conname constraintname,
251253
pg_catalog.Pg_get_constraintdef(con.oid, true) constraintKey
252254
FROM pg_catalog.pg_class c,
@@ -277,10 +279,12 @@ func GetPGIndexDDL() []DBIndex {
277279
Debugf("Extracting the unique indexes")
278280
var result []DBIndex
279281
query := `
280-
SELECT schemaname
281-
|| '.'
282-
|| tablename tablename,
283-
indexdef indexdef
282+
SELECT '"'
283+
|| schemaname
284+
|| '"."'
285+
|| tablename
286+
|| '"' tablename,
287+
indexdef indexdef
284288
FROM pg_indexes
285289
WHERE schemaname IN (SELECT nspname
286290
FROM pg_namespace
@@ -546,3 +550,25 @@ WHERE %[2]s = '%[6]s'
546550
Fatalf("Error when updating the foreign key for table %s, err: %v", key.Table, err)
547551
}
548552
}
553+
554+
// Delete the violating key
555+
func deleteViolatingConstraintKeys(tab string, column string) error {
556+
Debugf("Deleting the rows the table that violates the constraints: %s:(%s)", tab, column )
557+
query := `
558+
DELETE
559+
FROM %[1]s
560+
WHERE (
561+
%[2]s) IN
562+
(
563+
SELECT %[2]s
564+
FROM %[1]s
565+
GROUP BY %[2]s
566+
HAVING count(*) > 1);
567+
`
568+
query = fmt.Sprintf(query, tab, column)
569+
_, err := ExecuteDB(query)
570+
if err != nil {
571+
return err
572+
}
573+
return nil
574+
}

‎worker.go

Copy file name to clipboardExpand all lines: worker.go
+1-1Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -180,7 +180,7 @@ func GenerateTableName(tab, schema string) string {
180180
// Throw warning if there is skipped tables
181181
func skipTablesWarning() {
182182
if len(skippedTab) > 0 {
183-
Warnf("These tables are skipped since these datatypes are not supported by %s: %s",
183+
Warnf("These tables are skipped since these data types are not supported by %s: %s",
184184
programName, strings.Join(skippedTab, ","))
185185
}
186186
}

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.