forked from marklogic/node-client-api
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathbasic-loader.js
More file actions
122 lines (106 loc) · 3.6 KB
/
basic-loader.js
File metadata and controls
122 lines (106 loc) · 3.6 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
/*
* Copyright (c) 2020 MarkLogic Corporation
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
'use strict';
const fs = require('fs');
const path = require('path');
const {Writable} = require('stream');
const Vinyl = require('./optional.js').library('vinyl');
function getDatabaseClient(loader, options) {
if (options === void 0 || options == null) {
throw new Error(`${loader} requires options`);
}
const databaseClient = options.databaseClient;
if (databaseClient === void 0 || databaseClient == null) {
throw new Error(`${loader} requires databaseClient option`);
}
return databaseClient;
}
function loadFile(callback, options) {
const databaseClient = getDatabaseClient('loadFile', options);
const filePath = options.filePath;
if (filePath === void 0 || filePath == null) {
throw new Error('loadFile requires the filePath for the source file');
}
const documentDescriptor = options.documentDescriptor;
if (documentDescriptor === void 0 || documentDescriptor == null) {
throw new Error(`loadFile requires fileDescriptor option`);
}
fs.readFile(filePath, 'utf8', (err, data) => {
if (err) {
callback(err);
return;
}
documentDescriptor.content = data;
databaseClient.documents.write(documentDescriptor)
.result(output => callback(), callback);
});
}
function loadFileStream(options) {
const databaseClient = getDatabaseClient('loadFileStream', options);
const documentMetadata = options.documentMetadata;
if (documentMetadata === void 0 || documentMetadata == null) {
throw new Error(`loadFileStream requires documentMetadata option`);
}
let uriPrefix = options.uriPrefix;
if (uriPrefix === void 0 || uriPrefix == null) {
uriPrefix = '/';
} else if (!uriPrefix.endsWith('/')) {
uriPrefix += '/';
}
let uriStartDepth = options.uriStartDepth;
if (uriStartDepth === void 0 || uriStartDepth == null) {
uriStartDepth = 2;
}
uriStartDepth += path.resolve().split(path.sep).length - 1;
const bufferMax = 100;
const batchSeed = [documentMetadata];
const fileBuffer = new Array(bufferMax);
let bufferNext = 0;
return new Writable({
objectMode: true,
write(file, encoding, callback) {
if (!Vinyl.isVinyl(file) || file.isDirectory() || file.path === void 0 || !(file.path.length > 0)) {
console.trace('not Vinyl file: '+file);
callback();
return;
}
fileBuffer[bufferNext++] = {
uri: uriPrefix + file.path.split(path.sep).slice(uriStartDepth).join('/'),
content: file.contents
};
if (bufferNext < bufferMax) {
callback();
return;
}
databaseClient.documents.write(batchSeed.concat(fileBuffer))
.result(output => callback(), callback);
bufferNext = 0;
},
final(callback) {
if (bufferNext > 0) {
databaseClient.documents.write(batchSeed.concat(fileBuffer.slice(0, bufferNext)))
.result(output => callback(), callback);
bufferNext = 0;
} else {
callback();
}
}
});
}
module.exports = {
loadFile: loadFile,
loadFileStream: loadFileStream
};