From 186f3f8d242c4be1ebb470e6d12bfc10e1dafeee Mon Sep 17 00:00:00 2001 From: Siyuan Miao Date: Fri, 10 May 2019 16:07:50 +0800 Subject: [PATCH 1/4] :bug: fix _save_to_qiniu for bytes object in py3k --- leancloud/file_.py | 39 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/leancloud/file_.py b/leancloud/file_.py index ead158c4..4cffbced 100644 --- a/leancloud/file_.py +++ b/leancloud/file_.py @@ -74,7 +74,11 @@ def __init__(self, name='', data=None, mime_type=None): chunk = data.read(4096) if not chunk: break - checksum.update(chunk) + + try: + checksum.update(chunk) + except TypeError: + checksum.update(chunk.encode('utf-8')) self._metadata['_checksum'] = checksum.hexdigest() self._metadata['size'] = data.tell() @@ -164,10 +168,39 @@ def destroy(self): if response.status_code != 200: raise LeanCloudError(1, "the file is not sucessfully destroyed") + def _save_to_qiniu_internal_py3(self, token, key): + from qiniu.services.storage.uploader import crc32, _form_put, put_data + from qiniu.config import _BLOCK_SIZE + + final_data = '' + while True: + tmp_data = self._source.read(_BLOCK_SIZE) + if len(tmp_data) == 0: + break + elif len(final_data) == 0: + final_data = tmp_data + else: + final_data += tmp_data + else: + final_data = self._source.data + + crc = crc32(final_data) + return _form_put( + token, key, + final_data, None, + self.mime_type, crc + ) + def _save_to_qiniu(self, token, key): - import qiniu self._source.seek(0) - ret, info = qiniu.put_data(token, key, self._source) + + if six.PY3: + # use patched put_data implementation for py3k + ret, info = self._save_to_qiniu_internal_py3(token, key) + else: + import qiniu + # use put_data implementation provided by qiniu-sdk + ret, info = qiniu.put_data(token, key, self._source) self._source.seek(0) if info.status_code != 200: From dc9910612ea919dc5c76fd1e8feb62054df0a5ef Mon Sep 17 00:00:00 2001 From: Siyuan Miao Date: Fri, 10 May 2019 16:09:55 +0800 Subject: [PATCH 2/4] :arrow_up: upgrade urlib3 to 1.24.3 --- Pipfile.lock | 108 +++++++++++++++++++++++++-------------------------- 1 file changed, 54 insertions(+), 54 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index 1ccbce0c..579777f5 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -46,36 +46,36 @@ }, "cffi": { "hashes": [ - "sha256:00b97afa72c233495560a0793cdc86c2571721b4271c0667addc83c417f3d90f", - "sha256:0ba1b0c90f2124459f6966a10c03794082a2f3985cd699d7d63c4a8dae113e11", - "sha256:0bffb69da295a4fc3349f2ec7cbe16b8ba057b0a593a92cbe8396e535244ee9d", - "sha256:21469a2b1082088d11ccd79dd84157ba42d940064abbfa59cf5f024c19cf4891", - "sha256:2e4812f7fa984bf1ab253a40f1f4391b604f7fc424a3e21f7de542a7f8f7aedf", - "sha256:2eac2cdd07b9049dd4e68449b90d3ef1adc7c759463af5beb53a84f1db62e36c", - "sha256:2f9089979d7456c74d21303c7851f158833d48fb265876923edcb2d0194104ed", - "sha256:3dd13feff00bddb0bd2d650cdb7338f815c1789a91a6f68fdc00e5c5ed40329b", - "sha256:4065c32b52f4b142f417af6f33a5024edc1336aa845b9d5a8d86071f6fcaac5a", - "sha256:51a4ba1256e9003a3acf508e3b4f4661bebd015b8180cc31849da222426ef585", - "sha256:59888faac06403767c0cf8cfb3f4a777b2939b1fbd9f729299b5384f097f05ea", - "sha256:59c87886640574d8b14910840327f5cd15954e26ed0bbd4e7cef95fa5aef218f", - "sha256:610fc7d6db6c56a244c2701575f6851461753c60f73f2de89c79bbf1cc807f33", - "sha256:70aeadeecb281ea901bf4230c6222af0248c41044d6f57401a614ea59d96d145", - "sha256:71e1296d5e66c59cd2c0f2d72dc476d42afe02aeddc833d8e05630a0551dad7a", - "sha256:8fc7a49b440ea752cfdf1d51a586fd08d395ff7a5d555dc69e84b1939f7ddee3", - "sha256:9b5c2afd2d6e3771d516045a6cfa11a8da9a60e3d128746a7fe9ab36dfe7221f", - "sha256:9c759051ebcb244d9d55ee791259ddd158188d15adee3c152502d3b69005e6bd", - "sha256:b4d1011fec5ec12aa7cc10c05a2f2f12dfa0adfe958e56ae38dc140614035804", - "sha256:b4f1d6332339ecc61275bebd1f7b674098a66fea11a00c84d1c58851e618dc0d", - "sha256:c030cda3dc8e62b814831faa4eb93dd9a46498af8cd1d5c178c2de856972fd92", - "sha256:c2e1f2012e56d61390c0e668c20c4fb0ae667c44d6f6a2eeea5d7148dcd3df9f", - "sha256:c37c77d6562074452120fc6c02ad86ec928f5710fbc435a181d69334b4de1d84", - "sha256:c8149780c60f8fd02752d0429246088c6c04e234b895c4a42e1ea9b4de8d27fb", - "sha256:cbeeef1dc3c4299bd746b774f019de9e4672f7cc666c777cd5b409f0b746dac7", - "sha256:e113878a446c6228669144ae8a56e268c91b7f1fafae927adc4879d9849e0ea7", - "sha256:e21162bf941b85c0cda08224dade5def9360f53b09f9f259adb85fc7dd0e7b35", - "sha256:fb6934ef4744becbda3143d30c6604718871495a5e36c408431bf33d9c146889" - ], - "version": "==1.12.2" + "sha256:041c81822e9f84b1d9c401182e174996f0bae9991f33725d059b771744290774", + "sha256:046ef9a22f5d3eed06334d01b1e836977eeef500d9b78e9ef693f9380ad0b83d", + "sha256:066bc4c7895c91812eff46f4b1c285220947d4aa46fa0a2651ff85f2afae9c90", + "sha256:066c7ff148ae33040c01058662d6752fd73fbc8e64787229ea8498c7d7f4041b", + "sha256:2444d0c61f03dcd26dbf7600cf64354376ee579acad77aef459e34efcb438c63", + "sha256:300832850b8f7967e278870c5d51e3819b9aad8f0a2c8dbe39ab11f119237f45", + "sha256:34c77afe85b6b9e967bd8154e3855e847b70ca42043db6ad17f26899a3df1b25", + "sha256:46de5fa00f7ac09f020729148ff632819649b3e05a007d286242c4882f7b1dc3", + "sha256:4aa8ee7ba27c472d429b980c51e714a24f47ca296d53f4d7868075b175866f4b", + "sha256:4d0004eb4351e35ed950c14c11e734182591465a33e960a4ab5e8d4f04d72647", + "sha256:4e3d3f31a1e202b0f5a35ba3bc4eb41e2fc2b11c1eff38b362de710bcffb5016", + "sha256:50bec6d35e6b1aaeb17f7c4e2b9374ebf95a8975d57863546fa83e8d31bdb8c4", + "sha256:55cad9a6df1e2a1d62063f79d0881a414a906a6962bc160ac968cc03ed3efcfb", + "sha256:5662ad4e4e84f1eaa8efce5da695c5d2e229c563f9d5ce5b0113f71321bcf753", + "sha256:59b4dc008f98fc6ee2bb4fd7fc786a8d70000d058c2bbe2698275bc53a8d3fa7", + "sha256:73e1ffefe05e4ccd7bcea61af76f36077b914f92b76f95ccf00b0c1b9186f3f9", + "sha256:a1f0fd46eba2d71ce1589f7e50a9e2ffaeb739fb2c11e8192aa2b45d5f6cc41f", + "sha256:a2e85dc204556657661051ff4bab75a84e968669765c8a2cd425918699c3d0e8", + "sha256:a5457d47dfff24882a21492e5815f891c0ca35fefae8aa742c6c263dac16ef1f", + "sha256:a8dccd61d52a8dae4a825cdbb7735da530179fea472903eb871a5513b5abbfdc", + "sha256:ae61af521ed676cf16ae94f30fe202781a38d7178b6b4ab622e4eec8cefaff42", + "sha256:b012a5edb48288f77a63dba0840c92d0504aa215612da4541b7b42d849bc83a3", + "sha256:d2c5cfa536227f57f97c92ac30c8109688ace8fa4ac086d19d0af47d134e2909", + "sha256:d42b5796e20aacc9d15e66befb7a345454eef794fdb0737d1af593447c6c8f45", + "sha256:dee54f5d30d775f525894d67b1495625dd9322945e7fee00731952e0368ff42d", + "sha256:e070535507bd6aa07124258171be2ee8dfc19119c28ca94c9dfb7efd23564512", + "sha256:e1ff2748c84d97b065cc95429814cdba39bcbd77c9c85c89344b317dc0d9cbff", + "sha256:ed851c75d1e0e043cbf5ca9a8e1b13c4c90f3fbd863dacb01c0808e2b5204201" + ], + "version": "==1.12.3" }, "chardet": { "hashes": [ @@ -255,18 +255,18 @@ }, "urllib3": { "hashes": [ - "sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39", - "sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22" + "sha256:2393a695cd12afedd0dcb26fe5d50d0cf248e5a66f75dbd89a3d4eb333a61af4", + "sha256:a637e5fae88995b256e3409dc4d52c2e2e0ba32c42a6365fee8bbd2238de3cfb" ], - "version": "==1.24.1" + "version": "==1.24.3" }, "werkzeug": { "hashes": [ - "sha256:c3fd7a7d41976d9f44db327260e263132466836cef6f91512889ed60ad26557c", - "sha256:d5da73735293558eb1651ee2fddc4d0dedcfa06538b8813a2e20011583c9e49b" + "sha256:0a73e8bb2ff2feecfc5d56e6f458f5b99290ef34f565ffb2665801ff7de6af7a", + "sha256:7fad9770a8778f9576693f0cc29c7dcc36964df916b83734f4431c0e612a7fbc" ], "index": "pypi", - "version": "==0.14.1" + "version": "==0.15.2" } }, "develop": { @@ -346,10 +346,10 @@ }, "jinja2": { "hashes": [ - "sha256:74c935a1b8bb9a3947c50a54766a969d4846290e1e788ea44c1392163723c3bd", - "sha256:f84be1bb0040caca4cea721fcbbbbd61f9be9464ca236387158b0feea01914a4" + "sha256:065c4f02ebe7f7cf559e49ee5a95fb800a9e4528727aec6f24402a5374c65013", + "sha256:14dd6caf1527abb21f08f86c784eac40853ba93edb79552aa1e4b8aef1b61c7b" ], - "version": "==2.10" + "version": "==2.10.1" }, "markupsafe": { "hashes": [ @@ -402,24 +402,24 @@ }, "pygments": { "hashes": [ - "sha256:5ffada19f6203563680669ee7f53b64dabbeb100eb51b61996085e99c03b284a", - "sha256:e8218dd399a61674745138520d0d4cf2621d7e032439341bc3f647bff125818d" + "sha256:31cba6ffb739f099a85e243eff8cb717089fdd3c7300767d9fc34cb8e1b065f5", + "sha256:5ad302949b3c98dd73f8d9fcdc7e9cb592f120e32a18e23efd7f3dc51194472b" ], - "version": "==2.3.1" + "version": "==2.4.0" }, "pyparsing": { "hashes": [ - "sha256:66c9268862641abcac4a96ba74506e594c884e3f57690a696d21ad8210ed667a", - "sha256:f6c5ef0d7480ad048c054c37632c67fca55299990fff127850181659eea33fc3" + "sha256:1873c03321fc118f4e9746baf201ff990ceb915f433f23b395f5580d1840cb2a", + "sha256:9b6323ef4ab914af344ba97510e966d64ba91055d6b9afa6b30799340e89cc03" ], - "version": "==2.3.1" + "version": "==2.4.0" }, "pytz": { "hashes": [ - "sha256:32b0891edff07e28efe91284ed9c31e123d84bea3fd98e1f72be2508f43ef8d9", - "sha256:d5f05e487007e29e03409f9398d074e158d920d36eb82eaf66fb1136b0c5374c" + "sha256:303879e36b721603cc54604edcac9d20401bdbe31e1e4fdee5b9f98d5d31dfda", + "sha256:d747dd3d23d77ef44c6a3526e274af6efeb0a6f1afd5a69ba4d5be4098c8e141" ], - "version": "==2018.9" + "version": "==2019.1" }, "requests": { "hashes": [ @@ -478,18 +478,18 @@ }, "urllib3": { "hashes": [ - "sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39", - "sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22" + "sha256:2393a695cd12afedd0dcb26fe5d50d0cf248e5a66f75dbd89a3d4eb333a61af4", + "sha256:a637e5fae88995b256e3409dc4d52c2e2e0ba32c42a6365fee8bbd2238de3cfb" ], - "version": "==1.24.1" + "version": "==1.24.3" }, "werkzeug": { "hashes": [ - "sha256:c3fd7a7d41976d9f44db327260e263132466836cef6f91512889ed60ad26557c", - "sha256:d5da73735293558eb1651ee2fddc4d0dedcfa06538b8813a2e20011583c9e49b" + "sha256:0a73e8bb2ff2feecfc5d56e6f458f5b99290ef34f565ffb2665801ff7de6af7a", + "sha256:7fad9770a8778f9576693f0cc29c7dcc36964df916b83734f4431c0e612a7fbc" ], "index": "pypi", - "version": "==0.14.1" + "version": "==0.15.2" }, "wsgi-intercept": { "hashes": [ From b0dd68c89d4c4db60b79e79d29baf6834d813dda Mon Sep 17 00:00:00 2001 From: Siyuan Miao Date: Fri, 10 May 2019 16:42:59 +0800 Subject: [PATCH 3/4] :bug: fix timeout for uploading larger files to qiniu --- leancloud/file_.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/leancloud/file_.py b/leancloud/file_.py index 4cffbced..f3be912e 100644 --- a/leancloud/file_.py +++ b/leancloud/file_.py @@ -26,6 +26,8 @@ logger = logging.getLogger(__name__) +DEFAULT_TIMEOUT = 30 + class File(object): _class_name = '_File' # walks like a leancloud.Object @@ -35,6 +37,7 @@ def __init__(self, name='', data=None, mime_type=None): self._url = None self._acl = None self.current_user = leancloud.User.get_current() + self.timeout = 30 self._metadata = { 'owner': 'unknown' } @@ -83,6 +86,11 @@ def __init__(self, name='', data=None, mime_type=None): self._metadata['_checksum'] = checksum.hexdigest() self._metadata['size'] = data.tell() + # 3.5MB, 1Mbps * 30s + # increase timeout + if self._metadata['size'] > 3750000: + self.timeout = self.timeout * int(self._metadata['size'] / 3750000) + data.seek(0, os.SEEK_SET) self._source = data @@ -194,11 +202,13 @@ def _save_to_qiniu_internal_py3(self, token, key): def _save_to_qiniu(self, token, key): self._source.seek(0) + import qiniu + qiniu.set_default(connection_timeout=self.timeout) + if six.PY3: # use patched put_data implementation for py3k ret, info = self._save_to_qiniu_internal_py3(token, key) else: - import qiniu # use put_data implementation provided by qiniu-sdk ret, info = qiniu.put_data(token, key, self._source) self._source.seek(0) From 3e342ab6392a647119f5b1f6bb229912845c7f64 Mon Sep 17 00:00:00 2001 From: Siyuan Miao Date: Fri, 10 May 2019 16:53:55 +0800 Subject: [PATCH 4/4] :bookmark: release 2.1.11 --- changelog | 8 ++++++++ setup.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/changelog b/changelog index 45510bbc..18d5054b 100644 --- a/changelog +++ b/changelog @@ -1,3 +1,11 @@ +## [2.1.11] - 2019-05-10 +### Fixed +- Error -1 when uploading large files to Qiniu +- TypeError in py3k when uploading files to Qiniu using BytesIO / StringIO +- Upgraded urlib3 to 1.24.3 (CVE-2019-11324) +### Added +- email argument for User.login + ## [2.1.10] - 2019-03-18 ### Changed - Upgrade Qiniu SDK diff --git a/setup.py b/setup.py index 034a7ca3..59cf20c3 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ setup( name='leancloud', - version='2.1.10', + version='2.1.11', description='LeanCloud Python SDK', url='https://leancloud.cn/', author='asaka',