diff --git a/.codelingoignore b/.codelingoignore deleted file mode 100644 index 48b8bf9..0000000 --- a/.codelingoignore +++ /dev/null @@ -1 +0,0 @@ -vendor/ diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..975d258 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +version: 2 + +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" + - package-ecosystem: "gomod" + directory: "/" + schedule: + interval: "weekly" diff --git a/.github/workflows/actionlint.yml b/.github/workflows/actionlint.yml new file mode 100644 index 0000000..41ce66c --- /dev/null +++ b/.github/workflows/actionlint.yml @@ -0,0 +1,19 @@ +name: Lint GitHub Actions Workflows + +on: + push: + paths: + - .github/** + +permissions: + contents: read + +jobs: + actionlint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - name: "Check workflow files" + uses: docker://docker.mirror.hashicorp.services/rhysd/actionlint:latest + with: + args: -color diff --git a/.github/workflows/go-multierror.yml b/.github/workflows/go-multierror.yml new file mode 100644 index 0000000..b3ab73e --- /dev/null +++ b/.github/workflows/go-multierror.yml @@ -0,0 +1,134 @@ +name: hashicorp/go-multierror/go-multierror +on: + - push + - pull_request + +permissions: + contents: read + +jobs: + go-fmt: + runs-on: ubuntu-latest + steps: + - name: Get go-version + run: go version + - name: Checkout code + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - name: Setup go + uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1 + with: + go-version-file: go.mod + - name: check go fmt + run: |- + files="$(go fmt ./...)" + if [ -n "$files" ]; then + echo "The following file(s) do not conform to go fmt:" + echo "$files" + exit 1 + fi + + lint: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - name: Setup go + uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1 + with: + go-version-file: go.mod + - name: Run golangci-lint + uses: golangci/golangci-lint-action@08e2f20817b15149a52b5b3ebe7de50aff2ba8c5 + + linux-tests: + runs-on: ubuntu-latest + env: + TEST_RESULTS_PATH: '/tmp/test-results' + strategy: + matrix: + go-version: + - '1.13' # oldest supported; named in go.mod + - 'oldstable' + - 'stable' + steps: + - name: Checkout code + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - name: Setup go + uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1 + with: + go-version: ${{ matrix.go-version }} + - name: Install gotestsum + uses: autero1/action-gotestsum@7263b9d73912eec65f46337689e59fac865c425f # v2.0.0 + with: + gotestsum_version: 1.9.0 + - name: Get go version and env + run: | + go version + go env + - name: Create test directory + run: mkdir -p "$TEST_RESULTS_PATH/go-multierror" + - name: Run go tests + env: + PLATFORM: linux + REPORT_FILE: ${{ env.TEST_RESULTS_PATH }}/go-multierror/gotestsum-report.xml + run: |- + gotestsum --format=short-verbose --junitfile ${{ env.REPORT_FILE }} -- -p 2 -cover -coverprofile=coverage-linux.out ./... + - name: Upload and save artifacts + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 + with: + path: ${{ env.TEST_RESULTS_PATH }} + name: tests-linux-${{ matrix.go-version }} + - name: Upload coverage report + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 + with: + path: coverage-linux.out + name: Coverage-report-linux-${{matrix.go-version}} + - name: Display coverage report + run: go tool cover -func=coverage-linux.out + + windows-tests: + runs-on: windows-latest + env: + TEST_RESULTS_PATH: 'c:\Users\runneradmin\AppData\Local\Temp\test-results' + strategy: + matrix: + go-version: + - '1.13' # oldest supported; named in go.mod + - 'oldstable' + - 'stable' + steps: + - name: Checkout code + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - name: Setup Go + uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1 + with: + go-version: ${{ matrix.go-version }} + - name: Get go-version and env + run: | + go version + go env + - run: git config --global core.autocrlf false + - name: Download go modules + run: go mod download + - name: Install gotestsum + uses: autero1/action-gotestsum@7263b9d73912eec65f46337689e59fac865c425f # v2.0.0 + with: + gotestsum_version: 1.9.0 + - name: Run go tests + env: + PLATFORM: win + REPORT_FILE: ${{ env.TEST_RESULTS_PATH }}/go-multierror/gotestsum-report.xml + run: |- + gotestsum.exe --format=short-verbose --junitfile ${{ env.REPORT_FILE }} -- -p 2 -cover -coverprofile="coverage-win.out" ./... + - name: Upload and save artifacts + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 + with: + path: ${{ env.TEST_RESULTS_PATH }} + name: tests-windows-${{ matrix.go-version }} + - name: Upload coverage test + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 + with: + path: coverage-win.out + name: Coverage-report-win-${{matrix.go-version}} + - name: Display coverage report + run: go tool cover -func=coverage-win.out + shell: cmd diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 541ae8f..0000000 --- a/.gitignore +++ /dev/null @@ -1,10 +0,0 @@ -./titan -.idea -*.iml -*.swp -coverage.out -cover.cov -logs -vendor -.DS_Store -go.sum diff --git a/.go-version b/.go-version new file mode 100644 index 0000000..d3456a9 --- /dev/null +++ b/.go-version @@ -0,0 +1 @@ +1.13 diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 1c4763b..0000000 --- a/.travis.yml +++ /dev/null @@ -1,17 +0,0 @@ -language: go -go_import_path: github.com/distributedio/titan - -go: - - "1.13.3" - -cache: - directories: - - $GOPATH/pkg/mod - -before_script: - - go get golang.org/x/tools/cmd/cover - - go get github.com/mattn/goveralls - -script: - - make coverage - - $HOME/gopath/bin/goveralls -coverprofile=cover.cov -service=travis-ci diff --git a/CHANGES.md b/CHANGES.md deleted file mode 100644 index 481e520..0000000 --- a/CHANGES.md +++ /dev/null @@ -1,13 +0,0 @@ -# Changes from upstream - -This repository is a frozen fork of etcd used for Sourcegraph QA pipelines. The following directive was added to the `go.mod` file in the following commits. The rest of the repository content is the same as upstream (but frozen). - -``` -replace ( - go.uber.org/zap => github.com/sourcegraph-testing/zap v1.12.0 -) -``` - -`aef232fbec9089d4468ff06705a3a7f84ee50ea6` -> `fb38de395ba67f49978b218e099de1c45122fb50` -`33623cc32f8d9f999fd69189d29124d4368c20ab` -> `415ffd5a3ba7a92a07cd96c7d9f4b734f61248f7` -`0ad2e75d529bda74472a1dbb5e488ec095b07fe7` -> `f8307e394c512b4263fc0cd67ccf9fd46f1ad9a5` diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 0000000..3da71e0 --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1,13 @@ +# Each line is a file pattern followed by one or more owners. +# More on CODEOWNERS files: https://help.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners + +# Default owner +* @hashicorp/team-ip-compliance + +# Add override rules below. Each line is a file/folder pattern followed by one or more owners. +# Being an owner means those groups or individuals will be added as reviewers to PRs affecting +# those areas of the code. +# Examples: +# /docs/ @docs-team +# *.js @js-team +# *.go @go-team \ No newline at end of file diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md deleted file mode 100644 index b2dba33..0000000 --- a/CODE_OF_CONDUCT.md +++ /dev/null @@ -1,76 +0,0 @@ -# Contributor Covenant Code of Conduct - -## Our Pledge - -In the interest of fostering an open and welcoming environment, we as -contributors and maintainers pledge to making participation in our project and -our community a harassment-free experience for everyone, regardless of age, body -size, disability, ethnicity, sex characteristics, gender identity and expression, -level of experience, education, socio-economic status, nationality, personal -appearance, race, religion, or sexual identity and orientation. - -## Our Standards - -Examples of behavior that contributes to creating a positive environment -include: - -* Using welcoming and inclusive language -* Being respectful of differing viewpoints and experiences -* Gracefully accepting constructive criticism -* Focusing on what is best for the community -* Showing empathy towards other community members - -Examples of unacceptable behavior by participants include: - -* The use of sexualized language or imagery and unwelcome sexual attention or - advances -* Trolling, insulting/derogatory comments, and personal or political attacks -* Public or private harassment -* Publishing others' private information, such as a physical or electronic - address, without explicit permission -* Other conduct which could reasonably be considered inappropriate in a - professional setting - -## Our Responsibilities - -Project maintainers are responsible for clarifying the standards of acceptable -behavior and are expected to take appropriate and fair corrective action in -response to any instances of unacceptable behavior. - -Project maintainers have the right and responsibility to remove, edit, or -reject comments, commits, code, wiki edits, issues, and other contributions -that are not aligned to this Code of Conduct, or to ban temporarily or -permanently any contributor for other behaviors that they deem inappropriate, -threatening, offensive, or harmful. - -## Scope - -This Code of Conduct applies both within project spaces and in public spaces -when an individual is representing the project or its community. Examples of -representing a project or community include using an official project e-mail -address, posting via an official social media account, or acting as an appointed -representative at an online or offline event. Representation of a project may be -further defined and clarified by project maintainers. - -## Enforcement - -Instances of abusive, harassing, or otherwise unacceptable behavior may be -reported by contacting the project team at ryq@meitu.com. All -complaints will be reviewed and investigated and will result in a response that -is deemed necessary and appropriate to the circumstances. The project team is -obligated to maintain confidentiality with regard to the reporter of an incident. -Further details of specific enforcement policies may be posted separately. - -Project maintainers who do not follow or enforce the Code of Conduct in good -faith may face temporary or permanent repercussions as determined by other -members of the project's leadership. - -## Attribution - -This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, -available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html - -[homepage]: https://www.contributor-covenant.org - -For answers to common questions about this code of conduct, see -https://www.contributor-covenant.org/faq diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 6809f47..0000000 --- a/Dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -# Builder image -FROM golang:latest as builder - -RUN apk add --no-cache \ - make \ - git - -COPY . /go/src/github.com/distributedio/titan - -WORKDIR /go/src/github.com/distributedio/titan - -RUN env GOOS=linux CGO_ENABLED=0 make - -# Executable image -FROM alpine - -COPY --from=builder /go/src/github.com/distributedio/titan/titan /titan/bin/titan -COPY --from=builder /go/src/github.com/distributedio/titan/conf/titan.toml /titan/conf/titan.toml - -WORKDIR /titan - -EXPOSE 7369 - -ENTRYPOINT ["./bin/titan"] diff --git a/LICENSE b/LICENSE index 261eeb9..e25da5f 100644 --- a/LICENSE +++ b/LICENSE @@ -1,201 +1,355 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. +Copyright (c) 2014 HashiCorp, Inc. + +Mozilla Public License, version 2.0 + +1. Definitions + +1.1. “Contributor” + + means each individual or legal entity that creates, contributes to the + creation of, or owns Covered Software. + +1.2. “Contributor Version” + + means the combination of the Contributions of others (if any) used by a + Contributor and that particular Contributor’s Contribution. + +1.3. “Contribution” + + means Covered Software of a particular Contributor. + +1.4. “Covered Software” + + means Source Code Form to which the initial Contributor has attached the + notice in Exhibit A, the Executable Form of such Source Code Form, and + Modifications of such Source Code Form, in each case including portions + thereof. + +1.5. “Incompatible With Secondary Licenses” + means + + a. that the initial Contributor has attached the notice described in + Exhibit B to the Covered Software; or + + b. that the Covered Software was made available under the terms of version + 1.1 or earlier of the License, but not also under the terms of a + Secondary License. + +1.6. “Executable Form” + + means any form of the work other than Source Code Form. + +1.7. “Larger Work” + + means a work that combines Covered Software with other material, in a separate + file or files, that is not Covered Software. + +1.8. “License” + + means this document. + +1.9. “Licensable” + + means having the right to grant, to the maximum extent possible, whether at the + time of the initial grant or subsequently, any and all of the rights conveyed by + this License. + +1.10. “Modifications” + + means any of the following: + + a. any file in Source Code Form that results from an addition to, deletion + from, or modification of the contents of Covered Software; or + + b. any new file in Source Code Form that contains any Covered Software. + +1.11. “Patent Claims” of a Contributor + + means any patent claim(s), including without limitation, method, process, + and apparatus claims, in any patent Licensable by such Contributor that + would be infringed, but for the grant of the License, by the making, + using, selling, offering for sale, having made, import, or transfer of + either its Contributions or its Contributor Version. + +1.12. “Secondary License” + + means either the GNU General Public License, Version 2.0, the GNU Lesser + General Public License, Version 2.1, the GNU Affero General Public + License, Version 3.0, or any later versions of those licenses. + +1.13. “Source Code Form” + + means the form of the work preferred for making modifications. + +1.14. “You” (or “Your”) + + means an individual or a legal entity exercising rights under this + License. For legal entities, “You” includes any entity that controls, is + controlled by, or is under common control with You. For purposes of this + definition, “control” means (a) the power, direct or indirect, to cause + the direction or management of such entity, whether by contract or + otherwise, or (b) ownership of more than fifty percent (50%) of the + outstanding shares or beneficial ownership of such entity. + + +2. License Grants and Conditions + +2.1. Grants + + Each Contributor hereby grants You a world-wide, royalty-free, + non-exclusive license: + + a. under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or as + part of a Larger Work; and + + b. under Patent Claims of such Contributor to make, use, sell, offer for + sale, have made, import, and otherwise transfer either its Contributions + or its Contributor Version. + +2.2. Effective Date + + The licenses granted in Section 2.1 with respect to any Contribution become + effective for each Contribution on the date the Contributor first distributes + such Contribution. + +2.3. Limitations on Grant Scope + + The licenses granted in this Section 2 are the only rights granted under this + License. No additional rights or licenses will be implied from the distribution + or licensing of Covered Software under this License. Notwithstanding Section + 2.1(b) above, no patent license is granted by a Contributor: + + a. for any code that a Contributor has removed from Covered Software; or + + b. for infringements caused by: (i) Your and any other third party’s + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + + c. under Patent Claims infringed by Covered Software in the absence of its + Contributions. + + This License does not grant any rights in the trademarks, service marks, or + logos of any Contributor (except as may be necessary to comply with the + notice requirements in Section 3.4). + +2.4. Subsequent Licenses + + No Contributor makes additional grants as a result of Your choice to + distribute the Covered Software under a subsequent version of this License + (see Section 10.2) or under the terms of a Secondary License (if permitted + under the terms of Section 3.3). + +2.5. Representation + + Each Contributor represents that the Contributor believes its Contributions + are its original creation(s) or it has sufficient rights to grant the + rights to its Contributions conveyed by this License. + +2.6. Fair Use + + This License is not intended to limit any rights You have under applicable + copyright doctrines of fair use, fair dealing, or other equivalents. + +2.7. Conditions + + Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in + Section 2.1. + + +3. Responsibilities + +3.1. Distribution of Source Form + + All distribution of Covered Software in Source Code Form, including any + Modifications that You create or to which You contribute, must be under the + terms of this License. You must inform recipients that the Source Code Form + of the Covered Software is governed by the terms of this License, and how + they can obtain a copy of this License. You may not attempt to alter or + restrict the recipients’ rights in the Source Code Form. + +3.2. Distribution of Executable Form + + If You distribute Covered Software in Executable Form then: + + a. such Covered Software must also be made available in Source Code Form, + as described in Section 3.1, and You must inform recipients of the + Executable Form how they can obtain a copy of such Source Code Form by + reasonable means in a timely manner, at a charge no more than the cost + of distribution to the recipient; and + + b. You may distribute such Executable Form under the terms of this License, + or sublicense it under different terms, provided that the license for + the Executable Form does not attempt to limit or alter the recipients’ + rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + + You may create and distribute a Larger Work under terms of Your choice, + provided that You also comply with the requirements of this License for the + Covered Software. If the Larger Work is a combination of Covered Software + with a work governed by one or more Secondary Licenses, and the Covered + Software is not Incompatible With Secondary Licenses, this License permits + You to additionally distribute such Covered Software under the terms of + such Secondary License(s), so that the recipient of the Larger Work may, at + their option, further distribute the Covered Software under the terms of + either this License or such Secondary License(s). + +3.4. Notices + + You may not remove or alter the substance of any license notices (including + copyright notices, patent notices, disclaimers of warranty, or limitations + of liability) contained within the Source Code Form of the Covered + Software, except that You may alter any license notices to the extent + required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + + You may choose to offer, and to charge a fee for, warranty, support, + indemnity or liability obligations to one or more recipients of Covered + Software. However, You may do so only on Your own behalf, and not on behalf + of any Contributor. You must make it absolutely clear that any such + warranty, support, indemnity, or liability obligation is offered by You + alone, and You hereby agree to indemnify every Contributor for any + liability incurred by such Contributor as a result of warranty, support, + indemnity or liability terms You offer. You may include additional + disclaimers of warranty and limitations of liability specific to any + jurisdiction. + +4. Inability to Comply Due to Statute or Regulation + + If it is impossible for You to comply with any of the terms of this License + with respect to some or all of the Covered Software due to statute, judicial + order, or regulation then You must: (a) comply with the terms of this License + to the maximum extent possible; and (b) describe the limitations and the code + they affect. Such description must be placed in a text file included with all + distributions of the Covered Software under this License. Except to the + extent prohibited by statute or regulation, such description must be + sufficiently detailed for a recipient of ordinary skill to be able to + understand it. + +5. Termination + +5.1. The rights granted under this License will terminate automatically if You + fail to comply with any of its terms. However, if You become compliant, + then the rights granted under this License from a particular Contributor + are reinstated (a) provisionally, unless and until such Contributor + explicitly and finally terminates Your grants, and (b) on an ongoing basis, + if such Contributor fails to notify You of the non-compliance by some + reasonable means prior to 60 days after You have come back into compliance. + Moreover, Your grants from a particular Contributor are reinstated on an + ongoing basis if such Contributor notifies You of the non-compliance by + some reasonable means, this is the first time You have received notice of + non-compliance with this License from such Contributor, and You become + compliant prior to 30 days after Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent + infringement claim (excluding declaratory judgment actions, counter-claims, + and cross-claims) alleging that a Contributor Version directly or + indirectly infringes any patent, then the rights granted to You by any and + all Contributors for the Covered Software under Section 2.1 of this License + shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user + license agreements (excluding distributors and resellers) which have been + validly granted by You or Your distributors under this License prior to + termination shall survive termination. + +6. Disclaimer of Warranty + + Covered Software is provided under this License on an “as is” basis, without + warranty of any kind, either expressed, implied, or statutory, including, + without limitation, warranties that the Covered Software is free of defects, + merchantable, fit for a particular purpose or non-infringing. The entire + risk as to the quality and performance of the Covered Software is with You. + Should any Covered Software prove defective in any respect, You (not any + Contributor) assume the cost of any necessary servicing, repair, or + correction. This disclaimer of warranty constitutes an essential part of this + License. No use of any Covered Software is authorized under this License + except under this disclaimer. + +7. Limitation of Liability + + Under no circumstances and under no legal theory, whether tort (including + negligence), contract, or otherwise, shall any Contributor, or anyone who + distributes Covered Software as permitted above, be liable to You for any + direct, indirect, special, incidental, or consequential damages of any + character including, without limitation, damages for lost profits, loss of + goodwill, work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses, even if such party shall have been + informed of the possibility of such damages. This limitation of liability + shall not apply to liability for death or personal injury resulting from such + party’s negligence to the extent applicable law prohibits such limitation. + Some jurisdictions do not allow the exclusion or limitation of incidental or + consequential damages, so this exclusion and limitation may not apply to You. + +8. Litigation + + Any litigation relating to this License may be brought only in the courts of + a jurisdiction where the defendant maintains its principal place of business + and such litigation shall be governed by laws of that jurisdiction, without + reference to its conflict-of-law provisions. Nothing in this Section shall + prevent a party’s ability to bring cross-claims or counter-claims. + +9. Miscellaneous + + This License represents the complete agreement concerning the subject matter + hereof. If any provision of this License is held to be unenforceable, such + provision shall be reformed only to the extent necessary to make it + enforceable. Any law or regulation which provides that the language of a + contract shall be construed against the drafter shall not be used to construe + this License against a Contributor. + + +10. Versions of the License + +10.1. New Versions + + Mozilla Foundation is the license steward. Except as provided in Section + 10.3, no one other than the license steward has the right to modify or + publish new versions of this License. Each version will be given a + distinguishing version number. + +10.2. Effect of New Versions + + You may distribute the Covered Software under the terms of the version of + the License under which You originally received the Covered Software, or + under the terms of any subsequent version published by the license + steward. + +10.3. Modified Versions + + If you create software not governed by this License, and you want to + create a new license for such software, you may create and use a modified + version of this License if you rename the license and remove any + references to the name of the license steward (except to note that such + modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses + If You choose to distribute Source Code Form that is Incompatible With + Secondary Licenses under the terms of this version of the License, the + notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice + + This Source Code Form is subject to the + terms of the Mozilla Public License, v. + 2.0. If a copy of the MPL was not + distributed with this file, You can + obtain one at + http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular file, then +You may include the notice in a location (such as a LICENSE file in a relevant +directory) where a recipient would be likely to look for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - “Incompatible With Secondary Licenses” Notice + + This Source Code Form is “Incompatible + With Secondary Licenses”, as defined by + the Mozilla Public License, v. 2.0. diff --git a/Makefile b/Makefile index f397651..b97cd6e 100644 --- a/Makefile +++ b/Makefile @@ -1,41 +1,31 @@ -PROJECT_NAME := titan -PKG := github.com/distributedio/$(PROJECT_NAME) -PKG_LIST := $(shell go list ${PKG}/... | grep -v /vendor/) -GITHASH := $(shell git rev-parse --short HEAD) -GO_FILES := $(shell find . -name '*.go' | grep -v /vendor/ | grep -v _test.go) -GO_LOGS := $(shell git log --abbrev-commit --oneline -n 1 | sed 's/$(GITHASH)//g' | sed 's/"//g' | sed "s/'//g") - -LDFLAGS += -X "$(PKG)/context.ReleaseVersion=$(shell git tag --contains)" -LDFLAGS += -X "$(PKG)/context.BuildTS=$(shell date -u '+%Y-%m-%d %I:%M:%S')" -LDFLAGS += -X "$(PKG)/context.GitHash=$(GITHASH)" -LDFLAGS += -X "$(PKG)/context.GolangVersion=$(shell go version)" -LDFLAGS += -X "$(PKG)/context.GitLog=$(GO_LOGS)" -LDFLAGS += -X "$(PKG)/context.GitBranch=$(shell git rev-parse --abbrev-ref HEAD)" - -.PHONY: all build clean test coverage lint proto -all: build token - -test: - env GO111MODULE=on go test -short ${PKG_LIST} - -coverage: - env GO111MODULE=on go test -covermode=count -v -coverprofile cover.cov ${PKG_LIST} - -build: - env GO111MODULE=on go build -ldflags '$(LDFLAGS)' -o titan ./bin/titan/ - -token: tools/token/main.go command/common.go - env GO111MODULE=on go build -ldflags '$(LDFLAGS)' -o token ./tools/token/ - -clean: - rm -f ./titan - rm -rf ./token - -lint: - golangci-lint run -p=bugs,complexity,format,performance,style,unused - -proto: - cd ./db/zlistproto && protoc --gofast_out=plugins=grpc:. ./zlist.proto - -release: build - ./tools/release.sh +TEST?=./... + +default: test + +# test runs the test suite and vets the code. +test: generate + @echo "==> Running tests..." + @go list $(TEST) \ + | grep -v "/vendor/" \ + | xargs -n1 go test -timeout=60s -parallel=10 ${TESTARGS} + +# testrace runs the race checker +testrace: generate + @echo "==> Running tests (race)..." + @go list $(TEST) \ + | grep -v "/vendor/" \ + | xargs -n1 go test -timeout=60s -race ${TESTARGS} + +# updatedeps installs all the dependencies needed to run and build. +updatedeps: + @sh -c "'${CURDIR}/scripts/deps.sh' '${NAME}'" + +# generate runs `go generate` to build the dynamically generated source files. +generate: + @echo "==> Generating..." + @find . -type f -name '.DS_Store' -delete + @go list ./... \ + | grep -v "/vendor/" \ + | xargs -n1 go generate + +.PHONY: default test testrace updatedeps generate diff --git a/README.md b/README.md index 0e6f9a8..3beef97 100644 --- a/README.md +++ b/README.md @@ -1,122 +1,151 @@ -# Titan +# go-multierror -[![Build Status](https://travis-ci.org/distributedio/titan.svg?branch=master)](https://travis-ci.org/distributedio/titan) -[![Go Report Card](https://goreportcard.com/badge/github.com/distributedio/titan)](https://goreportcard.com/report/github.com/distributedio/titan) -[![Coverage Status](https://coveralls.io/repos/github/distributedio/titan/badge.svg?branch=master)](https://coveralls.io/github/distributedio/titan?branch=master) -[![Coverage Status](https://img.shields.io/badge/version-v0.3.1-brightgreen.svg)](https://github.com/distributedio/titan/releases) -[![Discourse status](https://img.shields.io/discourse/https/meta.discourse.org/status.svg)](https://titan-tech-group.slack.com) +[![CircleCI](https://img.shields.io/circleci/build/github/hashicorp/go-multierror/master)](https://circleci.com/gh/hashicorp/go-multierror) +[![Go Reference](https://pkg.go.dev/badge/github.com/hashicorp/go-multierror.svg)](https://pkg.go.dev/github.com/hashicorp/go-multierror) +![GitHub go.mod Go version](https://img.shields.io/github/go-mod/go-version/hashicorp/go-multierror) -A distributed implementation of __Redis compatible layer__ based on [TiKV](https://github.com/tikv/tikv/) +[circleci]: https://app.circleci.com/pipelines/github/hashicorp/go-multierror +[godocs]: https://pkg.go.dev/github.com/hashicorp/go-multierror -## Why Titan? +`go-multierror` is a package for Go that provides a mechanism for +representing a list of `error` values as a single `error`. -* Completely compatible with Redis protocol -* Full distributed transaction with strong consistency -* Multi-tenancy support -* No painful scale out -* High availability +This allows a function in Go to return an `error` that might actually +be a list of errors. If the caller knows this, they can unwrap the +list and access the errors. If the caller doesn't know, the error +formats to a nice human-readable format. -Thanks [TiKV](https://github.com/tikv/tikv/) for supporting the core features. The project is developed and open sourced by the Beijing Infrastructure Team at [Meitu](https://www.meitu.com/) and has been donated to [DistributedIO](https://github.com/distributedio) org. +`go-multierror` is fully compatible with the Go standard library +[errors](https://golang.org/pkg/errors/) package, including the +functions `As`, `Is`, and `Unwrap`. This provides a standardized approach +for introspecting on error values. -## Architecture +## Installation and Docs -![titan](docs/titan.jpeg) +Install using `go get github.com/hashicorp/go-multierror`. -## Quick start +Full documentation is available at +https://pkg.go.dev/github.com/hashicorp/go-multierror -Can't wait to experiment with Titan? Just follow 2 steps: +### Requires go version 1.13 or newer -1. `curl -s -O https://raw.githubusercontent.com/distributedio/titan/master/docker-compose.yml` -2. `docker-compose up` +`go-multierror` requires go version 1.13 or newer. Go 1.13 introduced +[error wrapping](https://golang.org/doc/go1.13#error_wrapping), which +this library takes advantage of. -Then connect to Titan using `redis-cli` +If you need to use an earlier version of go, you can use the +[v1.0.0](https://github.com/hashicorp/go-multierror/tree/v1.0.0) +tag, which doesn't rely on features in go 1.13. + +If you see compile errors that look like the below, it's likely that +you're on an older version of go: ``` -redis-cli -p 7369 +/go/src/github.com/hashicorp/go-multierror/multierror.go:112:9: undefined: errors.As +/go/src/github.com/hashicorp/go-multierror/multierror.go:117:9: undefined: errors.Is ``` -___Enjoy!___ - -## Installation +## Usage -### SetUp TiKV cluster +go-multierror is easy to use and purposely built to be unobtrusive in +existing Go applications/libraries that may not be aware of it. -Titan works with 2 TiDB components: +**Building a list of errors** -* TiKV -* PD +The `Append` function is used to create a list of errors. This function +behaves a lot like the Go built-in `append` function: it doesn't matter +if the first argument is nil, a `multierror.Error`, or any other `error`, +the function behaves as you would expect. -To setup TiKV and PD, please follow the official [instructions](https://pingcap.com/docs-cn/dev/how-to/deploy/orchestrated/ansible/) +```go +var result error -### Run Titan +if err := step1(); err != nil { + result = multierror.Append(result, err) +} +if err := step2(); err != nil { + result = multierror.Append(result, err) +} -* Build the binary - -``` -go get github.com/distributedio/titan -cd $GOPATH/src/github.com/distributedio/titan -make +return result ``` -* Edit the configration file +**Customizing the formatting of the errors** -``` -pd-addrs="tikv://your-pd-addrs:port" -``` +By specifying a custom `ErrorFormat`, you can customize the format +of the `Error() string` function: -* Run Titan +```go +var result *multierror.Error -``` -./titan -``` +// ... accumulate errors here, maybe using Append -For more details about [Deploy Titan](docs/ops/deploy.md), click here. +if result != nil { + result.ErrorFormat = func([]error) string { + return "errors!" + } +} +``` -## Commands supporting status +**Accessing the list of errors** -See the details of the commands [supporting status](docs/command_list.md) +`multierror.Error` implements `error` so if the caller doesn't know about +multierror, it will work just fine. But if you're aware a multierror might +be returned, you can use type switches to access the list of errors: -| command | status | -| ------------ | ----------------------- | -| Connections | Almost Fully Supported | -| Transactions | Supported | -| Server | Almost Fully Supported | -| Keys | Supported | -| Strings | Almost Fully Supported | -| List | Almost Fully Supported | -| Hashes | Supported | -| Sets | Almost Fully Supported | -| Sorted Sets | Almost Fully Supported | -| Geo | Not Supported Yet | -| Hyperloglog | Not Supported Yet | -| Pub/Sub | Not Supported Yet | -| Scripting | Not Supported Yet | -| Streams | Not Supported Yet | +```go +if err := something(); err != nil { + if merr, ok := err.(*multierror.Error); ok { + // Use merr.Errors + } +} +``` -## Benchmarks +You can also use the standard [`errors.Unwrap`](https://golang.org/pkg/errors/#Unwrap) +function. This will continue to unwrap into subsequent errors until none exist. -Refer to the [benchmark docs](https://pan.baidu.com/s/1m5yp5LsvFjsDKvHtaXwWvg) for more details. It's shared on Baidu Disks, use the code `hzt6` to gain the permission. +**Extracting an error** -Basic benchmarking result. +The standard library [`errors.As`](https://golang.org/pkg/errors/#As) +function can be used directly with a multierror to extract a specific error: -### Get +```go +// Assume err is a multierror value +err := somefunc() -![Get command benchmark](docs/benchmark/get-benchmark.png) +// We want to know if "err" has a "RichErrorType" in it and extract it. +var errRich RichErrorType +if errors.As(err, &errRich) { + // It has it, and now errRich is populated. +} +``` -### Set +**Checking for an exact error value** -![Set command benchmark](docs/benchmark/set-benchmark.png) +Some errors are returned as exact errors such as the [`ErrNotExist`](https://golang.org/pkg/os/#pkg-variables) +error in the `os` package. You can check if this error is present by using +the standard [`errors.Is`](https://golang.org/pkg/errors/#Is) function. -For more info, please vist here [Titan Benchmarks](docs/benchmark/benchmark.md) +```go +// Assume err is a multierror value +err := somefunc() +if errors.Is(err, os.ErrNotExist) { + // err contains os.ErrNotExist +} +``` -## FAQ +**Returning a multierror only if there are errors** -[FAQ](https://github.com/distributedio/titan/issues?utf8=%E2%9C%93&q=+label%3A%22good+first+issue%22) +If you build a `multierror.Error`, you can use the `ErrorOrNil` function +to return an `error` implementation only if there are errors to return: -## Roadmap +```go +var result *multierror.Error -View our [Roadmap](https://github.com/distributedio/titan/projects) +// ... accumulate errors here -## Release Note -* 20.4.21: add support for rpop and rpoplpush -Hello World 2 +// Return the `error` only if errors were added to the multierror, otherwise +// return nil since there are no errors. +return result.ErrorOrNil() +``` +Hello World diff --git a/append.go b/append.go new file mode 100644 index 0000000..683b665 --- /dev/null +++ b/append.go @@ -0,0 +1,46 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package multierror + +// Append is a helper function that will append more errors +// onto an Error in order to create a larger multi-error. +// +// If err is not a multierror.Error, then it will be turned into +// one. If any of the errs are multierr.Error, they will be flattened +// one level into err. +// Any nil errors within errs will be ignored. If err is nil, a new +// *Error will be returned. +func Append(err error, errs ...error) *Error { + switch err := err.(type) { + case *Error: + // Typed nils can reach here, so initialize if we are nil + if err == nil { + err = new(Error) + } + + // Go through each error and flatten + for _, e := range errs { + switch e := e.(type) { + case *Error: + if e != nil { + err.Errors = append(err.Errors, e.Errors...) + } + default: + if e != nil { + err.Errors = append(err.Errors, e) + } + } + } + + return err + default: + newErrs := make([]error, 0, len(errs)+1) + if err != nil { + newErrs = append(newErrs, err) + } + newErrs = append(newErrs, errs...) + + return Append(&Error{}, newErrs...) + } +} diff --git a/append_test.go b/append_test.go new file mode 100644 index 0000000..e5179bf --- /dev/null +++ b/append_test.go @@ -0,0 +1,85 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package multierror + +import ( + "errors" + "testing" +) + +func TestAppend_Error(t *testing.T) { + original := &Error{ + Errors: []error{errors.New("foo")}, + } + + result := Append(original, errors.New("bar")) + if len(result.Errors) != 2 { + t.Fatalf("wrong len: %d", len(result.Errors)) + } + + original = &Error{} + result = Append(original, errors.New("bar")) + if len(result.Errors) != 1 { + t.Fatalf("wrong len: %d", len(result.Errors)) + } + + // Test when a typed nil is passed + var e *Error + result = Append(e, errors.New("baz")) + if len(result.Errors) != 1 { + t.Fatalf("wrong len: %d", len(result.Errors)) + } + + // Test flattening + original = &Error{ + Errors: []error{errors.New("foo")}, + } + + result = Append(original, Append(nil, errors.New("foo"), errors.New("bar"))) + if len(result.Errors) != 3 { + t.Fatalf("wrong len: %d", len(result.Errors)) + } +} + +func TestAppend_NilError(t *testing.T) { + var err error + result := Append(err, errors.New("bar")) + if len(result.Errors) != 1 { + t.Fatalf("wrong len: %d", len(result.Errors)) + } +} + +func TestAppend_NilErrorArg(t *testing.T) { + var err error + var nilErr *Error + result := Append(err, nilErr) + if len(result.Errors) != 0 { + t.Fatalf("wrong len: %d", len(result.Errors)) + } +} + +func TestAppend_NilErrorIfaceArg(t *testing.T) { + var err error + var nilErr error + result := Append(err, nilErr) + if len(result.Errors) != 0 { + t.Fatalf("wrong len: %d", len(result.Errors)) + } +} + +func TestAppend_NonError(t *testing.T) { + original := errors.New("foo") + result := Append(original, errors.New("bar")) + if len(result.Errors) != 2 { + t.Fatalf("wrong len: %d", len(result.Errors)) + } +} + +func TestAppend_NonError_Error(t *testing.T) { + original := errors.New("foo") + result := Append(original, Append(nil, errors.New("bar"))) + if len(result.Errors) != 2 { + t.Fatalf("wrong len: %d", len(result.Errors)) + } +} diff --git a/bin/titan/main.go b/bin/titan/main.go deleted file mode 100644 index d7d58d4..0000000 --- a/bin/titan/main.go +++ /dev/null @@ -1,221 +0,0 @@ -package main - -import ( - "flag" - "fmt" - "io" - "net/http" - _ "net/http/pprof" - "os" - ospath "path" - "time" - - rolling "github.com/arthurkiller/rollingwriter" - "github.com/distributedio/configo" - "github.com/distributedio/continuous" - "github.com/sirupsen/logrus" - "go.uber.org/zap" - "go.uber.org/zap/zapcore" - - "github.com/distributedio/titan" - "github.com/distributedio/titan/conf" - "github.com/distributedio/titan/context" - "github.com/distributedio/titan/db" - "github.com/distributedio/titan/metrics" - "github.com/distributedio/titan/server" -) - -func main() { - var showVersion bool - var confPath string - var pdAddrs string - - flag.BoolVar(&showVersion, "v", false, "Show Version") - flag.StringVar(&confPath, "c", "conf/titan.toml", "conf file path") - flag.StringVar(&pdAddrs, "pd-addrs", "", "pd cluster addresses") - flag.Parse() - - if showVersion { - titan.PrintVersionInfo() - return - } - - config := &conf.Titan{} - if err := configo.Load(confPath, config); err != nil { - fmt.Printf("unmarshal config file failed, %s\n", err) - os.Exit(1) - } - if pdAddrs != "" { - config.TiKV.PdAddrs = pdAddrs - } - if config.TiKV.PdAddrs == "mocktikv://" { - fmt.Println("Warning: Titan is running in memory mode, all the data would be cleared after the process exit.") - fmt.Println("The memory mode can only be used for experience, configure the tikv.pd-addrs to use the TiKV as a backend") - } - - if err := ConfigureZap(config.Logger.Name, config.Logger.Path, config.Logger.Level, - config.Logger.TimeRotate, config.Logger.Compress); err != nil { - fmt.Printf("create logger failed, %s\n", err) - os.Exit(1) - } - - if err := ConfigureLogrus(config.TiKV.Logger.Path, config.TiKV.Logger.Level, - config.TiKV.Logger.TimeRotate, config.TiKV.Logger.Compress); err != nil { - fmt.Printf("create tikv logger failed, %s\n", err) - os.Exit(1) - } - - store, err := db.Open(&config.TiKV) - if err != nil { - zap.L().Fatal("open db failed", zap.Error(err)) - os.Exit(1) - } - - svr := metrics.NewServer(&config.Status) - - serv := titan.New(&context.ServerContext{ - RequirePass: config.Server.Auth, - Store: store, - ListZipThreshold: config.Server.ListZipThreshold, - }) - - var servOpts, statusOpts []continuous.ServerOption - - // titan server tls options - if config.Server.SSLCertFile != "" && config.Server.SSLKeyFile != "" { - tlsConfig, err := server.TLSConfig(config.Server.SSLCertFile, config.Server.SSLKeyFile) - if err != nil { - zap.L().Fatal("failed to load server TLS config", zap.Error(err)) - } - servOpts = append(servOpts, continuous.TLSConfig(tlsConfig)) - } - - // status server tls options - if config.Status.SSLCertFile != "" && config.Status.SSLKeyFile != "" { - tlsConfig, err := server.TLSConfig(config.Status.SSLCertFile, config.Status.SSLKeyFile) - if err != nil { - zap.L().Fatal("failed to load status server TLS config", zap.Error(err)) - } - statusOpts = append(statusOpts, continuous.TLSConfig(tlsConfig)) - } - - writer, err := Writer(config.Logger.Path, config.Logger.TimeRotate, config.Logger.Compress) - if err != nil { - zap.L().Fatal("create writer for continuous failed", zap.Error(err)) - } - cont := continuous.New(continuous.LoggerOutput(writer), continuous.PidFile(config.PIDFileName)) - if err := cont.AddServer(serv, &continuous.ListenOn{Network: "tcp", Address: config.Server.Listen}, servOpts...); err != nil { - zap.L().Fatal("add titan server failed:", zap.Error(err)) - } - - if err := cont.AddServer(svr, &continuous.ListenOn{Network: "tcp", Address: config.Status.Listen}, statusOpts...); err != nil { - zap.L().Fatal("add statues server failed:", zap.Error(err)) - } - - if err := cont.Serve(); err != nil { - zap.L().Fatal("run server failed:", zap.Error(err)) - } -} - -// ConfigureZap customize the zap logger -func ConfigureZap(name, path, level, pattern string, compress bool) error { - writer, err := Writer(path, pattern, compress) - if err != nil { - return err - } - - var lv = zap.NewAtomicLevel() - switch level { - case "debug": - lv.SetLevel(zap.DebugLevel) - case "info": - lv.SetLevel(zap.InfoLevel) - case "warn": - lv.SetLevel(zap.WarnLevel) - case "error": - lv.SetLevel(zap.ErrorLevel) - case "panic": - lv.SetLevel(zap.PanicLevel) - case "fatal": - lv.SetLevel(zap.FatalLevel) - default: - return fmt.Errorf("unknown log level(%s)", level) - } - timeEncoder := func(t time.Time, enc zapcore.PrimitiveArrayEncoder) { - enc.AppendString(t.Local().Format("2006-01-02 15:04:05.999999999")) - } - - encoderCfg := zapcore.EncoderConfig{ - NameKey: "Name", - StacktraceKey: "Stack", - MessageKey: "Message", - LevelKey: "Level", - TimeKey: "TimeStamp", - CallerKey: "Caller", - EncodeTime: timeEncoder, - EncodeLevel: zapcore.CapitalLevelEncoder, - EncodeDuration: zapcore.StringDurationEncoder, - EncodeCaller: zapcore.ShortCallerEncoder, - } - - output := zapcore.AddSync(writer) - var zapOpts []zap.Option - zapOpts = append(zapOpts, zap.AddCaller()) - zapOpts = append(zapOpts, zap.Hooks(metrics.Measure)) - - logger := zap.New(zapcore.NewCore(zapcore.NewJSONEncoder(encoderCfg), output, lv), zapOpts...) - logger.Named(name) - log := logger.With(zap.Int("PID", os.Getpid())) - zap.ReplaceGlobals(log) - //http change log level - http.Handle("/titan/log/level", lv) - - return nil -} - -// ConfigureLogrus customize the logrus logger, which is used by TiKV SDK -func ConfigureLogrus(path, level, pattern string, compress bool) error { - writer, err := Writer(path, pattern, compress) - if err != nil { - return err - } - logrus.SetOutput(writer) - switch level { - case "debug": - logrus.SetLevel(logrus.DebugLevel) - case "info": - logrus.SetLevel(logrus.InfoLevel) - case "warn": - logrus.SetLevel(logrus.WarnLevel) - case "error": - logrus.SetLevel(logrus.ErrorLevel) - case "panic": - logrus.SetLevel(logrus.PanicLevel) - case "fatal": - logrus.SetLevel(logrus.FatalLevel) - default: - return fmt.Errorf("unknown log level(%s)", level) - } - return nil -} - -//Writer generate the rollingWriter -func Writer(path, pattern string, compress bool) (io.Writer, error) { - if path == "stdout" { - return os.Stdout, nil - } else if path == "stderr" { - return os.Stderr, nil - } - var opts []rolling.Option - opts = append(opts, rolling.WithRollingTimePattern(pattern)) - if compress { - opts = append(opts, rolling.WithCompress()) - } - dir, filename := ospath.Split(path) - opts = append(opts, rolling.WithLogPath(dir), rolling.WithFileName(filename), rolling.WithLock()) - writer, err := rolling.NewWriter(opts...) - if err != nil { - return nil, fmt.Errorf("create IOWriter failed, %s", err) - } - return writer, nil -} diff --git a/bin/titan/main1_test.go b/bin/titan/main1_test.go deleted file mode 100644 index d3ad486..0000000 --- a/bin/titan/main1_test.go +++ /dev/null @@ -1,29 +0,0 @@ -package main - -import ( - "io/ioutil" - "os" - "path" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestWriter(t *testing.T) { - stream, _err := Writer("stdout", "", true) - assert.Equal(t, stream, os.Stdout) - assert.Nil(t, _err) - - stream, _err = Writer("stderr", "", true) - assert.Equal(t, stream, os.Stderr) - assert.Nil(t, _err) - - td, td_err := ioutil.TempDir("", "titan-test") - assert.Nil(t, td_err) - stream, _err = Writer(path.Join(td, "titan-test-log"), "* * * * *", true) - assert.Nil(t, _err) - - stream, _err = Writer(path.Join(td, "titan-test-log"), "", true) - assert.NotNil(t, _err) - -} diff --git a/client.go b/client.go deleted file mode 100644 index bc1b405..0000000 --- a/client.go +++ /dev/null @@ -1,177 +0,0 @@ -package titan - -import ( - "bufio" - "io" - "io/ioutil" - "net" - "strings" - "sync" - "time" - - "github.com/distributedio/titan/command" - "github.com/distributedio/titan/context" - "github.com/distributedio/titan/encoding/resp" - "go.uber.org/zap" -) - -type client struct { - cliCtx *context.ClientContext - server *Server - conn net.Conn - exec *command.Executor - r *bufio.Reader - - eofLock sync.Mutex //the lock of reading_writing 'eof' - eof bool //is over when read data from socket -} - -func newClient(cliCtx *context.ClientContext, s *Server, exec *command.Executor) *client { - return &client{ - cliCtx: cliCtx, - server: s, - exec: exec, - eof: false, - } -} - -func (c *client) readEof() { - c.eofLock.Lock() - defer c.eofLock.Unlock() - - c.eof = true -} - -func (c *client) isEof() bool { - c.eofLock.Lock() - defer c.eofLock.Unlock() - - return c.eof -} - -// Write to conn and log error if needed -func (c *client) Write(p []byte) (int, error) { - zap.L().Debug("write to client", zap.Int64("clientid", c.cliCtx.ID), zap.String("msg", string(p))) - n, err := c.conn.Write(p) - if err != nil { - c.conn.Close() - if err == io.EOF { - zap.L().Info("close connection", zap.String("addr", c.cliCtx.RemoteAddr), - zap.Int64("clientid", c.cliCtx.ID)) - } else { - zap.L().Error("write net failed", zap.String("addr", c.cliCtx.RemoteAddr), - zap.Int64("clientid", c.cliCtx.ID), - zap.String("namespace", c.cliCtx.Namespace), - zap.Bool("multi", c.cliCtx.Multi), - zap.Bool("watching", c.cliCtx.Txn != nil), - zap.String("command", c.cliCtx.LastCmd), - zap.String("error", err.Error())) - return 0, err - } - } - return n, nil -} - -func (c *client) serve(conn net.Conn) error { - c.conn = conn - c.r = bufio.NewReader(conn) - - var cmd []string - var err error - for { - select { - case <-c.cliCtx.Done: - return c.conn.Close() - default: - cmd, err = c.readCommand() - if err != nil { - c.conn.Close() - if err == io.EOF { - zap.L().Info("close connection", zap.String("addr", c.cliCtx.RemoteAddr), - zap.Int64("clientid", c.cliCtx.ID)) - return nil - } - zap.L().Error("read command failed", zap.String("addr", c.cliCtx.RemoteAddr), - zap.Int64("clientid", c.cliCtx.ID), zap.Error(err)) - return err - } - } - - if c.server.servCtx.Pause > 0 { - time.Sleep(c.server.servCtx.Pause) - c.server.servCtx.Pause = 0 - } - if len(cmd) == 0 { - continue - } - - c.cliCtx.Updated = time.Now() - c.cliCtx.LastCmd = cmd[0] - - ctx := &command.Context{ - Name: cmd[0], - Args: cmd[1:], - In: c.r, - Out: c, - TraceID: GenerateTraceID(), - } - - ctx.Context = context.New(c.cliCtx, c.server.servCtx) - zap.L().Debug("recv msg", zap.String("command", ctx.Name), zap.Strings("arguments", ctx.Args)) - - // Skip reply if necessary - if c.cliCtx.SkipN != 0 { - ctx.Out = ioutil.Discard - if c.cliCtx.SkipN > 0 { - c.cliCtx.SkipN-- - } - } - if env := zap.L().Check(zap.DebugLevel, "recv client command"); env != nil { - env.Write(zap.String("addr", c.cliCtx.RemoteAddr), - zap.Int64("clientid", c.cliCtx.ID), - zap.String("traceid", ctx.TraceID), - zap.String("command", ctx.Name)) - } - - c.exec.Execute(ctx) - } -} - -func (c *client) readInlineCommand() ([]string, error) { - buf, err := c.r.ReadBytes('\n') - if err != nil { - return nil, err - } - - line := strings.TrimRight(string(buf), "\r\n") - return strings.Fields(line), nil -} - -func (c *client) readCommand() ([]string, error) { - p, err := c.r.Peek(1) - if err != nil { - return nil, err - } - // not a bulk string - if p[0] != '*' { - return c.readInlineCommand() - } - - argc, err := resp.ReadArray(c.r) - if err != nil { - return nil, err - } - if argc == 0 { - return []string{}, nil - } - - argv := make([]string, argc) - for i := 0; i < argc; i++ { - arg, err := resp.ReadBulkString(c.r) - if err != nil { - return nil, err - } - argv[i] = arg - } - return argv, nil -} diff --git a/codelingo.yaml b/codelingo.yaml deleted file mode 100644 index 81ea436..0000000 --- a/codelingo.yaml +++ /dev/null @@ -1,4 +0,0 @@ -tenets: -- import: codelingo/go -- import: codelingo/effective-go -- import: codelingo/code-review-comments diff --git a/command/changetype_test.go b/command/changetype_test.go deleted file mode 100644 index dd910c2..0000000 --- a/command/changetype_test.go +++ /dev/null @@ -1,117 +0,0 @@ -package command - -import ( - "github.com/stretchr/testify/assert" - "testing" - "time" -) - -func TestNoExpireChange(t *testing.T){ - key := "string_key" - args := []string{key, "10"} - - ctx := ContextTest("set", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), "OK") - EqualGet(t, key, "10", nil) - - args = []string{key, "a", "b", "c"} - ctx1 := ContextTest("sadd", args...) - Call(ctx1) - assert.Contains(t, ctxString(ctx1.Out), "WRONGTYPE") - - args = []string{key} - ctx2 := ContextTest("scard", args...) - Call(ctx2) - assert.Contains(t, ctxString(ctx2.Out), "WRONGTYPE") - - args = []string{key, "1", "a"} - ctx3 := ContextTest("zadd", args...) - Call(ctx3) - assert.Contains(t, ctxString(ctx3.Out), "WRONGTYPE") -} -func TestExpireChange(t *testing.T){ - - Cfg.Expire.Disable = true - Cfg.GC.Disable = true - time.Sleep(time.Second) - - key := "string_key" - args := []string{key, "10", "ex", "4"} - - ctx := ContextTest("set", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), "OK") - - args = []string{key, "a", "b", "c"} - ctx1 := ContextTest("sadd", args...) - Call(ctx1) - assert.Contains(t, ctxString(ctx1.Out), "WRONGTYPE") - - time.Sleep(time.Second * 4) - - args = []string{key, "a", "b", "c"} - ctx2 := ContextTest("sadd", args...) - Call(ctx2) - assert.Contains(t, ctxString(ctx2.Out), "3") - - args = []string{key, "4"} - ctx = ContextTest("expire", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), "1") - - args = []string{key, "1", "a"} - ctx = ContextTest("zadd", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), "WRONGTYPE") - - time.Sleep(time.Second * 4) - - args = []string{key, "1", "a"} - ctx = ContextTest("zadd", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), "1") - - args = []string{key, "4"} - ctx = ContextTest("expire", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), "1") - - args = []string{key, "a", "1"} - ctx = ContextTest("hset", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), "WRONGTYPE") - - time.Sleep(time.Second * 4) - - args = []string{key, "0", "2"} - ctx = ContextTest("zrange", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), "0") - - args = []string{key, "1", "a"} - ctx = ContextTest("hset", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), "1") - - args = []string{key, "4"} - ctx = ContextTest("expire", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), "1") - - args = []string{key, "a"} - ctx = ContextTest("lpush", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), "WRONGTYPE") - - time.Sleep(time.Second * 4) - - args = []string{key, "a"} - ctx = ContextTest("lpush", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), "1") - - Cfg.Expire.Disable = false - Cfg.GC.Disable = false - time.Sleep(time.Second) -} diff --git a/command/command.go b/command/command.go deleted file mode 100644 index 83fc6ee..0000000 --- a/command/command.go +++ /dev/null @@ -1,310 +0,0 @@ -package command - -import ( - "io" - "strconv" - "strings" - "time" - - "github.com/distributedio/titan/context" - "github.com/distributedio/titan/db" - "github.com/distributedio/titan/encoding/resp" - "github.com/distributedio/titan/metrics" - "github.com/shafreeck/retry" - "go.uber.org/zap" -) - -// Context is the runtime context of a command -type Context struct { - Name string - Args []string - In io.Reader - Out io.Writer - TraceID string - *context.Context -} - -// Command is a redis command implementation -type Command func(ctx *Context) - -// OnCommit returns by TxnCommand and will be called after a transaction being committed -type OnCommit func() - -// SimpleString replies a simplestring when commit -func SimpleString(w io.Writer, s string) OnCommit { - return func() { - resp.ReplySimpleString(w, s) - } -} - -// BulkString replies a bulkstring when commit -func BulkString(w io.Writer, s string) OnCommit { - return func() { - resp.ReplyBulkString(w, s) - } -} - -// NullBulkString replies a null bulkstring when commit -func NullBulkString(w io.Writer) OnCommit { - return func() { - resp.ReplyNullBulkString(w) - } -} - -// Integer replies in integer when commit -func Integer(w io.Writer, v int64) OnCommit { - return func() { - resp.ReplyInteger(w, v) - } -} - -// BytesArray replies a [][]byte when commit -func BytesArray(w io.Writer, a [][]byte) OnCommit { - return func() { - start := time.Now() - resp.ReplyArray(w, len(a)) - zap.L().Debug("reply array size", zap.Int64("cost(us)", time.Since(start).Nanoseconds()/1000)) - start = time.Now() - for i := range a { - if a[i] == nil { - resp.ReplyNullBulkString(w) - continue - } - resp.ReplyBulkString(w, string(a[i])) - if i%10 == 9 { - zap.L().Debug("reply 10 bulk string", zap.Int64("cost(us)", time.Since(start).Nanoseconds()/1000)) - start = time.Now() - } - } - } -} - -// TxnCommand runs a command in transaction -type TxnCommand func(ctx *Context, txn *db.Transaction) (OnCommit, error) - -// Call a command -func Call(ctx *Context) { - ctx.Name = strings.ToLower(ctx.Name) - - if ctx.Name != "auth" && - ctx.Server.RequirePass != "" && - ctx.Client.Authenticated == false { - resp.ReplyError(ctx.Out, ErrNoAuth.Error()) - return - } - // Exec all queued commands if this is an exec command - if ctx.Name == "exec" { - if len(ctx.Args) != 0 { - resp.ReplyError(ctx.Out, ErrWrongArgs(ctx.Name).Error()) - return - } - // Exec must begin with multi - if !ctx.Client.Multi { - resp.ReplyError(ctx.Out, ErrExec.Error()) - return - } - - Exec(ctx) - feedMonitors(ctx) - return - } - // Discard all queued commands and return - if ctx.Name == "discard" { - if !ctx.Client.Multi { - resp.ReplyError(ctx.Out, ErrDiscard.Error()) - return - } - - Discard(ctx) - feedMonitors(ctx) - return - } - - cmdInfoCommand, ok := commands[ctx.Name] - if !ok { - resp.ReplyError(ctx.Out, ErrUnKnownCommand(ctx.Name).Error()) - return - } - argc := len(ctx.Args) + 1 // include the command name - arity := cmdInfoCommand.Cons.Arity - - if arity > 0 && argc != arity { - resp.ReplyError(ctx.Out, ErrWrongArgs(ctx.Name).Error()) - return - } - - if arity < 0 && argc < -arity { - resp.ReplyError(ctx.Out, ErrWrongArgs(ctx.Name).Error()) - return - } - - // We now in a multi block, queue the command and return - if ctx.Client.Multi { - if ctx.Name == "multi" { - resp.ReplyError(ctx.Out, ErrMultiNested.Error()) - return - } - commands := ctx.Client.Commands - commands = append(commands, &context.Command{Name: ctx.Name, Args: ctx.Args}) - ctx.Client.Commands = commands - resp.ReplySimpleString(ctx.Out, "QUEUED") - return - } - - feedMonitors(ctx) - start := time.Now() - cmdInfoCommand.Proc(ctx) - cost := time.Since(start) - - cmdInfoCommand.Stat.Calls++ - cmdInfoCommand.Stat.Microseconds += cost.Nanoseconds() / int64(1000) -} - -// TxnCall calls a command with transaction, it is used with multi/exec -func TxnCall(ctx *Context, txn *db.Transaction) (OnCommit, error) { - name := strings.ToLower(ctx.Name) - desc, ok := commands[name] - if !ok || desc.Txn == nil { - return nil, ErrUnKnownCommand(ctx.Name) - } - feedMonitors(ctx) - cmd := desc.Txn - return cmd(ctx, txn) -} - -// AutoCommit commits to database after run a txn command -func AutoCommit(cmd TxnCommand) Command { - return func(ctx *Context) { - retry.Ensure(ctx, func() error { - mt := metrics.GetMetrics() - start := time.Now() - txn, err := ctx.Client.DB.Begin() - key := "" - if len(ctx.Args) > 0 { - key = ctx.Args[0] - if len(ctx.Args) > 1 { - mt.CommandArgsNumHistogramVec.WithLabelValues(ctx.Client.Namespace, ctx.Name).Observe(float64(len(ctx.Args) - 1)) - } - } - cost := time.Since(start).Seconds() - zap.L().Debug("transation begin", zap.String("name", ctx.Name), zap.String("key", key), zap.Int64("cost(us)", int64(cost*1000000))) - mt.TxnBeginHistogramVec.WithLabelValues(ctx.Client.Namespace, ctx.Name).Observe(cost) - if err != nil { - mt.TxnFailuresCounterVec.WithLabelValues(ctx.Client.Namespace, ctx.Name).Inc() - resp.ReplyError(ctx.Out, "ERR "+err.Error()) - zap.L().Error("txn begin failed", - zap.Int64("clientid", ctx.Client.ID), - zap.String("command", ctx.Name), - zap.String("traceid", ctx.TraceID), - zap.Error(err)) - return err - } - - start = time.Now() - onCommit, err := cmd(ctx, txn) - cost = time.Since(start).Seconds() - zap.L().Debug("command done", zap.String("name", ctx.Name), zap.String("key", key), zap.Int64("cost(us)", int64(cost*1000000))) - mt.CommandFuncDoneHistogramVec.WithLabelValues(ctx.Client.Namespace, ctx.Name).Observe(cost) - if err != nil { - mt.TxnFailuresCounterVec.WithLabelValues(ctx.Client.Namespace, ctx.Name).Inc() - resp.ReplyError(ctx.Out, err.Error()) - txn.Rollback() - zap.L().Error("command process failed", - zap.Int64("clientid", ctx.Client.ID), - zap.String("command", ctx.Name), - zap.String("traceid", ctx.TraceID), - zap.Error(err)) - return err - } - - start = time.Now() - mtFunc := func() { - cost = time.Since(start).Seconds() - mt.TxnCommitHistogramVec.WithLabelValues(ctx.Client.Namespace, ctx.Name).Observe(cost) - } - if err := txn.Commit(ctx); err != nil { - txn.Rollback() - mt.TxnFailuresCounterVec.WithLabelValues(ctx.Client.Namespace, ctx.Name).Inc() - if db.IsRetryableError(err) { - mt.TxnRetriesCounterVec.WithLabelValues(ctx.Client.Namespace, ctx.Name).Inc() - mt.TxnConflictsCounterVec.WithLabelValues(ctx.Client.Namespace, ctx.Name).Inc() - mtFunc() - zap.L().Error("txn commit retry", - zap.Int64("clientid", ctx.Client.ID), - zap.String("command", ctx.Name), - zap.String("traceid", ctx.TraceID), - zap.Error(err)) - return retry.Retriable(err) - } - resp.ReplyError(ctx.Out, "ERR "+err.Error()) - mtFunc() - zap.L().Error("txn commit failed", - zap.Int64("clientid", ctx.Client.ID), - zap.String("command", ctx.Name), - zap.String("traceid", ctx.TraceID), - zap.Error(err)) - return err - } - zap.L().Debug("commit ", zap.String("name", ctx.Name), zap.String("key", key), zap.Int64("cost(us)", time.Since(start).Nanoseconds()/1000)) - - start = time.Now() - if onCommit != nil { - onCommit() - } - cost = time.Since(start).Seconds() - zap.L().Debug("onCommit ", zap.String("name", ctx.Name), zap.String("key", key), zap.Int64("cost(us)", int64(cost*1000000))) - mt.ReplyFuncDoneHistogramVec.WithLabelValues(ctx.Client.Namespace, ctx.Name).Observe(cost) - mtFunc() - return nil - }) - } -} - -func feedMonitors(ctx *Context) { - ctx.Server.Monitors.Range(func(k, v interface{}) bool { - mCtx := v.(*Context) - if mCtx.Client.Namespace != sysAdminNamespace && mCtx.Client.Namespace != ctx.Client.Namespace { - return true - } - - now := time.Now().UnixNano() / 1000 - ts := strconv.FormatFloat(float64(now)/1000000, 'f', -1, 64) - id := strconv.FormatInt(int64(ctx.Client.DB.ID), 10) - - line := ts + " [" + id + " " + ctx.Client.RemoteAddr + "]" + " " + ctx.Name + " " + strings.Join(ctx.Args, " ") - start := time.Now() - err := resp.ReplySimpleString(mCtx.Out, line) - zap.L().Debug("feedMonitors reply", zap.String("name", ctx.Name), zap.Int64("cost(us)", time.Since(start).Nanoseconds()/1000)) - if err != nil { - ctx.Server.Monitors.Delete(k) - } - - return true - }) -} - -// Executor executes a command -type Executor struct { - commands map[string]Desc -} - -// NewExecutor news a Executor -func NewExecutor() *Executor { - return &Executor{commands: commands} -} - -// Execute a command -func (e *Executor) Execute(ctx *Context) { - start := time.Now() - Call(ctx) - cost := time.Since(start).Seconds() - metrics.GetMetrics().CommandCallHistogramVec.WithLabelValues(ctx.Client.Namespace, ctx.Name).Observe(cost) -} - -// Desc describes a command with constraints -type Desc struct { - Proc Command - Txn TxnCommand - Stat Statistic - Cons Constraint -} diff --git a/command/command_test.go b/command/command_test.go deleted file mode 100644 index 3525337..0000000 --- a/command/command_test.go +++ /dev/null @@ -1,115 +0,0 @@ -package command - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func matchPrefixCase() map[string]string { - cs := map[string]string{ - "abc?[a-z]": "abc", - "?abc": "", - "\\*[^abc]?": "*", - } - return cs -} - -type patternMap map[string]bool - -func matchCase(nocase bool) map[string]*patternMap { - var cs map[string]*patternMap - if !nocase { - cs = map[string]*patternMap{ - "*": &patternMap{ - "": true, - "abcd": true, - "*[*]": true, - }, - "******a": &patternMap{ - "a": true, - "***a": true, - "bcdea": true, - "abcd": false, - }, - "\\*?aaa": &patternMap{ - "*caaa": true, - "abc": false, - }, - "[a-z][^0-9][z-a]?[a-z": &patternMap{ - "abz.a": true, - "a1z.*": false, - "abz.e": true, - }, - "[a-z]*cat*[h][^b]*eyes*": &patternMap{ - "my cat has very bright eyes": true, - "my dog has very bright eyes": false, - }, - "h?llo": &patternMap{ - "hello": true, - "healo": false, - }, - "h??lo": &patternMap{ - "hello": true, - }, - "h*o": &patternMap{ - "hello": true, - "ho": true, - }, - } - - } else { - cs = map[string]*patternMap{ - "[A-Z][0-9]*": &patternMap{ - "B1": true, - "B2000": true, - "b2000": false, - }, - "*A": &patternMap{ - "abcdA": true, - "abcda": false, - "Ae": false, - }, - "?A*C": &patternMap{ - "1AbcdC": true, - "cA12344C": true, - "1abcdc": false, - }, - } - } - - return cs -} - -func TestGlobMatchPrefix(t *testing.T) { - list := matchPrefixCase() - for match, exptected := range list { - val := globMatchPrefix([]byte(match)) - assert.Equal(t, exptected, string(val)) - } -} - -func TestPatternMatch(t *testing.T) { - cs := matchCase(false) - for pattern, vals := range cs { - for val, expected := range map[string]bool(*vals) { - actual := globMatch([]byte(pattern), []byte(val), false) - assert.Equal(t, expected, actual, "err log:", pattern, val) - } - } - - // check upper string - cs = matchCase(true) - for pattern, vals := range cs { - for val, expected := range map[string]bool(*vals) { - actual := globMatch([]byte(pattern), []byte(val), true) - assert.Equal(t, expected, actual, "err log:", pattern, val) - } - } -} - -func BenchmarkGlobMatch(b *testing.B) { - for i := 0; i < b.N; i++ { - globMatch([]byte("hellabcdlo"), []byte("h*lo"), false) - } -} diff --git a/command/common.go b/command/common.go deleted file mode 100644 index 1fe1043..0000000 --- a/command/common.go +++ /dev/null @@ -1,262 +0,0 @@ -package command - -import ( - "bytes" - "crypto/hmac" - "crypto/sha256" - "encoding/hex" - "errors" - "math" - "strconv" - "strings" -) - -//tokenSignLen token default len -const tokenSignLen = 11 - -//Base token base msg -type Base struct { - Version int8 `json:"version"` - CreateAt int64 `json:"create_at"` - Namespace []byte `json:"namespace"` - // Sign string `json:"-"` -} - -//MarshalBinary Namespace SHOULD NOT contains a colon -func (t *Base) MarshalBinary() (data []byte, err error) { - data = append(data, t.Namespace...) - data = append(data, '-') - data = append(data, []byte(strconv.FormatInt(t.CreateAt, 10))...) - data = append(data, '-') - data = append(data, []byte(strconv.FormatInt(int64(t.Version), 10))...) - return data, nil -} - -//UnmarshalBinary token base unmarshl -func (t *Base) UnmarshalBinary(data []byte) error { - fields := bytes.Split(data, []byte{'-'}) - l := len(fields) - if l < 3 { - return errors.New("invalid token") - } - - version, err := strconv.ParseInt(string(fields[l-1]), 10, 64) - if err != nil { - return err - } - t.Version = int8(version) - - createAt, err := strconv.ParseInt(string(fields[l-2]), 10, 64) - if err != nil { - return err - } - t.CreateAt = createAt - - t.Namespace = bytes.Join(fields[:l-2], []byte("")) - - return nil -} - -func Verify(token, key []byte) ([]byte, error) { - encodedSignLen := hex.EncodedLen(tokenSignLen) - if len(token) < encodedSignLen || len(key) == 0 { - return nil, errors.New("token or key is parameter illegal") - - } - - sign := make([]byte, tokenSignLen) - hex.Decode(sign, token[len(token)-encodedSignLen:]) - - meta := token[:len(token)-encodedSignLen-1] //counting in the ":" - mac := hmac.New(sha256.New, key) - mac.Write(meta) - - if !hmac.Equal(mac.Sum(nil)[:tokenSignLen], sign) { - return nil, errors.New("token mismatch") - } - - var t Base - if err := t.UnmarshalBinary(meta); err != nil { - return nil, err - } - return t.Namespace, nil -} - -//Token token create through key server namespace create time -func Token(key, namespace []byte, createAt int64) ([]byte, error) { - t := &Base{Namespace: namespace, CreateAt: createAt, Version: 1} - data, err := t.MarshalBinary() - if err != nil { - return nil, err - } - - mac := hmac.New(sha256.New, key) - mac.Write(data) - sign := mac.Sum(nil) - - //truncate to 32 byte: https://tools.ietf.org/html/rfc2104#section-5 - // we have 11 byte rigth of hmac,so the rest of data is token message - sign = sign[:tokenSignLen] - - encodedSign := make([]byte, hex.EncodedLen(len(sign))) - hex.Encode(encodedSign, sign) - var token []byte - token = append(token, data...) - token = append(token, '-') - token = append(token, encodedSign...) - return token, nil -} - -// globMatch matches s with pattern in glob-style -func globMatch(pattern, val []byte, nocase bool) bool { - if !nocase { - pattern = bytes.ToLower(pattern) - val = bytes.ToLower(val) - } - for len(pattern) > 0 { - switch pattern[0] { - case '*': - for len(pattern) >= 2 && pattern[1] == '*' { - pattern = pattern[1:] - } - if len(pattern) == 1 { - return true - } - for len(val) > 0 { - if globMatch(pattern[1:], val, nocase) { - return true - } - val = val[1:] - } - return false - case '?': - if len(val) == 0 { - return false - } - val = val[1:] - case '[': - pattern = pattern[1:] - not := false - if len(pattern) > 0 && pattern[0] == '^' { - not = true - pattern = pattern[1:] - } - - var match bool - for len(pattern) > 0 { - if len(pattern) >= 2 && pattern[0] == '\\' { - pattern = pattern[1:] - if pattern[0] == val[0] { - match = true - } - } else if pattern[0] == ']' { - break - } else if len(pattern) >= 3 && pattern[1] == '-' { - if val[0] >= pattern[0] && val[0] <= pattern[2] || val[0] <= pattern[0] && val[0] >= pattern[2] { - match = true - } - pattern = pattern[2:] - } else if pattern[0] == val[0] { - match = true - } else if len(pattern) == 1 { - break - } - if len(pattern) > 0 { - pattern = pattern[1:] - } - } - if not { - match = !match - } - if !match { - return false - } - val = val[1:] - case '\\': - if len(pattern) >= 2 { - pattern = pattern[1:] - } - fallthrough - default: - if pattern[0] != val[0] { - return false - } - val = val[1:] - } - if len(pattern) > 0 { - pattern = pattern[1:] - } - if len(val) == 0 { - for len(pattern) > 0 && pattern[0] == '*' { - pattern = pattern[1:] - } - break - } - } - if len(pattern) == 0 && len(val) == 0 { - return true - } - return false - -} - -//globMatchPrefix Glob-style patter prefix -func globMatchPrefix(val []byte) []byte { - var v []byte - pattern := val - for i := 0; i < len(pattern); i++ { - switch pattern[i] { - case '\\': - if i+1 < len(pattern) { - i++ - v = append(v, pattern[i]) - } - case '*', '[', ']', '?': - return v - default: - v = append(v, pattern[i]) - } - } - return v -} - -func getFloatAndInclude(strf string) (float64, bool, error) { - var f float64 - include := true - var err error - lowerStrf := strings.ToLower(strf) - if lowerStrf[0] == '(' { - include = false - lowerStrf = lowerStrf[1:] - } - if lowerStrf == "-inf" { - f = -math.MaxFloat64 - } else if lowerStrf == "+inf" || lowerStrf == "inf" { - f = math.MaxFloat64 - } else { - if f, err = strconv.ParseFloat(lowerStrf, 64); err != nil { - return f, include, err - } - } - return f, include, nil - -} - -func getLimitParameters(offsetCount []string) (int64, int64, error) { - if len(offsetCount) < 2 { - return 0, 0, ErrSyntax - } - var offset, count, tmp int64 - var err error - for i := 0; i < 2; i++ { - if tmp, err = strconv.ParseInt(offsetCount[i], 10, 64); err != nil { - return offset, count, ErrInteger - } - if i == 0 { - offset = tmp - } else { - count = tmp - } - } - return offset, count, nil -} diff --git a/command/connection.go b/command/connection.go deleted file mode 100644 index 7c0e656..0000000 --- a/command/connection.go +++ /dev/null @@ -1,74 +0,0 @@ -package command - -import ( - "strconv" - - "github.com/distributedio/titan/encoding/resp" - "github.com/distributedio/titan/metrics" -) - -// Auth verifies the client -func Auth(ctx *Context) { - args := ctx.Args - serverauth := []byte(ctx.Server.RequirePass) - if len(serverauth) == 0 { - resp.ReplyError(ctx.Out, "ERR Client sent AUTH, but no password is set") - return - } - - token := []byte(args[0]) - namespace, err := Verify(token, serverauth) - if err != nil { - resp.ReplyError(ctx.Out, "ERR invalid password") - return - } - metrics.GetMetrics().ConnectionOnlineGaugeVec.WithLabelValues(ctx.Client.Namespace).Dec() - metrics.GetMetrics().ConnectionOnlineGaugeVec.WithLabelValues(string(namespace)).Inc() - ctx.Client.Namespace = string(namespace) - ctx.Client.DB.Namespace = string(namespace) - ctx.Client.Authenticated = true - resp.ReplySimpleString(ctx.Out, OK) -} - -// Echo the given string -func Echo(ctx *Context) { - resp.ReplyBulkString(ctx.Out, ctx.Args[0]) -} - -// Ping the server -func Ping(ctx *Context) { - args := ctx.Args - if len(args) > 0 { - resp.ReplyBulkString(ctx.Out, args[0]) - return - } - resp.ReplySimpleString(ctx.Out, "PONG") -} - -// Select the logical database -func Select(ctx *Context) { - args := ctx.Args - idx, err := strconv.Atoi(args[0]) - if err != nil { - resp.ReplyError(ctx.Out, "ERR invalid DB index") - return - } - if idx < 0 || idx > 255 { - resp.ReplyError(ctx.Out, "ERR invalid DB index") - return - } - namespace := ctx.Client.Namespace - ctx.Client.DB = ctx.Server.Store.DB(namespace, idx) - resp.ReplySimpleString(ctx.Out, OK) -} - -// Quit asks the server to close the connection -func Quit(ctx *Context) { - close(ctx.Client.Done) - resp.ReplySimpleString(ctx.Out, OK) -} - -// SwapDB swaps two Redis databases -func SwapDB(ctx *Context) { - resp.ReplyError(ctx.Out, "ERR not supported") -} diff --git a/command/constraint.go b/command/constraint.go deleted file mode 100644 index e014688..0000000 --- a/command/constraint.go +++ /dev/null @@ -1,146 +0,0 @@ -package command - -// Constraint is the rule of command -type Constraint struct { - Arity int // number of arguments, it is possible to use -N to say >= N - Flags Flag - FirstKey int - LastKey int - KeyStep int -} - -// Flag is the redis command flag -type Flag int - -// Command flags -const ( - CmdWrite Flag = 1 << iota - CmdReadOnly - CmdDenyOOM - CmdModule - CmdAdmin - CmdPubsub - CmdNoScript - CmdRandom - CmdSortForScript - CmdLoading - CmdStale - CmdSkipMonitor - CmdAsking - CmdFast - CmdModuleGetKeys - CmdModuleNoCluster -) - -// String returns the string representation of flag -func (f Flag) String() string { - switch f { - case CmdWrite: - return "write" - case CmdReadOnly: - return "readonly" - case CmdDenyOOM: - return "denyoom" - case CmdModule: - return "module" - case CmdAdmin: - return "admin" - case CmdPubsub: - return "pubsub" - case CmdNoScript: - return "noscript" - case CmdRandom: - return "random" - case CmdSortForScript: - return "sort_for_script" - case CmdLoading: - return "loading" - case CmdStale: - return "stale" - case CmdSkipMonitor: - return "skip_monitor" - case CmdAsking: - return "asking" - case CmdFast: - return "fast" - case CmdModuleGetKeys: - return "module_getkeys" - case CmdModuleNoCluster: - return "module_no_cluster" - } - return "" -} - -// flags parse sflags to flags -// This is the meaning of the flags: -// -// w: write command (may modify the key space). -// r: read command (will never modify the key space). -// m: may increase memory usage once called. Don't allow if out of memory. -// a: admin command, like SAVE or SHUTDOWN. -// p: Pub/Sub related command. -// f: force replication of this command, regardless of server.dirty. -// s: command not allowed in scripts. -// R: random command. Command is not deterministic, that is, the same command -// with the same arguments, with the same key space, may have different -// results. For instance SPOP and RANDOMKEY are two random commands. -// S: Sort command output array if called from script, so that the output -// is deterministic. -// l: Allow command while loading the database. -// t: Allow command while a slave has stale data but is not allowed to -// server this data. Normally no command is accepted in this condition -// but just a few. -// M: Do not automatically propagate the command on MONITOR. -// k: Perform an implicit ASKING for this command, so the command will be -// accepted in cluster mode if the slot is marked as 'importing'. -// F: Fast command: O(1) or O(log(N)) command that should never delay -// its execution as long as the kernel scheduler is giving us time. -// Note that commands that may trigger a DEL as a side effect (like SET) -// are not fast commands. -func flags(s string) Flag { - flags := Flag(0) - for i := 0; i < len(s); i++ { - switch s[i] { - case 'w': - flags |= CmdWrite - case 'r': - flags |= CmdReadOnly - case 'm': - flags |= CmdDenyOOM - case 'a': - flags |= CmdAdmin - case 'p': - flags |= CmdPubsub - case 's': - flags |= CmdNoScript - case 'R': - flags |= CmdRandom - case 'S': - flags |= CmdSortForScript - case 'l': - flags |= CmdLoading - case 't': - flags |= CmdStale - case 'M': - flags |= CmdSkipMonitor - case 'k': - flags |= CmdAsking - case 'F': - flags |= CmdFast - default: - panic("Unsupported command flag") - } - } - return flags -} -func parseFlags(flags Flag) []string { - var s []string - // we have total 16 flags now - for i := uint(0); i < 16; i++ { - f := Flag(1 << i) - if f&flags != 0 { - s = append(s, f.String()) - } - } - return s -} diff --git a/command/error.go b/command/error.go deleted file mode 100644 index e4b4628..0000000 --- a/command/error.go +++ /dev/null @@ -1,112 +0,0 @@ -package command - -import ( - "errors" - "fmt" -) - -// RedisError defines the redis protocol error -type RedisError error - -const ( - // UnKnownCommandStr is the command not find - UnKnownCommandStr = "unknown command '%s'" - // WrongArgs is for wrong number of arguments error - WrongArgs = "ERR wrong number of arguments for '%s' command" -) - -var ( - // OK is the simple string "OK" returned to client - OK = "OK" - - // Queued is the simple string "QUEUED" return to client - Queued = "QUEUED" - - // ErrProtocol invalid request - // ErrProtocol = errors.New("ERR invalid request") - - // ErrNoAuth authentication required - ErrNoAuth = errors.New("NOAUTH Authentication required") - - // ErrAuthInvalid invalid password - ErrAuthInvalid = errors.New("ERR invalid password") - - // ErrAuthUnSet Client sent AUTH, but no password is set - ErrAuthUnSet = errors.New("ERR Client sent AUTH, but no password is set") - - // ErrInvalidDB invalid DB index - ErrInvalidDB = errors.New("ERR invalid DB index") - - //ErrExpire expire time in set - ErrExpire = errors.New("ERR invalid expire time in set") - - //ErrExpire expire time in setex - ErrExpireSetEx = errors.New("ERR invalid expire time in setex") - - // ErrInteger value is not an integer or out of range - ErrInteger = errors.New("ERR value is not an integer or out of range") - - // ErrFloat value is not a valid float - ErrFloat = errors.New("ERR value is not a valid float") - - // ErrBitInteger bit is not an integer or out of range - ErrBitInteger = errors.New("ERR bit is not an integer or out of range") - - // ErrBitInvaild the bit argument must be 1 or 0 - ErrBitInvaild = errors.New("ERR The bit argument must be 1 or 0") - - // ErrBitOffset bit offset is not an integer or out of range - ErrBitOffset = errors.New("ERR bit offset is not an integer or out of range") - - //ErrBitOp not must be called with a single source key. - ErrBitOp = errors.New("BITOP NOT must be called with a single source key.") - - // ErrOffset offset is out of range - ErrOffset = errors.New("ERR offset is out of range") - - // ErrIndex offset is out of range - ErrIndex = errors.New("ERR index out of range") - - // ErrSyntax syntax error - ErrSyntax = errors.New("ERR syntax error") - - // ErrMSet wrong number of arguments for MSET - ErrMSet = errors.New("ERR wrong number of arguments for MSET") - - // ErrNoSuchKey reteurn on lset for key which no exist - ErrNoSuchKey = errors.New("ERR no such key") - - // ErrReturnType return data type error - ErrReturnType = errors.New("ERR return data type error") - - //ErrMaximum allows the maximum size of a string - ErrMaximum = errors.New("ERR string exceeds maximum allowed size") - - // ErrMultiNested indicates a nested multi command which is not allowed - ErrMultiNested = errors.New("ERR MULTI calls can not be nested") - - // ErrTypeMismatch Operation against a key holding the wrong kind of value - ErrTypeMismatch = errors.New("WRONGTYPE Operation against a key holding the wrong kind of value") - - // ErrEmptyArray error - ErrEmptyArray = errors.New("EmptyArray error") - - //ErrExec exec without multi - ErrExec = errors.New("ERR EXEC without MULTI") - - //ErrDiscard without multi - ErrDiscard = errors.New("ERR DISCARD without MULTI") - - //argument min or max isn't float - ErrMinOrMaxNotFloat = errors.New("ERR min or max is not a float") -) - -//ErrUnKnownCommand return RedisError of the cmd -func ErrUnKnownCommand(cmd string) error { - return fmt.Errorf(UnKnownCommandStr, cmd) -} - -// ErrWrongArgs return RedisError of the cmd -func ErrWrongArgs(cmd string) error { - return fmt.Errorf(WrongArgs, cmd) -} diff --git a/command/extension.go b/command/extension.go deleted file mode 100644 index e772ccd..0000000 --- a/command/extension.go +++ /dev/null @@ -1,68 +0,0 @@ -package command - -import ( - "fmt" - "github.com/distributedio/titan/db" - "github.com/distributedio/titan/encoding/resp" - "math" - "strconv" -) - -// Escan scan the expiration list -// escan [from start] [to end] [count N] -func Escan(ctx *Context, txn *db.Transaction) (OnCommit, error) { - var from, to, count int64 - to = math.MaxInt64 - count = 10 - args := ctx.Args - if len(args)%2 != 0 { - return nil, ErrWrongArgs("escan") - } - - var p *int64 - for i := 0; i < len(args)-1; i++ { - switch args[i] { - case "from": - p = &from - case "to": - p = &to - case "count": - p = &count - default: - return nil, ErrSyntax - } - i++ - val, err := strconv.ParseInt(args[i], 10, 64) - if err != nil { - return nil, err - } - *p = val - } - if from > to { - return nil, db.ErrOutOfRange - } - at, keys, err := db.ScanExpiration(txn, from, to, count) - if err != nil { - return nil, err - } - n := len(keys) - if n == 0 { - return BytesArray(ctx.Out, nil), nil - } - // set the last ts as cursor - cursor := fmt.Sprintf("%d", at[n-1]) - - // set cursor to 0 if there is no more results - if int64(n) < count { - cursor = "0" - } - - return func() { - resp.ReplyArray(ctx.Out, 2) - resp.ReplyBulkString(ctx.Out, cursor) - resp.ReplyArray(ctx.Out, n) - for i := 0; i < n; i++ { - resp.ReplyBulkString(ctx.Out, fmt.Sprintf("%d %s", at[i], keys[i])) - } - }, nil -} diff --git a/command/hashes.go b/command/hashes.go deleted file mode 100644 index 5f4ab7f..0000000 --- a/command/hashes.go +++ /dev/null @@ -1,422 +0,0 @@ -package command - -import ( - "bytes" - "errors" - "strconv" - "strings" - - "github.com/distributedio/titan/db" - "github.com/distributedio/titan/encoding/resp" -) - -// HDel removes the specified fields from the hash stored at key -func HDel(ctx *Context, txn *db.Transaction) (OnCommit, error) { - hash, err := txn.Hash([]byte(ctx.Args[0])) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - - fields := make([][]byte, len(ctx.Args[1:])) - for i, field := range ctx.Args[1:] { - fields[i] = []byte(field) - } - c, err := hash.HDel(fields) - if err != nil { - return nil, errors.New("ERR " + err.Error()) - } - return Integer(ctx.Out, c), nil -} - -// HSet sets field in the hash stored at key to value -func HSet(ctx *Context, txn *db.Transaction) (OnCommit, error) { - key := []byte(ctx.Args[0]) - field := []byte(ctx.Args[1]) - value := []byte(ctx.Args[2]) - - hash, err := txn.Hash(key) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - - status, err := hash.HSet(field, value) - if err != nil { - return nil, errors.New("ERR " + err.Error()) - } - return Integer(ctx.Out, int64(status)), nil -} - -// HSetNX sets field in the hash stored at key to value, only if field does not yet exist -func HSetNX(ctx *Context, txn *db.Transaction) (OnCommit, error) { - key := []byte(ctx.Args[0]) - field := []byte(ctx.Args[1]) - value := []byte(ctx.Args[2]) - - hash, err := txn.Hash(key) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - - status, err := hash.HSetNX(field, value) - if err != nil { - return nil, errors.New("ERR " + err.Error()) - } - return Integer(ctx.Out, int64(status)), nil -} - -// HGet returns the value associated with field in the hash stored at key -func HGet(ctx *Context, txn *db.Transaction) (OnCommit, error) { - key := []byte(ctx.Args[0]) - field := []byte(ctx.Args[1]) - - hash, err := txn.Hash(key) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - val, err := hash.HGet(field) - if err != nil { - return nil, errors.New("ERR " + err.Error()) - } - if val == nil { - return NullBulkString(ctx.Out), nil - } - return BulkString(ctx.Out, string(val)), nil -} - -// HGetAll returns all fields and values of the hash stored at key -func HGetAll(ctx *Context, txn *db.Transaction) (OnCommit, error) { - key := []byte(ctx.Args[0]) - hash, err := txn.Hash(key) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - fields, vals, err := hash.HGetAll() - if err != nil { - return nil, errors.New("ERR " + err.Error()) - } - - results := make([][]byte, len(fields)*2) - for i := range fields { - results[i*2] = fields[i] - results[i*2+1] = vals[i] - } - - return BytesArray(ctx.Out, results), nil -} - -// HExists returns if field is an existing field in the hash stored at key -func HExists(ctx *Context, txn *db.Transaction) (OnCommit, error) { - key := []byte(ctx.Args[0]) - field := []byte(ctx.Args[1]) - hash, err := txn.Hash(key) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - exist, err := hash.HExists(field) - if err != nil { - return nil, errors.New("ERR " + err.Error()) - } - if exist { - return Integer(ctx.Out, 1), nil - } - return Integer(ctx.Out, 0), nil -} - -// HIncrBy increments the number stored at field in the hash stored at key by increment -func HIncrBy(ctx *Context, txn *db.Transaction) (OnCommit, error) { - key := []byte(ctx.Args[0]) - field := []byte(ctx.Args[1]) - incr, err := strconv.ParseInt(ctx.Args[2], 10, 64) - if err != nil { - return nil, ErrInteger - } - - hash, err := txn.Hash(key) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - - val, err := hash.HIncrBy(field, incr) - if err != nil { - return nil, errors.New("ERR " + err.Error()) - } - return Integer(ctx.Out, val), err -} - -// HIncrByFloat increment the specified field of a hash stored at key, -// and representing a floating point number, by the specified increment -func HIncrByFloat(ctx *Context, txn *db.Transaction) (OnCommit, error) { - key := []byte(ctx.Args[0]) - field := []byte(ctx.Args[1]) - incr, err := strconv.ParseFloat(ctx.Args[2], 64) - if err != nil { - return nil, ErrFloat - } - - hash, err := txn.Hash(key) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - - val, err := hash.HIncrByFloat(field, incr) - if err != nil { - return nil, errors.New("ERR " + err.Error()) - } - return BulkString(ctx.Out, strconv.FormatFloat(val, 'f', -1, 64)), nil -} - -// HKeys returns all field names in the hash stored at key -func HKeys(ctx *Context, txn *db.Transaction) (OnCommit, error) { - key := []byte(ctx.Args[0]) - hash, err := txn.Hash(key) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - fields, _, err := hash.HGetAll() - if err != nil { - return nil, errors.New("ERR " + err.Error()) - } - return BytesArray(ctx.Out, fields), nil -} - -// HVals returns all values in the hash stored at key -func HVals(ctx *Context, txn *db.Transaction) (OnCommit, error) { - key := []byte(ctx.Args[0]) - hash, err := txn.Hash(key) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - _, vals, err := hash.HGetAll() - if err != nil { - return nil, errors.New("ERR " + err.Error()) - } - return BytesArray(ctx.Out, vals), nil - -} - -// HLen returns the number of fields contained in the hash stored at key -func HLen(ctx *Context, txn *db.Transaction) (OnCommit, error) { - var size int64 - key := []byte(ctx.Args[0]) - hash, err := txn.Hash(key) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - size, err = hash.HLen() - if err != nil { - return nil, errors.New("ERR " + err.Error()) - } - return Integer(ctx.Out, size), nil -} - -// HStrLen returns the string length of the value associated with field in the hash stored at key -func HStrLen(ctx *Context, txn *db.Transaction) (OnCommit, error) { - key := []byte(ctx.Args[0]) - field := []byte(ctx.Args[1]) - hash, err := txn.Hash(key) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - val, err := hash.HGet(field) - if err != nil { - return nil, errors.New("ERR " + err.Error()) - } - return Integer(ctx.Out, int64(len(val))), nil -} - -// HMGet returns the values associated with the specified fields in the hash stored at key -func HMGet(ctx *Context, txn *db.Transaction) (OnCommit, error) { - key := []byte(ctx.Args[0]) - fields := make([][]byte, len(ctx.Args[1:])) - for i, field := range ctx.Args[1:] { - fields[i] = []byte(field) - } - - hash, err := txn.Hash(key) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - - vals, err := hash.HMGet(fields) - if err != nil { - return nil, errors.New("ERR " + err.Error()) - } - return BytesArray(ctx.Out, vals), nil -} - -// HMSet sets the specified fields to their respective values in the hash stored at key -func HMSet(ctx *Context, txn *db.Transaction) (OnCommit, error) { - key := []byte(ctx.Args[0]) - kvs := ctx.Args[1:] - mapping := make(map[string][]byte) - - if len(kvs)%2 != 0 { - return nil, errors.New("ERR wrong number of arguments for HMSET") - } - - // When there are multiple groups of the same field/val, - // take the last valid fields/val pair and save it in mapping - for i := 0; i < len(kvs)-1; i += 2 { - mapping[kvs[i]] = []byte(kvs[i+1]) - } - - fields := make([][]byte, 0, len(mapping)) - values := make([][]byte, 0, len(mapping)) - // Iterate mapping to get the field and value after de-duplication - for field, val := range mapping { - fields = append(fields, []byte(field)) - values = append(values, val) - } - - hash, err := txn.Hash(key) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - - if err := hash.HMSet(fields, values); err != nil { - return nil, errors.New("ERR " + err.Error()) - } - return SimpleString(ctx.Out, "OK"), nil -} - -//HScan incrementally iterate hash fields and associated values -func HScan(ctx *Context, txn *db.Transaction) (OnCommit, error) { - var ( - key []byte - cursor []byte - lastCursor = []byte("0") - count = uint64(defaultScanCount) - kvs = [][]byte{} - pattern []byte - prefix []byte - isAll bool - err error - ) - key = []byte(ctx.Args[0]) - if strings.Compare(ctx.Args[1], "0") != 0 { - cursor = []byte(ctx.Args[1]) - } - - // define return result - result := func() { - if _, err := resp.ReplyArray(ctx.Out, 2); err != nil { - return - } - resp.ReplyBulkString(ctx.Out, string(lastCursor)) - if _, err := resp.ReplyArray(ctx.Out, len(kvs)); err != nil { - return - } - for i := range kvs { - resp.ReplyBulkString(ctx.Out, string(kvs[i])) - } - } - hash, err := txn.Hash(key) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - - //check if hash is not exits return result - if !hash.Exists() { - return result, nil - } - - if len(ctx.Args)%2 != 0 { - return nil, ErrSyntax - } - - for i := 2; i < len(ctx.Args); i += 2 { - arg := strings.ToLower(ctx.Args[i]) - next := ctx.Args[i+1] - switch arg { - case "count": - if count, err = strconv.ParseUint(next, 10, 64); err != nil { - return nil, ErrInteger - } - if count > ScanMaxCount { - count = ScanMaxCount - } - if count == 0 { - count = uint64(defaultScanCount) - } - case "match": - pattern = []byte(next) - isAll = (pattern[0] == '*' && len(pattern) == 1) - } - } - - if len(pattern) == 0 { - isAll = true - } else { - prefix = globMatchPrefix(pattern) - if cursor == nil && prefix != nil { - cursor = prefix - } - } - - f := func(key, val []byte) bool { - if count <= 0 { - lastCursor = key - return false - } - if prefix != nil && !bytes.HasPrefix(key, prefix) { - return false - } - if isAll || globMatch(pattern, key, false) { - kvs = append(kvs, key) - kvs = append(kvs, val) - count-- - } - return true - } - - if err := hash.HScan(cursor, f); err != nil { - return nil, errors.New("ERR " + err.Error()) - } - return result, nil - -} diff --git a/command/hashes_test.go b/command/hashes_test.go deleted file mode 100644 index 4182e08..0000000 --- a/command/hashes_test.go +++ /dev/null @@ -1,493 +0,0 @@ -package command - -import ( - "strconv" - "testing" - - "github.com/stretchr/testify/assert" -) - -func initHashes(t *testing.T, key string, n int) { - args := []string{key} - for i := n; i > 0; i-- { - args = append(args, strconv.Itoa(i), "bar") - } - ctx := ContextTest("hmset", args...) - Call(ctx) - lines := ctxLines(ctx.Out) - assert.Equal(t, "+OK", lines[0]) -} - -func clearHashes(t *testing.T, key string) { - ctx := ContextTest("del", key) - Call(ctx) - lines := ctxLines(ctx.Out) - assert.Equal(t, ":1", lines[0]) -} - -func setHashes(args ...string) []string { - ctx := ContextTest("hmset", args...) - Call(ctx) - return ctxLines(ctx.Out) -} - -func TestHLen(t *testing.T) { - // init - key := "hash-key-len" - initHashes(t, key, 3) - - // case 1 - ctx := ContextTest("hlen", key) - Call(ctx) - lines := ctxLines(ctx.Out) - assert.Equal(t, ":3", lines[0]) - - // case 2 - lines = setHashes(key, "a", "a", "b", "b") - assert.Equal(t, "+OK", lines[0]) - ctx = ContextTest("hlen", key) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, ":5", lines[0]) - - // case 3 - lines = setHashes(key, "c", "c", "c", "d") - assert.Equal(t, "+OK", lines[0]) - ctx = ContextTest("hlen", key) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, ":6", lines[0]) - - // end - clearHashes(t, key) -} - -func TestHDel(t *testing.T) { - // init - key := "hash-key-del" - initHashes(t, key, 5) - - // case 1 - ctx := ContextTest("hdel", key, "1") - Call(ctx) - lines := ctxLines(ctx.Out) - assert.Equal(t, ":1", lines[0]) - ctx = ContextTest("hlen", key) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, ":4", lines[0]) - - // case 2 - ctx = ContextTest("hdel", key, "2") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, ":1", lines[0]) - ctx = ContextTest("hlen", key) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, ":3", lines[0]) - - // case 3 - ctx = ContextTest("hdel", key, "3", "4", "5") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, ":3", lines[0]) - ctx = ContextTest("hlen", key) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, ":0", lines[0]) - // then re-insert into hash - lines = setHashes(key, "a", "a", "b", "b") - assert.Equal(t, "+OK", lines[0]) - ctx = ContextTest("hlen", key) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, ":2", lines[0]) -} - -func TestHGet(t *testing.T) { - // init - key := "hash-key" - initHashes(t, key, 3) - - // case 1 - ctx := ContextTest("hget", key, "1") - Call(ctx) - lines := ctxLines(ctx.Out) - assert.Equal(t, "$3", lines[0]) - assert.Equal(t, "bar", lines[1]) - - // case 2 - ctx = ContextTest("hget", key, "5") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "$-1", lines[0]) - - // end - clearHashes(t, key) -} -func TestHMGet(t *testing.T) { - // init - key := "hash-key-mget" - initHashes(t, key, 3) - - // case 1 - ctx := ContextTest("hmget", key, "1", "2", "3") - Call(ctx) - lines := ctxLines(ctx.Out) - assert.Equal(t, "*3", lines[0]) - assert.Equal(t, "$3", lines[1]) - assert.Equal(t, "bar", lines[2]) - assert.Equal(t, "$3", lines[3]) - assert.Equal(t, "bar", lines[4]) - assert.Equal(t, "$3", lines[5]) - assert.Equal(t, "bar", lines[6]) - - // case 2 - ctx = ContextTest("hset", key, "foo", "haha") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, ":1", lines[0]) - - ctx = ContextTest("hmget", key, "1", "foo") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "*2", lines[0]) - assert.Equal(t, "$3", lines[1]) - assert.Equal(t, "bar", lines[2]) - assert.Equal(t, "$4", lines[3]) - assert.Equal(t, "haha", lines[4]) - - //case 3 - ctx = ContextTest("hmget", key, "ccc", "bbb") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "*2", lines[0]) - assert.Equal(t, "$-1", lines[1]) - assert.Equal(t, "$-1", lines[2]) - - // end - clearHashes(t, key) -} -func TestHMSet(t *testing.T) { - // init - key := "hash-key-mset" - initHashes(t, key, 3) - - // case 1 - ctx := ContextTest("hmset", key, "1", "ha", "2", "haha", "3", "hahaha") - Call(ctx) - lines := ctxLines(ctx.Out) - assert.Equal(t, "+OK", lines[0]) - ctx = ContextTest("hgetall", key) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "*6", lines[0]) - assert.Equal(t, "1", lines[2]) - assert.Equal(t, "ha", lines[4]) - assert.Equal(t, "2", lines[6]) - assert.Equal(t, "haha", lines[8]) - assert.Equal(t, "3", lines[10]) - assert.Equal(t, "hahaha", lines[12]) - - // case 2 - ctx = ContextTest("hmset", key, "foo", "bar") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "+OK", lines[0]) - ctx = ContextTest("hmget", key, "1", "foo") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "*2", lines[0]) - assert.Equal(t, "$2", lines[1]) - assert.Equal(t, "ha", lines[2]) - assert.Equal(t, "$3", lines[3]) - assert.Equal(t, "bar", lines[4]) - - // end - clearHashes(t, key) -} - -func TestHSetNX(t *testing.T) { - // init - key := "hash-key-setnx" - initHashes(t, key, 3) - - // case 1 - ctx := ContextTest("hsetnx", key, "1", "haha") - Call(ctx) - lines := ctxLines(ctx.Out) - assert.Equal(t, ":0", lines[0]) - ctx = ContextTest("hget", key, "1") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "$3", lines[0]) - assert.Equal(t, "bar", lines[1]) - - // case 2 - ctx = ContextTest("hsetnx", key, "4", "haha") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, ":1", lines[0]) - ctx = ContextTest("hget", key, "4") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "$4", lines[0]) - assert.Equal(t, "haha", lines[1]) - - // case 3 - ctx = ContextTest("hsetnx", key, "4", "bar") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, ":0", lines[0]) - ctx = ContextTest("hget", key, "4") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "$4", lines[0]) - assert.Equal(t, "haha", lines[1]) - - // end - clearHashes(t, key) -} - -func TestHSet(t *testing.T) { - // init - key := "hash-key-set" - initHashes(t, key, 3) - - // case 1 - ctx := ContextTest("hset", key, "1", "haha") - Call(ctx) - lines := ctxLines(ctx.Out) - assert.Equal(t, ":0", lines[0]) - ctx = ContextTest("hget", key, "1") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "$4", lines[0]) - assert.Equal(t, "haha", lines[1]) - - // case 2 - ctx = ContextTest("hset", key, "4", "haha") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, ":1", lines[0]) - ctx = ContextTest("hget", key, "4") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "$4", lines[0]) - assert.Equal(t, "haha", lines[1]) - - // case 3 - ctx = ContextTest("hset", key, "4", "bar") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, ":0", lines[0]) - ctx = ContextTest("hget", key, "4") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "$3", lines[0]) - assert.Equal(t, "bar", lines[1]) - - // end - clearHashes(t, key) -} - -func TestHGetAll(t *testing.T) { - // init - key := "hash-key-getall" - initHashes(t, key, 3) - - // case 1 - ctx := ContextTest("hgetall", key) - Call(ctx) - lines := ctxLines(ctx.Out) - assert.Equal(t, "*6", lines[0]) - assert.Equal(t, "$1", lines[1]) - assert.Equal(t, "1", lines[2]) - assert.Equal(t, "$3", lines[3]) - assert.Equal(t, "bar", lines[4]) - assert.Equal(t, "$1", lines[5]) - assert.Equal(t, "2", lines[6]) - assert.Equal(t, "$3", lines[7]) - assert.Equal(t, "bar", lines[8]) - assert.Equal(t, "$1", lines[9]) - assert.Equal(t, "3", lines[10]) - assert.Equal(t, "$3", lines[11]) - assert.Equal(t, "bar", lines[12]) - - // case 2 - ctx = ContextTest("hset", key, "foo", "haha") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, ":1", lines[0]) - - ctx = ContextTest("hgetall", key) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "*8", lines[0]) - assert.Equal(t, "$1", lines[1]) - assert.Equal(t, "1", lines[2]) - assert.Equal(t, "$3", lines[3]) - assert.Equal(t, "bar", lines[4]) - assert.Equal(t, "$1", lines[5]) - assert.Equal(t, "2", lines[6]) - assert.Equal(t, "$3", lines[7]) - assert.Equal(t, "bar", lines[8]) - assert.Equal(t, "$1", lines[9]) - assert.Equal(t, "3", lines[10]) - assert.Equal(t, "$3", lines[11]) - assert.Equal(t, "bar", lines[12]) - assert.Equal(t, "$3", lines[13]) - assert.Equal(t, "foo", lines[14]) - assert.Equal(t, "$4", lines[15]) - assert.Equal(t, "haha", lines[16]) - - // end - clearHashes(t, key) -} - -func TestHExists(t *testing.T) { - // init - key := "hash-key-exists" - initHashes(t, key, 3) - - // case 1 - ctx := ContextTest("hexists", key, "1") - Call(ctx) - lines := ctxLines(ctx.Out) - assert.Equal(t, ":1", lines[0]) - - //case 2 - ctx = ContextTest("hdel", key, "1") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, ":1", lines[0]) - ctx = ContextTest("hexists", key, "1") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, ":0", lines[0]) - - //case 3 - ctx = ContextTest("hexists", "hash_no_exists_key", "1") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, ":0", lines[0]) - clearList(t, key) -} - -func TestHIncrBy(t *testing.T) { - // init - key := "hash-key" - - // case 1 - ctx := ContextTest("hincrby", key, "one", "1") - Call(ctx) - lines := ctxLines(ctx.Out) - assert.Equal(t, ":1", lines[0]) - - // case 2 - ctx = ContextTest("hincrby", key, "one", "-2") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, ":-1", lines[0]) - clearList(t, key) - -} - -func TestHIncrByFloat(t *testing.T) { - // init - key := "hash-key" - - // case 1 - ctx := ContextTest("hincrbyfloat", key, "one", "1.1") - Call(ctx) - lines := ctxLines(ctx.Out) - assert.Equal(t, "1.1", lines[1]) - - // case 2 - ctx = ContextTest("hincrbyfloat", key, "one", "-2.2") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "-1.1", lines[1]) - clearList(t, key) - -} - -func TestHKeys(t *testing.T) { - // init - key := "hash-key-keys" - initHashes(t, key, 3) - - // case 1 - ctx := ContextTest("hkeys", key) - Call(ctx) - lines := ctxLines(ctx.Out) - assert.Equal(t, "*3", lines[0]) - assert.Equal(t, "1", lines[2]) - assert.Equal(t, "2", lines[4]) - assert.Equal(t, "3", lines[6]) - - // case 2 - ctx = ContextTest("hdel", key, "1") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, ":1", lines[0]) - ctx = ContextTest("hkeys", key) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "*2", lines[0]) - assert.Equal(t, "2", lines[2]) - assert.Equal(t, "3", lines[4]) - - clearList(t, key) - -} -func TestHStrLen(t *testing.T) { - // init - key := "hash-key" - - // case 1 - ctx := ContextTest("hset", key, "1", "abc") - Call(ctx) - lines := ctxLines(ctx.Out) - assert.Equal(t, ":1", lines[0]) - ctx = ContextTest("hstrlen", key, "1") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, ":3", lines[0]) - - clearList(t, key) -} -func TestHVals(t *testing.T) { - // init - key := "hash-key-val" - initHashes(t, key, 3) - - // case 1 - ctx := ContextTest("hvals", key) - Call(ctx) - lines := ctxLines(ctx.Out) - assert.Equal(t, "bar", lines[2]) - assert.Equal(t, "bar", lines[4]) - assert.Equal(t, "bar", lines[6]) - - clearList(t, key) - -} - -func TestHScan(t *testing.T) { - // init - key := "hash-key" - ctx := ContextTest("hmset", key, "field1", "1", "field2", "1", "field3", "1", "field4", "1") - Call(ctx) - lines := ctxLines(ctx.Out) - assert.Equal(t, "+OK", lines[0]) - - //case1 - ctx = ContextTest("hscan", key, "0", "match", "field*", "count", "2") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "$6", lines[1]) - assert.Equal(t, "field3", lines[2]) - assert.Equal(t, "*4", lines[3]) -} diff --git a/command/init.go b/command/init.go deleted file mode 100644 index a2d50b5..0000000 --- a/command/init.go +++ /dev/null @@ -1,131 +0,0 @@ -package command - -var commands map[string]Desc - -func init() { - // commands contains all commands that open to clients - // exec should not be in this table to avoid 'initialization loop', and it indeed not necessary be here in fact. - commands = map[string]Desc{ - // connections - "auth": Desc{Proc: Auth, Cons: Constraint{2, flags("sltF"), 0, 0, 0}}, - "echo": Desc{Proc: Echo, Cons: Constraint{2, flags("F"), 0, 0, 0}}, - "ping": Desc{Proc: Ping, Cons: Constraint{-1, flags("tF"), 0, 0, 0}}, - "quit": Desc{Proc: Quit, Cons: Constraint{1, 0, 0, 0, 0}}, - "select": Desc{Proc: Select, Cons: Constraint{2, flags("lF"), 0, 0, 0}}, - "swapdb": Desc{Proc: SwapDB, Cons: Constraint{3, flags("wF"), 0, 0, 0}}, - - // transactions, exec and discard should called explicitly, so they are registered here - "multi": Desc{Proc: Multi, Cons: Constraint{1, flags("sF"), 0, 0, 0}}, - "watch": Desc{Proc: Watch, Cons: Constraint{-2, flags("sF"), 1, -1, 1}}, - "unwatch": Desc{Proc: Unwatch, Cons: Constraint{1, flags("sF"), 0, 0, 0}}, - - // lists - "lindex": Desc{Proc: AutoCommit(LIndex), Txn: LIndex, Cons: Constraint{3, flags("r"), 1, 1, 1}}, - "linsert": Desc{Proc: AutoCommit(LInsert), Txn: LInsert, Cons: Constraint{5, flags("wm"), 1, 1, 1}}, - "llen": Desc{Proc: AutoCommit(LLen), Txn: LLen, Cons: Constraint{2, flags("rF"), 1, 1, 1}}, - "lpop": Desc{Proc: AutoCommit(LPop), Txn: LPop, Cons: Constraint{2, flags("wF"), 1, 1, 1}}, - "lpush": Desc{Proc: AutoCommit(LPush), Txn: LPush, Cons: Constraint{-3, flags("wmF"), 1, 1, 1}}, - "lpushx": Desc{Proc: AutoCommit(LPushx), Txn: LPushx, Cons: Constraint{3, flags("wmF"), 1, 1, 1}}, - "lrange": Desc{Proc: AutoCommit(LRange), Txn: LRange, Cons: Constraint{4, flags("r"), 1, 1, 1}}, - "lset": Desc{Proc: AutoCommit(LSet), Txn: LSet, Cons: Constraint{4, flags("wm"), 1, 1, 1}}, - "rpop": Desc{Proc: AutoCommit(RPop), Txn: RPop, Cons: Constraint{2, flags("wF"), 1, 1, 1}}, - "rpoplpush": Desc{Proc: AutoCommit(RPopLPush), Txn: RPopLPush, Cons: Constraint{3, flags("wF"), 1, 2, 1}}, - "rpush": Desc{Proc: AutoCommit(RPush), Txn: RPush, Cons: Constraint{-3, flags("wmF"), 1, 1, 1}}, - "rpushx": Desc{Proc: AutoCommit(RPushx), Txn: RPushx, Cons: Constraint{-3, flags("wmF"), 1, 1, 1}}, - - // strings - "get": Desc{Proc: AutoCommit(Get), Txn: Get, Cons: Constraint{2, flags("rF"), 1, 1, 1}}, - "set": Desc{Proc: AutoCommit(Set), Txn: Set, Cons: Constraint{-3, flags("wm"), 1, 1, 1}}, - "setnx": Desc{Proc: AutoCommit(SetNx), Txn: SetNx, Cons: Constraint{3, flags("wmF"), 1, 1, 1}}, - "setex": Desc{Proc: AutoCommit(SetEx), Txn: SetEx, Cons: Constraint{4, flags("wm"), 1, 1, 1}}, - "psetex": Desc{Proc: AutoCommit(PSetEx), Txn: PSetEx, Cons: Constraint{4, flags("wm"), 1, 1, 1}}, - "mget": Desc{Proc: AutoCommit(MGet), Txn: MGet, Cons: Constraint{-2, flags("rF"), 1, -1, 1}}, - "mset": Desc{Proc: AutoCommit(MSet), Txn: MSet, Cons: Constraint{-3, flags("wm"), 1, -1, 2}}, - "msetnx": Desc{Proc: AutoCommit(MSetNx), Txn: MSetNx, Cons: Constraint{-3, flags("wm"), 1, -1, 2}}, - "strlen": Desc{Proc: AutoCommit(Strlen), Txn: Strlen, Cons: Constraint{2, flags("rF"), 1, 1, 1}}, - "append": Desc{Proc: AutoCommit(Append), Txn: Append, Cons: Constraint{3, flags("wm"), 1, 1, 1}}, - "setrange": Desc{Proc: AutoCommit(SetRange), Txn: SetRange, Cons: Constraint{4, flags("wm"), 1, 1, 1}}, - "getrange": Desc{Proc: AutoCommit(GetRange), Txn: GetRange, Cons: Constraint{4, flags("r"), 1, 1, 1}}, - "incr": Desc{Proc: AutoCommit(Incr), Txn: Incr, Cons: Constraint{2, flags("wmF"), 1, 1, 1}}, - "decr": Desc{Proc: AutoCommit(Decr), Txn: Decr, Cons: Constraint{2, flags("wmF"), 1, 1, 1}}, - "incrby": Desc{Proc: AutoCommit(IncrBy), Txn: IncrBy, Cons: Constraint{3, flags("wmF"), 1, 1, 1}}, - "decrby": Desc{Proc: AutoCommit(DecrBy), Txn: DecrBy, Cons: Constraint{3, flags("wmF"), 1, 1, 1}}, - "incrbyfloat": Desc{Proc: AutoCommit(IncrByFloat), Txn: IncrByFloat, Cons: Constraint{3, flags("wmF"), 1, 1, 1}}, - "setbit": Desc{Proc: AutoCommit(SetBit), Txn: SetBit, Cons: Constraint{4, flags("wm"), 1, 1, 1}}, - // "bitop": Desc{Proc: AutoCommit(BitOp), Cons: Constraint{-4, flags("wm"), 2, -1, 1}}, - // "bitfield": Desc{Proc: AutoCommit(BitField), Cons: Constraint{-2, flags("wm"), 1, 1, 1}}, - "getbit": Desc{Proc: AutoCommit(GetBit), Txn: GetBit, Cons: Constraint{3, flags("r"), 1, 1, 1}}, - "bitcount": Desc{Proc: AutoCommit(BitCount), Txn: BitCount, Cons: Constraint{-2, flags("r"), 1, 1, 1}}, - "bitpos": Desc{Proc: AutoCommit(BitPos), Txn: BitPos, Cons: Constraint{-3, flags("r"), 1, 1, 1}}, - "getset": Desc{Proc: AutoCommit(GetSet), Txn: GetSet, Cons: Constraint{3, flags("wm"), 1, 1, 1}}, - - // keys - "type": Desc{Proc: AutoCommit(Type), Txn: Type, Cons: Constraint{2, flags("rF"), 1, 1, 1}}, - "exists": Desc{Proc: AutoCommit(Exists), Txn: Exists, Cons: Constraint{-2, flags("rF"), 1, -1, 1}}, - "keys": Desc{Proc: AutoCommit(Keys), Txn: Keys, Cons: Constraint{-2, flags("rS"), 0, 0, 0}}, - "del": Desc{Proc: AutoCommit(Delete), Txn: Delete, Cons: Constraint{-2, flags("w"), 1, -1, 1}}, - "unlink": Desc{Proc: AutoCommit(Delete), Txn: Delete, Cons: Constraint{-2, flags("w"), 1, -1, 1}}, - "expire": Desc{Proc: AutoCommit(Expire), Txn: Expire, Cons: Constraint{3, flags("wF"), 1, 1, 1}}, - "expireat": Desc{Proc: AutoCommit(ExpireAt), Txn: ExpireAt, Cons: Constraint{3, flags("wF"), 1, 1, 1}}, - "pexpire": Desc{Proc: AutoCommit(PExpire), Txn: PExpire, Cons: Constraint{3, flags("wF"), 1, 1, 1}}, - "pexpireat": Desc{Proc: AutoCommit(PExpireAt), Txn: PExpireAt, Cons: Constraint{3, flags("wF"), 1, 1, 1}}, - "persist": Desc{Proc: AutoCommit(Persist), Txn: Persist, Cons: Constraint{2, flags("wF"), 1, 1, 1}}, - "ttl": Desc{Proc: AutoCommit(TTL), Txn: TTL, Cons: Constraint{2, flags("rF"), 1, 1, 1}}, - "pttl": Desc{Proc: AutoCommit(PTTL), Txn: PTTL, Cons: Constraint{2, flags("rF"), 1, 1, 1}}, - "object": Desc{Proc: AutoCommit(Object), Txn: Object, Cons: Constraint{-2, flags("rR"), 0, 0, 0}}, - "scan": Desc{Proc: AutoCommit(Scan), Txn: Scan, Cons: Constraint{-2, flags("rR"), 0, 0, 0}}, - "randomkey": Desc{Proc: AutoCommit(RandomKey), Txn: RandomKey, Cons: Constraint{1, flags("rR"), 0, 0, 0}}, - "touch": Desc{Proc: AutoCommit(Touch), Txn: Touch, Cons: Constraint{-2, flags("rF"), 1, -1, 1}}, - - // server - "monitor": Desc{Proc: Monitor, Cons: Constraint{1, flags("as"), 0, 0, 0}}, - "client": Desc{Proc: Client, Cons: Constraint{-2, flags("as"), 0, 0, 0}}, - "debug": Desc{Proc: AutoCommit(Debug), Cons: Constraint{-2, flags("as"), 0, 0, 0}}, - "command": Desc{Proc: RedisCommand, Cons: Constraint{0, flags("lt"), 0, 0, 0}}, - "flushdb": Desc{Proc: AutoCommit(FlushDB), Cons: Constraint{-1, flags("w"), 0, 0, 0}}, - "flushall": Desc{Proc: AutoCommit(FlushAll), Cons: Constraint{-1, flags("w"), 0, 0, 0}}, - "time": Desc{Proc: Time, Cons: Constraint{1, flags("RF"), 0, 0, 0}}, - "info": Desc{Proc: Info, Cons: Constraint{-1, flags("lt"), 0, 0, 0}}, - - // hashes - "hdel": Desc{Proc: AutoCommit(HDel), Txn: HDel, Cons: Constraint{-3, flags("wF"), 1, 1, 1}}, - "hset": Desc{Proc: AutoCommit(HSet), Txn: HSet, Cons: Constraint{-4, flags("wmF"), 1, 1, 1}}, - "hget": Desc{Proc: AutoCommit(HGet), Txn: HGet, Cons: Constraint{3, flags("rF"), 1, 1, 1}}, - "hgetall": Desc{Proc: AutoCommit(HGetAll), Txn: HGetAll, Cons: Constraint{2, flags("r"), 1, 1, 1}}, - "hexists": Desc{Proc: AutoCommit(HExists), Txn: HExists, Cons: Constraint{3, flags("rF"), 1, 1, 1}}, - "hincrby": Desc{Proc: AutoCommit(HIncrBy), Txn: HIncrBy, Cons: Constraint{4, flags("wmF"), 1, 1, 1}}, - "hincrbyfloat": Desc{Proc: AutoCommit(HIncrByFloat), Txn: HIncrByFloat, Cons: Constraint{4, flags("wmF"), 1, 1, 1}}, - "hkeys": Desc{Proc: AutoCommit(HKeys), Txn: HKeys, Cons: Constraint{2, flags("rS"), 1, 1, 1}}, - "hvals": Desc{Proc: AutoCommit(HVals), Txn: HVals, Cons: Constraint{2, flags("rS"), 1, 1, 1}}, - "hlen": Desc{Proc: AutoCommit(HLen), Txn: HLen, Cons: Constraint{2, flags("rF"), 1, 1, 1}}, - "hstrlen": Desc{Proc: AutoCommit(HStrLen), Txn: HStrLen, Cons: Constraint{3, flags("rF"), 1, 1, 1}}, - "hsetnx": Desc{Proc: AutoCommit(HSetNX), Txn: HSetNX, Cons: Constraint{4, flags("wmF"), 1, 1, 1}}, - "hmget": Desc{Proc: AutoCommit(HMGet), Txn: HMGet, Cons: Constraint{-3, flags("rF"), 1, 1, 1}}, - "hmset": Desc{Proc: AutoCommit(HMSet), Txn: HMSet, Cons: Constraint{-3, flags("wmF"), 1, 1, 1}}, - "hscan": Desc{Proc: AutoCommit(HScan), Txn: HScan, Cons: Constraint{-3, flags("rR"), 0, 0, 0}}, - - // sets - "sadd": Desc{Proc: AutoCommit(SAdd), Txn: SAdd, Cons: Constraint{-3, flags("wmF"), 1, 1, 1}}, - "smembers": Desc{Proc: AutoCommit(SMembers), Txn: SMembers, Cons: Constraint{2, flags("rS"), 1, 1, 1}}, - "scard": Desc{Proc: AutoCommit(SCard), Txn: SCard, Cons: Constraint{2, flags("rF"), 1, 1, 1}}, - "sismember": Desc{Proc: AutoCommit(SIsmember), Txn: SIsmember, Cons: Constraint{3, flags("rF"), 1, 1, 1}}, - "spop": Desc{Proc: AutoCommit(SPop), Txn: SPop, Cons: Constraint{-2, flags("wRF"), 1, 1, 1}}, - "srem": Desc{Proc: AutoCommit(SRem), Txn: SRem, Cons: Constraint{-3, flags("wF"), 1, 1, 1}}, - "sunion": Desc{Proc: AutoCommit(SUnion), Txn: SUnion, Cons: Constraint{-2, flags("rS"), 1, -1, 1}}, - "sinter": Desc{Proc: AutoCommit(SInter), Txn: SInter, Cons: Constraint{-2, flags("rS"), 1, -1, 1}}, - "sdiff": Desc{Proc: AutoCommit(SDiff), Txn: SDiff, Cons: Constraint{-2, flags("rS"), 1, -1, 1}}, - "smove": Desc{Proc: AutoCommit(SMove), Txn: SMove, Cons: Constraint{4, flags("wF"), 1, 2, 1}}, - - // zsets - "zadd": Desc{Proc: AutoCommit(ZAdd), Txn: ZAdd, Cons: Constraint{-4, flags("wmF"), 1, 1, 1}}, - "zrange": Desc{Proc: AutoCommit(ZRange), Txn: ZRange, Cons: Constraint{-4, flags("rF"), 1, 1, 1}}, - "zrevrange": Desc{Proc: AutoCommit(ZRevRange), Txn: ZRevRange, Cons: Constraint{-4, flags("rF"), 1, 1, 1}}, - "zrangebyscore": {Proc: AutoCommit(ZRangeByScore), Txn: ZRangeByScore, Cons: Constraint{-4, flags("rF"), 1, 1, 1}}, - "zrem": Desc{Proc: AutoCommit(ZRem), Txn: ZRem, Cons: Constraint{-3, flags("wF"), 1, 1, 1}}, - "zcard": Desc{Proc: AutoCommit(ZCard), Txn: ZCard, Cons: Constraint{2, flags("rF"), 1, 1, 1}}, - "zscore": Desc{Proc: AutoCommit(ZScore), Txn: ZScore, Cons: Constraint{3, flags("rF"), 1, 1, 1}}, - - // extension commands - "escan": Desc{Proc: AutoCommit(Escan), Txn: Escan, Cons: Constraint{-1, flags("rR"), 0, 0, 0}}, - } -} diff --git a/command/keys.go b/command/keys.go deleted file mode 100644 index 57bc6d7..0000000 --- a/command/keys.go +++ /dev/null @@ -1,372 +0,0 @@ -package command - -import ( - "bytes" - "errors" - "fmt" - "strconv" - "strings" - "time" - - "github.com/distributedio/titan/db" - "github.com/distributedio/titan/encoding/resp" -) - -const ( - //ScanMaxCount is the max limitation of a single scan - ScanMaxCount = 255 - // defautlScanCout is used when no hints being supplied by clients - defaultScanCount = 10 -) - -// Delete removes the specified keys. A key is ignored if it does not exist -func Delete(ctx *Context, txn *db.Transaction) (OnCommit, error) { - kv := txn.Kv() - keys := make([][]byte, len(ctx.Args)) - for i := range ctx.Args { - keys[i] = []byte(ctx.Args[i]) - } - c, err := kv.Delete(keys) - if err != nil { - return nil, errors.New("ERR " + err.Error()) - } - return Integer(ctx.Out, c), nil -} - -// Exists returns if key exists -func Exists(ctx *Context, txn *db.Transaction) (OnCommit, error) { - kv := txn.Kv() - keys := make([][]byte, len(ctx.Args)) - for i := range ctx.Args { - keys[i] = []byte(ctx.Args[i]) - } - c, err := kv.Exists(keys) - if err != nil { - return nil, errors.New("ERR " + err.Error()) - } - return Integer(ctx.Out, c), nil -} - -// Expire sets a timeout on key -func Expire(ctx *Context, txn *db.Transaction) (OnCommit, error) { - kv := txn.Kv() - key := []byte(ctx.Args[0]) - seconds, err := strconv.ParseInt(ctx.Args[1], 10, 64) - if err != nil { - return nil, ErrInteger - } - - at := time.Now().Add(time.Second * time.Duration(seconds)).UnixNano() - if err := kv.ExpireAt(key, at); err != nil { - if err == db.ErrKeyNotFound { - return Integer(ctx.Out, 0), nil - } - return nil, errors.New("ERR " + err.Error()) - } - return Integer(ctx.Out, 1), nil -} - -// ExpireAt sets an absolute timestamp to expire on key -func ExpireAt(ctx *Context, txn *db.Transaction) (OnCommit, error) { - kv := txn.Kv() - key := []byte(ctx.Args[0]) - timestamp, err := strconv.ParseInt(ctx.Args[1], 10, 64) - if err != nil { - return nil, ErrInteger - } - - at := int64(time.Second * time.Duration(timestamp)) - if at <= 0 { - at = 1 - } - - if err := kv.ExpireAt(key, at); err != nil { - if err == db.ErrKeyNotFound { - return Integer(ctx.Out, 0), nil - } - return nil, errors.New("ERR " + err.Error()) - } - - return Integer(ctx.Out, 1), nil -} - -// Persist removes the existing timeout on key, turning the key from volatile to persistent -func Persist(ctx *Context, txn *db.Transaction) (OnCommit, error) { - kv := txn.Kv() - key := []byte(ctx.Args[0]) - obj, err := txn.Object(key) - if err != nil && err != db.ErrKeyNotFound { - return nil, errors.New("ERR " + err.Error()) - } - if err == db.ErrKeyNotFound || obj.ExpireAt == 0 { - return Integer(ctx.Out, 0), nil - } - - if err := kv.ExpireAt(key, 0); err != nil { - if err == db.ErrKeyNotFound { - return Integer(ctx.Out, 0), nil - } - return nil, errors.New("ERR " + err.Error()) - } - return Integer(ctx.Out, 1), nil -} - -// PExpire works exactly like expire but the time to live of the key is specified in milliseconds instead of seconds -func PExpire(ctx *Context, txn *db.Transaction) (OnCommit, error) { - kv := txn.Kv() - key := []byte(ctx.Args[0]) - ms, err := strconv.ParseInt(ctx.Args[1], 10, 64) - if err != nil { - return nil, ErrInteger - } - at := time.Now().Add(time.Millisecond * time.Duration(ms)).UnixNano() - if err := kv.ExpireAt(key, at); err != nil { - if err == db.ErrKeyNotFound { - return Integer(ctx.Out, 0), nil - } - return nil, errors.New("ERR " + err.Error()) - } - return Integer(ctx.Out, 1), nil - -} - -// PExpireAt has the same effect and semantic as expireAt, -// but the Unix time at which the key will expire is specified in milliseconds instead of seconds -func PExpireAt(ctx *Context, txn *db.Transaction) (OnCommit, error) { - kv := txn.Kv() - key := []byte(ctx.Args[0]) - ms, err := strconv.ParseInt(ctx.Args[1], 10, 64) - if err != nil { - return nil, ErrInteger - } - at := int64(time.Millisecond * time.Duration(ms)) - if at <= 0 { - at = 1 - } - if err := kv.ExpireAt(key, at); err != nil { - if err == db.ErrKeyNotFound { - return Integer(ctx.Out, 0), nil - } - return nil, errors.New("ERR " + err.Error()) - } - return Integer(ctx.Out, 1), nil -} - -// TTL returns the remaining time to live of a key that has a timeout -func TTL(ctx *Context, txn *db.Transaction) (OnCommit, error) { - key := []byte(ctx.Args[0]) - now := db.Now() - obj, err := txn.Object(key) - if err != nil { - if err == db.ErrKeyNotFound { - return Integer(ctx.Out, -2), nil - } - return nil, errors.New("ERR " + err.Error()) - } - if obj.ExpireAt == 0 { - return Integer(ctx.Out, -1), nil - } - ttl := (obj.ExpireAt - now) / int64(time.Second) - return Integer(ctx.Out, ttl), nil -} - -// PTTL likes TTL this command returns the remaining time to live of a key that has an expire set, -// with the sole difference that TTL returns the amount of remaining time in seconds while PTTL returns it in milliseconds -func PTTL(ctx *Context, txn *db.Transaction) (OnCommit, error) { - key := []byte(ctx.Args[0]) - now := db.Now() - obj, err := txn.Object(key) - if err != nil { - if err == db.ErrKeyNotFound { - return Integer(ctx.Out, -2), nil - } - return nil, errors.New("ERR " + err.Error()) - } - if db.IsExpired(obj, now) { - return Integer(ctx.Out, -2), nil - } - if obj.ExpireAt == 0 { - return Integer(ctx.Out, -1), nil - } - ttl := (obj.ExpireAt - now) / int64(time.Millisecond) - return Integer(ctx.Out, ttl), nil - -} - -// Object inspects the internals of Redis Objects -func Object(ctx *Context, txn *db.Transaction) (OnCommit, error) { - argc := len(ctx.Args) - subCmd := strings.ToLower(ctx.Args[0]) - cmdErr := fmt.Errorf("ERR Unknown subcommand or wrong number of arguments for '%s'. Try OBJECT help", subCmd) - if argc == 1 && subCmd == "help" { - - helpInfo := [][]byte{ - []byte("OBJECT key. Subcommands:"), - []byte("ENCODING -- Return the kind of internal representation used in order to store the value associated with a key."), - []byte("FREQ -- Return the access frequency index of the key. The returned integer is proportional to the logarithm of the recent access frequency of the key."), - []byte("IDLETIME -- Return the idle time of the key, that is the approximated number of seconds elapsed since the last access to the key."), - []byte("REFCOUNT -- Return the number of references of the value associated with the specified key."), - } - return BytesArray(ctx.Out, helpInfo), nil - } else if argc == 2 { - key := []byte(ctx.Args[1]) - obj, err := txn.Object(key) - if err != nil { - if err == db.ErrKeyNotFound { - return NullBulkString(ctx.Out), nil - } - return nil, errors.New("ERR " + err.Error()) - } - switch subCmd { - case "refcount", "freq": - return Integer(ctx.Out, 0), nil - case "idletime": - sec := int64(time.Since(time.Unix(0, obj.UpdatedAt)).Seconds()) - return Integer(ctx.Out, sec), nil - case "encoding": - return SimpleString(ctx.Out, obj.Encoding.String()), nil - } - } - return nil, cmdErr -} - -// Type returns the string representation of the type of the value stored at key -func Type(ctx *Context, txn *db.Transaction) (OnCommit, error) { - key := []byte(ctx.Args[0]) - obj, err := txn.Object(key) - if err != nil { - if err == db.ErrKeyNotFound { - return SimpleString(ctx.Out, "none"), nil - } - return nil, errors.New("ERR " + err.Error()) - } - - return SimpleString(ctx.Out, obj.Type.String()), nil -} - -// Keys returns all keys matching pattern -func Keys(ctx *Context, txn *db.Transaction) (OnCommit, error) { - list := make([][]byte, 0) - pattern := []byte(ctx.Args[0]) - all := (pattern[0] == '*' && len(pattern) == 1) - prefix := globMatchPrefix(pattern) - - kv := txn.Kv() - f := func(key []byte) bool { - if all || globMatch(pattern, key, false) { - list = append(list, key) - } - return true - } - - if err := kv.Keys(prefix, f); err != nil { - return nil, errors.New("ERR " + err.Error()) - } - return BytesArray(ctx.Out, list), nil -} - -// Scan incrementally iterates the key space -func Scan(ctx *Context, txn *db.Transaction) (OnCommit, error) { - var ( - start []byte - end = []byte("0") - count uint64 = defaultScanCount - pattern []byte - prefix []byte - all bool - err error - ) - if strings.Compare(ctx.Args[0], "0") != 0 { - start = []byte(ctx.Args[0]) - } - - if len(ctx.Args)%2 == 0 { - return nil, ErrInteger - } - - for i := 1; i < len(ctx.Args); i += 2 { - arg := strings.ToLower(ctx.Args[i]) - next := ctx.Args[i+1] - switch arg { - case "count": - if count, err = strconv.ParseUint(next, 10, 64); err != nil { - return nil, ErrInteger - } - if count > ScanMaxCount { - count = ScanMaxCount - } - if count == 0 { - count = defaultScanCount - } - case "match": - pattern = []byte(next) - all = (pattern[0] == '*' && len(pattern) == 1) - } - } - - if len(pattern) == 0 { - all = true - } else { - prefix = globMatchPrefix(pattern) - if start == nil && prefix != nil { - start = prefix - } - } - - kv := txn.Kv() - list := [][]byte{} - f := func(key []byte) bool { - if count <= 0 { - end = key - return false - } - if prefix != nil && !bytes.HasPrefix(key, prefix) { - return false - } - if all || globMatch(pattern, key, false) { - list = append(list, key) - count-- - } - return true - } - - if err := kv.Keys(start, f); err != nil { - return nil, errors.New("ERR " + err.Error()) - } - return func() { - resp.ReplyArray(ctx.Out, 2) - resp.ReplyBulkString(ctx.Out, string(end)) - resp.ReplyArray(ctx.Out, len(list)) - for i := range list { - resp.ReplyBulkString(ctx.Out, string(list[i])) - } - }, nil - -} - -// RandomKey returns a random key from the currently selected database -func RandomKey(ctx *Context, txn *db.Transaction) (OnCommit, error) { - kv := txn.Kv() - key, err := kv.RandomKey() - if err != nil { - return nil, errors.New("ERR " + err.Error()) - } - if key == nil { - return NullBulkString(ctx.Out), nil - } - return BulkString(ctx.Out, string(key)), nil -} - -// Touch alters the last access time of a key(s) -func Touch(ctx *Context, txn *db.Transaction) (OnCommit, error) { - kv := txn.Kv() - keys := make([][]byte, len(ctx.Args)) - for i := range ctx.Args { - keys[i] = []byte(ctx.Args[i]) - } - count, err := kv.Touch(keys) - if err != nil { - return nil, err - } - return Integer(ctx.Out, count), nil -} diff --git a/command/keys_test.go b/command/keys_test.go deleted file mode 100644 index 9b35bf8..0000000 --- a/command/keys_test.go +++ /dev/null @@ -1,407 +0,0 @@ -package command - -import ( - "strconv" - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func InitData(t *testing.T, keys []string, val string) { - for _, key := range keys { - ctx := ContextTest("set", key, val) - Call(ctx) - } -} - -func AddList(t *testing.T, key string, val string) { - ctx := ContextTest("lpush", key, val) - Call(ctx) -} - -func EquealKeyExists(t *testing.T, key string) { - ctx := ContextTest("exists", key) - Call(ctx) - lines := ctxLines(ctx.Out) - assert.Equal(t, ":1", lines[0]) - -} - -func NotEquealKeyExists(t *testing.T, key string) { - ctx := ContextTest("exists", key) - Call(ctx) - lines := ctxLines(ctx.Out) - assert.Equal(t, ":0", lines[0]) -} - -func TestDel(t *testing.T) { - keys := []string{ - "keys-del-key1", - "keys-del-key2", - "keys-del-key3", - "keys-del-key4", - "keys-del-key5", - "keys-del-key6", - } - - InitData(t, keys, "val") - ctx := ContextTest("del", keys[0], keys[1], keys[2], keys[3], keys[4], keys[5]) - Call(ctx) - lines := ctxLines(ctx.Out) - assert.Equal(t, ":6", lines[0]) - NotEquealKeyExists(t, keys[0]) - - InitData(t, keys, "val") - ctx = ContextTest("del", keys[0], keys[1], keys[2], keys[3], keys[4], keys[5], "keys-del-faild") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, ":6", lines[0]) - NotEquealKeyExists(t, keys[0]) -} - -func TestExists(t *testing.T) { - keys := []string{ - "keys-keyexists1", - "keys-keyexists2", - "keys-keyexists3", - "keys-keyexists4", - "keys-keyexists5", - "keys-keyexists6", - } - - InitData(t, keys, "val") - ctx := ContextTest("exists", keys[0], keys[1], keys[2], keys[3], keys[4], keys[5]) - Call(ctx) - lines := ctxLines(ctx.Out) - assert.Equal(t, ":6", lines[0]) - - ctx = ContextTest("exists", keys[0], keys[1], keys[2], keys[3], keys[4], keys[5], "keys-keyexists-faild") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, ":6", lines[0]) - - ctx = ContextTest("del", keys[0], keys[1], keys[2], keys[3], keys[4], keys[5]) - Call(ctx) -} - -func TestExpireAt(t *testing.T) { - keys := []string{"keys-expireat1", "keys-expireat2", "keys-expireat3"} - InitData(t, keys, "val") - - now := time.Now().Unix() - - time1 := now + 10 - ctx := ContextTest("expireat", keys[0], strconv.FormatInt(time1, 10)) - Call(ctx) - lines := ctxLines(ctx.Out) - assert.Equal(t, ":1", lines[0]) - EquealKeyExists(t, keys[0]) - - time2 := "0" - ctx = ContextTest("expireat", keys[1], time2) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, ":1", lines[0]) - NotEquealKeyExists(t, keys[1]) - - time3 := "-1" - ctx = ContextTest("expireat", keys[2], time3) - Call(ctx) - lines = ctxLines(ctx.Out) - - assert.Equal(t, ":1", lines[0]) - NotEquealKeyExists(t, keys[2]) -} - -func TestExpire(t *testing.T) { - key1 := "keys-expire1" - key2 := "keys-expire2" - key3 := "keys-expire3" - keys := []string{ - key1, key2, key3, - } - InitData(t, keys, "val") - - time1 := "10" - ctx := ContextTest("expire", keys[0], time1) - Call(ctx) - lines := ctxLines(ctx.Out) - assert.Equal(t, ":1", lines[0]) - EquealKeyExists(t, keys[0]) - - time2 := "0" - ctx = ContextTest("expire", keys[1], time2) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, ":1", lines[0]) - NotEquealKeyExists(t, keys[1]) - - time3 := "-1" - ctx = ContextTest("expire", keys[2], time3) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, ":1", lines[0]) - NotEquealKeyExists(t, keys[2]) -} - -func TestPExpire(t *testing.T) { - key1 := "keys-pexpire1" - key2 := "keys-pexpire2" - key3 := "keys-pexpire3" - keys := []string{ - key1, key2, key3, - } - InitData(t, keys, "val") - - time1 := "10" - ctx := ContextTest("pexpire", keys[0], time1) - Call(ctx) - lines := ctxLines(ctx.Out) - assert.Equal(t, ":1", lines[0]) - EquealKeyExists(t, keys[0]) - - time2 := "0" - ctx = ContextTest("pexpire", keys[1], time2) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, ":1", lines[0]) - NotEquealKeyExists(t, keys[1]) - - time3 := "-1" - ctx = ContextTest("pexpire", keys[2], time3) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, ":1", lines[0]) - NotEquealKeyExists(t, keys[2]) - -} - -func TestPExpireAt(t *testing.T) { - key1 := "keys-pexpireat1" - key2 := "keys-pexpireat2" - key3 := "keys-pexpireat3" - keys := []string{ - key1, key2, key3, - } - InitData(t, keys, "val") - - now := time.Now().Unix() - - time1 := (now + 10) * int64(time.Second) / int64(time.Millisecond) - ctx := ContextTest("pexpireat", keys[0], strconv.FormatInt(time1, 10)) - Call(ctx) - lines := ctxLines(ctx.Out) - assert.Equal(t, ":1", lines[0]) - EquealKeyExists(t, keys[0]) - - time2 := "0" - ctx = ContextTest("pexpireat", keys[1], time2) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, ":1", lines[0]) - NotEquealKeyExists(t, keys[1]) - - time3 := "-1" - ctx = ContextTest("pexpireat", keys[2], time3) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, ":1", lines[0]) - NotEquealKeyExists(t, keys[2]) -} - -func TestTTL(t *testing.T) { - val := "val" - key1 := "keys-ttl1" - - InitData(t, []string{key1}, val) - - ctx := ContextTest("ttl", key1) - Call(ctx) - lines := ctxLines(ctx.Out) - assert.Equal(t, ":-1", lines[0]) - - ctx = ContextTest("expire", key1, "2") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, ":1", lines[0]) - - ctx = ContextTest("ttl", key1) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.NotEqual(t, ":-1", lines[0]) - assert.NotEqual(t, ":-2", lines[0]) - assert.NotEqual(t, ":0", lines[0]) - - time.Sleep(1 * time.Second) - - ctx = ContextTest("ttl", key1) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.NotEqual(t, ":-2", lines[0]) -} - -func TestPTTL(t *testing.T) { - val := "val" - key1 := "keys-pttl1" - - InitData(t, []string{key1}, val) - - ctx := ContextTest("pttl", key1) - Call(ctx) - lines := ctxLines(ctx.Out) - assert.Equal(t, ":-1", lines[0]) - - ctx = ContextTest("expire", key1, "2") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, ":1", lines[0]) - - ctx = ContextTest("pttl", key1) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.NotEqual(t, ":-1", lines[0]) - assert.NotEqual(t, ":-2", lines[0]) - assert.NotEqual(t, ":0", lines[0]) - - time.Sleep(1 * time.Second) - - ctx = ContextTest("pttl", key1) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.NotEqual(t, ":-2", lines[0]) -} - -func TestPerist(t *testing.T) { - key := "keys-pexist1" - val := "val" - InitData(t, []string{key}, val) - - ctx := ContextTest("expire", key, "5") - Call(ctx) - lines := ctxLines(ctx.Out) - assert.Equal(t, ":1", lines[0]) - - ctx = ContextTest("persist", key) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, ":1", lines[0]) - - ctx = ContextTest("ttl", key) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, ":-1", lines[0]) - - ctx = ContextTest("persist", key) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, ":0", lines[0]) - -} - -func TestType(t *testing.T) { - key := "keys-type1" - val := "val" - InitData(t, []string{key}, val) - - lkey := "keys-type-list1" - AddList(t, lkey, val) - - ctx := ContextTest("type", key) - Call(ctx) - lines := ctxLines(ctx.Out) - assert.Equal(t, "+string", lines[0]) - - ctx = ContextTest("type", lkey) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "+list", lines[0]) - - ctx = ContextTest("type", "keys-type-faild") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "+none", lines[0]) -} - -func TestKeys(t *testing.T) { - keys := []string{ - "keys-abc1:keys", - "keys-acb1:keys", - "keys-abc2:keys", - } - val := "val" - InitData(t, keys, val) - - ctx := ContextTest("keys", "keys*:keys") - Call(ctx) - lines := ctxLines(ctx.Out) - assert.Equal(t, "*3", lines[0]) - assert.Contains(t, lines, "keys-abc1:keys") - - ctx = ContextTest("keys", "keys-ab*") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "*2", lines[0]) - assert.Contains(t, lines, "keys-abc1:keys") -} - -func TestScan(t *testing.T) { - keys := []string{ - "keys-scan1", - "keys-scan2", - "keys-scan3", - "keys-scan4", - "keys-sscan5", - } - val := "val" - InitData(t, keys, val) - - ctx := ContextTest("scan", "0", "count", "4", "match", "keys-scan*") - Call(ctx) - lines := ctxLines(ctx.Out) - assert.Equal(t, "*2", lines[0]) - assert.Contains(t, lines, "keys-scan4") - assert.Equal(t, "keys-sscan5", lines[2]) -} - -func TestObject(t *testing.T) { - key := "keys-object1" - val := "val" - InitData(t, []string{key}, val) - lkey := "keys-objectlist1" - AddList(t, lkey, val) - - ctx := ContextTest("object", "encoding", key) - Call(ctx) - lines := ctxLines(ctx.Out) - assert.Equal(t, "+raw", lines[0]) - - ctx = ContextTest("object", "encoding", lkey) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "+linkedlist", lines[0]) - - time.Sleep(time.Second) - ctx = ContextTest("object", "idletime", lkey) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.NotEqual(t, ":0", lines[0]) - -} - -func TestRandomkey(t *testing.T) { - keys := []string{ - "keyscan1", - "keyscan2", - "keyscan3", - "keyscan4", - "skeyscan5", - } - val := "val" - InitData(t, keys, val) - - ctx := ContextTest("randomkey") - Call(ctx) - lines := ctxLines(ctx.Out) - assert.NotEqual(t, 0, len(lines)) -} diff --git a/command/lists.go b/command/lists.go deleted file mode 100644 index b355e49..0000000 --- a/command/lists.go +++ /dev/null @@ -1,394 +0,0 @@ -package command - -import ( - "errors" - "strconv" - "strings" - - "github.com/distributedio/titan/db" -) - -//wangzongsheng note this, we use config item -// var ( -// // ListZipThreshold indicates to create a ziplist when it is exceeded to push elements -// ListZipThreshold = 100 -// ) - -// LPush inserts an entry to the head of the list -func LPush(ctx *Context, txn *db.Transaction) (OnCommit, error) { - args := ctx.Args - - // Create a ziplist if lpush with too much items - var opts []db.ListOption - if len(args[1:]) > ctx.Server.ListZipThreshold { //ListZipThreshold - opts = append(opts, db.UseZip()) - } - - // Number of args should be checked by caller - key := []byte(args[0]) - lst, err := txn.List(key, opts...) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - - for _, val := range args[1:] { - if err := lst.LPush([]byte(val)); err != nil { - return nil, errors.New("ERR " + err.Error()) - } - } - return Integer(ctx.Out, lst.Length()), nil -} - -// LPushx prepend a value to a list, only if the list exists -func LPushx(ctx *Context, txn *db.Transaction) (OnCommit, error) { - key := []byte(ctx.Args[0]) - lst, err := txn.List(key) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - - if !lst.Exist() { - return Integer(ctx.Out, 0), nil - } - for _, val := range ctx.Args[1:] { - if err := lst.LPush([]byte(val)); err != nil { - return nil, errors.New("ERR " + err.Error()) - } - } - return Integer(ctx.Out, lst.Length()), nil -} - -// LPop removes and returns the first element of the list stored at key -func LPop(ctx *Context, txn *db.Transaction) (OnCommit, error) { - args := ctx.Args - - // number of args should be checked by caller - key := []byte(args[0]) - - lst, err := txn.List(key) - - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - - if !lst.Exist() { - return NullBulkString(ctx.Out), nil - } - - val, err := lst.LPop() - if err != nil { - return nil, errors.New("ERR " + err.Error()) - } - return BulkString(ctx.Out, string(val)), nil -} - -// LRange get a range of elements from a list -func LRange(ctx *Context, txn *db.Transaction) (OnCommit, error) { - args := ctx.Args - key := []byte(args[0]) - - start, err := strconv.ParseInt(args[1], 10, 64) - if err != nil { - return nil, ErrInteger - } - stop, err := strconv.ParseInt(args[2], 10, 64) - if err != nil { - return nil, ErrInteger - } - - lst, err := txn.List(key) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - - if !lst.Exist() { - return BytesArray(ctx.Out, nil), nil - } - - items, err := lst.Range(start, stop) - if err != nil { - return nil, errors.New("ERR " + err.Error()) - } - if len(items) == 0 { - return BytesArray(ctx.Out, nil), nil - } - return BytesArray(ctx.Out, items), nil -} - -// LInsert insert an element before or after another element in a list -func LInsert(ctx *Context, txn *db.Transaction) (OnCommit, error) { - key := []byte(ctx.Args[0]) - - before := false - switch strings.ToLower(ctx.Args[1]) { - case "before": - before = true - case "after": - before = false - default: - return nil, ErrSyntax - } - - pivot := []byte(ctx.Args[2]) - value := []byte(ctx.Args[3]) - - lst, err := txn.List(key) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - - if !lst.Exist() { - return Integer(ctx.Out, -1), nil - } - - err = lst.Insert(pivot, value, before) - if err != nil { - return nil, errors.New("ERR " + err.Error()) - } - return Integer(ctx.Out, lst.Length()), nil -} - -//LIndex get an element from a list by its index -func LIndex(ctx *Context, txn *db.Transaction) (OnCommit, error) { - key := []byte(ctx.Args[0]) - n, err := strconv.ParseInt(string(ctx.Args[1]), 10, 64) - if err != nil { - return nil, ErrInteger - } - - lst, err := txn.List(key) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - if !lst.Exist() { - return NullBulkString(ctx.Out), nil - } - val, err := lst.Index(n) - if err != nil { - if err == db.ErrOutOfRange { - return NullBulkString(ctx.Out), nil - } - return nil, errors.New("ERR " + err.Error()) - - } - return BulkString(ctx.Out, string(val)), nil -} - -//LLen get the length of a list -func LLen(ctx *Context, txn *db.Transaction) (OnCommit, error) { - key := []byte(ctx.Args[0]) - lst, err := txn.List(key) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - - if !lst.Exist() { - return Integer(ctx.Out, 0), nil - } - - return Integer(ctx.Out, lst.Length()), nil -} - -//LRem remove elements from a list -func LRem(ctx *Context, txn *db.Transaction) (OnCommit, error) { - key := []byte(ctx.Args[0]) - n, err := strconv.ParseInt(string(ctx.Args[1]), 10, 64) - if err != nil { - return nil, ErrInteger - } - lst, err := txn.List(key) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - - if !lst.Exist() { - return Integer(ctx.Out, 0), nil - } - count, err := lst.LRem([]byte(ctx.Args[2]), n) - if err != nil { - return nil, errors.New("ERR " + err.Error()) - } - return Integer(ctx.Out, int64(count)), nil -} - -//LTrim trim a list to the specified range -func LTrim(ctx *Context, txn *db.Transaction) (OnCommit, error) { - key := []byte(ctx.Args[0]) - start, err := strconv.ParseInt(ctx.Args[1], 10, 64) - if err != nil { - return nil, ErrInteger - } - stop, err := strconv.ParseInt(ctx.Args[2], 10, 64) - if err != nil { - return nil, ErrInteger - } - lst, err := txn.List(key) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - - if !lst.Exist() { - return BulkString(ctx.Out, "OK"), nil - } - if err = lst.LTrim(start, stop); err != nil { - return nil, errors.New("ERR " + err.Error()) - } - - return BulkString(ctx.Out, "OK"), nil - -} - -//LSet set the value of an element in a list by its index -func LSet(ctx *Context, txn *db.Transaction) (OnCommit, error) { - key := []byte(ctx.Args[0]) - lst, err := txn.List(key) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - - if !lst.Exist() { - return nil, ErrNoSuchKey - } - n, err := strconv.ParseInt(ctx.Args[1], 10, 64) - if err != nil { - return nil, ErrInteger - } - if err := lst.Set(n, []byte(ctx.Args[2])); err != nil { - if err == db.ErrOutOfRange { - return nil, ErrIndex - } - return nil, errors.New("ERR " + err.Error()) - } - return SimpleString(ctx.Out, "OK"), nil -} - -//RPop remove and get the last element in a list -func RPop(ctx *Context, txn *db.Transaction) (OnCommit, error) { - key := []byte(ctx.Args[0]) - lst, err := txn.List(key) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - - if !lst.Exist() { - return NullBulkString(ctx.Out), nil - } - val, err := lst.RPop() - if err != nil { - return nil, errors.New("ERR " + err.Error()) - } - return BulkString(ctx.Out, string(val)), nil -} - -// RPopLPush remove the last element in a list, prepend it to another list and return it -func RPopLPush(ctx *Context, txn *db.Transaction) (OnCommit, error) { - listsrc, err := txn.List([]byte(ctx.Args[0])) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - - if !listsrc.Exist() { - return NullBulkString(ctx.Out), nil - } - val, err := listsrc.RPop() - if err != nil { - return nil, errors.New("ERR " + err.Error()) - } - - // create dst list on not exist - listdst, err := txn.List([]byte(ctx.Args[1])) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, ErrSyntax - } - if err = listdst.LPush(val); err != nil { - return nil, ErrTypeMismatch - } - return BulkString(ctx.Out, string(val)), nil -} - -// RPush append one or multiple values to a list -func RPush(ctx *Context, txn *db.Transaction) (OnCommit, error) { - args := ctx.Args - key := []byte(args[0]) - - // Create a ziplist if lpush with too much items - var opts []db.ListOption - if len(args[1:]) > ctx.Server.ListZipThreshold { //ListZipThreshold - opts = append(opts, db.UseZip()) - } - - lst, err := txn.List(key, opts...) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - - for _, val := range args[1:] { - if err := lst.RPush([]byte(val)); err != nil { - return nil, errors.New("ERR " + err.Error()) - } - } - return Integer(ctx.Out, lst.Length()), nil -} - -// RPushx append a value to a list, only if the list exists -func RPushx(ctx *Context, txn *db.Transaction) (OnCommit, error) { - key := []byte(ctx.Args[0]) - lst, err := txn.List(key) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - if !lst.Exist() { - return Integer(ctx.Out, 0), nil - } - for _, val := range ctx.Args[1:] { - if err := lst.RPush([]byte(val)); err != nil { - return nil, errors.New("ERR " + err.Error()) - } - } - return Integer(ctx.Out, lst.Length()), nil -} diff --git a/command/lists_test.go b/command/lists_test.go deleted file mode 100644 index 5d663f4..0000000 --- a/command/lists_test.go +++ /dev/null @@ -1,467 +0,0 @@ -package command - -import ( - "strconv" - "testing" - - "github.com/stretchr/testify/assert" -) - -func initList(t *testing.T, key string, length int) { - args := []string{key} - for i := length; i >= 1; i-- { - args = append(args, strconv.Itoa(i)) - } - ctx := ContextTest("lpush", args...) - Call(ctx) - lines := ctxLines(ctx.Out) - strlen := strconv.Itoa(length) - assert.Equal(t, ":"+strlen, lines[0]) -} - -func clearList(t *testing.T, key string) { - ctx := ContextTest("del", key) - Call(ctx) - lines := ctxLines(ctx.Out) - assert.Equal(t, ":1", lines[0]) -} - -func rpushList(t *testing.T, key, val string) []string { - ctx := ContextTest("rpush", key, val) - Call(ctx) - return ctxLines(ctx.Out) - -} - -func TestLLen(t *testing.T) { - // init - key := "list-llen-key" - initList(t, key, 3) - - // case 1 - ctx := ContextTest("llen", key) - Call(ctx) - lines := ctxLines(ctx.Out) - assert.Equal(t, ":3", lines[0]) - - // case 2 - lines = rpushList(t, key, "4") - assert.Equal(t, ":4", lines[0]) - ctx = ContextTest("llen", key) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, ":4", lines[0]) - // end - clearList(t, key) - - // zlist init - key = "list-llen-zlistkey" - initList(t, key, 600) - - ctx = ContextTest("llen", key) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, ":600", lines[0]) - - lines = rpushList(t, key, "lastval") - assert.Equal(t, ":601", lines[0]) - ctx = ContextTest("llen", key) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, ":601", lines[0]) - // end - clearList(t, key) - -} - -func TestLIndex(t *testing.T) { - // init - key := "list-lindex-list" - initList(t, key, 3) - - // case 1 - ctx := ContextTest("lindex", key, "0") - Call(ctx) - lines := ctxLines(ctx.Out) - assert.Equal(t, "1", lines[1]) - - // case 2 - ctx = ContextTest("lindex", key, "-1") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "3", lines[1]) - - // case 4 - ctx = ContextTest("lindex", key, "-100") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "$-1", lines[0]) - - // end - clearList(t, key) - - // init zlikst - key = "list-lindex-zlist" - initList(t, key, 600) - - // case 1 - ctx = ContextTest("lindex", key, "0") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "1", lines[1]) - - // case 2 - ctx = ContextTest("lindex", key, "-1") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "600", lines[1]) - - // case 4 - ctx = ContextTest("lindex", key, "-700") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "$-1", lines[0]) - - // end - clearList(t, key) - -} - -func TestLInsert(t *testing.T) { - key := "list-linsert-list" - initList(t, key, 3) - - // case 1 - ctx := ContextTest("linsert", key, "BeFoRe", "2", "a1") - Call(ctx) - lines := ctxLines(ctx.Out) - assert.Equal(t, ":4", lines[0]) - // case 2 - ctx = ContextTest("linsert", key, "afTER", "2", "b1") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, ":5", lines[0]) - - // end - ctx = ContextTest("lrange", key, "0", "-1") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "a1", lines[4]) - assert.Equal(t, "b1", lines[8]) - clearList(t, key) - - key = "list-linsert-zlist" - initList(t, key, 600) - - // case 1 - ctx = ContextTest("linsert", key, "BeFoRe", "2", "a1") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, ":601", lines[0]) - // case 2 - ctx = ContextTest("linsert", key, "afTER", "2", "b1") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, ":602", lines[0]) - - // end - ctx = ContextTest("lrange", key, "0", "-1") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "a1", lines[4]) - assert.Equal(t, "b1", lines[8]) - clearList(t, key) -} - -func TestLPop(t *testing.T) { - // init - key := "list-lpop-list" - initList(t, key, 3) - - // case 1 - ctx := ContextTest("lpop", key) - Call(ctx) - lines := ctxLines(ctx.Out) - assert.Equal(t, "1", lines[1]) - // case 2 - ctx = ContextTest("lpop", key) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "2", lines[1]) - // case 3 - ctx = ContextTest("lpop", key) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "3", lines[1]) - // case 4 - ctx = ContextTest("lpop", key) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "$-1", lines[0]) - - key = "list-lpop-zlist" - initList(t, key, 600) - - for i := 1; i <= 600; i++ { - stri := strconv.Itoa(i) - ctx = ContextTest("lpop", key) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, stri, lines[1], "test zlist lpop error") - } - ctx = ContextTest("lpop", key) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "$-1", lines[0]) - -} - -func TestLPush(t *testing.T) { - // init - key := "list-lpush-list" - initList(t, key, 3) - ctx := ContextTest("lpush", key, "b") - Call(ctx) - lines := ctxLines(ctx.Out) - assert.Equal(t, ":4", lines[0]) - ctx = ContextTest("lrange", key, "0", "-1") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Contains(t, lines, "b") - clearList(t, key) - - key = "list-lpush-zlist" - initList(t, key, 600) - ctx = ContextTest("lpush", key, "zlist") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, ":601", lines[0]) - ctx = ContextTest("lrange", key, "0", "-1") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Contains(t, lines, "zlist") - clearList(t, key) -} - -func TestLPushx(t *testing.T) { - key := "list-lpushx-list" - ctx := ContextTest("lpushx", key, "b") - Call(ctx) - lines := ctxLines(ctx.Out) - assert.Equal(t, ":0", lines[0]) - initList(t, key, 3) - ctx = ContextTest("lpushx", key, "b") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, ":4", lines[0]) - - ctx = ContextTest("lrange", key, "0", "-1") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Contains(t, lines, "b") - clearList(t, key) - - key = "list-lpushx-zlist" - initList(t, key, 600) - ctx = ContextTest("lpushx", key, "zlist") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, ":601", lines[0]) - ctx = ContextTest("lrange", key, "0", "-1") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Contains(t, lines, "zlist") - clearList(t, key) -} - -func TestLRange(t *testing.T) { - key := "list-lrange-list" - initList(t, key, 3) - ctx := ContextTest("lrange", key, "-6", "1") - Call(ctx) - lines := ctxLines(ctx.Out) - assert.Equal(t, "1", lines[2]) - assert.Equal(t, "2", lines[4]) - - ctx = ContextTest("lrange", key, "-7", "-2") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "1", lines[2]) - assert.Equal(t, "2", lines[4]) - - ctx = ContextTest("lrange", key, "-2", "-5") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "*0", lines[0]) - - ctx = ContextTest("lrange", key, "0", "2") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "1", lines[2]) - assert.Equal(t, "2", lines[4]) - assert.Equal(t, "3", lines[6]) - - ctx = ContextTest("lrange", key, "0", "0") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "1", lines[2]) - clearList(t, key) - - key = "list-lrange-zlist" - initList(t, key, 600) - ctx = ContextTest("lrange", key, "-6", "1") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "*0", lines[0]) - - ctx = ContextTest("lrange", key, "-7", "-2") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Contains(t, lines, "599") - - ctx = ContextTest("lrange", key, "-2", "-5") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "*0", lines[0]) - - ctx = ContextTest("lrange", key, "0", "2") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "1", lines[2]) - - ctx = ContextTest("lrange", key, "0", "0") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "1", lines[2]) - clearList(t, key) - -} - -func TestLSet(t *testing.T) { - key := "list-lset-list" - initList(t, key, 3) - - ctx := ContextTest("lset", key, "0") - Call(ctx) - lines := ctxLines(ctx.Out) - assert.Equal(t, "-ERR wrong number of arguments for 'lset' command", lines[0]) - - ctx = ContextTest("lset", key, "0", "testval") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "+OK", lines[0]) - - ctx = ContextTest("lindex", key, "0") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "testval", lines[1]) - - ctx = ContextTest("lset", key, "-1", "aa") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "+OK", lines[0]) - - ctx = ContextTest("lset", key, "100", "bb") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "-ERR index out of range", lines[0]) - clearList(t, key) - - key = "list-lset-zlist" - initList(t, key, 600) - - ctx = ContextTest("lset", key, "0") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "-ERR wrong number of arguments for 'lset' command", lines[0]) - - ctx = ContextTest("lset", key, "0", "testval") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "+OK", lines[0]) - - ctx = ContextTest("lindex", key, "0") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "testval", lines[1]) - - ctx = ContextTest("lset", key, "-1", "aa") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "+OK", lines[0]) - - ctx = ContextTest("lset", key, "10000", "bb") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "-ERR index out of range", lines[0]) - -} - -func TestRPush(t *testing.T) { - key := "list-rpush-list" - ctx := ContextTest("rpush", key, "first") - Call(ctx) - lines := ctxLines(ctx.Out) - assert.Equal(t, ":1", lines[0]) - ctx = ContextTest("lrange", key, "0", "-1") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Contains(t, lines, "first") - clearList(t, key) - - initList(t, key, 3) - ctx = ContextTest("rpush", key, "aa") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, ":4", lines[0]) - ctx = ContextTest("lrange", key, "0", "-1") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "aa", lines[8]) - clearList(t, key) - - key = "list-rpush-zlist" - initList(t, key, 600) - ctx = ContextTest("rpush", key, "aa") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, ":601", lines[0]) - ctx = ContextTest("lrange", key, "0", "-1") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "aa", lines[1202]) - clearList(t, key) - -} - -func TestRPushx(t *testing.T) { - key := "list-rpushx-list" - ctx := ContextTest("rpushx", key, "first") - Call(ctx) - lines := ctxLines(ctx.Out) - assert.Equal(t, ":0", lines[0]) - - initList(t, key, 3) - ctx = ContextTest("rpushx", key, "aa") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, ":4", lines[0]) - ctx = ContextTest("lrange", key, "0", "-1") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "aa", lines[8]) - clearList(t, key) - - key = "list-rpushx-zlist" - initList(t, key, 600) - ctx = ContextTest("rpushx", key, "aa") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, ":601", lines[0]) - ctx = ContextTest("lrange", key, "0", "-1") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "aa", lines[1202]) - clearList(t, key) - -} diff --git a/command/server.go b/command/server.go deleted file mode 100644 index 1ea4f1a..0000000 --- a/command/server.go +++ /dev/null @@ -1,396 +0,0 @@ -package command - -import ( - "errors" - "fmt" - "os" - "runtime" - "strconv" - "strings" - "time" - - "github.com/distributedio/titan/context" - "github.com/distributedio/titan/db" - "github.com/distributedio/titan/encoding/resp" -) - -const sysAdminNamespace = "$sys.admin" - -// Monitor streams back every command processed by the Titan server -func Monitor(ctx *Context) { - ctx.Server.Monitors.Store(ctx.Client.RemoteAddr, ctx) - resp.ReplySimpleString(ctx.Out, "OK") -} - -// Client manages client connections -func Client(ctx *Context) { - syntaxErr := "ERR Syntax error, try CLIENT (LIST | KILL | GETNAME | SETNAME | PAUSE | REPLY)" - list := func(ctx *Context) { - now := time.Now() - var lines []string - clients := &ctx.Server.Clients - clients.Range(func(k, v interface{}) bool { - client := v.(*context.ClientContext) - if ctx.Client.Namespace != sysAdminNamespace && client.Namespace != ctx.Client.Namespace { - return true - } - age := now.Sub(client.Created) / time.Second - idle := now.Sub(client.Updated) / time.Second - flags := "N" - if client.Multi { - flags = "x" - } - - // id=2 addr=127.0.0.1:39604 fd=6 name= age=196 idle=2 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=client - line := fmt.Sprintf("id=%d addr=%s fd=%d name=%s age=%d idle=%d "+ - "flags=%s db=%d sub=%d psub=%d multi=%d qbuf=%d qbuf-free=%d obl=%d oll=%d omem=%d events=%s cmd=%s\n", - client.ID, client.RemoteAddr, 0, client.Name, age, idle, flags, client.DB.ID, 0, 0, len(client.Commands), - 0, 0, 0, 0, 0, "rw", client.LastCmd) - lines = append(lines, line) - return true - }) - resp.ReplyBulkString(ctx.Out, strings.Join(lines, "")) - } - getname := func(ctx *Context) { - name := ctx.Client.Name - if len(name) != 0 { - resp.ReplyBulkString(ctx.Out, name) - return - } - resp.ReplyNullBulkString(ctx.Out) - } - setname := func(ctx *Context) { - args := ctx.Args[1:] - if len(args) != 1 { - resp.ReplyError(ctx.Out, syntaxErr) - return - } - ctx.Client.Name = args[0] - resp.ReplySimpleString(ctx.Out, "OK") - } - pause := func(ctx *Context) { - if ctx.Client.Namespace != sysAdminNamespace { - resp.ReplyError(ctx.Out, "ERR client pause can be used by $sys.admin only") - return - } - args := ctx.Args[1:] - if len(args) != 1 { - resp.ReplyError(ctx.Out, syntaxErr) - return - } - msec, err := strconv.ParseInt(args[0], 10, 64) - if err != nil { - resp.ReplyError(ctx.Out, "ERR timeout is not an integer or out of range") - return - } - ctx.Server.Pause = time.Duration(msec) * time.Millisecond - resp.ReplySimpleString(ctx.Out, "OK") - } - reply := func(ctx *Context) { - args := ctx.Args[1:] - if len(args) != 1 { - resp.ReplyError(ctx.Out, syntaxErr) - return - } - switch strings.ToLower(args[0]) { - case "on": - ctx.Client.SkipN = 0 - resp.ReplySimpleString(ctx.Out, "OK") - case "off": - ctx.Client.SkipN = -1 - case "skip": - ctx.Client.SkipN = 1 - } - } - kill := func(ctx *Context) { - args := ctx.Args[1:] - if len(args) < 1 { - resp.ReplyError(ctx.Out, syntaxErr) - return - } - var addr string - var id int64 - var ctype string // client type - var err error - skipSelf := true - if len(args) == 1 { - addr = args[0] - skipSelf = false // you can kill yourself in old fashion - } else if len(args)%2 != 0 { - resp.ReplyError(ctx.Out, syntaxErr) - return - } - for i := 0; i < len(args)-1; i += 2 { - switch strings.ToLower(string(args[i])) { - case "addr": - addr = string(args[i+1]) - case "type": - ctype = string(args[i+1]) - case "id": - id, err = strconv.ParseInt(string(args[i+1]), 10, 64) - if err != nil { - resp.ReplyError(ctx.Out, syntaxErr) - return - } - case "skipme": - answer := string(args[i+1]) - if strings.ToLower(answer) == "yes" { - skipSelf = true - } - if strings.ToLower(answer) == "no" { - skipSelf = false - } - } - } - // now kill clients with above rules - killed := 0 - closeSelf := false - ctx.Server.Clients.Range(func(k, v interface{}) bool { - cli := v.(*context.ClientContext) - - if cli.Namespace != sysAdminNamespace && cli.Namespace != ctx.Client.Namespace { - return true - } - - if id != 0 && cli.ID != id { - return true - } - if addr != "" && cli.RemoteAddr != addr { - return true - } - if ctype != "" && strings.ToLower(ctype) != "normal" { - return true - } - if ctx.Client.ID == cli.ID && skipSelf { - return true - } - if ctx.Client.ID == cli.ID { - killed++ - closeSelf = true - return true - } - - cli.Close() - killed++ - return true - }) - - if len(args) == 1 { - if killed == 0 { - resp.ReplyError(ctx.Out, "ERR No such client") - } else { - resp.ReplySimpleString(ctx.Out, "OK") - } - } else { - resp.ReplyInteger(ctx.Out, int64(killed)) - } - if closeSelf { - close(ctx.Client.Done) - } - } - - args := ctx.Args - switch strings.ToLower(args[0]) { - case "list": - list(ctx) - case "kill": - kill(ctx) - case "getname": - getname(ctx) - case "setname": - setname(ctx) - case "reply": - reply(ctx) - case "pause": - pause(ctx) - default: - resp.ReplyError(ctx.Out, syntaxErr) - } - -} - -// Debug the titan server -func Debug(ctx *Context, txn *db.Transaction) (OnCommit, error) { - switch strings.ToLower(ctx.Args[0]) { - case "object": - return debugObject(ctx, txn) - default: - return nil, errors.New("ERR not supported") - } -} -func debugObject(ctx *Context, txn *db.Transaction) (OnCommit, error) { - key := []byte(ctx.Args[1]) - obj, err := txn.Object(key) - if err != nil { - return nil, errors.New("ERR " + err.Error()) - } - return SimpleString(ctx.Out, obj.String()), nil -} - -// RedisCommand returns Array reply of details about all Redis commands -func RedisCommand(ctx *Context) { - count := func(ctx *Context) { - resp.ReplyInteger(ctx.Out, int64(len(commands))) - } - getkeys := func(ctx *Context) { - args := ctx.Args[1:] - if len(args) == 0 { - resp.ReplyError(ctx.Out, "ERR Unknown subcommand or wrong number of arguments.") - return - } - name := args[0] - cmdInfo, ok := commands[name] - if !ok { - resp.ReplyError(ctx.Out, "ERR Invalid command specified") - return - } - - if cmdInfo.Cons.Arity > 0 && len(args) != cmdInfo.Cons.Arity { - resp.ReplyError(ctx.Out, "ERR Invalid number of arguments specified for command") - return - } - if cmdInfo.Cons.Arity < 0 && len(args) < -cmdInfo.Cons.Arity { - resp.ReplyError(ctx.Out, "ERR Invalid number of arguments specified for command") - return - } - var keys []string - last := cmdInfo.Cons.LastKey - if last < 0 { - last += len(args) - } - for i := cmdInfo.Cons.FirstKey; i <= last; i += cmdInfo.Cons.KeyStep { - keys = append(keys, args[i]) - } - resp.ReplyArray(ctx.Out, len(keys)) - for _, key := range keys { - resp.ReplyBulkString(ctx.Out, key) - } - } - info := func(ctx *Context) { - names := ctx.Args[1:] - resp.ReplyArray(ctx.Out, len(names)) - for _, name := range names { - if cmd, ok := commands[name]; ok { - resp.ReplyArray(ctx.Out, 6) - resp.ReplyBulkString(ctx.Out, name) - resp.ReplyInteger(ctx.Out, int64(cmd.Cons.Arity)) - - flags := parseFlags(cmd.Cons.Flags) - resp.ReplyArray(ctx.Out, len(flags)) - for i := range flags { - resp.ReplyBulkString(ctx.Out, flags[i]) - } - - resp.ReplyInteger(ctx.Out, int64(cmd.Cons.FirstKey)) - resp.ReplyInteger(ctx.Out, int64(cmd.Cons.LastKey)) - resp.ReplyInteger(ctx.Out, int64(cmd.Cons.KeyStep)) - } else { - resp.ReplyNullBulkString(ctx.Out) - } - } - } - list := func(ctx *Context) { - resp.ReplyArray(ctx.Out, len(commands)) - for name, cmd := range commands { - resp.ReplyArray(ctx.Out, 6) - resp.ReplyBulkString(ctx.Out, name) - resp.ReplyInteger(ctx.Out, int64(cmd.Cons.Arity)) - - flags := parseFlags(cmd.Cons.Flags) - resp.ReplyArray(ctx.Out, len(flags)) - for i := range flags { - resp.ReplyBulkString(ctx.Out, flags[i]) - } - - resp.ReplyInteger(ctx.Out, int64(cmd.Cons.FirstKey)) - resp.ReplyInteger(ctx.Out, int64(cmd.Cons.LastKey)) - resp.ReplyInteger(ctx.Out, int64(cmd.Cons.KeyStep)) - } - - } - args := ctx.Args - if len(args) == 0 { - list(ctx) - return - } - switch strings.ToLower(args[0]) { - case "count": - count(ctx) - case "getkeys": - getkeys(ctx) - case "info": - info(ctx) - default: - resp.ReplyError(ctx.Out, "ERR Unknown subcommand or wrong number of arguments.") - } -} - -// FlushDB clears current db -// This function is **VERY DANGEROUS**. It's not only running on one single region, but it can -// delete a large range that spans over many regions, bypassing the Raft layer. -func FlushDB(ctx *Context, txn *db.Transaction) (OnCommit, error) { - kv := txn.Kv() - if err := kv.FlushDB(ctx); err != nil { - return nil, errors.New("ERR " + err.Error()) - } - return SimpleString(ctx.Out, "OK"), nil -} - -// FlushAll cleans up all databases -// This function is **VERY DANGEROUS**. It's not only running on one single region, but it can -// delete a large range that spans over many regions, bypassing the Raft layer. -func FlushAll(ctx *Context, txn *db.Transaction) (OnCommit, error) { - kv := txn.Kv() - if err := kv.FlushAll(ctx); err != nil { - return nil, errors.New("ERR " + err.Error()) - } - return SimpleString(ctx.Out, "OK"), nil -} - -// Time returns the server time -func Time(ctx *Context) { - now := time.Now().UnixNano() / int64(time.Microsecond) - sec := now / 1000000 - msec := now % sec - resp.ReplyArray(ctx.Out, 2) - resp.ReplyBulkString(ctx.Out, strconv.Itoa(int(sec))) - resp.ReplyBulkString(ctx.Out, strconv.Itoa(int(msec))) -} - -// Info returns information and statistics about the server in a format that is simple to parse by computers and easy to read by humans -func Info(ctx *Context) { - exe, err := os.Executable() - if err != nil { - resp.ReplyError(ctx.Out, "ERR "+err.Error()) - } - - // count the number of clients - var numberOfClients int - ctx.Server.Clients.Range(func(k, v interface{}) bool { - numberOfClients++ - return true - }) - - var lines []string - lines = append(lines, "# Server") - lines = append(lines, "titan_version:"+context.ReleaseVersion) - lines = append(lines, "titan_git_sha1:"+context.GitHash) - lines = append(lines, "titan_build_id:"+context.BuildTS) - lines = append(lines, "os:"+runtime.GOOS) - lines = append(lines, "arch_bits:"+runtime.GOARCH) - lines = append(lines, "go_version:"+context.GolangVersion) - lines = append(lines, "process_id:"+strconv.Itoa(os.Getpid())) - lines = append(lines, "uptime_in_seconds:"+strconv.FormatInt(int64(time.Since(ctx.Server.StartAt)/time.Second), 10)) - lines = append(lines, "uptime_in_days:"+strconv.FormatInt(int64(time.Since(ctx.Server.StartAt)/time.Second/86400), 10)) - lines = append(lines, "executable:"+exe) - - lines = append(lines, "# Clients") - lines = append(lines, "connected_clients:"+strconv.Itoa(numberOfClients)) - lines = append(lines, "client_longest_output_list:0") - lines = append(lines, "client_biggest_input_buf:0") - lines = append(lines, "blocked_clients:0") - lines = append(lines, "client_namespace:"+ctx.Client.Namespace) - - resp.ReplyBulkString(ctx.Out, strings.Join(lines, "\n")+"\n") - return -} diff --git a/command/server_test.go b/command/server_test.go deleted file mode 100644 index 7dbcd88..0000000 --- a/command/server_test.go +++ /dev/null @@ -1,95 +0,0 @@ -package command - -import ( - "bytes" - "io/ioutil" - "strings" - "testing" - "time" - - "github.com/distributedio/titan/context" - "github.com/distributedio/titan/db" - "github.com/stretchr/testify/assert" -) - -func TestInfo(t *testing.T) { - out := bytes.NewBuffer(nil) - ctx := &Context{ - Name: "info", - Args: nil, - In: nil, - Out: out, - Context: context.New(&context.ClientContext{ - Namespace: "$unittest", - }, &context.ServerContext{ - StartAt: time.Now(), - }), - } - Info(ctx) - // t.Log(out.String()) - if strings.Index(out.String(), "ERR") == 0 { - t.Fail() - } -} - -func TestMonitor(t *testing.T) { - assert := assert.New(t) - cli := &context.ClientContext{ - Namespace: "$unittest", - ID: 1, - RemoteAddr: "127.0.0.1", - DB: &db.DB{Namespace: "$unittest", ID: 0}, - } - serv := &context.ServerContext{} - - out := bytes.NewBuffer(nil) - ctx := &Context{ - Name: "monitor", - Args: nil, - In: nil, - Out: out, - Context: context.New(cli, serv), - } - - Monitor(ctx) - assert.Equal("+OK\r\n", out.String()) - - out.Reset() - ctx = &Context{ - Name: "ping", - Args: nil, - In: nil, - Out: ioutil.Discard, - Context: ctx.Context, - } - feedMonitors(ctx) - assert.Contains(out.String(), "[0 127.0.0.1] ping \r\n") -} - -func TestClient_List(t *testing.T) { - assert := assert.New(t) - now := time.Now() - cli := &context.ClientContext{ - Namespace: "$unittest", - ID: 1, - RemoteAddr: "127.0.0.1", - DB: &db.DB{Namespace: "$unittest", ID: 0}, - Created: now, - Updated: now, - } - serv := &context.ServerContext{} - serv.Clients.Store(cli.RemoteAddr, cli) - - out := bytes.NewBuffer(nil) - ctx := &Context{ - Name: "client", - Args: []string{"list"}, - In: nil, - Out: out, - Context: context.New(cli, serv), - } - - Client(ctx) - - assert.Contains(out.String(), "id=1 addr=127.0.0.1") -} diff --git a/command/sets.go b/command/sets.go deleted file mode 100644 index 5758173..0000000 --- a/command/sets.go +++ /dev/null @@ -1,336 +0,0 @@ -package command - -import ( - "bytes" - "container/heap" - "errors" - "strconv" - - "github.com/distributedio/titan/db" -) - -// SAdd adds the specified members to the set stored at key -func SAdd(ctx *Context, txn *db.Transaction) (OnCommit, error) { - key := []byte(ctx.Args[0]) - members := make([][]byte, len(ctx.Args[1:])) - for i, member := range ctx.Args[1:] { - members[i] = []byte(member) - } - set, err := txn.Set(key) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - added, err := set.SAdd(members...) - if err != nil { - return nil, errors.New("ERR " + err.Error()) - } - return Integer(ctx.Out, added), nil -} - -// SMembers returns all the members of the set value stored at key -func SMembers(ctx *Context, txn *db.Transaction) (OnCommit, error) { - key := []byte(ctx.Args[0]) - set, err := txn.Set(key) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - members, err := set.SMembers() - if err != nil { - return nil, errors.New("ERR " + err.Error()) - } - return BytesArray(ctx.Out, members), nil -} - -// SCard returns the set cardinality (number of elements) of the set stored at key -func SCard(ctx *Context, txn *db.Transaction) (OnCommit, error) { - key := []byte(ctx.Args[0]) - set, err := txn.Set(key) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - count, err := set.SCard() - if err != nil { - return nil, errors.New("ERR " + err.Error()) - } - return Integer(ctx.Out, int64(count)), nil -} - -// SIsmember returns if member is a member of the set stored at key -func SIsmember(ctx *Context, txn *db.Transaction) (OnCommit, error) { - key := []byte(ctx.Args[0]) - member := []byte(ctx.Args[1]) - set, err := txn.Set(key) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - count, err := set.SIsmember(member) - if err != nil { - return nil, errors.New("ERR " + err.Error()) - } - return Integer(ctx.Out, int64(count)), nil - -} - -// SPop removes and returns one or more random elements from the set value store at key -func SPop(ctx *Context, txn *db.Transaction) (OnCommit, error) { - if len(ctx.Args) > 2 { - return nil, ErrSyntax - } - - count := 1 - var err error - key := []byte(ctx.Args[0]) - if len(ctx.Args) == 2 { - count, err = strconv.Atoi(ctx.Args[1]) - if err != nil { - return nil, ErrInteger - } - } - - set, err := txn.Set(key) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - members, err := set.SPop(int64(count)) - if err != nil { - return nil, errors.New("ERR " + err.Error()) - } - return BytesArray(ctx.Out, members), nil -} - -// SRem removes the specified members from the set stored at key -func SRem(ctx *Context, txn *db.Transaction) (OnCommit, error) { - var members [][]byte - key := []byte(ctx.Args[0]) - for _, member := range ctx.Args[1:] { - members = append(members, []byte(member)) - } - set, err := txn.Set(key) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - count, err := set.SRem(members) - if err != nil { - return nil, errors.New("ERR " + err.Error()) - } - return Integer(ctx.Out, int64(count)), nil -} - -// SMove movies member from the set at source to the set at destination -func SMove(ctx *Context, txn *db.Transaction) (OnCommit, error) { - member := make([]byte, 0, len(ctx.Args[2])) - key := []byte(ctx.Args[0]) - destkey := []byte(ctx.Args[1]) - member = []byte(ctx.Args[2]) - set, err := txn.Set(key) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - count, err := set.SMove(destkey, member) - if err != nil { - return nil, errors.New("ERR " + err.Error()) - } - return Integer(ctx.Out, int64(count)), nil -} - -// SUnion returns the members of the set resulting from the union of all the given sets. -func SUnion(ctx *Context, txn *db.Transaction) (OnCommit, error) { - var members [][]byte - var setsIter []*db.SetIter - for _, key := range ctx.Args { - set, err := txn.Set([]byte(key)) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - if !set.Exists() { - continue - } - siter, err := set.Iter() - if err != nil { - return nil, errors.New("ERR " + err.Error()) - } - defer siter.Iter.Close() - setsIter = append(setsIter, siter) - } - - h := MinHeap(setsIter) - heap.Init(&h) - var last []byte - for len(h) != 0 { - min := h[0].Value() - - // ignore the duplicated member - if last == nil || !bytes.Equal(min, last) { - members = append(members, min) - } - last = min - - if err := h[0].Iter.Next(); err != nil { - return nil, err - } - if h[0].Valid() { - heap.Fix(&h, 0) - } else { - heap.Remove(&h, 0) - } - } - return BytesArray(ctx.Out, members), nil -} - -type MinHeap []*db.SetIter - -func (h MinHeap) Len() int { return len(h) } -func (h MinHeap) Less(i, j int) bool { return bytes.Compare(h[i].Value(), h[j].Value()) < 0 } -func (h MinHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] } -func (h *MinHeap) Push(x interface{}) { - item := x.(*db.SetIter) - *h = append(*h, item) -} - -func (h *MinHeap) Pop() interface{} { - old := *h - n := len(old) - item := old[n-1] - *h = old[0 : n-1] - return item -} - -// SInter returns the members of the set resulting from the intersection of all the given sets. -func SInter(ctx *Context, txn *db.Transaction) (OnCommit, error) { - var members [][]byte - setsIter := make([]*db.SetIter, len(ctx.Args)) - for i, key := range ctx.Args { - set, err := txn.Set([]byte(key)) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - // If the set corresponding to key does not exist, it is processed as an empty set - if !set.Exists() { - return BytesArray(ctx.Out, members), nil - } - siter, err := set.Iter() - if err != nil { - return nil, errors.New("ERR " + err.Error()) - } - defer siter.Iter.Close() - setsIter[i] = siter - } - - h := MinHeap(setsIter) - heap.Init(&h) - var last []byte - k := len(h) //k-way merge - n := k - for len(h) != 0 { - min := h[0].Value() - if last == nil || bytes.Equal(last, min) { - n-- - } else { - n = k - 1 - } - last = min - - // it is a member of intersection if there are k equal values continuously - if n == 0 { - members = append(members, min) - } - - if err := h[0].Iter.Next(); err != nil { - return nil, err - } - if h[0].Valid() { - heap.Fix(&h, 0) - } else { - heap.Remove(&h, 0) - } - } - return BytesArray(ctx.Out, members), nil -} - -// SDiff returns the members of the set resulting from the difference between the first set and all the successive sets. -func SDiff(ctx *Context, txn *db.Transaction) (OnCommit, error) { - var members [][]byte - var setsIter []*db.SetIter - for i, key := range ctx.Args { - set, err := txn.Set([]byte(key)) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - if !set.Exists() && i != 0 { - continue - } - siter, err := set.Iter() - if err != nil { - return nil, errors.New("ERR " + err.Error()) - } - defer siter.Iter.Close() - setsIter = append(setsIter, siter) - } - iter := setsIter[0] - h := MinHeap(setsIter[1:]) - heap.Init(&h) - for iter.Valid() { - member := iter.Value() - if len(h) == 0 { - members = append(members, member) - if err := iter.Iter.Next(); err != nil { - return nil, errors.New("ERR " + err.Error()) - } - continue - } - - min := h[0].Value() - switch bytes.Compare(member, min) { - case -1: - members = append(members, member) - if err := iter.Iter.Next(); err != nil { - return nil, errors.New("ERR " + err.Error()) - } - case 0: - if err := iter.Iter.Next(); err != nil { - return nil, errors.New("ERR " + err.Error()) - } - fallthrough - case 1: - if err := h[0].Iter.Next(); err != nil { - return nil, errors.New("ERR " + err.Error()) - } - if h[0].Valid() { - heap.Fix(&h, 0) - } else { - heap.Remove(&h, 0) - } - } - } - return BytesArray(ctx.Out, members), nil -} diff --git a/command/sets_test.go b/command/sets_test.go deleted file mode 100644 index b4efc4b..0000000 --- a/command/sets_test.go +++ /dev/null @@ -1,494 +0,0 @@ -package command - -import ( - "strconv" - "testing" - - "github.com/stretchr/testify/assert" -) - -func initSets(t *testing.T, key string, length int) { - args := []string{key} - for i := length; i > 0; i-- { - args = append(args, strconv.Itoa(i)) - } - ctx := ContextTest("sadd", args...) - Call(ctx) - lines := ctxLines(ctx.Out) - strlen := strconv.Itoa(length) - assert.Equal(t, ":"+strlen, lines[0]) -} - -func clearSets(t *testing.T, key string) { - ctx := ContextTest("del", key) - Call(ctx) - lines := ctxLines(ctx.Out) - assert.Equal(t, ":1", lines[0]) -} - -func setSets(t *testing.T, args ...string) []string { - ctx := ContextTest("sadd", args...) - Call(ctx) - return ctxLines(ctx.Out) -} - -func TestAdd(t *testing.T) { - key := "set-sadd" - testkey := "set-sadd-test" - //initSets(t, key, 3) - - //case1 - ctx := ContextTest("sadd", key, "1", "2", "3") - Call(ctx) - lines := ctxLines(ctx.Out) - assert.Equal(t, ":3", lines[0]) - - lines = setSets(t, key, "1", "2") - assert.Equal(t, ":0", lines[0]) - - lines = setSets(t, key, "3", "4") - assert.Equal(t, ":1", lines[0]) - - // end - clearSets(t, key) - - //case2 - ctx = ContextTest("sadd", testkey, "3", "1") - Call(ctx) - lines = ctxLines(ctx.Out) - - assert.Equal(t, ":2", lines[0]) - -} - -func TestSCard(t *testing.T) { - // init - key := "set-scard" - initSets(t, key, 3) - - // case 1 - ctx := ContextTest("scard", key) - Call(ctx) - lines := ctxLines(ctx.Out) - assert.Equal(t, ":3", lines[0]) - - // case 2 - lines = setSets(t, key, "a", "b") - assert.Equal(t, ":2", lines[0]) - ctx = ContextTest("scard", key) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, ":5", lines[0]) - - // case 3 - lines = setSets(t, key, "a", "c") - assert.Equal(t, ":1", lines[0]) - ctx = ContextTest("scard", key) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, ":6", lines[0]) - - // end - clearSets(t, key) -} - -func TestSMembers(t *testing.T) { - // init - key := "set-smembers" - //case1 - ctx := ContextTest("sadd", key, "1", "2", "3") - Call(ctx) - ctx = ContextTest("smembers", key) - Call(ctx) - lines := ctxLines(ctx.Out) - assert.Equal(t, "*3", lines[0]) - assert.Equal(t, "$1", lines[1]) - assert.Equal(t, "1", lines[2]) - assert.Equal(t, "$1", lines[3]) - assert.Equal(t, "2", lines[4]) - assert.Equal(t, "$1", lines[5]) - assert.Equal(t, "3", lines[6]) - - //end - clearSets(t, key) -} - -func TestSIsmember(t *testing.T) { - - key := "set-ismember" - - ctx := ContextTest("sadd", key, "1", "2", "3") - Call(ctx) - - //case 1 - ctx = ContextTest("sismember", key, "1") - Call(ctx) - lines := ctxLines(ctx.Out) - assert.Equal(t, ":1", lines[0]) - //case 2 - ctx = ContextTest("sismember", key, "4") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, ":0", lines[0]) - - //end - clearSets(t, key) -} -func TestSPop(t *testing.T) { - key := "set-spop" - - ctx := ContextTest("sadd", key, "1", "2", "3", "4", "5") - Call(ctx) - - //case 1 - ctx = ContextTest("spop", key) - Call(ctx) - lines := ctxLines(ctx.Out) - assert.Equal(t, "*1", lines[0]) - //assert.Equal(t, "$1", lines[1]) - //assert.Equal(t, "1", lines[2]) - - ctx = ContextTest("smembers", key) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "*4", lines[0]) - /* - assert.Equal(t, "$1", lines[1]) - assert.Equal(t, "2", lines[2]) - assert.Equal(t, "$1", lines[3]) - assert.Equal(t, "3", lines[4]) - assert.Equal(t, "$1", lines[5]) - assert.Equal(t, "4", lines[6]) - assert.Equal(t, "$1", lines[7]) - assert.Equal(t, "5", lines[8]) - */ - //case 2 - ctx = ContextTest("spop", key, "2") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "*2", lines[0]) - /* assert.Equal(t, "$1", lines[1]) - assert.Equal(t, "2", lines[2]) - assert.Equal(t, "$1", lines[3]) - assert.Equal(t, "3", lines[4]) - */ - ctx = ContextTest("smembers", key) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "*2", lines[0]) - /* assert.Equal(t, "$1", lines[1]) - assert.Equal(t, "4", lines[2]) - assert.Equal(t, "$1", lines[3]) - assert.Equal(t, "5", lines[4]) - */ - ctx = ContextTest("spop", key, "m1") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "-ERR value is not an integer or out of range", lines[0]) - ctx = ContextTest("smembers", key) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "*2", lines[0]) - assert.Equal(t, "$1", lines[1]) - assert.Equal(t, "4", lines[2]) - assert.Equal(t, "$1", lines[3]) - assert.Equal(t, "5", lines[4]) - - ctx = ContextTest("spop", key, "m1", "m2") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "-ERR syntax error", lines[0]) - ctx = ContextTest("smembers", key) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "*2", lines[0]) - assert.Equal(t, "$1", lines[1]) - assert.Equal(t, "4", lines[2]) - assert.Equal(t, "$1", lines[3]) - assert.Equal(t, "5", lines[4]) - - ctx = ContextTest("spop", key, "1", "2") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "-ERR syntax error", lines[0]) - ctx = ContextTest("smembers", key) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "*2", lines[0]) - assert.Equal(t, "$1", lines[1]) - assert.Equal(t, "4", lines[2]) - assert.Equal(t, "$1", lines[3]) - assert.Equal(t, "5", lines[4]) - - clearSets(t, key) -} -func TestSRem(t *testing.T) { - - key := "set-srem" - - ctx := ContextTest("sadd", key, "1", "2", "3") - Call(ctx) - - //case 1 - ctx = ContextTest("srem", key, "1") - Call(ctx) - lines := ctxLines(ctx.Out) - assert.Equal(t, ":1", lines[0]) - - ctx = ContextTest("smembers", key) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "*2", lines[0]) - assert.Equal(t, "$1", lines[1]) - assert.Equal(t, "2", lines[2]) - assert.Equal(t, "$1", lines[3]) - assert.Equal(t, "3", lines[4]) - //case 2 - ctx = ContextTest("srem", key, "5") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, ":0", lines[0]) - - ctx = ContextTest("smembers", key) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "*2", lines[0]) - assert.Equal(t, "$1", lines[1]) - assert.Equal(t, "2", lines[2]) - assert.Equal(t, "$1", lines[3]) - assert.Equal(t, "3", lines[4]) - - //end - clearSets(t, key) -} -func TestSMove(t *testing.T) { - - key := "set-smove" - destkey := "set-dest-key" - ctx := ContextTest("sadd", key, "1", "2", "3") - Call(ctx) - ctx = ContextTest("sadd", destkey, "3", "4") - Call(ctx) - - //case 1 - ctx = ContextTest("smove", key, destkey, "1") - Call(ctx) - lines := ctxLines(ctx.Out) - assert.Equal(t, ":1", lines[0]) - - ctx = ContextTest("smembers", key) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "*2", lines[0]) - assert.Equal(t, "$1", lines[1]) - assert.Equal(t, "2", lines[2]) - assert.Equal(t, "$1", lines[3]) - assert.Equal(t, "3", lines[4]) - ctx = ContextTest("smembers", destkey) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "*3", lines[0]) - assert.Equal(t, "$1", lines[1]) - assert.Equal(t, "1", lines[2]) - assert.Equal(t, "$1", lines[3]) - assert.Equal(t, "3", lines[4]) - assert.Equal(t, "$1", lines[5]) - assert.Equal(t, "4", lines[6]) - //case 2 - ctx = ContextTest("smove", key, destkey, "5") - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, ":0", lines[0]) - - ctx = ContextTest("smembers", key) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "*2", lines[0]) - assert.Equal(t, "$1", lines[1]) - assert.Equal(t, "2", lines[2]) - assert.Equal(t, "$1", lines[3]) - assert.Equal(t, "3", lines[4]) - - //end - clearSets(t, key) - clearSets(t, destkey) -} -func TestSUnion(t *testing.T) { - key1 := "set-sunion1" - key2 := "set-sunion2" - key3 := "set-sunion3" - - ctx := ContextTest("sadd", key1, "a", "b", "c", "d") - Call(ctx) - ctx = ContextTest("sadd", key2, "c", "d", "e") - Call(ctx) - ctx = ContextTest("sadd", key3, "") - - Call(ctx) - - //case 1 - ctx = ContextTest("sunion", key1, key2) - Call(ctx) - lines := ctxLines(ctx.Out) - assert.Equal(t, "*5", lines[0]) - - //case 2 - ctx = ContextTest("sunion", key1, key3) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "*5", lines[0]) - //case 3 - ctx = ContextTest("sunion", key2, key3) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "*4", lines[0]) - - //case 4 - ctx = ContextTest("sunion", key1, key2, key3) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "*6", lines[0]) - //end - clearSets(t, key1) - - clearSets(t, key2) - - clearSets(t, key3) -} - -func TestSInter(t *testing.T) { - key1 := "set-sinter1" - key2 := "set-sinter2" - key3 := "set-sinter3" - key4 := "set-sinter4" - key5 := "set-sinter5" - ctx := ContextTest("sadd", key1, "a", "b", "c", "d") - Call(ctx) - ctx = ContextTest("sadd", key2, "c") - Call(ctx) - ctx = ContextTest("sadd", key3, "a", "c", "e") - Call(ctx) - ctx = ContextTest("sadd", key4, "") - Call(ctx) - - //case 1 - ctx = ContextTest("sinter", key1, key2, key3) - Call(ctx) - lines := ctxLines(ctx.Out) - assert.Equal(t, "*1", lines[0]) - assert.Equal(t, "$1", lines[1]) - assert.Equal(t, "c", lines[2]) - - //case 2 - ctx = ContextTest("sinter", key1, key3) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "*2", lines[0]) - assert.Equal(t, "$1", lines[1]) - assert.Equal(t, "a", lines[2]) - assert.Equal(t, "$1", lines[3]) - assert.Equal(t, "c", lines[4]) - - //case 3 - ctx = ContextTest("sinter", key1, key3, key5) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "*0", lines[0]) - //case 4 - ctx = ContextTest("sinter", key1, key2, key4) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "*0", lines[0]) - //end - clearSets(t, key1) - - clearSets(t, key2) - - clearSets(t, key3) - - clearSets(t, key4) - -} - -func TestSDiff(t *testing.T) { - key1 := "set-sdiff1" - key2 := "set-sdiff2" - key3 := "set-sdiff3" - key4 := "set-sdiff4" - key5 := "set-sdiff5" - key6 := "set-sdiff6" - - ctx := ContextTest("sadd", key1, "a", "b", "c", "d") - Call(ctx) - ctx = ContextTest("sadd", key2, "c") - Call(ctx) - ctx = ContextTest("sadd", key3, "a", "c", "e") - Call(ctx) - ctx = ContextTest("sadd", key4, "a", "a", "a") - Call(ctx) - ctx = ContextTest("sadd", key5, "a", "d", "d") - Call(ctx) - ctx = ContextTest("sadd", key6, "a", "b", "c", "d") - Call(ctx) - - //case 1 - ctx = ContextTest("sdiff", key1, key2) - Call(ctx) - lines := ctxLines(ctx.Out) - assert.Equal(t, "*3", lines[0]) - assert.Equal(t, "$1", lines[1]) - assert.Equal(t, "a", lines[2]) - assert.Equal(t, "$1", lines[3]) - assert.Equal(t, "b", lines[4]) - assert.Equal(t, "$1", lines[5]) - assert.Equal(t, "d", lines[6]) - - //case 2 - ctx = ContextTest("sdiff", key1, key2, key3) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "*2", lines[0]) - assert.Equal(t, "$1", lines[1]) - assert.Equal(t, "b", lines[2]) - assert.Equal(t, "$1", lines[3]) - assert.Equal(t, "d", lines[4]) - //case 3 - ctx = ContextTest("sdiff", key1, key4) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "*3", lines[0]) - assert.Equal(t, "$1", lines[1]) - assert.Equal(t, "b", lines[2]) - assert.Equal(t, "$1", lines[3]) - assert.Equal(t, "c", lines[4]) - assert.Equal(t, "$1", lines[5]) - assert.Equal(t, "d", lines[6]) - //case 4 - ctx = ContextTest("sdiff", key1, key5) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "*2", lines[0]) - assert.Equal(t, "$1", lines[1]) - assert.Equal(t, "b", lines[2]) - assert.Equal(t, "$1", lines[3]) - assert.Equal(t, "c", lines[4]) - - //case 5 - ctx = ContextTest("sdiff", key1, key6) - Call(ctx) - lines = ctxLines(ctx.Out) - assert.Equal(t, "*0", lines[0]) - //end - clearSets(t, key1) - - clearSets(t, key2) - - clearSets(t, key3) - - clearSets(t, key4) - - clearSets(t, key5) - - clearSets(t, key6) - -} diff --git a/command/stats.go b/command/stats.go deleted file mode 100644 index 2d0dd96..0000000 --- a/command/stats.go +++ /dev/null @@ -1,7 +0,0 @@ -package command - -// Statistic for the redis command -type Statistic struct { - Microseconds int64 - Calls int64 -} diff --git a/command/strings.go b/command/strings.go deleted file mode 100644 index 3422742..0000000 --- a/command/strings.go +++ /dev/null @@ -1,664 +0,0 @@ -package command - -import ( - "errors" - "strconv" - "strings" - "time" - - "github.com/distributedio/titan/db" -) - -var ( - //MaxRangeInteger max index in setrange command - MaxRangeInteger = 2<<29 - 1 -) - -// Get the value of key -func Get(ctx *Context, txn *db.Transaction) (OnCommit, error) { - key := ctx.Args[0] - str, err := txn.String([]byte(key)) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - val, err := str.Get() - if err != nil { - if err == db.ErrKeyNotFound { - return NullBulkString(ctx.Out), nil - } - return nil, errors.New("ERR " + err.Error()) - } - return BulkString(ctx.Out, string(val)), nil -} - -// Set key to hold the string value -func Set(ctx *Context, txn *db.Transaction) (OnCommit, error) { - key := []byte(ctx.Args[0]) - value := []byte(ctx.Args[1]) - args := ctx.Args - - var next bool - var flag int // flag int // 0 -- null 1---nx 2---xx - var unit int64 // time ms - var expire string //expire = expire *unit - for i := 2; i < len(args); i++ { - if i+1 < len(args) { - next = true - } - if (args[i][0] == 'n' || args[i][0] == 'N') && - (args[i][1] == 'x' || args[i][1] == 'X') && - len(args[i]) == 2 { - flag = 1 - } else if (args[i][0] == 'x' || args[i][0] == 'X') && - (args[i][1] == 'x' || args[i][1] == 'X') && - len(args[i]) == 2 { - flag = 2 - } else if (args[i][0] == 'p' || args[i][0] == 'P') && - (args[i][1] == 'x' || args[i][1] == 'X') && - len(args[i]) == 2 && next { - expire = args[i+1] - i = i + 1 - unit = int64(time.Millisecond) - } else if (args[i][0] == 'e' || args[i][0] == 'E') && - (args[i][1] == 'x' || args[i][1] == 'X') && - len(args[i]) == 2 && next { - expire = args[i+1] - i = i + 1 - unit = int64(time.Second) - } else { - return nil, ErrSyntax - } - } - - //ex|px - if expire != "" { - ui, err := strconv.ParseInt(expire, 10, 64) - if err != nil { - return nil, ErrInteger - } - if ui <= 0 { - return nil, ErrExpire - } - unit = ui * unit - } - - obj, err := txn.Object(key) - if err != nil && err != db.ErrKeyNotFound { - return nil, errors.New("ERR " + err.Error()) - } - - //xx - if flag == 2 { - if err == db.ErrKeyNotFound { - return NullBulkString(ctx.Out), nil - } - } - //nx - if flag == 1 { - if err != db.ErrKeyNotFound { - return NullBulkString(ctx.Out), nil - } - } - - if err != db.ErrKeyNotFound { - txn.Destory(obj, key) - } - - s := db.NewString(txn, key) - if err := s.Set(value, unit); err != nil { - return nil, errors.New("ERR " + err.Error()) - } - return SimpleString(ctx.Out, OK), nil -} - -// MGet returns the values of all specified key -func MGet(ctx *Context, txn *db.Transaction) (OnCommit, error) { - count := len(ctx.Args) - values := make([][]byte, count) - - keys := make([][]byte, count) - for i := range ctx.Args { - keys[i] = []byte(ctx.Args[i]) - } - - strs, err := txn.Strings(keys) - if err != nil { - return nil, errors.New("ERR " + err.Error()) - } - for i, str := range strs { - if str == nil || !str.Exist() { - values[i] = nil - continue - } - values[i], _ = str.Get() - } - return BytesArray(ctx.Out, values), nil -} - -// MSet sets the given keys to their respective values -func MSet(ctx *Context, txn *db.Transaction) (OnCommit, error) { - argc := len(ctx.Args) - args := ctx.Args - if argc%2 != 0 { - return nil, ErrMSet - } - for i := 2; i <= argc; i += 2 { - ctx.Args = args[i-2 : i] - if _, err := Set(ctx, txn); err != nil { - return nil, err - } - } - return SimpleString(ctx.Out, OK), nil -} - -// MSetNx et multiple keys to multiple values,only if none of the keys exist -//TODO bug -func MSetNx(ctx *Context, txn *db.Transaction) (OnCommit, error) { - argc := len(ctx.Args) - args := ctx.Args - if argc%2 != 0 { - return nil, ErrMSet - } - for i := 2; i <= argc; i += 2 { - if str, _ := txn.String([]byte(ctx.Args[0])); !str.Exist() { - ctx.Args = append(args[i-2:i], "nx") - if _, err := Set(ctx, txn); err != nil { - return nil, errors.New("ERR " + err.Error()) - } - ctx.Args = ctx.Args[2:] - continue - } - return Integer(ctx.Out, int64(0)), nil - } - return Integer(ctx.Out, int64(1)), nil -} - -// Strlen returns the length of the string value stored at key -func Strlen(ctx *Context, txn *db.Transaction) (OnCommit, error) { - key := ctx.Args[0] - str, err := txn.String([]byte(key)) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - - if !str.Exist() { - return Integer(ctx.Out, int64(0)), nil - } - - v, _ := str.Len() - - return Integer(ctx.Out, int64(v)), nil -} - -// Append a value to a key -func Append(ctx *Context, txn *db.Transaction) (OnCommit, error) { - key := []byte(ctx.Args[0]) - value := []byte(ctx.Args[1]) - str, err := txn.String(key) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - - llen, err := str.Append(value) - if err != nil { - return nil, errors.New("ERR " + err.Error()) - } - return Integer(ctx.Out, int64(llen)), nil -} - -// GetSet sets the string value of a key and return its old value -func GetSet(ctx *Context, txn *db.Transaction) (OnCommit, error) { - key := []byte(ctx.Args[0]) - v := []byte(ctx.Args[1]) - str, err := txn.String(key) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - if !str.Exist() { - return NullBulkString(ctx.Out), nil - } - - value, err := str.GetSet(v) - if err != nil { - return nil, errors.New("ERR " + err.Error()) - } - return BulkString(ctx.Out, string(value)), nil -} - -// GetRange increments the integer value of a keys by the given amount -func GetRange(ctx *Context, txn *db.Transaction) (OnCommit, error) { - key := ctx.Args[0] - str, err := txn.String([]byte(key)) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - - start, err := strconv.Atoi(ctx.Args[1]) - if err != nil { - return nil, ErrInteger - } - end, err := strconv.Atoi(ctx.Args[2]) - if err != nil { - return nil, ErrInteger - } - - if !str.Exist() { - return NullBulkString(ctx.Out), nil - } - - value := str.GetRange(start, end) - if len(value) == 0 { - return NullBulkString(ctx.Out), nil - } - - return BulkString(ctx.Out, string(value)), nil -} - -// SetNx sets the value of a key ,only if the key does not exist -func SetNx(ctx *Context, txn *db.Transaction) (OnCommit, error) { - //get the key - key := []byte(ctx.Args[0]) - str, err := txn.String(key) - if err != nil { - if err == db.ErrTypeMismatch { - return Integer(ctx.Out, int64(0)), nil - } - return nil, errors.New("ERR " + err.Error()) - } - //key 不存在时，为 key 设置指定的值。设置成功，返回 1 。 设置失败，返回 0 。 - if str.Exist() { - return Integer(ctx.Out, int64(0)), nil - } - - if err := str.Set([]byte(ctx.Args[1])); err != nil { - return nil, errors.New("ERR " + err.Error()) - } - return Integer(ctx.Out, int64(1)), nil -} - -// SetEx sets the value and expiration of a key KEY_NAME TIMEOUT VALUE -func SetEx(ctx *Context, txn *db.Transaction) (OnCommit, error) { - //get the key - key := []byte(ctx.Args[0]) - obj, err := txn.Object(key) - if err != nil && err != db.ErrKeyNotFound { - return nil, errors.New("ERR " + err.Error()) - } - if err != db.ErrKeyNotFound { - txn.Destory(obj, key) - } - - s := db.NewString(txn, key) - ui, err := strconv.ParseInt(ctx.Args[1], 10, 64) - if err != nil { - return nil, ErrInteger - } - if ui <= 0 { - return nil, ErrExpireSetEx - } - unit := ui * int64(time.Second) - if err := s.Set([]byte(ctx.Args[2]), unit); err != nil { - return nil, errors.New("ERR " + err.Error()) - } - - return SimpleString(ctx.Out, OK), nil -} - -// PSetEx sets the value and expiration in milliseconds of a key -func PSetEx(ctx *Context, txn *db.Transaction) (OnCommit, error) { - //get the key - key := []byte(ctx.Args[0]) - obj, err := txn.Object(key) - if err != nil && err != db.ErrKeyNotFound { - return nil, errors.New("ERR " + err.Error()) - } - - if err != db.ErrKeyNotFound { - txn.Destory(obj, key) - } - - s := db.NewString(txn, key) - ui, err := strconv.ParseUint(ctx.Args[1], 10, 64) - if err != nil { - return nil, ErrInteger - } - unit := ui * uint64(time.Millisecond) - if err := s.Set([]byte(ctx.Args[2]), int64(unit)); err != nil { - return nil, errors.New("ERR " + err.Error()) - } - - return SimpleString(ctx.Out, OK), nil -} - -//SetRange overwrites part of the string stored at key, starting at the specified offset, for the entire length of value. -func SetRange(ctx *Context, txn *db.Transaction) (OnCommit, error) { - offset, err := strconv.Atoi(ctx.Args[1]) - if err != nil { - return nil, ErrInteger - } - - key := []byte(ctx.Args[0]) - if offset < 0 || offset > MaxRangeInteger { - return nil, ErrMaximum - } - - str, err := txn.String(key) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - - // If the offset is larger than the current length of the string at key, the string is padded with zero-bytes to make offset fit. - val, err := str.SetRange(int64(offset), []byte(ctx.Args[2])) - if err != nil { - return nil, errors.New("ERR " + err.Error()) - } - return Integer(ctx.Out, int64(len(val))), nil - -} - -// Incr increments the integer value of a key by one -func Incr(ctx *Context, txn *db.Transaction) (OnCommit, error) { - key := []byte(ctx.Args[0]) - str, err := txn.String(key) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - delta, err := str.Incr(1) - if err != nil { - return nil, errors.New("ERR " + err.Error()) - } - return Integer(ctx.Out, int64(delta)), nil -} - -// IncrBy increments the integer value of a key by the given amount -func IncrBy(ctx *Context, txn *db.Transaction) (OnCommit, error) { - key := []byte(ctx.Args[0]) - str, err := txn.String(key) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - delta, err := strconv.ParseInt(ctx.Args[1], 10, 0) - if err != nil { - return nil, ErrInteger - } - - delta, err = str.Incr(delta) - if err != nil { - return nil, errors.New("ERR " + err.Error()) - } - return Integer(ctx.Out, int64(delta)), nil -} - -// IncrByFloat increments the float value of a key by the given amount -func IncrByFloat(ctx *Context, txn *db.Transaction) (OnCommit, error) { - key := []byte(ctx.Args[0]) - str, err := txn.String([]byte(key)) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - delta, err := strconv.ParseFloat(ctx.Args[1], 64) - if err != nil { - return nil, ErrInteger - } - delta, err = str.Incrf(delta) - if err != nil { - return nil, errors.New("ERR " + err.Error()) - } - return BulkString(ctx.Out, strconv.FormatFloat(delta, 'f', 17, 64)), nil -} - -// Decr decrements the integer value of a key by one -func Decr(ctx *Context, txn *db.Transaction) (OnCommit, error) { - key := []byte(ctx.Args[0]) - str, err := txn.String(key) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - - delta, err := str.Incr(-1) - if err != nil { - return nil, errors.New("ERR " + err.Error()) - } - return Integer(ctx.Out, int64(delta)), nil -} - -// DecrBy decrements the integer value of a key by the given number -func DecrBy(ctx *Context, txn *db.Transaction) (OnCommit, error) { - key := []byte(ctx.Args[0]) - str, err := txn.String(key) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - delta, err := strconv.ParseInt(ctx.Args[1], 10, 64) - if err != nil { - return nil, ErrInteger - } - - delta, err = str.Incr(-delta) - if err != nil { - return nil, errors.New("ERR " + err.Error()) - } - return Integer(ctx.Out, int64(delta)), nil -} - -// SetBit sets or clears the bit at offset in the string value stored at key. -func SetBit(ctx *Context, txn *db.Transaction) (OnCommit, error) { - key := []byte(ctx.Args[0]) - offset, err := strconv.Atoi(ctx.Args[1]) - if err != nil { - return nil, ErrBitOffset - } - if offset < 0 { - return nil, ErrBitOffset - } - - on, err := strconv.Atoi(ctx.Args[2]) - if err != nil { - return nil, ErrBitInteger - } - - // Bits can only be set or cleared... - if (on & ^1) != 0 { - return nil, ErrBitInteger - } - - str, err := txn.String(key) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - val, err := str.SetBit(offset, on) - if err != nil { - return nil, errors.New("ERR " + err.Error()) - } - if val != 0 { - return Integer(ctx.Out, 1), nil - } - return Integer(ctx.Out, 0), nil -} - -// GetBit gets the bit at offset in the string value stored at key. -func GetBit(ctx *Context, txn *db.Transaction) (OnCommit, error) { - key := []byte(ctx.Args[0]) - offset, err := strconv.Atoi(ctx.Args[1]) - if err != nil || offset < 0 { - return nil, ErrBitOffset - } - - str, err := txn.String(key) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - - if !str.Exist() { - return Integer(ctx.Out, 0), nil - } - - val, err := str.GetBit(offset) - if err != nil { - return nil, errors.New("ERR " + err.Error()) - } - - if val != 0 { - return Integer(ctx.Out, 1), nil - } - return Integer(ctx.Out, 0), nil -} - -// BitCount counts the number of set bits (population counting) in a string. -func BitCount(ctx *Context, txn *db.Transaction) (OnCommit, error) { - key := []byte(ctx.Args[0]) - str, err := txn.String(key) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - - if !str.Exist() { - return Integer(ctx.Out, 0), nil - } - - var begin, end int - switch len(ctx.Args) { - case 3: - begin, err = strconv.Atoi(ctx.Args[1]) - if err != nil { - return nil, ErrInteger - } - end, err = strconv.Atoi(ctx.Args[2]) - if err != nil { - return nil, ErrInteger - } - case 1: - begin = 0 - end = len(str.Meta.Value) - 1 - default: - return nil, ErrSyntax - } - - val, err := str.BitCount(begin, end) - if err != nil { - return nil, errors.New("ERR " + err.Error()) - } - return Integer(ctx.Out, int64(val)), nil -} - -// BitPos finds first bit set or clear in a string -func BitPos(ctx *Context, txn *db.Transaction) (OnCommit, error) { - bit, err := strconv.Atoi(string(ctx.Args[1])) - if err != nil { - return nil, ErrInteger - } - - if (bit != 0) && (bit != 1) { - return nil, ErrBitInvaild - } - - key := []byte(ctx.Args[0]) - str, err := txn.String(key) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - - if !str.Exist() { - if bit == 1 { - return Integer(ctx.Out, -1), nil - } - return Integer(ctx.Out, 0), nil - } - - var begin, end int - switch len(ctx.Args) { - case 4: - begin, err = strconv.Atoi(ctx.Args[2]) - if err != nil { - return nil, ErrInteger - } - end, err = strconv.Atoi(ctx.Args[3]) - if err != nil { - return nil, ErrInteger - } - case 3: - begin, err = strconv.Atoi(ctx.Args[2]) - if err != nil { - return nil, ErrInteger - } - end = len(str.Meta.Value) - 1 - case 2: - begin = 0 - end = len(str.Meta.Value) - 1 - default: - return nil, ErrSyntax - } - - val, err := str.BitPos(bit, begin, end) - if err != nil { - return nil, errors.New("ERR " + err.Error()) - } - return Integer(ctx.Out, int64(val)), nil -} - -// BitOp performs bitwise operations between strings -func BitOp(ctx *Context, txn *db.Transaction) (OnCommit, error) { - //TODO - if strings.EqualFold(ctx.Args[0], "and") { - } else if strings.EqualFold(ctx.Args[0], "or") { - } else if strings.EqualFold(ctx.Args[0], "xor") { - } else if strings.EqualFold(ctx.Args[0], "not") { - if len(ctx.Args) != 3 { - return nil, ErrBitOp - } - } else { - return nil, ErrSyntax - } - return Integer(ctx.Out, int64(0)), nil -} - -// BitField performs arbitrary bitfield integer operations on strings -func BitField(ctx *Context, txn *db.Transaction) (OnCommit, error) { - //TODO - return Integer(ctx.Out, int64(0)), nil -} diff --git a/command/strings_test.go b/command/strings_test.go deleted file mode 100644 index a106509..0000000 --- a/command/strings_test.go +++ /dev/null @@ -1,688 +0,0 @@ -package command - -import ( - "strconv" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestSet(t *testing.T) { - ctx := ContextTest("set", "key", "value") - Call(ctx) -} - -func EqualGet(t *testing.T, key string, value string, e error) { - ctx := ContextTest("get", key) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), value) -} - -func EqualStrlen(t *testing.T, key string, ll int) { - ctx := ContextTest("strlen", key) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), strconv.Itoa(ll)) -} - -//bug,bug -func EqualMGet(t *testing.T, keys []string, values []string, e error) { - ctx := ContextTest("mget", keys...) - Call(ctx) - for _, v := range values { - assert.Contains(t, ctxString(ctx.Out), v) - } - - // assert.Len(t, ctxString(ctx.Out), len(value)) -} - -var ( - value = "value" -) - -func SetEXS(key string) []string { - args := make([]string, 5) - args[0] = key - args[1] = value - args[2] = "ex" - args[3] = "1000" - args[4] = "nx" - return args -} - -func SetPXS(key string) []string { - args := make([]string, 5) - args[0] = key - args[1] = value - args[2] = "px" - args[3] = "1000" - args[4] = "nx" - return args -} - -func SetFour(key string) []string { - args := make([]string, 4) - args[0] = key - args[1] = "value" - args[2] = "px" - args[3] = "1000" - return args -} - -// test set ex NX|XX|未知 -func TestStringSetEXS(t *testing.T) { - - key := "setexs" - args := SetEXS(key) - - ctx := ContextTest("set", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), "OK") - EqualGet(t, key, value, nil) - EqualStrlen(t, key, len(value)) - - //修改key失败 - args = SetEXS(key) - args[1] = "v2" - ctx = ContextTest("set", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), "-1") - EqualGet(t, key, value, nil) - - args = SetEXS(key) - args[1] = "v2" - args[4] = "xx" - - ctx = ContextTest("set", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), "OK") - EqualGet(t, key, "v2", nil) - EqualStrlen(t, key, len("v2")) - // 测试nx - // 修改key 失败 - args = SetEXS(key) - args[1] = "value" - ctx = ContextTest("set", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), "-1") - EqualGet(t, key, "v2", nil) - - //乱序测试 - args[0] = key - args[1] = "v1" - args[2] = "xx" - args[3] = "ex" - args[4] = "1000" - - ctx = ContextTest("set", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), "OK") - EqualGet(t, key, "v1", nil) - - //异常测试 - args = SetEXS(key) - ctx = ContextTest("set", args[:3]...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), ErrSyntax.Error()) - - //异常测试 - args = SetEXS(key) - args[3] = "bx" - ctx = ContextTest("set", args[:3]...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), ErrSyntax.Error()) -} - -// test set px NX|XX|未知 -func TestStringSetPXS(t *testing.T) { - - key := "setpx" - args := SetPXS(key) - - ctx := ContextTest("set", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), "OK") - EqualGet(t, key, value, nil) - - //修改key失败 - args = SetPXS(key) - args[1] = "v2" - - ctx = ContextTest("set", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), "-1") - EqualGet(t, key, value, nil) - - // 测试nx - args = SetPXS(key) - args[1] = "v2" - args[4] = "xx" - - ctx = ContextTest("set", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), "OK") - EqualGet(t, key, args[1], nil) - - // 修改key 失败 - // key = - args = SetPXS("kpx2") - args[4] = "xx" - - ctx = ContextTest("set", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), "-1") - EqualGet(t, key, "v2", nil) - - //异常测试 - ctx = ContextTest("set", args[:3]...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), ErrSyntax.Error()) - - //乱序测试 - args[0] = key - args[1] = "v1" - args[2] = "xx" - args[3] = "px" - args[4] = "10000" - - ctx = ContextTest("set", args[:3]...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), "OK") - EqualGet(t, key, "v1", nil) - - //异常测试 - args = SetPXS(key) - args[3] = "bx" - args[4] = "xx" - - ctx = ContextTest("set", args[:3]...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), ErrSyntax.Error()) -} - -// test set px|ex|未知 -func TestStringSetFour(t *testing.T) { - key := "setpxex" - args := SetFour(key) - ctx := ContextTest("set", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), "OK") - EqualGet(t, key, value, nil) - - args[2] = "ex" - ctx = ContextTest("set", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), "OK") - EqualGet(t, key, value, nil) - - args[3] = "x" - ctx = ContextTest("set", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), ErrInteger.Error()) - - args[2] = "zx" - - ctx = ContextTest("set", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), ErrSyntax.Error()) -} - -// test set NX|XX|未知 -func TestStringSetThree(t *testing.T) { - args := make([]string, 3) - key := "setxxnxt" - args[0] = key - args[1] = value - args[2] = "nx" - ctx := ContextTest("set", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), "OK") - EqualGet(t, key, value, nil) - - args[1] = "v1" - args[2] = "xx" - - ctx = ContextTest("set", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), "OK") - EqualGet(t, key, "v1", nil) - - args[2] = "zx" - ctx = ContextTest("set", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), ErrSyntax.Error()) -} - -func TestStringSet(t *testing.T) { - - args := make([]string, 2) - key := "set" - args[0] = key - args[1] = "value" - ctx := ContextTest("lpush", args...) - Call(ctx) - - ctx = ContextTest("set", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), "OK") - EqualGet(t, key, "value", nil) - // EqualMGet(t, []string{key}, []string{"value"}, nil) -} - -func TestStringSetEx(t *testing.T) { - args := make([]string, 3) - key := "setex" - args[0] = key - args[1] = "10000" - args[2] = value - - ctx := ContextTest("setex", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), "OK") - EqualGet(t, key, value, nil) - - args[1] = "x" - ctx = ContextTest("setex", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), ErrInteger.Error()) -} - -func TestStringSetNx(t *testing.T) { - args := make([]string, 2) - key := "setnx" - args[0] = key - args[1] = value - - ctx := ContextTest("setnx", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), "1") - EqualGet(t, key, value, nil) - - args[1] = "v1" - ctx = ContextTest("setnx", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), "0") - EqualGet(t, key, value, nil) -} - -func TestStringPSetEx(t *testing.T) { - args := make([]string, 3) - key := "psetex" - args[0] = key - args[1] = "100000" - args[2] = value - - ctx := ContextTest("psetex", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), "OK") - EqualGet(t, key, value, nil) - - args[1] = "x" - ctx = ContextTest("psetex", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), ErrInteger.Error()) - -} - -func TestStringSetRange(t *testing.T) { - args := make([]string, 3) - key := "setrange" - args[0] = key - args[1] = "3" - args[2] = value - - ctx := ContextTest("setrange", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), "8") - ctx = ContextTest("get", key) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), value) - - args[1] = "1" - args[2] = "lll" - ctx = ContextTest("setrange", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), "8") - ctx = ContextTest("get", key) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), "lllalue") - - args[1] = "10" - args[2] = value - ctx = ContextTest("setrange", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), "15") - ctx = ContextTest("get", key) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), "\x00lllalue\x00\x00value") - - args[1] = "s" - ctx = ContextTest("setrange", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), ErrInteger.Error()) - - args[1] = "-2" - ctx = ContextTest("setrange", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), ErrMaximum.Error()) -} -func TestStringIncr(t *testing.T) { - args := make([]string, 1) - args[0] = "incr" - ctx := ContextTest("incr", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), "1") - - args[0] = "setex" - ctx = ContextTest("incr", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), ErrInteger.Error()) -} - -func TestStringIncrBy(t *testing.T) { - args := make([]string, 2) - args[0] = "incrby" - args[1] = "2" - ctx := ContextTest("incrby", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), "2") - - args[1] = "-2" - ctx = ContextTest("incrby", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), "0") - - args[1] = "02" - ctx = ContextTest("incrby", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), "2") - -} - -//bug -func TestStringIncrByFloat(t *testing.T) { - args := make([]string, 2) - args[0] = "incrbyfloat" - args[1] = "2.0e2" - - ctx := ContextTest("incrbyfloat", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), "200") - - args[1] = "-2.0e2" - ctx = ContextTest("incrbyfloat", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), "0") - - args[1] = "02" - ctx = ContextTest("incrbyfloat", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), "0") -} - -func TestStringDecr(t *testing.T) { - args := make([]string, 1) - args[0] = "decr" - ctx := ContextTest("decr", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), "-1") - - args[0] = "setex" - ctx = ContextTest("decr", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), ErrInteger.Error()) -} - -func TestStringDecrBy(t *testing.T) { - args := make([]string, 2) - args[0] = "decrby" - args[1] = "2" - - ctx := ContextTest("decrby", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), "-2") - - args[1] = "-2" - ctx = ContextTest("decrby", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), "0") - - args[1] = "02" - ctx = ContextTest("decrby", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), "-2") -} - -func TestStringMset(t *testing.T) { - args := make([]string, 4) - args[0] = "Mset1" - args[1] = "Mset3" - args[2] = "Mset2" - args[3] = "Mset4" - - ctx := ContextTest("mset", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), "OK") - EqualMGet(t, []string{args[0], args[2]}, []string{args[1], args[3]}, nil) - - ctx = ContextTest("mset", args[:3]...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), ErrMSet.Error()) -} - -func TestStringMsetNx(t *testing.T) { - args := make([]string, 4) - args[0] = "MsetN1" - args[1] = "MsetN3" - args[2] = "MsetN2" - args[3] = "MsetN4" - - ctx := ContextTest("msetnx", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), "1") - EqualMGet(t, []string{args[0], args[2]}, []string{args[1], args[3]}, nil) - - ctx = ContextTest("msetnx", args[:3]...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), ErrMSet.Error()) - - args[2] = "MsetN5" - ctx = ContextTest("msetnx", args...) - Call(ctx) - assert.Contains(t, ctxString(ctx.Out), "0") - EqualGet(t, args[0], args[1], nil) -} - -func TestStringAppend(t *testing.T) { - args := make([]string, 2) - args[0] = "Append" - args[1] = "Ap" - - out := CallTest("append", args...) - assert.Contains(t, out.String(), strconv.Itoa(len(args[1]))) - - out = CallTest("append", args...) - assert.Contains(t, out.String(), strconv.Itoa(len(args[1])*2)) -} - -func TestStringSetBit(t *testing.T) { - tests := []struct { - name string - args []string - want string - }{ - { - name: "1", - args: []string{"setbit", "1", "0", "0"}, - want: "-ERR wrong number of arguments for 'setbit' command", - }, - { - name: "2", - args: []string{"setbit", "x", "0"}, - want: ErrBitOffset.Error(), - }, - { - name: "3", - args: []string{"setbit", "1", "x"}, - want: ErrBitInteger.Error(), - }, - { - name: "4", - args: []string{"setbit", "1", "2"}, - want: ErrBitInteger.Error(), - }, - { - name: "5", - args: []string{"setbit", "1", "1"}, - want: ":0", - }, - { - name: "6", - args: []string{"setbit", "1", "0"}, - want: ":1", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - out := CallTest("setbit", tt.args...) - assert.Contains(t, out.String(), tt.want) - }) - } -} - -func TestStringGetBit(t *testing.T) { - CallTest("setbit", "getbit", "5", "1") - tests := []struct { - name string - args []string - want string - }{ - { - name: "1", - args: []string{"getbit", "1", "0", "0"}, - want: "-ERR wrong number of arguments for 'getbit' command", - }, - { - name: "2", - args: []string{"getbit", "x"}, - want: ErrBitOffset.Error(), - }, - { - name: "3", - args: []string{"getbit", "1"}, - want: ":0", - }, - { - name: "5", - args: []string{"getbit", "5"}, - want: ":1", - }, - { - name: "6", - args: []string{"getbit", "10"}, - want: ":0", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - out := CallTest("getbit", tt.args...) - assert.Contains(t, out.String(), tt.want) - }) - } -} - -func TestStringBitCount(t *testing.T) { - CallTest("setbit", "bit-count", "5", "1") - CallTest("setbit", "bit-count", "9", "1") - tests := []struct { - name string - args []string - want string - }{ - { - name: "1", - args: []string{"bit-count", "1"}, - want: ErrSyntax.Error(), - }, - { - name: "2", - args: []string{"bit-count", "x", "1"}, - want: ErrInteger.Error(), - }, - { - name: "3", - args: []string{"bit-count", "1", "x"}, - want: ErrInteger.Error(), - }, - { - name: "4", - args: []string{"bit-count"}, - want: ":2", - }, - { - name: "5", - args: []string{"bit-count", "1", "1"}, - want: ":1", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - out := CallTest("bitcount", tt.args...) - assert.Contains(t, out.String(), tt.want) - }) - } -} - -func TestStringBitPos(t *testing.T) { - CallTest("set", "bit-pos", "5") - tests := []struct { - name string - args []string - want string - }{ - { - name: "1", - args: []string{"bit-pos", "2"}, - want: ErrBitInvaild.Error(), - }, - { - name: "2", - args: []string{"bitnpos", "0", "0", "1"}, - want: ":0", - }, - { - name: "3", - args: []string{"bitnpos", "1", "0", "1"}, - want: ":-1", - }, - { - name: "4", - args: []string{"bit-pos", "1", "1", "1"}, - want: ":-1", - }, - { - name: "5", - args: []string{"bit-pos", "1", "1"}, - want: ":-1", - }, - { - name: "6", - args: []string{"bit-pos", "1"}, - want: ":2", - }, - { - name: "7", - args: []string{"bit-pos", "1", "1", "1", "1", "1"}, - want: ErrSyntax.Error(), - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - out := CallTest("bitpos", tt.args...) - assert.Contains(t, out.String(), tt.want) - }) - } -} diff --git a/command/test_test.go b/command/test_test.go deleted file mode 100644 index b34af0b..0000000 --- a/command/test_test.go +++ /dev/null @@ -1,52 +0,0 @@ -package command - -import ( - "bytes" - "io" - "strings" - - "github.com/distributedio/titan/conf" - "github.com/distributedio/titan/context" - "github.com/distributedio/titan/db" -) - -var Cfg = &conf.MockConf().TiKV -var mockdb *db.RedisStore - -func init() { - mockdb, _ = db.Open(Cfg) -} - -func ContextTest(name string, args ...string) *Context { - cliCtx := &context.ClientContext{ - DB: mockdb.DB("defalut", 1), - } - servCtx := &context.ServerContext{ - RequirePass: "", - ListZipThreshold: 100, - Store: mockdb, - } - rootCtx, _ := context.WithCancel(context.New(cliCtx, servCtx)) - return &Context{ - Name: name, - Args: args, - In: &bytes.Buffer{}, - Out: &bytes.Buffer{}, - Context: rootCtx, - } -} - -func CallTest(name string, args ...string) *bytes.Buffer { - ctx := ContextTest(name, args...) - Call(ctx) - return ctx.Out.(*bytes.Buffer) -} - -func ctxString(buf io.Writer) string { - return buf.(*bytes.Buffer).String() -} - -func ctxLines(buf io.Writer) []string { - str := ctxString(buf) - return strings.Split(str, "\r\n") -} diff --git a/command/transactions.go b/command/transactions.go deleted file mode 100644 index f381ac4..0000000 --- a/command/transactions.go +++ /dev/null @@ -1,184 +0,0 @@ -package command - -import ( - "bytes" - "strings" - "time" - - "github.com/distributedio/titan/db" - "github.com/distributedio/titan/encoding/resp" - "github.com/distributedio/titan/metrics" - "github.com/shafreeck/retry" - "go.uber.org/zap" -) - -// Multi starts a transaction which will block subsequent commands until 'exec' -func Multi(ctx *Context) { - ctx.Client.Multi = true - resp.ReplySimpleString(ctx.Out, OK) -} - -// Exec all the commands queued in client -func Exec(ctx *Context) { - ctx.Client.Multi = false - pendings := ctx.Client.Commands - if len(pendings) == 0 { - resp.ReplyArray(ctx.Out, 0) - return - } - ctx.Client.Commands = nil - - // Has watch command been issued - watching := ctx.Client.Txn != nil - txn := ctx.Client.Txn - ctx.Client.Txn = nil - - size := len(pendings) - var err error - var outputs []*bytes.Buffer - var onCommits []OnCommit - err = retry.Ensure(ctx, func() error { - if !watching { - txn, err = ctx.Client.DB.Begin() - if err != nil { - zap.L().Error("begin txn failed", - zap.Int64("clientid", ctx.Client.ID), - zap.String("command", ctx.Name), - zap.String("traceid", ctx.TraceID), - zap.Error(err)) - resp.ReplyArray(ctx.Out, 0) - return err - } - } - outputs = make([]*bytes.Buffer, size) - onCommits = make([]OnCommit, size) - commandCount := 0 - for i, cmd := range pendings { - var onCommit OnCommit - out := bytes.NewBuffer(nil) - subCtx := &Context{ - Name: cmd.Name, - Args: cmd.Args, - In: ctx.In, - Out: out, - Context: ctx.Context, - } - name := strings.ToLower(cmd.Name) - if desc, ok := commands[name]; ok && desc.Txn != nil { - start := time.Now() - onCommit, err = TxnCall(subCtx, txn) - zap.L().Debug("execute", zap.String("command", subCtx.Name), zap.Int64("cost(us)", time.Since(start).Nanoseconds()/1000)) - if err != nil { - resp.ReplyError(out, err.Error()) - } - } else { - Call(subCtx) - } - onCommits[i] = onCommit - outputs[i] = out - commandCount++ - } - start := time.Now() - mt := metrics.GetMetrics() - mt.MultiCommandHistogramVec.WithLabelValues(ctx.Client.Namespace, ctx.Name).Observe(float64(commandCount)) - defer func() { - cost := time.Since(start).Seconds() - mt.TxnCommitHistogramVec.WithLabelValues(ctx.Client.Namespace, ctx.Name).Observe(cost) - }() - start = time.Now() - err = txn.Commit(ctx) - zap.L().Debug("commit", zap.String("command", ctx.Name), zap.Int64("cost(us)", time.Since(start).Nanoseconds()/1000)) - if err != nil { - mt.TxnFailuresCounterVec.WithLabelValues(ctx.Client.Namespace, ctx.Name).Inc() - if db.IsRetryableError(err) && !watching { - mt.TxnRetriesCounterVec.WithLabelValues(ctx.Client.Namespace, ctx.Name).Inc() - mt.TxnConflictsCounterVec.WithLabelValues(ctx.Client.Namespace, ctx.Name).Inc() - zap.L().Error("txn commit retry", - zap.Int64("clientid", ctx.Client.ID), - zap.String("command", ctx.Name), - zap.String("traceid", ctx.TraceID), - zap.Error(err)) - return retry.Retriable(err) - } - zap.L().Error("commit failed", - zap.Int64("clientid", ctx.Client.ID), - zap.String("command", ctx.Name), - zap.String("traceid", ctx.TraceID), - zap.Error(err)) - return err - } - return nil - }) - if err != nil { - zap.L().Error("txn failed", - zap.Int64("clientid", ctx.Client.ID), - zap.String("command", ctx.Name), - zap.String("traceid", ctx.TraceID), - zap.Error(err)) - if watching { - resp.ReplyArray(ctx.Out, 0) - return - } - resp.ReplyError(ctx.Out, "EXECABORT Transaction discarded because of txn conflicts") - return - } - - resp.ReplyArray(ctx.Out, size) - // run OnCommit that fill reply to outputs - for i := range onCommits { - c := onCommits[i] - if c != nil { - c() - } - - if _, err := ctx.Out.Write(outputs[i].Bytes()); err != nil { - zap.L().Error("reply to client failed", - zap.Int64("clientid", ctx.Client.ID), - zap.String("command", ctx.Name), - zap.String("traceid", ctx.TraceID), - zap.Error(err)) - break - } - } -} - -// Watch starts a transaction, watch is a global transaction and is not key associated(this is different from redis) -func Watch(ctx *Context) { - txn, err := ctx.Client.DB.Begin() - if err != nil { - resp.ReplyError(ctx.Out, "Err "+err.Error()) - return - } - keys := make([][]byte, len(ctx.Args)) - for i := range ctx.Args { - keys[i] = []byte(ctx.Args[i]) - } - if err := txn.LockKeys(keys...); err != nil { - txn.Rollback() - resp.ReplyError(ctx.Out, "Err "+err.Error()) - return - } - ctx.Client.Txn = txn - resp.ReplySimpleString(ctx.Out, OK) -} - -// Discard flushes all previously queued commands in a transaction and restores the connection state to normal -func Discard(ctx *Context) { - // in watch state, the txn has begun, rollback it - if ctx.Client.Txn != nil { - ctx.Client.Txn.Rollback() - ctx.Client.Txn = nil - } - ctx.Client.Commands = nil - ctx.Client.Multi = false - resp.ReplySimpleString(ctx.Out, OK) -} - -// Unwatch flushes all the previously watched keys for a transaction -func Unwatch(ctx *Context) { - if ctx.Client.Txn != nil { - ctx.Client.Txn.Rollback() - ctx.Client.Txn = nil - } - resp.ReplySimpleString(ctx.Out, OK) -} diff --git a/command/zsets.go b/command/zsets.go deleted file mode 100644 index 05f633c..0000000 --- a/command/zsets.go +++ /dev/null @@ -1,242 +0,0 @@ -package command - -import ( - "errors" - "fmt" - "math" - "strconv" - "strings" - - "github.com/distributedio/titan/db" -) - -// ZAdd adds the specified members with scores to the sorted set -func ZAdd(ctx *Context, txn *db.Transaction) (OnCommit, error) { - key := []byte(ctx.Args[0]) - - fmt.Println("zadd", ctx.Args) - - kvs := ctx.Args[1:] - if len(kvs)%2 != 0 { - return nil, errors.New("ERR syntax error") - } - - uniqueMembers := make(map[string]bool) - count := len(kvs) / 2 - members := make([][]byte, 0, count) - scores := make([]float64, 0, count) - for i := 0; i < len(kvs)-1; i += 2 { - member := kvs[i+1] - if _, ok := uniqueMembers[member]; ok { - continue - } - - members = append(members, []byte(member)) - score, err := strconv.ParseFloat(kvs[i], 64) - if err != nil || math.IsNaN(score) { - return nil, ErrFloat - } - scores = append(scores, score) - - uniqueMembers[member] = true - } - - zset, err := txn.ZSet(key) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - - added, err := zset.ZAdd(members, scores) - if err != nil { - return nil, errors.New("ERR " + err.Error()) - } - - return Integer(ctx.Out, added), nil -} - -func ZRange(ctx *Context, txn *db.Transaction) (OnCommit, error) { - return zAnyOrderRange(ctx, txn, true) -} - -func ZRevRange(ctx *Context, txn *db.Transaction) (OnCommit, error) { - return zAnyOrderRange(ctx, txn, false) -} - -func zAnyOrderRange(ctx *Context, txn *db.Transaction, positiveOrder bool) (OnCommit, error) { - key := []byte(ctx.Args[0]) - start, err := strconv.ParseInt(ctx.Args[1], 10, 64) - if err != nil { - return nil, ErrInteger - } - stop, err := strconv.ParseInt(ctx.Args[2], 10, 64) - if err != nil { - return nil, ErrInteger - } - withScore := bool(false) - if len(ctx.Args) >= 4 { - if strings.ToUpper(ctx.Args[3]) == "WITHSCORES" { - withScore = true - } - } - - zset, err := txn.ZSet(key) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - if !zset.Exist() { - return BytesArray(ctx.Out, nil), nil - } - - items, err := zset.ZAnyOrderRange(start, stop, withScore, positiveOrder) - if err != nil { - return nil, errors.New("ERR " + err.Error()) - } - if len(items) == 0 { - return BytesArray(ctx.Out, nil), nil - } - return BytesArray(ctx.Out, items), nil -} - -func ZRangeByScore(ctx *Context, txn *db.Transaction) (OnCommit, error) { - return zAnyOrderRangeByScore(ctx, txn, true) -} - -func ZRevRangeByScore(ctx *Context, txn *db.Transaction) (OnCommit, error) { - return zAnyOrderRangeByScore(ctx, txn, false) -} - -func zAnyOrderRangeByScore(ctx *Context, txn *db.Transaction, positiveOrder bool) (OnCommit, error) { - key := []byte(ctx.Args[0]) - - startScore, startInclude, err := getFloatAndInclude(ctx.Args[1]) - if err != nil { - return nil, ErrMinOrMaxNotFloat - } - endScore, endInclude, err := getFloatAndInclude(ctx.Args[2]) - if err != nil { - return nil, ErrMinOrMaxNotFloat - } - - withScore := bool(false) - offset := int64(0) - count := int64(math.MaxInt64) - for i := 3; i < len(ctx.Args); i++ { - switch strings.ToUpper(ctx.Args[i]) { - case "WITHSCORES": - withScore = true - case "LIMIT": - if offset, count, err = getLimitParameters(ctx.Args[i+1:]); err != nil { - return nil, err - } - i += 2 - default: - return nil, ErrSyntax - } - } - - zset, err := txn.ZSet(key) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - if !zset.Exist() { - return BytesArray(ctx.Out, nil), nil - } - - items, err := zset.ZAnyOrderRangeByScore(startScore, startInclude, - endScore, endInclude, - withScore, - offset, count, - positiveOrder) - if err != nil { - return nil, errors.New("ERR " + err.Error()) - } - if len(items) == 0 { - return BytesArray(ctx.Out, nil), nil - } - return BytesArray(ctx.Out, items), nil -} - -func ZRem(ctx *Context, txn *db.Transaction) (OnCommit, error) { - key := []byte(ctx.Args[0]) - - uniqueMembers := make(map[string]bool) - members := make([][]byte, 0, len(ctx.Args)-1) - for _, member := range ctx.Args[1:] { - if _, ok := uniqueMembers[member]; ok { - continue - } - - members = append(members, []byte(member)) - uniqueMembers[member] = true - } - - zset, err := txn.ZSet(key) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - if !zset.Exist() { - return Integer(ctx.Out, 0), nil - } - - deleted, err := zset.ZRem(members) - if err != nil { - return nil, errors.New("ERR " + err.Error()) - } - - return Integer(ctx.Out, deleted), nil -} - -func ZCard(ctx *Context, txn *db.Transaction) (OnCommit, error) { - key := []byte(ctx.Args[0]) - - zset, err := txn.ZSet(key) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - if !zset.Exist() { - return Integer(ctx.Out, 0), nil - } - - return Integer(ctx.Out, zset.ZCard()), nil -} - -func ZScore(ctx *Context, txn *db.Transaction) (OnCommit, error) { - key := []byte(ctx.Args[0]) - member := []byte(ctx.Args[1]) - - zset, err := txn.ZSet(key) - if err != nil { - if err == db.ErrTypeMismatch { - return nil, ErrTypeMismatch - } - return nil, errors.New("ERR " + err.Error()) - } - if !zset.Exist() { - return NullBulkString(ctx.Out), nil - } - - score, err := zset.ZScore(member) - if err != nil { - return nil, errors.New("ERR " + err.Error()) - } - if score == nil { - return NullBulkString(ctx.Out), nil - } - - return BulkString(ctx.Out, string(score)), nil -} diff --git a/conf/config.go b/conf/config.go deleted file mode 100644 index 0da2d90..0000000 --- a/conf/config.go +++ /dev/null @@ -1,90 +0,0 @@ -package conf - -import "time" - -// Titan configuration center -type Titan struct { - Server Server `cfg:"server"` - Status Status `cfg:"status"` - TiKV TiKV `cfg:"tikv"` - Logger Logger `cfg:"logger"` - PIDFileName string `cfg:"pid-filename; titan.pid; ; the file name to record connd PID"` -} - -// Server config is the config of titan server -type Server struct { - Auth string `cfg:"auth;;;client connetion auth"` - Listen string `cfg:"listen; 0.0.0.0:7369; netaddr; address to listen"` - SSLCertFile string `cfg:"ssl-cert-file;;;server SSL certificate file (enables SSL support)"` - SSLKeyFile string `cfg:"ssl-key-file;;;server SSL key file"` - MaxConnection int64 `cfg:"max-connection;1000;numeric;client connection count"` - ListZipThreshold int `cfg:"list-zip-threshold;100;numeric;the max limit length of elements in list"` -} - -// TiKV config is the config of tikv sdk -type TiKV struct { - PdAddrs string `cfg:"pd-addrs; mocktikv://; ;pd address in tidb"` - GC GC `cfg:"gc"` - Expire Expire `cfg:"expire"` - ZT ZT `cfg:"zt"` - TiKVGC TiKVGC `cfg:"tikv-gc"` - Logger TiKVLogger `cfg:"logger"` -} - -// TiKVGC config is the config of implement tikv sdk gcwork -type TiKVGC struct { - Disable bool `cfg:"disable; false; boolean; false is used to disable tikvgc "` - Interval time.Duration `cfg:"interval;20m;;gc work tick interval"` - LeaderLifeTime time.Duration `cfg:"leader-life-time;30m;;lease flush leader interval"` - SafePointLifeTime time.Duration `cfg:"safe-point-life-time;10m;;safe point life time "` - Concurrency int `cfg:"concurrency;2;;gc work concurrency"` -} - -// GC config is the config of Titan GC work -type GC struct { - Disable bool `cfg:"disable; false; boolean; false is used to disable gc"` - Interval time.Duration `cfg:"interval;1s;;gc work tick interval"` - LeaderLifeTime time.Duration `cfg:"leader-life-time;3m;;lease flush leader interval"` - BatchLimit int `cfg:"batch-limit;256;numeric;key count limitation per-transection"` -} - -// Expire config is the config of Titan expire work -type Expire struct { - Disable bool `cfg:"disable; false; boolean; false is used to disable expire"` - Interval time.Duration `cfg:"interval;1s;;expire work tick interval"` - LeaderLifeTime time.Duration `cfg:"leader-life-time;3m;;lease flush leader interval"` - BatchLimit int `cfg:"batch-limit;256;numeric;key count limitation per-transection"` -} - -// ZT config is the config of zlist -type ZT struct { - Disable bool `cfg:"disable; false; boolean; false is used to disable zt"` - Workers int `cfg:"workers;5;numeric;parallel workers count"` - BatchCount int `cfg:"batch;10;numeric;object transfer limitation per-transection"` - QueueDepth int `cfg:"depth;100;numeric;ZT Worker queue depth"` - Interval time.Duration `cfg:"interval;1000ms; ;Queue fill interval in milsecond"` -} - -// Logger config is the config of default zap log -type Logger struct { - Name string `cfg:"name; titan; ; the default logger name"` - Path string `cfg:"path; logs/titan; ; the default log path (or stdout/stderr)"` - Level string `cfg:"level; info; ; log level(debug, info, warn, error, panic, fatal)"` - Compress bool `cfg:"compress; false; boolean; true for enabling log compress"` - TimeRotate string `cfg:"time-rotate; 0 0 0 * * *; ; log time rotate pattern(s m h D M W)"` -} - -// TiKVLogger config is the config of tikv log -type TiKVLogger struct { - Path string `cfg:"path; logs/tikv;nonempty ; the default log path (or stdout/stderr)"` - Level string `cfg:"level; info; ; log level(debug, info, warn, error, panic, fatal)"` - Compress bool `cfg:"compress; false; boolean; true for enabling log compress"` - TimeRotate string `cfg:"time-rotate; 0 0 0 * * *; ; log time rotate pattern(s m h D M W)"` -} - -// Status config is the config of exported server -type Status struct { - Listen string `cfg:"listen;0.0.0.0:7345;nonempty; listen address of http server"` - SSLCertFile string `cfg:"ssl-cert-file;;;status server SSL certificate file (enables SSL support)"` - SSLKeyFile string `cfg:"ssl-key-file;;;status server SSL key file"` -} diff --git a/conf/mockconfig.go b/conf/mockconfig.go deleted file mode 100644 index d57250f..0000000 --- a/conf/mockconfig.go +++ /dev/null @@ -1,38 +0,0 @@ -package conf - -import "time" - -// MockConf init and return titan mock conf -func MockConf() *Titan { - return &Titan{ - TiKV: TiKV{ - PdAddrs: "mocktikv://", - GC: GC{ - Disable: false, - Interval: time.Second, - LeaderLifeTime: 3 * time.Minute, - BatchLimit: 256, - }, - Expire: Expire{ - Disable: false, - Interval: time.Second, - LeaderLifeTime: 3 * time.Minute, - BatchLimit: 256, - }, - ZT: ZT{ - Disable: false, - Workers: 5, - BatchCount: 10, - QueueDepth: 100, - Interval: 1000 * time.Millisecond, - }, - TiKVGC: TiKVGC{ - Disable: false, - Interval: 20 * time.Minute, - LeaderLifeTime: 30 * time.Minute, - SafePointLifeTime: 10 * time.Minute, - Concurrency: 2, - }, - }, - } -} diff --git a/conf/titan.toml b/conf/titan.toml deleted file mode 100644 index ab6099e..0000000 --- a/conf/titan.toml +++ /dev/null @@ -1,148 +0,0 @@ -#type: string, description: the file name to record connd PID, default: titan.pid -#pid-filename = "titan.pid" - - -[server] - -#type: string, description: client connetion auth -auth = "" - -#type: string, rules: netaddr, description: address to listen, default: 0.0.0.0:7369 -#listen = "0.0.0.0:7369" - -#type: string, description: server SSL certificate file (enables SSL support) -ssl-cert-file = "" - -#type: string, description: server SSL key file -ssl-key-file = "" - -#type: int64, rules: numeric, description: client connection count, default: 1000 -#max-connection = 1000 - -#type: int, rules: numeric, description: the max limit length of elements in list, default: 100 -#list-zip-threshold = 100 - - - -[status] - -#type: string, rules: nonempty, description: listen address of http server, default: 0.0.0.0:7345 -#listen = "0.0.0.0:7345" - -#type: string, description: status server SSL certificate file (enables SSL support) -ssl-cert-file = "" - -#type: string, description: status server SSL key file -ssl-key-file = "" - - - -[tikv] - -#type: string, description: pd address in tidb, default: mocktikv:// -#pd-addrs = "mocktikv://" - - -[tikv.gc] - -#type: bool, rules: boolean, description: false is used to disable gc, default: false -#disable = false - -#type: time.Duration, description: gc work tick interval, default: 1s -#interval = "1s" - -#type: time.Duration, description: lease flush leader interval, default: 3m -#leader-life-time = "3m0s" - -#type: int, rules: numeric, description: key count limitation per-transection, default: 256 -#batch-limit = 256 - - - -[tikv.expire] - -#type: bool, rules: boolean, description: false is used to disable expire, default: false -#disable = false - -#type: time.Duration, description: expire work tick interval, default: 1s -#interval = "1s" - -#type: time.Duration, description: lease flush leader interval, default: 3m -#leader-life-time = "3m0s" - -#type: int, rules: numeric, description: key count limitation per-transection, default: 256 -#batch-limit = 256 - - - -[tikv.zt] - -#type: bool, rules: boolean, description: false is used to disable zt, default: false -#disable = false - -#type: int, rules: numeric, description: parallel workers count, default: 5 -#workers = 5 - -#type: int, rules: numeric, description: object transfer limitation per-transection, default: 10 -#batch = 10 - -#type: int, rules: numeric, description: ZT Worker queue depth, default: 100 -#depth = 100 - -#type: time.Duration, description: Queue fill interval in milsecond, default: 1000ms -#interval = "1s" - - - -[tikv.tikv-gc] - -#type: bool, rules: boolean, description: false is used to disable tikvgc, default: false -#disable = false - -#type: time.Duration, description: gc work tick interval, default: 20m -#interval = "20m0s" - -#type: time.Duration, description: lease flush leader interval, default: 30m -#leader-life-time = "30m0s" - -#type: time.Duration, description: safe point life time, default: 10m -#safe-point-life-time = "10m0s" - -#type: int, description: gc work concurrency, default: 2 -#concurrency = 2 - - - -[tikv.logger] - -#type: string, rules: nonempty, description: the default log path (or stdout/stderr), default: logs/tikv -#path = "logs/tikv" - -#type: string, description: log level(debug, info, warn, error, panic, fatal), default: info -#level = "info" - -#type: bool, rules: boolean, description: true for enabling log compress, default: false -#compress = false - -#type: string, description: log time rotate pattern(s m h D M W), default: 0 0 0 * * * -#time-rotate = "0 0 0 * * *" - - - -[logger] - -#type: string, description: the default logger name, default: titan -#name = "titan" - -#type: string, description: the default log path (or stdout/stderr), default: logs/titan -#path = "logs/titan" - -#type: string, description: log level(debug, info, warn, error, panic, fatal), default: info -#level = "info" - -#type: bool, rules: boolean, description: true for enabling log compress, default: false -#compress = false - -#type: string, description: log time rotate pattern(s m h D M W), default: 0 0 0 * * * -#time-rotate = "0 0 0 * * *" - diff --git a/context/context.go b/context/context.go deleted file mode 100644 index 5082436..0000000 --- a/context/context.go +++ /dev/null @@ -1,129 +0,0 @@ -package context - -import ( - "context" - "net" - "sync" - "time" - - "github.com/distributedio/titan/db" -) - -const ( - // DefaultNamespace default namespce - DefaultNamespace = "default" -) - -// Version information. -var ( - ReleaseVersion = "None" - BuildTS = "None" - GitHash = "None" - GitBranch = "None" - GitLog = "None" - GolangVersion = "None" - ConfigFile = "None" -) - -// Command releated context -type Command struct { - Name string - Args []string -} - -// ClientContext is the runtime context of a client -type ClientContext struct { - DB *db.DB - Authenticated bool // Client has be authenticated - Namespace string // Namespace of database - RemoteAddr string // Client remote address - ID int64 // Client uniq ID - Name string // Name is set by client setname - Created time.Time - Updated time.Time - LastCmd string - SkipN int // Skip N following commands, (-1 for skipping all commands) - Close func() error - - // When client is in multi...exec block, the Txn is assigned and Multi is set to be true - // Before exec, all command called will be queued in Commands - Txn *db.Transaction // Txn is set when client is in transaction which is triggered by watch command - Multi bool - Commands []*Command - - Done chan struct{} -} - -// NewClientContext new client context object ,id must be uniq -func NewClientContext(id int64, conn net.Conn) *ClientContext { - now := time.Now() - cli := &ClientContext{ - ID: id, - Created: now, - Updated: now, - Namespace: DefaultNamespace, - RemoteAddr: conn.RemoteAddr().String(), - Authenticated: false, - Multi: false, - Done: make(chan struct{}), - Close: conn.Close, - } - return cli -} - -// ServerContext is the runtime context of the server -type ServerContext struct { - RequirePass string - Store *db.RedisStore - Monitors sync.Map - Clients sync.Map - Pause time.Duration // elapse to pause all clients - StartAt time.Time - ListZipThreshold int -} - -// Context combines the client and server context -type Context struct { - context.Context - Client *ClientContext - Server *ServerContext -} - -// New a context -func New(c *ClientContext, s *ServerContext) *Context { - return &Context{Context: context.Background(), Client: c, Server: s} -} - -// CancelFunc tells an operation to abandon its work -type CancelFunc context.CancelFunc - -// WithCancel returns a copy of parent with a new Done channel -func WithCancel(parent *Context) (*Context, CancelFunc) { - ctx := *parent - child, cancel := context.WithCancel(parent.Context) - ctx.Context = child - return &ctx, CancelFunc(cancel) -} - -// WithDeadline returns a copy of the parent context with the deadline adjusted to be no later than d -func WithDeadline(parent *Context, d time.Time) (*Context, CancelFunc) { - ctx := *parent - child, cancel := context.WithDeadline(parent.Context, d) - ctx.Context = child - return &ctx, CancelFunc(cancel) -} - -// WithTimeout returns WithDeadline(parent, time.Now().Add(timeout)). -func WithTimeout(parent *Context, timeout time.Duration) (*Context, CancelFunc) { - ctx := *parent - child, cancel := context.WithTimeout(parent.Context, timeout) - ctx.Context = child - return &ctx, CancelFunc(cancel) -} - -// WithValue returns a copy of parent in which the value associated with key is val. -func WithValue(parent *Context, key, val interface{}) *Context { - ctx := *parent - ctx.Context = context.WithValue(parent.Context, key, val) - return &ctx -} diff --git a/db/codec.go b/db/codec.go deleted file mode 100644 index 349a13f..0000000 --- a/db/codec.go +++ /dev/null @@ -1,94 +0,0 @@ -package db - -import ( - "bytes" - "encoding/binary" - "math" -) - -// EncodeObject encode the object to binary -func EncodeObject(obj *Object) []byte { - b := make([]byte, 42) - copy(b, obj.ID[:16]) - binary.BigEndian.PutUint64(b[16:], uint64(obj.CreatedAt)) - binary.BigEndian.PutUint64(b[24:], uint64(obj.UpdatedAt)) - binary.BigEndian.PutUint64(b[32:], uint64(obj.ExpireAt)) - b[40], b[41] = byte(obj.Type), byte(obj.Encoding) - return b -} - -// DecodeObject decode the object from binary -func DecodeObject(b []byte) (obj *Object, err error) { - if len(b) < ObjectEncodingLength { - return nil, ErrInvalidLength - } - obj = &Object{ - ID: make([]byte, 16), - CreatedAt: int64(binary.BigEndian.Uint64(b[16:])), - UpdatedAt: int64(binary.BigEndian.Uint64(b[24:])), - ExpireAt: int64(binary.BigEndian.Uint64(b[32:])), // 40 bit fields - Type: ObjectType(b[40]), // 41 bit - Encoding: ObjectEncoding(b[41]), // 42 bit - } - copy(obj.ID, b[0:16]) - return obj, nil -} - -// EncodeInt64 encode the int64 object to binary -func EncodeInt64(v int64) ([]byte, error) { - var buf bytes.Buffer - if v < 0 { - v = int64(uint64(v) & 0x7FFFFFFFFFFFFFFF) - } else if v >= 0 { - v = int64(uint64(v) | 0x8000000000000000) - } - - // Ignore the error returned here, because buf is a memory io.Writer, can should not fail here - if err := binary.Write(&buf, binary.BigEndian, v); err != nil { - return nil, err - } - return buf.Bytes(), nil -} - -// DecodeInt64 decode the int64 object from binary -func DecodeInt64(b []byte) int64 { - v := int64(binary.BigEndian.Uint64(b)) - if v < 0 { - v = int64(uint64(v) & 0x7FFFFFFFFFFFFFFF) - } else if v >= 0 { - v = int64(uint64(v) | 0x8000000000000000) - } - return v -} - -// EncodeFloat64 encode the float64 object to binary -func EncodeFloat64(v float64) ([]byte, error) { - var buf bytes.Buffer - // keep the same pattern of 0.0 and -0.0 - if v == 0.0 { - v = 0.0 - } - - vi := int64(math.Float64bits(v)) - vi = ((vi ^ (vi >> 63)) | int64(uint64(^vi)&0x8000000000000000)) - - // Ignore the error returned here, because buf is a memory io.Writer, can should not fail here - if err := binary.Write(&buf, binary.BigEndian, vi); err != nil { - return nil, err - } - return buf.Bytes(), nil -} - -// DecodeFloat64 decode the float64 object from binary -func DecodeFloat64(d []byte) float64 { - vi := int64(binary.BigEndian.Uint64(d)) - if vi == 0 { - return 0.0 - } - if vi > 0 { - vi = ^vi - } else { - vi = (vi & 0x7FFFFFFFFFFFFFFF) - } - return math.Float64frombits(uint64(vi)) -} diff --git a/db/codec_test.go b/db/codec_test.go deleted file mode 100644 index 4d43fd2..0000000 --- a/db/codec_test.go +++ /dev/null @@ -1,73 +0,0 @@ -package db - -import ( - "bytes" - "math" - "reflect" - "testing" -) - -func TestCodecObject(t *testing.T) { - now := Now() - tests := []*Object{ - &Object{ - ID: []byte("1234567890123456"), - CreatedAt: now, - UpdatedAt: now, - ExpireAt: 0, - Type: ObjectString, - Encoding: ObjectEncodingRaw, - }, - } - for _, obj := range tests { - got, err := DecodeObject(EncodeObject(obj)) - if err != nil { - t.Fatalf("decode object get error: %s", err) - } - if !reflect.DeepEqual(got, obj) { - t.Fatalf("decode failed want=%v, got=%v", obj, got) - } - } -} - -func TestCodecEncodeInt64(t *testing.T) { - values := []int64{math.MinInt64, -1, 0, 1, math.MaxInt64} - for i := 0; i < len(values)-1; i++ { - e1, _ := EncodeInt64(values[i]) - e2, _ := EncodeInt64(values[i+1]) - if bytes.Compare(e1, e2) >= 0 { - t.Fatal("EncodeInt64 is not memcomparable") - } - } -} - -func TestCodecEncodeFloat64(t *testing.T) { - values := []float64{-1.0, 0.0, 1.0, math.MaxFloat64} - for i := 0; i < len(values)-1; i++ { - e1, _ := EncodeFloat64(values[i]) - e2, _ := EncodeFloat64(values[i+1]) - if bytes.Compare(e1, e2) >= 0 { - t.Fatal("EncodeFloat64 is not memcomparable") - } - } -} - -func TestCodecInt64(t *testing.T) { - for _, i := range []int64{0, 1, -1, math.MaxInt64, math.MinInt64} { - encode, _ := EncodeInt64(i) - v := DecodeInt64(encode) - if v != i { - t.Fatalf("decode failed want=%v, got=%v", i, v) - } - } -} - -func TestCodecFloat64(t *testing.T) { - for _, f := range []float64{math.MaxFloat64, math.SmallestNonzeroFloat64, 0, 1, -1, 0.1, -0.1} { - encode, _ := EncodeFloat64(f) - v := DecodeFloat64(encode) - if v != f { - t.Fatalf("decode failed want=%v, got=%v", f, v) - } - } -} diff --git a/db/db.go b/db/db.go deleted file mode 100644 index e6cae59..0000000 --- a/db/db.go +++ /dev/null @@ -1,407 +0,0 @@ -package db - -import ( - "bytes" - "context" - "encoding/binary" - "errors" - "fmt" - "strconv" - "time" - - "go.uber.org/zap" - - "github.com/distributedio/titan/conf" - "github.com/distributedio/titan/db/store" - "github.com/distributedio/titan/metrics" -) - -var ( - // ErrTypeMismatch indicates object type of key is not as expect - ErrTypeMismatch = errors.New("type mismatch") - - // ErrKeyNotFound key not exist - ErrKeyNotFound = errors.New("key not found") - - // ErrInteger valeu is not interge - ErrInteger = errors.New("value is not an integer or out of range") - - // ErrPrecision list index reach precision limitatin - ErrPrecision = errors.New("list reaches precision limitation, rebalance now") - - // ErrOutOfRange index/offset out of range - ErrOutOfRange = errors.New("error index/offset out of range") - - // ErrInvalidLength data length is invalid for unmarshaler" - ErrInvalidLength = errors.New("error data length is invalid for unmarshaler") - - // ErrEncodingMismatch object encoding type - ErrEncodingMismatch = errors.New("error object encoding type") - - // ErrStorageRetry storage err and try again later - ErrStorageRetry = errors.New("Storage err and try again later") - - //ErrSetNilValue means the value corresponding to key is a non-zero value - ErrSetNilValue = errors.New("The value corresponding to key is a non-zero value") - - // IsErrNotFound returns true if the key is not found, otherwise return false - IsErrNotFound = store.IsErrNotFound - - // IsRetryableError returns true if the error is temporary and can be retried - IsRetryableError = store.IsRetryableError - - // IsConflictError return true if the error is conflict - IsConflictError = store.IsConflictError - - // sysNamespace default namespace - sysNamespace = "$sys" - - // sysDatabaseID default db id - sysDatabaseID = 0 - - NilValue = []byte{0} -) - -// Iterator store.Iterator -type Iterator store.Iterator - -// DBID is the redis database ID -type DBID byte - -// String returns the string format of DBID -func (id DBID) String() string { - return fmt.Sprintf("%03d", id) -} - -// Bytes DBID returns a byte slice -func (id DBID) Bytes() []byte { - return []byte(id.String()) -} - -func toDBID(v []byte) DBID { - id, _ := strconv.Atoi(string(v)) - return DBID(id) -} - -// BatchGetValues issues batch requests to get values -func BatchGetValues(txn *Transaction, keys [][]byte) ([][]byte, error) { - kvs, err := store.BatchGetValues(txn.t, keys) - if err != nil { - return nil, err - } - values := make([][]byte, len(keys)) - for i := range keys { - values[i] = kvs[string(keys[i])] - } - return values, nil -} - -// DB is a redis compatible data structure storage -type DB struct { - Namespace string - ID DBID - kv *RedisStore -} - -// RedisStore wraps store.Storage -type RedisStore struct { - store.Storage - conf *conf.TiKV -} - -// Open a storage instance -func Open(conf *conf.TiKV) (*RedisStore, error) { - s, err := store.Open(conf.PdAddrs) - if err != nil { - return nil, err - } - rds := &RedisStore{Storage: s, conf: conf} - sysdb := rds.DB(sysNamespace, sysDatabaseID) - go StartGC(sysdb, &conf.GC) - go StartExpire(sysdb, &conf.Expire) - go StartZT(sysdb, &conf.ZT) - go StartTiKVGC(sysdb, &conf.TiKVGC) - return rds, nil -} - -// DB returns a DB object with sepcific ID -func (rds *RedisStore) DB(namesapce string, id int) *DB { - return &DB{Namespace: namesapce, ID: DBID(id), kv: rds} -} - -// Close the storage instance -func (rds *RedisStore) Close() error { - return rds.Storage.Close() -} - -// Transaction supplies transaction for data structures -type Transaction struct { - t store.Transaction - db *DB -} - -// Begin a transaction -func (db *DB) Begin() (*Transaction, error) { - txn, err := db.kv.Begin() - if err != nil { - return nil, err - } - return &Transaction{t: txn, db: db}, nil -} - -// Prefix returns the prefix of a DB object -func (db *DB) Prefix() []byte { - return dbPrefix(db.Namespace, db.ID.Bytes()) -} - -// Commit a transaction -func (txn *Transaction) Commit(ctx context.Context) error { - return txn.t.Commit(ctx) -} - -// Rollback a transaction -func (txn *Transaction) Rollback() error { - return txn.t.Rollback() -} - -// listOption for get a list -type listOption struct { - useZip bool -} - -// ListOption customize how to get a list -type ListOption func(o *listOption) - -// UseZip will create a ziplist if set when the key is missing -func UseZip() ListOption { - return func(o *listOption) { - o.useZip = true - } -} - -// List return a lists object, a new list is created if the key dose not exist. -func (txn *Transaction) List(key []byte, opts ...ListOption) (List, error) { - return GetList(txn, key, opts...) -} - -// String returns a string object, but the object is unsafe, maybe the object is expire,or not exist -func (txn *Transaction) String(key []byte) (*String, error) { - return GetString(txn, key) -} - -// Strings returns a slice of String -func (txn *Transaction) Strings(keys [][]byte) ([]*String, error) { - sobjs := make([]*String, len(keys)) - tkeys := make([][]byte, len(keys)) - for i, key := range keys { - tkeys[i] = MetaKey(txn.db, key) - } - mdata, err := store.BatchGetValues(txn.t, tkeys) - if err != nil { - return nil, err - } - for i, key := range keys { - obj := NewString(txn, key) - if data, ok := mdata[string(tkeys[i])]; ok { - if err := obj.decode(data); err != nil { - if logEnv := zap.L().Check(zap.DebugLevel, "Strings decoded value error"); logEnv != nil { - logEnv.Write(zap.ByteString("key", key), zap.Error(err)) - } - } - } - sobjs[i] = obj - } - return sobjs, nil -} - -// Kv returns a kv object -func (txn *Transaction) Kv() *Kv { - return GetKv(txn) -} - -// Hash returns a hash object -func (txn *Transaction) Hash(key []byte) (*Hash, error) { - return GetHash(txn, key) -} - -// Set returns a set object -func (txn *Transaction) Set(key []byte) (*Set, error) { - return GetSet(txn, key) -} - -// ZSet returns a zset object -func (txn *Transaction) ZSet(key []byte) (*ZSet, error) { - return GetZSet(txn, key) -} - -// LockKeys tries to lock the entries with the keys in KV store. -func (txn *Transaction) LockKeys(keys ...[]byte) error { - return store.LockKeys(txn.t, keys) -} - -// MetaKey build to metakey from a redis key -func MetaKey(db *DB, key []byte) []byte { - var mkey []byte - mkey = append(mkey, []byte(db.Namespace)...) - mkey = append(mkey, ':') - mkey = append(mkey, db.ID.Bytes()...) - mkey = append(mkey, ':', 'M', ':') - mkey = append(mkey, key...) - return mkey -} - -// DataKey builds a datakey from a redis key -func DataKey(db *DB, key []byte) []byte { - var dkey []byte - dkey = append(dkey, []byte(db.Namespace)...) - dkey = append(dkey, ':') - dkey = append(dkey, db.ID.Bytes()...) - dkey = append(dkey, ':', 'D', ':') - dkey = append(dkey, key...) - return dkey -} - -func dbPrefix(ns string, id []byte) []byte { - prefix := []byte(ns) - prefix = append(prefix, ':') - if id != nil { - prefix = append(prefix, id...) - prefix = append(prefix, ':') - } - return prefix -} - -func flushLease(txn store.Transaction, key, id []byte, interval time.Duration) error { - databytes := make([]byte, 24) - copy(databytes, id) - ts := uint64((time.Now().Add(interval).Unix())) - binary.BigEndian.PutUint64(databytes[16:], ts) - - if err := txn.Set(key, databytes); err != nil { - return err - } - return nil -} - -func checkLeader(txn store.Transaction, key, id []byte, interval time.Duration) (bool, error) { - val, err := txn.Get(key) - if err != nil { - if !IsErrNotFound(err) { - zap.L().Error("query leader message faild", - zap.ByteString("key", key), - zap.ByteString("id", id), - zap.Error(err)) - return false, err - } - - if env := zap.L().Check(zap.DebugLevel, "no leader now, create new lease"); env != nil { - env.Write(zap.ByteString("key", key), - zap.ByteString("id", id), - zap.Duration("interval", interval)) - } - - if err := flushLease(txn, key, id, interval); err != nil { - zap.L().Error("create lease failed", - zap.ByteString("key", key), - zap.ByteString("id", id), - zap.Duration("interval", interval), - zap.Error(err)) - return false, err - } - - return true, nil - } - - curID := val[0:16] - ts := int64(binary.BigEndian.Uint64(val[16:])) - - if time.Now().Unix() > ts { - if err := flushLease(txn, key, id, interval); err != nil { - zap.L().Error("create lease failed", - zap.ByteString("key", key), - zap.ByteString("id", id), - zap.Int64("last_ts", ts), - zap.Error(err)) - return false, err - } - return true, nil - } - - if bytes.Equal(curID, id) { - if err := flushLease(txn, key, id, interval); err != nil { - zap.L().Error("flush lease failed", - zap.ByteString("key", key), - zap.ByteString("curid", curID), - zap.ByteString("id", id), - zap.Error(err)) - return false, err - } - return true, nil - } - return false, nil -} - -func isLeader(db *DB, leader []byte, id []byte, interval time.Duration) (bool, error) { - count := 0 - label := "default" - switch { - case bytes.Equal(leader, sysZTLeader): - label = "ZT" - case bytes.Equal(leader, sysGCLeader): - label = "GC" - case bytes.Equal(leader, sysExpireLeader): - label = "EX" - case bytes.Equal(leader, sysTiKVGCLeader): - label = "TGC" - - } - - for { - txn, err := db.Begin() - if err != nil { - zap.L().Error("transection begin failed", - zap.ByteString("leader", leader), - zap.Error(err)) - continue - } - - isLeader, err := checkLeader(txn.t, leader, id, interval) - mtFunc := func() { - if isLeader { - metrics.GetMetrics().IsLeaderGaugeVec.WithLabelValues(label).Set(1) - return - } - metrics.GetMetrics().IsLeaderGaugeVec.WithLabelValues(label).Set(0) - } - - if err != nil { - if err := txn.Rollback(); err != nil { - return isLeader, err - } - if IsRetryableError(err) { - count++ - if count < 3 { - continue - } - } - mtFunc() - return isLeader, err - } - - if err := txn.Commit(context.Background()); err != nil { - if err := txn.Rollback(); err != nil { - return isLeader, err - } - if IsRetryableError(err) { - count++ - if count < 3 { - continue - } - } - mtFunc() - return isLeader, err - } - mtFunc() - return isLeader, err - } -} diff --git a/db/db_test.go b/db/db_test.go deleted file mode 100644 index a4a5c51..0000000 --- a/db/db_test.go +++ /dev/null @@ -1,36 +0,0 @@ -package db - -import ( - "context" - "os" - "testing" - - "github.com/distributedio/titan/conf" - "github.com/pingcap/tidb/store/mockstore" - "github.com/stretchr/testify/assert" -) - -var mockDB *DB - -func TestMain(m *testing.M) { - store, err := mockstore.NewMockTikvStore() - if err != nil { - panic(err) - } - mockConf := conf.MockConf() - mockDB = &DB{ - Namespace: "mockdb-ns", - ID: 1, - kv: &RedisStore{Storage: store, conf: &mockConf.TiKV}, - } - - os.Exit(m.Run()) -} - -func MockTest(t *testing.T, callFunc func(txn *Transaction)) { - txn, err := mockDB.Begin() - assert.NoError(t, err) - callFunc(txn) - err = txn.Commit(context.TODO()) - assert.NoError(t, err) -} diff --git a/db/expire.go b/db/expire.go deleted file mode 100644 index f04d328..0000000 --- a/db/expire.go +++ /dev/null @@ -1,294 +0,0 @@ -package db - -import ( - "bytes" - "context" - "time" - - "github.com/distributedio/titan/conf" - "github.com/distributedio/titan/db/store" - "github.com/distributedio/titan/metrics" - "github.com/pingcap/tidb/kv" - "go.uber.org/zap" -) - -var ( - expireKeyPrefix = []byte("$sys:0:at:") - sysExpireLeader = []byte("$sys:0:EXL:EXLeader") - - // $sys:0:at:{ts}:{metaKey} - expireTimestampOffset = len(expireKeyPrefix) - expireMetakeyOffset = expireTimestampOffset + 8 /*sizeof(int64)*/ + len(":") -) - -// IsExpired judge object expire through now -func IsExpired(obj *Object, now int64) bool { - if obj.ExpireAt == 0 || obj.ExpireAt > now { - return false - } - return true -} - -func expireKey(key []byte, ts int64) ([]byte, error) { - var buf []byte - buf = append(buf, expireKeyPrefix...) - encode, err := EncodeInt64(ts) - if err != nil { - return nil, err - } - buf = append(buf, encode...) - buf = append(buf, ':') - buf = append(buf, key...) - return buf, nil -} - -func expireAt(txn store.Transaction, mkey []byte, objID []byte, objType ObjectType, oldAt int64, newAt int64) error { - oldKey, err := expireKey(mkey, oldAt) - if err != nil { - return err - } - - newKey, err := expireKey(mkey, newAt) - if err != nil { - return err - } - - if oldAt > 0 { - if err := txn.Delete(oldKey); err != nil { - return err - } - } - - if newAt > 0 { - if err := txn.Set(newKey, objID); err != nil { - return err - } - } - action := "" - if oldAt > 0 && newAt > 0 { - action = "updated" - } else if oldAt > 0 { - action = "removed" - } else if newAt > 0 { - action = "added" - } - if action != "" { - metrics.GetMetrics().ExpireKeysTotal.WithLabelValues(action).Inc() - } - return nil -} - -func unExpireAt(txn store.Transaction, mkey []byte, expireAt int64) error { - if expireAt == 0 { - return nil - } - - oldKey, err := expireKey(mkey, expireAt) - if err != nil { - return err - } - - if err := txn.Delete(oldKey); err != nil { - return err - } - metrics.GetMetrics().ExpireKeysTotal.WithLabelValues("removed").Inc() - return nil -} - -// StartExpire get leader from db -func StartExpire(db *DB, conf *conf.Expire) { - ticker := time.NewTicker(conf.Interval) - defer ticker.Stop() - id := UUID() - for range ticker.C { - if conf.Disable { - continue - } - isLeader, err := isLeader(db, sysExpireLeader, id, conf.LeaderLifeTime) - if err != nil { - zap.L().Error("[Expire] check expire leader failed", zap.Error(err)) - continue - } - if !isLeader { - if logEnv := zap.L().Check(zap.DebugLevel, "[Expire] not expire leader"); logEnv != nil { - logEnv.Write(zap.ByteString("leader", sysExpireLeader), - zap.ByteString("uuid", id), - zap.Duration("leader-life-time", conf.LeaderLifeTime)) - } - continue - } - runExpire(db, conf.BatchLimit) - } -} - -// split a meta key with format: {namespace}:{id}:M:{key} -func splitMetaKey(key []byte) ([]byte, DBID, []byte) { - idx := bytes.Index(key, []byte{':'}) - namespace := key[:idx] - id := toDBID(key[idx+1 : idx+4]) - rawkey := key[idx+6:] - return namespace, id, rawkey -} - -func toTiKVDataKey(namespace []byte, id DBID, key []byte) []byte { - var b []byte - b = append(b, namespace...) - b = append(b, ':') - b = append(b, id.Bytes()...) - b = append(b, ':', 'D', ':') - b = append(b, key...) - return b -} - -func runExpire(db *DB, batchLimit int) { - txn, err := db.Begin() - if err != nil { - zap.L().Error("[Expire] txn begin failed", zap.Error(err)) - return - } - store.SetOption(txn.t, store.Priority, store.PriorityLow) - - endPrefix := kv.Key(expireKeyPrefix).PrefixNext() - iter, err := txn.t.Iter(expireKeyPrefix, endPrefix) - if err != nil { - zap.L().Error("[Expire] seek failed", zap.ByteString("prefix", expireKeyPrefix), zap.Error(err)) - if err := txn.Rollback(); err != nil { - zap.L().Error("[Expire] seek rollback failed", zap.ByteString("prefix", expireKeyPrefix), zap.Error(err)) - } - return - } - limit := batchLimit - now := time.Now().UnixNano() - - for iter.Valid() && iter.Key().HasPrefix(expireKeyPrefix) && limit > 0 { - rawKey := iter.Key() - ts := DecodeInt64(rawKey[expireTimestampOffset : expireTimestampOffset+8]) - if ts > now { - if logEnv := zap.L().Check(zap.DebugLevel, "[Expire] not need to expire key"); logEnv != nil { - logEnv.Write(zap.String("raw-key", string(rawKey)), zap.Int64("last-timestamp", ts)) - } - break - } - mkey := rawKey[expireMetakeyOffset:] - if err := doExpire(txn, mkey, iter.Value()); err != nil { - if err := txn.Rollback(); err != nil { - zap.L().Error("[Expire] seek rollback failed", zap.ByteString("prefix", mkey), zap.Error(err)) - } - return - } - - // Remove from expire list - if err := txn.t.Delete(rawKey); err != nil { - zap.L().Error("[Expire] delete failed", - zap.ByteString("mkey", mkey), - zap.Error(err)) - if err := txn.Rollback(); err != nil { - zap.L().Error("[Expire] seek rollback failed", zap.ByteString("prefix", rawKey), zap.Error(err)) - } - return - } - - if logEnv := zap.L().Check(zap.DebugLevel, "[Expire] delete expire list item"); logEnv != nil { - logEnv.Write(zap.ByteString("mkey", mkey)) - } - - if err := iter.Next(); err != nil { - zap.L().Error("[Expire] next failed", - zap.ByteString("mkey", mkey), - zap.Error(err)) - if err := txn.Rollback(); err != nil { - zap.L().Error("[Expire] seek rollback failed", zap.ByteString("prefix", mkey), zap.Error(err)) - } - return - } - limit-- - } - - if err := txn.Commit(context.Background()); err != nil { - if err := txn.Rollback(); err != nil { - zap.L().Error("[Expire] seek rollback failed", zap.Error(err)) - } - zap.L().Error("[Expire] commit failed", zap.Error(err)) - } - - if logEnv := zap.L().Check(zap.DebugLevel, "[Expire] expired end"); logEnv != nil { - logEnv.Write(zap.Int("expired_num", batchLimit-limit)) - } - - metrics.GetMetrics().ExpireKeysTotal.WithLabelValues("expired").Add(float64(batchLimit - limit)) -} - -func gcDataKey(txn *Transaction, namespace []byte, dbid DBID, key, id []byte) error { - dkey := toTiKVDataKey(namespace, dbid, id) - if err := gc(txn.t, dkey); err != nil { - zap.L().Error("[Expire] gc failed", - zap.ByteString("key", key), - zap.ByteString("namepace", namespace), - zap.Int64("db_id", int64(dbid)), - zap.ByteString("obj_id", id), - zap.Error(err)) - return err - } - if logEnv := zap.L().Check(zap.DebugLevel, "[Expire] gc data key"); logEnv != nil { - logEnv.Write(zap.ByteString("obj_id", id)) - } - return nil -} -func doExpire(txn *Transaction, mkey, id []byte) error { - namespace, dbid, key := splitMetaKey(mkey) - obj, err := getObject(txn, mkey) - // Check for dirty data due to copying or flushdb/flushall - if err == ErrKeyNotFound { - return gcDataKey(txn, namespace, dbid, key, id) - } - if err != nil { - return err - } - idLen := len(obj.ID) - if len(id) > idLen { - id = id[:idLen] - } - if !bytes.Equal(obj.ID, id) { - return gcDataKey(txn, namespace, dbid, key, id) - } - - // Delete object meta - if err := txn.t.Delete(mkey); err != nil { - zap.L().Error("[Expire] delete failed", - zap.ByteString("key", key), - zap.Error(err)) - return err - } - - if logEnv := zap.L().Check(zap.DebugLevel, "[Expire] delete metakey"); logEnv != nil { - logEnv.Write(zap.ByteString("mkey", mkey)) - } - if obj.Type == ObjectString { - return nil - } - return gcDataKey(txn, namespace, dbid, key, id) -} - -// ScanExpiration scans the expiration list -func ScanExpiration(txn *Transaction, from, to, count int64) ([]int64, [][]byte, error) { - // from/to are valid integers so no errors will happen here - start, _ := expireKey(nil, from) - end, _ := expireKey(nil, to) - iter, err := txn.t.Iter(start, end) - if err != nil { - return nil, nil, err - } - defer iter.Close() - var at []int64 - var keys [][]byte - for iter.Valid() && iter.Key().HasPrefix(expireKeyPrefix) && count > 0 { - rawKey := iter.Key() - ts := rawKey[expireTimestampOffset : expireMetakeyOffset-1] - at = append(at, DecodeInt64(ts)) - metaKey := rawKey[expireMetakeyOffset:] - keys = append(keys, metaKey) - count-- - iter.Next() - } - return at, keys, nil -} diff --git a/db/expire_test.go b/db/expire_test.go deleted file mode 100644 index 796eb8c..0000000 --- a/db/expire_test.go +++ /dev/null @@ -1,356 +0,0 @@ -package db - -import ( - "bytes" - "context" - "fmt" - "math" - "testing" - "time" - - "github.com/distributedio/titan/db/store" - "github.com/stretchr/testify/assert" -) - -func getTxn(t *testing.T) *Transaction { - txn, err := mockDB.Begin() - assert.NotNil(t, txn) - assert.NoError(t, err) - return txn -} - -func Test_runExpire(t *testing.T) { - hashKey := []byte("TestExpiredHash") - strKey := []byte("TestExpiredString") - expireAt := (time.Now().Unix() - 30) * int64(time.Second) - hashCall := func(t *testing.T, key []byte) []byte { - hash, txn, err := getHash(t, []byte(key)) - oldID := hash.meta.ID - assert.NoError(t, err) - assert.NotNil(t, txn) - assert.NotNil(t, hash) - hash.HSet([]byte("field1"), []byte("val")) - kv := GetKv(txn) - err = kv.ExpireAt([]byte(key), expireAt) - assert.NoError(t, err) - txn.Commit(context.TODO()) - - hash, txn, err = getHash(t, []byte(key)) - newID := hash.meta.ID - if bytes.Equal(oldID, newID) { - assert.Fail(t, "old hash is not expired") - return nil - } - assert.NoError(t, err) - assert.NotNil(t, txn) - assert.NotNil(t, hash) - hash.HSet([]byte("field1"), []byte("val")) - txn.Commit(context.TODO()) - return oldID - } - - stringCall := func(t *testing.T, key []byte) []byte { - hash, txn, err := getHash(t, []byte(key)) - oldID := hash.meta.ID - assert.NoError(t, err) - assert.NotNil(t, txn) - assert.NotNil(t, hash) - hash.HSet([]byte("field1"), []byte("val")) - kv := GetKv(txn) - err = kv.ExpireAt([]byte(key), expireAt) - assert.NoError(t, err) - txn.Commit(context.TODO()) - - txn = getTxn(t) - s, err := GetString(txn, key) - assert.NoError(t, err) - newID := s.Meta.ID - if bytes.Equal(oldID, newID) { - assert.Fail(t, "old hash is not expired") - return nil - } - err = s.Set([]byte("val")) - assert.NoError(t, err) - txn.Commit(context.TODO()) - return oldID - } - - type args struct { - key []byte - call func(*testing.T, []byte) []byte - } - type want struct { - gckey bool - } - - tests := []struct { - name string - args args - want want - }{ - { - name: "TestExpiredHash", - args: args{ - key: hashKey, - call: hashCall, - }, - want: want{ - gckey: true, - }, - }, - { - name: "TestExpiredString", - args: args{ - key: strKey, - call: stringCall, - }, - want: want{ - gckey: true, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - id := tt.args.call(t, tt.args.key) - txn := getTxn(t) - runExpire(txn.db, 1) - txn.Commit(context.TODO()) - - txn = getTxn(t) - gcKey := toTiKVGCKey(toTiKVDataKey([]byte(txn.db.Namespace), txn.db.ID, id)) - - _, err := txn.t.Get(gcKey) - txn.Commit(context.TODO()) - if tt.want.gckey { - assert.NoError(t, err) - } else { - assert.Equal(t, true, store.IsErrNotFound(err)) - } - }) - } - -} - -func Test_doExpire(t *testing.T) { - initHash := func(t *testing.T, key []byte) []byte { - hash, txn, err := getHash(t, key) - assert.NoError(t, err) - assert.NotNil(t, txn) - assert.NotNil(t, hash) - hash.HSet([]byte("field1"), []byte("val")) - txn.Commit(context.TODO()) - return hash.meta.ID - } - - expireAt := (time.Now().Unix() - 30) * int64(time.Second) - hashCall := func(t *testing.T, key []byte) ([]byte, []byte) { - hash, txn, err := getHash(t, []byte(key)) - oldID := hash.meta.ID - assert.NoError(t, err) - assert.NotNil(t, txn) - assert.NotNil(t, hash) - hash.HSet([]byte("field1"), []byte("val")) - kv := GetKv(txn) - err = kv.ExpireAt([]byte(key), expireAt) - assert.NoError(t, err) - txn.Commit(context.TODO()) - - hash, txn, err = getHash(t, []byte(key)) - newID := hash.meta.ID - if bytes.Equal(oldID, newID) { - assert.Fail(t, "old hash is not expired") - return nil, nil - } - assert.NoError(t, err) - assert.NotNil(t, txn) - assert.NotNil(t, hash) - hash.HSet([]byte("field1"), []byte("val")) - txn.Commit(context.TODO()) - return oldID, newID - } - - hashId := initHash(t, []byte("TestExpiredHash")) - rHashId, nmateHashId := hashCall(t, []byte("TestExpiredRewriteHash")) - - dirtyDataHashID := initHash(t, []byte("TestExpiredHash_dirty_data")) - rDHashId, _ := hashCall(t, []byte("TestExpiredRewriteHash_dirty_data")) - txn := getTxn(t) - type args struct { - mkey []byte - id []byte - tp byte - } - type want struct { - gckey bool - } - - tests := []struct { - name string - args args - want want - }{ - { - name: "TestExpiredHash", - args: args{ - mkey: MetaKey(txn.db, []byte("TestExpiredHash")), - id: hashId, - }, - want: want{ - gckey: true, - }, - }, - { - name: "TestExpiredRewriteHash", - args: args{ - mkey: MetaKey(txn.db, []byte("TestExpiredRewriteHash")), - id: rHashId, - }, - want: want{ - gckey: true, - }, - }, - { - name: "TestExpiredNotExistsMeta", - args: args{ - mkey: MetaKey(txn.db, []byte("TestExpiredRewriteHash")), - id: nmateHashId, - }, - want: want{ - gckey: true, - }, - }, - { - name: "TestExpiredHash_dirty_data", - args: args{ - mkey: MetaKey(txn.db, []byte("TestExpiredHash_dirty_data")), - id: dirtyDataHashID, - tp: byte(ObjectHash), - }, - want: want{ - gckey: true, - }, - }, - { - name: "TestExpiredRewriteHash_dirty_data", - args: args{ - mkey: MetaKey(txn.db, []byte("TestExpiredRewriteHash_dirty_data")), - id: rDHashId, - tp: byte(ObjectHash), - }, - want: want{ - gckey: true, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - txn := getTxn(t) - id := tt.args.id - if tt.args.tp == byte(ObjectHash) { - id = append(id, tt.args.tp) - } - err := doExpire(txn, tt.args.mkey, id) - txn.Commit(context.TODO()) - assert.NoError(t, err) - - txn = getTxn(t) - gcKey := toTiKVGCKey(toTiKVDataKey([]byte(txn.db.Namespace), txn.db.ID, tt.args.id)) - - _, err = txn.t.Get(gcKey) - txn.Commit(context.TODO()) - if tt.want.gckey { - assert.NoError(t, err) - } else { - assert.Equal(t, true, store.IsErrNotFound(err)) - } - }) - } - -} - -func TestScanExpiration(t *testing.T) { - var at []int64 - var mkeys [][]byte - - // setUp fill the expiration list - now := Now() - setUp := func() { - txn, err := mockDB.Begin() - assert.NoError(t, err) - - // cleanup the keys left by other tests(TODO these dirty data should be deleted where it is generated) - iter, err := txn.t.Iter(expireKeyPrefix, nil) - assert.NoError(t, err) - defer iter.Close() - for iter.Valid() && iter.Key().HasPrefix(expireKeyPrefix) { - txn.t.Delete(iter.Key()) - iter.Next() - } - - for i := 0; i < 10; i++ { - ts := now - 10 + int64(i)*int64(time.Second) - mkey := MetaKey(txn.db, []byte(fmt.Sprintf("expire_key_%d", i))) - err := expireAt(txn.t, mkey, mkey, ObjectString, 0, ts) - assert.NoError(t, err) - - at = append(at, ts) - mkeys = append(mkeys, mkey) - } - assert.NoError(t, txn.Commit(context.Background())) - } - tearDown := func() { - txn, err := mockDB.Begin() - assert.NoError(t, err) - for i := range at { - assert.NoError(t, unExpireAt(txn.t, mkeys[i], at[i])) - } - assert.NoError(t, txn.Commit(context.Background())) - } - - setUp() - - type args struct { - from int64 - to int64 - count int64 - } - type want struct { - s int // start index of the result - e int // end index of the result(not included) - } - - tests := []struct { - name string - args args - want want - }{ - {"escan 0 max 10", args{0, math.MaxInt64, 10}, want{0, 10}}, - {"escan 0 max 1", args{0, math.MaxInt64, 1}, want{0, 1}}, - {"escan 0 0 1", args{0, 0, 1}, want{0, 0}}, - {"escan max max 1", args{math.MaxInt64, math.MaxInt64, 1}, want{0, 0}}, - {"escan 0 max 20", args{0, math.MaxInt64, 10}, want{0, 10}}, - {"escan at[2] at[8] 10", args{at[2], at[8], 10}, want{2, 8}}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Log(tt.name) - - txn, err := mockDB.Begin() - assert.NoError(t, err) - - tses, keys, err := ScanExpiration(txn, tt.args.from, tt.args.to, tt.args.count) - assert.NoError(t, err) - assert.NoError(t, txn.Commit(context.Background())) - - assert.Equal(t, tt.want.e-tt.want.s, len(tses)) - assert.Equal(t, tt.want.e-tt.want.s, len(keys)) - for i := range tses { - assert.Equal(t, at[tt.want.s+i], tses[i]) - assert.Equal(t, mkeys[tt.want.s+i], keys[i]) - } - }) - } - - tearDown() -} diff --git a/db/gc.go b/db/gc.go deleted file mode 100644 index de54f47..0000000 --- a/db/gc.go +++ /dev/null @@ -1,189 +0,0 @@ -package db - -import ( - "context" - "time" - - "github.com/distributedio/titan/conf" - "github.com/distributedio/titan/db/store" - "github.com/distributedio/titan/metrics" - "github.com/pingcap/tidb/kv" - "go.uber.org/zap" -) - -var ( - sysGCLeader = []byte("$sys:0:GCL:GCLeader") -) - -func toTiKVGCKey(key []byte) []byte { - b := []byte{} - b = append(b, sysNamespace...) - b = append(b, ':', byte(sysDatabaseID)) - b = append(b, ':', 'G', 'C', ':') - b = append(b, key...) - return b -} - -// {sys.ns}:{sys.id}:{GC}:{prefix} -// prefix: {user.ns}:{user.id}:{M/D}:{user.objectID} -func gc(txn store.Transaction, prefixes ...[]byte) error { - for _, prefix := range prefixes { - if logEnv := zap.L().Check(zap.DebugLevel, "[GC] add to gc"); logEnv != nil { - logEnv.Write(zap.ByteString("prefix", prefix)) - } - metrics.GetMetrics().GCKeysCounterVec.WithLabelValues("gc_add").Inc() - if err := txn.Set(toTiKVGCKey(prefix), []byte{0}); err != nil { - return err - } - } - return nil -} - -func gcDeleteRange(txn store.Transaction, prefix []byte, limit int) (int, error) { - var ( - resultErr error - count int - ) - endPrefix := kv.Key(prefix).PrefixNext() - itr, err := txn.Iter(prefix, endPrefix) - if err != nil { - return count, err - } - defer itr.Close() - callback := func(k kv.Key) bool { - if resultErr = txn.Delete(itr.Key()); resultErr != nil { - return true - } - count++ - if limit > 0 && count >= limit { - return true - } - return false - } - if err := kv.NextUntil(itr, callback); err != nil { - return 0, err - } - if resultErr != nil { - return 0, resultErr - } - return count, nil -} - -func doGC(db *DB, limit int) error { - gcPrefix := toTiKVGCKey(nil) - endGCPrefix := kv.Key(gcPrefix).PrefixNext() - dbTxn, err := db.Begin() - if err != nil { - zap.L().Error("[GC] transection begin failed", - zap.ByteString("gcprefix", gcPrefix), - zap.Int("limit", limit), - zap.Error(err)) - return err - } - txn := dbTxn.t - store.SetOption(txn, store.KeyOnly, true) - store.SetOption(txn, store.Priority, store.PriorityLow) - - itr, err := txn.Iter(gcPrefix, endGCPrefix) - if err != nil { - return err - } - defer itr.Close() - if !itr.Valid() || !itr.Key().HasPrefix(gcPrefix) { - if logEnv := zap.L().Check(zap.DebugLevel, "[GC] not need to gc item"); logEnv != nil { - logEnv.Write(zap.ByteString("gcprefix", gcPrefix), zap.Int("limit", limit)) - } - return nil - } - gcKeyCount := 0 - dataKeyCount := 0 - var resultErr error - callback := func(k kv.Key) bool { - dataPrefix := k[len(gcPrefix):] - count := 0 - if logEnv := zap.L().Check(zap.DebugLevel, "[GC] start to delete prefix"); logEnv != nil { - logEnv.Write(zap.ByteString("data-prefix", dataPrefix), zap.Int("limit", limit)) - } - if count, resultErr = gcDeleteRange(txn, dataPrefix, limit); resultErr != nil { - return true - } - - //check and delete gc key - if limit > 0 && count < limit || limit <= 0 && count > 0 { - if logEnv := zap.L().Check(zap.DebugLevel, "[GC] delete prefix succeed"); logEnv != nil { - logEnv.Write(zap.ByteString("data-prefix", dataPrefix), zap.Int("limit", limit)) - } - - if resultErr = txn.Delete(k); resultErr != nil { - return true - } - gcKeyCount++ - } - - dataKeyCount += count - return limit-(gcKeyCount+dataKeyCount) <= 0 - } - if err := kv.NextUntil(itr, callback); err != nil { - zap.L().Error("[GC] iter prefix err", zap.ByteString("gc-prefix", gcPrefix), zap.Error(err)) - return err - } - if resultErr != nil { - if err := txn.Rollback(); err != nil { - zap.L().Error("[GC] rollback err", zap.Error(err)) - } - return resultErr - } - if err := txn.Commit(context.Background()); err != nil { - if err := txn.Rollback(); err != nil { - zap.L().Error("[GC] rollback err", zap.Error(err)) - } - return err - } - if logEnv := zap.L().Check(zap.DebugLevel, "[GC] txn commit success"); logEnv != nil { - logEnv.Write(zap.Int("limit", limit), - zap.Int("gcKeyCount", gcKeyCount), - zap.Int("dataKeyCount", dataKeyCount)) - } - metrics.GetMetrics().GCKeysCounterVec.WithLabelValues("data_delete").Add(float64(dataKeyCount)) - metrics.GetMetrics().GCKeysCounterVec.WithLabelValues("gc_delete").Add(float64(gcKeyCount)) - return nil -} - -// StartGC start gc -//1.获取leader许可 -//2.leader 执行清理任务 -func StartGC(db *DB, conf *conf.GC) { - ticker := time.NewTicker(conf.Interval) - defer ticker.Stop() - id := UUID() - for range ticker.C { - if conf.Disable { - continue - } - isLeader, err := isLeader(db, sysGCLeader, id, conf.LeaderLifeTime) - if err != nil { - zap.L().Error("[GC] check GC leader failed", - zap.ByteString("leader", sysGCLeader), - zap.ByteString("uuid", id), - zap.Duration("leader-life-time", conf.LeaderLifeTime), - zap.Error(err)) - continue - } - if !isLeader { - if logEnv := zap.L().Check(zap.DebugLevel, "[GC] current is not gc leader"); logEnv != nil { - logEnv.Write(zap.ByteString("leader", sysGCLeader), - zap.ByteString("uuid", id), - zap.Duration("leader-life-time", conf.LeaderLifeTime)) - } - continue - } - if err := doGC(db, conf.BatchLimit); err != nil { - zap.L().Error("[GC] do GC failed", - zap.ByteString("leader", sysGCLeader), - zap.ByteString("uuid", id), - zap.Duration("leader-life-time", conf.LeaderLifeTime), - zap.Error(err)) - continue - } - } -} diff --git a/db/gc_test.go b/db/gc_test.go deleted file mode 100644 index fd11bc1..0000000 --- a/db/gc_test.go +++ /dev/null @@ -1,134 +0,0 @@ -package db - -import ( - "context" - "testing" - - "github.com/distributedio/titan/db/store" - "github.com/pingcap/tidb/kv" - "github.com/stretchr/testify/assert" -) - -func TestGC(t *testing.T) { - hashCall := func(t *testing.T, key []byte, count int64) []byte { - hash, txn, err := getHash(t, []byte(key)) - assert.NoError(t, err) - assert.NotNil(t, txn) - assert.NotNil(t, hash) - for count > 0 { - encode, _ := EncodeInt64(count) - hash.HSet(encode, []byte("val")) - count-- - } - kv := GetKv(txn) - assert.NotNil(t, kv) - c, err := kv.Delete([][]byte{key}) - assert.NoError(t, err) - assert.Equal(t, c, int64(1)) - txn.Commit(context.TODO()) - return hash.meta.ID - } - - type args struct { - key []byte - fieldCount int64 - gcCount int - call func(*testing.T, []byte, int64) []byte - } - type want struct { - keyExists bool - } - - tests := []struct { - name string - args args - want want - }{ - { - name: "TestGCIterVal", - args: args{ - key: []byte("TestGCHash1"), - fieldCount: 10, - gcCount: 5, - call: hashCall, - }, - want: want{ - keyExists: true, - }, - }, - { - name: "TestGCAll", - args: args{ - key: []byte("TestGCHash2"), - fieldCount: 10, - gcCount: 17, - call: hashCall, - }, - want: want{ - keyExists: false, - }, - }, - { - name: "TestLimitZero", - args: args{ - key: []byte("TestGCZeroHash3"), - fieldCount: 10, - gcCount: 0, - call: hashCall, - }, - want: want{ - keyExists: false, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if err := clearGCData(t); err != nil { - assert.NoError(t, err) - } - id := tt.args.call(t, tt.args.key, tt.args.fieldCount) - txn := getTxn(t) - doGC(txn.db, tt.args.gcCount) - - txn = getTxn(t) - gcKey := toTiKVGCKey(toTiKVDataKey([]byte(txn.db.Namespace), txn.db.ID, id)) - - _, err := txn.t.Get(gcKey) - txn.Commit(context.TODO()) - if tt.want.keyExists { - assert.NoError(t, err) - } else { - assert.Equal(t, true, store.IsErrNotFound(err)) - } - }) - } - -} - -func clearGCData(t *testing.T) error { - gcPrefix := toTiKVGCKey(nil) - endGCPrefix := kv.Key(gcPrefix).PrefixNext() - - txn := getTxn(t) - itr, err := txn.t.Iter(gcPrefix, endGCPrefix) - if err != nil { - return err - } - defer itr.Close() - if !itr.Valid() || !itr.Key().HasPrefix(gcPrefix) { - return nil - } - call := func(k kv.Key) bool { - if resultErr := txn.t.Delete(k); resultErr != nil { - return true - } - return false - } - if err := kv.NextUntil(itr, call); err != nil { - return err - } - if err := txn.Commit(context.TODO()); err != nil { - return err - } - return nil -} diff --git a/db/hash.go b/db/hash.go deleted file mode 100644 index 157e3e1..0000000 --- a/db/hash.go +++ /dev/null @@ -1,444 +0,0 @@ -package db - -import ( - "errors" - "strconv" - - "github.com/distributedio/titan/db/store" - "github.com/pingcap/tidb/kv" -) - -// HashMeta is the meta data of the hashtable -type HashMeta struct { - Object -} - -//EncodeHashMeta encodes meta data into byte slice -func EncodeHashMeta(meta *HashMeta) []byte { - return EncodeObject(&meta.Object) -} - -//DecodeHashMeta decode meta data into meta field -func DecodeHashMeta(b []byte) (*HashMeta, error) { - obj, err := DecodeObject(b) - if err != nil { - return nil, err - } - return &HashMeta{Object: *obj}, nil -} - -// Hash implements the hashtable -type Hash struct { - meta *HashMeta - key []byte - exists bool - txn *Transaction -} - -// GetHash returns a hash object, create new one if nonexists -func GetHash(txn *Transaction, key []byte) (*Hash, error) { - hash := newHash(txn, key) - mkey := MetaKey(txn.db, key) - meta, err := txn.t.Get(mkey) - if err != nil { - if IsErrNotFound(err) { - return hash, nil - } - return nil, err - } - hmeta, err := DecodeHashMeta(meta) - if err != nil { - return nil, err - } - if IsExpired(&hmeta.Object, Now()) { - return hash, nil - } - if hmeta.Type != ObjectHash { - return nil, ErrTypeMismatch - } - hash.meta = hmeta - hash.exists = true - return hash, nil -} - -//newHash creates a hash object -func newHash(txn *Transaction, key []byte) *Hash { - now := Now() - return &Hash{ - txn: txn, - key: key, - meta: &HashMeta{ - Object: Object{ - ID: UUID(), - CreatedAt: now, - UpdatedAt: now, - ExpireAt: 0, - Type: ObjectHash, - Encoding: ObjectEncodingHT, - }, - }, - } -} - -//hashItemKey spits field into metakey -func hashItemKey(key []byte, field []byte) []byte { - var dkey []byte - dkey = append(dkey, key...) - dkey = append(dkey, ':') - return append(dkey, field...) -} - -// HDel removes the specified fields from the hash stored at key -func (hash *Hash) HDel(fields [][]byte) (int64, error) { - var ( - fieldsMap = make(map[string]struct{}, len(fields)) - num int64 - retainMeta bool - ) - - if !hash.Exists() { - return 0, nil - } - - dkey := DataKey(hash.txn.db, hash.meta.ID) - for _, f := range fields { - field := hashItemKey(dkey, f) - fieldsMap[string(field)] = struct{}{} - } - prefix := kv.Key(hashItemKey(dkey, nil)) - endPrefix := prefix.PrefixNext() - - var delErr error - callback := func(k kv.Key) bool { - if _, ok := fieldsMap[string(k)]; ok { - if delErr = hash.txn.t.Delete(k); delErr != nil { - return true - } - num++ - return false - } - retainMeta = true - return num == int64(len(fieldsMap)) - } - - store.SetOption(hash.txn.t, store.KeyOnly, true) - iter, err := hash.txn.t.Iter(prefix, endPrefix) - if err != nil { - return 0, err - } - if err := kv.NextUntil(iter, callback); err != nil { - return 0, err - } - if delErr != nil { - return 0, delErr - } - if !retainMeta { - if err := hash.delMeta(); err != nil { - return 0, err - } - } - return num, nil -} - -// HSet sets field in the hash stored at key to value -func (hash *Hash) HSet(field []byte, value []byte) (int, error) { - dkey := DataKey(hash.txn.db, hash.meta.ID) - ikey := hashItemKey(dkey, field) - newField := false - _, err := hash.txn.t.Get(ikey) - if err != nil { - if !IsErrNotFound(err) { - return 0, err - } - newField = true - } - - if err := hash.txn.t.Set(ikey, value); err != nil { - return 0, err - } - - if !hash.Exists() { - if err := hash.setMeta(); err != nil { - return 0, err - } - } - if !newField { - return 0, nil - } - - return 1, nil -} - -// HSetNX sets field in the hash stored at key to value, only if field does not yet exist -func (hash *Hash) HSetNX(field []byte, value []byte) (int, error) { - dkey := DataKey(hash.txn.db, hash.meta.ID) - ikey := hashItemKey(dkey, field) - - _, err := hash.txn.t.Get(ikey) - if !IsErrNotFound(err) { - return 0, err - } - if err := hash.txn.t.Set(ikey, value); err != nil { - return 0, err - } - - //update and save meta - if !hash.Exists() { - if err := hash.setMeta(); err != nil { - return 0, err - } - } - return 1, nil -} - -// HGet returns the value associated with field in the hash stored at key -func (hash *Hash) HGet(field []byte) ([]byte, error) { - if !hash.Exists() { - return nil, nil - } - dkey := DataKey(hash.txn.db, hash.meta.ID) - ikey := hashItemKey(dkey, field) - val, err := hash.txn.t.Get(ikey) - if err != nil { - if IsErrNotFound(err) { - return nil, nil - } - return nil, err - } - return val, nil -} - -// HGetAll returns all fields and values of the hash stored at key -func (hash *Hash) HGetAll() ([][]byte, [][]byte, error) { - if !hash.Exists() { - return nil, nil, nil - } - dkey := DataKey(hash.txn.db, hash.meta.ID) - prefix := hashItemKey(dkey, nil) - endPrefix := kv.Key(prefix).PrefixNext() - iter, err := hash.txn.t.Iter(prefix, endPrefix) - if err != nil { - return nil, nil, err - } - var fields [][]byte - var vals [][]byte - for iter.Valid() && iter.Key().HasPrefix(prefix) { - fields = append(fields, []byte(iter.Key()[len(prefix):])) - vals = append(vals, iter.Value()) - if err := iter.Next(); err != nil { - return nil, nil, err - } - } - return fields, vals, nil -} - -// HExists returns if field is an existing field in the hash stored at key -func (hash *Hash) HExists(field []byte) (bool, error) { - if !hash.Exists() { - return false, nil - } - dkey := DataKey(hash.txn.db, hash.meta.ID) - ikey := hashItemKey(dkey, field) - if _, err := hash.txn.t.Get(ikey); err != nil { - if IsErrNotFound(err) { - return false, nil - } - return false, err - } - return true, nil -} - -// HIncrBy increments the number stored at field in the hash stored at key by increment -func (hash *Hash) HIncrBy(field []byte, v int64) (int64, error) { - var n int64 - dkey := DataKey(hash.txn.db, hash.meta.ID) - ikey := hashItemKey(dkey, field) - - if hash.Exists() { - val, err := hash.txn.t.Get(ikey) - if err != nil { - if !IsErrNotFound(err) { - return 0, err - } - } else { - n, err = strconv.ParseInt(string(val), 10, 64) - if err != nil { - return 0, errors.New("hash value is not an integer") - } - - } - } - n += v - - val := []byte(strconv.FormatInt(n, 10)) - if err := hash.txn.t.Set(ikey, val); err != nil { - return 0, err - } - - if !hash.Exists() { - if err := hash.setMeta(); err != nil { - return 0, err - } - } - - return n, nil -} - -// HIncrByFloat increment the specified field of a hash stored at key, -// and representing a floating point number, by the specified increment -func (hash *Hash) HIncrByFloat(field []byte, v float64) (float64, error) { - var n float64 - dkey := DataKey(hash.txn.db, hash.meta.ID) - ikey := hashItemKey(dkey, field) - if hash.Exists() { - val, err := hash.txn.t.Get(ikey) - if err != nil { - if !IsErrNotFound(err) { - return 0, err - } - } else { - n, err = strconv.ParseFloat(string(val), 64) - if err != nil { - return 0, errors.New("hash value is not an float") - } - - } - } - n += v - - val := []byte(strconv.FormatFloat(n, 'f', -1, 64)) - if err := hash.txn.t.Set(ikey, val); err != nil { - return 0, err - } - - if !hash.Exists() { - if err := hash.setMeta(); err != nil { - return 0, err - } - } - - return n, nil -} - -// HLen returns the number of fields contained in the hash stored at key -func (hash *Hash) HLen() (int64, error) { - if !hash.Exists() { - return 0, nil - } - dkey := DataKey(hash.txn.db, hash.meta.ID) - prefix := hashItemKey(dkey, nil) - endPrefix := kv.Key(prefix).PrefixNext() - var length int64 - callback := func(k kv.Key) bool { - length++ - return false - } - store.SetOption(hash.txn.t, store.KeyOnly, true) - iter, err := hash.txn.t.Iter(prefix, endPrefix) - if err != nil { - return 0, err - } - if err := kv.NextUntil(iter, callback); err != nil { - return 0, err - } - - return length, nil -} - -// HMGet returns the values associated with the specified fields in the hash stored at key -func (hash *Hash) HMGet(fields [][]byte) ([][]byte, error) { - values := make([][]byte, len(fields)) - if !hash.Exists() { - return values, nil - } - ikeys := make([][]byte, len(fields)) - dkey := DataKey(hash.txn.db, hash.meta.ID) - for i := range fields { - ikeys[i] = hashItemKey(dkey, fields[i]) - } - - return BatchGetValues(hash.txn, ikeys) -} - -// HMSet sets the specified fields to their respective values in the hash stored at key -func (hash *Hash) HMSet(fields, values [][]byte) error { - var added int64 - oldValues, err := hash.HMGet(fields) - if err != nil { - return err - } - dkey := DataKey(hash.txn.db, hash.meta.ID) - for i := range fields { - ikey := hashItemKey(dkey, fields[i]) - if err := hash.txn.t.Set(ikey, values[i]); err != nil { - return err - } - if oldValues[i] == nil { - added++ - } - } - if added == 0 { - return nil - } - if !hash.Exists() { - if err := hash.setMeta(); err != nil { - return err - } - } - return nil -} - -// HScan incrementally iterate hash fields and associated values -func (hash *Hash) HScan(cursor []byte, f func(key, val []byte) bool) error { - if !hash.Exists() { - return nil - } - dkey := DataKey(hash.txn.db, hash.meta.ID) - prefix := hashItemKey(dkey, nil) - endPrefix := kv.Key(prefix).PrefixNext() - ikey := hashItemKey(dkey, cursor) - iter, err := hash.txn.t.Iter(ikey, endPrefix) - if err != nil { - return err - } - for iter.Valid() && iter.Key().HasPrefix(prefix) { - key := iter.Key() - if !f(key[len(prefix):], iter.Value()) { - break - } - if err := iter.Next(); err != nil { - return err - } - } - return nil -} - -// Exists check hashes exist -func (hash *Hash) Exists() bool { - return hash.exists -} - -func (hash *Hash) setMeta() error { - meta := EncodeHashMeta(hash.meta) - err := hash.txn.t.Set(MetaKey(hash.txn.db, hash.key), meta) - if err != nil { - return err - } - if !hash.Exists() { - hash.exists = true - } - return nil - -} - -func (hash *Hash) delMeta() error { - err := hash.txn.t.Delete(MetaKey(hash.txn.db, hash.key)) - if err != nil { - return err - } - if hash.Exists() { - hash.exists = false - } - return nil - -} diff --git a/db/hash_test.go b/db/hash_test.go deleted file mode 100644 index 10f1f6a..0000000 --- a/db/hash_test.go +++ /dev/null @@ -1,907 +0,0 @@ -package db - -import ( - "bytes" - "context" - "fmt" - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -// compareGetString skip CreatedAt UpdatedAt ID compare -func compareGetHash(want, get *Hash) error { - switch { - case !bytes.Equal(want.key, get.key): - return fmt.Errorf("set key not equal, want=%s, get=%s", string(want.key), string(get.key)) - case want.meta.ExpireAt != get.meta.ExpireAt: - return fmt.Errorf("meta.expireAt not equal, want=%v, get=%v", want.meta.ExpireAt, get.meta.ExpireAt) - case want.meta.Type != get.meta.Type: - return fmt.Errorf("meta.Type not equal, want=%v, get=%v", want.meta.Type, get.meta.Type) - case want.exists != get.exists: - return fmt.Errorf("exists not equal, want=%v, get=%v", want.exists, get.exists) - } - return nil -} -func compareNewHash(want, get *Hash) error { - switch { - case !bytes.Equal(want.key, get.key): - return fmt.Errorf("set key not equal, want=%s, get=%s", string(want.key), string(get.key)) - } - return nil -} - -var ( - TestHashExistKey = []byte("HashKey") - TesyHashField = []byte("HashField") -) - -func Test_newHash(t *testing.T) { - txn, err := mockDB.Begin() - assert.NotNil(t, txn) - assert.NoError(t, err) - type args struct { - txn *Transaction - key []byte - } - tests := []struct { - name string - args args - want *Hash - }{ - { - name: "TestNewHash", - args: args{ - txn: txn, - key: []byte("TestNewHash"), - }, - want: &Hash{ - meta: &HashMeta{}, - key: []byte("TestNewHash"), - txn: txn, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - txn, err := mockDB.Begin() - assert.NotNil(t, txn) - assert.NoError(t, err) - got := newHash(tt.args.txn, tt.args.key) - txn.Commit(context.TODO()) - if err := compareNewHash(tt.want, got); err != nil { - t.Errorf("NewHash() = %v, want %v", got, tt.want) - } - }) - } - txn.Commit(context.TODO()) -} - -func setHashMeta(t *testing.T, txn *Transaction, key []byte, metaSlot int64) error { - h := newHash(txn, key) - mkey := MetaKey(txn.db, key) - hm := &HashMeta{ - Object: h.meta.Object, - } - meta := EncodeHashMeta(hm) - err := txn.t.Set(mkey, meta) - assert.NoError(t, err) - assert.NotNil(t, txn) - return nil -} - -func getHashMeta(t *testing.T, txn *Transaction, key []byte) *HashMeta { - mkey := MetaKey(txn.db, key) - rawMeta, err := txn.t.Get(mkey) - assert.NoError(t, err) - meta, err1 := DecodeHashMeta(rawMeta) - assert.NoError(t, err1) - return meta -} - -//删除设置的meta信息 -func destoryHashMeta(t *testing.T, txn *Transaction, key []byte) error { - metakey := MetaKey(txn.db, key) - if err := txn.t.Delete(metakey); err != nil { - return err - } - return nil -} - -func getHash(t *testing.T, key []byte) (*Hash, *Transaction, error) { - txn, err := mockDB.Begin() - assert.NotNil(t, txn) - assert.NoError(t, err) - hash, err := GetHash(txn, key) - assert.NotNil(t, hash) - assert.NoError(t, err) - - return hash, txn, nil -} -func compareKvMap(t *testing.T, get, want map[string][]byte) error { - switch { - case !bytes.Equal(want["TestHashdelHashFiled1"], get["TestHashdelHashFiled1"]): - return fmt.Errorf("set key not equal, want=%s, get=%s", string(want["TestHashdelHashFiled1"]), string(get["TestHashdelHashFiled1"])) - case !bytes.Equal(want["TestHashdelHashFiled2"], get["TestHashdelHashFiled2"]): - return fmt.Errorf("set key not equal, want=%s, get=%s", string(want["TestHashdelHashFiled2"]), string(get["Tes tHashdelHashFiled2"])) - case !bytes.Equal(want["TestHashdelHashFiled3"], get["TestHashdelHashFiled3"]): - return fmt.Errorf("set key not equal, want=%s, get=%s", string(want["TestHashdelHashFiled3"]), string(get["Tes tHashdelHashFiled3"])) - } - return nil -} -func TestGetHash(t *testing.T) { - txn, err := mockDB.Begin() - assert.NoError(t, err) - assert.NotNil(t, txn) - - setHashMeta(t, txn, []byte("TestGetHashExistKey"), 0) - setHashMeta(t, txn, []byte("TestGetHashSlotKey"), 100) - type args struct { - txn *Transaction - key []byte - } - type want struct { - hash *Hash - err error - } - tests := []struct { - name string - args args - want want - err error - wantErr bool - }{ - { - name: "TestGetHashNoExistKey", - args: args{ - txn: txn, - key: []byte("TestGetHashNoExistKey"), - }, - want: want{ - hash: &Hash{ - meta: &HashMeta{ - Object: Object{ - Type: ObjectHash, - }, - }, - key: []byte("TestGetHashNoExistKey"), - exists: false, - txn: txn, - }, - err: nil, - }, - err: nil, - wantErr: false, - }, - { - name: "TestGetHashExistKey", - args: args{ - txn: txn, - key: []byte("TestGetHashExistKey")}, - want: want{ - hash: &Hash{meta: &HashMeta{ - Object: Object{ - Type: ObjectHash, - }, - }, - key: []byte("TestGetHashExistKey"), - exists: true, - txn: txn, - }, - err: nil, - }, - err: nil, - wantErr: false, - }, - { - name: "TestGetHashSlotKey", - args: args{ - txn: txn, - key: []byte("TestGetHashSlotKey")}, - want: want{ - hash: &Hash{meta: &HashMeta{ - Object: Object{ - Type: ObjectHash, - }, - }, - key: []byte("TestGetHashSlotKey"), - exists: true, - txn: txn, - }, - err: nil, - }, - err: nil, - wantErr: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - txn, err := mockDB.Begin() - if err != nil { - t.Errorf("db.Begin error %s", err) - } - got, err := GetHash(tt.args.txn, tt.args.key) - if err = txn.Commit(context.TODO()); err != nil { - t.Errorf("GetString() txn.Commit error = %v", err) - return - } - if (err != nil) != tt.wantErr { - t.Errorf("GetHash() error = %v, wantErr %v", err, tt.wantErr) - return - } - if err := compareGetHash(tt.want.hash, got); err != nil { - t.Errorf("GetHash() = %v, want %v", got, tt.want) - } - if tt.want.err != tt.err { - t.Errorf("GetHash() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } - destoryHashMeta(t, txn, []byte("TestGetHashExistKey")) - destoryHashMeta(t, txn, []byte("TestGetHashSlotKey")) - txn.Commit(context.TODO()) -} - -func TestHashHSet(t *testing.T) { - type args struct { - field []byte - value []byte - } - type want struct { - num int - len int - } - tests := []struct { - name string - args args - want want - }{ - { - name: "TestHashHSetNoExistKey", - args: args{ - field: []byte("HashField"), - value: []byte("HashValue"), - }, - want: want{ - num: 1, - len: 1, - }, - }, - { - name: "TestHashHSetExistKey", - args: args{ - field: []byte("HashField"), - value: []byte("HashValue2"), - }, - want: want{ - num: 0, - len: 1, - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - hash, txn, err := getHash(t, []byte("TestHashHSet")) - assert.NoError(t, err) - assert.NotNil(t, txn) - assert.NotNil(t, hash) - - got, err := hash.HSet(tt.args.field, tt.args.value) - assert.NoError(t, err) - assert.NotNil(t, got) - txn.Commit(context.TODO()) - - assert.Equal(t, got, tt.want.num) - }) - } -} -func TestHashHDel(t *testing.T) { - hash, txn, err := getHash(t, []byte("TestHashHDel")) - assert.NoError(t, err) - assert.NotNil(t, txn) - assert.NotNil(t, hash) - - var fileds [][]byte - fileds = append(fileds, []byte("TestHashDelFiled1")) - fileds = append(fileds, []byte("TestHashDelFiled2")) - fileds = append(fileds, []byte("TestHashDelFiled3")) - - hash.HSet([]byte("TestHashDelFiled1"), []byte("TestDelHashValue1")) - hash.HSet([]byte("TestHashDelFiled2"), []byte("TestDelHashValue2")) - hash.HSet([]byte("TestHashDelFiled3"), []byte("TestDelHashValue3")) - txn.Commit(context.TODO()) - - type args struct { - fields [][]byte - } - - type want struct { - num int64 - len int64 - } - - tests := []struct { - name string - args args - want want - }{ - { - name: "TestHashDelCase1", - args: args{ - fields: fileds[:1], - }, - want: want{ - num: 1, - len: 2, - }, - }, - { - name: "TestHashDelCase2", - args: args{ - fields: fileds[1:], - }, - want: want{ - num: 2, - len: 0, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - //test hdel method - hash, txn, err := getHash(t, []byte("TestHashHDel")) - assert.NoError(t, err) - assert.NotNil(t, txn) - assert.NotNil(t, hash) - - got, err := hash.HDel(tt.args.fields) - assert.NoError(t, err) - assert.NotNil(t, got) - - txn.Commit(context.TODO()) - - assert.Equal(t, got, tt.want.num) - - //use hlen check - hash, txn, err = getHash(t, []byte("TestHashHDel")) - hlen, err1 := hash.HLen() - assert.NoError(t, err1) - assert.Equal(t, hlen, tt.want.len) - txn.Commit(context.TODO()) - }) - } -} - -func TestHashHSetNX(t *testing.T) { - - type args struct { - field []byte - value []byte - } - tests := []struct { - name string - args args - want int - }{ - - { - name: "TestHash_HSetNXNoExist", - args: args{ - field: []byte("TestHashSetNxField"), - value: []byte("TestHashSetNxValue"), - }, - want: 1, - }, - - { - name: "TestHash_HSetNXExist", - args: args{ - field: []byte("TestHashSetNxField"), - value: []byte("TestHashSetNxValue"), - }, - want: 0, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - hash, txn, err := getHash(t, []byte("TestHashHSetNX")) - assert.NoError(t, err) - assert.NotNil(t, txn) - assert.NotNil(t, hash) - - got, err := hash.HSetNX(tt.args.field, tt.args.value) - - txn.Commit(context.TODO()) - assert.Equal(t, got, tt.want) - }) - } -} - -func TestHash_HGet(t *testing.T) { - hash, txn, err := getHash(t, []byte("TestHashHGet")) - assert.NoError(t, err) - assert.NotNil(t, txn) - assert.NotNil(t, hash) - - hash.HSet([]byte("TestHashHGetFiled"), []byte("TestHashHGetValue")) - txn.Commit(context.TODO()) - type args struct { - field []byte - } - tests := []struct { - name string - args args - want []byte - }{ - { - name: "TestHashHGet", - args: args{ - field: []byte("TestHashHGetFiled"), - }, - want: []byte("TestHashHGetValue"), - }, - { - name: "TestHashHGetNoExist", - args: args{ - field: []byte("TestHashHGetFiled1"), - }, - want: nil, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - - hash, txn, err := getHash(t, []byte("TestHashHGet")) - assert.NoError(t, err) - assert.NotNil(t, txn) - assert.NotNil(t, hash) - - got, err := hash.HGet(tt.args.field) - txn.Commit(context.TODO()) - assert.Equal(t, got, tt.want) - }) - } -} - -func TestHashHGetAll(t *testing.T) { - - hash, txn, err := getHash(t, []byte("TestHashHGetAll")) - assert.NoError(t, err) - assert.NotNil(t, txn) - assert.NotNil(t, hash) - - hash.HSet([]byte("TestHashHGetAllFiled1"), []byte("TestHashHGetAllValue1")) - hash.HSet([]byte("TestHashHGetAllFiled2"), []byte("TestHashHGetAllValue2")) - hash.HSet([]byte("TestHashHGetAllFiled3"), []byte("TestHashHGetAllValue3")) - txn.Commit(context.TODO()) - type want struct { - fields [][]byte - value [][]byte - } - - var fields [][]byte - var value [][]byte - - fields = append(fields, []byte("TestHashHGetAllFiled1")) - fields = append(fields, []byte("TestHashHGetAllFiled2")) - fields = append(fields, []byte("TestHashHGetAllFiled3")) - value = append(value, []byte("TestHashHGetAllValue1")) - value = append(value, []byte("TestHashHGetAllValue2")) - value = append(value, []byte("TestHashHGetAllValue3")) - tests := []struct { - name string - want want - }{ - - { - name: "TestHashHGetAll", - want: want{ - fields: fields, - value: value, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - hash, txn, err := getHash(t, []byte("TestHashHGetAll")) - assert.NoError(t, err) - assert.NotNil(t, txn) - assert.NotNil(t, hash) - got, got1, err := hash.HGetAll() - assert.NoError(t, err) - assert.NotNil(t, got) - assert.NotNil(t, got1) - txn.Commit(context.TODO()) - - assert.Equal(t, got, tt.want.fields) - assert.Equal(t, got1, tt.want.value) - }) - } -} - -func TestHash_HExists(t *testing.T) { - hash, txn, err := getHash(t, []byte("TestHashExists")) - assert.NoError(t, err) - assert.NotNil(t, txn) - assert.NotNil(t, hash) - - hash.HSet([]byte("TestHashDestory"), []byte("TestHashDestoryValue1")) - - txn.Commit(context.TODO()) - - type args struct { - field []byte - } - tests := []struct { - name string - args args - want bool - }{ - { - name: "TestHashExists", - args: args{ - field: []byte("TestHashDestory"), - }, - want: true, - }, - { - name: "TestHashNoExists", - args: args{ - field: []byte("TestHashNoExistsDestory"), - }, - want: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - - hash, txn, err := getHash(t, []byte("TestHashExists")) - assert.NoError(t, err) - assert.NotNil(t, txn) - assert.NotNil(t, hash) - - got, err := hash.HExists(tt.args.field) - txn.Commit(context.TODO()) - assert.Equal(t, got, tt.want) - assert.NoError(t, err) - }) - } -} - -func TestHashHIncrByFloat(t *testing.T) { - hash, txn, err := getHash(t, []byte("TestHashHIncrByFloat")) - assert.NoError(t, err) - assert.NotNil(t, txn) - assert.NotNil(t, hash) - - hash.HSet([]byte("TestHashHIncrByFloat"), []byte("10.50")) - - txn.Commit(context.TODO()) - type args struct { - field []byte - v float64 - } - tests := []struct { - name string - args args - want float64 - }{ - { - name: "TestHashHIncrByFloat", - args: args{ - field: []byte("TestHashHIncrByFloat"), - v: float64(0.1), - }, - want: float64(10.6), - }, - { - name: "TestHashHIncrByFloat2", - args: args{ - field: []byte("TestHashHIncrByFloat"), - v: float64(-5), - }, - want: float64(5.6), - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - hash, txn, err := getHash(t, []byte("TestHashHIncrByFloat")) - assert.NoError(t, err) - assert.NotNil(t, txn) - assert.NotNil(t, hash) - got, err := hash.HIncrByFloat(tt.args.field, tt.args.v) - txn.Commit(context.TODO()) - assert.Equal(t, got, tt.want) - assert.NoError(t, err) - }) - } -} - -func TestHashHLen(t *testing.T) { - hash, txn, err := getHash(t, []byte("TestHashHLen")) - assert.NoError(t, err) - assert.NotNil(t, txn) - assert.NotNil(t, hash) - // hashslot, err := getHash(t, []byte("TestHashSlotHLen")) - // hashslot.HMSlot(10) - - hash.HSet([]byte("TestHashHlenField1"), []byte("TestHashHlenValue")) - hash.HSet([]byte("TestHashHlenField2"), []byte("TestHashHlenValue")) - hash.HSet([]byte("TestHashHlenField3"), []byte("TestHashHlenValue")) - - txn.Commit(context.TODO()) - hashslot, txn, err := getHash(t, []byte("TestHashSlotHLen")) - assert.NoError(t, err) - assert.NotNil(t, txn) - assert.NotNil(t, hashslot) - // hashslot, err := getHash(t, []byte("TestHashSlotHLen")) - // hashslot.HMSlot(10) - - hashslot.HSet([]byte("TestHashHlenField1"), []byte("TestHashHlenValue")) - - hashslot.HSet([]byte("TestHashHlenField2"), []byte("TestHashHlenValue")) - hashslot.HSet([]byte("TestHashHlenField3"), []byte("TestHashHlenValue")) - - txn.Commit(context.TODO()) - // hashslot.HSet([]byte("TestHashslotHlenField1"), []byte("TestHashHlenValue")) - // hashslot.HSet([]byte("TestHashslotHlenField2"), []byte("TestHashHlenValue")) - // hashslot.HSet([]byte("TestHashslotHlenField3"), []byte("TestHashHlenValue")) - tests := []struct { - name string - - want int64 - }{ - { - name: "TestHashHLen", - - want: 3, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - hash, txn, err := getHash(t, []byte("TestHashHLen")) - assert.NoError(t, err) - assert.NotNil(t, txn) - assert.NotNil(t, hash) - - got, err := hash.HLen() - txn.Commit(context.TODO()) - assert.Equal(t, got, tt.want) - assert.NoError(t, err) - - hashslot, txn, err := getHash(t, []byte("TestHashSlotHLen")) - assert.NoError(t, err) - assert.NotNil(t, txn) - assert.NotNil(t, hashslot) - - got1, err := hashslot.HLen() - txn.Commit(context.TODO()) - assert.Equal(t, got1, tt.want) - assert.NoError(t, err) - }) - } -} - -func TestHash_HScan(t *testing.T) { - hash, txn, err := getHash(t, []byte("TestHashHScan")) - assert.NoError(t, err) - assert.NotNil(t, txn) - assert.NotNil(t, hash) - - hash.HSet([]byte("TestHashHScanFiled1"), []byte("TestHashHScanValue1")) - hash.HSet([]byte("TestHashHScanFiled2"), []byte("TestHashHScanValue2")) - hash.HSet([]byte("TestHashHScanFiled3"), []byte("TestHashHScanValue3")) - - txn.Commit(context.TODO()) - - type args struct { - cursor []byte - f func(key, val []byte) bool - } - var value [][]byte - count := 2 - - tests := []struct { - name string - args args - want [][]byte - }{ - { - name: "TestHashHScan", - args: args{ - cursor: []byte("TestHashHScanFiled"), - f: func(key, val []byte) bool { - if count == 0 { - return false - } - value = append(value, key, val) - count-- - return true - - }, - }, - want: append(value, []byte("TestHashHScanFiled1"), []byte("TestHashHScanValue1"), []byte("TestHashHScanFiled2"), []byte("TestHashHScanValue2")), - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - hash, txn, err := getHash(t, []byte("TestHashHScan")) - assert.NoError(t, err) - assert.NotNil(t, txn) - assert.NotNil(t, hash) - - err = hash.HScan(tt.args.cursor, tt.args.f) - txn.Commit(context.TODO()) - - assert.Equal(t, value, tt.want) - assert.NoError(t, err) - }) - } -} - -func TestHash_HMGet(t *testing.T) { - hash, txn, err := getHash(t, []byte("TestHashHMGet")) - assert.NoError(t, err) - assert.NotNil(t, txn) - assert.NotNil(t, hash) - - hash.HSet([]byte("TestHashHMGetFiled1"), []byte("TestHashHGetValue1")) - hash.HSet([]byte("TestHashHMGetFiled2"), []byte("TestHashHGetValue2")) - hash.HSet([]byte("TestHashHMGetFiled3"), []byte("TestHashHGetValue3")) - - txn.Commit(context.TODO()) - - var fields [][]byte - var value [][]byte - - fields = append(fields, []byte("TestHashHMGetFiled1")) - fields = append(fields, []byte("TestHashHMGetFiled2")) - fields = append(fields, []byte("TestHashHMGetFiled3")) - value = append(value, []byte("TestHashHGetValue1")) - value = append(value, []byte("TestHashHGetValue2")) - value = append(value, []byte("TestHashHGetValue3")) - type args struct { - fields [][]byte - } - tests := []struct { - name string - args args - want [][]byte - }{ - { - name: "TestHashHMGet", - args: args{ - fields: fields, - }, - want: value, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - hash, txn, err := getHash(t, []byte("TestHashHMGet")) - assert.NoError(t, err) - assert.NotNil(t, txn) - assert.NotNil(t, hash) - got, err := hash.HMGet(tt.args.fields) - - txn.Commit(context.TODO()) - assert.Equal(t, got, tt.want) - assert.NoError(t, err) - }) - } -} - -func TestHashHMSet(t *testing.T) { - - var fields [][]byte - var value [][]byte - - fields = append(fields, []byte("TestHashHMSetFiled1")) - fields = append(fields, []byte("TestHashHMSetFiled2")) - fields = append(fields, []byte("TestHashHMSetFiled3")) - value = append(value, []byte("TestHashHSetValue1")) - value = append(value, []byte("TestHashHSetValue2")) - value = append(value, []byte("TestHashHSetValue3")) - type args struct { - fields [][]byte - values [][]byte - } - type want struct { - value [][]byte - len int64 - } - tests := []struct { - name string - args args - want want - }{ - { - name: "TestHashHMSet", - args: args{ - fields: fields, - values: value, - }, - want: want{ - value: value, - len: int64(3), - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - - hash, txn, err := getHash(t, []byte("TestHashHMSet")) - assert.NoError(t, err) - assert.NotNil(t, txn) - assert.NotNil(t, hash) - err = hash.HMSet(tt.args.fields, tt.args.values) - assert.NoError(t, err) - txn.Commit(context.TODO()) - got, err := hash.HMGet(fields) - - assert.Equal(t, got, tt.want.value) - assert.NoError(t, err) - }) - } -} - -func TestHashExpired(t *testing.T) { - setExpireAt := func(t *testing.T, key []byte, expireAt int64) []byte { - hash, txn, err := getHash(t, key) - assert.NoError(t, err) - assert.NotNil(t, txn) - assert.NotNil(t, hash) - id := hash.meta.Object.ID - hash.meta.Object.ExpireAt = expireAt - hash.HSet([]byte("TestHashExpiredfield"), []byte("TestHashExpiredval")) - txn.Commit(context.TODO()) - return id - } - - ts := Now() - type args struct { - key []byte - expireAt int64 - } - tests := []struct { - name string - args args - want bool - }{ - { - name: "TestHashExpired", - args: args{ - key: []byte("TestHashExpired"), - expireAt: ts - 3*int64(time.Second), - }, - want: false, - }, - { - name: "TestHashNotExpired", - args: args{ - key: []byte("TestHashNotExpired"), - expireAt: ts + 10*int64(time.Second), - }, - want: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - oldID := setExpireAt(t, tt.args.key, tt.args.expireAt) - hash, txn, err := getHash(t, tt.args.key) - assert.NoError(t, err) - assert.NotNil(t, txn) - assert.NotNil(t, hash) - newID := hash.meta.Object.ID - txn.Commit(context.TODO()) - if tt.want { - assert.Equal(t, newID, oldID) - } else { - assert.NotEqual(t, newID, oldID) - } - }) - } -} diff --git a/db/kv.go b/db/kv.go deleted file mode 100644 index 0e19481..0000000 --- a/db/kv.go +++ /dev/null @@ -1,352 +0,0 @@ -package db - -import ( - "bytes" - "context" - "errors" - "math/rand" - "sync" - - "github.com/distributedio/titan/db/store" - "github.com/pingcap/kvproto/pkg/kvrpcpb" - "github.com/pingcap/kvproto/pkg/metapb" - sdk_kv "github.com/pingcap/tidb/kv" - "github.com/pingcap/tidb/store/tikv" - "github.com/pingcap/tidb/store/tikv/tikvrpc" - "go.uber.org/zap" -) - -// Kv supplies key releated operations -type Kv struct { - txn *Transaction -} - -// GetKv returns a Kv object -func GetKv(txn *Transaction) *Kv { - return &Kv{txn} -} - -// Keys iterator all keys in db -func (kv *Kv) Keys(start []byte, f func(key []byte) bool) error { - mkey := MetaKey(kv.txn.db, start) - prefix := MetaKey(kv.txn.db, nil) - endPrefix := sdk_kv.Key(prefix).PrefixNext() - iter, err := kv.txn.t.Iter(mkey, endPrefix) - if err != nil { - return err - } - defer iter.Close() - - now := Now() - for iter.Valid() { - key := iter.Key() - if !bytes.HasPrefix(key, prefix) { - break - } - - obj, err := DecodeObject(iter.Value()) - if err != nil { - return err - } - if !IsExpired(obj, now) && !f(key[len(prefix):]) { - break - } - if err := iter.Next(); err != nil { - return err - } - } - return nil -} - -// Delete specific keys, ignore if non exist -func (kv *Kv) Delete(keys [][]byte) (int64, error) { - var ( - count int64 - metaKeys [][]byte - mapping = make(map[string][]byte) - now = Now() - ) - // use mapping to filter duplicate keys - for _, key := range keys { - mkey := MetaKey(kv.txn.db, key) - if _, ok := mapping[string(mkey)]; !ok { - mapping[string(mkey)] = key - metaKeys = append(metaKeys, mkey) - } - } - - values, err := store.BatchGetValues(kv.txn.t, metaKeys) - if err != nil { - return count, err - } - for k, val := range values { - if val != nil { - obj, err := DecodeObject(val) - if err != nil { - return count, err - } - if IsExpired(obj, now) { - continue - } - if err := kv.txn.Destory(obj, mapping[k]); err != nil { - return count, err - } - count++ - } - } - return count, nil -} - -// ExpireAt set a timeout on key -func (kv *Kv) ExpireAt(key []byte, at int64) error { - mkey := MetaKey(kv.txn.db, key) - now := Now() - - meta, err := kv.txn.t.Get(mkey) - if err != nil { - if IsErrNotFound(err) { - return ErrKeyNotFound - } - return err - } - obj, err := DecodeObject(meta) - if err != nil { - return err - } - if IsExpired(obj, now) { - return ErrKeyNotFound - } - if at == 0 && obj.ExpireAt != 0 { - if err = unExpireAt(kv.txn.t, mkey, obj.ExpireAt); err != nil { - return err - } - } - - if at > 0 { - if err := expireAt(kv.txn.t, mkey, obj.ID, obj.Type, obj.ExpireAt, at); err != nil { - return err - } - } - obj.ExpireAt = at - updated := EncodeObject(obj) - updated = append(updated, meta[ObjectEncodingLength:]...) - return kv.txn.t.Set(mkey, updated) -} - -//Exists check if the given keys exist -func (kv *Kv) Exists(keys [][]byte) (int64, error) { - var count int64 - now := Now() - mkeys := make([][]byte, len(keys)) - for i, key := range keys { - mkeys[i] = MetaKey(kv.txn.db, key) - } - - values, err := store.BatchGetValues(kv.txn.t, mkeys) - if err != nil { - return count, err - } - for _, val := range values { - if val != nil { - obj, err := DecodeObject(val) - if err != nil { - return count, err - } - if IsExpired(obj, now) { - continue - } - count++ - } - } - return count, nil -} - -// FlushDB clear current db. -func (kv *Kv) FlushDB(ctx context.Context) error { - prefix := kv.txn.db.Prefix() - endPrefix := sdk_kv.Key(prefix).PrefixNext() - if err := unsafeDeleteRange(ctx, kv.txn.db, prefix, endPrefix); err != nil { - zap.L().Error("flushdb data unsafe clear err", - zap.ByteString("start", prefix), - zap.ByteString("end", endPrefix), - zap.Error(err)) - - return ErrStorageRetry - } - - if err := clearSysRangeData(ctx, kv.txn.db, prefix, endPrefix); err != nil { - return ErrStorageRetry - } - - return nil -} - -// FlushAll clean up all databases. -func (kv *Kv) FlushAll(ctx context.Context) error { - prefix := dbPrefix(kv.txn.db.Namespace, nil) - endPrefix := sdk_kv.Key(prefix).PrefixNext() - if err := unsafeDeleteRange(ctx, kv.txn.db, prefix, endPrefix); err != nil { - zap.L().Error("flushall data unsafe clear err", - zap.ByteString("start", prefix), - zap.ByteString("end", endPrefix), - zap.Error(err)) - return ErrStorageRetry - } - if err := clearSysRangeData(ctx, kv.txn.db, prefix, endPrefix); err != nil { - return ErrStorageRetry - } - - return nil -} - -// RandomKey return a key from current db randomly -// Now we use an static length(64) to generate the key spaces, it means it is random for keys -// that len(key) <= 64, it is enough for most cases -func (kv *Kv) RandomKey() ([]byte, error) { - buf := make([]byte, 64) - // Read for rand here always return a nil error - rand.Read(buf) - - mkey := MetaKey(kv.txn.db, buf) - prefix := MetaKey(kv.txn.db, nil) - - randKeys := func(iter sdk_kv.Iterator) []byte { - keys := make([][]byte, 0) - for iter.Valid() && iter.Key().HasPrefix(prefix) { - keys = append(keys, iter.Key()[len(prefix):]) - iter.Next() - } - count := len(keys) - if count == 0 { - return nil - } - return keys[rand.Int31n(int32(count))] - } - - iter, err := kv.txn.t.Iter(mkey, nil) - if err != nil { - return nil, err - } - - if key := randKeys(iter); key != nil { - return key, nil - } - - iter, err = kv.txn.t.IterReverse(mkey) - if err != nil { - return nil, err - } - if key := randKeys(iter); key != nil { - return key, nil - } - return nil, nil -} - -// Touch alters the last access time of a key(s) -func (kv *Kv) Touch(keys [][]byte) (int64, error) { - ts := Now() - count := int64(0) - - mkeys := make([][]byte, len(keys)) - for i, key := range keys { - mkeys[i] = MetaKey(kv.txn.db, key) - } - - values, err := store.BatchGetValues(kv.txn.t, mkeys) - if err != nil { - return 0, err - } - - for mkey, meta := range values { - if meta == nil { - continue - } - obj, err := DecodeObject(meta) - if err != nil { - return 0, err - } - if IsExpired(obj, ts) { - continue - } - - obj.UpdatedAt = ts - count++ - updated := append(EncodeObject(obj), meta[ObjectEncodingLength:]...) - if err := kv.txn.t.Set([]byte(mkey), updated); err != nil { - return 0, err - } - } - return count, nil -} - -//clear system range data(GC/ZT) -func clearSysRangeData(ctx context.Context, db *DB, startKey, endKey []byte) error { - gcStart := toTiKVGCKey(startKey) - gcEnd := toTiKVGCKey(endKey) - if err := unsafeDeleteRange(ctx, db, gcStart, gcEnd); err != nil { - zap.L().Error("[GC] unsafe clear err", - zap.ByteString("start", gcStart), - zap.ByteString("end", gcEnd), - zap.Error(err)) - return err - } - - ztStart := toZTKey(startKey) - ztEnd := toZTKey(endKey) - if err := unsafeDeleteRange(ctx, db, ztStart, ztEnd); err != nil { - zap.L().Error("[ZT] unsafe clear err", - zap.ByteString("start", ztStart), - zap.ByteString("end", ztEnd), - zap.Error(err)) - return err - } - return nil -} - -func unsafeDeleteRange(ctx context.Context, db *DB, startKey, endKey []byte) error { - storage, ok := db.kv.Storage.(tikv.Storage) - if !ok { - zap.L().Error("delete ranges: storage conversion PDClient failed") - return errors.New("Storage not available") - } - stores, err := storage.GetRegionCache().PDClient().GetAllStores(ctx) - if err != nil { - zap.L().Error("delete ranges: got an error while trying to get store list from PD:", zap.Error(err)) - return err - } - - req := &tikvrpc.Request{ - Type: tikvrpc.CmdUnsafeDestroyRange, - UnsafeDestroyRange: &kvrpcpb.UnsafeDestroyRangeRequest{ - StartKey: startKey, - EndKey: endKey, - }, - } - tikvCli := storage.GetTiKVClient() - - var wg sync.WaitGroup - for _, store := range stores { - if store.State != metapb.StoreState_Up { - continue - } - - address := store.Address - storeID := store.Id - wg.Add(1) - go func() { - defer wg.Done() - _, storeErr := tikvCli.SendRequest(ctx, address, req, tikv.UnsafeDestroyRangeTimeout) - if storeErr != nil { - zap.L().Error("destroy range on store failed with ", - zap.Uint64("store_id", storeID), - zap.String("addr", address), - zap.ByteString("start", startKey), - zap.ByteString("end", endKey), - zap.Error(storeErr)) - err = storeErr - } - }() - } - wg.Wait() - return err -} diff --git a/db/kv_test.go b/db/kv_test.go deleted file mode 100644 index 25c63e6..0000000 --- a/db/kv_test.go +++ /dev/null @@ -1,153 +0,0 @@ -package db - -import ( - "context" - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func SetVal(t *testing.T, db *DB, key, val []byte) { - txn, err := db.Begin() - assert.NoError(t, err) - str := NewString(txn, key) - assert.NoError(t, err) - err = str.Set(val) - assert.NoError(t, err) - txn.Commit(context.Background()) -} - -func CheckNotFoundKey(t *testing.T, db *DB, key []byte) (bool, error) { - txn, err := db.Begin() - assert.NoError(t, err) - obj, err := txn.Object(key) - txn.Commit(context.Background()) - if obj != nil { - return false, err - } - return true, err -} - -func EqualExpireAt(t *testing.T, db *DB, key []byte, expected int64) { - txn, err := db.Begin() - assert.NoError(t, err) - obj, err := txn.Object(key) - txn.Commit(context.Background()) - assert.NoError(t, err) - assert.NotNil(t, obj) - assert.Equal(t, expected, obj.ExpireAt) -} - -func TestDelete(t *testing.T) { - key := []byte("keys-key-del") - val := []byte("keys-val-del") - db := MockDB() - SetVal(t, db, key, val) - txn, err := db.Begin() - assert.NoError(t, err) - kv := txn.Kv() - assert.NoError(t, err) - keys := [][]byte{key} - _, err = kv.Delete(keys) - assert.NoError(t, err) - txn.Commit(context.Background()) - notFound, _ := CheckNotFoundKey(t, db, key) - assert.Equal(t, true, notFound) -} - -func TestExists(t *testing.T) { - db := MockDB() - key := []byte("key-ex") - val := []byte("val-ex") - SetVal(t, db, key, val) - txn, err := db.Begin() - assert.NoError(t, err) - kv := txn.Kv() - assert.NoError(t, err) - keys := [][]byte{key} - _, err = kv.Exists(keys) - txn.Commit(context.Background()) - assert.NoError(t, err) -} - -func TestExpireAt(t *testing.T) { - db := MockDB() - key := []byte("key-ex") - val := []byte("val-ex") - SetVal(t, db, key, val) - now := time.Now().UnixNano() - - time1 := now + int64(100*time.Second) - txn, err := db.Begin() - assert.NoError(t, err) - kv := txn.Kv() - assert.NoError(t, err) - err = kv.ExpireAt(key, time1) - txn.Commit(context.Background()) - assert.NoError(t, err) - EqualExpireAt(t, db, key, time1) - -} - -func TestKeys(t *testing.T) { - list := [][]byte{ - []byte("keys"), - []byte("keys12"), - []byte("keys13"), - []byte("keys14"), - []byte("keys15"), - } - - db := MockDB() - val := []byte("val") - for _, key := range list { - SetVal(t, db, key, val) - } - - txn, err := db.Begin() - assert.NoError(t, err) - kv := txn.Kv() - assert.NoError(t, err) - var actualkeys [][]byte - call := func(key []byte) bool { - actualkeys = append(actualkeys, key) - return true - } - err = kv.Keys([]byte("keys"), call) - assert.NoError(t, err) - txn.Commit(context.Background()) - assert.Equal(t, list, actualkeys) - -} - -func TestRandomKey(t *testing.T) { - list := [][]byte{ - []byte("randomkey1"), - []byte("randomkey2"), - []byte("randomkey3"), - []byte("randomkey4"), - []byte("randomkey5"), - } - - db := MockDB() - val := []byte("val") - for _, key := range list { - SetVal(t, db, key, val) - } - - mapkey := make(map[string]int) - for i := 0; i < 5; i++ { - txn, err := db.Begin() - assert.NoError(t, err) - kv := txn.Kv() - assert.NoError(t, err) - - tmp1, err := kv.RandomKey() - assert.NoError(t, err) - mapkey[string(tmp1)]++ - txn.Commit(context.Background()) - - } - // assert.NotEqual(t, 1, len(mapkey)) -} diff --git a/db/lease.go b/db/lease.go deleted file mode 100644 index d7f66e2..0000000 --- a/db/lease.go +++ /dev/null @@ -1,8 +0,0 @@ -package db - -// Lease is an object that can be associated with other objects -// those can share the same ttl of the lease -type Lease struct { - Object - TouchedAt int64 -} diff --git a/db/list.go b/db/list.go deleted file mode 100644 index d9ff548..0000000 --- a/db/list.go +++ /dev/null @@ -1,63 +0,0 @@ -package db - -// List defines the list interface -type List interface { - Index(n int64) (data []byte, err error) - Insert(pivot, v []byte, before bool) error - LPop() (data []byte, err error) - LPush(data ...[]byte) (err error) - RPop() (data []byte, err error) - RPush(data ...[]byte) (err error) - Range(left, right int64) (value [][]byte, err error) - LRem(v []byte, n int64) (int, error) - Set(n int64, data []byte) error - LTrim(start int64, stop int64) error - Length() int64 - Exist() bool - Destory() error -} - -// GetList returns a List object, it creates a new one if the key does not exist, -// when UseZip() is set, it will create a ziplist instead of a linklist -func GetList(txn *Transaction, key []byte, opts ...ListOption) (List, error) { - opt := &listOption{} - for _, o := range opts { - if o == nil { - continue - } - o(opt) - } - list := NewLList - if opt.useZip { - list = NewZList - } - - metaKey := MetaKey(txn.db, key) - val, err := txn.t.Get(metaKey) - if err != nil { - if IsErrNotFound(err) { // error NotFound - return list(txn, key) - } - return nil, err - } - - // exist - obj, err := DecodeObject(val) - if err != nil { - return nil, err - } - if IsExpired(obj, Now()) { - return list(txn, key) - } - - if obj.Type != ObjectList { - return nil, ErrTypeMismatch - } - - if obj.Encoding == ObjectEncodingLinkedlist { - return GetLList(txn, metaKey, obj, val) - } else if obj.Encoding == ObjectEncodingZiplist { - return GetZList(txn, metaKey, obj, val) - } - return nil, ErrEncodingMismatch -} diff --git a/db/list_test.go b/db/list_test.go deleted file mode 100644 index c3d2d6c..0000000 --- a/db/list_test.go +++ /dev/null @@ -1,8 +0,0 @@ -package db - -import "testing" - -func TestEncodeComleteCode(t *testing.T) {} -func TestDecodeComleteCode(t *testing.T) {} -func TestCaculateIndex(t *testing.T) {} -func TestPrecision(t *testing.T) {} diff --git a/db/llist.go b/db/llist.go deleted file mode 100644 index 3baed34..0000000 --- a/db/llist.go +++ /dev/null @@ -1,697 +0,0 @@ -package db - -import ( - "bytes" - "encoding/binary" - "math" - "time" - - "github.com/distributedio/titan/metrics" - - "github.com/pingcap/tidb/kv" -) - -// LListMeta keeps all meta info of a list object -// after marshaled, listmeta raw data should organized like below -// |----object 34----| -// |------len 8------| -// |----linedx 8-----| -// |----rindex 8-----| -type LListMeta struct { - Object - Len int64 - Lindex float64 - Rindex float64 -} - -// LList is a distributed object that works like a double link list -// {Object schema}:{index} -> value -type LList struct { - LListMeta - rawMetaKey []byte - rawDataKeyPrefix []byte - txn *Transaction -} - -//GetLList returns a list -func GetLList(txn *Transaction, metaKey []byte, obj *Object, val []byte) (List, error) { - l := &LList{ - txn: txn, - rawMetaKey: metaKey, - } - if err := l.LListMeta.Unmarshal(obj, val); err != nil { - return nil, err - } - l.rawDataKeyPrefix = DataKey(txn.db, l.Object.ID) - l.rawDataKeyPrefix = append(l.rawDataKeyPrefix, []byte(Separator)...) - return l, nil -} - -//NewLList creates a new list -func NewLList(txn *Transaction, key []byte) (List, error) { - now := Now() - metaKey := MetaKey(txn.db, key) - obj := Object{ - ExpireAt: 0, - CreatedAt: now, - UpdatedAt: now, - Type: ObjectList, - ID: UUID(), - Encoding: ObjectEncodingLinkedlist, - } - l := &LList{ - LListMeta: LListMeta{ - Object: obj, - Len: 0, - Lindex: 0, - Rindex: 0, - }, - txn: txn, - rawMetaKey: metaKey, - } - l.rawDataKeyPrefix = DataKey(txn.db, l.Object.ID) - l.rawDataKeyPrefix = append(l.rawDataKeyPrefix, []byte(Separator)...) - return l, nil - -} - -// Marshal encodes meta data into byte slice -func (l *LListMeta) Marshal() []byte { - b := EncodeObject(&l.Object) - meta := make([]byte, 24) - binary.BigEndian.PutUint64(meta[0:8], uint64(l.Len)) - binary.BigEndian.PutUint64(meta[8:16], math.Float64bits(l.Lindex)) - binary.BigEndian.PutUint64(meta[16:24], math.Float64bits(l.Rindex)) - return append(b, meta...) -} - -// Unmarshal parses meta data into meta field -func (l *LListMeta) Unmarshal(obj *Object, b []byte) (err error) { - if len(b[ObjectEncodingLength:]) != 24 { - return ErrInvalidLength - } - meta := b[ObjectEncodingLength:] - l.Object = *obj - l.Len = int64(binary.BigEndian.Uint64(meta[:8])) - l.Lindex = math.Float64frombits(binary.BigEndian.Uint64(meta[8:16])) - l.Rindex = math.Float64frombits(binary.BigEndian.Uint64(meta[16:24])) - return nil -} - -// Length returns length of the list -func (l *LList) Length() int64 { return l.LListMeta.Len } - -// LPush adds new elements to the left -// 1. calculate index -// 2. encode object and call kv -// 3. modify the new index in meta -func (l *LList) LPush(data ...[]byte) (err error) { - for i := range data { - l.Lindex-- - encode, err := EncodeFloat64(l.Lindex) - if err != nil { - return err - } - if err = l.txn.t.Set(append(l.rawDataKeyPrefix, encode...), data[i]); err != nil { - return err - } - l.Len++ - if l.Len == 1 { - l.Rindex = l.Lindex - } - } - return l.txn.t.Set(l.rawMetaKey, l.LListMeta.Marshal()) -} - -// RPush pushes elements into right side of list -func (l *LList) RPush(data ...[]byte) (err error) { - for i := range data { - l.Rindex++ - encode, err := EncodeFloat64(l.Rindex) - if err != nil { - return err - } - - if err = l.txn.t.Set(append(l.rawDataKeyPrefix, encode...), data[i]); err != nil { - return err - } - l.Len++ - if l.Len == 1 { - l.Lindex = l.Rindex - } - } - return l.txn.t.Set(l.rawMetaKey, l.LListMeta.Marshal()) -} - -// Set the index object with given value, return ErrIndex on out of range error. -func (l *LList) Set(n int64, data []byte) error { - if n < 0 { - n = l.Len + n - } - if n < 0 || n >= l.Len { - return ErrOutOfRange - } - realidx, _, err := l.index(n) - if err != nil { - return err - } - - encode, err := EncodeFloat64(realidx) - if err != nil { - return err - } - return l.txn.t.Set(append(l.rawDataKeyPrefix, encode...), data) -} - -// Insert value in the list stored at key either before or after the reference value pivot -// 1. pivot berfore/ pivot/ next --> real indexs -func (l *LList) Insert(pivot, v []byte, before bool) error { - idxs, err := l.indexValue(pivot) - if err != nil { - return err - } - - var idx float64 - if before { - if idxs[0] == math.MaxFloat64 { // LPUSH - l.LListMeta.Lindex-- - idx = l.LListMeta.Lindex - } else if idx, err = calculateIndex(idxs[0], idxs[1]); err != nil { - return err - } - } else { - if idxs[2] == math.MaxFloat64 { // RPUSH - l.LListMeta.Rindex++ - idx = l.LListMeta.Rindex - } else if idx, err = calculateIndex(idxs[1], idxs[2]); err != nil { - return err - } - } - l.Len++ - - encode, err := EncodeFloat64(idx) - if err != nil { - return err - } - if err = l.txn.t.Set(append(l.rawDataKeyPrefix, encode...), v); err != nil { - return err - } - return l.txn.t.Set(l.rawMetaKey, l.LListMeta.Marshal()) -} - -// Index returns the element at index n in the list stored at key -func (l *LList) Index(n int64) (data []byte, err error) { - if n < 0 { - n = l.Len + n - } - if n < 0 || n >= l.Len { - return nil, ErrOutOfRange - } - _, val, err := l.index(n) - if err != nil { - return nil, err - } - return val, nil -} - -// LPop returns and deletes the left most element -// 0. calculate data key -// 1. iterate to last value -// 2. get the key and call kv delete -// 3. modify the new index in meta -func (l *LList) LPop() (data []byte, err error) { - if l.Len == 0 { - // XXX should delete this? - return nil, ErrKeyNotFound - } - - encode, err := EncodeFloat64(l.LListMeta.Lindex) - if err != nil { - return nil, err - } - leftKey := append(l.rawDataKeyPrefix, encode...) - - // find the left object - iter, err := l.txn.t.Iter(leftKey, nil) - if err != nil { - return nil, err - } - if !iter.Valid() || !iter.Key().HasPrefix(l.rawDataKeyPrefix) { - return nil, ErrKeyNotFound - } - val := iter.Value() - - if err = l.txn.t.Delete(iter.Key()); err != nil { - return nil, err - } - - if l.Len == 1 { - return val, l.txn.t.Delete(l.rawMetaKey) - } - - // get the next data object and check if get - err = iter.Next() - if err != nil { - return nil, err - } - if !iter.Valid() || !iter.Key().HasPrefix(l.rawDataKeyPrefix) { - return nil, ErrKeyNotFound - } - l.LListMeta.Len-- - l.LListMeta.Lindex = DecodeFloat64(iter.Key()[len(l.rawDataKeyPrefix):]) // trim prefix with list data key - return val, l.txn.t.Set(l.rawMetaKey, l.LListMeta.Marshal()) -} - -// RPop returns and deletes the right most element -func (l *LList) RPop() ([]byte, error) { - if l.Len == 0 { - return nil, ErrKeyNotFound - } - - encode, err := EncodeFloat64(l.LListMeta.Rindex) - if err != nil { - return nil, err - } - // rightKey: {DB.ns}:{DB.id}:D:{linedx} - rightKey := append(l.rawDataKeyPrefix, encode...) - - // find the left object - iter, err := l.txn.t.IterReverse(rightKey) - if err != nil { - return nil, err - } - if !iter.Valid() || !iter.Key().HasPrefix(l.rawDataKeyPrefix) { - return nil, ErrKeyNotFound - } - val := iter.Value() - if err = l.txn.t.Delete(iter.Key()); err != nil { - return nil, err - } - - if l.Len == 1 { - return val, l.txn.t.Delete(l.rawMetaKey) - } - - // get the next data object and check if get - err = iter.Next() - if err != nil { - return nil, err - } - if !iter.Valid() || !iter.Key().HasPrefix(l.rawDataKeyPrefix) { - return nil, ErrKeyNotFound - } - l.LListMeta.Len-- - l.LListMeta.Rindex = DecodeFloat64(iter.Key()[len(l.rawDataKeyPrefix):]) // trim prefix with list data key - return val, l.txn.t.Set(l.rawMetaKey, l.LListMeta.Marshal()) -} - -// Range returns the elements in [left, right] -func (l *LList) Range(left, right int64) (value [][]byte, err error) { - if right < 0 { - if right = l.Len + right; right < 0 { - return [][]byte{}, nil - } - } - if left < 0 { - if left = l.Len + left; left < 0 { - left = 0 - } - } - // return 0 elements - if left > right { - return [][]byte{}, nil - } - _, v, err := l.scan(left, right+1) - return v, err -} - -// LTrim an existing list so that it will contain only the specified range of elements specified -func (l *LList) LTrim(start int64, stop int64) error { - if start < 0 { - if start = l.Len + start; start < 0 { - start = 0 - } - } - if stop < 0 { - stop = l.Len + stop - } - if stop > l.Len-1 { - stop = l.Len - 1 - } - - if start > stop { - return l.Destory() - } - - lIndex := l.Lindex - rIndex := l.Rindex - var err error - // Iter from left to start, when start is 0, do not need to seek - if start > 0 { - lIndex, err = l.remove(l.LListMeta.Lindex, start) - if err != nil { - return err - } - } - if stop+1 < l.Len { - /* - TODO bug - rIndex, _, err = l.index(stop) // stop is included to reserve - if err != nil { - return err - } - */ - iter, err := l.seekIndex(stop) - if err != nil { - return err - } - defer iter.Close() - rIndex = DecodeFloat64(iter.Key()[len(l.rawDataKeyPrefix):]) - if err := iter.Next(); err != nil { - return err - } - - toRemove := DecodeFloat64(iter.Key()[len(l.rawDataKeyPrefix):]) - _, err = l.remove(toRemove, l.Len-stop-1) - if err != nil { - return err - } - } - l.Lindex = lIndex - l.Rindex = rIndex - l.Len = stop - start + 1 - if l.LListMeta.Len == 0 { // destory if len comes to 0 - return l.txn.t.Delete(l.rawMetaKey) - } - return l.txn.t.Set(l.rawMetaKey, l.LListMeta.Marshal()) -} - -// seekIndex will return till we get the last element not larger than index -func (l *LList) seekIndex(index int64) (Iterator, error) { - encode, err := EncodeFloat64(l.Lindex) - if err != nil { - return nil, err - } - key := append(l.rawDataKeyPrefix, encode...) - iter, err := l.txn.t.Iter(key, nil) - if err != nil { - return nil, err - } - for ; iter.Valid() && iter.Key().HasPrefix(l.rawDataKeyPrefix) && index != 0 && err == nil; err = iter.Next() { - index-- - } - if err != nil { - return nil, err - } - return iter, nil -} - -// remove n elements from start, return next index after delete -func (l *LList) remove(start float64, n int64) (float64, error) { - encode, err := EncodeFloat64(start) - if err != nil { - return 0, err - } - startKey := append(l.rawDataKeyPrefix, encode...) - iter, err := l.txn.t.Iter(startKey, nil) - if err != nil { - return 0, err - } - for iter.Valid() && iter.Key().HasPrefix(l.rawDataKeyPrefix) && n != 0 { - if err := l.txn.t.Delete(iter.Key()); err != nil { - return 0, err - } - if err := iter.Next(); err != nil { - return 0, err - } - n-- - } - if n > 0 { - return l.Rindex, nil - } - - if err := iter.Next(); err != nil { - return 0, err - } - nextKey := iter.Key() - if !nextKey.HasPrefix(l.rawDataKeyPrefix) { - return l.Rindex, nil - } - - nextIdx := DecodeFloat64(nextKey[len(l.rawDataKeyPrefix):]) - return nextIdx, nil -} - -// index return the index and value of the n index list data -// n should be positive -func (l *LList) index(n int64) (realindex float64, value []byte, err error) { - if n < 0 || n >= l.Len { - return 0, nil, ErrOutOfRange - } - encode, err := EncodeFloat64(l.LListMeta.Lindex) - if err != nil { - return 0, nil, err - } - - // case1: only 1 object in list - if l.Len == 1 { - val, err := l.txn.t.Get(append(l.rawDataKeyPrefix, encode...)) - if err != nil { - return 0, nil, err - } - return l.Lindex, val, nil - } - - // case2: only 2 object in list - if l.Len == 2 { - idx := l.LListMeta.Rindex - if n == 0 { - idx = l.LListMeta.Lindex - } - encode, err := EncodeFloat64(idx) - if err != nil { - return 0, nil, err - } - - val, err := l.txn.t.Get(append(l.rawDataKeyPrefix, encode...)) - if err != nil { - return 0, nil, err - } - return idx, val, nil - } - - idxs, vals, err := l.scan(n, n+1) - if err != nil { - return 0, nil, err - } - return idxs[0], vals[0], nil -} - -// LRem removes the first count occurrences of elements equal to value from the list stored at key -func (l *LList) LRem(v []byte, n int64) (int, error) { - idxs, err := l.indexValueN(v, n) - if err != nil { - return 0, err - } - - for i := range idxs { - encode, err := EncodeFloat64(idxs[i]) - if err != nil { - return 0, err - } - - if err = l.txn.t.Delete(append(l.rawDataKeyPrefix, encode...)); err != nil { - return 0, err - } - } - - l.LListMeta.Len -= int64(len(idxs)) - if l.LListMeta.Len == 0 { // destory if len comes to 0 - return len(idxs), l.txn.t.Delete(l.rawMetaKey) - } - - // TODO maybe we can find a new way to avoid these seek - // update list index and left right index - encode, err := EncodeFloat64(l.LListMeta.Rindex) - if err != nil { - return 0, err - } - - rightKey := append(l.rawDataKeyPrefix, encode...) - iter, err := l.txn.t.IterReverse(rightKey) - if err != nil { - return 0, err - } - if !iter.Valid() || !iter.Key().HasPrefix(l.rawDataKeyPrefix) { - return 0, ErrKeyNotFound - } - l.LListMeta.Rindex = DecodeFloat64(iter.Key()[len(l.rawDataKeyPrefix):]) // trim prefix with list data key - - encode, err = EncodeFloat64(l.LListMeta.Lindex) - if err != nil { - return 0, err - } - - leftKey := append(l.rawDataKeyPrefix, encode...) - iter, err = l.txn.t.Iter(leftKey, nil) - if err != nil { - return 0, err - } - if !iter.Valid() || !iter.Key().HasPrefix(l.rawDataKeyPrefix) { - return 0, ErrKeyNotFound - } - l.LListMeta.Lindex = DecodeFloat64(iter.Key()[len(l.rawDataKeyPrefix):]) // trim prefix with list data key - - return len(idxs), l.txn.t.Set(l.rawMetaKey, l.LListMeta.Marshal()) -} - -// indexValueN return the index of the given list data value. -func (l *LList) indexValueN(v []byte, n int64) (realidxs []float64, err error) { - var iter kv.Iterator - if n < 0 { - n = -n - encode, err := EncodeFloat64(l.LListMeta.Rindex) - if err != nil { - return nil, err - } - if iter, err = l.txn.t.IterReverse(append(l.rawDataKeyPrefix, encode...)); err != nil { - return nil, err - } - } else if n > 0 { - encode, err := EncodeFloat64(l.LListMeta.Lindex) - if err != nil { - return nil, err - } - if iter, err = l.txn.t.Iter(append(l.rawDataKeyPrefix, encode...), nil); err != nil { - return nil, err - } - } else { - n = l.Len - } - - // for loop iterate all objects and check if valid until reach pivot value - for count := int64(0); count < n && err == nil && iter.Valid() && iter.Key().HasPrefix(l.rawDataKeyPrefix); err = iter.Next() { - // reset the rindex/lindex here - if bytes.Equal(iter.Value(), v) { // found the value! now iter the next and return - realidxs = append(realidxs, DecodeFloat64(iter.Key()[len(l.rawDataKeyPrefix):])) - count++ - } - } - return realidxs, err -} - -// indexValue return the [befor, real, after] index and value of the given list data value. -func (l *LList) indexValue(v []byte) (realidxs []float64, err error) { - realidxs = []float64{math.MaxFloat64, math.MaxFloat64, math.MaxFloat64} - encode, err := EncodeFloat64(l.LListMeta.Lindex) - if err != nil { - return nil, err - } - - iter, err := l.txn.t.Iter(append(l.rawDataKeyPrefix, encode...), nil) - if err != nil { // dup - return nil, err - } - - flag := false //if we find the v key - rawDataKeyPrefixLen := len(l.rawDataKeyPrefix) - - // for loop iterate all objects and check if valid until reach pivot value - for ; iter.Valid() && iter.Key().HasPrefix(l.rawDataKeyPrefix) && err == nil; err = iter.Next() { - realidxs[0] = realidxs[1] - realidxs[1] = realidxs[2] - realidxs[2] = DecodeFloat64(iter.Key()[rawDataKeyPrefixLen:]) - if bytes.Equal(iter.Value(), v) { // found the value! now iter the next and return - flag = true - realidxs[0] = realidxs[1] - realidxs[1] = realidxs[2] - if err = iter.Next(); err == nil && iter.Valid() && iter.Key().HasPrefix(l.rawDataKeyPrefix) { - // found next value index - realidxs[2] = DecodeFloat64(iter.Key()[rawDataKeyPrefixLen:]) - } else { - // did not found next value - // 1. {before, key, next} - // 1.1 {NULL, key, next} - // 2. {before, key, NULL} - // 2.1 {NULL, key, NULL} - realidxs[2] = math.MaxFloat64 - } - break - } - } - if err != nil { - return nil, err - } - if flag { - return realidxs, nil - } - return nil, ErrKeyNotFound -} - -// scan return objects between [left, right) range -// if left == right , we should return the firsh element -// else case: iterator all keys and get the object index value -// rightidx may not larger than right -func (l *LList) scan(left, right int64) (realidxs []float64, values [][]byte, err error) { - realidxs = make([]float64, 0, right-left) - values = make([][]byte, 0, right-left) - - // seek start indecate the seek first key start time. - encode, err := EncodeFloat64(l.LListMeta.Lindex) - if err != nil { - return nil, nil, err - } - start := time.Now() - iter, err := l.txn.t.Iter(append(l.rawDataKeyPrefix, encode...), nil) - - var idx int64 - // for loop iterate all objects to get the next data object and check if valid - for idx = 0; idx < left && err == nil && iter.Valid() && iter.Key().HasPrefix(l.rawDataKeyPrefix); err = iter.Next() { - idx++ - } - if err != nil { // err tikv error - return nil, nil, err - } - // if list not exist, return the 0 - if idx != left { - return []float64{}, [][]byte{}, nil - } - - //monitor the seek first key cost - metrics.GetMetrics().LRangeSeekHistogram.Observe(time.Since(start).Seconds()) - - // for loop iterate all objects to get objects and check if valid - for ; idx < right && err == nil && iter.Valid() && iter.Key().HasPrefix(l.rawDataKeyPrefix); err = iter.Next() { - // found - realidxs = append(realidxs, DecodeFloat64(iter.Key()[len(l.rawDataKeyPrefix):])) - values = append(values, iter.Value()) - idx++ - } - if err != nil { - return nil, nil, err - } - return realidxs, values, nil -} - -// Exist checks if a list exists -func (l *LList) Exist() bool { - return l.Len != 0 -} - -// Destory the list -func (l *LList) Destory() error { - // delete the meta data - if err := l.txn.t.Delete(l.rawMetaKey); err != nil { - return err - } - // leaving the data to gc - return gc(l.txn.t, l.rawDataKeyPrefix) -} - -// calculateIndex return the real index between left and right, return ErrPerc= -func calculateIndex(left, right float64) (float64, error) { - if f := (left + right) / 2; f != left && f != right { - return f, nil - } - return 0, ErrPrecision -} diff --git a/db/object.go b/db/object.go deleted file mode 100644 index b52750b..0000000 --- a/db/object.go +++ /dev/null @@ -1,161 +0,0 @@ -package db - -import ( - "fmt" -) - -const ( - // Separator of the key segment - Separator = ":" - // ObjectEncodingLength indecate current object marshaled length - ObjectEncodingLength = 42 -) - -// ObjectEncoding is the encoding type of an object -type ObjectEncoding byte - -// Encoding values, see https://github.com/antirez/redis/blob/unstable/src/server.h#L581 -const ( - ObjectEncodingRaw = ObjectEncoding(iota) - ObjectEncodingInt - ObjectEncodingHT - ObjectEncodingZipmap - ObjectEncodingLinkedlist - ObjectEncodingZiplist - ObjectEncodingIntset - ObjectEncodingSkiplist - ObjectEncodingEmbstr - ObjectEncodingQuicklist -) - -// String representation of ObjectEncoding -func (enc ObjectEncoding) String() string { - switch enc { - case ObjectEncodingRaw: - return "raw" - case ObjectEncodingInt: - return "int" - case ObjectEncodingHT: - return "hashtable" - case ObjectEncodingZipmap: - return "zipmap" - case ObjectEncodingLinkedlist: - return "linkedlist" - case ObjectEncodingZiplist: - return "ziplist" - case ObjectEncodingIntset: - return "intset" - case ObjectEncodingSkiplist: - return "skiplist" - case ObjectEncodingEmbstr: - return "embstr" - case ObjectEncodingQuicklist: - return "quicklist" - default: - return "unknown" - } -} - -// ObjectType is the type of a data structure -type ObjectType byte - -// String representation of object type -func (t ObjectType) String() string { - switch t { - case ObjectString: - return "string" - case ObjectList: - return "list" - case ObjectSet: - return "set" - case ObjectZSet: - return "zset" - case ObjectHash: - return "hash" - } - return "none" -} - -// Object types, see https://github.com/antirez/redis/blob/unstable/src/server.h#L461 -const ( - ObjectString = ObjectType(iota) - ObjectList - ObjectSet - ObjectZSet - ObjectHash -) - -// Object meta schema -// Layout {DB}:{TAG}:{Key} -// DB [0-255] -// Key Usersapce key -// TAG M(Meta), D(Data) -// Object data schema -// Layout: {DB}:{TAG}:{ID}:{Others} -// ID Object ID, ID is not used for meta -// String schema (associated value with meta) -// Layout: {DB}:M:{key} -type Object struct { - ID []byte - Type ObjectType //refer to redis - Encoding ObjectEncoding //refer to redis - CreatedAt int64 - UpdatedAt int64 - ExpireAt int64 -} - -// String representation of an object -func (obj *Object) String() string { - return fmt.Sprintf("ID:%s type:%s encoding:%s createdat:%d updatedat:%d expireat:%d", - UUIDString(obj.ID), obj.Type, obj.Encoding, obj.CreatedAt, obj.UpdatedAt, obj.ExpireAt) -} - -// Object returns the object associated with the key -func (txn *Transaction) Object(key []byte) (*Object, error) { - mkey := MetaKey(txn.db, key) - obj, err := getObject(txn, mkey) - if err != nil { - return nil, err - } - if IsExpired(obj, Now()) { - return nil, ErrKeyNotFound - } - return obj, nil -} - -// Destory the object -func (txn *Transaction) Destory(obj *Object, key []byte) error { - mkey := MetaKey(txn.db, key) - dkey := DataKey(txn.db, obj.ID) - if err := txn.t.Delete(mkey); err != nil { - return err - } - if obj.Type != ObjectString { - if err := gc(txn.t, dkey); err != nil { - return err - } - } - - if obj.ExpireAt > 0 { - if err := unExpireAt(txn.t, mkey, obj.ExpireAt); err != nil { - return err - } - } - - return nil -} - -func getObject(txn *Transaction, metaKey []byte) (*Object, error) { - meta, err := txn.t.Get(metaKey) - if err != nil { - if IsErrNotFound(err) { - return nil, ErrKeyNotFound - } - return nil, err - } - obj, err := DecodeObject(meta) - if err != nil { - return nil, err - } - return obj, nil -} diff --git a/db/set.go b/db/set.go deleted file mode 100644 index e0fa232..0000000 --- a/db/set.go +++ /dev/null @@ -1,349 +0,0 @@ -package db - -import ( - "bytes" - "encoding/binary" - - "github.com/pingcap/tidb/kv" -) - -// SetNilValue is the value set to a tikv key for tikv do not support a real empty value -var SetNilValue = []byte{0} - -// SetMeta is the meta data of the set -type SetMeta struct { - Object - Len int64 -} - -// Set implements the set data structure -type Set struct { - meta *SetMeta - key []byte - exists bool - txn *Transaction -} - -// GetSet returns a set object, create new one if nonexists -func GetSet(txn *Transaction, key []byte) (*Set, error) { - set := newSet(txn, key) - mkey := MetaKey(txn.db, key) - meta, err := txn.t.Get(mkey) - if err != nil { - if IsErrNotFound(err) { - return set, nil - } - return nil, err - } - - obj, err := DecodeObject(meta) - if err != nil { - return nil, err - } - if IsExpired(obj, Now()) { - return set, nil - } - if obj.Type != ObjectSet { - return nil, ErrTypeMismatch - } - - m := meta[ObjectEncodingLength:] - if len(m) != 8 { - return nil, ErrInvalidLength - } - set.meta.Object = *obj - set.meta.Len = int64(binary.BigEndian.Uint64(m[:8])) - set.exists = true - return set, nil -} - -// SetIter is the struct of Iterator and prefix -type SetIter struct { - Iter Iterator - Prefix []byte -} - -// Iter returns the SetIter object -func (set *Set) Iter() (*SetIter, error) { - var siter SetIter - dkey := DataKey(set.txn.db, set.meta.ID) - prefix := append(dkey, ':') - endPrefix := kv.Key(prefix).PrefixNext() - iter, err := set.txn.t.Iter(prefix, endPrefix) - if err != nil { - return nil, err - } - siter.Iter = iter - siter.Prefix = prefix - return &siter, nil -} - -// Value returns the member pointed by iter -func (siter *SetIter) Value() []byte { - res := siter.Iter.Key()[len(siter.Prefix):] - return res -} - -// Valid judgies whether the key directed by iter has the same prifix -func (siter *SetIter) Valid() bool { - return siter.Iter.Key().HasPrefix(siter.Prefix) -} - -//newSet create new Set object -func newSet(txn *Transaction, key []byte) *Set { - now := Now() - return &Set{ - txn: txn, - key: key, - meta: &SetMeta{ - Object: Object{ - ID: UUID(), - CreatedAt: now, - UpdatedAt: now, - ExpireAt: 0, - Type: ObjectSet, - Encoding: ObjectEncodingHT, - }, - Len: 0, - }, - } -} - -//encodeSetMeta encodes meta data into byte slice -func encodeSetMeta(meta *SetMeta) []byte { - b := EncodeObject(&meta.Object) - m := make([]byte, 8) - binary.BigEndian.PutUint64(m[:8], uint64(meta.Len)) - return append(b, m...) -} - -func setItemKey(key []byte, member []byte) []byte { - var ikeys []byte - ikeys = append(ikeys, key...) - ikeys = append(ikeys, ':') - ikeys = append(ikeys, member...) - return ikeys -} - -func (set *Set) updateMeta() error { - meta := encodeSetMeta(set.meta) - err := set.txn.t.Set(MetaKey(set.txn.db, set.key), meta) - if err != nil { - return err - } - set.meta.UpdatedAt = Now() - if !set.exists { - set.exists = true - } - return nil -} - -// SAdd adds the specified members to the set stored at key -func (set *Set) SAdd(members ...[]byte) (int64, error) { - // Namespace:DBID:D:ObjectID - dkey := DataKey(set.txn.db, set.meta.ID) - // Remove the duplicate - ms := RemoveRepByMap(members) - ikeys := make([][]byte, len(ms)) - for i := range ms { - ikeys[i] = setItemKey(dkey, ms[i]) - } - // {Namespace}:{DBID}:{D}:{ObjectID}:{ms[i]} - values, err := BatchGetValues(set.txn, ikeys) - if err != nil { - return 0, nil - } - added := int64(0) - for i := range ikeys { - if values[i] == nil { - added++ - } - if err := set.txn.t.Set(ikeys[i], SetNilValue); err != nil { - return 0, err - } - } - set.meta.Len += added - if err := set.updateMeta(); err != nil { - return 0, err - } - - return added, nil -} - -// RemoveRepByMap filters duplicate elements through the map's unique primary key feature -func RemoveRepByMap(members [][]byte) [][]byte { - result := [][]byte{} - // uniq saves non-repeating primary keys - uniq := map[string]struct{}{} - for _, m := range members { - mStr := string(m) - _, ok := uniq[mStr] - if !ok { - result = append(result, m) - uniq[mStr] = struct{}{} - } - } - return result -} - -// Exists check set exist -func (set *Set) Exists() bool { - return set.exists -} - -// SMembers returns all the members of the set value stored at key -func (set *Set) SMembers() ([][]byte, error) { - if !set.Exists() { - return nil, nil - } - dkey := DataKey(set.txn.db, set.meta.ID) - prefix := append(dkey, ':') - endPrefix := kv.Key(prefix).PrefixNext() - count := set.meta.Len - members := make([][]byte, 0, count) - iter, err := set.txn.t.Iter(prefix, endPrefix) - if err != nil { - return nil, err - } - for iter.Valid() && iter.Key().HasPrefix(prefix) && count != 0 { - members = append(members, iter.Key()[len(prefix):]) - if err := iter.Next(); err != nil { - return nil, err - } - count-- - } - return members, nil -} - -// SCard returns the set cardinality (number of elements) of the set stored at key -func (set *Set) SCard() (int64, error) { - if !set.Exists() { - return 0, nil - } - return set.meta.Len, nil -} - -// SIsmember returns if member is a member of the set stored at key -func (set *Set) SIsmember(member []byte) (int64, error) { - if !set.Exists() { - return 0, nil - } - dkey := DataKey(set.txn.db, set.meta.ID) - ikey := setItemKey(dkey, member) - - value, err := set.txn.t.Get(ikey) - if err != nil { - if IsErrNotFound(err) { - return 0, nil - } - return 0, err - } - if !bytes.Equal(value, SetNilValue) { - return 0, ErrSetNilValue - } - return 1, nil -} - -// SPop removes and returns one or more random elements from the set value store at key. -func (set *Set) SPop(count int64) ([][]byte, error) { - if !set.Exists() || set.meta.Len == 0 { - return make([][]byte, 0), nil - } - dkey := DataKey(set.txn.db, set.meta.ID) - prefix := append(dkey, ':') - endPrefix := kv.Key(prefix).PrefixNext() - iter, err := set.txn.t.Iter(prefix, endPrefix) - if err != nil { - return nil, err - } - defer iter.Close() - var deleted int64 - var members [][]byte - for iter.Valid() && iter.Key().HasPrefix(prefix) && count != 0 { - members = append(members, iter.Key()[len(prefix):]) - if err := set.txn.t.Delete([]byte(iter.Key())); err != nil { - return nil, err - } - deleted++ - count-- - if err := iter.Next(); err != nil { - return nil, err - } - } - set.meta.Len -= deleted - if err := set.updateMeta(); err != nil { - return nil, err - } - return members, nil -} - -// SRem removes the specified members from the set stored at key -func (set *Set) SRem(members [][]byte) (int64, error) { - var num int64 - if !set.Exists() { - return 0, nil - } - dkey := DataKey(set.txn.db, set.meta.ID) - ms := RemoveRepByMap(members) - ikeys := make([][]byte, len(ms)) - for i := range ms { - ikeys[i] = setItemKey(dkey, ms[i]) - value, err := set.txn.t.Get(ikeys[i]) - if err != nil { - if IsErrNotFound(err) { - continue - } - return 0, err - } - if bytes.Equal(value, SetNilValue) { - if err := set.txn.t.Delete([]byte(ikeys[i])); err != nil { - return 0, err - } - num++ - } - } - set.meta.Len -= num - if err := set.updateMeta(); err != nil { - return 0, err - } - return num, nil -} - -// SMove movies member from the set at source to the set at destination -func (set *Set) SMove(destination []byte, member []byte) (int64, error) { - if !set.Exists() { - return 0, nil - } - res, err := set.SIsmember(member) - if err != nil { - return 0, err - } - if res == 0 { - return 0, nil - } - destset, err := GetSet(set.txn, destination) - if err != nil { - return 0, nil - } - - res, err = destset.SIsmember(member) - if err != nil { - return 0, err - } - if res == 0 { - if _, err := destset.SAdd(member); err != nil { - return 0, err - } - destset.meta.Len++ - } - dkey := DataKey(set.txn.db, set.meta.ID) - ikey := setItemKey(dkey, member) - if err := set.txn.t.Delete([]byte(ikey)); err != nil { - return 0, err - } - set.meta.Len-- - if err := set.updateMeta(); err != nil { - return 0, err - } - return 1, nil -} diff --git a/db/set_test.go b/db/set_test.go deleted file mode 100644 index 58b64f3..0000000 --- a/db/set_test.go +++ /dev/null @@ -1,653 +0,0 @@ -package db - -import ( - "bytes" - "context" - "fmt" - "reflect" - "testing" - - "github.com/stretchr/testify/assert" -) - -// compareSet skip CreatedAt UpdatedAt ID compare -func compareSet(want, got *Set) error { - switch { - case !bytes.Equal(want.key, got.key): - return fmt.Errorf("set key not equal, want=%s, got=%s", string(want.key), string(got.key)) - case want.meta.ExpireAt != got.meta.ExpireAt: - return fmt.Errorf("meta.ExpireAt not equal, want=%v, got=%v", want.meta.ExpireAt, got.meta.ExpireAt) - case want.meta.Type != got.meta.Type: - return fmt.Errorf("meta.Type not equal, want=%v, got=%v", want.meta.Type, got.meta.Type) - case want.meta.Encoding != got.meta.Encoding: - return fmt.Errorf("meta.Encoding not equal, want=%v, got=%v", want.meta.Encoding, got.meta.Encoding) - case want.meta.Len != got.meta.Len: - return fmt.Errorf("meta.Len not equal, want=%v, got=%v", want.meta.Len, got.meta.Len) - case want.exists != got.exists: - return fmt.Errorf("exists not equal, want=%v, get=%v", want.exists, got.exists) - } - return nil -} - -func testAddData(t *testing.T, key []byte, values ...[]byte) { - var txn *Transaction - var err error - var set *Set - if txn, err = mockDB.Begin(); err != nil { - t.Errorf("TestGetSet db.Begin error %s", err) - } - if set, err = GetSet(txn, key); err != nil { - t.Errorf("Set.SAdd() error = %v", err) - } - _, err = set.SAdd(values...) - if err != nil { - t.Errorf("Set.SAdd() error = %v", err) - return - } - if err = txn.Commit(context.TODO()); err != nil { - t.Errorf("Set.SAdd() txn.Commit error = %v", err) - return - } -} - -func setSetMeta(t *testing.T, txn *Transaction, key []byte) error { - h := newSet(txn, key) - mkey := MetaKey(txn.db, key) - sm := &SetMeta{ - Object: h.meta.Object, - Len: 1, - } - meta := encodeSetMeta(sm) - err := txn.t.Set(mkey, meta) - assert.NoError(t, err) - assert.NotNil(t, txn) - return nil -} -func destorySetMeta(t *testing.T, txn *Transaction, key []byte) error { - mkey := MetaKey(txn.db, key) - if err := txn.t.Delete(mkey); err != nil { - return err - } - return nil -} -func Test_newSet(t *testing.T) { - txn, err := mockDB.Begin() - assert.NotNil(t, txn) - assert.NoError(t, err) - type args struct { - txn *Transaction - key []byte - } - tests := []struct { - name string - args args - want *Set - }{ - { - name: "TestNewSet", - args: args{ - txn: txn, - key: []byte("TestNewSet"), - }, - want: &Set{ - meta: &SetMeta{ - Object: Object{ - ExpireAt: 0, - Type: ObjectSet, - Encoding: ObjectEncodingHT, - }, - Len: 0, - }, - key: []byte("TestNewSet"), - txn: txn, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - txn, err := mockDB.Begin() - assert.NotNil(t, txn) - assert.NoError(t, err) - got := newSet(tt.args.txn, tt.args.key) - txn.Commit(context.TODO()) - if compareSet(tt.want, got); err != nil { - t.Errorf("newSet() = %v, want %v", got, tt.want) - } - }) - } - txn.Commit(context.TODO()) -} - -func TestGetSet(t *testing.T) { - var testNotExistSetKey = []byte("not_exist_key") - var testExistSetKey = []byte("exist_key") - - txn, err := mockDB.Begin() - assert.NotNil(t, txn) - assert.NoError(t, err) - setSetMeta(t, txn, testExistSetKey) - type args struct { - txn *Transaction - key []byte - } - type want struct { - set *Set - err *error - } - tests := []struct { - name string - args args - want want - }{ - { - name: "not exist set", - args: args{ - txn: txn, - key: testNotExistSetKey, - }, - want: want{ - set: &Set{ - key: testNotExistSetKey, - meta: &SetMeta{ - Object: Object{ - ExpireAt: 0, - Type: ObjectSet, - Encoding: ObjectEncodingHT, - }, - Len: 0, - }, - exists: false, - txn: txn, - }, - err: nil, - }, - }, - { - name: "exist set", - args: args{ - txn: txn, - key: testExistSetKey, - }, - want: want{ - set: &Set{ - key: testExistSetKey, - meta: &SetMeta{ - Object: Object{ - ExpireAt: 0, - Type: ObjectSet, - Encoding: ObjectEncodingHT, - }, - Len: 1, - }, - exists: true, - txn: txn, - }, - err: nil, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - txn, err := mockDB.Begin() - assert.NotNil(t, txn) - assert.NoError(t, err) - got, err := GetSet(tt.args.txn, tt.args.key) - if err = txn.Commit(context.TODO()); err != nil { - t.Errorf("GetString() txn.Commit error = %v", err) - return - } - if err := compareSet(tt.want.set, got); err != nil { - t.Errorf("GetSet() = %v, want %v", got, tt.want) - } - }) - } - destorySetMeta(t, txn, testExistSetKey) - txn.Commit(context.TODO()) -} - -func TestSet_SAdd(t *testing.T) { - var testSetSAddKey = []byte("set_sadd_key") - tests := []struct { - name string - key []byte - members [][]byte - want int64 - }{ - { - name: "empty", - key: testSetSAddKey, - members: [][]byte{[]byte("value1")}, - want: 1, - }, - { - name: "duplicate", - key: testSetSAddKey, - members: [][]byte{[]byte("value1")}, - want: 0, - }, - { - name: "set_mutil", - key: testSetSAddKey, - members: [][]byte{[]byte("value2"), []byte("value3"), []byte("value4")}, - want: 3, - }, - { - name: "set_duplicate_some", - key: testSetSAddKey, - members: [][]byte{[]byte("value4"), []byte("value5"), []byte("value6")}, - want: 2, - }, - { - name: "set_duplicate_some", - key: testSetSAddKey, - members: [][]byte{[]byte("value4"), []byte("value4"), []byte("value4"), []byte("value5"), []byte("value6")}, - want: 0, - }, - { - name: "set_duplicate_some", - key: testSetSAddKey, - members: [][]byte{[]byte("value7"), []byte("value7"), []byte("value8")}, - want: 2, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - txn, err := mockDB.Begin() - assert.NoError(t, err) - assert.NotNil(t, txn) - - set, err := GetSet(txn, tt.key) - assert.NoError(t, err) - assert.NotNil(t, set) - - got, err := set.SAdd(tt.members...) - assert.NoError(t, err) - assert.NotNil(t, got) - - txn.Commit(context.TODO()) - - assert.Equal(t, got, tt.want) - }) - } -} - -func TestSet_SMembers(t *testing.T) { - var testSetSMembersKeyEmpty = []byte("set_key_empty") - var testSetSMembersKeyOne = []byte("set_key_one") - var testSetSMembersKeyTwo = []byte("set_key_two") - testAddData(t, testSetSMembersKeyOne, []byte("value1")) - testAddData(t, testSetSMembersKeyTwo, []byte("value1"), []byte("value2")) - tests := []struct { - name string - key []byte - want [][]byte - }{ - { - name: "empty", - key: testSetSMembersKeyEmpty, - want: [][]byte{}, - }, - { - name: "one", - key: testSetSMembersKeyOne, - want: [][]byte{[]byte("value1")}, - }, - { - name: "two", - key: testSetSMembersKeyTwo, - want: [][]byte{[]byte("value1"), []byte("value2")}, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - txn, err := mockDB.Begin() - assert.NoError(t, err) - assert.NotNil(t, txn) - - set, err := GetSet(txn, tt.key) - assert.NoError(t, err) - assert.NotNil(t, set) - - got, err := set.SMembers() - assert.NoError(t, err) - - txn.Commit(context.TODO()) - - assert.Equal(t, len(got), len(tt.want)) - - for i := range got { - assert.Equal(t, got[i], tt.want[i]) - } - }) - } -} - -func TestSet_SCard(t *testing.T) { - var testSetSCardKey = []byte("SCardKey") - testAddData(t, testSetSCardKey, []byte("ExistsValue1")) - testAddData(t, testSetSCardKey, []byte("ExistsValue2")) - tests := []struct { - name string - key []byte - want int64 - }{ - { - name: "SCardKey", - key: testSetSCardKey, - want: 2, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - txn, err := mockDB.Begin() - assert.NotNil(t, txn) - assert.NoError(t, err) - - set, err := GetSet(txn, tt.key) - assert.NoError(t, err) - assert.NotNil(t, set) - - got, err := set.SCard() - assert.NotNil(t, got) - assert.NoError(t, err) - - if err = txn.Commit(context.TODO()); err != nil { - t.Errorf("SCard() txn.Commit error = %v", err) - return - } - assert.Equal(t, got, tt.want) - }) - } -} - -func TestSet_SIsmember(t *testing.T) { - var testSetSIsMembersKey = []byte("SIsmemberKey") - testAddData(t, testSetSIsMembersKey, []byte("ExistsValue")) - type args struct { - member []byte - } - tests := []struct { - name string - key []byte - args args - want int64 - }{ - { - name: "testExistMember", - key: testSetSIsMembersKey, - args: args{ - member: []byte("ExistsValue"), - }, - want: 1, - }, - { - name: "testExistMember", - key: testSetSIsMembersKey, - args: args{ - member: []byte("NoExistsValue"), - }, - want: 0, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - txn, err := mockDB.Begin() - assert.NotNil(t, txn) - assert.NoError(t, err) - - set, err := GetSet(txn, tt.key) - assert.NoError(t, err) - assert.NotNil(t, set) - - got, err := set.SIsmember(tt.args.member) - assert.NotNil(t, got) - assert.NoError(t, err) - - if err = txn.Commit(context.TODO()); err != nil { - t.Errorf("SIsmember() txn.Commit error = %v", err) - return - } - assert.Equal(t, got, tt.want) - }) - } -} - -func TestSet_SPop(t *testing.T) { - var testSPopKey = []byte("SPopKey") - testAddData(t, testSPopKey, []byte("1"), []byte("2"), []byte("3"), []byte("4"), []byte("5")) - type args struct { - count int64 - } - tests := []struct { - name string - key []byte - args args - wantMembersCount int64 - }{ - { - name: "TestSPopZero", - key: testSPopKey, - args: args{ - count: 1, - }, - wantMembersCount: 1, - }, - { - name: "TestSPopNotZero", - key: testSPopKey, - args: args{ - count: 2, - }, - wantMembersCount: 2, - }, - { - name: "TestSPopBigCount", - key: testSPopKey, - args: args{ - count: 6, - }, - wantMembersCount: 2, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - txn, err := mockDB.Begin() - assert.NotNil(t, txn) - assert.NoError(t, err) - - set, err := GetSet(txn, tt.key) - assert.NoError(t, err) - assert.NotNil(t, set) - - gotMembers, err := set.SPop(tt.args.count) - assert.NoError(t, err) - assert.NotNil(t, gotMembers) - - if err = txn.Commit(context.TODO()); err != nil { - t.Errorf("SPop() txn.Commit error = %v", err) - return - } - assert.Equal(t, int64(len(gotMembers)), tt.wantMembersCount) - }) - } -} - -func TestSet_SRem(t *testing.T) { - var testSRemKey = []byte("testSRemKey") - testAddData(t, testSRemKey, []byte("1"), []byte("2"), []byte("3"), []byte("4"), []byte("5")) - type args struct { - members [][]byte - } - m1 := [][]byte{[]byte("1")} - m2 := [][]byte{[]byte("4"), []byte("2"), []byte("3")} - tests := []struct { - name string - key []byte - args args - want int64 - }{ - { - name: "testSRemOneMember", - key: testSRemKey, - args: args{ - members: m1, - }, - want: 1, - }, - { - name: "testSRemMoreMember", - key: testSRemKey, - args: args{ - members: m2, - }, - want: 3, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - txn, err := mockDB.Begin() - assert.NotNil(t, txn) - assert.NoError(t, err) - - set, err := GetSet(txn, tt.key) - assert.NoError(t, err) - assert.NotNil(t, set) - - got, err := set.SRem(tt.args.members) - assert.NoError(t, err) - assert.NotNil(t, got) - - if err = txn.Commit(context.TODO()); err != nil { - t.Errorf("SRem() txn.Commit error = %v", err) - return - } - assert.Equal(t, got, tt.want) - }) - } -} - -func TestRemoveRepByMap(t *testing.T) { - type args struct { - members [][]byte - } - tests := []struct { - name string - args args - want [][]byte - }{ - { - name: "single member", - args: args{ - members : [][]byte{[]byte("value1")}, - }, - want: [][]byte{[]byte("value1")}, - }, - { - name: "multi members", - args: args{ - members : [][]byte{[]byte("value1"),[]byte("value2"),[]byte("value3")}, - }, - want: [][]byte{[]byte("value1"),[]byte("value2"),[]byte("value3")}, - }, - { - name: "with duplicate members", - args: args{ - members : [][]byte{[]byte("value1"),[]byte("value2"),[]byte("value1")}, - }, - want: [][]byte{[]byte("value1"),[]byte("value2")}, - }, - - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := RemoveRepByMap(tt.args.members); !reflect.DeepEqual(got, tt.want) { - t.Errorf("RemoveRepByMap() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestSet_SMove(t *testing.T) { - var testSMoveSouceKey = []byte("testSMoveSouceKey") - var testSMoveDestinationKey = []byte("testSMoveDestinationKey") - testAddData(t, testSMoveSouceKey, []byte("1"), []byte("2"), []byte("3"), []byte("4"), []byte("5")) - testAddData(t, testSMoveDestinationKey, []byte("3"), []byte("4"), []byte("5")) - m1 := []byte("2") - m2 := []byte("6") - m3 := []byte("3") - type args struct { - destination []byte - member []byte - } - tests := []struct { - name string - key []byte - args args - mres int64 - want int64 - }{ - { - name: "TestSet_SMove1", - key: testSMoveSouceKey, - args: args{ - destination: testSMoveDestinationKey, - member: m1, - }, - mres: int64(1), - want: 1, - }, - { - name: "TestSet_SMove2", - key: testSMoveSouceKey, - args: args{ - destination: testSMoveDestinationKey, - member: m2, - }, - mres: int64(0), - want: 0, - }, - { - name: "TestSet_SMove3", - key: testSMoveSouceKey, - args: args{ - destination: testSMoveDestinationKey, - member: m3, - }, - mres: int64(1), - want: 1, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - txn, err := mockDB.Begin() - assert.NotNil(t, txn) - assert.NoError(t, err) - - set, err := GetSet(txn, tt.key) - assert.NoError(t, err) - assert.NotNil(t, set) - - destset, err := GetSet(txn, tt.args.destination) - assert.NoError(t, err) - assert.NotNil(t, destset) - - got, err := set.SMove(tt.args.destination, tt.args.member) - assert.NoError(t, err) - assert.NotNil(t, got) - if err = txn.Commit(context.TODO()); err != nil { - t.Errorf("SMove() txn.Commit error = %v", err) - return - } - iss, err := set.SIsmember(tt.args.member) - assert.NoError(t, err) - assert.Equal(t, iss, int64(0)) - - isdest, err := destset.SIsmember(tt.args.member) - assert.NoError(t, err) - assert.Equal(t, isdest, tt.mres) - - assert.Equal(t, got, tt.want) - - }) - } -} diff --git a/db/store/mock_store.go b/db/store/mock_store.go deleted file mode 100644 index 5c6428c..0000000 --- a/db/store/mock_store.go +++ /dev/null @@ -1,12 +0,0 @@ -package store - -import "github.com/pingcap/tidb/store/mockstore" - -//MockAddr default mock tikv addr -var MockAddr = "mocktikv://" - -// MockOpen create fake tikv db -func MockOpen(addrs string) (r Storage, e error) { - var driver mockstore.MockDriver - return driver.Open(MockAddr) -} diff --git a/db/store/store.go b/db/store/store.go deleted file mode 100644 index b1495c4..0000000 --- a/db/store/store.go +++ /dev/null @@ -1,105 +0,0 @@ -package store - -import ( - "context" - "strings" - "unsafe" - - "github.com/pingcap/tidb/kv" - "github.com/pingcap/tidb/store/tikv" -) - -// Transaction options -const ( - // PresumeKeyNotExists indicates that when dealing with a Get operation but failing to read data from cache, - // we presume that the key does not exist in Store. The actual existence will be checked before the - // transaction's commit. - // This option is an optimization for frequent checks during a transaction, e.g. batch inserts. - PresumeKeyNotExists Option = iota + 1 - // PresumeKeyNotExistsError is the option key for error. - // When PresumeKeyNotExists is set and condition is not match, should throw the error. - PresumeKeyNotExistsError - // BinlogInfo contains the binlog data and client. - BinlogInfo - // SchemaChecker is used for checking schema-validity. - SchemaChecker - // IsolationLevel sets isolation level for current transaction. The default level is SI. - IsolationLevel - // Priority marks the priority of this transaction. - Priority - // NotFillCache makes this request do not touch the LRU cache of the underlying storage. - NotFillCache - // SyncLog decides whether the WAL(write-ahead log) of this request should be synchronized. - SyncLog - // KeyOnly retrieve only keys, it can be used in scan now. - KeyOnly -) - -// Priority value for transaction priority. -const ( - PriorityNormal = iota - PriorityLow - PriorityHigh -) - -//type rename tidb kv type -type ( - // Storage defines the interface for storage. - Storage kv.Storage - // Transaction defines the interface for operations inside a Transaction. - Transaction kv.Transaction - // Iterator is the interface for a iterator on KV store. - Iterator kv.Iterator - // Option is used for customizing kv store's behaviors during a transaction. - Option kv.Option -) - -//Open create tikv db ,create fake db if addr contains mockaddr -func Open(addrs string) (r Storage, e error) { - if strings.Contains(addrs, MockAddr) { - return MockOpen(addrs) - } - return tikv.Driver{}.Open(addrs) -} - -// IsErrNotFound checks if err is a kind of NotFound error. -func IsErrNotFound(err error) bool { - return kv.IsErrNotFound(err) -} - -// IsRetryableError checks if err is a kind of RetryableError error. -func IsRetryableError(err error) bool { - return kv.IsTxnRetryableError(err) -} - -func IsConflictError(err error) bool { - return kv.ErrWriteConflict.Equal(err) - -} - -func RunInNewTxn(store Storage, retryable bool, f func(txn kv.Transaction) error) error { - return kv.RunInNewTxn(store, retryable, f) -} - -// LockKeys tries to lock the entries with the keys in KV store. -func LockKeys(txn Transaction, keys [][]byte) error { - kvKeys := make([]kv.Key, len(keys)) - for i := range keys { - kvKeys[i] = kv.Key(keys[i]) - } - return txn.LockKeys(context.Background(), &kv.LockCtx{}, kvKeys...) -} - -// BatchGetValues issue batch requests to get values -func BatchGetValues(txn Transaction, keys [][]byte) (map[string][]byte, error) { - kvkeys := *(*[]kv.Key)(unsafe.Pointer(&keys)) - return txn.BatchGet(kvkeys) -} - -func SetOption(txn Transaction, opt Option, val interface{}) { - txn.SetOption(kv.Option(opt), val) -} - -func DelOption(txn Transaction, opt Option) { - txn.DelOption(kv.Option(opt)) -} diff --git a/db/string.go b/db/string.go deleted file mode 100644 index c816baf..0000000 --- a/db/string.go +++ /dev/null @@ -1,296 +0,0 @@ -package db - -import ( - "strconv" - - "go.uber.org/zap" -) - -//StringMeta string meta msg -type StringMeta struct { - Object - Value []byte -} - -// String object operate tikv -type String struct { - Meta StringMeta - key []byte - txn *Transaction -} - -// GetString return string object , -// if key is exist , object load meta -// otherwise object is null if key is not exist and err is not found -// otherwise return err -func GetString(txn *Transaction, key []byte) (*String, error) { - str := NewString(txn, key) - mkey := MetaKey(txn.db, key) - now := Now() - Meta, err := txn.t.Get(mkey) - if err != nil { - if IsErrNotFound(err) { - return str, nil - } - return nil, err - } - if err := str.decode(Meta); err != nil && err != ErrKeyNotFound { - return nil, err - } - - str.Meta.UpdatedAt = now - return str, nil -} - -// NewString create new string object -func NewString(txn *Transaction, key []byte) *String { - str := &String{txn: txn, key: key} - now := Now() - str.Meta.CreatedAt = now - str.Meta.UpdatedAt = now - str.Meta.ExpireAt = 0 - str.Meta.ID = UUID() - str.Meta.Type = ObjectString - str.Meta.Encoding = ObjectEncodingRaw - return str -} - -// Get the value information for the key from db -func (s *String) Get() ([]byte, error) { - if !s.Exist() { - return nil, ErrKeyNotFound - } - return s.Meta.Value, nil -} - -// Set set the string value of a key -// the num of expire slice is not zero and expire[0] is not zero ,the key add exprie queue -// otherwise the delete expire queue -func (s *String) Set(val []byte, expire ...int64) error { - timestamp := Now() - mkey := MetaKey(s.txn.db, s.key) - if len(expire) != 0 && expire[0] > 0 { - old := s.Meta.ExpireAt - s.Meta.ExpireAt = timestamp + expire[0] - if err := expireAt(s.txn.t, mkey, s.Meta.ID, s.Meta.Type, old, s.Meta.ExpireAt); err != nil { - return err - } - } else { - //maybe key is not expire queue,so unExpireAt will return err,but it is not relationship - if err := unExpireAt(s.txn.t, mkey, s.Meta.ExpireAt); err != nil { - zap.L().Error("add expire failed", zap.String("key", string(mkey))) - } - s.Meta.ExpireAt = 0 - } - s.Meta.Value = val - return s.txn.t.Set(mkey, s.encode()) -} - -// Len value len -func (s *String) Len() (int, error) { - return len(s.Meta.Value), nil -} - -// Exist returns ture if key exist -func (s *String) Exist() bool { - return s.Meta.Value != nil -} - -// Append appends a value to key -func (s *String) Append(value []byte) (int, error) { - s.Meta.Value = append(s.Meta.Value, value...) - if err := s.txn.t.Set(MetaKey(s.txn.db, s.key), s.encode()); err != nil { - return 0, err - } - return len(s.Meta.Value), nil -} - -// GetSet returns old value ,value replace old value -func (s *String) GetSet(value []byte) ([]byte, error) { - v := s.Meta.Value - if err := s.Set(value); err != nil { - return nil, err - } - return v, nil -} - -// GetRange returns string from the absolute of start to the absolute of end -func (s *String) GetRange(start, end int) []byte { - vlen := len(s.Meta.Value) - if end < 0 { - end = vlen + end - } - if start < 0 { - start = vlen + start - } - if start > end || start > vlen || end < 0 { - return nil - } - if end > vlen { - end = vlen - 1 - } - if start < 0 { - start = 0 - } - return s.Meta.Value[start : end+1] -} - -// SetRange overwrites part of the string stored at key, starting at the specified offset, for the entire length of value. -func (s *String) SetRange(offset int64, value []byte) ([]byte, error) { - val := s.Meta.Value - if int64(len(val)) < offset+int64(len(value)) { - val = append(val, make([]byte, offset+int64(len(value))-int64(len(val)))...) - } - copy(val[offset:], value) - if err := s.Set(val); err != nil { - return nil, err - } - - return val, nil -} - -// Incr increments the integer value by the given amount -// the old value must be integer -func (s *String) Incr(delta int64) (int64, error) { - value := s.Meta.Value - if value != nil { - v, err := strconv.ParseInt(string(value), 10, 64) - if err != nil { - return 0, ErrInteger - } - delta = v + delta - } - - vs := strconv.FormatInt(delta, 10) - if err := s.Set([]byte(vs)); err != nil { - return 0, err - } - return delta, nil - -} - -// Incrf increments the float value by the given amount -// the old value must be float -func (s *String) Incrf(delta float64) (float64, error) { - value := s.Meta.Value - if value != nil { - v, err := strconv.ParseFloat(string(value), 64) - if err != nil { - return 0, ErrInteger - } - delta = v + delta - } - - vs := strconv.FormatFloat(delta, 'e', -1, 64) - if err := s.Set([]byte(vs)); err != nil { - return 0, err - } - return delta, nil -} - -// SetBit key offset bitvalue -// return the off postion of value -func (s *String) SetBit(offset, on int) (int, error) { - val := s.Meta.Value - bitoff := offset >> 3 - llen := int(bitoff) - len(val) + 1 - if llen > 0 { - val = append(val, make([]byte, llen)...) - } - - /* Get current values */ - byteval := int(val[bitoff]) - bit := uint(7 - (offset & 0x7)) - bitval := byteval & (1 << bit) - - /* Update byte with new bit value and return original value */ - byteval &= (^(1 << bit)) - byteval = byteval | ((on & 0x1) << bit) - val[bitoff] = byte(byteval) - if err := s.Set(val); err != nil { - return 0, err - } - return bitval, nil -} - -// GetBit key offset bitvalye -// offset / 8 > the index of value -// offset mod 8 +1 -func (s *String) GetBit(offset int) (int, error) { - val := s.Meta.Value - bitoff := offset >> 3 - if int(bitoff) > len(val)-1 { - return 0, nil - } - - /* Get current values */ - byteval := int(val[bitoff]) - bit := uint(7 - (offset & 0x7)) - bitval := byteval & (1 << bit) - - return bitval, nil -} - -// BitCount counts the number of set bits (population counting) in a string. -func (s *String) BitCount(begin, end int) (int, error) { - begin, end = initCursor(begin, end, len(s.Meta.Value)) - if begin > end { - return 0, nil - } - return redisPopcount(s.Meta.Value[begin : end+1]), nil -} - -// BitPos finds first bit set or clear in a string -func (s *String) BitPos(bit, begin, end int) (int, error) { - begin, end = initCursor(begin, end, len(s.Meta.Value)) - // For empty ranges (start > end) we return -1 as an empty range does - // not contain a 0 nor a 1. - if begin > end { - return -1, nil - } - return redisBitpos(s.Meta.Value[begin:end+1], bit), nil -} - -func (s *String) BitOpAnd() { -} - -func (s *String) BitOpOr() { -} - -func (s *String) BitOpXor() { -} - -func (s *String) BitOpNot() { -} - -// encode because of the value is small size , value and meta decode together -func (s *String) encode() []byte { - b := EncodeObject(&s.Meta.Object) - b = append(b, s.Meta.Value...) - return b -} - -// decode if obj has been existed , stop parse -func (s *String) decode(b []byte) error { - obj, err := DecodeObject(b) - if err != nil { - return err - } - - if IsExpired(obj, Now()) { - return ErrKeyNotFound - } - - if obj.Type != ObjectString { - return ErrTypeMismatch - } - - if obj.Encoding != ObjectEncodingRaw { - return ErrTypeMismatch - } - s.Meta.Object = *obj - if len(b) >= ObjectEncodingLength { - s.Meta.Value = b[ObjectEncodingLength:] - } - return nil -} diff --git a/db/string_test.go b/db/string_test.go deleted file mode 100644 index c1f0e86..0000000 --- a/db/string_test.go +++ /dev/null @@ -1,809 +0,0 @@ -package db - -import ( - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -// compareGetString skip CreatedAt UpdatedAt ID compare -func compareGetString(t *testing.T, want, get *String) { - assert.Equal(t, want.key, get.key) - assert.Equal(t, want.Meta.ExpireAt, get.Meta.ExpireAt) - assert.Equal(t, want.Meta.Type, get.Meta.Type) - assert.Equal(t, want.Meta.Encoding, get.Meta.Encoding) - assert.Equal(t, want.Meta.Value, get.Meta.Value) -} - -func setValue(t *testing.T, TestKey []byte, value []byte) { - callFunc := func(txn *Transaction) { - s, err := GetString(txn, TestKey) - assert.NoError(t, err) - err = s.Set(value) - } - MockTest(t, callFunc) -} - -func getValue(t *testing.T, key []byte, value []byte) { - callFunc := func(txn *Transaction) { - s, err := GetString(txn, key) - assert.NoError(t, err) - val, err := s.Get() - assert.NoError(t, err) - assert.Equal(t, value, val) - } - MockTest(t, callFunc) -} - -var ( - TestExistKey = []byte("StringKey") - TestExpireExistKey = []byte("ExpireStringKey") - TestNoExistKey = []byte("NoExitStringKey") - - value = []byte("StringValue") - - NoExist = "No Exist Key" - Exist = "Exist Key" -) - -func TestGetString(t *testing.T) { - txn, err := mockDB.Begin() - assert.NoError(t, err) - NewString(txn, TestExistKey) - type args struct { - txn *Transaction - key []byte - } - type want struct { - key *String - val []byte - } - tests := []struct { - name string - args args - want want - }{ - { - name: "GetExistString", - args: args{ - txn: txn, - key: TestExistKey, - }, - want: want{ - key: &String{ - key: TestExistKey, - txn: txn, - }, - val: value, - }, - }, - { - name: "GetNoExistString", - args: args{ - txn: txn, - key: TestNoExistKey, - }, - want: want{ - key: &String{ - key: TestNoExistKey, - txn: txn, - }, - val: nil, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - callFunc := func(txn *Transaction) { - got, err := GetString(tt.args.txn, tt.args.key) - assert.NoError(t, err) - compareGetString(t, tt.want.key, got) - } - MockTest(t, callFunc) - }) - } -} - -func TestStringGet(t *testing.T) { - setValue(t, TestExistKey, value) - tests := []struct { - name string - key []byte - value []byte - want []byte - }{ - { - name: "TestString_GetExitKey", - key: TestExistKey, - value: value, - want: value, - }, - { - name: "TestString_GetNoExitKey", - key: TestNoExistKey, - value: nil, - want: nil, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - callFunc := func(txn *Transaction) { - s, err := GetString(txn, tt.key) - assert.NoError(t, err) - got, err := s.Get() - assert.Equal(t, got, tt.want) - } - MockTest(t, callFunc) - }) - } -} - -func TestStringSet(t *testing.T) { - var exp = []int64{int64(2 * time.Second)} - type args struct { - val []byte - expire []int64 - } - type want struct { - val []byte - err error - } - tests := []struct { - name string - key []byte - args args - want want - }{ - { - name: "set no expire", - key: TestExistKey, - args: args{ - val: value, - expire: nil, - }, - want: want{ - val: value, - err: nil, - }, - }, - { - name: "set expire", - key: TestExpireExistKey, - args: args{ - val: value, - expire: exp, - }, - want: want{ - val: nil, - err: ErrKeyNotFound, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - callFunc := func(txn *Transaction) { - s, err := GetString(txn, tt.key) - assert.NoError(t, err) - err = s.Set(tt.args.val, tt.args.expire...) - assert.NoError(t, err) - } - MockTest(t, callFunc) - - if len(tt.args.expire) != 0 { - callFunc = func(txn *Transaction) { - s, err := GetString(txn, tt.key) - assert.NoError(t, err) - val, err := s.Get() - assert.NoError(t, err) - assert.Equal(t, tt.args.val, val) - } - MockTest(t, callFunc) - time.Sleep(3 * time.Second) - } - callFunc = func(txn *Transaction) { - s, err := GetString(txn, tt.key) - assert.NoError(t, err) - val, err := s.Get() - assert.Equal(t, tt.want.err, err) - assert.Equal(t, tt.want.val, val) - } - MockTest(t, callFunc) - }) - } -} - -func TestStringLen(t *testing.T) { - setValue(t, TestExistKey, value) - tests := []struct { - name string - key []byte - want int - }{ - { - name: "TestString_Len", - key: TestExistKey, - want: 11, - }, - { - name: "TestString_Len", - key: TestNoExistKey, - want: 0, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - callFunc := func(txn *Transaction) { - s, err := GetString(txn, tt.key) - assert.NoError(t, err) - got, err := s.Len() - assert.Equal(t, tt.want, got) - } - MockTest(t, callFunc) - }) - } -} - -func TestStringExist(t *testing.T) { - setValue(t, TestExistKey, value) - // write exist string - tests := []struct { - name string - key []byte - want bool - }{ - { - name: "Exist", - key: TestExistKey, - want: true, - }, - { - name: "NoExist", - key: TestNoExistKey, - want: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - callFunc := func(txn *Transaction) { - s, err := GetString(txn, tt.key) - assert.NoError(t, err) - got := s.Exist() - assert.Equal(t, tt.want, got) - } - MockTest(t, callFunc) - }) - } - -} - -func TestStringAppend(t *testing.T) { - setValue(t, TestExistKey, value) - type args struct { - value []byte - } - tests := []struct { - name string - args args - key []byte - want int - }{ - { - name: "TestString_AppendExistKey", - args: args{ - value: value, - }, - key: TestExistKey, - want: 22, - }, - { - name: "TestString_AppendNoExistKey", - args: args{ - value: value, - }, - key: TestNoExistKey, - want: 11, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - callFunc := func(txn *Transaction) { - s, err := GetString(txn, tt.key) - assert.NoError(t, err) - got, err := s.Append(tt.args.value) - assert.Equal(t, tt.want, got) - } - MockTest(t, callFunc) - }) - } -} - -func TestStringGetSet(t *testing.T) { - setValue(t, []byte("GetSetExistKey"), value) - type args struct { - value []byte - } - tests := []struct { - name string - args args - key []byte - want []byte - }{ - { - name: "GetSet_ExistKey", - args: args{ - value: []byte("NewVlaue"), - }, - key: []byte("GetSetExistKey"), - want: value, - }, - { - name: "GetSet_NoExistKey", - args: args{ - value: []byte("NewVlaue"), - }, - key: []byte("GetSetNoExistKey"), - want: nil, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - callFunc := func(txn *Transaction) { - s, err := GetString(txn, tt.key) - assert.NoError(t, err) - got, err := s.GetSet(tt.args.value) - assert.NoError(t, err) - assert.Equal(t, tt.want, got) - value, err := s.Get() - assert.NoError(t, err) - assert.Equal(t, value, []byte("NewVlaue")) - } - MockTest(t, callFunc) - }) - } -} - -func TestStringGetRange(t *testing.T) { - type args struct { - start int - end int - } - tests := []struct { - name string - args args - want []byte - }{ - { - name: "GetRange1", - //"StringValue" - args: args{ - start: 4, - end: -1, - }, - want: []byte("ngValue"), - }, - - { - name: "GetRange2", - args: args{ - start: 4, - end: -20, - }, - want: nil, - }, - - { - name: "GetRange3", - args: args{ - start: -1, - end: 3, - }, - want: nil, - }, - { - name: "GetRange4", - args: args{ - start: 22, - end: 3, - }, - want: nil, - }, - - { - name: "GetRange5", - args: args{ - start: 3, - end: 20, - }, - want: []byte("ingValue"), - }, - { - name: "GetRange6", - args: args{ - start: -22, - end: 3, - }, - want: []byte("Stri"), - }, - { - name: "GetRange7", - args: args{ - start: 2, - end: 10, - }, - want: []byte("ringValue"), - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - callFunc := func(txn *Transaction) { - s, err := GetString(txn, []byte("GetRangeExistKey")) - assert.NoError(t, err) - err = s.Set(value) - assert.NoError(t, err) - got := s.GetRange(tt.args.start, tt.args.end) - assert.Equal(t, tt.want, got) - } - MockTest(t, callFunc) - }) - } -} - -func TestStringSetRange(t *testing.T) { - setValue(t, []byte("SetRangeExistKey"), value) - type args struct { - offset int64 - value []byte - } - - tests := []struct { - name string - args args - key []byte - want []byte - }{ - { - name: "SetRange_ExistKey", - args: args{ - offset: 6, - value: []byte("lllll"), - }, - key: []byte("SetRangeExistKey"), - want: []byte("Stringlllll"), - }, - { - name: "SetRange_NoExistKey", - args: args{ - offset: 6, - value: []byte("lllll"), - }, - key: []byte("SetRangeNoExistKey"), - want: []byte("\x00\x00\x00\x00\x00\x00lllll"), - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - callFunc := func(txn *Transaction) { - s, err := GetString(txn, tt.key) - assert.NoError(t, err) - got, err := s.SetRange(tt.args.offset, tt.args.value) - assert.NoError(t, err) - assert.Equal(t, tt.want, got) - } - MockTest(t, callFunc) - }) - } -} - -func TestStringIncr(t *testing.T) { - type args struct { - delta int64 - } - tests := []struct { - name string - args args - want int64 - }{ - { - name: "Incr", - args: args{ - delta: 10, - }, - want: int64(20), - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - callFunc := func(txn *Transaction) { - s, err := GetString(txn, value) - assert.NoError(t, err) - err = s.Set([]byte("10")) - assert.NoError(t, err) - got, err := s.Incr(tt.args.delta) - assert.NoError(t, err) - assert.Equal(t, tt.want, got) - } - MockTest(t, callFunc) - }) - } -} - -func TestStringIncrf(t *testing.T) { - type args struct { - delta float64 - } - tests := []struct { - name string - args args - want float64 - }{ - { - name: "Incr", - args: args{ - delta: float64(0.2), - }, - want: float64(10.3), - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - callFunc := func(txn *Transaction) { - s, err := GetString(txn, value) - assert.NoError(t, err) - err = s.Set([]byte("10.1")) - assert.NoError(t, err) - got, err := s.Incrf(tt.args.delta) - assert.NoError(t, err) - if got < tt.want-0.000000000000001 { - t.Errorf("String.Incrf() = %v, want %v", got, tt.want) - } - } - MockTest(t, callFunc) - }) - } -} - -func TestStringSetBit(t *testing.T) { - type args struct { - on int - off int - } - type want struct { - retval int - value []byte - } - tests := []struct { - name string - args args - want want - }{ - { - name: "one", - args: args{ - off: 1, - on: 1, - }, - want: want{ - retval: 0, - value: []byte{0x40}, - }, - }, - { - name: "two", - args: args{ - off: 5, - on: 1, - }, - want: want{ - retval: 0, - value: []byte{0x44}, - }, - }, - { - name: "three", - args: args{ - off: 5, - on: 0, - }, - want: want{ - retval: 0x4, - value: []byte{0x40}, - }, - }, - { - name: "four", - args: args{ - off: 12, - on: 1, - }, - want: want{ - retval: 0, - value: []byte{0x40, 0x08}, - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - key := []byte("set-bit") - callFunc := func(txn *Transaction) { - s, err := GetString(txn, key) - assert.NoError(t, err) - want, err := s.SetBit(tt.args.off, tt.args.on) - assert.NoError(t, err) - assert.Equal(t, tt.want.retval, want) - } - MockTest(t, callFunc) - getValue(t, key, tt.want.value) - }) - } -} - -func TestStringGetBit(t *testing.T) { - key := []byte("get-bits") - callFunc := func(txn *Transaction) { - s, err := GetString(txn, key) - assert.NoError(t, err) - s.SetBit(4, 1) - } - MockTest(t, callFunc) - - type args struct { - off int - } - type want struct { - retval int - } - tests := []struct { - name string - args args - want want - }{ - { - name: "one", - args: args{ - off: 1, - }, - want: want{ - retval: 0, - }, - }, - { - name: "two", - args: args{ - off: 100, - }, - want: want{ - retval: 0, - }, - }, - { - name: "three", - args: args{ - off: 4, - }, - want: want{ - retval: 8, - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - callFunc := func(txn *Transaction) { - s, err := GetString(txn, key) - assert.NoError(t, err) - want, err := s.GetBit(tt.args.off) - assert.NoError(t, err) - assert.Equal(t, tt.want.retval, want) - } - MockTest(t, callFunc) - }) - } -} - -func TestStringCountBit(t *testing.T) { - key := []byte("bit-count") - callFunc := func(txn *Transaction) { - s, err := GetString(txn, key) - assert.NoError(t, err) - s.SetBit(4, 1) - } - MockTest(t, callFunc) - - type args struct { - begin int - end int - } - type want struct { - count int - } - tests := []struct { - name string - args args - want want - }{ - { - name: "one", - args: args{ - begin: 0, - end: 10, - }, - want: want{ - count: 1, - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - callFunc := func(txn *Transaction) { - s, err := GetString(txn, key) - assert.NoError(t, err) - count, err := s.BitCount(tt.args.begin, tt.args.end) - assert.NoError(t, err) - assert.Equal(t, tt.want.count, count) - } - MockTest(t, callFunc) - }) - } -} - -func TestStringBitPos(t *testing.T) { - key := []byte("bit-pos") - callFunc := func(txn *Transaction) { - s, err := GetString(txn, key) - assert.NoError(t, err) - s.Set([]byte("1")) - } - MockTest(t, callFunc) - - type args struct { - bit int - begin int - end int - } - type want struct { - pos int - } - tests := []struct { - name string - args args - want want - }{ - { - name: "one", - args: args{ - bit: 1, - begin: 0, - end: 10, - }, - want: want{ - pos: 2, - }, - }, - { - name: "two", - args: args{ - bit: 0, - begin: 100, - end: 0, - }, - want: want{ - pos: -1, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - callFunc := func(txn *Transaction) { - s, err := GetString(txn, key) - assert.NoError(t, err) - count, err := s.BitPos(tt.args.bit, tt.args.begin, tt.args.end) - assert.NoError(t, err) - assert.Equal(t, tt.want.pos, count) - } - MockTest(t, callFunc) - }) - } -} diff --git a/db/test_test.go b/db/test_test.go deleted file mode 100644 index c90fc3d..0000000 --- a/db/test_test.go +++ /dev/null @@ -1,18 +0,0 @@ -package db - -import ( - "github.com/distributedio/titan/conf" - "github.com/pingcap/tidb/store/mockstore" - "go.uber.org/zap" -) - -func MockDB() *DB { - zap.ReplaceGlobals(zap.NewNop()) - store, err := mockstore.NewMockTikvStore() - if err != nil { - panic(err) - } - mockConf := conf.MockConf() - redis := &RedisStore{Storage: store, conf: &mockConf.TiKV} - return &DB{Namespace: "ns", ID: DBID(1), kv: redis} -} diff --git a/db/tikvgc.go b/db/tikvgc.go deleted file mode 100644 index c07504f..0000000 --- a/db/tikvgc.go +++ /dev/null @@ -1,140 +0,0 @@ -package db - -import ( - "context" - "time" - - "github.com/distributedio/titan/conf" - "github.com/pingcap/tidb/store/tikv" - "github.com/pingcap/tidb/store/tikv/gcworker" - "github.com/pingcap/tidb/store/tikv/oracle" - "go.uber.org/zap" -) - -var ( - sysTiKVGCLeader = []byte("$sys:0:TGC:GCLeader") - sysTiKVGCLastSafePoint = []byte("$sys:0:TGC:LastSafePoint") -) - -const ( - tikvGcTimeFormat = "20060102-15:04:05 -0700 MST" -) - -// StartTiKVGC start tikv gcwork -func StartTiKVGC(db *DB, tikvCfg *conf.TiKVGC) { - ticker := time.NewTicker(tikvCfg.Interval) - defer ticker.Stop() - uuid := UUID() - ctx := context.Background() - for range ticker.C { - if tikvCfg.Disable { - continue - } - isLeader, err := isLeader(db, sysTiKVGCLeader, uuid, tikvCfg.LeaderLifeTime) - if err != nil { - zap.L().Error("[TiKVGC] check TiKVGC leader failed", zap.Error(err)) - continue - } - if !isLeader { - if logEnv := zap.L().Check(zap.DebugLevel, "[TiKVGC] not TiKVGC leader"); logEnv != nil { - logEnv.Write(zap.ByteString("leader", sysTiKVGCLeader), - zap.ByteString("uuid", uuid), - zap.Duration("leader-life-time", tikvCfg.LeaderLifeTime), - zap.Duration("safe-point-life-time", tikvCfg.SafePointLifeTime), - zap.Int("concurrency", tikvCfg.Concurrency)) - } - continue - } - if err := runTiKVGC(ctx, db, uuid, tikvCfg.SafePointLifeTime, tikvCfg.Concurrency); err != nil { - zap.L().Error("[TiKVGC] do TiKVGC failed", zap.Error(err)) - continue - } - } -} - -func runTiKVGC(ctx context.Context, db *DB, uuid []byte, lifeTime time.Duration, concurrency int) error { - newPoint, err := getNewSafePoint(db, lifeTime) - if err != nil { - return err - } - - lastPoint, err := getLastSafePoint(db) - if err != nil { - return err - } - - if lastPoint != nil && newPoint.Before(*lastPoint) { - zap.L().Info("[TiKVGC] last safe point is later than current on,no need to gc.", - zap.Time("last", *lastPoint), zap.Time("current", *newPoint)) - return nil - } - - if lastPoint == nil { - zap.L().Info("[TiKVGC] current safe point ", zap.Time("current", *newPoint)) - } else { - zap.L().Info("[TiKVGC] current safe point ", zap.Time("current", *newPoint), zap.Time("last", *lastPoint)) - } - - if err := saveLastSafePoint(ctx, db, newPoint); err != nil { - zap.L().Error("[TiKVGC] save last safe point err ", zap.Time("current", *newPoint)) - return err - } - safePoint := oracle.ComposeTS(oracle.GetPhysical(*newPoint), 0) - if err := gcworker.RunGCJob(ctx, db.kv.Storage.(tikv.Storage), safePoint, UUIDString(uuid), concurrency); err != nil { - return err - } - return nil - -} - -func saveLastSafePoint(ctx context.Context, db *DB, safePoint *time.Time) error { - txn, err := db.Begin() - if err != nil { - return err - } - if err := txn.t.Set(sysTiKVGCLastSafePoint, []byte(safePoint.Format(tikvGcTimeFormat))); err != nil { - return err - } - if err := txn.t.Commit(ctx); err != nil { - if err := txn.Rollback(); err != nil { - zap.L().Error("rollback failed", zap.Error(err)) - } - return err - } - return nil -} - -func getNewSafePoint(db *DB, lifeTime time.Duration) (*time.Time, error) { - currentVer, err := db.kv.CurrentVersion() - if err != nil { - return nil, err - } - physical := oracle.ExtractPhysical(currentVer.Ver) - sec, nsec := physical/1e3, (physical%1e3)*1e6 - now := time.Unix(sec, nsec) - safePoint := now.Add(-lifeTime) - return &safePoint, nil -} - -func getLastSafePoint(db *DB) (*time.Time, error) { - txn, err := db.Begin() - if err != nil { - return nil, err - } - val, err := txn.t.Get(sysTiKVGCLastSafePoint) - if err != nil { - if IsErrNotFound(err) { - return nil, nil - } - return nil, err - } - str := string(val) - if str == "" { - return nil, nil - } - t, err := time.Parse(tikvGcTimeFormat, str) - if err != nil { - return nil, err - } - return &t, nil -} diff --git a/db/util.go b/db/util.go deleted file mode 100644 index a1b4e77..0000000 --- a/db/util.go +++ /dev/null @@ -1,117 +0,0 @@ -package db - -import ( - "encoding/binary" - "math" - "math/bits" - "time" - - "github.com/satori/go.uuid" -) - -// UUID allocates an unique object ID. -func UUID() []byte { return uuid.NewV4().Bytes() } - -// UUIDString returns canonical string representation of UUID: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -func UUIDString(id []byte) string { return uuid.FromBytesOrNil(id).String() } - -// Now returns the current unix nano timestamp. -func Now() int64 { return time.Now().UnixNano() } - -func redisPopcount(s []byte) int { - // Count initial bytes not aligned to 32 bit. - bitval := s - begin := 0 - ll := 4 - len(bitval)%4 - if ll != 4 { - bitval = append(bitval, make([]byte, ll)...) - } - - sum := 0 - for begin < len(s) { - num := binary.BigEndian.Uint32(bitval[begin : begin+4]) - sum += bits.OnesCount32(num) - begin += 4 - } - - return sum -} - -func initCursor(begin, end, llen int) (int, int) { - if begin < 0 { - begin = llen + begin - } - if end < 0 { - end = llen + end - } - - if begin < 0 { - begin = 0 - } - if end < 0 { - end = 0 - } - if end >= llen { - end = llen - 1 - } - return begin, end -} - -func redisBitpos(val []byte, bit int) int { - // fill in four bytes - bitval := val - ll := 4 - len(bitval)%4 - if ll != 4 { - bitval = append(bitval, make([]byte, ll)...) - } - - // calculated in term of four bytes - var skipval uint32 - if bit == 0 { - skipval = math.MaxUint32 - } - pos := 0 - for pos < len(bitval) { - num := binary.BigEndian.Uint32(bitval[pos : pos+4]) - if num != skipval { - break - } - pos += 4 - } - - // calculated in term of one bytes - var one uint8 - if bit == 0 { - one = math.MaxUint8 - } - for pos < len(bitval) { - num := uint8(bitval[pos]) - if num != one { - break - } - pos += 1 - } - if pos == len(bitval) && bit == 1 { - return -1 - } - - // find the corresponding bit - one = math.MaxUint8 /* All bitval set to 1.*/ - one >>= 1 /* All bitval set to 1 but the MSB. */ - one = ^one /* All bitval set to 0 but the MSB. */ - word := bitval[pos] - pos = pos * 8 - bbit := false - if bit == 1 { - bbit = true - } - for one != 0 { - // fmt.Println(one, word, (one & word), pos, uint8(bit)) - if ((one & word) != 0) == bbit { - return pos - } - one >>= 1 - pos++ - } - return pos -} diff --git a/db/util_test.go b/db/util_test.go deleted file mode 100644 index ad284f2..0000000 --- a/db/util_test.go +++ /dev/null @@ -1,138 +0,0 @@ -package db - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestInitCursor(t *testing.T) { - type args struct { - begin int - end int - llen int - } - type want struct { - begin int - end int - } - - tests := []struct { - name string - args args - want want - }{ - { - name: "1", - args: args{ - begin: -11, - end: -12, - llen: 10, - }, - want: want{ - begin: 0, - end: 0, - }, - }, - { - name: "2", - args: args{ - begin: -11, - end: 12, - llen: 10, - }, - want: want{ - begin: 0, - end: 9, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - begin, end := initCursor(tt.args.begin, tt.args.end, tt.args.llen) - assert.Equal(t, tt.want.begin, begin) - assert.Equal(t, tt.want.end, end) - }) - } -} - -func TestBitpos(t *testing.T) { - type args struct { - val []byte - bit int - } - tests := []struct { - name string - args args - want int - }{ - { - name: "1", - args: args{ - val: []byte{byte(255), byte(255), byte(255), byte(255), byte(255), byte(255)}, - bit: 1, - }, - want: 0, - }, - { - name: "2", - args: args{ - val: []byte{byte(255), byte(255), byte(255), byte(255), byte(255), byte(255)}, - bit: 0, - }, - want: 48, - }, - { - name: "3", - args: args{ - val: []byte{byte(0), byte(0), byte(0), byte(0), byte(0)}, - bit: 1, - }, - want: -1, - }, - { - name: "4", - args: args{ - val: []byte{byte(0), byte(0), byte(0), byte(0), byte(0)}, - bit: 0, - }, - want: 0, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.want, redisBitpos(tt.args.val, tt.args.bit)) - }) - } -} - -func TestRedisPopCount(t *testing.T) { - type args struct { - val []byte - } - tests := []struct { - name string - args args - want int - }{ - { - name: "1", - args: args{ - val: []byte{byte(255), byte(255), byte(255), byte(255), byte(255), byte(255)}, - }, - want: 48, - }, - { - name: "2", - args: args{ - val: []byte{byte(0), byte(0), byte(0), byte(0)}, - }, - want: 0, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.want, redisPopcount(tt.args.val)) - }) - } -} diff --git a/db/zlist.go b/db/zlist.go deleted file mode 100644 index 03a9ee3..0000000 --- a/db/zlist.go +++ /dev/null @@ -1,301 +0,0 @@ -package db - -import ( - "bytes" - - pb "github.com/distributedio/titan/db/zlistproto" - "github.com/golang/protobuf/proto" -) - -// GetZList generate List objectm with auto reation, if zip is true, zipped list will be choose -func GetZList(txn *Transaction, metaKey []byte, obj *Object, val []byte) (*ZList, error) { - l := &ZList{ - rawMetaKey: metaKey, - txn: txn, - } - if err := l.Unmarshal(obj, val); err != nil { - return nil, err - } - return l, nil -} - -//NewZList create new list object ,the key is not checked for presence -func NewZList(txn *Transaction, key []byte) (List, error) { - metaKey := MetaKey(txn.db, key) - ts := Now() - obj := Object{ - ExpireAt: 0, - CreatedAt: ts, - UpdatedAt: ts, - Type: ObjectList, - ID: UUID(), - Encoding: ObjectEncodingZiplist, - } - l := &ZList{ - Object: obj, - value: pb.Zlistvalue{}, - rawMetaKey: metaKey, - txn: txn, - } - if err := PutZList(txn, metaKey); err != nil { - return nil, err - } - return l, nil -} - -// ZList ZListMeta defined zip list, with only objectMeta info. -type ZList struct { - Object - rawMetaKey []byte - value pb.Zlistvalue //[][]byte - txn *Transaction -} - -//Exist if zlist is effective return true ,otherwise return false -func (l *ZList) Exist() bool { - return l.value.V != nil -} - -// Marshal encode zlist into byte slice -func (l *ZList) Marshal() ([]byte, error) { - b := EncodeObject(&l.Object) - meta, err := proto.Marshal(&l.value) - if err != nil { - return nil, err - } - return append(b, meta...), nil -} - -// zlistCommit try to marshal zlist values and then do set -func (l *ZList) zlistCommit() error { - b, err := l.Marshal() - if err != nil { - return err - } - return l.txn.t.Set(l.rawMetaKey, b) -} - -// Unmarshal parse meta data into meta field -func (l *ZList) Unmarshal(obj *Object, b []byte) (err error) { - l.Object = *obj - if err := proto.Unmarshal(b[ObjectEncodingLength:], &l.value); err != nil { - return err - } - return nil -} - -// Length return z list length -func (l *ZList) Length() int64 { return int64(len(l.value.V)) } - -//LPush append new elements to the object values -func (l *ZList) LPush(data ...[]byte) (err error) { - cv := make([][]byte, len(data), len(data)+len(l.value.V)) - - j := 0 // data->[] lpush - for i := len(data) - 1; i >= 0; i-- { - cv[j] = data[i] - j++ - } - cv = append(cv, l.value.V...) - l.value.V = cv - return l.zlistCommit() -} - -// RPush insert data befroe object values -func (l *ZList) RPush(data ...[]byte) (err error) { - l.value.V = append(l.value.V, data...) // []<-data rpush - return l.zlistCommit() -} - -// Set the index object with given value, return ErrIndex on out of range error. -func (l *ZList) Set(n int64, data []byte) error { - if n < 0 { - n = int64(len(l.value.V)) + n - } - if n < 0 || n >= int64(len(l.value.V)) { - return ErrOutOfRange - } - l.value.V[n] = data - return l.zlistCommit() -} - -// Insert v before/after pivot in zlist -func (l *ZList) Insert(pivot, v []byte, before bool) error { - index := -1 - for index = range l.value.V { - if bytes.Equal(l.value.V[index], pivot) { - break - } - } - // if pivot not exist, index will reach len(l.valus.V) - if index == len(l.value.V) { - return ErrKeyNotFound - } - - if !before { - index++ - } - - cv := make([][]byte, len(l.value.V)+1) - copy(cv[:index], l.value.V[:index]) - cv[index] = v - copy(cv[index+1:], l.value.V[index:]) - - l.value.V = cv - return l.zlistCommit() -} - -//Index return the value at index -func (l *ZList) Index(n int64) (data []byte, err error) { - if n < 0 { - n += int64(len(l.value.V)) - } - if n < 0 || n >= int64(len(l.value.V)) { - return nil, ErrOutOfRange - } - return l.value.V[n], nil -} - -// LPop return and delete the left most element -func (l *ZList) LPop() (data []byte, err error) { - v := l.value.V[0] - l.value.V = l.value.V[1:] - - //destory on last key - if len(l.value.V) == 0 { - return v, l.Destory() - } - return v, l.zlistCommit() -} - -// RPop return and delete the right most element -func (l *ZList) RPop() ([]byte, error) { - v := l.value.V[len(l.value.V)-1] - l.value.V = l.value.V[:len(l.value.V)-1] - //destory on last key - if len(l.value.V) == 0 { - return v, l.Destory() - } - return v, l.zlistCommit() -} - -// Range return the elements in [left, right] -func (l *ZList) Range(left, right int64) (value [][]byte, err error) { - if right < 0 { - if right = int64(len(l.value.V)) + right; right < 0 { - return [][]byte{}, nil - } - } - if left < 0 { - if left = int64(len(l.value.V)) + left; left < 0 { - left = 0 - } - } - if right >= int64(len(l.value.V)) { - right = int64(len(l.value.V) - 1) - } - // return 0 elements - if left > right { - return [][]byte{}, nil - } - return l.value.V[left : right+1], nil -} - -// LTrim get keys from start index to stop index -func (l *ZList) LTrim(start int64, stop int64) error { - if start < 0 { - if start = int64(len(l.value.V)) + start; start < 0 { - start = 0 - } - } - if stop < 0 { - stop += int64(len(l.value.V)) - } - if stop > int64(len(l.value.V)-1) { - stop = int64(len(l.value.V) - 1) - } - - if start > stop { - return l.Destory() - } - - l.value.V = l.value.V[start : stop+1] - return l.zlistCommit() -} - -// LRem begin delete the count of n key from v -func (l *ZList) LRem(v []byte, n int64) (int, error) { - cv := make([][]byte, len(l.value.V)) - count := 0 - if n < 0 { // delete from tail to head, then trim the cv - j := len(l.value.V) - 1 - for i := j; i >= 0 && count < int(n); i-- { - if !bytes.Equal(l.value.V[i], v) { - cv[j] = l.value.V[i] - j-- - } else { - count++ - } - } - l.value.V = cv[j+1:] - } else if n == 0 { - j := 0 - for i := range l.value.V { - if !bytes.Equal(l.value.V[i], v) { - cv[j] = l.value.V[i] - j++ - } else { - count++ - } - } - l.value.V = cv[:j] - } else { - j := 0 - for i := j; i < len(l.value.V) && count < int(n); i-- { - if !bytes.Equal(l.value.V[i], v) { - cv[j] = l.value.V[i] - j++ - } else { - count++ - } - } - l.value.V = cv[:j] - } - return count, l.zlistCommit() -} - -// Destory the zlist -func (l *ZList) Destory() error { - // delete the meta data - return l.txn.t.Delete(l.rawMetaKey) -} - -// TransferToLList create an llist and put values into llist from zlist, LList will inheritance -// information from ZList -func (l *ZList) TransferToLList(dbns []byte, dbid DBID, key []byte) (*LList, error) { - ll := &LList{ - LListMeta: LListMeta{ - Object: Object{ - ExpireAt: 0, - CreatedAt: l.CreatedAt, - UpdatedAt: l.UpdatedAt, - Type: ObjectList, - ID: l.ID, - Encoding: ObjectEncodingLinkedlist, - }, - Len: 0, - Lindex: 0, - Rindex: 0, - }, - txn: l.txn, - rawMetaKey: l.rawMetaKey, - } - dataKeyPrefix := []byte{} - dataKeyPrefix = append(dataKeyPrefix, dbns...) - dataKeyPrefix = append(dataKeyPrefix, ':') - dataKeyPrefix = append(dataKeyPrefix, dbid.Bytes()...) - dataKeyPrefix = append(dataKeyPrefix, ':', 'D', ':') - dataKeyPrefix = append(dataKeyPrefix, ll.Object.ID...) - ll.rawDataKeyPrefix = append(dataKeyPrefix, ':') - return ll, ll.RPush(l.value.V...) -} diff --git a/db/zlistproto/zlist.pb.go b/db/zlistproto/zlist.pb.go deleted file mode 100644 index 24fd2a9..0000000 --- a/db/zlistproto/zlist.pb.go +++ /dev/null @@ -1,317 +0,0 @@ -// Code generated by protoc-gen-gogo. -// source: zlist.proto -// DO NOT EDIT! - -/* -Package zlistproto is a generated protocol buffer package. - -It is generated from these files: - zlist.proto - -It has these top-level messages: - Zlistvalue -*/ -package zlistproto - -import proto "github.com/golang/protobuf/proto" -import fmt "fmt" -import math "math" - -import io "io" - -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package - -type Zlistvalue struct { - V [][]byte `protobuf:"bytes,1,rep,name=v" json:"v,omitempty"` -} - -func (m *Zlistvalue) Reset() { *m = Zlistvalue{} } -func (m *Zlistvalue) String() string { return proto.CompactTextString(m) } -func (*Zlistvalue) ProtoMessage() {} -func (*Zlistvalue) Descriptor() ([]byte, []int) { return fileDescriptorZlist, []int{0} } - -func init() { - proto.RegisterType((*Zlistvalue)(nil), "zlistproto.zlistvalue") -} -func (m *Zlistvalue) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *Zlistvalue) MarshalTo(dAtA []byte) (int, error) { - var i int - _ = i - var l int - _ = l - if len(m.V) > 0 { - for _, b := range m.V { - dAtA[i] = 0xa - i++ - i = encodeVarintZlist(dAtA, i, uint64(len(b))) - i += copy(dAtA[i:], b) - } - } - return i, nil -} - -func encodeFixed64Zlist(dAtA []byte, offset int, v uint64) int { - dAtA[offset] = uint8(v) - dAtA[offset+1] = uint8(v >> 8) - dAtA[offset+2] = uint8(v >> 16) - dAtA[offset+3] = uint8(v >> 24) - dAtA[offset+4] = uint8(v >> 32) - dAtA[offset+5] = uint8(v >> 40) - dAtA[offset+6] = uint8(v >> 48) - dAtA[offset+7] = uint8(v >> 56) - return offset + 8 -} -func encodeFixed32Zlist(dAtA []byte, offset int, v uint32) int { - dAtA[offset] = uint8(v) - dAtA[offset+1] = uint8(v >> 8) - dAtA[offset+2] = uint8(v >> 16) - dAtA[offset+3] = uint8(v >> 24) - return offset + 4 -} -func encodeVarintZlist(dAtA []byte, offset int, v uint64) int { - for v >= 1<<7 { - dAtA[offset] = uint8(v&0x7f | 0x80) - v >>= 7 - offset++ - } - dAtA[offset] = uint8(v) - return offset + 1 -} -func (m *Zlistvalue) Size() (n int) { - var l int - _ = l - if len(m.V) > 0 { - for _, b := range m.V { - l = len(b) - n += 1 + l + sovZlist(uint64(l)) - } - } - return n -} - -func sovZlist(x uint64) (n int) { - for { - n++ - x >>= 7 - if x == 0 { - break - } - } - return n -} -func sozZlist(x uint64) (n int) { - return sovZlist(uint64((x << 1) ^ uint64((int64(x) >> 63)))) -} -func (m *Zlistvalue) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowZlist - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: zlistvalue: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: zlistvalue: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field V", wireType) - } - var byteLen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowZlist - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - byteLen |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - if byteLen < 0 { - return ErrInvalidLengthZlist - } - postIndex := iNdEx + byteLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.V = append(m.V, make([]byte, postIndex-iNdEx)) - copy(m.V[len(m.V)-1], dAtA[iNdEx:postIndex]) - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipZlist(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthZlist - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func skipZlist(dAtA []byte) (n int, err error) { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowZlist - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - wireType := int(wire & 0x7) - switch wireType { - case 0: - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowZlist - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - iNdEx++ - if dAtA[iNdEx-1] < 0x80 { - break - } - } - return iNdEx, nil - case 1: - iNdEx += 8 - return iNdEx, nil - case 2: - var length int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowZlist - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - length |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - iNdEx += length - if length < 0 { - return 0, ErrInvalidLengthZlist - } - return iNdEx, nil - case 3: - for { - var innerWire uint64 - var start int = iNdEx - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowZlist - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - innerWire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - innerWireType := int(innerWire & 0x7) - if innerWireType == 4 { - break - } - next, err := skipZlist(dAtA[start:]) - if err != nil { - return 0, err - } - iNdEx = start + next - } - return iNdEx, nil - case 4: - return iNdEx, nil - case 5: - iNdEx += 4 - return iNdEx, nil - default: - return 0, fmt.Errorf("proto: illegal wireType %d", wireType) - } - } - panic("unreachable") -} - -var ( - ErrInvalidLengthZlist = fmt.Errorf("proto: negative length found during unmarshaling") - ErrIntOverflowZlist = fmt.Errorf("proto: integer overflow") -) - -func init() { proto.RegisterFile("zlist.proto", fileDescriptorZlist) } - -var fileDescriptorZlist = []byte{ - // 90 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0xae, 0xca, 0xc9, 0x2c, - 0x2e, 0xd1, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0x02, 0x73, 0xc0, 0x6c, 0x25, 0x29, 0x2e, - 0x08, 0xaf, 0x2c, 0x31, 0xa7, 0x34, 0x55, 0x88, 0x87, 0x8b, 0xb1, 0x4c, 0x82, 0x51, 0x81, 0x59, - 0x83, 0x27, 0x88, 0xb1, 0xcc, 0x49, 0xe0, 0xc4, 0x23, 0x39, 0xc6, 0x0b, 0x8f, 0xe4, 0x18, 0x1f, - 0x3c, 0x92, 0x63, 0x9c, 0xf1, 0x58, 0x8e, 0x21, 0x89, 0x0d, 0xac, 0xc9, 0x18, 0x10, 0x00, 0x00, - 0xff, 0xff, 0x1b, 0xe9, 0xd7, 0x42, 0x4f, 0x00, 0x00, 0x00, -} diff --git a/db/zlistproto/zlist.proto b/db/zlistproto/zlist.proto deleted file mode 100644 index ed9a570..0000000 --- a/db/zlistproto/zlist.proto +++ /dev/null @@ -1,6 +0,0 @@ -syntax = "proto3"; -package zlistproto; - -message zlistvalue{ - repeated bytes v = 1; -} diff --git a/db/zset.go b/db/zset.go deleted file mode 100644 index fd0fea1..0000000 --- a/db/zset.go +++ /dev/null @@ -1,472 +0,0 @@ -package db - -import ( - "encoding/binary" - "strconv" - "time" - - "github.com/pingcap/tidb/kv" - - "go.uber.org/zap" -) - -// ZSetMeta is the meta data of the sorted set -type ZSetMeta struct { - Object - Len int64 -} - -// ZSet implements the the sorted set -type ZSet struct { - meta ZSetMeta - key []byte - txn *Transaction -} - -type MemberScore struct { - Member string - Score float64 -} - -const byteScoreLen = 8 - -func newZSet(txn *Transaction, key []byte) *ZSet { - now := Now() - return &ZSet{ - txn: txn, - key: key, - meta: ZSetMeta{ - Object: Object{ - ID: UUID(), - CreatedAt: now, - UpdatedAt: now, - ExpireAt: 0, - Type: ObjectZSet, - Encoding: ObjectEncodingHT, - }, - Len: 0, - }, - } -} - -// GetZSet returns a sorted set, create new one if don't exists -func GetZSet(txn *Transaction, key []byte) (*ZSet, error) { - zset := newZSet(txn, key) - - mkey := MetaKey(txn.db, key) - start := time.Now() - meta, err := txn.t.Get(mkey) - zap.L().Debug("zset get metaKey", zap.Int64("cost(us)", time.Since(start).Nanoseconds()/1000)) - if err != nil { - if IsErrNotFound(err) { - return zset, nil - } - return nil, err - } - - obj, err := DecodeObject(meta) - if err != nil { - return nil, err - } - if IsExpired(obj, Now()) { - return zset, nil - } - if obj.Type != ObjectZSet { - return nil, ErrTypeMismatch - } - - m := meta[ObjectEncodingLength:] - if len(m) != 8 { - return nil, ErrInvalidLength - } - zset.meta.Object = *obj - zset.meta.Len = int64(binary.BigEndian.Uint64(m[:8])) - - return zset, nil -} - -func (zset *ZSet) ZAdd(members [][]byte, scores []float64) (int64, error) { - added := int64(0) - - oldValues := make([][]byte, len(members)) - var err error - if zset.meta.Len > 0 { - start := time.Now() - oldValues, err = zset.MGet(members) - zap.L().Debug("zset mget", zap.Int64("cost(us)", time.Since(start).Nanoseconds()/1000)) - if err != nil { - return 0, err - } - } - - dkey := DataKey(zset.txn.db, zset.meta.ID) - var found bool - var start time.Time - costDel, costSetMem, costSetScore := int64(0), int64(0), int64(0) - for i := range members { - found = false - if oldValues[i] != nil { - found = true - oldScore := DecodeFloat64(oldValues[i]) - if scores[i] == oldScore { - continue - } - oldScoreKey := zsetScoreKey(dkey, oldValues[i], members[i]) - start = time.Now() - err = zset.txn.t.Delete(oldScoreKey) - costDel += time.Since(start).Nanoseconds() - if err != nil { - return added, err - } - } - memberKey := zsetMemberKey(dkey, members[i]) - bytesScore, err := EncodeFloat64(scores[i]) - if err != nil { - return 0, err - } - start = time.Now() - err = zset.txn.t.Set(memberKey, bytesScore) - costSetMem += time.Since(start).Nanoseconds() - if err != nil { - return added, err - } - - scoreKey := zsetScoreKey(dkey, bytesScore, members[i]) - start = time.Now() - err = zset.txn.t.Set(scoreKey, NilValue) - costSetScore += time.Since(start).Nanoseconds() - if err != nil { - return added, err - } - - if !found { - added += 1 - } - } - zap.L().Debug("zset cost(us)", zap.Int64("del oldScoreKey", costDel/1000), - zap.Int64("set memberKey", costSetMem/1000), - zap.Int64("set scoreKey", costSetScore/1000)) - - zset.meta.Len += added - start = time.Now() - if err = zset.updateMeta(); err != nil { - return 0, err - } - zap.L().Debug("zset update meta key", zap.Int64("cost(us)", time.Since(start).Nanoseconds()/1000)) - - return added, nil -} - -func (zset *ZSet) MGet(members [][]byte) ([][]byte, error) { - ikeys := make([][]byte, len(members)) - dkey := DataKey(zset.txn.db, zset.meta.ID) - for i := range members { - ikeys[i] = zsetMemberKey(dkey, members[i]) - } - - return BatchGetValues(zset.txn, ikeys) -} - -func (zset *ZSet) updateMeta() error { - meta := zset.encodeMeta(zset.meta) - return zset.txn.t.Set(MetaKey(zset.txn.db, zset.key), meta) -} - -func (zset *ZSet) encodeMeta(meta ZSetMeta) []byte { - b := EncodeObject(&meta.Object) - m := make([]byte, 8) - binary.BigEndian.PutUint64(m[:8], uint64(meta.Len)) - return append(b, m...) -} - -func (zset *ZSet) Exist() bool { - return zset.meta.Len != 0 -} - -func (zset *ZSet) ZAnyOrderRange(start int64, stop int64, withScore bool, positiveOrder bool) ([][]byte, error) { - if stop < 0 { - if stop = zset.meta.Len + stop; stop < 0 { - return [][]byte{}, nil - } - } else if stop >= zset.meta.Len { - stop = zset.meta.Len - 1 - } - if start < 0 { - if start = zset.meta.Len + start; start < 0 { - start = 0 - } - } - // return 0 elements - if start > stop || start >= zset.meta.Len { - return [][]byte{}, nil - } - dkey := DataKey(zset.txn.db, zset.meta.ID) - scorePrefix := ZSetScorePrefix(dkey) - var iter Iterator - var err error - startTime := time.Now() - if positiveOrder { - iter, err = zset.txn.t.Iter(scorePrefix, nil) - } else { - //tikv sdk didn't implement SeekReverse(), so we just use seek() to implement zrevrange now - //because tikv sdk scan 256 keys in next(), for the zset which have <=256 members, - // its performance should be similar with seekReverse, for >256 zset, it should be bad - //iter, err = zset.txn.t.SeekReverse(scorePrefix) - iter, err = zset.txn.t.Iter(scorePrefix, nil) - tmp := start - start = zset.meta.Len - 1 - stop - stop = zset.meta.Len - 1 - tmp - } - zap.L().Debug("zset seek", zap.Int64("cost(us)", time.Since(startTime).Nanoseconds()/1000)) - - if err != nil { - return nil, err - } - - var items [][]byte - cost := int64(0) - for i := int64(0); err == nil && i <= stop && iter.Valid() && iter.Key().HasPrefix(scorePrefix); i++ { - if i >= start { - if len(iter.Key()) <= len(scorePrefix)+byteScoreLen+len(":") { - zap.L().Error("score&member's length isn't enough to be decoded", - zap.ByteString("meta key", zset.key), zap.ByteString("data key", iter.Key())) - startTime = time.Now() - err = iter.Next() - cost += time.Since(startTime).Nanoseconds() - continue - } - - scoreAndMember := iter.Key()[len(scorePrefix):] - score := scoreAndMember[0:byteScoreLen] - member := scoreAndMember[byteScoreLen+len(":"):] - items = append(items, member) - if withScore { - val := []byte(strconv.FormatFloat(DecodeFloat64(score), 'f', -1, 64)) - items = append(items, val) - if !positiveOrder { - items[len(items)-1], items[len(items)-2] = items[len(items)-2], items[len(items)-1] - } - } - } - - startTime = time.Now() - err = iter.Next() - cost += time.Since(startTime).Nanoseconds() - } - zap.L().Debug("zset all next", zap.Int64("cost(us)", cost/1000)) - - if !positiveOrder { - for i, j := 0, len(items)-1; i < j; i, j = i+1, j-1 { - items[i], items[j] = items[j], items[i] - } - } - - return items, nil -} - -// ZAnyOrderRangeByScore returns the items of a zset in specific order -func (zset *ZSet) ZAnyOrderRangeByScore(startScore float64, startInclude bool, - stopScore float64, stopInclude bool, - withScore bool, - offset int64, count int64, - positiveOrder bool) ([][]byte, error) { - if positiveOrder && startScore > stopScore { - return nil, nil - } - if !positiveOrder && startScore < stopScore { - return nil, nil - } - if startScore == stopScore && (!startInclude || !stopInclude) { - return nil, nil - } - if offset < 0 || count == 0 { - return nil, nil - } - - dkey := DataKey(zset.txn.db, zset.meta.ID) - scorePrefix := ZSetScorePrefix(dkey) - - startPrefix := make([]byte, len(scorePrefix)+byteScoreLen) - copy(startPrefix, scorePrefix) - byteStartScore, err := EncodeFloat64(startScore) - if err != nil { - return nil, err - } - copy(startPrefix[len(scorePrefix):], byteStartScore) - - stopPrefix := make([]byte, len(scorePrefix)+byteScoreLen) - copy(stopPrefix, scorePrefix) - byteStopScore, err := EncodeFloat64(stopScore) - if err != nil { - return nil, err - } - copy(stopPrefix[len(scorePrefix):], byteStopScore) - - var iter Iterator - if positiveOrder { - upperBoundKey := kv.Key(stopPrefix).PrefixNext() - iter, err = zset.txn.t.Iter(startPrefix, upperBoundKey) - } else { - iter, err = zset.txn.t.IterReverse(startPrefix) - } - if err != nil { - return nil, err - } - - var items [][]byte - countN := int64(0) - startComFinished := false - for i := int64(0); err == nil && iter.Valid() && iter.Key().HasPrefix(scorePrefix); i, err = i+1, iter.Next() { - key := iter.Key() - if len(key) <= len(scorePrefix)+byteScoreLen+len(":") { - zap.L().Error("score&member's length isn't enough to be decoded", - zap.ByteString("meta key", zset.key), zap.ByteString("data key", iter.Key())) - continue - } - - curPrefix := key[:len(scorePrefix)+byteScoreLen] - if !startInclude && !startComFinished { - if curPrefix.Cmp(startPrefix) == 0 { - offset += 1 - continue - } else { - startComFinished = true - } - } - - comWithStop := curPrefix.Cmp(stopPrefix) - if (!stopInclude && comWithStop == 0) || - (positiveOrder && comWithStop > 0) || - (!positiveOrder && comWithStop < 0) { - break - } - - if i < offset { - continue - } - countN += 1 - if count > 0 && countN > count { - break - } - - scoreAndMember := key[len(scorePrefix):] - score := scoreAndMember[0:byteScoreLen] - member := scoreAndMember[byteScoreLen+len(":"):] - items = append(items, member) - if withScore { - val := []byte(strconv.FormatFloat(DecodeFloat64(score), 'f', -1, 64)) - items = append(items, val) - } - } - - return items, nil -} - -func (zset *ZSet) ZRem(members [][]byte) (int64, error) { - deleted := int64(0) - - start := time.Now() - scores, err := zset.MGet(members) - zap.L().Debug("zrem mget", zap.Int64("cost(us)", time.Since(start).Nanoseconds()/1000)) - if err != nil { - return 0, err - } - - dkey := DataKey(zset.txn.db, zset.meta.ID) - costDelMem, costDelScore := int64(0), int64(0) - for i := range members { - if scores[i] == nil { - continue - } - - scoreKey := zsetScoreKey(dkey, scores[i], members[i]) - start = time.Now() - err = zset.txn.t.Delete(scoreKey) - costDelScore += time.Since(start).Nanoseconds() - if err != nil { - return deleted, err - } - - memberKey := zsetMemberKey(dkey, members[i]) - start = time.Now() - err = zset.txn.t.Delete(memberKey) - costDelMem += time.Since(start).Nanoseconds() - if err != nil { - return deleted, err - } - - deleted += 1 - } - zap.L().Debug("zrem cost(us)", zap.Int64("del memberKey", costDelMem/1000), - zap.Int64("del scoreKey", costDelScore/1000)) - zset.meta.Len -= deleted - - if zset.meta.Len == 0 { - mkey := MetaKey(zset.txn.db, zset.key) - start = time.Now() - err = zset.txn.t.Delete(mkey) - zap.L().Debug("zrem delete meta key", zap.Int64("cost(us)", time.Since(start).Nanoseconds()/1000)) - if err != nil { - return deleted, err - } - if zset.meta.Object.ExpireAt > 0 { - start = time.Now() - err := unExpireAt(zset.txn.t, mkey, zset.meta.Object.ExpireAt) - zap.L().Debug("zrem delete expire key", zap.Int64("cost(us)", time.Since(start).Nanoseconds()/1000)) - if err != nil { - return deleted, err - } - } - return deleted, nil - } - - start = time.Now() - err = zset.updateMeta() - zap.L().Debug("zrem update meta key", zap.Int64("cost(us)", time.Since(start).Nanoseconds()/1000)) - return deleted, err -} -func (zset *ZSet) ZCard() int64 { - return zset.meta.Len -} - -func (zset *ZSet) ZScore(member []byte) ([]byte, error) { - dkey := DataKey(zset.txn.db, zset.meta.ID) - memberKey := zsetMemberKey(dkey, member) - bytesScore, err := zset.txn.t.Get(memberKey) - if err != nil { - if IsErrNotFound(err) { - return nil, nil - } - return nil, err - } - - fscore := DecodeFloat64(bytesScore) - sscore := strconv.FormatFloat(fscore, 'f', -1, 64) - return []byte(sscore), nil -} - -func zsetMemberKey(dkey []byte, member []byte) []byte { - var memberKey []byte - memberKey = append(memberKey, dkey...) - memberKey = append(memberKey, ':', 'M', ':') - memberKey = append(memberKey, member...) - return memberKey -} - -// ZSetScorePrefix builds a score key prefix from a redis key -func ZSetScorePrefix(dkey []byte) []byte { - var sPrefix []byte - sPrefix = append(sPrefix, dkey...) - sPrefix = append(sPrefix, ':', 'S', ':') - return sPrefix -} - -func zsetScoreKey(dkey []byte, score []byte, member []byte) []byte { - var scoreKey []byte - scoreKey = append(scoreKey, dkey...) - scoreKey = append(scoreKey, ':', 'S', ':') - scoreKey = append(scoreKey, score...) - scoreKey = append(scoreKey, ':') - scoreKey = append(scoreKey, member...) - return scoreKey -} diff --git a/db/ztransfer.go b/db/ztransfer.go deleted file mode 100644 index f465267..0000000 --- a/db/ztransfer.go +++ /dev/null @@ -1,246 +0,0 @@ -package db - -import ( - "context" - "time" - - "github.com/distributedio/titan/conf" - "github.com/distributedio/titan/metrics" - "github.com/pingcap/tidb/kv" - "go.uber.org/zap" -) - -var ( - sysZTLeader = []byte("$sys:0:ZTL:ZTLeader") - sysZTKeyPrefixLength = len(toZTKey([]byte{})) - sysZTLeaderFlushInterval = 10 * time.Second - - ztQueue chan []byte -) - -// loadZList is read only, so ZList did not call Destroy() -func loadZList(txn *Transaction, metaKey []byte) (*ZList, error) { - val, err := txn.t.Get(metaKey) - if err != nil { - return nil, err - } - - obj, err := DecodeObject(val) - if err != nil { - return nil, err - } - if obj.Type != ObjectList { - return nil, ErrTypeMismatch - } - if obj.Encoding != ObjectEncodingZiplist { - zap.L().Error("[ZT] error in trans zlist, encoding type error", zap.Error(err)) - return nil, ErrEncodingMismatch - } - if obj.ExpireAt != 0 && obj.ExpireAt < Now() { - return nil, ErrKeyNotFound - } - - l := &ZList{ - rawMetaKey: metaKey, - txn: txn, - } - if err = l.Unmarshal(obj, val); err != nil { - return nil, err - } - return l, nil -} - -// toZTKey convert meta key to ZT key -// {sys.ns}:{sys.id}:{ZT}:{metakey} -// NOTE put this key to sys db. -func toZTKey(metakey []byte) []byte { - b := []byte{} - b = append(b, sysNamespace...) - b = append(b, ':', byte(sysDatabaseID)) - b = append(b, ':', 'Z', 'T', ':') - b = append(b, metakey...) - return b -} - -// PutZList should be called after ZList created -func PutZList(txn *Transaction, metakey []byte) error { - if logEnv := zap.L().Check(zap.DebugLevel, "[ZT] Zlist recorded in txn"); logEnv != nil { - logEnv.Write(zap.String("key", string(metakey))) - } - return txn.t.Set(toZTKey(metakey), []byte{0}) -} - -// RemoveZTKey remove an metakey from ZT -func RemoveZTKey(txn *Transaction, metakey []byte) error { - return txn.t.Delete(toZTKey(metakey)) -} - -// doZListTransfer get zt key, create zlist and transfer to llist, after that, delete zt key -func doZListTransfer(txn *Transaction, metakey []byte) (int, error) { - zlist, err := loadZList(txn, metakey) - if err != nil { - if err == ErrTypeMismatch || err == ErrEncodingMismatch || err == ErrKeyNotFound { - if err = RemoveZTKey(txn, metakey); err != nil { - zap.L().Error("[ZT] error in remove ZTKkey", zap.Error(err)) - return 0, err - } - return 0, nil - } - zap.L().Error("[ZT] error in create zlist", zap.Error(err)) - return 0, err - } - - llist, err := zlist.TransferToLList(splitMetaKey(metakey)) - if err != nil { - zap.L().Error("[ZT] error in convert zlist", zap.Error(err)) - return 0, err - } - // clean the zt key, after success - if err = RemoveZTKey(txn, metakey); err != nil { - zap.L().Error("[ZT] error in remove ZTKkey", zap.Error(err)) - return 0, err - } - - return int(llist.Len), nil -} - -func ztWorker(db *DB, batch int, interval time.Duration) { - var txn *Transaction - var err error - var n int - - txnstart := false - batchCount := 0 - sum := 0 - commit := func(t *Transaction) { - if err = t.Commit(context.Background()); err != nil { - zap.L().Error("[ZT] error in commit transfer", zap.Error(err)) - if err := txn.Rollback(); err != nil { - zap.L().Error("[ZT] rollback failed", zap.Error(err)) - } - } else { - metrics.GetMetrics().ZTInfoCounterVec.WithLabelValues("zlist").Add(float64(batchCount)) - metrics.GetMetrics().ZTInfoCounterVec.WithLabelValues("key").Add(float64(sum)) - if logEnv := zap.L().Check(zap.DebugLevel, "[ZT] transfer zlist succeed"); logEnv != nil { - logEnv.Write(zap.Int("count", batchCount), - zap.Int("n", sum)) - } - } - txnstart = false - batchCount = 0 - sum = 0 - } - - // create zlist and transfer to llist, after that, delete zt key - for { - select { - case metakey := <-ztQueue: - if !txnstart { - if txn, err = db.Begin(); err != nil { - zap.L().Error("[ZT] zt worker error in kv begin", zap.Error(err)) - continue - } - txnstart = true - } - - if n, err = doZListTransfer(txn, metakey); err != nil { - if err := txn.Rollback(); err != nil { - zap.L().Error("[ZT] rollback failed", zap.Error(err)) - } - txnstart = false - continue - } - sum += n - batchCount++ - if batchCount >= batch { - commit(txn) - } - default: - if batchCount > 0 { - commit(txn) - } else { - time.Sleep(interval) - txnstart = false - } - } - } -} - -func runZT(db *DB, prefix []byte, tick <-chan time.Time) ([]byte, error) { - txn, err := db.Begin() - if err != nil { - zap.L().Error("[ZT] error in kv begin", zap.Error(err)) - return toZTKey(nil), nil - } - endPrefix := kv.Key(prefix).PrefixNext() - iter, err := txn.t.Iter(prefix, endPrefix) - if err != nil { - zap.L().Error("[ZT] error in seek", zap.ByteString("prefix", prefix), zap.Error(err)) - return toZTKey(nil), err - } - - for ; iter.Valid() && iter.Key().HasPrefix(prefix); err = iter.Next() { - if err != nil { - zap.L().Error("[ZT] error in iter next", zap.Error(err)) - return toZTKey(nil), err - } - select { - case ztQueue <- iter.Key()[sysZTKeyPrefixLength:]: - case <-tick: - return iter.Key(), nil - default: - return iter.Key(), nil - } - } - if logEnv := zap.L().Check(zap.DebugLevel, "[ZT] no more ZT item, retrive iterator"); logEnv != nil { - logEnv.Write(zap.ByteString("prefix", prefix)) - } - - return toZTKey(nil), txn.Commit(context.Background()) -} - -// StartZT start ZT fill in the queue(channel), and start the worker to consume. -func StartZT(db *DB, conf *conf.ZT) { - ztQueue = make(chan []byte, conf.QueueDepth) - for i := 0; i < conf.Workers; i++ { - go ztWorker(db, conf.BatchCount, conf.Interval) - } - - // check leader and fill the channel - prefix := toZTKey(nil) - ticker := time.NewTicker(conf.Interval) - defer ticker.Stop() - id := UUID() - for range ticker.C { - if conf.Disable { - continue - } - isLeader, err := isLeader(db, sysZTLeader, id, sysZTLeaderFlushInterval) - if err != nil { - zap.L().Error("[ZT] check ZT leader failed", - zap.Int64("dbid", int64(db.ID)), - zap.Error(err)) - continue - } - if !isLeader { - if logEnv := zap.L().Check(zap.DebugLevel, "[ZT] not ZT leader"); logEnv != nil { - logEnv.Write(zap.ByteString("leader", sysZTLeader), - zap.ByteString("uuid", id), - zap.Int("workers", conf.Workers), - zap.Int("batchcount", conf.BatchCount), - zap.Int("queue-depth", conf.QueueDepth), - zap.Duration("interval", conf.Interval), - ) - } - continue - } - - if prefix, err = runZT(db, prefix, ticker.C); err != nil { - zap.L().Error("[ZT] error in run ZT", - zap.Int64("dbid", int64(db.ID)), - zap.ByteString("prefix", prefix), - zap.Error(err)) - continue - } - } -} diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 85b5086..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,48 +0,0 @@ -version: '2.1' - -services: - pd0: - image: pingcap/pd:latest - ports: - - "2379" - volumes: - - ./data:/data - - ./logs:/logs - command: - - --name=pd0 - - --client-urls=http://0.0.0.0:2379 - - --peer-urls=http://0.0.0.0:2380 - - --advertise-client-urls=http://pd0:2379 - - --advertise-peer-urls=http://pd0:2380 - - --initial-cluster=pd0=http://pd0:2380 - - --data-dir=/data/pd0 - - --log-file=/logs/pd0.log - restart: on-failure - - tikv0: - image: pingcap/tikv:latest - volumes: - - ./data:/data - - ./logs:/logs - command: - - --addr=0.0.0.0:20160 - - --advertise-addr=tikv0:20160 - - --data-dir=/data/tikv0 - - --pd=pd0:2379 - - --log-file=/logs/tikv0.log - depends_on: - - "pd0" - restart: on-failure - - titan: - image: distributedio/titan - volumes: - - ./logs:/titan/logs - ports: - - "7369:7369" - - "7345:7345" - command: - - --pd-addrs=tikv://pd0:2379 - depends_on: - - "tikv0" - restart: on-failure diff --git a/docs-cn/README.md b/docs-cn/README.md deleted file mode 100644 index df58919..0000000 --- a/docs-cn/README.md +++ /dev/null @@ -1,6 +0,0 @@ -# Titan - -[设计](./design.md) -[部署](./ops/deploy.md) -[性能测试](./benchmark.md) -Hello World 2 diff --git a/docs-cn/benchmark.md b/docs-cn/benchmark.md deleted file mode 100644 index a8c22b9..0000000 --- a/docs-cn/benchmark.md +++ /dev/null @@ -1 +0,0 @@ -# Titan 性能压测 diff --git a/docs-cn/design.md b/docs-cn/design.md deleted file mode 100644 index cb20582..0000000 --- a/docs-cn/design.md +++ /dev/null @@ -1 +0,0 @@ -# Titan 架构设计 diff --git a/docs-cn/introduce.md b/docs-cn/introduce.md deleted file mode 100644 index e10b99d..0000000 --- a/docs-cn/introduce.md +++ /dev/null @@ -1 +0,0 @@ -# Introduction diff --git a/docs-cn/ops/deploy.md b/docs-cn/ops/deploy.md deleted file mode 100644 index 17dd772..0000000 --- a/docs-cn/ops/deploy.md +++ /dev/null @@ -1 +0,0 @@ -# Titan 安装与部署 diff --git a/docs/README.md b/docs/README.md deleted file mode 100644 index dbf8f8f..0000000 --- a/docs/README.md +++ /dev/null @@ -1,6 +0,0 @@ -# Titan - -[Design](./design.md) -[Deploy](./ops/deploy.md) -[Benchmark](./benchmark.md) -Hello World 2 diff --git a/docs/benchmark/benchmark.md b/docs/benchmark/benchmark.md deleted file mode 100644 index c54e521..0000000 --- a/docs/benchmark/benchmark.md +++ /dev/null @@ -1,68 +0,0 @@ -# Benchmarks of Titan - -## Environment - -``` -Titan Cluster(HDD sata) -192.168.134.48 (CPU：Intel(R) Xeon(R) CPU E5-2630 v3 @ 2.40GHz,  Mem：32G) -192.168.134.70 (CPU: Intel(R) Xeon(R) CPU E5-2630 v3 @ 2.40GHz, Mem: 32G) - -TiKV Cluster (SSD sata, 4 TiKV instance on one physical machine) -192.168.135.41:20171 (CPU: Intel(R) Xeon(R) CPU E5-2630 v4 @ 2.20GHz, Mem: 96G, Disk: 5*480G) -192.168.135.41:20172 -192.168.135.41:20174 -192.168.135.41:20173 - -192.168.135.54:20171 -192.168.135.54:20172 -192.168.135.54:20173 -192.168.135.54:20174 - -192.168.133.114:20171 -192.168.133.114:20172 -192.168.133.114:20173 -192.168.133.114:20174 - -pd (SSD sata) -192.168.135.41:2379 -192.168.135.54:2379 -192.168.133.114:2379 - -TiKV tunning -[server] -grpc-concurrency = 8 - -[raftstore] -sync-log = false - - -[rocksdb.defaultcf] -block-cache-size = "9.6GB" - -[rocksdb.writecf] -block-cache-size = “3.6GB” -``` - -Fill into 500G first before benchmarking - -## Results - -### Get - -![Get command benchmark](./get-benchmark.png) - -### Set - -![Set command benchmark](./set-benchmark.png) - -### LPush - -![LPush command benchmark](./lpush-benchmark.png) - -### LPop - -![LPop command benchmark](./lpop-benchmark.png) - -### LRange - -![LRange command benchmark](./lrange-benchmark.png) diff --git a/docs/benchmark/get-benchmark.png b/docs/benchmark/get-benchmark.png deleted file mode 100644 index 6f6ab86..0000000 Binary files a/docs/benchmark/get-benchmark.png and /dev/null differ diff --git a/docs/benchmark/lpop-benchmark.png b/docs/benchmark/lpop-benchmark.png deleted file mode 100644 index 5eb0343..0000000 Binary files a/docs/benchmark/lpop-benchmark.png and /dev/null differ diff --git a/docs/benchmark/lpush-benchmark.png b/docs/benchmark/lpush-benchmark.png deleted file mode 100644 index 566aa85..0000000 Binary files a/docs/benchmark/lpush-benchmark.png and /dev/null differ diff --git a/docs/benchmark/lrange-benchmark.png b/docs/benchmark/lrange-benchmark.png deleted file mode 100644 index da826c6..0000000 Binary files a/docs/benchmark/lrange-benchmark.png and /dev/null differ diff --git a/docs/benchmark/set-benchmark.png b/docs/benchmark/set-benchmark.png deleted file mode 100644 index ea2818d..0000000 Binary files a/docs/benchmark/set-benchmark.png and /dev/null differ diff --git a/docs/command_list.md b/docs/command_list.md deleted file mode 100644 index f50cf21..0000000 --- a/docs/command_list.md +++ /dev/null @@ -1,206 +0,0 @@ -## Commands list - -### Connections -- [x] auth -- [x] echo -- [x] ping -- [x] quit -- [x] select -- [ ] swapdb, not supported - -### Transactions -- [x] multi -- [x] exec -- [x] discard -- [x] watch -- [x] unwatch - -### Server -- [x] client list -- [x] client kill -- [x] client pause -- [x] client reply -- [x] client getname -- [x] client setname -- [x] monitor -- [x] debug object -- [x] flushdb -- [x] flushall -- [x] time -- [x] command -- [x] command count -- [x] command getkeys -- [x] command info -- [x] info -- [ ] slowlog - -### Keys -- [x] del -- [x] type -- [x] exists -- [x] expire -- [x] expireat -- [x] object -- [x] pexpire -- [x] pexpireat -- [x] ttl -- [x] pttl -- [x] randomkey -- [x] touch -- [x] keys -- [x] scan -- [x] unlink - -### Strings - -- [x] get -- [x] set -- [x] mget -- [x] mset -- [x] strlen -- [x] incr -- [x] incrby -- [x] decr -- [x] decrby -- [x] append -- [ ] bitcount -- [ ] bitfield -- [ ] bitop -- [ ] bitpos -- [ ] getbit -- [x] getrange -- [x] getset -- [x] incrbyfloat -- [ ] msetnx -- [x] psetex -- [ ] setbit -- [x] setex -- [x] setnx -- [ ] setrange - -### List - -- [x] lrange -- [x] linsert -- [x] lindex -- [x] llen -- [x] lset -- [x] lpush -- [x] lpop -- [x] lpushx -- [x] ltrim -- [x] lrem -- [x] rpop -- [x] rpoplpush -- [x] rpush -- [x] rpushx -- [ ] blpop -- [ ] brpop -- [ ] brpoplpush - -### Hashes -- [x] hset -- [x] hget -- [x] hgetall -- [x] hdel -- [x] hexists -- [x] hincrby -- [x] hincrbyfloat -- [x] hkeys -- [x] hlen -- [x] hmget -- [x] hmset -- [x] hscan -- [x] hsetnx -- [x] hstrlen -- [x] hvals - -### Sets - -- [x] sadd -- [x] scard -- [x] sdiff -- [ ] sdiffstore -- [x] sinter -- [ ] sinterstore -- [x] sismember -- [x] smembers -- [x] smove -- [x] spop -- [ ] srandmember -- [x] srem -- [x] sunion -- [ ] sunionstore -- [ ] sscan - -### Sorted Sets - -- [ ] bzpopmin -- [ ] bzpopmax -- [x] zadd -- [x] zcard -- [ ] zcount -- [ ] zincrby -- [ ] zinterstore -- [ ] zlexcount -- [ ] zpopmax -- [ ] zpopmin -- [x] zrange -- [ ] zrangebylex -- [ ] zrevrangebylex -- [x] zrangebyscore -- [ ] zrank -- [x] zrem -- [ ] zremrangebylex -- [ ] zremrangebyrank -- [ ] zremrangebyscore -- [x] zrevrange -- [ ] zrevrangebyscore -- [ ] zrevrank -- [x] zscore -- [ ] zunionstore -- [ ] zscan - -### Geo - -- [ ] geoadd -- [ ] geohash -- [ ] geopos -- [ ] geodist -- [ ] georadius -- [ ] georadiusbymember - -### hyperloglog - -- [ ] pfadd -- [ ] pfcount -- [ ] pfmerge - -### Pub/Sub - -- [ ] psubscribe -- [ ] pubsub -- [ ] publish -- [ ] punsubscribe -- [ ] subscribe -- [ ] unsubscribe - -### Scripting - -- [ ] eval -- [ ] evalsha -- [ ] script debug -- [ ] script exists -- [ ] script flush -- [ ] script kill -- [ ] script load - -### Streams - -- [ ] xadd -- [ ] xrange -- [ ] xrevrange -- [ ] xlen -- [ ] xread -- [ ] xreadgroup -- [ ] xpending diff --git a/docs/commit-message-style.md b/docs/commit-message-style.md deleted file mode 100644 index 554052a..0000000 --- a/docs/commit-message-style.md +++ /dev/null @@ -1,103 +0,0 @@ -# Commit Message Style - -It is friendly to have a good commit message style for someone who want to learn about the commitment history. -We follow the style of [AngularJS](https://docs.google.com/document/d/1QrDFcIiPjSLDn3EL15IJygNPiHORgU1_OOAqWjiDU5Y/edit#heading=h.greljkmo14y0) and supply more details specific to this project. - -## Format of the commit message - -```html -(): - - - -